diff options
Diffstat (limited to 'vendor/unicode-normalization/scripts/unicode.py')
| -rw-r--r-- | vendor/unicode-normalization/scripts/unicode.py | 621 |
1 files changed, 621 insertions, 0 deletions
diff --git a/vendor/unicode-normalization/scripts/unicode.py b/vendor/unicode-normalization/scripts/unicode.py new file mode 100644 index 000000000..bfe49c8b2 --- /dev/null +++ b/vendor/unicode-normalization/scripts/unicode.py @@ -0,0 +1,621 @@ +#!/usr/bin/env python +# +# Copyright 2011-2018 The Rust Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution and at +# http://rust-lang.org/COPYRIGHT. +# +# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +# This script uses the following Unicode tables: +# - DerivedNormalizationProps.txt +# - NormalizationTest.txt +# - UnicodeData.txt +# - StandardizedVariants.txt +# +# Since this should not require frequent updates, we just store this +# out-of-line and check the tables.rs and normalization_tests.rs files into git. +import collections +import urllib.request + +UNICODE_VERSION = "14.0.0" +UCD_URL = "https://www.unicode.org/Public/%s/ucd/" % UNICODE_VERSION + +PREAMBLE = """// Copyright 2012-2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// NOTE: The following code was generated by "scripts/unicode.py", do not edit directly + +#![allow(missing_docs)] +""" + +NormalizationTest = collections.namedtuple( + "NormalizationTest", + ["source", "nfc", "nfd", "nfkc", "nfkd"], +) + +# Mapping taken from Table 12 from: +# http://www.unicode.org/reports/tr44/#General_Category_Values +expanded_categories = { + 'Lu': ['LC', 'L'], 'Ll': ['LC', 'L'], 'Lt': ['LC', 'L'], + 'Lm': ['L'], 'Lo': ['L'], + 'Mn': ['M'], 'Mc': ['M'], 'Me': ['M'], + 'Nd': ['N'], 'Nl': ['N'], 'No': ['No'], + 'Pc': ['P'], 'Pd': ['P'], 'Ps': ['P'], 'Pe': ['P'], + 'Pi': ['P'], 'Pf': ['P'], 'Po': ['P'], + 'Sm': ['S'], 'Sc': ['S'], 'Sk': ['S'], 'So': ['S'], + 'Zs': ['Z'], 'Zl': ['Z'], 'Zp': ['Z'], + 'Cc': ['C'], 'Cf': ['C'], 'Cs': ['C'], 'Co': ['C'], 'Cn': ['C'], +} + +# Constants from Unicode 9.0.0 Section 3.12 Conjoining Jamo Behavior +# http://www.unicode.org/versions/Unicode9.0.0/ch03.pdf#M9.32468.Heading.310.Combining.Jamo.Behavior +S_BASE, L_COUNT, V_COUNT, T_COUNT = 0xAC00, 19, 21, 28 +S_COUNT = L_COUNT * V_COUNT * T_COUNT + +class UnicodeData(object): + def __init__(self): + self._load_unicode_data() + self.norm_props = self._load_norm_props() + self.norm_tests = self._load_norm_tests() + + self.canon_comp = self._compute_canonical_comp() + self.canon_fully_decomp, self.compat_fully_decomp = self._compute_fully_decomposed() + + self.cjk_compat_variants_fully_decomp = {} + self._load_cjk_compat_ideograph_variants() + + def stats(name, table): + count = sum(len(v) for v in table.values()) + print("%s: %d chars => %d decomposed chars" % (name, len(table), count)) + + print("Decomposition table stats:") + stats("Canonical decomp", self.canon_decomp) + stats("Compatible decomp", self.compat_decomp) + stats("Canonical fully decomp", self.canon_fully_decomp) + stats("Compatible fully decomp", self.compat_fully_decomp) + stats("CJK Compat Variants fully decomp", self.cjk_compat_variants_fully_decomp) + + self.ss_leading, self.ss_trailing = self._compute_stream_safe_tables() + + def _fetch(self, filename): + resp = urllib.request.urlopen(UCD_URL + filename) + return resp.read().decode('utf-8') + + def _load_unicode_data(self): + self.name_to_char_int = {} + self.combining_classes = {} + self.compat_decomp = {} + self.canon_decomp = {} + self.general_category_mark = [] + self.general_category_public_assigned = [] + + assigned_start = 0; + prev_char_int = -1; + prev_name = ""; + + for line in self._fetch("UnicodeData.txt").splitlines(): + # See ftp://ftp.unicode.org/Public/3.0-Update/UnicodeData-3.0.0.html + pieces = line.split(';') + assert len(pieces) == 15 + char, name, category, cc, decomp = pieces[0], pieces[1], pieces[2], pieces[3], pieces[5] + char_int = int(char, 16) + + name = pieces[1].strip() + self.name_to_char_int[name] = char_int + + if cc != '0': + self.combining_classes[char_int] = cc + + if decomp.startswith('<'): + self.compat_decomp[char_int] = [int(c, 16) for c in decomp.split()[1:]] + elif decomp != '': + self.canon_decomp[char_int] = [int(c, 16) for c in decomp.split()] + + if category == 'M' or 'M' in expanded_categories.get(category, []): + self.general_category_mark.append(char_int) + + assert category != 'Cn', "Unexpected: Unassigned codepoint in UnicodeData.txt" + if category not in ['Co', 'Cs']: + if char_int != prev_char_int + 1 and not is_first_and_last(prev_name, name): + self.general_category_public_assigned.append((assigned_start, prev_char_int)) + assigned_start = char_int + prev_char_int = char_int + prev_name = name; + + self.general_category_public_assigned.append((assigned_start, prev_char_int)) + + def _load_cjk_compat_ideograph_variants(self): + for line in self._fetch("StandardizedVariants.txt").splitlines(): + strip_comments = line.split('#', 1)[0].strip() + if not strip_comments: + continue + + variation_sequence, description, differences = strip_comments.split(';') + description = description.strip() + + # Don't use variations that only apply in particular shaping environments. + if differences: + continue + + # Look for entries where the description field is a codepoint name. + if description not in self.name_to_char_int: + continue + + # Only consider the CJK Compatibility Ideographs. + if not description.startswith('CJK COMPATIBILITY IDEOGRAPH-'): + continue + + char_int = self.name_to_char_int[description] + + assert not char_int in self.combining_classes, "Unexpected: CJK compat variant with a combining class" + assert not char_int in self.compat_decomp, "Unexpected: CJK compat variant and compatibility decomposition" + assert len(self.canon_decomp[char_int]) == 1, "Unexpected: CJK compat variant and non-singleton canonical decomposition" + # If we ever need to handle Hangul here, we'll need to handle it separately. + assert not (S_BASE <= char_int < S_BASE + S_COUNT) + + cjk_compat_variant_parts = [int(c, 16) for c in variation_sequence.split()] + for c in cjk_compat_variant_parts: + assert not c in self.canon_decomp, "Unexpected: CJK compat variant is unnormalized (canon)" + assert not c in self.compat_decomp, "Unexpected: CJK compat variant is unnormalized (compat)" + self.cjk_compat_variants_fully_decomp[char_int] = cjk_compat_variant_parts + + def _load_norm_props(self): + props = collections.defaultdict(list) + + for line in self._fetch("DerivedNormalizationProps.txt").splitlines(): + (prop_data, _, _) = line.partition("#") + prop_pieces = prop_data.split(";") + + if len(prop_pieces) < 2: + continue + + assert len(prop_pieces) <= 3 + (low, _, high) = prop_pieces[0].strip().partition("..") + + prop = prop_pieces[1].strip() + + data = None + if len(prop_pieces) == 3: + data = prop_pieces[2].strip() + + props[prop].append((low, high, data)) + + return props + + def _load_norm_tests(self): + tests = [] + for line in self._fetch("NormalizationTest.txt").splitlines(): + (test_data, _, _) = line.partition("#") + test_pieces = test_data.split(";") + + if len(test_pieces) < 5: + continue + + source, nfc, nfd, nfkc, nfkd = [[c.strip() for c in p.split()] for p in test_pieces[:5]] + tests.append(NormalizationTest(source, nfc, nfd, nfkc, nfkd)) + + return tests + + def _compute_canonical_comp(self): + canon_comp = {} + comp_exclusions = [ + (int(low, 16), int(high or low, 16)) + for low, high, _ in self.norm_props["Full_Composition_Exclusion"] + ] + for char_int, decomp in self.canon_decomp.items(): + if any(lo <= char_int <= hi for lo, hi in comp_exclusions): + continue + + assert len(decomp) == 2 + assert (decomp[0], decomp[1]) not in canon_comp + canon_comp[(decomp[0], decomp[1])] = char_int + + return canon_comp + + def _compute_fully_decomposed(self): + """ + Even though the decomposition algorithm is recursive, it is possible + to precompute the recursion at table generation time with modest + increase to the table size. Then, for these precomputed tables, we + note that 1) compatible decomposition is a subset of canonical + decomposition and 2) they mostly agree on their intersection. + Therefore, we don't store entries in the compatible table for + characters that decompose the same way under canonical decomposition. + + Decomposition table stats: + Canonical decomp: 2060 chars => 3085 decomposed chars + Compatible decomp: 3662 chars => 5440 decomposed chars + Canonical fully decomp: 2060 chars => 3404 decomposed chars + Compatible fully decomp: 3678 chars => 5599 decomposed chars + + The upshot is that decomposition code is very simple and easy to inline + at mild code size cost. + """ + def _decompose(char_int, compatible): + # 7-bit ASCII never decomposes + if char_int <= 0x7f: + yield char_int + return + + # Assert that we're handling Hangul separately. + assert not (S_BASE <= char_int < S_BASE + S_COUNT) + + decomp = self.canon_decomp.get(char_int) + if decomp is not None: + for decomposed_ch in decomp: + for fully_decomposed_ch in _decompose(decomposed_ch, compatible): + yield fully_decomposed_ch + return + + if compatible and char_int in self.compat_decomp: + for decomposed_ch in self.compat_decomp[char_int]: + for fully_decomposed_ch in _decompose(decomposed_ch, compatible): + yield fully_decomposed_ch + return + + yield char_int + return + + end_codepoint = max( + max(self.canon_decomp.keys()), + max(self.compat_decomp.keys()), + ) + + canon_fully_decomp = {} + compat_fully_decomp = {} + + for char_int in range(0, end_codepoint + 1): + # Always skip Hangul, since it's more efficient to represent its + # decomposition programmatically. + if S_BASE <= char_int < S_BASE + S_COUNT: + continue + + canon = list(_decompose(char_int, False)) + if not (len(canon) == 1 and canon[0] == char_int): + canon_fully_decomp[char_int] = canon + + compat = list(_decompose(char_int, True)) + if not (len(compat) == 1 and compat[0] == char_int): + compat_fully_decomp[char_int] = compat + + # Since canon_fully_decomp is a subset of compat_fully_decomp, we don't + # need to store their overlap when they agree. When they don't agree, + # store the decomposition in the compatibility table since we'll check + # that first when normalizing to NFKD. + assert set(canon_fully_decomp) <= set(compat_fully_decomp) + + for ch in set(canon_fully_decomp) & set(compat_fully_decomp): + if canon_fully_decomp[ch] == compat_fully_decomp[ch]: + del compat_fully_decomp[ch] + + return canon_fully_decomp, compat_fully_decomp + + def _compute_stream_safe_tables(self): + """ + To make a text stream-safe with the Stream-Safe Text Process (UAX15-D4), + we need to be able to know the number of contiguous non-starters *after* + applying compatibility decomposition to each character. + + We can do this incrementally by computing the number of leading and + trailing non-starters for each character's compatibility decomposition + with the following rules: + + 1) If a character is not affected by compatibility decomposition, look + up its canonical combining class to find out if it's a non-starter. + 2) All Hangul characters are starters, even under decomposition. + 3) Otherwise, very few decomposing characters have a nonzero count + of leading or trailing non-starters, so store these characters + with their associated counts in a separate table. + """ + leading_nonstarters = {} + trailing_nonstarters = {} + + for c in set(self.canon_fully_decomp) | set(self.compat_fully_decomp): + decomposed = self.compat_fully_decomp.get(c) or self.canon_fully_decomp[c] + + num_leading = 0 + for d in decomposed: + if d not in self.combining_classes: + break + num_leading += 1 + + num_trailing = 0 + for d in reversed(decomposed): + if d not in self.combining_classes: + break + num_trailing += 1 + + if num_leading > 0: + leading_nonstarters[c] = num_leading + if num_trailing > 0: + trailing_nonstarters[c] = num_trailing + + return leading_nonstarters, trailing_nonstarters + +hexify = lambda c: '{:04X}'.format(c) + +# Test whether `first` and `last` are corresponding "<..., First>" and +# "<..., Last>" markers. +def is_first_and_last(first, last): + if not first.startswith('<') or not first.endswith(', First>'): + return False + if not last.startswith('<') or not last.endswith(', Last>'): + return False + return first[1:-8] == last[1:-7] + +def gen_mph_data(name, d, kv_type, kv_callback): + (salt, keys) = minimal_perfect_hash(d) + out.write("pub(crate) const %s_SALT: &[u16] = &[\n" % name.upper()) + for s in salt: + out.write(" 0x{:x},\n".format(s)) + out.write("];\n") + out.write("pub(crate) const {}_KV: &[{}] = &[\n".format(name.upper(), kv_type)) + for k in keys: + out.write(" {},\n".format(kv_callback(k))) + out.write("];\n\n") + +def gen_combining_class(combining_classes, out): + gen_mph_data('canonical_combining_class', combining_classes, 'u32', + lambda k: "0x{:X}".format(int(combining_classes[k]) | (k << 8))) + +def gen_composition_table(canon_comp, out): + table = {} + for (c1, c2), c3 in canon_comp.items(): + if c1 < 0x10000 and c2 < 0x10000: + table[(c1 << 16) | c2] = c3 + (salt, keys) = minimal_perfect_hash(table) + gen_mph_data('COMPOSITION_TABLE', table, '(u32, char)', + lambda k: "(0x%s, '\\u{%s}')" % (hexify(k), hexify(table[k]))) + + out.write("pub(crate) fn composition_table_astral(c1: char, c2: char) -> Option<char> {\n") + out.write(" match (c1, c2) {\n") + for (c1, c2), c3 in sorted(canon_comp.items()): + if c1 >= 0x10000 and c2 >= 0x10000: + out.write(" ('\\u{%s}', '\\u{%s}') => Some('\\u{%s}'),\n" % (hexify(c1), hexify(c2), hexify(c3))) + + out.write(" _ => None,\n") + out.write(" }\n") + out.write("}\n") + +def gen_decomposition_tables(canon_decomp, compat_decomp, cjk_compat_variants_decomp, out): + tables = [(canon_decomp, 'canonical'), (compat_decomp, 'compatibility'), (cjk_compat_variants_decomp, 'cjk_compat_variants')] + for table, name in tables: + offsets = {} + offset = 0 + out.write("pub(crate) const %s_DECOMPOSED_CHARS: &[char] = &[\n" % name.upper()) + for k, v in table.items(): + offsets[k] = offset + offset += len(v) + for c in v: + out.write(" '\\u{%s}',\n" % hexify(c)) + # The largest offset must fit in a u16. + assert offset < 65536 + out.write("];\n") + gen_mph_data(name + '_decomposed', table, "(u32, (u16, u16))", + lambda k: "(0x{:x}, ({}, {}))".format(k, offsets[k], len(table[k]))) + +def gen_qc_match(prop_table, out): + out.write(" match c {\n") + + for low, high, data in prop_table: + assert data in ('N', 'M') + result = "No" if data == 'N' else "Maybe" + if high: + out.write(r" '\u{%s}'...'\u{%s}' => %s," % (low, high, result)) + else: + out.write(r" '\u{%s}' => %s," % (low, result)) + out.write("\n") + + out.write(" _ => Yes,\n") + out.write(" }\n") + +def gen_nfc_qc(prop_tables, out): + out.write("#[inline]\n") + out.write("#[allow(ellipsis_inclusive_range_patterns)]\n") + out.write("pub fn qc_nfc(c: char) -> IsNormalized {\n") + gen_qc_match(prop_tables['NFC_QC'], out) + out.write("}\n") + +def gen_nfkc_qc(prop_tables, out): + out.write("#[inline]\n") + out.write("#[allow(ellipsis_inclusive_range_patterns)]\n") + out.write("pub fn qc_nfkc(c: char) -> IsNormalized {\n") + gen_qc_match(prop_tables['NFKC_QC'], out) + out.write("}\n") + +def gen_nfd_qc(prop_tables, out): + out.write("#[inline]\n") + out.write("#[allow(ellipsis_inclusive_range_patterns)]\n") + out.write("pub fn qc_nfd(c: char) -> IsNormalized {\n") + gen_qc_match(prop_tables['NFD_QC'], out) + out.write("}\n") + +def gen_nfkd_qc(prop_tables, out): + out.write("#[inline]\n") + out.write("#[allow(ellipsis_inclusive_range_patterns)]\n") + out.write("pub fn qc_nfkd(c: char) -> IsNormalized {\n") + gen_qc_match(prop_tables['NFKD_QC'], out) + out.write("}\n") + +def gen_combining_mark(general_category_mark, out): + gen_mph_data('combining_mark', general_category_mark, 'u32', + lambda k: '0x{:04x}'.format(k)) + +def gen_public_assigned(general_category_public_assigned, out): + # This could be done as a hash but the table is somewhat small. + out.write("#[inline]\n") + out.write("pub fn is_public_assigned(c: char) -> bool {\n") + out.write(" match c {\n") + + start = True + for first, last in general_category_public_assigned: + if start: + out.write(" ") + start = False + else: + out.write(" | ") + if first == last: + out.write("'\\u{%s}'\n" % hexify(first)) + else: + out.write("'\\u{%s}'..='\\u{%s}'\n" % (hexify(first), hexify(last))) + out.write(" => true,\n") + + out.write(" _ => false,\n") + out.write(" }\n") + out.write("}\n") + out.write("\n") + +def gen_stream_safe(leading, trailing, out): + # This could be done as a hash but the table is very small. + out.write("#[inline]\n") + out.write("pub fn stream_safe_leading_nonstarters(c: char) -> usize {\n") + out.write(" match c {\n") + + for char, num_leading in sorted(leading.items()): + out.write(" '\\u{%s}' => %d,\n" % (hexify(char), num_leading)) + + out.write(" _ => 0,\n") + out.write(" }\n") + out.write("}\n") + out.write("\n") + + gen_mph_data('trailing_nonstarters', trailing, 'u32', + lambda k: "0x{:X}".format(int(trailing[k]) | (k << 8))) + +def gen_tests(tests, out): + out.write("""#[derive(Debug)] +pub struct NormalizationTest { + pub source: &'static str, + pub nfc: &'static str, + pub nfd: &'static str, + pub nfkc: &'static str, + pub nfkd: &'static str, +} + +""") + + out.write("pub const NORMALIZATION_TESTS: &[NormalizationTest] = &[\n") + str_literal = lambda s: '"%s"' % "".join("\\u{%s}" % c for c in s) + + for test in tests: + out.write(" NormalizationTest {\n") + out.write(" source: %s,\n" % str_literal(test.source)) + out.write(" nfc: %s,\n" % str_literal(test.nfc)) + out.write(" nfd: %s,\n" % str_literal(test.nfd)) + out.write(" nfkc: %s,\n" % str_literal(test.nfkc)) + out.write(" nfkd: %s,\n" % str_literal(test.nfkd)) + out.write(" },\n") + + out.write("];\n") + +# Guaranteed to be less than n. +def my_hash(x, salt, n): + # This is hash based on the theory that multiplication is efficient + mask_32 = 0xffffffff + y = ((x + salt) * 2654435769) & mask_32 + y ^= (x * 0x31415926) & mask_32 + return (y * n) >> 32 + +# Compute minimal perfect hash function, d can be either a dict or list of keys. +def minimal_perfect_hash(d): + n = len(d) + buckets = dict((h, []) for h in range(n)) + for key in d: + h = my_hash(key, 0, n) + buckets[h].append(key) + bsorted = [(len(buckets[h]), h) for h in range(n)] + bsorted.sort(reverse = True) + claimed = [False] * n + salts = [0] * n + keys = [0] * n + for (bucket_size, h) in bsorted: + # Note: the traditional perfect hashing approach would also special-case + # bucket_size == 1 here and assign any empty slot, rather than iterating + # until rehash finds an empty slot. But we're not doing that so we can + # avoid the branch. + if bucket_size == 0: + break + else: + for salt in range(1, 32768): + rehashes = [my_hash(key, salt, n) for key in buckets[h]] + # Make sure there are no rehash collisions within this bucket. + if all(not claimed[hash] for hash in rehashes): + if len(set(rehashes)) < bucket_size: + continue + salts[h] = salt + for key in buckets[h]: + rehash = my_hash(key, salt, n) + claimed[rehash] = True + keys[rehash] = key + break + if salts[h] == 0: + print("minimal perfect hashing failed") + # Note: if this happens (because of unfortunate data), then there are + # a few things that could be done. First, the hash function could be + # tweaked. Second, the bucket order could be scrambled (especially the + # singletons). Right now, the buckets are sorted, which has the advantage + # of being deterministic. + # + # As a more extreme approach, the singleton bucket optimization could be + # applied (give the direct address for singleton buckets, rather than + # relying on a rehash). That is definitely the more standard approach in + # the minimal perfect hashing literature, but in testing the branch was a + # significant slowdown. + exit(1) + return (salts, keys) + +if __name__ == '__main__': + data = UnicodeData() + with open("tables.rs", "w", newline = "\n") as out: + out.write(PREAMBLE) + out.write("use crate::quick_check::IsNormalized;\n") + out.write("use crate::quick_check::IsNormalized::*;\n") + out.write("\n") + + version = "(%s, %s, %s)" % tuple(UNICODE_VERSION.split(".")) + out.write("#[allow(unused)]\n") + out.write("pub const UNICODE_VERSION: (u8, u8, u8) = %s;\n\n" % version) + + gen_combining_class(data.combining_classes, out) + out.write("\n") + + gen_composition_table(data.canon_comp, out) + out.write("\n") + + gen_decomposition_tables(data.canon_fully_decomp, data.compat_fully_decomp, data.cjk_compat_variants_fully_decomp, out) + + gen_combining_mark(data.general_category_mark, out) + out.write("\n") + + gen_public_assigned(data.general_category_public_assigned, out) + out.write("\n") + + gen_nfc_qc(data.norm_props, out) + out.write("\n") + + gen_nfkc_qc(data.norm_props, out) + out.write("\n") + + gen_nfd_qc(data.norm_props, out) + out.write("\n") + + gen_nfkd_qc(data.norm_props, out) + out.write("\n") + + gen_stream_safe(data.ss_leading, data.ss_trailing, out) + out.write("\n") + + with open("normalization_tests.rs", "w", newline = "\n") as out: + out.write(PREAMBLE) + gen_tests(data.norm_tests, out) |