summaryrefslogtreecommitdiffstats
path: root/intl/unicharutil/tools
diff options
context:
space:
mode:
Diffstat (limited to 'intl/unicharutil/tools')
-rw-r--r--intl/unicharutil/tools/README.txt3
-rwxr-xr-xintl/unicharutil/tools/genSpecialCasingData.pl309
-rwxr-xr-xintl/unicharutil/tools/genUnicodePropertyData.pl495
3 files changed, 807 insertions, 0 deletions
diff --git a/intl/unicharutil/tools/README.txt b/intl/unicharutil/tools/README.txt
new file mode 100644
index 0000000000..7295547f11
--- /dev/null
+++ b/intl/unicharutil/tools/README.txt
@@ -0,0 +1,3 @@
+Instructions for using these tools are on the Mozilla Wiki.
+
+https://wiki.mozilla.org/I18n:Updating_Unicode_version
diff --git a/intl/unicharutil/tools/genSpecialCasingData.pl b/intl/unicharutil/tools/genSpecialCasingData.pl
new file mode 100755
index 0000000000..9800688683
--- /dev/null
+++ b/intl/unicharutil/tools/genSpecialCasingData.pl
@@ -0,0 +1,309 @@
+#!/usr/bin/env perl
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This tool is used to extract "special" (one-to-many) case mappings
+# into a form that can be used by nsTextRunTransformations.
+
+use strict;
+
+if ($#ARGV != 1) {
+ print <<__EOT;
+# Run this tool using a command line of the form
+#
+# perl genSpecialCasingData.pl UnicodeData.txt SpecialCasing.txt
+#
+# The nsSpecialCasingData.cpp file will be written to standard output.
+#
+# This tool will also write up-to-date versions of the test files
+# all-{upper,lower,title}.html
+# and corresponding -ref files in the current directory.
+#
+__EOT
+ exit 0;
+}
+
+my %allLower;
+my %allUpper;
+my %allTitle;
+my %compositions;
+my %gc;
+open FH, "< $ARGV[0]" or die "can't open $ARGV[0] (should be UnicodeData.txt)\n";
+while (<FH>) {
+ chomp;
+ my @fields = split /;/;
+ next if ($fields[1] =~ /</); # ignore ranges etc
+ my $usv = hex "0x$fields[0]";
+ $allUpper{$usv} = $fields[12] if $fields[12] ne '';
+ $allLower{$usv} = $fields[13] if $fields[13] ne '';
+ $allTitle{$usv} = $fields[14] if $fields[14] ne '';
+ $gc{$usv} = $fields[2];
+ # we only care about non-singleton canonical decomps
+ my $decomp = $fields[5];
+ next if $decomp eq '' or $decomp =~ /</ or not $decomp =~ / /;
+ $compositions{$decomp} = sprintf("%04X", $usv);
+}
+close FH;
+
+my %specialLower;
+my %specialUpper;
+my %specialTitle;
+my %charName;
+my @headerLines;
+open FH, "< $ARGV[1]" or die "can't open $ARGV[1] (should be SpecialCasing.txt)\n";
+while (<FH>) {
+ chomp;
+ m/#\s*(.+)$/;
+ my $comment = $1;
+ if ($comment =~ /^(SpecialCasing-|Date:)/) {
+ push @headerLines, $comment;
+ next;
+ }
+ s/#.*//;
+ s/;\s*$//;
+ next if $_ eq '';
+ my @fields = split /; */;
+ next unless (scalar @fields) == 4;
+ my $usv = hex "0x$fields[0]";
+ addIfSpecial(\%specialLower, $usv, $fields[1]);
+ addIfSpecial(\%specialTitle, $usv, $fields[2]);
+ addIfSpecial(\%specialUpper, $usv, $fields[3]);
+ $charName{$usv} = $comment;
+}
+close FH;
+
+print <<__END__;
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* Auto-generated from files in the Unicode Character Database
+ by genSpecialCasingData.pl - do not edit! */
+
+#include "nsSpecialCasingData.h"
+#include "mozilla/ArrayUtils.h" // for ArrayLength
+#include <stdlib.h> // for bsearch
+
+__END__
+map { print "/* $_ */\n" } @headerLines;
+
+print <<__END__;
+
+using mozilla::unicode::MultiCharMapping;
+
+__END__
+
+printMappings('Lower', \%specialLower);
+printMappings('Upper', \%specialUpper);
+printMappings('Title', \%specialTitle);
+
+print <<__END__;
+static int CompareMCM(const void* aKey, const void* aElement)
+{
+ const uint32_t ch = *static_cast<const uint32_t*>(aKey);
+ const MultiCharMapping* mcm = static_cast<const MultiCharMapping*>(aElement);
+ return int(ch) - int(mcm->mOriginalChar);
+}
+
+#define MAKE_SPECIAL_CASE_ACCESSOR(which) \\
+ const MultiCharMapping* \\
+ Special##which(uint32_t aChar) \\
+ { \\
+ const void* p = bsearch(&aChar, CaseSpecials_##which, \\
+ mozilla::ArrayLength(CaseSpecials_##which), \\
+ sizeof(MultiCharMapping), CompareMCM); \\
+ return static_cast<const MultiCharMapping*>(p); \\
+ }
+
+namespace mozilla {
+namespace unicode {
+
+MAKE_SPECIAL_CASE_ACCESSOR(Lower)
+MAKE_SPECIAL_CASE_ACCESSOR(Upper)
+MAKE_SPECIAL_CASE_ACCESSOR(Title)
+
+} // namespace unicode
+} // namespace mozilla
+__END__
+
+addSpecialsTo(\%allLower, \%specialLower);
+addSpecialsTo(\%allUpper, \%specialUpper);
+addSpecialsTo(\%allTitle, \%specialTitle);
+
+my $testFont = "../fonts/dejavu-sans/DejaVuSans.ttf";
+genTest('lower', \%allLower);
+genTest('upper', \%allUpper);
+genTitleTest();
+
+sub printMappings {
+ my ($whichMapping, $hash) = @_;
+ print "static const MultiCharMapping CaseSpecials_${whichMapping}[] = {\n";
+ foreach my $key (sort { $a <=> $b } keys %$hash) {
+ my @chars = split(/ /, $hash->{$key});
+ printf " { 0x%04x, {0x%04x, 0x%04x, 0x%04x} }, // %s\n", $key,
+ hex "0x0$chars[0]", hex "0x0$chars[1]", hex "0x0$chars[2]",
+ "$charName{$key}";
+ }
+ print "};\n\n";
+};
+
+sub addIfSpecial {
+ my ($hash, $usv, $mapping) = @_;
+ return unless $mapping =~ / /;
+ # only do compositions that start with the initial char
+ foreach (keys %compositions) {
+ $mapping =~ s/^$_/$compositions{$_}/;
+ }
+ $hash->{$usv} = $mapping;
+};
+
+sub addSpecialsTo {
+ my ($hash, $specials) = @_;
+ foreach my $key (keys %$specials) {
+ $hash->{$key} = $specials->{$key};
+ }
+};
+
+sub genTest {
+ my ($whichMapping, $hash) = @_;
+ open OUT, "> all-$whichMapping.html";
+ print OUT <<__END__;
+<!DOCTYPE html>
+<!-- GENERATED FILE, DO NOT EDIT -->
+<html>
+ <head>
+ <meta http-equiv="Content-type" content="text/html; charset=utf-8">
+ <style type="text/css">
+ \@font-face { font-family: foo; src: url($testFont); }
+ p { font-family: foo; font-size: 12px; text-transform: ${whichMapping}case; }
+ </style>
+ </head>
+ <body>
+ <p>
+__END__
+ foreach my $key (sort { $a <=> $b } keys %$hash) {
+ # Bug 1476304: we exclude Georgian letters U+10D0..10FF because of lack
+ # of widespread font support for the corresponding Mtavruli characters
+ # at this time (July 2018).
+ # This condition is to be removed once the major platforms ship with
+ # fonts that support U+1C90..1CBF.
+ my $skippedGeorgian = $whichMapping eq "upper" && $key >= 0x10D0 && $key <= 0x10FF;
+ print OUT "<!-- " if $skippedGeorgian;
+ printf OUT "&#x%04X;", $key;
+ print OUT " -->" if $skippedGeorgian;
+ print OUT " <!-- $charName{$key} -->" if exists $charName{$key};
+ print OUT " <!-- Temporarily skipped, see bug 1476304. -->" if $skippedGeorgian;
+ print OUT "\n";
+ }
+ print OUT <<__END__;
+ </p>
+ </body>
+</html>
+__END__
+ close OUT;
+
+ open OUT, "> all-$whichMapping-ref.html";
+ print OUT <<__END__;
+<!DOCTYPE html>
+<!-- GENERATED FILE, DO NOT EDIT -->
+<html>
+ <head>
+ <meta http-equiv="Content-type" content="text/html; charset=utf-8">
+ <style type="text/css">
+ \@font-face { font-family: foo; src: url($testFont); }
+ p { font-family: foo; font-size: 12px; }
+ </style>
+ </head>
+ <body>
+ <p>
+__END__
+ foreach my $key (sort { $a <=> $b } keys %$hash) {
+ # Bug 1476304: we exclude Georgian letters U+10D0..10FF because of lack
+ # of widespread font support for the corresponding Mtavruli characters
+ # at this time (July 2018).
+ # This condition is to be removed once the major platforms ship with
+ # fonts that support U+1C90..1CBF.
+ my $skippedGeorgian = $whichMapping eq "upper" && $key >= 0x10D0 && $key <= 0x10FF;
+ print OUT "<!-- " if $skippedGeorgian;
+ print OUT join('', map { sprintf("&#x%s;", $_) } split(/ /, $hash->{$key}));
+ print OUT " -->" if $skippedGeorgian;
+ print OUT " <!-- $charName{$key} -->" if exists $charName{$key};
+ print OUT " <!-- Temporarily skipped, see bug 1476304. -->" if $skippedGeorgian;
+ print OUT "\n";
+ }
+ print OUT <<__END__;
+ </p>
+ </body>
+</html>
+__END__
+ close OUT;
+};
+
+sub genTitleTest {
+ open OUT, "> all-title.html";
+ print OUT <<__END__;
+<!DOCTYPE html>
+<!-- GENERATED FILE, DO NOT EDIT -->
+<html>
+ <head>
+ <meta http-equiv="Content-type" content="text/html; charset=utf-8">
+ <style type="text/css">
+ \@font-face { font-family: foo; src: url($testFont); }
+ p { font-family: foo; text-transform: capitalize; }
+ </style>
+ </head>
+ <body>
+ <p>
+__END__
+ foreach my $key (sort { $a <=> $b } keys %allTitle) {
+ printf OUT "&#x%04X;x", $key;
+ print OUT " <!-- $charName{$key} -->" if exists $charName{$key};
+ print OUT "\n";
+ }
+ print OUT <<__END__;
+ </p>
+ </body>
+</html>
+__END__
+ close OUT;
+
+ open OUT, "> all-title-ref.html";
+ print OUT <<__END__;
+<!DOCTYPE html>
+<!-- GENERATED FILE, DO NOT EDIT -->
+<html>
+ <head>
+ <meta http-equiv="Content-type" content="text/html; charset=utf-8">
+ <style type="text/css">
+ \@font-face { font-family: foo; src: url($testFont); }
+ p { font-family: foo; }
+ </style>
+ </head>
+ <body>
+ <p>
+__END__
+ foreach my $key (sort { $a <=> $b } keys %allTitle) {
+ # capitalize is only applied to characters with GC=L* or N*...
+ if ($gc{$key} =~ /^[LN]/) {
+ # ...and those that are already uppercase are not transformed
+ if (exists $allUpper{$key}) {
+ print OUT join('', map { sprintf("&#x%s;", $_) } split(/ /, $allTitle{$key}));
+ } else {
+ printf OUT "&#x%04X;", $key;
+ }
+ print OUT "x";
+ } else {
+ printf OUT "&#x%04X;X", $key;
+ }
+ print OUT " <!-- $charName{$key} -->" if exists $charName{$key};
+ print OUT "\n";
+ }
+ print OUT <<__END__;
+ </p>
+ </body>
+</html>
+__END__
+ close OUT;
+};
diff --git a/intl/unicharutil/tools/genUnicodePropertyData.pl b/intl/unicharutil/tools/genUnicodePropertyData.pl
new file mode 100755
index 0000000000..f72f18f1ae
--- /dev/null
+++ b/intl/unicharutil/tools/genUnicodePropertyData.pl
@@ -0,0 +1,495 @@
+#!/usr/bin/env perl
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This tool is used to prepare lookup tables of Unicode character properties
+# needed by gfx code to support text shaping operations. The properties are
+# read from the Unicode Character Database and compiled into multi-level arrays
+# for efficient lookup.
+#
+# Note that for most properties, we now rely on ICU; this tool and the tables
+# it generates are used only for a couple of properties not readily exposed
+# via ICU APIs.
+#
+# To regenerate the tables in nsUnicodePropertyData.cpp:
+#
+# (1) Download the current Unicode data files from
+#
+# https://www.unicode.org/Public/UNIDATA/
+#
+# NB: not all the files are actually needed; currently, we require
+# - UnicodeData.txt
+# - ReadMe.txt (to record version/date of the UCD)
+# - Unihan_Variants.txt (from Unihan.zip)
+# though this may change if we find a need for additional properties.
+#
+# The Unicode data files listed above should be together in one directory.
+#
+# We also require the file
+# https://www.unicode.org/Public/security/latest/IdentifierStatus.txt
+# This file should be in a sub-directory "security" immediately below the
+# directory containing the other Unicode data files.
+#
+# We also require the latest data file for UTR50, currently revision-17:
+# https://www.unicode.org/Public/vertical/revision-17/VerticalOrientation-17.txt
+# This file should be in a sub-directory "vertical" immediately below the
+# directory containing the other Unicode data files.
+#
+#
+# (2) Run this tool using a command line of the form
+#
+# perl genUnicodePropertyData.pl \
+# /path/to/icu/common/unicode \
+# /path/to/UCD-directory
+#
+# This will generate (or overwrite!) the files
+#
+# nsUnicodePropertyData.cpp
+# UnicodeScriptCodes.h
+#
+# in the current directory.
+
+use strict;
+use List::Util qw(first);
+
+if ($#ARGV != 1) {
+ print <<__EOT;
+# Run this tool using a command line of the form
+#
+# perl genUnicodePropertyData.pl \\
+# /path/to/icu/common/unicode \\
+# /path/to/UCD-directory
+#
+# where icu/common/unicode is the directory containing ICU 'common' headers,
+# and UCD-directory is a directory containing the current Unicode Character
+# Database files (UnicodeData.txt, etc), available from
+# https://www.unicode.org/Public/UNIDATA/, with additional resources as
+# detailed in the source comments.
+#
+# This will generate (or overwrite!) the files
+#
+# nsUnicodePropertyData.cpp
+# UnicodeScriptCodes.h
+#
+# in the current directory.
+__EOT
+ exit 0;
+}
+
+my $ICU = $ARGV[0];
+my $UNICODE = $ARGV[1];
+
+my @scriptCodeToName;
+my @idtype;
+
+my $sc = -1;
+
+sub readIcuHeader
+{
+ my $file = shift;
+ open FH, "< $ICU/$file" or die "can't open ICU header $ICU/$file\n";
+ while (<FH>) {
+ # adjust for ICU vs UCD naming discrepancies
+ s/LANNA/TAI_THAM/;
+ s/MEITEI_MAYEK/MEETEI_MAYEK/;
+ s/ORKHON/OLD_TURKIC/;
+ s/MENDE/MENDE_KIKAKUI/;
+ s/SIGN_WRITING/SIGNWRITING/;
+ if (m|USCRIPT_([A-Z_]+)\s*=\s*([0-9]+),\s*/\*\s*([A-Z][a-z]{3})\s*\*/|) {
+ $sc = $2;
+ $scriptCodeToName[$sc] = $1;
+ }
+ }
+ close FH;
+}
+
+&readIcuHeader("uscript.h");
+
+die "didn't find ICU script codes\n" if $sc == -1;
+
+# We don't currently store these values; %idType is used only to check that
+# properties listed in the IdentifierType.txt file are recognized. We record
+# only the %mappedIdType values that are used by nsIDNService::isLabelSafe.
+# In practice, it would be sufficient for us to read only the last value in
+# IdentifierType.txt, but we check that all values are known so that we'll get
+# a warning if future updates introduce new ones, and can consider whether
+# they need to be taken into account.
+my %idType = (
+ "Not_Character" => 0,
+ "Recommended" => 1,
+ "Inclusion" => 2,
+ "Uncommon_Use" => 3,
+ "Technical" => 4,
+ "Obsolete" => 5,
+ "Aspirational" => 6,
+ "Limited_Use" => 7,
+ "Exclusion" => 8,
+ "Not_XID" => 9,
+ "Not_NFKC" => 10,
+ "Default_Ignorable" => 11,
+ "Deprecated" => 12
+);
+
+# These match the IdentifierType enum in UnicodeProperties.h.
+my %mappedIdType = (
+ "Restricted" => 0,
+ "Allowed" => 1
+);
+
+my %verticalOrientationCode = (
+ 'U' => 0, # U - Upright, the same orientation as in the code charts
+ 'R' => 1, # R - Rotated 90 degrees clockwise compared to the code charts
+ 'Tu' => 2, # Tu - Transformed typographically, with fallback to Upright
+ 'Tr' => 3 # Tr - Transformed typographically, with fallback to Rotated
+);
+
+# initialize default properties
+my @hanVariant;
+my @fullWidth;
+my @fullWidthInverse;
+my @verticalOrientation;
+for (my $i = 0; $i < 0x110000; ++$i) {
+ $hanVariant[$i] = 0;
+ $fullWidth[$i] = 0;
+ $fullWidthInverse[$i] = 0;
+ $verticalOrientation[$i] = 1; # default for unlisted codepoints is 'R'
+}
+
+# read ReadMe.txt
+my @versionInfo;
+open FH, "< $UNICODE/ReadMe.txt" or die "can't open Unicode ReadMe.txt file\n";
+while (<FH>) {
+ chomp;
+ push @versionInfo, $_;
+}
+close FH;
+
+# read UnicodeData.txt
+open FH, "< $UNICODE/UnicodeData.txt" or die "can't open UCD file UnicodeData.txt\n";
+while (<FH>) {
+ chomp;
+ my @fields = split /;/;
+ if ($fields[1] =~ /First/) {
+ my $first = hex "0x$fields[0]";
+ $_ = <FH>;
+ @fields = split /;/;
+ if ($fields[1] =~ /Last/) {
+ my $last = hex "0x$fields[0]";
+ do {
+ if ($fields[1] =~ /CJK/) {
+ @hanVariant[$first] = 3;
+ }
+ $first++;
+ } while ($first <= $last);
+ } else {
+ die "didn't find Last code for range!\n";
+ }
+ } else {
+ my $usv = hex "0x$fields[0]";
+ if ($fields[1] =~ /CJK/) {
+ @hanVariant[$usv] = 3;
+ }
+ if ($fields[5] =~ /^<narrow>/) {
+ my $wideChar = hex(substr($fields[5], 9));
+ die "didn't expect supplementary-plane values here" if $usv > 0xffff || $wideChar > 0xffff;
+ $fullWidth[$usv] = $wideChar;
+ $fullWidthInverse[$wideChar] = $usv;
+ }
+ elsif ($fields[5] =~ /^<wide>/) {
+ my $narrowChar = hex(substr($fields[5], 7));
+ die "didn't expect supplementary-plane values here" if $usv > 0xffff || $narrowChar > 0xffff;
+ $fullWidth[$narrowChar] = $usv;
+ $fullWidthInverse[$usv] = $narrowChar;
+ }
+ }
+}
+close FH;
+
+# read IdentifierStatus.txt
+open FH, "< $UNICODE/security/IdentifierStatus.txt" or die "can't open UCD file IdentifierStatus.txt\n";
+push @versionInfo, "";
+while (<FH>) {
+ chomp;
+ s/\xef\xbb\xbf//;
+ push @versionInfo, $_;
+ last if /Date:/;
+}
+while (<FH>) {
+ if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s+;\s+Allowed/) {
+ my $start = hex "0x$1";
+ my $end = (defined $2) ? hex "0x$2" : $start;
+ for (my $i = $start; $i <= $end; ++$i) {
+ $idtype[$i] = $mappedIdType{'Allowed'};
+ }
+ }
+}
+close FH;
+
+open FH, "< $UNICODE/Unihan_Variants.txt" or die "can't open UCD file Unihan_Variants.txt (from Unihan.zip)\n";
+push @versionInfo, "";
+while (<FH>) {
+ chomp;
+ push @versionInfo, $_;
+ last if /Date:/;
+}
+my $savedusv = 0;
+my $hasTC = 0;
+my $hasSC = 0;
+while (<FH>) {
+ chomp;
+ if (m/U\+([0-9A-F]{4,6})\s+k([^ ]+)Variant/) {
+ my $usv = hex "0x$1";
+ if ($usv != $savedusv) {
+ unless ($savedusv == 0) {
+ if ($hasTC && !$hasSC) {
+ $hanVariant[$savedusv] = 1;
+ } elsif (!$hasTC && $hasSC) {
+ $hanVariant[$savedusv] = 2;
+ }
+ }
+ $savedusv = $usv;
+ $hasTC = 0;
+ $hasSC = 0;
+ }
+ if ($2 eq "Traditional") {
+ $hasTC = 1;
+ }
+ if ($2 eq "Simplified") {
+ $hasSC = 1;
+ }
+ }
+}
+close FH;
+
+# read VerticalOrientation-17.txt
+open FH, "< $UNICODE/vertical/VerticalOrientation-17.txt" or die "can't open UTR50 data file VerticalOrientation-17.txt\n";
+push @versionInfo, "";
+while (<FH>) {
+ chomp;
+ push @versionInfo, $_;
+ last if /Date:/;
+}
+while (<FH>) {
+ chomp;
+ s/#.*//;
+ if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s*;\s*([^ ]+)/) {
+ my $vo = $3;
+ warn "unknown Vertical_Orientation code $vo"
+ unless exists $verticalOrientationCode{$vo};
+ $vo = $verticalOrientationCode{$vo};
+ my $start = hex "0x$1";
+ my $end = (defined $2) ? hex "0x$2" : $start;
+ for (my $i = $start; $i <= $end; ++$i) {
+ $verticalOrientation[$i] = $vo;
+ }
+ }
+}
+close FH;
+
+my $timestamp = gmtime();
+
+open DATA_TABLES, "> nsUnicodePropertyData.cpp" or die "unable to open nsUnicodePropertyData.cpp for output";
+
+my $licenseBlock = q[/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * Derived from the Unicode Character Database by genUnicodePropertyData.pl
+ *
+ * For Unicode terms of use, see http://www.unicode.org/terms_of_use.html
+ */
+];
+
+my $versionInfo = join("\n", @versionInfo);
+
+print DATA_TABLES <<__END;
+$licenseBlock
+/*
+ * Created on $timestamp from UCD data files with version info:
+ *
+
+$versionInfo
+
+ *
+ * * * * * This file contains MACHINE-GENERATED DATA, do not edit! * * * * *
+ */
+
+#include <stdint.h>
+#include "harfbuzz/hb.h"
+
+__END
+
+open HEADER, "> UnicodeScriptCodes.h" or die "unable to open UnicodeScriptCodes.h for output";
+
+print HEADER <<__END;
+$licenseBlock
+/*
+ * Created on $timestamp from UCD data files with version info:
+ *
+
+$versionInfo
+
+ *
+ * * * * * This file contains MACHINE-GENERATED DATA, do not edit! * * * * *
+ */
+
+#ifndef intl_components_UnicodeScriptCodes_h_
+#define intl_components_UnicodeScriptCodes_h_
+
+__END
+
+our $totalData = 0;
+
+sub sprintCharProps2_short
+{
+ my $usv = shift;
+ return sprintf("{%d,%d},",
+ $verticalOrientation[$usv], $idtype[$usv]);
+}
+&genTables("CharProp2", "", "nsCharProps2", 9, 7, \&sprintCharProps2_short, 16, 1, 1);
+
+sub sprintHanVariants
+{
+ my $baseUsv = shift;
+ my $varShift = 0;
+ my $val = 0;
+ while ($varShift < 8) {
+ $val |= $hanVariant[$baseUsv++] << $varShift;
+ $varShift += 2;
+ }
+ return sprintf("0x%02x,", $val);
+}
+## Han Variant data currently unused but may be needed in future, see bug 857481
+## &genTables("HanVariant", "", "uint8_t", 9, 7, \&sprintHanVariants, 2, 1, 4);
+
+sub sprintFullWidth
+{
+ my $usv = shift;
+ return sprintf("0x%04x,", $fullWidth[$usv]);
+}
+&genTables("FullWidth", "", "uint16_t", 10, 6, \&sprintFullWidth, 0, 2, 1);
+
+sub sprintFullWidthInverse
+{
+ my $usv = shift;
+ return sprintf("0x%04x,", $fullWidthInverse[$usv]);
+}
+&genTables("FullWidthInverse", "", "uint16_t", 10, 6, \&sprintFullWidthInverse, 0, 2, 1);
+
+print STDERR "Total data = $totalData\n";
+
+sub genTables
+{
+ my ($prefix, $typedef, $type, $indexBits, $charBits, $func, $maxPlane, $bytesPerEntry, $charsPerEntry) = @_;
+
+ if ($typedef ne '') {
+ print HEADER "$typedef\n";
+ }
+
+ print DATA_TABLES "#define k${prefix}MaxPlane $maxPlane\n";
+ print DATA_TABLES "#define k${prefix}IndexBits $indexBits\n";
+ print DATA_TABLES "#define k${prefix}CharBits $charBits\n";
+
+ my $indexLen = 1 << $indexBits;
+ my $charsPerPage = 1 << $charBits;
+ my %charIndex = ();
+ my %pageMapIndex = ();
+ my @pageMap = ();
+ my @char = ();
+
+ my $planeMap = "\x00" x $maxPlane;
+ foreach my $plane (0 .. $maxPlane) {
+ my $pageMap = "\x00" x $indexLen * 2;
+ foreach my $page (0 .. $indexLen - 1) {
+ my $charValues = "";
+ for (my $ch = 0; $ch < $charsPerPage; $ch += $charsPerEntry) {
+ my $usv = $plane * 0x10000 + $page * $charsPerPage + $ch;
+ $charValues .= &$func($usv);
+ }
+ chop $charValues;
+
+ unless (exists $charIndex{$charValues}) {
+ $charIndex{$charValues} = scalar keys %charIndex;
+ $char[$charIndex{$charValues}] = $charValues;
+ }
+ substr($pageMap, $page * 2, 2) = pack('S', $charIndex{$charValues});
+ }
+
+ unless (exists $pageMapIndex{$pageMap}) {
+ $pageMapIndex{$pageMap} = scalar keys %pageMapIndex;
+ $pageMap[$pageMapIndex{$pageMap}] = $pageMap;
+ }
+ if ($plane > 0) {
+ substr($planeMap, $plane - 1, 1) = pack('C', $pageMapIndex{$pageMap});
+ }
+ }
+
+ if ($maxPlane) {
+ print DATA_TABLES "static const uint8_t s${prefix}Planes[$maxPlane] = {";
+ print DATA_TABLES join(',', map { sprintf("%d", $_) } unpack('C*', $planeMap));
+ print DATA_TABLES "};\n\n";
+ }
+
+ my $chCount = scalar @char;
+ my $pmBits = $chCount > 255 ? 16 : 8;
+ my $pmCount = scalar @pageMap;
+ if ($maxPlane == 0) {
+ die "there should only be one pageMap entry!" if $pmCount > 1;
+ print DATA_TABLES "static const uint${pmBits}_t s${prefix}Pages[$indexLen] = {\n";
+ } else {
+ print DATA_TABLES "static const uint${pmBits}_t s${prefix}Pages[$pmCount][$indexLen] = {\n";
+ }
+ for (my $i = 0; $i < scalar @pageMap; ++$i) {
+ print DATA_TABLES $maxPlane > 0 ? " {" : " ";
+ print DATA_TABLES join(',', map { sprintf("%d", $_) } unpack('S*', $pageMap[$i]));
+ print DATA_TABLES $maxPlane > 0 ? ($i < $#pageMap ? "},\n" : "}\n") : "\n";
+ }
+ print DATA_TABLES "};\n\n";
+
+ my $pageLen = $charsPerPage / $charsPerEntry;
+ print DATA_TABLES "static const $type s${prefix}Values[$chCount][$pageLen] = {\n";
+ for (my $i = 0; $i < scalar @char; ++$i) {
+ print DATA_TABLES " {";
+ print DATA_TABLES $char[$i];
+ print DATA_TABLES $i < $#char ? "},\n" : "}\n";
+ }
+ print DATA_TABLES "};\n";
+
+ my $dataSize = $pmCount * $indexLen * $pmBits/8 +
+ $chCount * $pageLen * $bytesPerEntry +
+ $maxPlane;
+ $totalData += $dataSize;
+
+ print STDERR "Data for $prefix = $dataSize\n";
+}
+
+print DATA_TABLES <<__END;
+/*
+ * * * * * This file contains MACHINE-GENERATED DATA, do not edit! * * * * *
+ */
+__END
+
+close DATA_TABLES;
+
+print HEADER "namespace mozilla::intl {\n";
+print HEADER "enum class Script : int16_t {\n";
+for (my $i = 0; $i < scalar @scriptCodeToName; ++$i) {
+ print HEADER " ", $scriptCodeToName[$i], " = ", $i, ",\n";
+}
+print HEADER "\n NUM_SCRIPT_CODES = ", scalar @scriptCodeToName, ",\n";
+print HEADER "\n INVALID = -1\n";
+print HEADER "};\n";
+print HEADER "} // namespace mozilla::intl\n\n";
+
+print HEADER <<__END;
+#endif
+/*
+ * * * * * This file contains MACHINE-GENERATED DATA, do not edit! * * * * *
+ */
+__END
+
+close HEADER;
+