summaryrefslogtreecommitdiffstats
path: root/vendor/bstr/scripts/generate-unicode-data
blob: b8341c5a64f25d64b4406dd4dd3a44158677489a (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
#!/bin/sh

set -e
D="$(dirname "$0")"

# Convenience function for checking that a command exists.
requires() {
    cmd="$1"
    if ! command -v "$cmd" > /dev/null 2>&1; then
        echo "DEPENDENCY MISSING: $cmd must be installed" >&2
        exit 1
    fi
}

# Test if an array ($2) contains a particular element ($1).
array_exists() {
    needle="$1"
    shift

    for el in "$@"; do
        if [ "$el" = "$needle" ]; then
            return 0
        fi
    done
    return 1
}

graphemes() {
    regex="$(sh "$D/regex/grapheme.sh")"

    echo "generating forward grapheme DFA"
    ucd-generate dfa \
        --name GRAPHEME_BREAK_FWD \
        --sparse --minimize --anchored --state-size 2 \
        src/unicode/fsm/ \
        "$regex"

    echo "generating reverse grapheme DFA"
    ucd-generate dfa \
        --name GRAPHEME_BREAK_REV \
        --reverse --longest \
        --sparse --minimize --anchored --state-size 2 \
        src/unicode/fsm/ \
        "$regex"
}

words() {
    regex="$(sh "$D/regex/word.sh")"

    echo "generating forward word DFA (this can take a while)"
    ucd-generate dfa \
        --name WORD_BREAK_FWD \
        --sparse --minimize --anchored --state-size 4 \
        src/unicode/fsm/ \
        "$regex"
}

sentences() {
    regex="$(sh "$D/regex/sentence.sh")"

    echo "generating forward sentence DFA (this can take a while)"
    ucd-generate dfa \
        --name SENTENCE_BREAK_FWD \
        --minimize \
        --sparse --anchored --state-size 4 \
        src/unicode/fsm/ \
        "$regex"
}

regional_indicator() {
    # For finding all occurrences of region indicators. This is used to handle
    # regional indicators as a special case for the reverse grapheme iterator
    # and the reverse word iterator.
    echo "generating regional indicator DFA"
    ucd-generate dfa \
        --name REGIONAL_INDICATOR_REV \
        --reverse \
        --classes --minimize --anchored --premultiply --state-size 1 \
        src/unicode/fsm/ \
        "\p{gcb=Regional_Indicator}"
}

simple_word() {
    echo "generating forward simple word DFA"
    ucd-generate dfa \
        --name SIMPLE_WORD_FWD \
        --sparse --minimize --state-size 2 \
        src/unicode/fsm/ \
        "\w"
}

whitespace() {
    echo "generating forward whitespace DFA"
    ucd-generate dfa \
        --name WHITESPACE_ANCHORED_FWD \
        --anchored --classes --premultiply --minimize --state-size 1 \
        src/unicode/fsm/ \
        "\s+"

    echo "generating reverse whitespace DFA"
    ucd-generate dfa \
        --name WHITESPACE_ANCHORED_REV \
        --reverse \
        --anchored --classes --premultiply --minimize --state-size 2 \
        src/unicode/fsm/ \
        "\s+"
}

main() {
    if array_exists "-h" "$@" || array_exists "--help" "$@"; then
        echo "Usage: $(basename "$0") [--list-commands] [<command>] ..." >&2
        exit
    fi

    commands="
        graphemes
        sentences
        words
        regional-indicator
        simple-word
        whitespace
    "
    if array_exists "--list-commands" "$@"; then
        for cmd in $commands; do
            echo "$cmd"
        done
        exit
    fi

    # ucd-generate is used to compile regexes into DFAs.
    requires ucd-generate

    mkdir -p src/unicode/fsm/

    cmds=$*
    if [ $# -eq 0 ] || array_exists "all" "$@"; then
        cmds=$commands
    fi
    for cmd in $cmds; do
        if array_exists "$cmd" $commands; then
            fun="$(echo "$cmd" | sed 's/-/_/g')"
            eval "$fun"
        else
            echo "unrecognized command: $cmd" >&2
        fi
    done
}

main "$@"