Adding upstream version 3.8.1.upstream/3.8.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

8 files changed, 5043 insertions, 0 deletions

diff --git a/contrib/snowball/compiler/analyser.c b/contrib/snowball/compiler/analyser.c
new file mode 100644
index 0000000..dffa555
--- /dev/null
+++ b/contrib/snowball/compiler/analyser.c
@@ -0,0 +1,1380 @@
+
+#include <stdio.h>   /* printf etc */
+#include <stdlib.h>  /* exit */
+#include <string.h>  /* memmove */
+#include "header.h"
+
+typedef enum {
+    e_token_omitted = 0,
+    e_unexpected_token = 1,
+    e_string_omitted = 2,
+    e_unexpected_token_in_among = 3,
+    /* For codes above here, report "after " t->previous_token after the error. */
+    e_unresolved_substring = 14,
+    e_not_allowed_inside_reverse = 15,
+    e_empty_grouping = 16,
+    e_already_backwards = 17,
+    e_empty_among = 18,
+    e_adjacent_bracketed_in_among = 19,
+    e_substring_preceded_by_substring = 20,
+    /* For codes below here, tokeniser->b is printed before the error. */
+    e_redeclared = 30,
+    e_undeclared = 31,
+    e_declared_as_different_mode = 32,
+    e_not_of_type_x = 33,
+    e_not_of_type_string_or_integer = 34,
+    e_misplaced = 35,
+    e_redefined = 36,
+    e_misused = 37
+} error_code;
+
+/* recursive usage: */
+
+static void read_program_(struct analyser * a, int terminator);
+static struct node * read_C(struct analyser * a);
+static struct node * C_style(struct analyser * a, const char * s, int token);
+
+
+static void print_node_(struct node * p, int n, const char * s) {
+
+    int i;
+    for (i = 0; i < n; i++) fputs(i == n - 1 ? s : "  ", stdout);
+    printf("%s ", name_of_token(p->type));
+    if (p->name) report_b(stdout, p->name->b);
+    if (p->literalstring) {
+        printf("'");
+        report_b(stdout, p->literalstring);
+        printf("'");
+    }
+    printf("\n");
+    if (p->AE) print_node_(p->AE, n+1, "# ");
+    if (p->left) print_node_(p->left, n+1, "  ");
+    if (p->aux) print_node_(p->aux, n+1, "@ ");
+    if (p->right) print_node_(p->right, n, "  ");
+}
+
+extern void print_program(struct analyser * a) {
+    print_node_(a->program, 0, "  ");
+}
+
+static struct node * new_node(struct analyser * a, int type) {
+    NEW(node, p);
+    p->next = a->nodes; a->nodes = p;
+    p->left = 0;
+    p->right = 0;
+    p->aux = 0;
+    p->AE = 0;
+    p->name = 0;
+    p->literalstring = 0;
+    p->mode = a->mode;
+    p->line_number = a->tokeniser->line_number;
+    p->type = type;
+    return p;
+}
+
+static const char * name_of_mode(int n) {
+    switch (n) {
+        case m_backward: return "string backward";
+        case m_forward:  return "string forward";
+    /*  case m_integer:  return "integer";  */
+    }
+    fprintf(stderr, "Invalid mode %d in name_of_mode()\n", n);
+    exit(1);
+}
+
+static const char * name_of_type(int n) {
+    switch (n) {
+        case 's': return "string";
+        case 'i': return "integer";
+        case 'r': return "routine";
+        case 'R': return "routine or grouping";
+        case 'g': return "grouping";
+    }
+    fprintf(stderr, "Invalid type %d in name_of_type()\n", n);
+    exit(1);
+}
+
+static const char * name_of_name_type(int code) {
+    switch (code) {
+        case t_string: return "string";
+        case t_boolean: return "boolean";
+        case t_integer: return "integer";
+        case t_routine: return "routine";
+        case t_external: return "external";
+        case t_grouping: return "grouping";
+    }
+    fprintf(stderr, "Invalid type code %d in name_of_name_type()\n", code);
+    exit(1);
+}
+
+static void count_error(struct analyser * a) {
+    struct tokeniser * t = a->tokeniser;
+    if (t->error_count >= 20) { fprintf(stderr, "... etc\n"); exit(1); }
+    t->error_count++;
+}
+
+static void error2(struct analyser * a, error_code n, int x) {
+    struct tokeniser * t = a->tokeniser;
+    count_error(a);
+    fprintf(stderr, "%s:%d: ", t->file, t->line_number);
+    if ((int)n >= (int)e_redeclared) report_b(stderr, t->b);
+    switch (n) {
+        case e_token_omitted:
+            fprintf(stderr, "%s omitted", name_of_token(t->omission)); break;
+        case e_unexpected_token_in_among:
+            fprintf(stderr, "in among(...), ");
+            /* fall through */
+        case e_unexpected_token:
+            fprintf(stderr, "unexpected %s", name_of_token(t->token));
+            if (t->token == c_number) fprintf(stderr, " %d", t->number);
+            if (t->token == c_name) {
+                fprintf(stderr, " ");
+                report_b(stderr, t->b);
+            } break;
+        case e_string_omitted:
+            fprintf(stderr, "string omitted"); break;
+
+        case e_unresolved_substring:
+            fprintf(stderr, "unresolved substring on line %d", x); break;
+        case e_not_allowed_inside_reverse:
+            fprintf(stderr, "%s not allowed inside reverse(...)", name_of_token(t->token)); break;
+        case e_empty_grouping:
+            fprintf(stderr, "empty grouping"); break;
+        case e_already_backwards:
+            fprintf(stderr, "backwards used when already in this mode"); break;
+        case e_empty_among:
+            fprintf(stderr, "empty among(...)"); break;
+        case e_adjacent_bracketed_in_among:
+            fprintf(stderr, "two adjacent bracketed expressions in among(...)"); break;
+        case e_substring_preceded_by_substring:
+            fprintf(stderr, "substring preceded by another substring on line %d", x); break;
+
+        case e_redeclared:
+            fprintf(stderr, " re-declared"); break;
+        case e_undeclared:
+            fprintf(stderr, " undeclared"); break;
+        case e_declared_as_different_mode:
+            fprintf(stderr, " declared as %s mode; used as %s mode",
+                            name_of_mode(a->mode), name_of_mode(x)); break;
+        case e_not_of_type_x:
+            fprintf(stderr, " not of type %s", name_of_type(x)); break;
+        case e_not_of_type_string_or_integer:
+            fprintf(stderr, " not of type string or integer"); break;
+        case e_misplaced:
+            fprintf(stderr, " misplaced"); break;
+        case e_redefined:
+            fprintf(stderr, " redefined"); break;
+        case e_misused:
+            fprintf(stderr, " mis-used as %s mode",
+                            name_of_mode(x)); break;
+    }
+    if ((int)n < (int)e_unresolved_substring && t->previous_token > 0)
+        fprintf(stderr, " after %s", name_of_token(t->previous_token));
+    fprintf(stderr, "\n");
+}
+
+static void error(struct analyser * a, error_code n) { error2(a, n, 0); }
+
+static void error4(struct analyser * a, struct name * q) {
+    count_error(a);
+    fprintf(stderr, "%s:%d: ", a->tokeniser->file, q->used->line_number);
+    report_b(stderr, q->b);
+    fprintf(stderr, " undefined\n");
+}
+
+static void omission_error(struct analyser * a, int n) {
+    a->tokeniser->omission = n;
+    error(a, e_token_omitted);
+}
+
+static int check_token(struct analyser * a, int code) {
+    struct tokeniser * t = a->tokeniser;
+    if (t->token != code) { omission_error(a, code); return false; }
+    return true;
+}
+
+static int get_token(struct analyser * a, int code) {
+    struct tokeniser * t = a->tokeniser;
+    read_token(t);
+    {
+        int x = check_token(a, code);
+        if (!x) t->token_held = true;
+        return x;
+    }
+}
+
+static struct name * look_for_name(struct analyser * a) {
+    symbol * q = a->tokeniser->b;
+    struct name * p;
+    for (p = a->names; p; p = p->next) {
+        symbol * b = p->b;
+        int n = SIZE(b);
+        if (n == SIZE(q) && memcmp(q, b, n * sizeof(symbol)) == 0) {
+            p->referenced = true;
+            return p;
+        }
+    }
+    return 0;
+}
+
+static struct name * find_name(struct analyser * a) {
+    struct name * p = look_for_name(a);
+    if (p == 0) error(a, e_undeclared);
+    return p;
+}
+
+static void check_routine_mode(struct analyser * a, struct name * p, int mode) {
+    if (p->mode < 0) p->mode = mode; else
+    if (p->mode != mode) error2(a, e_misused, mode);
+}
+
+static void check_name_type(struct analyser * a, struct name * p, int type) {
+    switch (type) {
+        case 's':
+            if (p->type == t_string) return;
+            break;
+        case 'i':
+            if (p->type == t_integer) return;
+            break;
+        case 'b':
+            if (p->type == t_boolean) return;
+            break;
+        case 'R':
+            if (p->type == t_grouping) return;
+            /* FALLTHRU */
+        case 'r':
+            if (p->type == t_routine || p->type == t_external) return;
+            break;
+        case 'g':
+            if (p->type == t_grouping) return;
+            break;
+    }
+    error2(a, e_not_of_type_x, type);
+}
+
+static void read_names(struct analyser * a, int type) {
+    struct tokeniser * t = a->tokeniser;
+    if (!get_token(a, c_bra)) return;
+    while (true) {
+        int token = read_token(t);
+        switch (token) {
+            case c_len: {
+                /* Context-sensitive token - once declared as a name, it loses
+                 * its special meaning, for compatibility with older versions
+                 * of snowball.
+                 */
+                static const symbol c_len_lit[] = {
+                    'l', 'e', 'n'
+                };
+                MOVE_TO_B(t->b, c_len_lit);
+                goto handle_as_name;
+            }
+            case c_lenof: {
+                /* Context-sensitive token - once declared as a name, it loses
+                 * its special meaning, for compatibility with older versions
+                 * of snowball.
+                 */
+                static const symbol c_lenof_lit[] = {
+                    'l', 'e', 'n', 'o', 'f'
+                };
+                MOVE_TO_B(t->b, c_lenof_lit);
+                goto handle_as_name;
+            }
+            case c_name:
+handle_as_name:
+                if (look_for_name(a) != 0) error(a, e_redeclared); else {
+                    NEW(name, p);
+                    p->b = copy_b(t->b);
+                    p->type = type;
+                    p->mode = -1; /* routines, externals */
+                    /* We defer assigning counts until after we've eliminated
+                     * variables whose values are never used. */
+                    p->count = -1;
+                    p->referenced = false;
+                    p->used_in_among = false;
+                    p->used = 0;
+                    p->value_used = false;
+                    p->initialised = false;
+                    p->used_in_definition = false;
+                    p->local_to = 0;
+                    p->grouping = 0;
+                    p->definition = 0;
+                    p->declaration_line_number = t->line_number;
+                    p->next = a->names;
+                    a->names = p;
+                    if (token != c_name) {
+                        disable_token(t, token);
+                    }
+                }
+                break;
+            default:
+                if (!check_token(a, c_ket)) t->token_held = true;
+                return;
+        }
+    }
+}
+
+static symbol * new_literalstring(struct analyser * a) {
+    NEW(literalstring, p);
+    p->b = copy_b(a->tokeniser->b);
+    p->next = a->literalstrings;
+    a->literalstrings = p;
+    return p->b;
+}
+
+static int read_AE_test(struct analyser * a) {
+
+    struct tokeniser * t = a->tokeniser;
+    switch (read_token(t)) {
+        case c_assign: return c_mathassign;
+        case c_plusassign:
+        case c_minusassign:
+        case c_multiplyassign:
+        case c_divideassign:
+        case c_eq:
+        case c_ne:
+        case c_gr:
+        case c_ge:
+        case c_ls:
+        case c_le: return t->token;
+        default: error(a, e_unexpected_token); t->token_held = true; return c_eq;
+    }
+}
+
+static int binding(int t) {
+    switch (t) {
+        case c_plus: case c_minus: return 1;
+        case c_multiply: case c_divide: return 2;
+        default: return -2;
+    }
+}
+
+static void mark_used_in(struct analyser * a, struct name * q, struct node * p) {
+    if (!q->used) {
+        q->used = p;
+        q->local_to = a->program_end->name;
+    } else if (q->local_to) {
+        if (q->local_to != a->program_end->name) {
+            /* Used in more than one routine/external. */
+            q->local_to = NULL;
+        }
+    }
+}
+
+static void name_to_node(struct analyser * a, struct node * p, int type) {
+    struct name * q = find_name(a);
+    if (q) {
+        check_name_type(a, q, type);
+        mark_used_in(a, q, p);
+    }
+    p->name = q;
+}
+
+static struct node * read_AE(struct analyser * a, int B) {
+    struct tokeniser * t = a->tokeniser;
+    struct node * p;
+    struct node * q;
+    switch (read_token(t)) {
+        case c_minus: /* monadic */
+            q = read_AE(a, 100);
+            if (q->type == c_neg) {
+                /* Optimise away double negation, which avoids generators
+                 * having to worry about generating "--" (decrement operator
+                 * in many languages).
+                 */
+                p = q->right;
+                /* Don't free q, it's in the linked list a->nodes. */
+                break;
+            }
+            p = new_node(a, c_neg);
+            p->right = q;
+            break;
+        case c_bra:
+            p = read_AE(a, 0);
+            get_token(a, c_ket);
+            break;
+        case c_name:
+            p = new_node(a, c_name);
+            name_to_node(a, p, 'i');
+            if (p->name) p->name->value_used = true;
+            break;
+        case c_maxint:
+        case c_minint:
+            a->int_limits_used = true;
+            /* fall through */
+        case c_cursor:
+        case c_limit:
+        case c_len:
+        case c_size:
+            p = new_node(a, t->token);
+            break;
+        case c_number:
+            p = new_node(a, c_number);
+            p->number = t->number;
+            break;
+        case c_lenof:
+        case c_sizeof:
+            p = C_style(a, "s", t->token);
+            break;
+        default:
+            error(a, e_unexpected_token);
+            t->token_held = true;
+            return 0;
+    }
+    while (true) {
+        int token = read_token(t);
+        int b = binding(token);
+        if (binding(token) <= B) {
+            t->token_held = true;
+            return p;
+        }
+        q = new_node(a, token);
+        q->left = p;
+        q->right = read_AE(a, b);
+        p = q;
+    }
+}
+
+static struct node * read_C_connection(struct analyser * a, struct node * q, int op) {
+    struct tokeniser * t = a->tokeniser;
+    struct node * p = new_node(a, op);
+    struct node * p_end = q;
+    p->left = q;
+    do {
+        q = read_C(a);
+        p_end->right = q; p_end = q;
+    } while (read_token(t) == op);
+    t->token_held = true;
+    return p;
+}
+
+static struct node * read_C_list(struct analyser * a) {
+    struct tokeniser * t = a->tokeniser;
+    struct node * p = new_node(a, c_bra);
+    struct node * p_end = 0;
+    while (true) {
+        int token = read_token(t);
+        if (token == c_ket) return p;
+        if (token < 0) { omission_error(a, c_ket); return p; }
+        t->token_held = true;
+        {
+            struct node * q = read_C(a);
+            while (true) {
+                token = read_token(t);
+                if (token != c_and && token != c_or) {
+                    t->token_held = true;
+                    break;
+                }
+                q = read_C_connection(a, q, token);
+            }
+            if (p_end == 0) p->left = q; else p_end->right = q;
+            p_end = q;
+        }
+    }
+}
+
+static struct node * C_style(struct analyser * a, const char * s, int token) {
+    int i;
+    struct node * p = new_node(a, token);
+    for (i = 0; s[i] != 0; i++) switch (s[i]) {
+        case 'C':
+            p->left = read_C(a); continue;
+        case 'D':
+            p->aux = read_C(a); continue;
+        case 'A':
+            p->AE = read_AE(a, 0); continue;
+        case 'f':
+            get_token(a, c_for); continue;
+        case 'S':
+            {
+                int str_token = read_token(a->tokeniser);
+                if (str_token == c_name) name_to_node(a, p, 's'); else
+                if (str_token == c_literalstring) p->literalstring = new_literalstring(a);
+                else error(a, e_string_omitted);
+            }
+            continue;
+        case 'b':
+        case 's':
+        case 'i':
+            if (get_token(a, c_name)) name_to_node(a, p, s[i]);
+            continue;
+    }
+    return p;
+}
+
+static struct node * read_literalstring(struct analyser * a) {
+    struct node * p = new_node(a, c_literalstring);
+    p->literalstring = new_literalstring(a);
+    return p;
+}
+
+static void reverse_b(symbol * b) {
+    int i = 0; int j = SIZE(b) - 1;
+    while (i < j) {
+        int ch1 = b[i]; int ch2 = b[j];
+        b[i++] = ch2; b[j--] = ch1;
+    }
+}
+
+static int compare_amongvec(const void *pv, const void *qv) {
+    const struct amongvec * p = (const struct amongvec*)pv;
+    const struct amongvec * q = (const struct amongvec*)qv;
+    symbol * b_p = p->b; int p_size = p->size;
+    symbol * b_q = q->b; int q_size = q->size;
+    int smaller_size = p_size < q_size ? p_size : q_size;
+    int i;
+    for (i = 0; i < smaller_size; i++)
+        if (b_p[i] != b_q[i]) return b_p[i] - b_q[i];
+    if (p_size - q_size)
+        return p_size - q_size;
+    return p->line_number - q->line_number;
+}
+
+#define PTR_NULL_CHECK(P, Q) do {\
+        if ((Q) == NULL) {\
+            if ((P) != NULL) return 1;\
+        } else {\
+            if ((P) == NULL) return -1;\
+        }\
+    } while (0)
+
+static int compare_node(const struct node *p, const struct node *q) {
+    PTR_NULL_CHECK(p, q);
+    if (q == NULL) {
+        /* p must be NULL too. */
+        return 0;
+    }
+
+    if (p->type != q->type) return p->type > q->type ? 1 : -1;
+    if (p->mode != q->mode) return p->mode > q->mode ? 1 : -1;
+    if (p->type == c_number) {
+        if (p->number != q->number)
+            return p->number > q->number ? 1 : -1;
+    }
+
+    PTR_NULL_CHECK(p->left, q->left);
+    if (p->left) {
+        int r = compare_node(p->left, q->left);
+        if (r != 0) return r;
+    }
+
+    PTR_NULL_CHECK(p->AE, q->AE);
+    if (p->AE) {
+        int r = compare_node(p->AE, q->AE);
+        if (r != 0) return r;
+    }
+
+    PTR_NULL_CHECK(p->aux, q->aux);
+    if (p->aux) {
+        int r = compare_node(p->aux, q->aux);
+        if (r != 0) return r;
+    }
+
+    PTR_NULL_CHECK(p->name, q->name);
+    if (p->name) {
+        int r;
+        if (SIZE(p->name->b) != SIZE(q->name->b)) {
+            return SIZE(p->name->b) - SIZE(q->name->b);
+        }
+        r = memcmp(p->name->b, q->name->b,
+                   SIZE(p->name->b) * sizeof(symbol));
+        if (r != 0) return r;
+    }
+
+    PTR_NULL_CHECK(p->literalstring, q->literalstring);
+    if (p->literalstring) {
+        int r;
+        if (SIZE(p->literalstring) != SIZE(q->literalstring)) {
+            return SIZE(p->literalstring) - SIZE(q->literalstring);
+        }
+        r = memcmp(p->literalstring, q->literalstring,
+                   SIZE(p->literalstring) * sizeof(symbol));
+        if (r != 0) return r;
+    }
+
+    return compare_node(p->right, q->right);
+}
+
+static void make_among(struct analyser * a, struct node * p, struct node * substring) {
+
+    NEW(among, x);
+    NEWVEC(amongvec, v, p->number);
+    struct node * q = p->left;
+    struct amongvec * w0 = v;
+    struct amongvec * w1 = v;
+    int result = 1;
+
+    int direction = substring != 0 ? substring->mode : p->mode;
+    int backward = direction == m_backward;
+
+    if (a->amongs == 0) a->amongs = x; else a->amongs_end->next = x;
+    a->amongs_end = x;
+    x->next = 0;
+    x->b = v;
+    x->number = a->among_count++;
+    x->function_count = 0;
+    x->starter = 0;
+    x->nocommand_count = 0;
+    x->amongvar_needed = false;
+
+    if (q->type == c_bra) { x->starter = q; q = q->right; }
+
+    while (q) {
+        if (q->type == c_literalstring) {
+            symbol * b = q->literalstring;
+            w1->b = b;           /* pointer to case string */
+            w1->action = NULL;   /* action gets filled in below */
+            w1->line_number = q->line_number;
+            w1->size = SIZE(b);  /* number of characters in string */
+            w1->i = -1;          /* index of longest substring */
+            w1->result = -1;     /* number of corresponding case expression */
+            if (q->left) {
+                struct name * function = q->left->name;
+                w1->function = function;
+                function->used_in_among = true;
+                check_routine_mode(a, function, direction);
+                x->function_count++;
+            } else {
+                w1->function = 0;
+            }
+            w1++;
+        } else if (q->left == 0) {
+            /* empty command: () */
+            w0 = w1;
+        } else {
+            /* Check for previous action which is the same as this one and use
+             * the same action code if we find one.
+             */
+            int among_result = -1;
+            struct amongvec * w;
+            for (w = v; w < w0; ++w) {
+                if (w->action && compare_node(w->action->left, q->left) == 0) {
+                    if (w->result <= 0) {
+                        printf("Among code %d isn't positive\n", w->result);
+                        exit(1);
+                    }
+                    among_result = w->result;
+                    break;
+                }
+            }
+            if (among_result < 0) {
+                among_result = result++;
+            }
+
+            while (w0 != w1) {
+                w0->action = q;
+                w0->result = among_result;
+                w0++;
+            }
+        }
+        q = q->right;
+    }
+    if (w1-v != p->number) { fprintf(stderr, "oh! %d %d\n", (int)(w1-v), p->number); exit(1); }
+    x->command_count = result - 1;
+    {
+        NEWVEC(node*, commands, x->command_count);
+        memset(commands, 0, x->command_count * sizeof(struct node*));
+        for (w0 = v; w0 < w1; w0++) {
+            if (w0->result > 0) {
+                /* result == -1 when there's no command. */
+                if (w0->result > x->command_count) {
+                    fprintf(stderr, "More among codes than expected\n");
+                    exit(1);
+                }
+                if (!commands[w0->result - 1])
+                    commands[w0->result - 1] = w0->action;
+            } else {
+                ++x->nocommand_count;
+            }
+            if (backward) reverse_b(w0->b);
+        }
+        x->commands = commands;
+    }
+    qsort(v, w1 - v, sizeof(struct amongvec), compare_amongvec);
+
+    /* the following loop is O(n squared) */
+    for (w0 = w1 - 1; w0 >= v; w0--) {
+        symbol * b = w0->b;
+        int size = w0->size;
+        struct amongvec * w;
+
+        for (w = w0 - 1; w >= v; w--) {
+            if (w->size < size && memcmp(w->b, b, w->size * sizeof(symbol)) == 0) {
+                w0->i = w - v;  /* fill in index of longest substring */
+                break;
+            }
+        }
+    }
+    if (backward) for (w0 = v; w0 < w1; w0++) reverse_b(w0->b);
+
+    for (w0 = v; w0 < w1 - 1; w0++)
+        if (w0->size == (w0 + 1)->size &&
+            memcmp(w0->b, (w0 + 1)->b, w0->size * sizeof(symbol)) == 0) {
+            count_error(a);
+            fprintf(stderr, "%s:%d: among(...) has repeated string '",
+                    a->tokeniser->file, (w0 + 1)->line_number);
+            report_b(stderr, (w0 + 1)->b);
+            fprintf(stderr, "'\n");
+            count_error(a);
+            fprintf(stderr, "%s:%d: previously seen here\n",
+                    a->tokeniser->file, w0->line_number);
+        }
+
+    x->literalstring_count = p->number;
+    p->among = x;
+
+    x->substring = substring;
+    if (substring != 0) substring->among = x;
+    if (x->command_count > 1 ||
+        (x->command_count == 1 && x->nocommand_count > 0) ||
+        x->starter != 0) {
+        /* We need to set among_var rather than just checking if find_among*()
+         * returns zero or not.
+         */
+        x->amongvar_needed = a->amongvar_needed = true;
+    }
+}
+
+static struct node * read_among(struct analyser * a) {
+    struct tokeniser * t = a->tokeniser;
+    struct node * p = new_node(a, c_among);
+    struct node * p_end = 0;
+    int previous_token = -1;
+    struct node * substring = a->substring;
+
+    a->substring = 0;
+    p->number = 0; /* counts the number of literals */
+    if (!get_token(a, c_bra)) return p;
+    while (true) {
+        struct node * q;
+        int token = read_token(t);
+        switch (token) {
+            case c_literalstring:
+                q = read_literalstring(a);
+                if (read_token(t) == c_name) {
+                    struct node * r = new_node(a, c_name);
+                    name_to_node(a, r, 'r');
+                    q->left = r;
+                }
+                else t->token_held = true;
+                p->number++; break;
+            case c_bra:
+                if (previous_token == c_bra) error(a, e_adjacent_bracketed_in_among);
+                q = read_C_list(a); break;
+            default:
+                error(a, e_unexpected_token_in_among);
+                previous_token = token;
+                continue;
+            case c_ket:
+                if (p->number == 0) error(a, e_empty_among);
+                if (t->error_count == 0) make_among(a, p, substring);
+                return p;
+        }
+        previous_token = token;
+        if (p_end == 0) p->left = q; else p_end->right = q;
+        p_end = q;
+    }
+}
+
+static struct node * read_substring(struct analyser * a) {
+
+    struct node * p = new_node(a, c_substring);
+    if (a->substring != 0) error2(a, e_substring_preceded_by_substring, a->substring->line_number);
+    a->substring = p;
+    return p;
+}
+
+static void check_modifyable(struct analyser * a) {
+    if (!a->modifyable) error(a, e_not_allowed_inside_reverse);
+}
+
+static struct node * read_C(struct analyser * a) {
+    struct tokeniser * t = a->tokeniser;
+    int token = read_token(t);
+    switch (token) {
+        case c_bra:
+            return read_C_list(a);
+        case c_backwards:
+            {
+                int mode = a->mode;
+                if (a->mode == m_backward) error(a, e_already_backwards); else a->mode = m_backward;
+                {   struct node * p = C_style(a, "C", token);
+                    a->mode = mode;
+                    return p;
+                }
+            }
+        case c_reverse:
+            {
+                int mode = a->mode;
+                int modifyable = a->modifyable;
+                a->modifyable = false;
+                a->mode = mode == m_forward ? m_backward : m_forward;
+                {
+                    struct node * p = C_style(a, "C", token);
+                    a->mode = mode;
+                    a->modifyable = modifyable;
+                    return p;
+                }
+            }
+        case c_not:
+        case c_try:
+        case c_fail:
+        case c_test:
+        case c_do:
+        case c_goto:
+        case c_gopast:
+        case c_repeat:
+            return C_style(a, "C", token);
+        case c_loop:
+        case c_atleast:
+            return C_style(a, "AC", token);
+        case c_setmark: {
+            struct node * n = C_style(a, "i", token);
+            if (n->name) n->name->initialised = true;
+            return n;
+        }
+        case c_tomark:
+        case c_atmark:
+        case c_hop:
+            return C_style(a, "A", token);
+        case c_delete:
+            check_modifyable(a);
+            /* fall through */
+        case c_next:
+        case c_tolimit:
+        case c_atlimit:
+        case c_leftslice:
+        case c_rightslice:
+        case c_true:
+        case c_false:
+        case c_debug:
+            return new_node(a, token);
+        case c_assignto:
+        case c_sliceto: {
+            struct node *n;
+            check_modifyable(a);
+            n = C_style(a, "s", token);
+            if (n->name) n->name->initialised = true;
+            return n;
+        }
+        case c_assign:
+        case c_insert:
+        case c_attach:
+        case c_slicefrom: {
+            struct node *n;
+            check_modifyable(a);
+            n = C_style(a, "S", token);
+            if (n->name) n->name->value_used = true;
+            return n;
+        }
+        case c_setlimit:
+            return C_style(a, "CfD", token);
+        case c_set:
+        case c_unset: {
+            struct node * n = C_style(a, "b", token);
+            if (n->name) n->name->initialised = true;
+            return n;
+        }
+        case c_dollar: {
+            struct tokeniser * t = a->tokeniser;
+            read_token(t);
+            if (t->token == c_bra) {
+                /* Handle newer $(AE REL_OP AE) syntax. */
+                struct node * n = read_AE(a, 0);
+                read_token(t);
+                switch (t->token) {
+                    case c_eq:
+                    case c_ne:
+                    case c_gr:
+                    case c_ge:
+                    case c_ls:
+                    case c_le: {
+                        struct node * lhs = n;
+                        n = new_node(a, t->token);
+                        n->left = lhs;
+                        n->AE = read_AE(a, 0);
+                        get_token(a, c_ket);
+                        break;
+                    }
+                    default:
+                        error(a, e_unexpected_token);
+                        t->token_held = true;
+                        break;
+                }
+                return n;
+            }
+
+            if (t->token == c_name) {
+                struct node * p;
+                struct name * q = find_name(a);
+                int mode = a->mode;
+                int modifyable = a->modifyable;
+                if (q && q->type == t_string) {
+                    /* Assume for now that $ on string both initialises and
+                     * uses the string variable.  FIXME: Can we do better?
+                     */
+                    q->initialised = true;
+                    q->value_used = true;
+                    a->mode = m_forward;
+                    a->modifyable = true;
+                    p = new_node(a, c_dollar);
+                    p->left = read_C(a);
+                    p->name = q;
+                } else {
+                    if (q && q->type != t_integer) {
+                        /* If $ is used on an unknown name or a name which
+                         * isn't a string or an integer then we assume the
+                         * unknown name is an integer as $ is used more often
+                         * on integers than strings, so hopefully this it less
+                         * likely to cause an error avalanche.
+                         *
+                         * For an unknown name, we'll already have reported an
+                         * error.
+                         */
+                        error(a, e_not_of_type_string_or_integer);
+                        q = NULL;
+                    }
+                    p = new_node(a, read_AE_test(a));
+                    p->AE = read_AE(a, 0);
+
+                    if (q) {
+                        switch (p->type) {
+                            case c_mathassign:
+                                q->initialised = true;
+                                p->name = q;
+                                break;
+                            default:
+                                /* +=, etc don't "initialise" as they only
+                                 * amend an existing value.  Similarly, they
+                                 * don't count as using the value.
+                                 */
+                                p->name = q;
+                                break;
+                            case c_eq:
+                            case c_ne:
+                            case c_gr:
+                            case c_ge:
+                            case c_ls:
+                            case c_le:
+                                p->left = new_node(a, c_name);
+                                p->left->name = q;
+                                q->value_used = true;
+                                break;
+                        }
+                    }
+                }
+                if (q) mark_used_in(a, q, p);
+                a->mode = mode;
+                a->modifyable = modifyable;
+                return p;
+            }
+
+            error(a, e_unexpected_token);
+            t->token_held = true;
+            return new_node(a, c_dollar);
+        }
+        case c_name:
+            {
+                struct name * q = find_name(a);
+                struct node * p = new_node(a, c_name);
+                if (q) {
+                    mark_used_in(a, q, p);
+                    switch (q->type) {
+                        case t_boolean:
+                            p->type = c_booltest;
+                            q->value_used = true;
+                            break;
+                        case t_integer:
+                            error(a, e_misplaced); /* integer name misplaced */
+                            break;
+                        case t_string:
+                            q->value_used = true;
+                            break;
+                        case t_routine:
+                        case t_external:
+                            p->type = c_call;
+                            check_routine_mode(a, q, a->mode);
+                            break;
+                        case t_grouping:
+                            p->type = c_grouping; break;
+                    }
+                }
+                p->name = q;
+                return p;
+            }
+        case c_non:
+            {
+                struct node * p = new_node(a, token);
+                read_token(t);
+                if (t->token == c_minus) read_token(t);
+                if (!check_token(a, c_name)) { omission_error(a, c_name); return p; }
+                name_to_node(a, p, 'g');
+                return p;
+            }
+        case c_literalstring:
+            return read_literalstring(a);
+        case c_among: return read_among(a);
+        case c_substring: return read_substring(a);
+        default: error(a, e_unexpected_token); return 0;
+    }
+}
+
+static int next_symbol(symbol * p, symbol * W, int utf8) {
+    if (utf8) {
+        int ch;
+        int j = get_utf8(p, & ch);
+        W[0] = ch; return j;
+    } else {
+        W[0] = p[0]; return 1;
+    }
+}
+
+static symbol * alter_grouping(symbol * p, symbol * q, int style, int utf8) {
+    int j = 0;
+    symbol W[1];
+    int width;
+    if (style == c_plus) {
+        while (j < SIZE(q)) {
+            width = next_symbol(q + j, W, utf8);
+            p = add_to_b(p, 1, W);
+            j += width;
+        }
+    } else {
+        while (j < SIZE(q)) {
+            int i;
+            width = next_symbol(q + j, W, utf8);
+            for (i = 0; i < SIZE(p); i++) {
+                if (p[i] == W[0]) {
+                    memmove(p + i, p + i + 1, (SIZE(p) - i - 1) * sizeof(symbol));
+                    SIZE(p)--;
+                }
+            }
+            j += width;
+        }
+    }
+    return p;
+}
+
+static void read_define_grouping(struct analyser * a, struct name * q) {
+    struct tokeniser * t = a->tokeniser;
+    int style = c_plus;
+    {
+        NEW(grouping, p);
+        if (a->groupings == 0) a->groupings = p; else a->groupings_end->next = p;
+        a->groupings_end = p;
+        if (q) q->grouping = p;
+        p->next = 0;
+        p->name = q;
+        p->line_number = a->tokeniser->line_number;
+        p->b = create_b(0);
+        while (true) {
+            switch (read_token(t)) {
+                case c_name:
+                    {
+                        struct name * r = find_name(a);
+                        if (r) {
+                            check_name_type(a, r, 'g');
+                            p->b = alter_grouping(p->b, r->grouping->b, style, false);
+                            r->used_in_definition = true;
+                        }
+                    }
+                    break;
+                case c_literalstring:
+                    p->b = alter_grouping(p->b, t->b, style, (a->encoding == ENC_UTF8));
+                    break;
+                default: error(a, e_unexpected_token); return;
+            }
+            switch (read_token(t)) {
+                case c_plus:
+                case c_minus: style = t->token; break;
+                default: goto label0;
+            }
+        }
+    label0:
+        {
+            int i;
+            int max = 0;
+            int min = 1<<16;
+            for (i = 0; i < SIZE(p->b); i++) {
+                if (p->b[i] > max) max = p->b[i];
+                if (p->b[i] < min) min = p->b[i];
+            }
+            p->largest_ch = max;
+            p->smallest_ch = min;
+            if (min == 1<<16) error(a, e_empty_grouping);
+        }
+        t->token_held = true; return;
+    }
+}
+
+static void read_define_routine(struct analyser * a, struct name * q) {
+    struct node * p = new_node(a, c_define);
+    a->amongvar_needed = false;
+    if (q) {
+        check_name_type(a, q, 'R');
+        if (q->definition != 0) error(a, e_redefined);
+        if (q->mode < 0) q->mode = a->mode; else
+        if (q->mode != a->mode) error2(a, e_declared_as_different_mode, q->mode);
+    }
+    p->name = q;
+    if (a->program == 0) a->program = p; else a->program_end->right = p;
+    a->program_end = p;
+    get_token(a, c_as);
+    p->left = read_C(a);
+    if (q) q->definition = p->left;
+
+    if (a->substring != 0) {
+        error2(a, e_unresolved_substring, a->substring->line_number);
+        a->substring = 0;
+    }
+    p->amongvar_needed = a->amongvar_needed;
+}
+
+static void read_define(struct analyser * a) {
+    if (get_token(a, c_name)) {
+        struct name * q = find_name(a);
+        int type;
+        if (q) {
+            type = q->type;
+        } else {
+            /* No declaration, so sniff next token - if it is 'as' then parse
+             * as a routine, otherwise as a grouping.
+             */
+            if (read_token(a->tokeniser) == c_as) {
+                type = t_routine;
+            } else {
+                type = t_grouping;
+            }
+            a->tokeniser->token_held = true;
+        }
+
+        if (type == t_grouping) {
+            read_define_grouping(a, q);
+        } else {
+            read_define_routine(a, q);
+        }
+    }
+}
+
+static void read_backwardmode(struct analyser * a) {
+    int mode = a->mode;
+    a->mode = m_backward;
+    if (get_token(a, c_bra)) {
+        read_program_(a, c_ket);
+        check_token(a, c_ket);
+    }
+    a->mode = mode;
+}
+
+static void read_program_(struct analyser * a, int terminator) {
+    struct tokeniser * t = a->tokeniser;
+    while (true) {
+        switch (read_token(t)) {
+            case c_strings:     read_names(a, t_string); break;
+            case c_booleans:    read_names(a, t_boolean); break;
+            case c_integers:    read_names(a, t_integer); break;
+            case c_routines:    read_names(a, t_routine); break;
+            case c_externals:   read_names(a, t_external); break;
+            case c_groupings:   read_names(a, t_grouping); break;
+            case c_define:      read_define(a); break;
+            case c_backwardmode:read_backwardmode(a); break;
+            case c_ket:
+                if (terminator == c_ket) return;
+                /* fall through */
+            default:
+                error(a, e_unexpected_token); break;
+            case -1:
+                if (terminator >= 0) omission_error(a, c_ket);
+                return;
+        }
+    }
+}
+
+static void remove_dead_assignments(struct node * p, struct name * q) {
+    if (p->name == q) {
+        switch (p->type) {
+            case c_assignto:
+            case c_sliceto:
+            case c_mathassign:
+            case c_plusassign:
+            case c_minusassign:
+            case c_multiplyassign:
+            case c_divideassign:
+            case c_setmark:
+            case c_set:
+            case c_unset:
+            case c_dollar:
+                /* c_true is a no-op. */
+                p->type = c_true;
+                break;
+            default:
+                /* There are no read accesses to this variable, so any
+                 * references must be assignments.
+                 */
+                fprintf(stderr, "Unhandled type of dead assignment via %s\n",
+                        name_of_token(p->type));
+                exit(1);
+        }
+    }
+    if (p->AE) remove_dead_assignments(p->AE, q);
+    if (p->left) remove_dead_assignments(p->left, q);
+    if (p->aux) remove_dead_assignments(p->aux, q);
+    if (p->right) remove_dead_assignments(p->right, q);
+}
+
+extern void read_program(struct analyser * a) {
+    read_program_(a, -1);
+    {
+        struct name * q = a->names;
+        while (q) {
+            switch (q->type) {
+                case t_external: case t_routine:
+                    if (q->used && q->definition == 0) error4(a, q);
+                    break;
+                case t_grouping:
+                    if (q->used && q->grouping == 0) error4(a, q);
+                    break;
+            }
+            q = q->next;
+        }
+    }
+
+    if (a->tokeniser->error_count == 0) {
+        struct name * q = a->names;
+        struct name ** ptr = &(a->names);
+        while (q) {
+            if (!q->referenced) {
+                fprintf(stderr, "%s:%d: warning: %s '",
+                        a->tokeniser->file,
+                        q->declaration_line_number,
+                        name_of_name_type(q->type));
+                report_b(stderr, q->b);
+                if (q->type == t_routine ||
+                    q->type == t_external ||
+                    q->type == t_grouping) {
+                    fprintf(stderr, "' declared but not defined\n");
+                } else {
+                    fprintf(stderr, "' defined but not used\n");
+                    q = q->next;
+                    *ptr = q;
+                    continue;
+                }
+            } else if (q->type == t_routine || q->type == t_grouping) {
+                /* It's OK to define a grouping but only use it to define other
+                 * groupings.
+                 */
+                if (!q->used && !q->used_in_definition) {
+                    int line_num;
+                    if (q->type == t_routine) {
+                        line_num = q->definition->line_number;
+                    } else {
+                        line_num = q->grouping->line_number;
+                    }
+                    fprintf(stderr, "%s:%d: warning: %s '",
+                            a->tokeniser->file,
+                            line_num,
+                            name_of_name_type(q->type));
+                    report_b(stderr, q->b);
+                    fprintf(stderr, "' defined but not used\n");
+                }
+            } else if (q->type == t_external) {
+                /* Unused is OK. */
+            } else if (!q->initialised) {
+                fprintf(stderr, "%s:%d: warning: %s '",
+                        a->tokeniser->file,
+                        q->declaration_line_number,
+                        name_of_name_type(q->type));
+                report_b(stderr, q->b);
+                fprintf(stderr, "' is never initialised\n");
+            } else if (!q->value_used) {
+                fprintf(stderr, "%s:%d: warning: %s '",
+                        a->tokeniser->file,
+                        q->declaration_line_number,
+                        name_of_name_type(q->type));
+                report_b(stderr, q->b);
+                fprintf(stderr, "' is set but never used\n");
+                remove_dead_assignments(a->program, q);
+                q = q->next;
+                *ptr = q;
+                continue;
+            }
+            ptr = &(q->next);
+            q = q->next;
+        }
+
+        {
+            /* Now we've eliminated variables whose values are never used we
+             * can number the variables, which is used by some generators.
+             */
+            int * name_count = a->name_count;
+            struct name * n;
+            for (n = a->names; n; n = n->next) {
+                n->count = name_count[n->type]++;
+            }
+        }
+    }
+}
+
+extern struct analyser * create_analyser(struct tokeniser * t) {
+    NEW(analyser, a);
+    a->tokeniser = t;
+    a->nodes = 0;
+    a->names = 0;
+    a->literalstrings = 0;
+    a->program = 0;
+    a->amongs = 0;
+    a->among_count = 0;
+    a->groupings = 0;
+    a->mode = m_forward;
+    a->modifyable = true;
+    { int i; for (i = 0; i < t_size; i++) a->name_count[i] = 0; }
+    a->substring = 0;
+    a->int_limits_used = false;
+    return a;
+}
+
+extern void close_analyser(struct analyser * a) {
+    {
+        struct node * q = a->nodes;
+        while (q) {
+            struct node * q_next = q->next;
+            FREE(q);
+            q = q_next;
+        }
+    }
+    {
+        struct name * q = a->names;
+        while (q) {
+            struct name * q_next = q->next;
+            lose_b(q->b); FREE(q);
+            q = q_next;
+        }
+    }
+    {
+        struct literalstring * q = a->literalstrings;
+        while (q) {
+            struct literalstring * q_next = q->next;
+            lose_b(q->b); FREE(q);
+            q = q_next;
+        }
+    }
+    {
+        struct among * q = a->amongs;
+        while (q) {
+            struct among * q_next = q->next;
+            FREE(q->b);
+            FREE(q->commands);
+            FREE(q);
+            q = q_next;
+        }
+    }
+    {
+        struct grouping * q = a->groupings;
+        while (q) {
+            struct grouping * q_next = q->next;
+            lose_b(q->b); FREE(q);
+            q = q_next;
+        }
+    }
+    FREE(a);
+}
diff --git a/contrib/snowball/compiler/driver.c b/contrib/snowball/compiler/driver.c
new file mode 100644
index 0000000..587028f
--- /dev/null
+++ b/contrib/snowball/compiler/driver.c
@@ -0,0 +1,574 @@
+#include <ctype.h>   /* for toupper etc */
+#include <stdio.h>   /* for fprintf etc */
+#include <stdlib.h>  /* for free etc */
+#include <string.h>  /* for strcmp */
+#include "header.h"
+
+#define DEFAULT_JAVA_PACKAGE "org.tartarus.snowball.ext"
+#define DEFAULT_JAVA_BASE_CLASS "org.tartarus.snowball.SnowballProgram"
+#define DEFAULT_JAVA_AMONG_CLASS "org.tartarus.snowball.Among"
+#define DEFAULT_JAVA_STRING_CLASS "java.lang.StringBuilder"
+
+#define DEFAULT_GO_PACKAGE "snowball"
+#define DEFAULT_GO_SNOWBALL_RUNTIME "github.com/snowballstem/snowball/go"
+
+#define DEFAULT_CS_NAMESPACE "Snowball"
+#define DEFAULT_CS_BASE_CLASS "Stemmer"
+#define DEFAULT_CS_AMONG_CLASS "Among"
+#define DEFAULT_CS_STRING_CLASS "StringBuilder"
+
+#define DEFAULT_JS_BASE_CLASS "BaseStemmer"
+
+#define DEFAULT_PYTHON_BASE_CLASS "BaseStemmer"
+
+static int eq(const char * s1, const char * s2) {
+    return strcmp(s1, s2) == 0;
+}
+
+__attribute__((noreturn))
+static void print_arglist(int exit_code) {
+    FILE * f = exit_code ? stderr : stdout;
+    fprintf(f, "Usage: snowball SOURCE_FILE... [OPTIONS]\n\n"
+               "Supported options:\n"
+               "  -o[utput] file\n"
+               "  -s[yntax]\n"
+               "  -comments\n"
+#ifndef DISABLE_JAVA
+               "  -j[ava]\n"
+#endif
+#ifndef DISABLE_CSHARP
+               "  -cs[harp]\n"
+#endif
+               "  -c++\n"
+#ifndef DISABLE_PASCAL
+               "  -pascal\n"
+#endif
+#ifndef DISABLE_PYTHON
+               "  -py[thon]\n"
+#endif
+#ifndef DISABLE_JS
+               "  -js\n"
+#endif
+#ifndef DISABLE_RUST
+               "  -rust\n"
+#endif
+#ifndef DISABLE_GO
+               "  -go\n"
+#endif
+               "  -w[idechars]\n"
+               "  -u[tf8]\n"
+               "  -n[ame] class name\n"
+               "  -ep[refix] string\n"
+               "  -vp[refix] string\n"
+               "  -i[nclude] directory\n"
+               "  -r[untime] path to runtime headers\n"
+               "  -p[arentclassname] fully qualified parent class name\n"
+#if !defined(DISABLE_JAVA) || !defined(DISABLE_CSHARP)
+               "  -P[ackage] package name for stemmers\n"
+               "  -S[tringclass] StringBuffer-compatible class\n"
+               "  -a[mongclass] fully qualified name of the Among class\n"
+#endif
+#ifndef DISABLE_GO
+               "  -gop[ackage] Go package name for stemmers\n"
+               "  -gor[untime] Go snowball runtime package\n"
+#endif
+               "  --help        display this help and exit\n"
+               "  --version     output version information and exit\n"
+           );
+    exit(exit_code);
+}
+
+static void check_lim(int i, int argc) {
+    if (i >= argc) {
+        fprintf(stderr, "argument list is one short\n");
+        print_arglist(1);
+    }
+}
+
+static FILE * get_output(symbol * b) {
+    char * s = b_to_s(b);
+    FILE * output = fopen(s, "w");
+    if (output == 0) {
+        fprintf(stderr, "Can't open output %s\n", s);
+        exit(1);
+    }
+    free(s);
+    return output;
+}
+
+static int read_options(struct options * o, int argc, char * argv[]) {
+    char * s;
+    int i = 1;
+    int new_argc = 1;
+    /* Note down the last option used to specify an explicit encoding so
+     * we can warn we ignored it for languages with a fixed encoding.
+     */
+    const char * encoding_opt = NULL;
+
+    /* set defaults: */
+
+    o->output_file = 0;
+    o->syntax_tree = false;
+    o->comments = false;
+    o->externals_prefix = NULL;
+    o->variables_prefix = 0;
+    o->runtime_path = 0;
+    o->parent_class_name = NULL;
+    o->string_class = NULL;
+    o->among_class = NULL;
+    o->package = NULL;
+    o->go_snowball_runtime = DEFAULT_GO_SNOWBALL_RUNTIME;
+    o->name = NULL;
+    o->make_lang = LANG_C;
+    o->includes = 0;
+    o->includes_end = 0;
+    o->encoding = ENC_SINGLEBYTE;
+
+    /* read options: */
+
+    while (i < argc) {
+        s = argv[i++];
+        if (s[0] != '-') {
+            /* Non-option argument - shuffle down. */
+            argv[new_argc++] = s;
+            continue;
+        }
+
+        {
+            if (eq(s, "-o") || eq(s, "-output")) {
+               check_lim(i, argc);
+                o->output_file = argv[i++];
+                continue;
+            }
+            if (eq(s, "-n") || eq(s, "-name")) {
+                check_lim(i, argc);
+                o->name = argv[i++];
+                continue;
+            }
+#ifndef DISABLE_JS
+            if (eq(s, "-js")) {
+                o->make_lang = LANG_JAVASCRIPT;
+                continue;
+            }
+#endif
+#ifndef DISABLE_RUST
+            if (eq(s, "-rust")) {
+                o->make_lang = LANG_RUST;
+                continue;
+            }
+#endif
+#ifndef DISABLE_GO
+            if (eq(s, "-go")) {
+                o->make_lang = LANG_GO;
+                continue;
+            }
+#endif
+#ifndef DISABLE_JAVA
+            if (eq(s, "-j") || eq(s, "-java")) {
+                o->make_lang = LANG_JAVA;
+                continue;
+            }
+#endif
+#ifndef DISABLE_CSHARP
+            if (eq(s, "-cs") || eq(s, "-csharp")) {
+                o->make_lang = LANG_CSHARP;
+                continue;
+            }
+#endif
+            if (eq(s, "-c++")) {
+                o->make_lang = LANG_CPLUSPLUS;
+                continue;
+            }
+#ifndef DISABLE_PASCAL
+            if (eq(s, "-pascal")) {
+                o->make_lang = LANG_PASCAL;
+                continue;
+            }
+#endif
+#ifndef DISABLE_PYTHON
+            if (eq(s, "-py") || eq(s, "-python")) {
+                o->make_lang = LANG_PYTHON;
+                continue;
+            }
+#endif
+            if (eq(s, "-w") || eq(s, "-widechars")) {
+                encoding_opt = s;
+                o->encoding = ENC_WIDECHARS;
+                continue;
+            }
+            if (eq(s, "-s") || eq(s, "-syntax")) {
+                o->syntax_tree = true;
+                continue;
+            }
+            if (eq(s, "-comments")) {
+                o->comments = true;
+                continue;
+            }
+            if (eq(s, "-ep") || eq(s, "-eprefix")) {
+                check_lim(i, argc);
+                o->externals_prefix = argv[i++];
+                continue;
+            }
+            if (eq(s, "-vp") || eq(s, "-vprefix")) {
+                check_lim(i, argc);
+                o->variables_prefix = argv[i++];
+                continue;
+            }
+            if (eq(s, "-i") || eq(s, "-include")) {
+                check_lim(i, argc);
+
+                {
+                    NEW(include, p);
+                    symbol * b = add_s_to_b(0, argv[i++]);
+                    b = add_s_to_b(b, "/");
+                    p->next = 0; p->b = b;
+
+                    if (o->includes == 0) o->includes = p; else
+                                          o->includes_end->next = p;
+                    o->includes_end = p;
+                }
+                continue;
+            }
+            if (eq(s, "-r") || eq(s, "-runtime")) {
+                check_lim(i, argc);
+                o->runtime_path = argv[i++];
+                continue;
+            }
+            if (eq(s, "-u") || eq(s, "-utf8")) {
+                encoding_opt = s;
+                o->encoding = ENC_UTF8;
+                continue;
+            }
+            if (eq(s, "-p") || eq(s, "-parentclassname")) {
+                check_lim(i, argc);
+                o->parent_class_name = argv[i++];
+                continue;
+            }
+#if !defined(DISABLE_JAVA) || !defined(DISABLE_CSHARP)
+            if (eq(s, "-P") || eq(s, "-Package")) {
+                check_lim(i, argc);
+                o->package = argv[i++];
+                continue;
+            }
+            if (eq(s, "-S") || eq(s, "-stringclass")) {
+                check_lim(i, argc);
+                o->string_class = argv[i++];
+                continue;
+            }
+            if (eq(s, "-a") || eq(s, "-amongclass")) {
+                check_lim(i, argc);
+                o->among_class = argv[i++];
+                continue;
+            }
+#endif
+#ifndef DISABLE_GO
+            if (eq(s, "-gop") || eq(s, "-gopackage")) {
+                check_lim(i, argc);
+                o->package = argv[i++];
+                continue;
+            }
+            if (eq(s, "-gor") || eq(s, "-goruntime")) {
+                check_lim(i, argc);
+                o->go_snowball_runtime = argv[i++];
+                continue;
+            }
+#endif
+            if (eq(s, "--help")) {
+                print_arglist(0);
+            }
+
+            if (eq(s, "--version")) {
+                printf("Snowball compiler version " SNOWBALL_VERSION "\n");
+                exit(0);
+            }
+
+            fprintf(stderr, "'%s' misplaced\n", s);
+            print_arglist(1);
+        }
+    }
+    if (new_argc == 1) {
+        fprintf(stderr, "no source files specified\n");
+        print_arglist(1);
+    }
+    argv[new_argc] = NULL;
+
+    /* Set language-dependent defaults. */
+    switch (o->make_lang) {
+        case LANG_C:
+        case LANG_CPLUSPLUS:
+            encoding_opt = NULL;
+            break;
+        case LANG_CSHARP:
+            o->encoding = ENC_WIDECHARS;
+            if (!o->parent_class_name)
+                o->parent_class_name = DEFAULT_CS_BASE_CLASS;
+            if (!o->string_class)
+                o->string_class = DEFAULT_CS_STRING_CLASS;
+            if (!o->among_class)
+                o->among_class = DEFAULT_CS_AMONG_CLASS;
+            if (!o->package)
+                o->package = DEFAULT_CS_NAMESPACE;
+            break;
+        case LANG_GO:
+            o->encoding = ENC_UTF8;
+            if (!o->package)
+                o->package = DEFAULT_GO_PACKAGE;
+            break;
+        case LANG_JAVA:
+            o->encoding = ENC_WIDECHARS;
+            if (!o->parent_class_name)
+                o->parent_class_name = DEFAULT_JAVA_BASE_CLASS;
+            if (!o->string_class)
+                o->string_class = DEFAULT_JAVA_STRING_CLASS;
+            if (!o->among_class)
+                o->among_class = DEFAULT_JAVA_AMONG_CLASS;
+            if (!o->package)
+                o->package = DEFAULT_JAVA_PACKAGE;
+            break;
+        case LANG_JAVASCRIPT:
+            o->encoding = ENC_WIDECHARS;
+            if (!o->parent_class_name)
+                o->parent_class_name = DEFAULT_JS_BASE_CLASS;
+            break;
+        case LANG_PYTHON:
+            o->encoding = ENC_WIDECHARS;
+            if (!o->parent_class_name)
+                o->parent_class_name = DEFAULT_PYTHON_BASE_CLASS;
+            break;
+        case LANG_RUST:
+            o->encoding = ENC_UTF8;
+            break;
+        default:
+            break;
+    }
+
+    if (encoding_opt) {
+        fprintf(stderr, "warning: %s only meaningful for C and C++\n",
+                encoding_opt);
+    }
+
+    if (o->make_lang != LANG_C && o->make_lang != LANG_CPLUSPLUS) {
+        if (o->runtime_path) {
+            fprintf(stderr, "warning: -r/-runtime only meaningful for C and C++\n");
+        }
+        if (o->externals_prefix) {
+            fprintf(stderr, "warning: -ep/-eprefix only meaningful for C and C++\n");
+        }
+    }
+    if (!o->externals_prefix) o->externals_prefix = "";
+
+    if (!o->name && o->output_file) {
+        /* Default class name to basename of output_file - this is the standard
+         * convention for at least Java and C#.
+         */
+        const char * slash = strrchr(o->output_file, '/');
+        size_t len;
+        const char * leaf = (slash == NULL) ? o->output_file : slash + 1;
+
+        slash = strrchr(leaf, '\\');
+        if (slash != NULL) leaf = slash + 1;
+
+        {
+            const char * dot = strchr(leaf, '.');
+            len = (dot == NULL) ? strlen(leaf) : (size_t)(dot - leaf);
+        }
+
+        {
+            char * new_name = malloc(len + 1);
+            switch (o->make_lang) {
+                case LANG_CSHARP:
+                case LANG_PASCAL:
+                    /* Upper case initial letter. */
+                    memcpy(new_name, leaf, len);
+                    new_name[0] = toupper(new_name[0]);
+                    break;
+                case LANG_JAVASCRIPT:
+                case LANG_PYTHON: {
+                    /* Upper case initial letter and change each
+                     * underscore+letter or hyphen+letter to an upper case
+                     * letter.
+                     */
+                    size_t i, j = 0;
+                    int uc_next = true;
+                    for (i = 0; i != len; ++i) {
+                        unsigned char ch = leaf[i];
+                        if (ch == '_' || ch == '-') {
+                            uc_next = true;
+                        } else {
+                            if (uc_next) {
+                                new_name[j] = toupper(ch);
+                                uc_next = false;
+                            } else {
+                                new_name[j] = ch;
+                            }
+                            ++j;
+                        }
+                    }
+                    len = j;
+                    break;
+                }
+                default:
+                    /* Just copy. */
+                    memcpy(new_name, leaf, len);
+                    break;
+            }
+            new_name[len] = '\0';
+            o->name = new_name;
+        }
+    }
+
+    return new_argc;
+}
+
+extern int main(int argc, char * argv[]) {
+
+    int i;
+    NEW(options, o);
+    argc = read_options(o, argc, argv);
+    {
+        char * file = argv[1];
+        symbol * u = get_input(file);
+        if (u == 0) {
+            fprintf(stderr, "Can't open input %s\n", file);
+            exit(1);
+        }
+        {
+            struct tokeniser * t = create_tokeniser(u, file);
+            struct analyser * a = create_analyser(t);
+            struct input ** next_input_ptr = &(t->next);
+            a->encoding = t->encoding = o->encoding;
+            t->includes = o->includes;
+            /* If multiple source files are specified, set up the others to be
+             * read after the first in order, using the same mechanism as
+             * 'get' uses. */
+            for (i = 2; i != argc; ++i) {
+                NEW(input, q);
+                file = argv[i];
+                u = get_input(file);
+                if (u == 0) {
+                    fprintf(stderr, "Can't open input %s\n", file);
+                    exit(1);
+                }
+                q->p = u;
+                q->c = 0;
+                q->file = file;
+                q->file_needs_freeing = false;
+                q->line_number = 1;
+                *next_input_ptr = q;
+                next_input_ptr = &(q->next);
+            }
+            *next_input_ptr = NULL;
+            read_program(a);
+            if (t->error_count > 0) exit(1);
+            if (o->syntax_tree) print_program(a);
+            close_tokeniser(t);
+            if (!o->syntax_tree) {
+                struct generator * g;
+
+                const char * s = o->output_file;
+                if (!s) {
+                    fprintf(stderr, "Please include the -o option\n");
+                    print_arglist(1);
+                }
+                g = create_generator(a, o);
+                if (o->make_lang == LANG_C || o->make_lang == LANG_CPLUSPLUS) {
+                    symbol * b = add_s_to_b(0, s);
+                    b = add_s_to_b(b, ".h");
+                    o->output_h = get_output(b);
+                    b[SIZE(b) - 1] = 'c';
+                    if (o->make_lang == LANG_CPLUSPLUS) {
+                        b = add_s_to_b(b, "c");
+                    }
+                    o->output_src = get_output(b);
+                    lose_b(b);
+
+                    generate_program_c(g);
+                    fclose(o->output_src);
+                    fclose(o->output_h);
+                }
+#ifndef DISABLE_JAVA
+                if (o->make_lang == LANG_JAVA) {
+                    symbol * b = add_s_to_b(0, s);
+                    b = add_s_to_b(b, ".java");
+                    o->output_src = get_output(b);
+                    lose_b(b);
+                    generate_program_java(g);
+                    fclose(o->output_src);
+                }
+#endif
+#ifndef DISABLE_PASCAL
+                if (o->make_lang == LANG_PASCAL) {
+                    symbol *b = add_s_to_b(0, s);
+                    b = add_s_to_b(b, ".pas");
+                    o->output_src = get_output(b);
+                    lose_b(b);
+                    generate_program_pascal(g);
+                    fclose(o->output_src);
+                }
+#endif
+#ifndef DISABLE_PYTHON
+                if (o->make_lang == LANG_PYTHON) {
+                    symbol * b = add_s_to_b(0, s);
+                    b = add_s_to_b(b, ".py");
+                    o->output_src = get_output(b);
+                    lose_b(b);
+                    generate_program_python(g);
+                    fclose(o->output_src);
+                }
+#endif
+#ifndef DISABLE_JS
+                if (o->make_lang == LANG_JAVASCRIPT) {
+                    symbol * b = add_s_to_b(0, s);
+                    b = add_s_to_b(b, ".js");
+                    o->output_src = get_output(b);
+                    lose_b(b);
+                    generate_program_js(g);
+                    fclose(o->output_src);
+                }
+#endif
+#ifndef DISABLE_CSHARP
+                if (o->make_lang == LANG_CSHARP) {
+                    symbol * b = add_s_to_b(0, s);
+                    b = add_s_to_b(b, ".cs");
+                    o->output_src = get_output(b);
+                    lose_b(b);
+                    generate_program_csharp(g);
+                    fclose(o->output_src);
+                }
+#endif
+#ifndef DISABLE_RUST
+                if (o->make_lang == LANG_RUST) {
+                    symbol * b = add_s_to_b(0, s);
+                    b = add_s_to_b(b, ".rs");
+                    o->output_src = get_output(b);
+                    lose_b(b);
+                    generate_program_rust(g);
+                    fclose(o->output_src);
+                }
+#endif
+#ifndef DISABLE_GO
+                if (o->make_lang == LANG_GO) {
+                    symbol * b = add_s_to_b(0, s);
+                    b = add_s_to_b(b, ".go");
+                    o->output_src = get_output(b);
+                    lose_b(b);
+                    generate_program_go(g);
+                    fclose(o->output_src);
+                }
+#endif
+                close_generator(g);
+            }
+            close_analyser(a);
+        }
+        lose_b(u);
+    }
+    {   struct include * p = o->includes;
+        while (p) {
+            struct include * q = p->next;
+            lose_b(p->b); FREE(p); p = q;
+        }
+    }
+    FREE(o);
+    if (space_count) fprintf(stderr, "%d blocks unfreed\n", space_count);
+    return 0;
+}
diff --git a/contrib/snowball/compiler/generator.c b/contrib/snowball/compiler/generator.c
new file mode 100644
index 0000000..eed86c1
--- /dev/null
+++ b/contrib/snowball/compiler/generator.c
@@ -0,0 +1,1725 @@
+
+#include <limits.h>  /* for INT_MAX */
+#include <stdio.h>   /* for fprintf etc */
+#include <stdlib.h>  /* for free etc */
+#include <string.h>  /* for strlen */
+#include "header.h"
+
+/* Define this to get warning messages when optimisations can't be used. */
+/* #define OPTIMISATION_WARNINGS */
+
+/* recursive use: */
+
+static void generate(struct generator * g, struct node * p);
+
+static int new_label(struct generator * g) {
+    return g->next_label++;
+}
+
+/* Write routines for simple entities */
+
+/* Write a space if the preceding character was not whitespace */
+static void ws_opt_space(struct generator * g, const char * s) {
+    int ch = str_back(g->outbuf);
+    if (ch != ' ' && ch != '\n' && ch != '\t' && ch != -1)
+        write_char(g, ' ');
+    write_string(g, s);
+}
+
+static void wi3(struct generator * g, int i) {
+    if (i < 100) write_char(g, ' ');
+    if (i < 10)  write_char(g, ' ');
+    write_int(g, i); /* integer (width 3) */
+}
+
+
+/* Write routines for items from the syntax tree */
+
+static void write_varname(struct generator * g, struct name * p) {
+
+    int ch = "SIIrxg"[p->type];
+    switch (p->type) {
+        case t_external:
+            write_string(g, g->options->externals_prefix); break;
+        case t_string:
+        case t_boolean:
+        case t_integer: {
+            int count = p->count;
+            if (count < 0) {
+                fprintf(stderr, "Reference to optimised out variable ");
+                report_b(stderr, p->b);
+                fprintf(stderr, " attempted\n");
+                exit(1);
+            }
+            if (p->type == t_boolean) {
+                /* We use a single array for booleans and integers, with the
+                 * integers first.
+                 */
+                count += g->analyser->name_count[t_integer];
+            }
+            write_char(g, ch);
+            write_char(g, '[');
+            write_int(g, count);
+            write_char(g, ']');
+            return;
+        }
+        default:
+            write_char(g, ch); write_char(g, '_');
+    }
+    write_b(g, p->b);
+}
+
+static void write_varref(struct generator * g, struct name * p) {  /* reference to variable */
+    if (p->type < t_routine) write_string(g, "z->");
+    write_varname(g, p);
+}
+
+static void write_hexdigit(struct generator * g, int i) {
+    str_append_ch(g->outbuf, "0123456789ABCDEF"[i & 0xF]); /* hexchar */
+}
+
+static void write_hex(struct generator * g, int i) {
+    if (i >> 4) write_hex(g, i >> 4);
+    write_hexdigit(g, i); /* hex integer */
+}
+
+/* write character literal */
+static void wlitch(struct generator * g, int ch) {
+    if (32 <= ch && ch < 127) {
+        write_char(g, '\'');
+        if (ch == '\'' || ch == '\\') {
+            write_char(g, '\\');
+        }
+        write_char(g, ch);
+        write_char(g, '\'');
+    } else {
+        write_string(g, "0x"); write_hex(g, ch);
+    }
+}
+
+static void wlitarray(struct generator * g, symbol * p) {  /* write literal array */
+
+    write_string(g, "{ ");
+    {
+        int i;
+        for (i = 0; i < SIZE(p); i++) {
+            wlitch(g, p[i]);
+            if (i < SIZE(p) - 1) write_string(g, ", ");
+        }
+    }
+    write_string(g, " }");
+}
+
+static void wlitref(struct generator * g, symbol * p) {  /* write ref to literal array */
+
+    if (SIZE(p) == 0) {
+        write_char(g, '0');
+    } else {
+        struct str * s = g->outbuf;
+        g->outbuf = g->declarations;
+        write_string(g, "static const symbol s_"); write_int(g, g->literalstring_count); write_string(g, "[] = ");
+        wlitarray(g, p);
+        write_string(g, ";\n");
+        g->outbuf = s;
+        write_string(g, "s_"); write_int(g, g->literalstring_count);
+        g->literalstring_count++;
+    }
+}
+
+static void write_margin(struct generator * g) {
+    int i;
+    for (i = 0; i < g->margin; i++) write_string(g, "    ");
+}
+
+void write_comment_content(struct generator * g, struct node * p) {
+    switch (p->type) {
+        case c_mathassign:
+        case c_plusassign:
+        case c_minusassign:
+        case c_multiplyassign:
+        case c_divideassign:
+            if (p->name) {
+                write_char(g, '$');
+                write_b(g, p->name->b);
+                write_char(g, ' ');
+            }
+            write_string(g, name_of_token(p->type));
+            write_string(g, " <integer expression>");
+            break;
+        case c_eq:
+        case c_ne:
+        case c_gr:
+        case c_ge:
+        case c_ls:
+        case c_le:
+            write_string(g, "$(<integer expression> ");
+            write_string(g, name_of_token(p->type));
+            write_string(g, " <integer expression>)");
+            break;
+        default:
+            write_string(g, name_of_token(p->type));
+            if (p->name) {
+                write_char(g, ' ');
+                write_b(g, p->name->b);
+            }
+    }
+    write_string(g, ", line ");
+    write_int(g, p->line_number);
+}
+
+static void write_comment(struct generator * g, struct node * p) {
+    if (g->options->comments) {
+        ws_opt_space(g, "/* ");
+        write_comment_content(g, p);
+        write_string(g, " */");
+    }
+    write_newline(g);
+}
+
+static void wms(struct generator * g, const char * s) {
+    write_margin(g); write_string(g, s);   } /* margin + string */
+
+static void write_block_start(struct generator * g) { /* block start */
+    wms(g, "{   ");
+    g->margin++;
+}
+
+static void write_block_end(struct generator * g) {    /* block end */
+
+    if (g->line_labelled == g->line_count) { wms(g, ";"); write_newline(g); }
+    g->margin--;
+    wms(g, "}"); write_newline(g);
+}
+
+static void w(struct generator * g, const char * s);
+
+/* keep c */
+static void wk(struct generator * g, struct node * p, int keep_limit) {
+    ++g->keep_count;
+    if (p->mode == m_forward) {
+        write_string(g, "int c");
+        write_int(g, g->keep_count);
+        write_string(g, " = z->c");
+        if (keep_limit) {
+            write_string(g, ", mlimit");
+            write_int(g, g->keep_count);
+        }
+        write_char(g, ';');
+    } else {
+        write_string(g, "int m");
+        write_int(g, g->keep_count);
+        write_string(g, " = z->l - z->c");
+        if (keep_limit) {
+            write_string(g, ", mlimit");
+            write_int(g, g->keep_count);
+        }
+        write_string(g, "; (void)m");
+        write_int(g, g->keep_count);
+        write_char(g, ';');
+    }
+}
+
+static void wrestore(struct generator * g, struct node * p, int keep_token) {     /* restore c */
+    if (p->mode == m_forward) {
+        write_string(g, "z->c = c");
+    } else {
+        write_string(g, "z->c = z->l - m");
+    }
+    write_int(g, keep_token); write_char(g, ';');
+}
+
+static void wrestorelimit(struct generator * g, struct node * p, int keep_token) {     /* restore limit */
+    if (p->mode == m_forward) {
+        w(g, "z->l += mlimit");
+    } else {
+        w(g, "z->lb = mlimit");
+    }
+    write_int(g, keep_token); write_string(g, ";");
+}
+
+static void winc(struct generator * g, struct node * p) {     /* increment c */
+    write_string(g, p->mode == m_forward ? "z->c++;" :
+                                 "z->c--;");
+}
+
+static void wsetl(struct generator * g, int n) {
+
+    g->margin--;
+    wms(g, "lab"); write_int(g, n); write_char(g, ':'); write_newline(g);
+    g->line_labelled = g->line_count;
+    g->margin++;
+}
+
+static void wgotol(struct generator * g, int n) {
+    wms(g, "goto lab"); write_int(g, n); write_char(g, ';'); write_newline(g);
+}
+
+static void write_failure(struct generator * g, struct node * p) {          /* fail */
+    if (g->failure_keep_count != 0) {
+        write_string(g, "{ ");
+        if (g->failure_keep_count > 0) {
+            wrestore(g, p, g->failure_keep_count);
+        } else {
+            wrestorelimit(g, p, -g->failure_keep_count);
+        }
+        write_char(g, ' ');
+    }
+    switch (g->failure_label) {
+        case x_return:
+            write_string(g, "return 0;");
+            break;
+        default:
+            write_string(g, "goto lab");
+            write_int(g, g->failure_label);
+            write_char(g, ';');
+            g->label_used = 1;
+    }
+    if (g->failure_keep_count != 0) write_string(g, " }");
+}
+
+
+/* if at limit fail */
+static void write_check_limit(struct generator * g, struct node * p) {
+
+    write_string(g, p->mode == m_forward ? "if (z->c >= z->l) " :
+                                 "if (z->c <= z->lb) ");
+    write_failure(g, p);
+}
+
+static void write_data_address(struct generator * g, struct node * p) {
+    symbol * b = p->literalstring;
+    if (b != 0) {
+        write_int(g, SIZE(b)); w(g, ", ");
+        wlitref(g, b);
+    } else {
+        write_varref(g, p->name);
+    }
+}
+
+/* Formatted write. */
+static void writef(struct generator * g, const char * input, struct node * p) {
+    int i = 0;
+    int l = strlen(input);
+
+    while (i < l) {
+        int ch = input[i++];
+        if (ch != '~') {
+            write_char(g, ch);
+            continue;
+        }
+        switch (input[i++]) {
+            default: write_char(g, input[i - 1]); continue;
+            case 'C': write_comment(g, p); continue;
+            case 'k': wk(g, p, false); continue;
+            case 'K': wk(g, p, true); continue;
+            case 'i': winc(g, p); continue;
+            case 'l': write_check_limit(g, p); continue;
+            case 'f': write_failure(g, p); continue;
+            case 'M': write_margin(g); continue;
+            case 'N': write_newline(g); continue;
+            case '{': write_block_start(g); continue;
+            case '}': write_block_end(g); continue;
+            case 'S': write_string(g, g->S[input[i++] - '0']); continue;
+            case 'I': write_int(g, g->I[input[i++] - '0']); continue;
+            case 'J': wi3(g, g->I[input[i++] - '0']); continue;
+            case 'V': write_varref(g, g->V[input[i++] - '0']); continue;
+            case 'W': write_varname(g, g->V[input[i++] - '0']); continue;
+            case 'L': wlitref(g, g->L[input[i++] - '0']); continue;
+            case 'A': wlitarray(g, g->L[input[i++] - '0']); continue;
+            case 'c': wlitch(g, g->I[input[i++] - '0']); continue;
+            case 'a': write_data_address(g, p); continue;
+            case '+': g->margin++; continue;
+            case '-': g->margin--; continue;
+            case '$': /* insert_s, insert_v etc */
+                write_char(g, p->literalstring == 0 ? 'v' : 's');
+                continue;
+            case 'p': write_string(g, g->options->externals_prefix); continue;
+        }
+    }
+}
+
+static void w(struct generator * g, const char * s) {
+    writef(g, s, 0);
+}
+
+static void generate_AE(struct generator * g, struct node * p) {
+    const char * s;
+    switch (p->type) {
+        case c_name:
+            write_varref(g, p->name); break;
+        case c_number:
+            write_int(g, p->number); break;
+        case c_maxint:
+            write_string(g, "MAXINT"); break;
+        case c_minint:
+            write_string(g, "MININT"); break;
+        case c_neg:
+            write_char(g, '-'); generate_AE(g, p->right); break;
+        case c_multiply:
+            s = " * "; goto label0;
+        case c_plus:
+            s = " + "; goto label0;
+        case c_minus:
+            s = " - "; goto label0;
+        case c_divide:
+            s = " / ";
+        label0:
+            write_char(g, '('); generate_AE(g, p->left);
+            write_string(g, s); generate_AE(g, p->right); write_char(g, ')'); break;
+        case c_cursor:
+            w(g, "z->c"); break;
+        case c_limit:
+            w(g, p->mode == m_forward ? "z->l" : "z->lb"); break;
+        case c_len:
+            if (g->options->encoding == ENC_UTF8) {
+                w(g, "len_utf8(z->p)");
+                break;
+            }
+            /* FALLTHRU */
+        case c_size:
+            w(g, "SIZE(z->p)");
+            break;
+        case c_lenof:
+            if (g->options->encoding == ENC_UTF8) {
+                g->V[0] = p->name;
+                w(g, "len_utf8(~V0)");
+                break;
+            }
+            /* FALLTHRU */
+        case c_sizeof:
+            g->V[0] = p->name;
+            w(g, "SIZE(~V0)");
+            break;
+    }
+}
+
+/* K_needed() tests to see if we really need to keep c. Not true when the
+   command does not touch the cursor. This and repeat_score() could be
+   elaborated almost indefinitely.
+*/
+
+static int K_needed_(struct generator * g, struct node * p, int call_depth) {
+    while (p) {
+        switch (p->type) {
+            case c_atlimit:
+            case c_do:
+            case c_dollar:
+            case c_leftslice:
+            case c_rightslice:
+            case c_mathassign:
+            case c_plusassign:
+            case c_minusassign:
+            case c_multiplyassign:
+            case c_divideassign:
+            case c_eq:
+            case c_ne:
+            case c_gr:
+            case c_ge:
+            case c_ls:
+            case c_le:
+            case c_sliceto:
+            case c_booltest:
+            case c_set:
+            case c_unset:
+            case c_true:
+            case c_false:
+            case c_debug:
+                break;
+
+            case c_call:
+                /* Recursive functions aren't typical in snowball programs, so
+                 * make the pessimistic assumption that keep is needed if we
+                 * hit a generous limit on recursion.  It's not likely to make
+                 * a difference to any real world program, but means we won't
+                 * recurse until we run out of stack for pathological cases.
+                 */
+                if (call_depth >= 100) return true;
+                if (K_needed_(g, p->name->definition, call_depth + 1))
+                    return true;
+                break;
+
+            case c_bra:
+                if (K_needed_(g, p->left, call_depth)) return true;
+                break;
+
+            default: return true;
+        }
+        p = p->right;
+    }
+    return false;
+}
+
+extern int K_needed(struct generator * g, struct node * p) {
+    return K_needed_(g, p, 0);
+}
+
+static int repeat_score(struct generator * g, struct node * p, int call_depth) {
+    int score = 0;
+    while (p) {
+        switch (p->type) {
+            case c_dollar:
+            case c_leftslice:
+            case c_rightslice:
+            case c_mathassign:
+            case c_plusassign:
+            case c_minusassign:
+            case c_multiplyassign:
+            case c_divideassign:
+            case c_eq:
+            case c_ne:
+            case c_gr:
+            case c_ge:
+            case c_ls:
+            case c_le:
+            case c_sliceto:   /* case c_not: must not be included here! */
+            case c_debug:
+                break;
+
+            case c_call:
+                /* Recursive functions aren't typical in snowball programs, so
+                 * make the pessimistic assumption that repeat requires cursor
+                 * reinstatement if we hit a generous limit on recursion.  It's
+                 * not likely to make a difference to any real world program,
+                 * but means we won't recurse until we run out of stack for
+                 * pathological cases.
+                 */
+                if (call_depth >= 100) {
+                    return 2;
+                }
+                score += repeat_score(g, p->name->definition, call_depth + 1);
+                if (score >= 2)
+                    return score;
+                break;
+
+            case c_bra:
+                score += repeat_score(g, p->left, call_depth);
+                if (score >= 2)
+                    return score;
+                break;
+
+            case c_name:
+            case c_literalstring:
+            case c_next:
+            case c_grouping:
+            case c_non:
+            case c_hop:
+                if (++score >= 2)
+                    return score;
+                break;
+
+            default:
+                return 2;
+        }
+        p = p->right;
+    }
+    return score;
+}
+
+/* tests if an expression requires cursor reinstatement in a repeat */
+
+extern int repeat_restore(struct generator * g, struct node * p) {
+    return repeat_score(g, p, 0) >= 2;
+}
+
+static void generate_bra(struct generator * g, struct node * p) {
+    p = p->left;
+    while (p) {
+        generate(g, p);
+        p = p->right;
+    }
+}
+
+static void generate_and(struct generator * g, struct node * p) {
+    int keep_c = 0;
+    if (K_needed(g, p->left)) {
+        writef(g, "~{~k~C", p);
+        keep_c = g->keep_count;
+    } else {
+        writef(g, "~M~C", p);
+    }
+    p = p->left;
+    while (p) {
+        generate(g, p);
+        if (keep_c && p->right != 0) {
+            w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+        }
+        p = p->right;
+    }
+    if (keep_c) w(g, "~}");
+}
+
+static void generate_or(struct generator * g, struct node * p) {
+    int keep_c = 0;
+
+    int used = g->label_used;
+    int a0 = g->failure_label;
+    int a1 = g->failure_keep_count;
+
+    int out_lab = new_label(g);
+
+    if (K_needed(g, p->left)) {
+        writef(g, "~{~k~C", p);
+        keep_c = g->keep_count;
+    } else {
+        writef(g, "~M~C", p);
+    }
+    p = p->left;
+    g->failure_keep_count = 0;
+    while (p->right) {
+        g->failure_label = new_label(g);
+        g->label_used = 0;
+        generate(g, p);
+        wgotol(g, out_lab);
+        if (g->label_used)
+            wsetl(g, g->failure_label);
+        if (keep_c) {
+            w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+        }
+        p = p->right;
+    }
+    g->label_used = used;
+    g->failure_label = a0;
+    g->failure_keep_count = a1;
+
+    generate(g, p);
+    if (keep_c) w(g, "~}");
+    wsetl(g, out_lab);
+}
+
+static void generate_backwards(struct generator * g, struct node * p) {
+
+    writef(g, "~Mz->lb = z->c; z->c = z->l;~C~N", p);
+    generate(g, p->left);
+    w(g, "~Mz->c = z->lb;~N");
+}
+
+
+static void generate_not(struct generator * g, struct node * p) {
+    int keep_c = 0;
+
+    int used = g->label_used;
+    int a0 = g->failure_label;
+    int a1 = g->failure_keep_count;
+
+    if (K_needed(g, p->left)) {
+        writef(g, "~{~k~C", p);
+        keep_c = g->keep_count;
+    } else {
+        writef(g, "~M~C", p);
+    }
+
+    g->failure_label = new_label(g);
+    g->label_used = 0;
+    g->failure_keep_count = 0;
+    generate(g, p->left);
+
+    {
+        int l = g->failure_label;
+        int u = g->label_used;
+
+        g->label_used = used;
+        g->failure_label = a0;
+        g->failure_keep_count = a1;
+
+        writef(g, "~M~f~N", p);
+        if (u)
+            wsetl(g, l);
+    }
+    if (keep_c) {
+        w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N~}");
+    }
+}
+
+
+static void generate_try(struct generator * g, struct node * p) {
+    int keep_c = 0;
+    if (K_needed(g, p->left)) {
+        writef(g, "~{~k~C", p);
+        keep_c = g->keep_count;
+    } else {
+        writef(g, "~M~C", p);
+    }
+    g->failure_keep_count = keep_c;
+
+    g->failure_label = new_label(g);
+    g->label_used = 0;
+    generate(g, p->left);
+
+    if (g->label_used)
+        wsetl(g, g->failure_label);
+
+    if (keep_c) w(g, "~}");
+}
+
+static void generate_set(struct generator * g, struct node * p) {
+    g->V[0] = p->name; writef(g, "~M~V0 = 1;~C", p);
+}
+
+static void generate_unset(struct generator * g, struct node * p) {
+    g->V[0] = p->name; writef(g, "~M~V0 = 0;~C", p);
+}
+
+static void generate_fail(struct generator * g, struct node * p) {
+    generate(g, p->left);
+    writef(g, "~M~f~C", p);
+}
+
+/* generate_test() also implements 'reverse' */
+
+static void generate_test(struct generator * g, struct node * p) {
+    int keep_c = 0;
+    if (K_needed(g, p->left)) {
+        keep_c = ++g->keep_count;
+        w(g, p->mode == m_forward ? "~{int c_test" :
+                                    "~{int m_test");
+        write_int(g, keep_c);
+        w(g, p->mode == m_forward ? " = z->c;" :
+                                    " = z->l - z->c;");
+        writef(g, "~C", p);
+    } else writef(g, "~M~C", p);
+
+    generate(g, p->left);
+
+    if (keep_c) {
+        w(g, p->mode == m_forward ? "~Mz->c = c_test" :
+                                    "~Mz->c = z->l - m_test");
+        write_int(g, keep_c);
+        writef(g, ";~N~}", p);
+    }
+}
+
+static void generate_do(struct generator * g, struct node * p) {
+    int keep_c = 0;
+    if (K_needed(g, p->left)) {
+        writef(g, "~{~k~C", p);
+        keep_c = g->keep_count;
+    } else {
+        writef(g, "~M~C", p);
+    }
+
+    if (p->left->type == c_call) {
+        /* Optimise do <call> */
+        g->V[0] = p->left->name;
+        writef(g, "~{int ret = ~V0(z);~C", p->left);
+        w(g, "~Mif (ret < 0) return ret;~N~}");
+    } else {
+        g->failure_label = new_label(g);
+        g->label_used = 0;
+        g->failure_keep_count = 0;
+        generate(g, p->left);
+
+        if (g->label_used)
+            wsetl(g, g->failure_label);
+    }
+    if (keep_c) {
+        w(g, "~M"); wrestore(g, p, keep_c);
+        w(g, "~N~}");
+    }
+}
+
+static void generate_next(struct generator * g, struct node * p) {
+    if (g->options->encoding == ENC_UTF8) {
+        if (p->mode == m_forward)
+            w(g, "~{int ret = skip_utf8(z->p, z->c, 0, z->l, 1");
+        else
+            w(g, "~{int ret = skip_utf8(z->p, z->c, z->lb, 0, -1");
+        writef(g, ");~N"
+              "~Mif (ret < 0) ~f~N"
+              "~Mz->c = ret;~C"
+              "~}", p);
+    } else
+        writef(g, "~M~l~N"
+              "~M~i~C", p);
+}
+
+static void generate_GO_grouping(struct generator * g, struct node * p, int is_goto, int complement) {
+
+    struct grouping * q = p->name->grouping;
+    g->S[0] = p->mode == m_forward ? "" : "_b";
+    g->S[1] = complement ? "in" : "out";
+    g->S[2] = g->options->encoding == ENC_UTF8 ? "_U" : "";
+    g->V[0] = p->name;
+    g->I[0] = q->smallest_ch;
+    g->I[1] = q->largest_ch;
+    if (is_goto) {
+        writef(g, "~Mif (~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 1) < 0) ~f~C", p);
+    } else {
+        writef(g, "~{~C"
+              "~Mint ret = ~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 1);~N"
+              "~Mif (ret < 0) ~f~N", p);
+        if (p->mode == m_forward)
+            w(g, "~Mz->c += ret;~N");
+        else
+            w(g, "~Mz->c -= ret;~N");
+        w(g, "~}");
+    }
+}
+
+static void generate_GO(struct generator * g, struct node * p, int style) {
+    int keep_c = 0;
+
+    int used = g->label_used;
+    int a0 = g->failure_label;
+    int a1 = g->failure_keep_count;
+
+    if (p->left->type == c_grouping || p->left->type == c_non) {
+        /* Special case for "goto" or "gopast" when used on a grouping or an
+         * inverted grouping - the movement of c by the matching action is
+         * exactly what we want! */
+#ifdef OPTIMISATION_WARNINGS
+        printf("Optimising %s %s\n", style ? "goto" : "gopast", p->left->type == c_non ? "non" : "grouping");
+#endif
+        if (g->options->comments) {
+            writef(g, "~M~C", p);
+        }
+        generate_GO_grouping(g, p->left, style, p->left->type == c_non);
+        return;
+    }
+
+    w(g, "~Mwhile(1) {"); writef(g, "~C~+", p);
+
+    if (style == 1 || repeat_restore(g, p->left)) {
+        writef(g, "~M~k~N", p);
+        keep_c = g->keep_count;
+    }
+
+    g->failure_label = new_label(g);
+    g->label_used = 0;
+    generate(g, p->left);
+
+    if (style == 1) {
+        /* include for goto; omit for gopast */
+        w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+    }
+    w(g, "~Mbreak;~N");
+    if (g->label_used)
+        wsetl(g, g->failure_label);
+    if (keep_c) {
+        w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+    }
+
+    g->label_used = used;
+    g->failure_label = a0;
+    g->failure_keep_count = a1;
+
+/*  writef(g, "~M~l~N"
+          "~M~i~N", p);  */
+    generate_next(g, p);
+    w(g, "~}");
+}
+
+static void generate_loop(struct generator * g, struct node * p) {
+    w(g, "~{int i; for (i = "); generate_AE(g, p->AE); writef(g, "; i > 0; i--)~C"
+            "~{", p);
+
+    generate(g, p->left);
+
+    w(g,    "~}"
+         "~}");
+}
+
+static void generate_repeat_or_atleast(struct generator * g, struct node * p, int atleast_case) {
+    int keep_c = 0;
+    if (atleast_case) {
+        writef(g, "~Mwhile(1) {~+~N", p);
+    } else {
+        writef(g, "~Mwhile(1) {~+~C", p);
+    }
+
+    if (repeat_restore(g, p->left)) {
+        writef(g, "~M~k~N", p);
+        keep_c = g->keep_count;
+    }
+
+    g->failure_label = new_label(g);
+    g->label_used = 0;
+    g->failure_keep_count = 0;
+    generate(g, p->left);
+
+    if (atleast_case) w(g, "~Mi--;~N");
+
+    w(g, "~Mcontinue;~N");
+    if (g->label_used)
+        wsetl(g, g->failure_label);
+
+    if (keep_c) {
+        w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+    }
+
+    w(g, "~Mbreak;~N"
+      "~}");
+}
+
+static void generate_repeat(struct generator * g, struct node * p) {
+    generate_repeat_or_atleast(g, p, false);
+}
+
+static void generate_atleast(struct generator * g, struct node * p) {
+    w(g, "~{int i = "); generate_AE(g, p->AE); w(g, ";~C");
+    {
+        int used = g->label_used;
+        int a0 = g->failure_label;
+        int a1 = g->failure_keep_count;
+
+        generate_repeat_or_atleast(g, p, true);
+
+        g->label_used = used;
+        g->failure_label = a0;
+        g->failure_keep_count = a1;
+    }
+    writef(g, "~Mif (i > 0) ~f~N"
+       "~}", p);
+}
+
+static void generate_setmark(struct generator * g, struct node * p) {
+    g->V[0] = p->name;
+    writef(g, "~M~V0 = z->c;~C", p);
+}
+
+static void generate_tomark(struct generator * g, struct node * p) {
+    g->S[0] = p->mode == m_forward ? ">" : "<";
+
+    w(g, "~Mif (z->c ~S0 "); generate_AE(g, p->AE); writef(g, ") ~f~N", p);
+    w(g, "~Mz->c = "); generate_AE(g, p->AE); writef(g, ";~C", p);
+}
+
+static void generate_atmark(struct generator * g, struct node * p) {
+
+    w(g, "~Mif (z->c != "); generate_AE(g, p->AE); writef(g, ") ~f~C", p);
+}
+
+static void generate_hop(struct generator * g, struct node * p) {
+    g->S[0] = p->mode == m_forward ? "+" : "-";
+    g->S[1] = p->mode == m_forward ? "0" : "z->lb";
+    if (g->options->encoding == ENC_UTF8) {
+        w(g, "~{int ret = skip_utf8(z->p, z->c, ~S1, z->l, ~S0 ");
+        generate_AE(g, p->AE); writef(g, ");~C", p);
+        writef(g, "~Mif (ret < 0) ~f~N", p);
+    } else {
+        w(g, "~{int ret = z->c ~S0 ");
+        generate_AE(g, p->AE); writef(g, ";~C", p);
+        writef(g, "~Mif (~S1 > ret || ret > z->l) ~f~N", p);
+    }
+    writef(g, "~Mz->c = ret;~N"
+          "~}", p);
+}
+
+static void generate_delete(struct generator * g, struct node * p) {
+    writef(g, "~{int ret = slice_del(z);~C", p);
+    writef(g, "~Mif (ret < 0) return ret;~N"
+          "~}", p);
+}
+
+static void generate_tolimit(struct generator * g, struct node * p) {
+    g->S[0] = p->mode == m_forward ? "" : "b";
+    writef(g, "~Mz->c = z->l~S0;~C", p);
+}
+
+static void generate_atlimit(struct generator * g, struct node * p) {
+    g->S[0] = p->mode == m_forward ? "" : "b";
+    g->S[1] = p->mode == m_forward ? "<" : ">";
+    writef(g, "~Mif (z->c ~S1 z->l~S0) ~f~C", p);
+}
+
+static void generate_leftslice(struct generator * g, struct node * p) {
+    g->S[0] = p->mode == m_forward ? "bra" : "ket";
+    writef(g, "~Mz->~S0 = z->c;~C", p);
+}
+
+static void generate_rightslice(struct generator * g, struct node * p) {
+    g->S[0] = p->mode == m_forward ? "ket" : "bra";
+    writef(g, "~Mz->~S0 = z->c;~C", p);
+}
+
+static void generate_assignto(struct generator * g, struct node * p) {
+    g->V[0] = p->name;
+    writef(g, "~M~V0 = assign_to(z, ~V0);~C"
+          "~Mif (~V0 == 0) return -1;~C", p);
+}
+
+static void generate_sliceto(struct generator * g, struct node * p) {
+    g->V[0] = p->name;
+    writef(g, "~M~V0 = slice_to(z, ~V0);~C"
+          "~Mif (~V0 == 0) return -1;~N", p);
+}
+
+static void generate_insert(struct generator * g, struct node * p, int style) {
+
+    int keep_c = style == c_attach;
+    if (p->mode == m_backward) keep_c = !keep_c;
+    writef(g, "~{int ret;~N", p);
+    if (keep_c) w(g, "~{int saved_c = z->c;~N");
+    writef(g, "~Mret = insert_~$(z, z->c, z->c, ~a);~C", p);
+    if (keep_c) w(g, "~Mz->c = saved_c;~N~}");
+    writef(g, "~Mif (ret < 0) return ret;~N"
+          "~}", p);
+}
+
+static void generate_assignfrom(struct generator * g, struct node * p) {
+
+    int keep_c = p->mode == m_forward; /* like 'attach' */
+    writef(g, "~{int ret;~N", p);
+    if (keep_c) writef(g, "~{int saved_c = z->c;~N", p);
+    w(g, "~Mret = ");
+    writef(g, keep_c ? "insert_~$(z, z->c, z->l, ~a);~C" : "insert_~$(z, z->lb, z->c, ~a);~C", p);
+    if (keep_c) w(g, "~Mz->c = saved_c;~N~}");
+    writef(g, "~Mif (ret < 0) return ret;~N"
+          "~}", p);
+}
+
+/* bugs marked <======= fixed 22/7/02. Similar fixes required for Java */
+
+static void generate_slicefrom(struct generator * g, struct node * p) {
+
+/*  w(g, "~Mslice_from_s(z, ");   <============= bug! should be: */
+    writef(g, "~{int ret = slice_from_~$(z, ~a);~C", p);
+    writef(g, "~Mif (ret < 0) return ret;~N"
+          "~}", p);
+}
+
+static void generate_setlimit(struct generator * g, struct node * p) {
+    int keep_c;
+    if (p->left && p->left->type == c_tomark) {
+        /* Special case for:
+         *
+         *   setlimit tomark AE for C
+         *
+         * All uses of setlimit in the current stemmers we ship follow this
+         * pattern, and by special-casing we can avoid having to save and
+         * restore c.
+         */
+        struct node * q = p->left;
+
+        ++g->keep_count;
+        writef(g, "~N~{int mlimit", p);
+        write_int(g, g->keep_count);
+        writef(g, ";~C", p);
+        keep_c = g->keep_count;
+
+        g->S[0] = q->mode == m_forward ? ">" : "<";
+
+        w(g, "~Mif (z->c ~S0 "); generate_AE(g, q->AE); writef(g, ") ~f~N", q);
+        w(g, "~Mmlimit");
+        write_int(g, keep_c);
+        if (p->mode == m_forward) {
+            w(g, " = z->l - z->c; z->l = ");
+        } else {
+            w(g, " = z->lb; z->lb = ");
+        }
+        generate_AE(g, q->AE);
+        w(g, ";~N");
+    } else {
+        writef(g, "~{~K~C", p);
+        keep_c = g->keep_count;
+        generate(g, p->left);
+
+        w(g, "~Mmlimit");
+        write_int(g, keep_c);
+        if (p->mode == m_forward)
+            w(g, " = z->l - z->c; z->l = z->c;~N");
+        else
+            w(g, " = z->lb; z->lb = z->c;~N");
+        w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+    }
+
+    g->failure_keep_count = -keep_c;
+    generate(g, p->aux);
+    w(g, "~M");
+    wrestorelimit(g, p, -g->failure_keep_count);
+    w(g, "~N"
+      "~}");
+}
+
+/* dollar sets snowball up to operate on a string variable as if it were the
+ * current string */
+static void generate_dollar(struct generator * g, struct node * p) {
+
+    int used = g->label_used;
+    int a0 = g->failure_label;
+    int a1 = g->failure_keep_count;
+    int keep_token;
+    g->failure_label = new_label(g);
+    g->label_used = 0;
+    g->failure_keep_count = 0;
+
+    keep_token = ++g->keep_count;
+    g->I[0] = keep_token;
+    writef(g, "~{struct SN_env env~I0 = * z;~C", p);
+    g->V[0] = p->name;
+    /* Assume failure. */
+    writef(g, "~Mint failure = 1;~N"
+          "~Mz->p = ~V0;~N"
+          "~Mz->lb = z->c = 0;~N"
+          "~Mz->l = SIZE(z->p);~N", p);
+    generate(g, p->left);
+    /* Mark success. */
+    w(g, "~Mfailure = 0;~N");
+    if (g->label_used)
+        wsetl(g, g->failure_label);
+    g->V[0] = p->name; /* necessary */
+
+    g->label_used = used;
+    g->failure_label = a0;
+    g->failure_keep_count = a1;
+
+    g->I[0] = keep_token;
+    writef(g, "~M~V0 = z->p;~N"
+          "~M* z = env~I0;~N"
+          "~Mif (failure) ~f~N~}", p);
+}
+
+static void generate_integer_assign(struct generator * g, struct node * p, char * s) {
+
+    g->V[0] = p->name;
+    g->S[0] = s;
+    w(g, "~M~V0 ~S0 "); generate_AE(g, p->AE); writef(g, ";~C", p);
+}
+
+static void generate_integer_test(struct generator * g, struct node * p, char * s) {
+
+    w(g, "~Mif (!(");
+    generate_AE(g, p->left);
+    write_char(g, ' ');
+    write_string(g, s);
+    write_char(g, ' ');
+    generate_AE(g, p->AE);
+    writef(g, ")) ~f~C", p);
+}
+
+static void generate_call(struct generator * g, struct node * p) {
+
+    g->V[0] = p->name;
+    writef(g, "~{int ret = ~V0(z);~C", p);
+    if (g->failure_keep_count == 0 && g->failure_label == x_return) {
+        /* Combine the two tests in this special case for better optimisation
+         * and clearer generated code. */
+        writef(g, "~Mif (ret <= 0) return ret;~N~}", p);
+    } else {
+        writef(g, "~Mif (ret == 0) ~f~N"
+              "~Mif (ret < 0) return ret;~N~}", p);
+    }
+}
+
+static void generate_grouping(struct generator * g, struct node * p, int complement) {
+
+    struct grouping * q = p->name->grouping;
+    g->S[0] = p->mode == m_forward ? "" : "_b";
+    g->S[1] = complement ? "out" : "in";
+    g->S[2] = g->options->encoding == ENC_UTF8 ? "_U" : "";
+    g->V[0] = p->name;
+    g->I[0] = q->smallest_ch;
+    g->I[1] = q->largest_ch;
+    writef(g, "~Mif (~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 0)) ~f~C", p);
+}
+
+static void generate_namedstring(struct generator * g, struct node * p) {
+
+    g->S[0] = p->mode == m_forward ? "" : "_b";
+    g->V[0] = p->name;
+    writef(g, "~Mif (!(eq_v~S0(z, ~V0))) ~f~C", p);
+}
+
+static void generate_literalstring(struct generator * g, struct node * p) {
+    symbol * b = p->literalstring;
+    if (SIZE(b) == 1) {
+        /* It's quite common to compare with a single character literal string,
+         * so just inline the simpler code for this case rather than making a
+         * function call.  In UTF-8 mode, only do this for the ASCII subset,
+         * since multi-byte characters are more complex to test against.
+         */
+        if (g->options->encoding == ENC_UTF8 && *b >= 128) {
+            printf("single byte %d\n", *b);
+            exit(1);
+        }
+        g->I[0] = *b;
+        if (p->mode == m_forward) {
+            writef(g, "~Mif (z->c == z->l || z->p[z->c] != ~c0) ~f~C"
+                  "~Mz->c++;~N", p);
+        } else {
+            writef(g, "~Mif (z->c <= z->lb || z->p[z->c - 1] != ~c0) ~f~C"
+                  "~Mz->c--;~N", p);
+        }
+    } else {
+        g->S[0] = p->mode == m_forward ? "" : "_b";
+        g->I[0] = SIZE(b);
+        g->L[0] = b;
+
+        writef(g, "~Mif (!(eq_s~S0(z, ~I0, ~L0))) ~f~C", p);
+    }
+}
+
+static void generate_define(struct generator * g, struct node * p) {
+    struct name * q = p->name;
+    g->next_label = 0;
+
+    g->S[0] = q->type == t_routine ? "static" : "extern";
+    g->V[0] = q;
+
+    w(g, "~N~S0 int ~V0(struct SN_env * z) {");
+    if (g->options->comments) {
+        write_string(g, p->mode == m_forward ? " /* forwardmode */" : " /* backwardmode */");
+    }
+    w(g, "~N~+");
+    if (p->amongvar_needed) w(g, "~Mint among_var;~N");
+    g->failure_keep_count = 0;
+    g->failure_label = x_return;
+    g->label_used = 0;
+    g->keep_count = 0;
+    generate(g, p->left);
+    w(g, "~Mreturn 1;~N~}");
+}
+
+static void generate_substring(struct generator * g, struct node * p) {
+
+    struct among * x = p->among;
+    int block = -1;
+    unsigned int bitmap = 0;
+    struct amongvec * among_cases = x->b;
+    int c;
+    int empty_case = -1;
+    int n_cases = 0;
+    symbol cases[2];
+    int shortest_size = INT_MAX;
+    int shown_comment = 0;
+
+    g->S[0] = p->mode == m_forward ? "" : "_b";
+    g->I[0] = x->number;
+    g->I[1] = x->literalstring_count;
+
+    /* In forward mode with non-ASCII UTF-8 characters, the first character
+     * of the string will often be the same, so instead look at the last
+     * common character position.
+     *
+     * In backward mode, we can't match if there are fewer characters before
+     * the current position than the minimum length.
+     */
+    for (c = 0; c < x->literalstring_count; ++c) {
+        int size = among_cases[c].size;
+        if (size != 0 && size < shortest_size) {
+            shortest_size = size;
+        }
+    }
+
+    for (c = 0; c < x->literalstring_count; ++c) {
+        symbol ch;
+        if (among_cases[c].size == 0) {
+            empty_case = c;
+            continue;
+        }
+        if (p->mode == m_forward) {
+            ch = among_cases[c].b[shortest_size - 1];
+        } else {
+            ch = among_cases[c].b[among_cases[c].size - 1];
+        }
+        if (n_cases == 0) {
+            block = ch >> 5;
+        } else if (ch >> 5 != block) {
+            block = -1;
+            if (n_cases > 2) break;
+        }
+        if (block == -1) {
+            if (n_cases > 0 && ch == cases[0]) continue;
+            if (n_cases < 2) {
+                cases[n_cases++] = ch;
+            } else if (ch != cases[1]) {
+                ++n_cases;
+                break;
+            }
+        } else {
+            if ((bitmap & (1u << (ch & 0x1f))) == 0) {
+                bitmap |= 1u << (ch & 0x1f);
+                if (n_cases < 2)
+                    cases[n_cases] = ch;
+                ++n_cases;
+            }
+        }
+    }
+
+    if (block != -1 || n_cases <= 2) {
+        char buf[64];
+        g->I[2] = block;
+        g->I[3] = bitmap;
+        g->I[4] = shortest_size - 1;
+        if (p->mode == m_forward) {
+            sprintf(buf, "z->p[z->c + %d]", shortest_size - 1);
+            g->S[1] = buf;
+            if (shortest_size == 1) {
+                writef(g, "~Mif (z->c >= z->l", p);
+            } else {
+                writef(g, "~Mif (z->c + ~I4 >= z->l", p);
+            }
+        } else {
+            g->S[1] = "z->p[z->c - 1]";
+            if (shortest_size == 1) {
+                writef(g, "~Mif (z->c <= z->lb", p);
+            } else {
+                writef(g, "~Mif (z->c - ~I4 <= z->lb", p);
+            }
+        }
+        if (n_cases == 0) {
+            /* We get this for the degenerate case: among { '' }
+             * This doesn't seem to be a useful construct, but it is
+             * syntactically valid.
+             */
+        } else if (n_cases == 1) {
+            g->I[4] = cases[0];
+            writef(g, " || ~S1 != ~I4", p);
+        } else if (n_cases == 2) {
+            g->I[4] = cases[0];
+            g->I[5] = cases[1];
+            writef(g, " || (~S1 != ~I4 && ~S1 != ~I5)", p);
+        } else {
+            writef(g, " || ~S1 >> 5 != ~I2 || !((~I3 >> (~S1 & 0x1f)) & 1)", p);
+        }
+        write_string(g, ") ");
+        if (empty_case != -1) {
+            /* If the among includes the empty string, it can never fail
+             * so not matching the bitmap means we match the empty string.
+             */
+            g->I[4] = among_cases[empty_case].result;
+            writef(g, "among_var = ~I4; else~C", p);
+        } else {
+            writef(g, "~f~C", p);
+        }
+        shown_comment = 1;
+    } else {
+#ifdef OPTIMISATION_WARNINGS
+        printf("Couldn't shortcut among %d\n", x->number);
+#endif
+    }
+
+    if (!x->amongvar_needed) {
+        writef(g, "~Mif (!(find_among~S0(z, a_~I0, ~I1))) ~f", p);
+        writef(g, shown_comment ? "~N" : "~C", p);
+    } else {
+        writef(g, "~Mamong_var = find_among~S0(z, a_~I0, ~I1);", p);
+        writef(g, shown_comment ? "~N" : "~C", p);
+        writef(g, "~Mif (!(among_var)) ~f~N", p);
+    }
+}
+
+static void generate_among(struct generator * g, struct node * p) {
+
+    struct among * x = p->among;
+
+    if (x->substring == 0) generate_substring(g, p);
+
+    if (x->starter != 0) generate(g, x->starter);
+
+    if (x->command_count == 1 && x->nocommand_count == 0) {
+        /* Only one outcome ("no match" already handled). */
+        generate(g, x->commands[0]);
+    } else if (x->command_count > 0) {
+        int i;
+        writef(g, "~Mswitch (among_var) {~C~+", p);
+        for (i = 1; i <= x->command_count; i++) {
+            g->I[0] = i;
+            w(g, "~Mcase ~I0:~N~+");
+            generate(g, x->commands[i - 1]);
+            w(g, "~Mbreak;~N~-");
+        }
+        w(g, "~}");
+    }
+}
+
+static void generate_booltest(struct generator * g, struct node * p) {
+
+    g->V[0] = p->name;
+    writef(g, "~Mif (!(~V0)) ~f~C", p);
+}
+
+static void generate_false(struct generator * g, struct node * p) {
+
+    writef(g, "~M~f~C", p);
+}
+
+static void generate_debug(struct generator * g, struct node * p) {
+
+    g->I[0] = g->debug_count++;
+    g->I[1] = p->line_number;
+    writef(g, "~Mdebug(z, ~I0, ~I1);~C", p);
+
+}
+
+static void generate(struct generator * g, struct node * p) {
+
+    int used = g->label_used;
+    int a0 = g->failure_label;
+    int a1 = g->failure_keep_count;
+
+    switch (p->type) {
+        case c_define:        generate_define(g, p); break;
+        case c_bra:           generate_bra(g, p); break;
+        case c_and:           generate_and(g, p); break;
+        case c_or:            generate_or(g, p); break;
+        case c_backwards:     generate_backwards(g, p); break;
+        case c_not:           generate_not(g, p); break;
+        case c_set:           generate_set(g, p); break;
+        case c_unset:         generate_unset(g, p); break;
+        case c_try:           generate_try(g, p); break;
+        case c_fail:          generate_fail(g, p); break;
+        case c_reverse:
+        case c_test:          generate_test(g, p); break;
+        case c_do:            generate_do(g, p); break;
+        case c_goto:          generate_GO(g, p, 1); break;
+        case c_gopast:        generate_GO(g, p, 0); break;
+        case c_repeat:        generate_repeat(g, p); break;
+        case c_loop:          generate_loop(g, p); break;
+        case c_atleast:       generate_atleast(g, p); break;
+        case c_setmark:       generate_setmark(g, p); break;
+        case c_tomark:        generate_tomark(g, p); break;
+        case c_atmark:        generate_atmark(g, p); break;
+        case c_hop:           generate_hop(g, p); break;
+        case c_delete:        generate_delete(g, p); break;
+        case c_next:          generate_next(g, p); break;
+        case c_tolimit:       generate_tolimit(g, p); break;
+        case c_atlimit:       generate_atlimit(g, p); break;
+        case c_leftslice:     generate_leftslice(g, p); break;
+        case c_rightslice:    generate_rightslice(g, p); break;
+        case c_assignto:      generate_assignto(g, p); break;
+        case c_sliceto:       generate_sliceto(g, p); break;
+        case c_assign:        generate_assignfrom(g, p); break;
+        case c_insert:
+        case c_attach:        generate_insert(g, p, p->type); break;
+        case c_slicefrom:     generate_slicefrom(g, p); break;
+        case c_setlimit:      generate_setlimit(g, p); break;
+        case c_dollar:        generate_dollar(g, p); break;
+        case c_mathassign:    generate_integer_assign(g, p, "="); break;
+        case c_plusassign:    generate_integer_assign(g, p, "+="); break;
+        case c_minusassign:   generate_integer_assign(g, p, "-="); break;
+        case c_multiplyassign:generate_integer_assign(g, p, "*="); break;
+        case c_divideassign:  generate_integer_assign(g, p, "/="); break;
+        case c_eq:            generate_integer_test(g, p, "=="); break;
+        case c_ne:            generate_integer_test(g, p, "!="); break;
+        case c_gr:            generate_integer_test(g, p, ">"); break;
+        case c_ge:            generate_integer_test(g, p, ">="); break;
+        case c_ls:            generate_integer_test(g, p, "<"); break;
+        case c_le:            generate_integer_test(g, p, "<="); break;
+        case c_call:          generate_call(g, p); break;
+        case c_grouping:      generate_grouping(g, p, false); break;
+        case c_non:           generate_grouping(g, p, true); break;
+        case c_name:          generate_namedstring(g, p); break;
+        case c_literalstring: generate_literalstring(g, p); break;
+        case c_among:         generate_among(g, p); break;
+        case c_substring:     generate_substring(g, p); break;
+        case c_booltest:      generate_booltest(g, p); break;
+        case c_false:         generate_false(g, p); break;
+        case c_true:          break;
+        case c_debug:         generate_debug(g, p); break;
+        default: fprintf(stderr, "%d encountered\n", p->type);
+                 exit(1);
+    }
+
+    if (g->failure_label != a0)
+        g->label_used = used;
+    g->failure_label = a0;
+    g->failure_keep_count = a1;
+}
+
+void write_generated_comment_content(struct generator * g) {
+    w(g, "Generated by Snowball " SNOWBALL_VERSION
+         " - https://snowballstem.org/");
+}
+
+void write_start_comment(struct generator * g,
+                         const char * comment_start,
+                         const char * comment_end) {
+    write_margin(g);
+    w(g, comment_start);
+    write_generated_comment_content(g);
+    if (comment_end) {
+        w(g, comment_end);
+    }
+    w(g, "~N~N");
+}
+
+static void generate_head(struct generator * g) {
+
+    w(g, "#include \"");
+    if (g->options->runtime_path) {
+        write_string(g, g->options->runtime_path);
+        if (g->options->runtime_path[strlen(g->options->runtime_path) - 1] != '/')
+            write_char(g, '/');
+    }
+    w(g, "header.h\"~N~N");
+}
+
+static void generate_routine_headers(struct generator * g) {
+    struct name * q;
+    for (q = g->analyser->names; q; q = q->next) {
+        g->V[0] = q;
+        switch (q->type) {
+            case t_routine:
+                w(g, "static int ~W0(struct SN_env * z);~N");
+                break;
+            case t_external:
+                w(g,
+                  "#ifdef __cplusplus~N"
+                  "extern \"C\" {~N"
+                  "#endif~N"
+                  "extern int ~W0(struct SN_env * z);~N"
+                  "#ifdef __cplusplus~N"
+                  "}~N"
+                  "#endif~N"
+                  );
+                break;
+        }
+    }
+}
+
+static void generate_among_table(struct generator * g, struct among * x) {
+
+    struct amongvec * v = x->b;
+
+    g->I[0] = x->number;
+    {
+        int i;
+        for (i = 0; i < x->literalstring_count; i++) {
+            g->I[1] = i;
+            g->I[2] = v->size;
+            g->L[0] = v->b;
+            if (v->size)
+                w(g, "static const symbol s_~I0_~I1[~I2] = ~A0;~N");
+            v++;
+        }
+    }
+
+    g->I[1] = x->literalstring_count;
+    w(g, "~N~Mstatic const struct among a_~I0[~I1] =~N{~N");
+
+    v = x->b;
+    {
+        int i;
+        for (i = 0; i < x->literalstring_count; i++) {
+            g->I[1] = i;
+            g->I[2] = v->size;
+            g->I[3] = v->i;
+            g->I[4] = v->result;
+            g->S[0] = i < x->literalstring_count - 1 ? "," : "";
+
+            if (g->options->comments) {
+                w(g, "/*~J1 */ ");
+            }
+            w(g, "{ ~I2, ");
+            if (v->size == 0) {
+                w(g, "0,");
+            } else {
+                w(g, "s_~I0_~I1,");
+            }
+            w(g, " ~I3, ~I4, ");
+            if (v->function == 0) {
+                write_char(g, '0');
+            } else {
+                write_varname(g, v->function);
+            }
+            w(g, "}~S0~N");
+            v++;
+        }
+    }
+    w(g, "};~N~N");
+}
+
+static void generate_amongs(struct generator * g) {
+    struct among * x;
+    for (x = g->analyser->amongs; x; x = x->next) {
+        generate_among_table(g, x);
+    }
+}
+
+static void set_bit(symbol * b, int i) { b[i/8] |= 1 << i%8; }
+
+static void generate_grouping_table(struct generator * g, struct grouping * q) {
+
+    int range = q->largest_ch - q->smallest_ch + 1;
+    int size = (range + 7)/ 8;  /* assume 8 bits per symbol */
+    symbol * b = q->b;
+    symbol * map = create_b(size);
+    int i;
+    for (i = 0; i < size; i++) map[i] = 0;
+
+    for (i = 0; i < SIZE(b); i++) set_bit(map, b[i] - q->smallest_ch);
+
+    g->V[0] = q->name;
+
+    w(g, "static const unsigned char ~V0[] = { ");
+    for (i = 0; i < size; i++) {
+        write_int(g, map[i]);
+        if (i < size - 1) w(g, ", ");
+    }
+    w(g, " };~N~N");
+    lose_b(map);
+}
+
+static void generate_groupings(struct generator * g) {
+    struct grouping * q;
+    for (q = g->analyser->groupings; q; q = q->next) {
+        if (q->name->used)
+            generate_grouping_table(g, q);
+    }
+}
+
+static void generate_create(struct generator * g) {
+
+    int * p = g->analyser->name_count;
+    g->I[0] = p[t_string];
+    g->I[1] = p[t_integer] + p[t_boolean];
+    w(g, "~N"
+         "extern struct SN_env * ~pcreate_env(void) { return SN_create_env(~I0, ~I1); }"
+         "~N");
+}
+
+static void generate_close(struct generator * g) {
+
+    int * p = g->analyser->name_count;
+    g->I[0] = p[t_string];
+    w(g, "~Nextern void ~pclose_env(struct SN_env * z) { SN_close_env(z, ~I0); }~N~N");
+}
+
+static void generate_create_and_close_templates(struct generator * g) {
+    w(g, "~N"
+         "extern struct SN_env * ~pcreate_env(void);~N"
+         "extern void ~pclose_env(struct SN_env * z);~N"
+         "~N");
+}
+
+static void generate_header_file(struct generator * g) {
+
+    struct name * q;
+    const char * vp = g->options->variables_prefix;
+    g->S[0] = vp;
+
+    w(g, "#ifdef __cplusplus~N"
+         "extern \"C\" {~N"
+         "#endif~N");            /* for C++ */
+
+    generate_create_and_close_templates(g);
+    for (q = g->analyser->names; q; q = q->next) {
+        g->V[0] = q;
+        switch (q->type) {
+            case t_external:
+                w(g, "extern int ~W0(struct SN_env * z);~N");
+                break;
+            case t_string:
+            case t_integer:
+            case t_boolean:
+                if (vp) {
+                    int count = q->count;
+                    if (count < 0) {
+                        /* Unused variables should get removed from `names`. */
+                        fprintf(stderr, "Optimised out variable ");
+                        report_b(stderr, q->b);
+                        fprintf(stderr, " still in names list\n");
+                        exit(1);
+                    }
+                    if (q->type == t_boolean) {
+                        /* We use a single array for booleans and integers,
+                         * with the integers first.
+                         */
+                        count += g->analyser->name_count[t_integer];
+                    }
+                    g->I[0] = count;
+                    g->I[1] = "SIIrxg"[q->type];
+                    w(g, "#define ~S0");
+                    write_b(g, q->b);
+                    w(g, " (~c1[~I0])~N");
+                }
+                break;
+        }
+    }
+
+    w(g, "~N"
+         "#ifdef __cplusplus~N"
+         "}~N"
+         "#endif~N");            /* for C++ */
+
+    w(g, "~N");
+}
+
+extern void generate_program_c(struct generator * g) {
+
+    g->outbuf = str_new();
+    write_start_comment(g, "/* ", " */");
+    generate_head(g);
+    generate_routine_headers(g);
+    w(g, "#ifdef __cplusplus~N"
+         "extern \"C\" {~N"
+         "#endif~N"
+         "~N");
+    generate_create_and_close_templates(g);
+    w(g, "~N"
+         "#ifdef __cplusplus~N"
+         "}~N"
+         "#endif~N");
+    generate_amongs(g);
+    generate_groupings(g);
+    g->declarations = g->outbuf;
+    g->outbuf = str_new();
+    g->literalstring_count = 0;
+    {
+        struct node * p = g->analyser->program;
+        while (p) { generate(g, p); p = p->right; }
+    }
+    generate_create(g);
+    generate_close(g);
+    output_str(g->options->output_src, g->declarations);
+    str_delete(g->declarations);
+    output_str(g->options->output_src, g->outbuf);
+    str_clear(g->outbuf);
+
+    write_start_comment(g, "/* ", " */");
+    generate_header_file(g);
+    output_str(g->options->output_h, g->outbuf);
+    str_delete(g->outbuf);
+}
+
+/* Generator functions common to multiple languages. */
+
+extern struct generator * create_generator(struct analyser * a, struct options * o) {
+    NEW(generator, g);
+    g->analyser = a;
+    g->options = o;
+    g->margin = 0;
+    g->debug_count = 0;
+    g->copy_from_count = 0;
+    g->line_count = 0;
+    g->line_labelled = 0;
+    g->failure_label = -1;
+    g->unreachable = false;
+#ifndef DISABLE_PYTHON
+    g->max_label = 0;
+#endif
+    return g;
+}
+
+extern void close_generator(struct generator * g) {
+    FREE(g);
+}
+
+/* Write routines for simple entities */
+
+extern void write_char(struct generator * g, int ch) {
+    str_append_ch(g->outbuf, ch); /* character */
+}
+
+extern void write_newline(struct generator * g) {
+    str_append_ch(g->outbuf, '\n'); /* newline */
+    g->line_count++;
+}
+
+extern void write_string(struct generator * g, const char * s) {
+    str_append_string(g->outbuf, s);
+}
+
+extern void write_int(struct generator * g, int i) {
+    str_append_int(g->outbuf, i);
+}
+
+extern void write_b(struct generator * g, symbol * b) {
+
+    str_append_b(g->outbuf, b);
+}
+
+extern void write_str(struct generator * g, struct str * str) {
+
+    str_append(g->outbuf, str);
+}
diff --git a/contrib/snowball/compiler/header.h b/contrib/snowball/compiler/header.h
new file mode 100644
index 0000000..dadbee3
--- /dev/null
+++ b/contrib/snowball/compiler/header.h
@@ -0,0 +1,411 @@
+#include <stdio.h>
+
+#define SNOWBALL_VERSION "2.0.0"
+
+typedef unsigned char byte;
+typedef unsigned short symbol;
+
+#define true 1
+#define false 0
+
+#define MALLOC check_malloc
+#define FREE check_free
+
+#define NEW(type, p) struct type * p = (struct type *) MALLOC(sizeof(struct type))
+#define NEWVEC(type, p, n) struct type * p = (struct type *) MALLOC(sizeof(struct type) * (n))
+
+#define SIZE(p)     ((int *)(p))[-1]
+#define CAPACITY(p) ((int *)(p))[-2]
+
+extern symbol * create_b(int n);
+extern void report_b(FILE * out, const symbol * p);
+extern void lose_b(symbol * p);
+extern symbol * increase_capacity(symbol * p, int n);
+extern symbol * move_to_b(symbol * p, int n, const symbol * q);
+extern symbol * add_to_b(symbol * p, int n, const symbol * q);
+extern symbol * copy_b(const symbol * p);
+extern char * b_to_s(const symbol * p);
+extern symbol * add_s_to_b(symbol * p, const char * s);
+
+#define MOVE_TO_B(B, LIT) \
+    move_to_b(B, sizeof(LIT) / sizeof(LIT[0]), LIT)
+
+struct str; /* defined in space.c */
+
+extern struct str * str_new(void);
+extern void str_delete(struct str * str);
+extern void str_append(struct str * str, const struct str * add);
+extern void str_append_ch(struct str * str, char add);
+extern void str_append_b(struct str * str, const symbol * q);
+extern void str_append_b_tail(struct str * str, const symbol * q, int skip);
+extern void str_append_string(struct str * str, const char * s);
+extern void str_append_int(struct str * str, int i);
+extern void str_clear(struct str * str);
+extern void str_assign(struct str * str, const char * s);
+extern struct str * str_copy(const struct str * old);
+extern symbol * str_data(const struct str * str);
+extern int str_len(const struct str * str);
+extern int str_back(const struct str *str);
+extern int get_utf8(const symbol * p, int * slot);
+extern int put_utf8(int ch, symbol * p);
+extern void output_str(FILE * outfile, struct str * str);
+
+typedef enum { ENC_SINGLEBYTE, ENC_UTF8, ENC_WIDECHARS } enc;
+
+struct m_pair {
+
+    struct m_pair * next;
+    symbol * name;
+    symbol * value;
+
+};
+
+/* struct input must be a prefix of struct tokeniser. */
+struct input {
+
+    struct input * next;
+    symbol * p;
+    int c;
+    char * file;
+    int file_needs_freeing;
+    int line_number;
+
+};
+
+struct include {
+
+    struct include * next;
+    symbol * b;
+
+};
+
+enum token_codes {
+
+#include "syswords2.h"
+
+    c_mathassign,
+    c_name,
+    c_number,
+    c_literalstring,
+    c_neg,
+    c_call,
+    c_grouping,
+    c_booltest,
+
+    NUM_TOKEN_CODES
+};
+
+enum uplus_modes {
+    UPLUS_NONE,
+    UPLUS_DEFINED,
+    UPLUS_UNICODE
+};
+
+/* struct input must be a prefix of struct tokeniser. */
+struct tokeniser {
+
+    struct input * next;
+    symbol * p;
+    int c;
+    char * file;
+    int file_needs_freeing;
+    int line_number;
+    symbol * b;
+    symbol * b2;
+    int number;
+    int m_start;
+    int m_end;
+    struct m_pair * m_pairs;
+    int get_depth;
+    int error_count;
+    int token;
+    int previous_token;
+    byte token_held;
+    enc encoding;
+
+    int omission;
+    struct include * includes;
+
+    /* Mode in which U+ has been used:
+     * UPLUS_NONE - not used yet
+     * UPLUS_DEFINED - stringdef U+xxxx ....
+     * UPLUS_UNICODE - {U+xxxx} used with implicit meaning
+     */
+    int uplusmode;
+
+    char token_disabled[NUM_TOKEN_CODES];
+};
+
+extern symbol * get_input(const char * filename);
+extern struct tokeniser * create_tokeniser(symbol * b, char * file);
+extern int read_token(struct tokeniser * t);
+extern const char * name_of_token(int code);
+extern void disable_token(struct tokeniser * t, int code);
+extern void close_tokeniser(struct tokeniser * t);
+
+extern int space_count;
+extern void * check_malloc(int n);
+extern void check_free(void * p);
+
+struct node;
+
+struct name {
+
+    struct name * next;
+    symbol * b;
+    int type;                   /* t_string etc */
+    int mode;                   /*    )_  for routines, externals */
+    struct node * definition;   /*    )                           */
+    int count;                  /* 0, 1, 2 for each type */
+    struct grouping * grouping; /* for grouping names */
+    byte referenced;
+    byte used_in_among;         /* Function used in among? */
+    byte value_used;            /* (For variables) is its value ever used? */
+    byte initialised;           /* (For variables) is it ever initialised? */
+    byte used_in_definition;    /* (grouping) used in grouping definition? */
+    struct node * used;         /* First use, or NULL if not used */
+    struct name * local_to;     /* Local to one routine/external */
+    int declaration_line_number;/* Line number of declaration */
+
+};
+
+struct literalstring {
+
+    struct literalstring * next;
+    symbol * b;
+
+};
+
+struct amongvec {
+
+    symbol * b;      /* the string giving the case */
+    int size;        /* - and its size */
+    struct node * action; /* the corresponding action */
+    int i;           /* the amongvec index of the longest substring of b */
+    int result;      /* the numeric result for the case */
+    int line_number; /* for diagnostics and stable sorting */
+    struct name * function;
+
+};
+
+struct among {
+
+    struct among * next;
+    struct amongvec * b;      /* pointer to the amongvec */
+    int number;               /* amongs are numbered 0, 1, 2 ... */
+    int literalstring_count;  /* in this among */
+    int command_count;        /* in this among */
+    int nocommand_count;      /* number of "no command" entries in this among */
+    int function_count;       /* in this among */
+    int amongvar_needed;      /* do we need to set among_var? */
+    struct node * starter;    /* i.e. among( (starter) 'string' ... ) */
+    struct node * substring;  /* i.e. substring ... among ( ... ) */
+    struct node ** commands;  /* array with command_count entries */
+};
+
+struct grouping {
+
+    struct grouping * next;
+    symbol * b;               /* the characters of this group */
+    int largest_ch;           /* character with max code */
+    int smallest_ch;          /* character with min code */
+    struct name * name;       /* so g->name->grouping == g */
+    int line_number;
+};
+
+struct node {
+
+    struct node * next;
+    struct node * left;
+    struct node * aux;     /* used in setlimit */
+    struct among * among;  /* used in among */
+    struct node * right;
+    int type;
+    int mode;
+    struct node * AE;
+    struct name * name;
+    symbol * literalstring;
+    int number;
+    int line_number;
+    int amongvar_needed;   /* used in routine definitions */
+};
+
+enum name_types {
+
+    t_size = 6,
+
+    t_string = 0, t_boolean = 1, t_integer = 2, t_routine = 3, t_external = 4,
+    t_grouping = 5
+
+/*  If this list is extended, adjust wvn in generator.c  */
+};
+
+/*  In name_count[i] below, remember that
+    type   is
+    ----+----
+      0 |  string
+      1 |  boolean
+      2 |  integer
+      3 |  routine
+      4 |  external
+      5 |  grouping
+*/
+
+struct analyser {
+
+    struct tokeniser * tokeniser;
+    struct node * nodes;
+    struct name * names;
+    struct literalstring * literalstrings;
+    int mode;
+    byte modifyable;          /* false inside reverse(...) */
+    struct node * program;
+    struct node * program_end;
+    int name_count[t_size];   /* name_count[i] counts the number of names of type i */
+    struct among * amongs;
+    struct among * amongs_end;
+    int among_count;
+    int amongvar_needed;      /* used in reading routine definitions */
+    struct grouping * groupings;
+    struct grouping * groupings_end;
+    struct node * substring;  /* pending 'substring' in current routine definition */
+    enc encoding;
+    byte int_limits_used;     /* are maxint or minint used? */
+};
+
+enum analyser_modes {
+
+    m_forward = 0, m_backward /*, m_integer */
+
+};
+
+extern void print_program(struct analyser * a);
+extern struct analyser * create_analyser(struct tokeniser * t);
+extern void close_analyser(struct analyser * a);
+
+extern void read_program(struct analyser * a);
+
+struct generator {
+
+    struct analyser * analyser;
+    struct options * options;
+    int unreachable;           /* 0 if code can be reached, 1 if current code
+                                * is unreachable. */
+    int var_number;            /* Number of next variable to use. */
+    struct str * outbuf;       /* temporary str to store output */
+    struct str * declarations; /* str storing variable declarations */
+    int next_label;
+#ifndef DISABLE_PYTHON
+    int max_label;
+#endif
+    int margin;
+
+    /* if > 0, keep_count to restore in case of a failure;
+     * if < 0, the negated keep_count for the limit to restore in case of
+     * failure. */
+    int failure_keep_count;
+#if !defined(DISABLE_JAVA) && !defined(DISABLE_JS) && !defined(DISABLE_PYTHON) && !defined(DISABLE_CSHARP)
+    struct str * failure_str;  /* This is used by some generators instead of failure_keep_count */
+#endif
+
+    int label_used;     /* Keep track of whether the failure label is used. */
+    int failure_label;
+    int debug_count;
+    int copy_from_count; /* count of calls to copy_from() */
+
+    const char * S[10];  /* strings */
+    symbol * B[10];      /* blocks */
+    int I[10];           /* integers */
+    struct name * V[5];  /* variables */
+    symbol * L[5];       /* literals, used in formatted write */
+
+    int line_count;      /* counts number of lines output */
+    int line_labelled;   /* in ISO C, will need extra ';' if it is a block end */
+    int literalstring_count;
+    int keep_count;      /* used to number keep/restore pairs to avoid compiler warnings
+                            about shadowed variables */
+};
+
+/* Special values for failure_label in struct generator. */
+enum special_labels {
+    x_return = -1
+};
+
+struct options {
+
+    /* for the command line: */
+
+    const char * output_file;
+    const char * name;
+    FILE * output_src;
+    FILE * output_h;
+    byte syntax_tree;
+    byte comments;
+    enc encoding;
+    enum { LANG_JAVA, LANG_C, LANG_CPLUSPLUS, LANG_CSHARP, LANG_PASCAL, LANG_PYTHON, LANG_JAVASCRIPT, LANG_RUST, LANG_GO } make_lang;
+    const char * externals_prefix;
+    const char * variables_prefix;
+    const char * runtime_path;
+    const char * parent_class_name;
+    const char * package;
+    const char * go_snowball_runtime;
+    const char * string_class;
+    const char * among_class;
+    struct include * includes;
+    struct include * includes_end;
+};
+
+/* Generator functions common to several backends. */
+
+extern struct generator * create_generator(struct analyser * a, struct options * o);
+extern void close_generator(struct generator * g);
+
+extern void write_char(struct generator * g, int ch);
+extern void write_newline(struct generator * g);
+extern void write_string(struct generator * g, const char * s);
+extern void write_int(struct generator * g, int i);
+extern void write_b(struct generator * g, symbol * b);
+extern void write_str(struct generator * g, struct str * str);
+
+extern void write_comment_content(struct generator * g, struct node * p);
+extern void write_generated_comment_content(struct generator * g);
+extern void write_start_comment(struct generator * g,
+                                const char * comment_start,
+                                const char * comment_end);
+
+extern int K_needed(struct generator * g, struct node * p);
+extern int repeat_restore(struct generator * g, struct node * p);
+
+/* Generator for C code. */
+extern void generate_program_c(struct generator * g);
+
+#ifndef DISABLE_JAVA
+/* Generator for Java code. */
+extern void generate_program_java(struct generator * g);
+#endif
+
+#ifndef DISABLE_CSHARP
+/* Generator for C# code. */
+extern void generate_program_csharp(struct generator * g);
+#endif
+
+#ifndef DISABLE_PASCAL
+extern void generate_program_pascal(struct generator * g);
+#endif
+
+#ifndef DISABLE_PYTHON
+/* Generator for Python code. */
+extern void generate_program_python(struct generator * g);
+#endif
+
+#ifndef DISABLE_JS
+extern void generate_program_js(struct generator * g);
+#endif
+
+#ifndef DISABLE_RUST
+extern void generate_program_rust(struct generator * g);
+#endif
+
+#ifndef DISABLE_GO
+extern void generate_program_go(struct generator * g);
+#endif
diff --git a/contrib/snowball/compiler/space.c b/contrib/snowball/compiler/space.c
new file mode 100644
index 0000000..5b05876
--- /dev/null
+++ b/contrib/snowball/compiler/space.c
@@ -0,0 +1,287 @@
+
+#include <stdio.h>    /* for printf */
+#include <stdlib.h>   /* malloc, free */
+#include <string.h>   /* memmove */
+
+#include "header.h"
+
+#define HEAD 2*sizeof(int)
+#define EXTENDER 40
+
+
+/*  This modules provides a simple mechanism for arbitrary length writable
+    strings, called 'blocks'. They are 'symbol *' items rather than 'char *'
+    items however.
+
+    The calls are:
+
+        symbol * b = create_b(n);
+            - create an empty block b with room for n symbols
+        b = increase_capacity(b, n);
+            - increase the capacity of block b by n symbols (b may change)
+        b2 = copy_b(b)
+            - copy block b into b2
+        lose_b(b);
+            - lose block b
+        b = move_to_b(b, n, p);
+            - set the data in b to be the n symbols at address p
+        b = add_to_b(b, n, p);
+            - add the n symbols at address p to the end of the data in b
+        SIZE(b)
+            - is the number of symbols in b
+        For example:
+
+        symbol * b = create_b(0);
+        {   int i;
+            char p[10];
+            for (i = 0; i < 100; i++) {
+                sprintf(p, " %d", i);
+                add_s_to_b(b, p);
+            }
+        }
+
+    and b contains " 0 1 2 ... 99" spaced out as symbols.
+*/
+
+/*  For a block b, SIZE(b) is the number of symbols so far written into it,
+    CAPACITY(b) the total number it can contain, so SIZE(b) <= CAPACITY(b).
+    In fact blocks have 1 extra character over the promised capacity so
+    they can be zero terminated by 'b[SIZE(b)] = 0;' without fear of
+    overwriting.
+*/
+
+extern symbol * create_b(int n) {
+    symbol * p = (symbol *) (HEAD + (char *) MALLOC(HEAD + (n + 1) * sizeof(symbol)));
+    CAPACITY(p) = n;
+    SIZE(p) = 0;
+    return p;
+}
+
+extern void report_b(FILE * out, const symbol * p) {
+    int i;
+    for (i = 0; i < SIZE(p); i++) {
+        if (p[i] > 255) {
+            printf("In report_b, can't convert p[%d] to char because it's 0x%02x\n", i, (int)p[i]);
+            exit(1);
+        }
+        putc(p[i], out);
+    }
+}
+
+extern void output_str(FILE * outfile, struct str * str) {
+    report_b(outfile, str_data(str));
+}
+
+extern void lose_b(symbol * p) {
+    if (p == 0) return;
+    FREE((char *) p - HEAD);
+}
+
+extern symbol * increase_capacity(symbol * p, int n) {
+    symbol * q = create_b(CAPACITY(p) + n + EXTENDER);
+    memmove(q, p, CAPACITY(p) * sizeof(symbol));
+    SIZE(q) = SIZE(p);
+    lose_b(p); return q;
+}
+
+extern symbol * move_to_b(symbol * p, int n, const symbol * q) {
+    int x = n - CAPACITY(p);
+    if (x > 0) p = increase_capacity(p, x);
+    memmove(p, q, n * sizeof(symbol)); SIZE(p) = n; return p;
+}
+
+extern symbol * add_to_b(symbol * p, int n, const symbol * q) {
+    int x = SIZE(p) + n - CAPACITY(p);
+    if (x > 0) p = increase_capacity(p, x);
+    memmove(p + SIZE(p), q, n * sizeof(symbol)); SIZE(p) += n; return p;
+}
+
+extern symbol * copy_b(const symbol * p) {
+    int n = SIZE(p);
+    symbol * q = create_b(n);
+    move_to_b(q, n, p);
+    return q;
+}
+
+int space_count = 0;
+
+extern void * check_malloc(int n) {
+    space_count++;
+    return malloc(n);
+}
+
+extern void check_free(void * p) {
+    space_count--;
+    free(p);
+}
+
+/* To convert a block to a zero terminated string:  */
+
+extern char * b_to_s(const symbol * p) {
+    int n = SIZE(p);
+    char * s = (char *)malloc(n + 1);
+    {
+        int i;
+        for (i = 0; i < n; i++) {
+            if (p[i] > 255) {
+                printf("In b_to_s, can't convert p[%d] to char because it's 0x%02x\n", i, (int)p[i]);
+                exit(1);
+            }
+            s[i] = (char)p[i];
+        }
+    }
+    s[n] = 0;
+    return s;
+}
+
+/* To add a zero terminated string to a block. If p = 0 the
+   block is created. */
+
+extern symbol * add_s_to_b(symbol * p, const char * s) {
+    int n = strlen(s);
+    int k;
+    if (p == 0) p = create_b(n);
+    k = SIZE(p);
+    {
+        int x = k + n - CAPACITY(p);
+        if (x > 0) p = increase_capacity(p, x);
+    }
+    {
+        int i;
+        for (i = 0; i < n; i++) p[i + k] = s[i];
+    }
+    SIZE(p) += n;
+    return p;
+}
+
+/* The next section defines string handling capabilities in terms
+   of the lower level block handling capabilities of space.c */
+/* -------------------------------------------------------------*/
+
+struct str {
+    symbol * data;
+};
+
+/* Create a new string. */
+extern struct str * str_new(void) {
+
+    struct str * output = (struct str *) malloc(sizeof(struct str));
+    output->data = create_b(0);
+    return output;
+}
+
+/* Delete a string. */
+extern void str_delete(struct str * str) {
+
+    lose_b(str->data);
+    free(str);
+}
+
+/* Append a str to this str. */
+extern void str_append(struct str * str, const struct str * add) {
+
+    symbol * q = add->data;
+    str->data = add_to_b(str->data, SIZE(q), q);
+}
+
+/* Append a character to this str. */
+extern void str_append_ch(struct str * str, char add) {
+
+    symbol q[1];
+    q[0] = add;
+    str->data = add_to_b(str->data, 1, q);
+}
+
+/* Append a low level block to a str. */
+extern void str_append_b(struct str * str, const symbol * q) {
+
+    str->data = add_to_b(str->data, SIZE(q), q);
+}
+
+/* Append the tail of a low level block to a str. */
+extern void str_append_b_tail(struct str * str, const symbol * q, int skip) {
+    if (skip < 0 || skip >= SIZE(q)) return;
+
+    str->data = add_to_b(str->data, SIZE(q) - skip, q + skip);
+}
+
+/* Append a (char *, null terminated) string to a str. */
+extern void str_append_string(struct str * str, const char * s) {
+
+    str->data = add_s_to_b(str->data, s);
+}
+
+/* Append an integer to a str. */
+extern void str_append_int(struct str * str, int i) {
+
+    char s[30];
+    sprintf(s, "%d", i);
+    str_append_string(str, s);
+}
+
+/* Clear a string */
+extern void str_clear(struct str * str) {
+
+    SIZE(str->data) = 0;
+}
+
+/* Set a string */
+extern void str_assign(struct str * str, const char * s) {
+
+    str_clear(str);
+    str_append_string(str, s);
+}
+
+/* Copy a string. */
+extern struct str * str_copy(const struct str * old) {
+
+    struct str * newstr = str_new();
+    str_append(newstr, old);
+    return newstr;
+}
+
+/* Get the data stored in this str. */
+extern symbol * str_data(const struct str * str) {
+
+    return str->data;
+}
+
+/* Get the length of the str. */
+extern int str_len(const struct str * str) {
+
+    return SIZE(str->data);
+}
+
+/* Get the last character of the str.
+ *
+ * Or -1 if the string is empty.
+ */
+extern int str_back(const struct str *str) {
+    return SIZE(str->data) ? str->data[SIZE(str->data) - 1] : -1;
+}
+
+extern int get_utf8(const symbol * p, int * slot) {
+    int b0, b1;
+    b0 = *p++;
+    if (b0 < 0xC0) {   /* 1100 0000 */
+        * slot = b0; return 1;
+    }
+    b1 = *p++;
+    if (b0 < 0xE0) {   /* 1110 0000 */
+        * slot = (b0 & 0x1F) << 6 | (b1 & 0x3F); return 2;
+    }
+    * slot = (b0 & 0xF) << 12 | (b1 & 0x3F) << 6 | (*p & 0x3F); return 3;
+}
+
+extern int put_utf8(int ch, symbol * p) {
+    if (ch < 0x80) {
+        p[0] = ch; return 1;
+    }
+    if (ch < 0x800) {
+        p[0] = (ch >> 6) | 0xC0;
+        p[1] = (ch & 0x3F) | 0x80; return 2;
+    }
+    p[0] = (ch >> 12) | 0xE0;
+    p[1] = ((ch >> 6) & 0x3F) | 0x80;
+    p[2] = (ch & 0x3F) | 0x80; return 3;
+}
diff --git a/contrib/snowball/compiler/syswords.h b/contrib/snowball/compiler/syswords.h
new file mode 100644
index 0000000..c401d3e
--- /dev/null
+++ b/contrib/snowball/compiler/syswords.h
@@ -0,0 +1,86 @@
+static const struct system_word vocab[82+1] = {
+  { 0, (const byte *)"", 82+1},
+
+  { 1, (const byte *)"$",             c_dollar },
+  { 1, (const byte *)"(",             c_bra },
+  { 1, (const byte *)")",             c_ket },
+  { 1, (const byte *)"*",             c_multiply },
+  { 1, (const byte *)"+",             c_plus },
+  { 1, (const byte *)"-",             c_minus },
+  { 1, (const byte *)"/",             c_divide },
+  { 1, (const byte *)"<",             c_ls },
+  { 1, (const byte *)"=",             c_assign },
+  { 1, (const byte *)">",             c_gr },
+  { 1, (const byte *)"?",             c_debug },
+  { 1, (const byte *)"[",             c_leftslice },
+  { 1, (const byte *)"]",             c_rightslice },
+  { 2, (const byte *)"!=",            c_ne },
+  { 2, (const byte *)"*=",            c_multiplyassign },
+  { 2, (const byte *)"+=",            c_plusassign },
+  { 2, (const byte *)"-=",            c_minusassign },
+  { 2, (const byte *)"->",            c_sliceto },
+  { 2, (const byte *)"/*",            c_comment2 },
+  { 2, (const byte *)"//",            c_comment1 },
+  { 2, (const byte *)"/=",            c_divideassign },
+  { 2, (const byte *)"<+",            c_insert },
+  { 2, (const byte *)"<-",            c_slicefrom },
+  { 2, (const byte *)"<=",            c_le },
+  { 2, (const byte *)"==",            c_eq },
+  { 2, (const byte *)"=>",            c_assignto },
+  { 2, (const byte *)">=",            c_ge },
+  { 2, (const byte *)"as",            c_as },
+  { 2, (const byte *)"do",            c_do },
+  { 2, (const byte *)"or",            c_or },
+  { 3, (const byte *)"and",           c_and },
+  { 3, (const byte *)"for",           c_for },
+  { 3, (const byte *)"get",           c_get },
+  { 3, (const byte *)"hex",           c_hex },
+  { 3, (const byte *)"hop",           c_hop },
+  { 3, (const byte *)"len",           c_len },
+  { 3, (const byte *)"non",           c_non },
+  { 3, (const byte *)"not",           c_not },
+  { 3, (const byte *)"set",           c_set },
+  { 3, (const byte *)"try",           c_try },
+  { 4, (const byte *)"fail",          c_fail },
+  { 4, (const byte *)"goto",          c_goto },
+  { 4, (const byte *)"loop",          c_loop },
+  { 4, (const byte *)"next",          c_next },
+  { 4, (const byte *)"size",          c_size },
+  { 4, (const byte *)"test",          c_test },
+  { 4, (const byte *)"true",          c_true },
+  { 5, (const byte *)"among",         c_among },
+  { 5, (const byte *)"false",         c_false },
+  { 5, (const byte *)"lenof",         c_lenof },
+  { 5, (const byte *)"limit",         c_limit },
+  { 5, (const byte *)"unset",         c_unset },
+  { 6, (const byte *)"atmark",        c_atmark },
+  { 6, (const byte *)"attach",        c_attach },
+  { 6, (const byte *)"cursor",        c_cursor },
+  { 6, (const byte *)"define",        c_define },
+  { 6, (const byte *)"delete",        c_delete },
+  { 6, (const byte *)"gopast",        c_gopast },
+  { 6, (const byte *)"insert",        c_insert },
+  { 6, (const byte *)"maxint",        c_maxint },
+  { 6, (const byte *)"minint",        c_minint },
+  { 6, (const byte *)"repeat",        c_repeat },
+  { 6, (const byte *)"sizeof",        c_sizeof },
+  { 6, (const byte *)"tomark",        c_tomark },
+  { 7, (const byte *)"atleast",       c_atleast },
+  { 7, (const byte *)"atlimit",       c_atlimit },
+  { 7, (const byte *)"decimal",       c_decimal },
+  { 7, (const byte *)"reverse",       c_reverse },
+  { 7, (const byte *)"setmark",       c_setmark },
+  { 7, (const byte *)"strings",       c_strings },
+  { 7, (const byte *)"tolimit",       c_tolimit },
+  { 8, (const byte *)"booleans",      c_booleans },
+  { 8, (const byte *)"integers",      c_integers },
+  { 8, (const byte *)"routines",      c_routines },
+  { 8, (const byte *)"setlimit",      c_setlimit },
+  { 9, (const byte *)"backwards",     c_backwards },
+  { 9, (const byte *)"externals",     c_externals },
+  { 9, (const byte *)"groupings",     c_groupings },
+  { 9, (const byte *)"stringdef",     c_stringdef },
+  { 9, (const byte *)"substring",     c_substring },
+ { 12, (const byte *)"backwardmode",  c_backwardmode },
+ { 13, (const byte *)"stringescapes", c_stringescapes }
+};
diff --git a/contrib/snowball/compiler/syswords2.h b/contrib/snowball/compiler/syswords2.h
new file mode 100644
index 0000000..e853e32
--- /dev/null
+++ b/contrib/snowball/compiler/syswords2.h
@@ -0,0 +1,13 @@
+    c_among = 4, c_and, c_as, c_assign, c_assignto, c_atleast,
+    c_atlimit, c_atmark, c_attach, c_backwardmode, c_backwards,
+    c_booleans, c_bra, c_comment1, c_comment2, c_cursor, c_debug,
+    c_decimal, c_define, c_delete, c_divide, c_divideassign, c_do,
+    c_dollar, c_eq, c_externals, c_fail, c_false, c_for, c_ge, c_get,
+    c_gopast, c_goto, c_gr, c_groupings, c_hex, c_hop, c_insert,
+    c_integers, c_ket, c_le, c_leftslice, c_len, c_lenof, c_limit, c_loop,
+    c_ls, c_maxint, c_minint, c_minus, c_minusassign, c_multiply,
+    c_multiplyassign, c_ne, c_next, c_non, c_not, c_or, c_plus,
+    c_plusassign, c_repeat, c_reverse, c_rightslice, c_routines,
+    c_set, c_setlimit, c_setmark, c_size, c_sizeof, c_slicefrom,
+    c_sliceto, c_stringdef, c_stringescapes, c_strings, c_substring,
+    c_test, c_tolimit, c_tomark, c_true, c_try, c_unset,
diff --git a/contrib/snowball/compiler/tokeniser.c b/contrib/snowball/compiler/tokeniser.c
new file mode 100644
index 0000000..e6c6386
--- /dev/null
+++ b/contrib/snowball/compiler/tokeniser.c
@@ -0,0 +1,567 @@
+
+#include <stdio.h>   /* stderr etc */
+#include <stdlib.h>  /* malloc free */
+#include <string.h>  /* strlen */
+#include <ctype.h>   /* isalpha etc */
+#include "header.h"
+
+struct system_word {
+    int s_size;      /* size of system word */
+    const byte * s;  /* pointer to the system word */
+    int code;        /* its internal code */
+};
+
+
+/* ASCII collating assumed in syswords.c */
+
+#include "syswords.h"
+
+#define INITIAL_INPUT_BUFFER_SIZE 8192
+
+static int hex_to_num(int ch);
+
+static int smaller(int a, int b) { return a < b ? a : b; }
+
+extern symbol * get_input(const char * filename) {
+    FILE * input = fopen(filename, "r");
+    if (input == 0) { return 0; }
+    {
+        symbol * u = create_b(INITIAL_INPUT_BUFFER_SIZE);
+        int size = 0;
+        while (true) {
+            int ch = getc(input);
+            if (ch == EOF) break;
+            if (size >= CAPACITY(u)) u = increase_capacity(u, size);
+            u[size++] = ch;
+        }
+        fclose(input);
+        SIZE(u) = size;
+        return u;
+    }
+}
+
+static void error(struct tokeniser * t, const char * s1, int n, symbol * p, const char * s2) {
+    if (t->error_count == 20) { fprintf(stderr, "... etc\n"); exit(1); }
+    fprintf(stderr, "%s:%d: ", t->file, t->line_number);
+    if (s1) fprintf(stderr, "%s", s1);
+    if (p) {
+        int i;
+        for (i = 0; i < n; i++) fprintf(stderr, "%c", p[i]);
+    }
+    if (s2) fprintf(stderr, "%s", s2);
+    fprintf(stderr, "\n");
+    t->error_count++;
+}
+
+static void error1(struct tokeniser * t, const char * s) {
+    error(t, s, 0,0, 0);
+}
+
+static void error2(struct tokeniser * t, const char * s) {
+    error(t, "unexpected end of text after ", 0,0, s);
+}
+
+static int compare_words(int m, symbol * p, int n, const byte * q) {
+    if (m != n) return m - n;
+    {
+        int i; for (i = 0; i < n; i++) {
+            int diff = p[i] - q[i];
+            if (diff) return diff;
+        }
+    }
+    return 0;
+}
+
+static int find_word(int n, symbol * p) {
+    int i = 0; int j = vocab->code;
+    do {
+        int k = i + (j - i)/2;
+        const struct system_word * w = vocab + k;
+        int diff = compare_words(n, p, w->s_size, w->s);
+        if (diff == 0) return w->code;
+        if (diff < 0) j = k; else i = k;
+    } while (j - i != 1);
+    return -1;
+}
+
+static int get_number(int n, symbol * p) {
+    int x = 0;
+    int i; for (i = 0; i < n; i++) x = 10*x + p[i] - '0';
+    return x;
+}
+
+static int eq_s(struct tokeniser * t, const char * s) {
+    int l = strlen(s);
+    if (SIZE(t->p) - t->c < l) return false;
+    {
+        int i;
+        for (i = 0; i < l; i++) if (t->p[t->c + i] != s[i]) return false;
+    }
+    t->c += l; return true;
+}
+
+static int white_space(struct tokeniser * t, int ch) {
+    switch (ch) {
+        case '\n':
+            t->line_number++;
+            /* fall through */
+        case '\r':
+        case '\t':
+        case ' ':
+            return true;
+    }
+    return false;
+}
+
+static symbol * find_in_m(struct tokeniser * t, int n, symbol * p) {
+    struct m_pair * q;
+    for (q = t->m_pairs; q; q = q->next) {
+        symbol * name = q->name;
+        if (n == SIZE(name) && memcmp(name, p, n * sizeof(symbol)) == 0) return q->value;
+    }
+    return 0;
+}
+
+static int read_literal_string(struct tokeniser * t, int c) {
+    symbol * p = t->p;
+    int ch;
+    SIZE(t->b) = 0;
+    while (true) {
+        if (c >= SIZE(p)) { error2(t, "'"); return c; }
+        ch = p[c];
+        if (ch == '\n') { error1(t, "string not terminated"); return c; }
+        c++;
+        if (ch == t->m_start) {
+            /* Inside insert characters. */
+            int c0 = c;
+            int newlines = false; /* no newlines as yet */
+            int black_found = false; /* no printing chars as yet */
+            while (true) {
+                if (c >= SIZE(p)) { error2(t, "'"); return c; }
+                ch = p[c]; c++;
+                if (ch == t->m_end) break;
+                if (!white_space(t, ch)) black_found = true;
+                if (ch == '\n') newlines = true;
+                if (newlines && black_found) {
+                    error1(t, "string not terminated");
+                    return c;
+                }
+            }
+            if (!newlines) {
+                int n = c - c0 - 1;    /* macro size */
+                int firstch = p[c0];
+                symbol * q = find_in_m(t, n, p + c0);
+                if (q == 0) {
+                    if (n == 1 && (firstch == '\'' || firstch == t->m_start))
+                        t->b = add_to_b(t->b, 1, p + c0);
+                    else if (n >= 3 && firstch == 'U' && p[c0 + 1] == '+') {
+                        int codepoint = 0;
+                        int x;
+                        if (t->uplusmode == UPLUS_DEFINED) {
+                            /* See if found with xxxx upper-cased. */
+                            symbol * uc = create_b(n);
+                            int i;
+                            for (i = 0; i != n; ++i) {
+                                uc[i] = toupper(p[c0 + i]);
+                            }
+                            q = find_in_m(t, n, uc);
+                            lose_b(uc);
+                            if (q != 0) {
+                                t->b = add_to_b(t->b, SIZE(q), q);
+                                continue;
+                            }
+                            error1(t, "Some U+xxxx stringdefs seen but not this one");
+                        } else {
+                            t->uplusmode = UPLUS_UNICODE;
+                        }
+                        for (x = c0 + 2; x != c - 1; ++x) {
+                            int hex = hex_to_num(p[x]);
+                            if (hex < 0) {
+                                error1(t, "Bad hex digit following U+");
+                                break;
+                            }
+                            codepoint = (codepoint << 4) | hex;
+                        }
+                        if (t->encoding == ENC_UTF8) {
+                            if (codepoint < 0 || codepoint > 0x01ffff) {
+                                error1(t, "character values exceed 0x01ffff");
+                            }
+                            /* Ensure there's enough space for a max length
+                             * UTF-8 sequence. */
+                            if (CAPACITY(t->b) < SIZE(t->b) + 3) {
+                                t->b = increase_capacity(t->b, 3);
+                            }
+                            SIZE(t->b) += put_utf8(codepoint, t->b + SIZE(t->b));
+                        } else {
+                            symbol sym;
+                            if (t->encoding == ENC_SINGLEBYTE) {
+                                /* Only ISO-8859-1 is handled this way - for
+                                 * other single-byte character sets you need
+                                 * stringdef all the U+xxxx codes you use
+                                 * like - e.g.:
+                                 *
+                                 * stringdef U+0171   hex 'FB'
+                                 */
+                                if (codepoint < 0 || codepoint > 0xff) {
+                                    error1(t, "character values exceed 256");
+                                }
+                            } else {
+                                if (codepoint < 0 || codepoint > 0xffff) {
+                                    error1(t, "character values exceed 64K");
+                                }
+                            }
+                            sym = codepoint;
+                            t->b = add_to_b(t->b, 1, &sym);
+                        }
+                    } else
+                        error(t, "string macro '", n, p + c0, "' undeclared");
+                } else
+                    t->b = add_to_b(t->b, SIZE(q), q);
+            }
+        } else {
+            if (ch == '\'') return c;
+            if (ch < 0 || ch >= 0x80) {
+                if (t->encoding != ENC_WIDECHARS) {
+                    /* We don't really want people using non-ASCII literal
+                     * strings, but historically it's worked for single-byte
+                     * and UTF-8 if the source encoding matches what the
+                     * generated stemmer works in and it seems unfair to just
+                     * suddenly make this a hard error.`
+                     */
+                    fprintf(stderr,
+                            "%s:%d: warning: Non-ASCII literal strings aren't "
+                            "portable - use stringdef instead\n",
+                            t->file, t->line_number);
+                } else {
+                    error1(t, "Non-ASCII literal strings aren't "
+                              "portable - use stringdef instead");
+                }
+            }
+            t->b = add_to_b(t->b, 1, p + c - 1);
+        }
+    }
+}
+
+static int next_token(struct tokeniser * t) {
+    symbol * p = t->p;
+    int c = t->c;
+    int ch;
+    int code = -1;
+    while (true) {
+        if (c >= SIZE(p)) { t->c = c; return -1; }
+        ch = p[c];
+        if (white_space(t, ch)) { c++; continue; }
+        if (isalpha(ch)) {
+            int c0 = c;
+            while (c < SIZE(p) && (isalnum(p[c]) || p[c] == '_')) c++;
+            code = find_word(c - c0, p + c0);
+            if (code < 0 || t->token_disabled[code]) {
+                t->b = move_to_b(t->b, c - c0, p + c0);
+                code = c_name;
+            }
+        } else
+        if (isdigit(ch)) {
+            int c0 = c;
+            while (c < SIZE(p) && isdigit(p[c])) c++;
+            t->number = get_number(c - c0, p + c0);
+            code = c_number;
+        } else
+        if (ch == '\'') {
+            c = read_literal_string(t, c + 1);
+            code = c_literalstring;
+        } else
+        {
+            int lim = smaller(2, SIZE(p) - c);
+            int i;
+            for (i = lim; i > 0; i--) {
+                code = find_word(i, p + c);
+                if (code >= 0) { c += i; break; }
+            }
+        }
+        if (code >= 0) {
+            t->c = c;
+            return code;
+        }
+        error(t, "'", 1, p + c, "' unknown");
+        c++;
+        continue;
+    }
+}
+
+static int next_char(struct tokeniser * t) {
+    if (t->c >= SIZE(t->p)) return -1;
+    return t->p[t->c++];
+}
+
+static int next_real_char(struct tokeniser * t) {
+    while (true) {
+        int ch = next_char(t);
+        if (!white_space(t, ch)) return ch;
+    }
+}
+
+static void read_chars(struct tokeniser * t) {
+    int ch = next_real_char(t);
+    if (ch < 0) { error2(t, "stringdef"); return; }
+    {
+        int c0 = t->c-1;
+        while (true) {
+            ch = next_char(t);
+            if (white_space(t, ch) || ch < 0) break;
+        }
+        t->b2 = move_to_b(t->b2, t->c - c0 - 1, t->p + c0);
+    }
+}
+
+static int decimal_to_num(int ch) {
+    if ('0' <= ch && ch <= '9') return ch - '0';
+    return -1;
+}
+
+static int hex_to_num(int ch) {
+    if ('0' <= ch && ch <= '9') return ch - '0';
+    if ('a' <= ch && ch <= 'f') return ch - 'a' + 10;
+    if ('A' <= ch && ch <= 'F') return ch - 'A' + 10;
+    return -1;
+}
+
+static void convert_numeric_string(struct tokeniser * t, symbol * p, int base) {
+    int c = 0; int d = 0;
+    while (true) {
+        while (c < SIZE(p) && p[c] == ' ') c++;
+        if (c == SIZE(p)) break;
+        {
+            int number = 0;
+            while (c != SIZE(p)) {
+                int ch = p[c];
+                if (ch == ' ') break;
+                if (base == 10) {
+                    ch = decimal_to_num(ch);
+                    if (ch < 0) {
+                        error1(t, "decimal string contains non-digits");
+                        return;
+                    }
+                } else {
+                    ch = hex_to_num(ch);
+                    if (ch < 0) {
+                        error1(t, "hex string contains non-hex characters");
+                        return;
+                    }
+                }
+                number = base * number + ch;
+                c++;
+            }
+            if (t->encoding == ENC_SINGLEBYTE) {
+                if (number < 0 || number > 0xff) {
+                    error1(t, "character values exceed 256");
+                    return;
+                }
+            } else {
+                if (number < 0 || number > 0xffff) {
+                    error1(t, "character values exceed 64K");
+                    return;
+                }
+            }
+            if (t->encoding == ENC_UTF8)
+                d += put_utf8(number, p + d);
+            else
+                p[d++] = number;
+        }
+    }
+    SIZE(p) = d;
+}
+
+extern int read_token(struct tokeniser * t) {
+    symbol * p = t->p;
+    int held = t->token_held;
+    t->token_held = false;
+    if (held) return t->token;
+    while (true) {
+        int code = next_token(t);
+        switch (code) {
+            case c_comment1: /*  slash-slash comment */
+                while (t->c < SIZE(p) && p[t->c] != '\n') t->c++;
+                continue;
+            case c_comment2: /* slash-star comment */
+                while (true) {
+                    if (t->c >= SIZE(p)) {
+                        error1(t, "/* comment not terminated");
+                        t->token = -1;
+                        return -1;
+                    }
+                    if (p[t->c] == '\n') t->line_number++;
+                    if (eq_s(t, "*/")) break;
+                    t->c++;
+                }
+                continue;
+            case c_stringescapes: {
+                int ch1 = next_real_char(t);
+                int ch2 = next_real_char(t);
+                if (ch2 < 0) {
+                    error2(t, "stringescapes");
+                    continue;
+                }
+                if (ch1 == '\'') {
+                    error1(t, "first stringescape cannot be '");
+                    continue;
+                }
+                t->m_start = ch1;
+                t->m_end = ch2;
+                continue;
+            }
+            case c_stringdef: {
+                int base = 0;
+                read_chars(t);
+                code = read_token(t);
+                if (code == c_hex) { base = 16; code = read_token(t); } else
+                if (code == c_decimal) { base = 10; code = read_token(t); }
+                if (code != c_literalstring) {
+                    error1(t, "string omitted after stringdef");
+                    continue;
+                }
+                if (base > 0) convert_numeric_string(t, t->b, base);
+                {   NEW(m_pair, q);
+                    q->next = t->m_pairs;
+                    q->name = copy_b(t->b2);
+                    q->value = copy_b(t->b);
+                    t->m_pairs = q;
+                    if (t->uplusmode != UPLUS_DEFINED &&
+                        (SIZE(t->b2) >= 3 && t->b2[0] == 'U' && t->b2[1] == '+')) {
+                        if (t->uplusmode == UPLUS_UNICODE) {
+                            error1(t, "U+xxxx already used with implicit meaning");
+                        } else {
+                            t->uplusmode = UPLUS_DEFINED;
+                        }
+                    }
+                }
+                continue;
+            }
+            case c_get:
+                code = read_token(t);
+                if (code != c_literalstring) {
+                    error1(t, "string omitted after get"); continue;
+                }
+                t->get_depth++;
+                if (t->get_depth > 10) {
+                    fprintf(stderr, "get directives go 10 deep. Looping?\n");
+                    exit(1);
+                }
+                {
+                    NEW(input, q);
+                    char * file = b_to_s(t->b);
+                    symbol * u = get_input(file);
+                    if (u == 0) {
+                        struct include * r;
+                        for (r = t->includes; r; r = r->next) {
+                            symbol * b = copy_b(r->b);
+                            b = add_to_b(b, SIZE(t->b), t->b);
+                            free(file);
+                            file = b_to_s(b);
+                            u = get_input(file);
+                            lose_b(b);
+                            if (u != 0) break;
+                        }
+                    }
+                    if (u == 0) {
+                        error(t, "Can't get '", SIZE(t->b), t->b, "'");
+                        exit(1);
+                    }
+                    memmove(q, t, sizeof(struct input));
+                    t->next = q;
+                    t->p = u;
+                    t->c = 0;
+                    t->file = file;
+                    t->file_needs_freeing = true;
+                    t->line_number = 1;
+                }
+                p = t->p;
+                continue;
+            case -1:
+                if (t->next) {
+                    lose_b(p);
+                    {
+                        struct input * q = t->next;
+                        memmove(t, q, sizeof(struct input)); p = t->p;
+                        FREE(q);
+                    }
+                    t->get_depth--;
+                    continue;
+                }
+                /* fall through */
+            default:
+                t->previous_token = t->token;
+                t->token = code;
+                return code;
+        }
+    }
+}
+
+extern const char * name_of_token(int code) {
+    int i;
+    for (i = 1; i < vocab->code; i++)
+        if ((vocab + i)->code == code) return (const char *)(vocab + i)->s;
+    switch (code) {
+        case c_mathassign:   return "=";
+        case c_name:         return "name";
+        case c_number:       return "number";
+        case c_literalstring:return "literal";
+        case c_neg:          return "neg";
+        case c_grouping:     return "grouping";
+        case c_call:         return "call";
+        case c_booltest:     return "Boolean test";
+        case -2:             return "start of text";
+        case -1:             return "end of text";
+        default:             return "?";
+    }
+}
+
+extern void disable_token(struct tokeniser * t, int code) {
+    t->token_disabled[code] = 1;
+}
+
+extern struct tokeniser * create_tokeniser(symbol * p, char * file) {
+    NEW(tokeniser, t);
+    t->next = 0;
+    t->p = p;
+    t->c = 0;
+    t->file = file;
+    t->file_needs_freeing = false;
+    t->line_number = 1;
+    t->b = create_b(0);
+    t->b2 = create_b(0);
+    t->m_start = -1;
+    t->m_pairs = 0;
+    t->get_depth = 0;
+    t->error_count = 0;
+    t->token_held = false;
+    t->token = -2;
+    t->previous_token = -2;
+    t->uplusmode = UPLUS_NONE;
+    memset(t->token_disabled, 0, sizeof(t->token_disabled));
+    return t;
+}
+
+extern void close_tokeniser(struct tokeniser * t) {
+    lose_b(t->b);
+    lose_b(t->b2);
+    {
+        struct m_pair * q = t->m_pairs;
+        while (q) {
+            struct m_pair * q_next = q->next;
+            lose_b(q->name);
+            lose_b(q->value);
+            FREE(q);
+            q = q_next;
+        }
+    }
+    {
+        struct input * q = t->next;
+        while (q) {
+            struct input * q_next = q->next;
+            FREE(q);
+            q = q_next;
+        }
+    }
+    if (t->file_needs_freeing) free(t->file);
+    FREE(t);
+}