diff options
Diffstat (limited to '')
| -rw-r--r-- | contrib/snowball/compiler/analyser.c | 1380 | ||||
| -rw-r--r-- | contrib/snowball/compiler/driver.c | 574 | ||||
| -rw-r--r-- | contrib/snowball/compiler/generator.c | 1725 | ||||
| -rw-r--r-- | contrib/snowball/compiler/header.h | 411 | ||||
| -rw-r--r-- | contrib/snowball/compiler/space.c | 287 | ||||
| -rw-r--r-- | contrib/snowball/compiler/syswords.h | 86 | ||||
| -rw-r--r-- | contrib/snowball/compiler/syswords2.h | 13 | ||||
| -rw-r--r-- | contrib/snowball/compiler/tokeniser.c | 567 |
8 files changed, 5043 insertions, 0 deletions
diff --git a/contrib/snowball/compiler/analyser.c b/contrib/snowball/compiler/analyser.c new file mode 100644 index 0000000..dffa555 --- /dev/null +++ b/contrib/snowball/compiler/analyser.c @@ -0,0 +1,1380 @@ + +#include <stdio.h> /* printf etc */ +#include <stdlib.h> /* exit */ +#include <string.h> /* memmove */ +#include "header.h" + +typedef enum { + e_token_omitted = 0, + e_unexpected_token = 1, + e_string_omitted = 2, + e_unexpected_token_in_among = 3, + /* For codes above here, report "after " t->previous_token after the error. */ + e_unresolved_substring = 14, + e_not_allowed_inside_reverse = 15, + e_empty_grouping = 16, + e_already_backwards = 17, + e_empty_among = 18, + e_adjacent_bracketed_in_among = 19, + e_substring_preceded_by_substring = 20, + /* For codes below here, tokeniser->b is printed before the error. */ + e_redeclared = 30, + e_undeclared = 31, + e_declared_as_different_mode = 32, + e_not_of_type_x = 33, + e_not_of_type_string_or_integer = 34, + e_misplaced = 35, + e_redefined = 36, + e_misused = 37 +} error_code; + +/* recursive usage: */ + +static void read_program_(struct analyser * a, int terminator); +static struct node * read_C(struct analyser * a); +static struct node * C_style(struct analyser * a, const char * s, int token); + + +static void print_node_(struct node * p, int n, const char * s) { + + int i; + for (i = 0; i < n; i++) fputs(i == n - 1 ? s : " ", stdout); + printf("%s ", name_of_token(p->type)); + if (p->name) report_b(stdout, p->name->b); + if (p->literalstring) { + printf("'"); + report_b(stdout, p->literalstring); + printf("'"); + } + printf("\n"); + if (p->AE) print_node_(p->AE, n+1, "# "); + if (p->left) print_node_(p->left, n+1, " "); + if (p->aux) print_node_(p->aux, n+1, "@ "); + if (p->right) print_node_(p->right, n, " "); +} + +extern void print_program(struct analyser * a) { + print_node_(a->program, 0, " "); +} + +static struct node * new_node(struct analyser * a, int type) { + NEW(node, p); + p->next = a->nodes; a->nodes = p; + p->left = 0; + p->right = 0; + p->aux = 0; + p->AE = 0; + p->name = 0; + p->literalstring = 0; + p->mode = a->mode; + p->line_number = a->tokeniser->line_number; + p->type = type; + return p; +} + +static const char * name_of_mode(int n) { + switch (n) { + case m_backward: return "string backward"; + case m_forward: return "string forward"; + /* case m_integer: return "integer"; */ + } + fprintf(stderr, "Invalid mode %d in name_of_mode()\n", n); + exit(1); +} + +static const char * name_of_type(int n) { + switch (n) { + case 's': return "string"; + case 'i': return "integer"; + case 'r': return "routine"; + case 'R': return "routine or grouping"; + case 'g': return "grouping"; + } + fprintf(stderr, "Invalid type %d in name_of_type()\n", n); + exit(1); +} + +static const char * name_of_name_type(int code) { + switch (code) { + case t_string: return "string"; + case t_boolean: return "boolean"; + case t_integer: return "integer"; + case t_routine: return "routine"; + case t_external: return "external"; + case t_grouping: return "grouping"; + } + fprintf(stderr, "Invalid type code %d in name_of_name_type()\n", code); + exit(1); +} + +static void count_error(struct analyser * a) { + struct tokeniser * t = a->tokeniser; + if (t->error_count >= 20) { fprintf(stderr, "... etc\n"); exit(1); } + t->error_count++; +} + +static void error2(struct analyser * a, error_code n, int x) { + struct tokeniser * t = a->tokeniser; + count_error(a); + fprintf(stderr, "%s:%d: ", t->file, t->line_number); + if ((int)n >= (int)e_redeclared) report_b(stderr, t->b); + switch (n) { + case e_token_omitted: + fprintf(stderr, "%s omitted", name_of_token(t->omission)); break; + case e_unexpected_token_in_among: + fprintf(stderr, "in among(...), "); + /* fall through */ + case e_unexpected_token: + fprintf(stderr, "unexpected %s", name_of_token(t->token)); + if (t->token == c_number) fprintf(stderr, " %d", t->number); + if (t->token == c_name) { + fprintf(stderr, " "); + report_b(stderr, t->b); + } break; + case e_string_omitted: + fprintf(stderr, "string omitted"); break; + + case e_unresolved_substring: + fprintf(stderr, "unresolved substring on line %d", x); break; + case e_not_allowed_inside_reverse: + fprintf(stderr, "%s not allowed inside reverse(...)", name_of_token(t->token)); break; + case e_empty_grouping: + fprintf(stderr, "empty grouping"); break; + case e_already_backwards: + fprintf(stderr, "backwards used when already in this mode"); break; + case e_empty_among: + fprintf(stderr, "empty among(...)"); break; + case e_adjacent_bracketed_in_among: + fprintf(stderr, "two adjacent bracketed expressions in among(...)"); break; + case e_substring_preceded_by_substring: + fprintf(stderr, "substring preceded by another substring on line %d", x); break; + + case e_redeclared: + fprintf(stderr, " re-declared"); break; + case e_undeclared: + fprintf(stderr, " undeclared"); break; + case e_declared_as_different_mode: + fprintf(stderr, " declared as %s mode; used as %s mode", + name_of_mode(a->mode), name_of_mode(x)); break; + case e_not_of_type_x: + fprintf(stderr, " not of type %s", name_of_type(x)); break; + case e_not_of_type_string_or_integer: + fprintf(stderr, " not of type string or integer"); break; + case e_misplaced: + fprintf(stderr, " misplaced"); break; + case e_redefined: + fprintf(stderr, " redefined"); break; + case e_misused: + fprintf(stderr, " mis-used as %s mode", + name_of_mode(x)); break; + } + if ((int)n < (int)e_unresolved_substring && t->previous_token > 0) + fprintf(stderr, " after %s", name_of_token(t->previous_token)); + fprintf(stderr, "\n"); +} + +static void error(struct analyser * a, error_code n) { error2(a, n, 0); } + +static void error4(struct analyser * a, struct name * q) { + count_error(a); + fprintf(stderr, "%s:%d: ", a->tokeniser->file, q->used->line_number); + report_b(stderr, q->b); + fprintf(stderr, " undefined\n"); +} + +static void omission_error(struct analyser * a, int n) { + a->tokeniser->omission = n; + error(a, e_token_omitted); +} + +static int check_token(struct analyser * a, int code) { + struct tokeniser * t = a->tokeniser; + if (t->token != code) { omission_error(a, code); return false; } + return true; +} + +static int get_token(struct analyser * a, int code) { + struct tokeniser * t = a->tokeniser; + read_token(t); + { + int x = check_token(a, code); + if (!x) t->token_held = true; + return x; + } +} + +static struct name * look_for_name(struct analyser * a) { + symbol * q = a->tokeniser->b; + struct name * p; + for (p = a->names; p; p = p->next) { + symbol * b = p->b; + int n = SIZE(b); + if (n == SIZE(q) && memcmp(q, b, n * sizeof(symbol)) == 0) { + p->referenced = true; + return p; + } + } + return 0; +} + +static struct name * find_name(struct analyser * a) { + struct name * p = look_for_name(a); + if (p == 0) error(a, e_undeclared); + return p; +} + +static void check_routine_mode(struct analyser * a, struct name * p, int mode) { + if (p->mode < 0) p->mode = mode; else + if (p->mode != mode) error2(a, e_misused, mode); +} + +static void check_name_type(struct analyser * a, struct name * p, int type) { + switch (type) { + case 's': + if (p->type == t_string) return; + break; + case 'i': + if (p->type == t_integer) return; + break; + case 'b': + if (p->type == t_boolean) return; + break; + case 'R': + if (p->type == t_grouping) return; + /* FALLTHRU */ + case 'r': + if (p->type == t_routine || p->type == t_external) return; + break; + case 'g': + if (p->type == t_grouping) return; + break; + } + error2(a, e_not_of_type_x, type); +} + +static void read_names(struct analyser * a, int type) { + struct tokeniser * t = a->tokeniser; + if (!get_token(a, c_bra)) return; + while (true) { + int token = read_token(t); + switch (token) { + case c_len: { + /* Context-sensitive token - once declared as a name, it loses + * its special meaning, for compatibility with older versions + * of snowball. + */ + static const symbol c_len_lit[] = { + 'l', 'e', 'n' + }; + MOVE_TO_B(t->b, c_len_lit); + goto handle_as_name; + } + case c_lenof: { + /* Context-sensitive token - once declared as a name, it loses + * its special meaning, for compatibility with older versions + * of snowball. + */ + static const symbol c_lenof_lit[] = { + 'l', 'e', 'n', 'o', 'f' + }; + MOVE_TO_B(t->b, c_lenof_lit); + goto handle_as_name; + } + case c_name: +handle_as_name: + if (look_for_name(a) != 0) error(a, e_redeclared); else { + NEW(name, p); + p->b = copy_b(t->b); + p->type = type; + p->mode = -1; /* routines, externals */ + /* We defer assigning counts until after we've eliminated + * variables whose values are never used. */ + p->count = -1; + p->referenced = false; + p->used_in_among = false; + p->used = 0; + p->value_used = false; + p->initialised = false; + p->used_in_definition = false; + p->local_to = 0; + p->grouping = 0; + p->definition = 0; + p->declaration_line_number = t->line_number; + p->next = a->names; + a->names = p; + if (token != c_name) { + disable_token(t, token); + } + } + break; + default: + if (!check_token(a, c_ket)) t->token_held = true; + return; + } + } +} + +static symbol * new_literalstring(struct analyser * a) { + NEW(literalstring, p); + p->b = copy_b(a->tokeniser->b); + p->next = a->literalstrings; + a->literalstrings = p; + return p->b; +} + +static int read_AE_test(struct analyser * a) { + + struct tokeniser * t = a->tokeniser; + switch (read_token(t)) { + case c_assign: return c_mathassign; + case c_plusassign: + case c_minusassign: + case c_multiplyassign: + case c_divideassign: + case c_eq: + case c_ne: + case c_gr: + case c_ge: + case c_ls: + case c_le: return t->token; + default: error(a, e_unexpected_token); t->token_held = true; return c_eq; + } +} + +static int binding(int t) { + switch (t) { + case c_plus: case c_minus: return 1; + case c_multiply: case c_divide: return 2; + default: return -2; + } +} + +static void mark_used_in(struct analyser * a, struct name * q, struct node * p) { + if (!q->used) { + q->used = p; + q->local_to = a->program_end->name; + } else if (q->local_to) { + if (q->local_to != a->program_end->name) { + /* Used in more than one routine/external. */ + q->local_to = NULL; + } + } +} + +static void name_to_node(struct analyser * a, struct node * p, int type) { + struct name * q = find_name(a); + if (q) { + check_name_type(a, q, type); + mark_used_in(a, q, p); + } + p->name = q; +} + +static struct node * read_AE(struct analyser * a, int B) { + struct tokeniser * t = a->tokeniser; + struct node * p; + struct node * q; + switch (read_token(t)) { + case c_minus: /* monadic */ + q = read_AE(a, 100); + if (q->type == c_neg) { + /* Optimise away double negation, which avoids generators + * having to worry about generating "--" (decrement operator + * in many languages). + */ + p = q->right; + /* Don't free q, it's in the linked list a->nodes. */ + break; + } + p = new_node(a, c_neg); + p->right = q; + break; + case c_bra: + p = read_AE(a, 0); + get_token(a, c_ket); + break; + case c_name: + p = new_node(a, c_name); + name_to_node(a, p, 'i'); + if (p->name) p->name->value_used = true; + break; + case c_maxint: + case c_minint: + a->int_limits_used = true; + /* fall through */ + case c_cursor: + case c_limit: + case c_len: + case c_size: + p = new_node(a, t->token); + break; + case c_number: + p = new_node(a, c_number); + p->number = t->number; + break; + case c_lenof: + case c_sizeof: + p = C_style(a, "s", t->token); + break; + default: + error(a, e_unexpected_token); + t->token_held = true; + return 0; + } + while (true) { + int token = read_token(t); + int b = binding(token); + if (binding(token) <= B) { + t->token_held = true; + return p; + } + q = new_node(a, token); + q->left = p; + q->right = read_AE(a, b); + p = q; + } +} + +static struct node * read_C_connection(struct analyser * a, struct node * q, int op) { + struct tokeniser * t = a->tokeniser; + struct node * p = new_node(a, op); + struct node * p_end = q; + p->left = q; + do { + q = read_C(a); + p_end->right = q; p_end = q; + } while (read_token(t) == op); + t->token_held = true; + return p; +} + +static struct node * read_C_list(struct analyser * a) { + struct tokeniser * t = a->tokeniser; + struct node * p = new_node(a, c_bra); + struct node * p_end = 0; + while (true) { + int token = read_token(t); + if (token == c_ket) return p; + if (token < 0) { omission_error(a, c_ket); return p; } + t->token_held = true; + { + struct node * q = read_C(a); + while (true) { + token = read_token(t); + if (token != c_and && token != c_or) { + t->token_held = true; + break; + } + q = read_C_connection(a, q, token); + } + if (p_end == 0) p->left = q; else p_end->right = q; + p_end = q; + } + } +} + +static struct node * C_style(struct analyser * a, const char * s, int token) { + int i; + struct node * p = new_node(a, token); + for (i = 0; s[i] != 0; i++) switch (s[i]) { + case 'C': + p->left = read_C(a); continue; + case 'D': + p->aux = read_C(a); continue; + case 'A': + p->AE = read_AE(a, 0); continue; + case 'f': + get_token(a, c_for); continue; + case 'S': + { + int str_token = read_token(a->tokeniser); + if (str_token == c_name) name_to_node(a, p, 's'); else + if (str_token == c_literalstring) p->literalstring = new_literalstring(a); + else error(a, e_string_omitted); + } + continue; + case 'b': + case 's': + case 'i': + if (get_token(a, c_name)) name_to_node(a, p, s[i]); + continue; + } + return p; +} + +static struct node * read_literalstring(struct analyser * a) { + struct node * p = new_node(a, c_literalstring); + p->literalstring = new_literalstring(a); + return p; +} + +static void reverse_b(symbol * b) { + int i = 0; int j = SIZE(b) - 1; + while (i < j) { + int ch1 = b[i]; int ch2 = b[j]; + b[i++] = ch2; b[j--] = ch1; + } +} + +static int compare_amongvec(const void *pv, const void *qv) { + const struct amongvec * p = (const struct amongvec*)pv; + const struct amongvec * q = (const struct amongvec*)qv; + symbol * b_p = p->b; int p_size = p->size; + symbol * b_q = q->b; int q_size = q->size; + int smaller_size = p_size < q_size ? p_size : q_size; + int i; + for (i = 0; i < smaller_size; i++) + if (b_p[i] != b_q[i]) return b_p[i] - b_q[i]; + if (p_size - q_size) + return p_size - q_size; + return p->line_number - q->line_number; +} + +#define PTR_NULL_CHECK(P, Q) do {\ + if ((Q) == NULL) {\ + if ((P) != NULL) return 1;\ + } else {\ + if ((P) == NULL) return -1;\ + }\ + } while (0) + +static int compare_node(const struct node *p, const struct node *q) { + PTR_NULL_CHECK(p, q); + if (q == NULL) { + /* p must be NULL too. */ + return 0; + } + + if (p->type != q->type) return p->type > q->type ? 1 : -1; + if (p->mode != q->mode) return p->mode > q->mode ? 1 : -1; + if (p->type == c_number) { + if (p->number != q->number) + return p->number > q->number ? 1 : -1; + } + + PTR_NULL_CHECK(p->left, q->left); + if (p->left) { + int r = compare_node(p->left, q->left); + if (r != 0) return r; + } + + PTR_NULL_CHECK(p->AE, q->AE); + if (p->AE) { + int r = compare_node(p->AE, q->AE); + if (r != 0) return r; + } + + PTR_NULL_CHECK(p->aux, q->aux); + if (p->aux) { + int r = compare_node(p->aux, q->aux); + if (r != 0) return r; + } + + PTR_NULL_CHECK(p->name, q->name); + if (p->name) { + int r; + if (SIZE(p->name->b) != SIZE(q->name->b)) { + return SIZE(p->name->b) - SIZE(q->name->b); + } + r = memcmp(p->name->b, q->name->b, + SIZE(p->name->b) * sizeof(symbol)); + if (r != 0) return r; + } + + PTR_NULL_CHECK(p->literalstring, q->literalstring); + if (p->literalstring) { + int r; + if (SIZE(p->literalstring) != SIZE(q->literalstring)) { + return SIZE(p->literalstring) - SIZE(q->literalstring); + } + r = memcmp(p->literalstring, q->literalstring, + SIZE(p->literalstring) * sizeof(symbol)); + if (r != 0) return r; + } + + return compare_node(p->right, q->right); +} + +static void make_among(struct analyser * a, struct node * p, struct node * substring) { + + NEW(among, x); + NEWVEC(amongvec, v, p->number); + struct node * q = p->left; + struct amongvec * w0 = v; + struct amongvec * w1 = v; + int result = 1; + + int direction = substring != 0 ? substring->mode : p->mode; + int backward = direction == m_backward; + + if (a->amongs == 0) a->amongs = x; else a->amongs_end->next = x; + a->amongs_end = x; + x->next = 0; + x->b = v; + x->number = a->among_count++; + x->function_count = 0; + x->starter = 0; + x->nocommand_count = 0; + x->amongvar_needed = false; + + if (q->type == c_bra) { x->starter = q; q = q->right; } + + while (q) { + if (q->type == c_literalstring) { + symbol * b = q->literalstring; + w1->b = b; /* pointer to case string */ + w1->action = NULL; /* action gets filled in below */ + w1->line_number = q->line_number; + w1->size = SIZE(b); /* number of characters in string */ + w1->i = -1; /* index of longest substring */ + w1->result = -1; /* number of corresponding case expression */ + if (q->left) { + struct name * function = q->left->name; + w1->function = function; + function->used_in_among = true; + check_routine_mode(a, function, direction); + x->function_count++; + } else { + w1->function = 0; + } + w1++; + } else if (q->left == 0) { + /* empty command: () */ + w0 = w1; + } else { + /* Check for previous action which is the same as this one and use + * the same action code if we find one. + */ + int among_result = -1; + struct amongvec * w; + for (w = v; w < w0; ++w) { + if (w->action && compare_node(w->action->left, q->left) == 0) { + if (w->result <= 0) { + printf("Among code %d isn't positive\n", w->result); + exit(1); + } + among_result = w->result; + break; + } + } + if (among_result < 0) { + among_result = result++; + } + + while (w0 != w1) { + w0->action = q; + w0->result = among_result; + w0++; + } + } + q = q->right; + } + if (w1-v != p->number) { fprintf(stderr, "oh! %d %d\n", (int)(w1-v), p->number); exit(1); } + x->command_count = result - 1; + { + NEWVEC(node*, commands, x->command_count); + memset(commands, 0, x->command_count * sizeof(struct node*)); + for (w0 = v; w0 < w1; w0++) { + if (w0->result > 0) { + /* result == -1 when there's no command. */ + if (w0->result > x->command_count) { + fprintf(stderr, "More among codes than expected\n"); + exit(1); + } + if (!commands[w0->result - 1]) + commands[w0->result - 1] = w0->action; + } else { + ++x->nocommand_count; + } + if (backward) reverse_b(w0->b); + } + x->commands = commands; + } + qsort(v, w1 - v, sizeof(struct amongvec), compare_amongvec); + + /* the following loop is O(n squared) */ + for (w0 = w1 - 1; w0 >= v; w0--) { + symbol * b = w0->b; + int size = w0->size; + struct amongvec * w; + + for (w = w0 - 1; w >= v; w--) { + if (w->size < size && memcmp(w->b, b, w->size * sizeof(symbol)) == 0) { + w0->i = w - v; /* fill in index of longest substring */ + break; + } + } + } + if (backward) for (w0 = v; w0 < w1; w0++) reverse_b(w0->b); + + for (w0 = v; w0 < w1 - 1; w0++) + if (w0->size == (w0 + 1)->size && + memcmp(w0->b, (w0 + 1)->b, w0->size * sizeof(symbol)) == 0) { + count_error(a); + fprintf(stderr, "%s:%d: among(...) has repeated string '", + a->tokeniser->file, (w0 + 1)->line_number); + report_b(stderr, (w0 + 1)->b); + fprintf(stderr, "'\n"); + count_error(a); + fprintf(stderr, "%s:%d: previously seen here\n", + a->tokeniser->file, w0->line_number); + } + + x->literalstring_count = p->number; + p->among = x; + + x->substring = substring; + if (substring != 0) substring->among = x; + if (x->command_count > 1 || + (x->command_count == 1 && x->nocommand_count > 0) || + x->starter != 0) { + /* We need to set among_var rather than just checking if find_among*() + * returns zero or not. + */ + x->amongvar_needed = a->amongvar_needed = true; + } +} + +static struct node * read_among(struct analyser * a) { + struct tokeniser * t = a->tokeniser; + struct node * p = new_node(a, c_among); + struct node * p_end = 0; + int previous_token = -1; + struct node * substring = a->substring; + + a->substring = 0; + p->number = 0; /* counts the number of literals */ + if (!get_token(a, c_bra)) return p; + while (true) { + struct node * q; + int token = read_token(t); + switch (token) { + case c_literalstring: + q = read_literalstring(a); + if (read_token(t) == c_name) { + struct node * r = new_node(a, c_name); + name_to_node(a, r, 'r'); + q->left = r; + } + else t->token_held = true; + p->number++; break; + case c_bra: + if (previous_token == c_bra) error(a, e_adjacent_bracketed_in_among); + q = read_C_list(a); break; + default: + error(a, e_unexpected_token_in_among); + previous_token = token; + continue; + case c_ket: + if (p->number == 0) error(a, e_empty_among); + if (t->error_count == 0) make_among(a, p, substring); + return p; + } + previous_token = token; + if (p_end == 0) p->left = q; else p_end->right = q; + p_end = q; + } +} + +static struct node * read_substring(struct analyser * a) { + + struct node * p = new_node(a, c_substring); + if (a->substring != 0) error2(a, e_substring_preceded_by_substring, a->substring->line_number); + a->substring = p; + return p; +} + +static void check_modifyable(struct analyser * a) { + if (!a->modifyable) error(a, e_not_allowed_inside_reverse); +} + +static struct node * read_C(struct analyser * a) { + struct tokeniser * t = a->tokeniser; + int token = read_token(t); + switch (token) { + case c_bra: + return read_C_list(a); + case c_backwards: + { + int mode = a->mode; + if (a->mode == m_backward) error(a, e_already_backwards); else a->mode = m_backward; + { struct node * p = C_style(a, "C", token); + a->mode = mode; + return p; + } + } + case c_reverse: + { + int mode = a->mode; + int modifyable = a->modifyable; + a->modifyable = false; + a->mode = mode == m_forward ? m_backward : m_forward; + { + struct node * p = C_style(a, "C", token); + a->mode = mode; + a->modifyable = modifyable; + return p; + } + } + case c_not: + case c_try: + case c_fail: + case c_test: + case c_do: + case c_goto: + case c_gopast: + case c_repeat: + return C_style(a, "C", token); + case c_loop: + case c_atleast: + return C_style(a, "AC", token); + case c_setmark: { + struct node * n = C_style(a, "i", token); + if (n->name) n->name->initialised = true; + return n; + } + case c_tomark: + case c_atmark: + case c_hop: + return C_style(a, "A", token); + case c_delete: + check_modifyable(a); + /* fall through */ + case c_next: + case c_tolimit: + case c_atlimit: + case c_leftslice: + case c_rightslice: + case c_true: + case c_false: + case c_debug: + return new_node(a, token); + case c_assignto: + case c_sliceto: { + struct node *n; + check_modifyable(a); + n = C_style(a, "s", token); + if (n->name) n->name->initialised = true; + return n; + } + case c_assign: + case c_insert: + case c_attach: + case c_slicefrom: { + struct node *n; + check_modifyable(a); + n = C_style(a, "S", token); + if (n->name) n->name->value_used = true; + return n; + } + case c_setlimit: + return C_style(a, "CfD", token); + case c_set: + case c_unset: { + struct node * n = C_style(a, "b", token); + if (n->name) n->name->initialised = true; + return n; + } + case c_dollar: { + struct tokeniser * t = a->tokeniser; + read_token(t); + if (t->token == c_bra) { + /* Handle newer $(AE REL_OP AE) syntax. */ + struct node * n = read_AE(a, 0); + read_token(t); + switch (t->token) { + case c_eq: + case c_ne: + case c_gr: + case c_ge: + case c_ls: + case c_le: { + struct node * lhs = n; + n = new_node(a, t->token); + n->left = lhs; + n->AE = read_AE(a, 0); + get_token(a, c_ket); + break; + } + default: + error(a, e_unexpected_token); + t->token_held = true; + break; + } + return n; + } + + if (t->token == c_name) { + struct node * p; + struct name * q = find_name(a); + int mode = a->mode; + int modifyable = a->modifyable; + if (q && q->type == t_string) { + /* Assume for now that $ on string both initialises and + * uses the string variable. FIXME: Can we do better? + */ + q->initialised = true; + q->value_used = true; + a->mode = m_forward; + a->modifyable = true; + p = new_node(a, c_dollar); + p->left = read_C(a); + p->name = q; + } else { + if (q && q->type != t_integer) { + /* If $ is used on an unknown name or a name which + * isn't a string or an integer then we assume the + * unknown name is an integer as $ is used more often + * on integers than strings, so hopefully this it less + * likely to cause an error avalanche. + * + * For an unknown name, we'll already have reported an + * error. + */ + error(a, e_not_of_type_string_or_integer); + q = NULL; + } + p = new_node(a, read_AE_test(a)); + p->AE = read_AE(a, 0); + + if (q) { + switch (p->type) { + case c_mathassign: + q->initialised = true; + p->name = q; + break; + default: + /* +=, etc don't "initialise" as they only + * amend an existing value. Similarly, they + * don't count as using the value. + */ + p->name = q; + break; + case c_eq: + case c_ne: + case c_gr: + case c_ge: + case c_ls: + case c_le: + p->left = new_node(a, c_name); + p->left->name = q; + q->value_used = true; + break; + } + } + } + if (q) mark_used_in(a, q, p); + a->mode = mode; + a->modifyable = modifyable; + return p; + } + + error(a, e_unexpected_token); + t->token_held = true; + return new_node(a, c_dollar); + } + case c_name: + { + struct name * q = find_name(a); + struct node * p = new_node(a, c_name); + if (q) { + mark_used_in(a, q, p); + switch (q->type) { + case t_boolean: + p->type = c_booltest; + q->value_used = true; + break; + case t_integer: + error(a, e_misplaced); /* integer name misplaced */ + break; + case t_string: + q->value_used = true; + break; + case t_routine: + case t_external: + p->type = c_call; + check_routine_mode(a, q, a->mode); + break; + case t_grouping: + p->type = c_grouping; break; + } + } + p->name = q; + return p; + } + case c_non: + { + struct node * p = new_node(a, token); + read_token(t); + if (t->token == c_minus) read_token(t); + if (!check_token(a, c_name)) { omission_error(a, c_name); return p; } + name_to_node(a, p, 'g'); + return p; + } + case c_literalstring: + return read_literalstring(a); + case c_among: return read_among(a); + case c_substring: return read_substring(a); + default: error(a, e_unexpected_token); return 0; + } +} + +static int next_symbol(symbol * p, symbol * W, int utf8) { + if (utf8) { + int ch; + int j = get_utf8(p, & ch); + W[0] = ch; return j; + } else { + W[0] = p[0]; return 1; + } +} + +static symbol * alter_grouping(symbol * p, symbol * q, int style, int utf8) { + int j = 0; + symbol W[1]; + int width; + if (style == c_plus) { + while (j < SIZE(q)) { + width = next_symbol(q + j, W, utf8); + p = add_to_b(p, 1, W); + j += width; + } + } else { + while (j < SIZE(q)) { + int i; + width = next_symbol(q + j, W, utf8); + for (i = 0; i < SIZE(p); i++) { + if (p[i] == W[0]) { + memmove(p + i, p + i + 1, (SIZE(p) - i - 1) * sizeof(symbol)); + SIZE(p)--; + } + } + j += width; + } + } + return p; +} + +static void read_define_grouping(struct analyser * a, struct name * q) { + struct tokeniser * t = a->tokeniser; + int style = c_plus; + { + NEW(grouping, p); + if (a->groupings == 0) a->groupings = p; else a->groupings_end->next = p; + a->groupings_end = p; + if (q) q->grouping = p; + p->next = 0; + p->name = q; + p->line_number = a->tokeniser->line_number; + p->b = create_b(0); + while (true) { + switch (read_token(t)) { + case c_name: + { + struct name * r = find_name(a); + if (r) { + check_name_type(a, r, 'g'); + p->b = alter_grouping(p->b, r->grouping->b, style, false); + r->used_in_definition = true; + } + } + break; + case c_literalstring: + p->b = alter_grouping(p->b, t->b, style, (a->encoding == ENC_UTF8)); + break; + default: error(a, e_unexpected_token); return; + } + switch (read_token(t)) { + case c_plus: + case c_minus: style = t->token; break; + default: goto label0; + } + } + label0: + { + int i; + int max = 0; + int min = 1<<16; + for (i = 0; i < SIZE(p->b); i++) { + if (p->b[i] > max) max = p->b[i]; + if (p->b[i] < min) min = p->b[i]; + } + p->largest_ch = max; + p->smallest_ch = min; + if (min == 1<<16) error(a, e_empty_grouping); + } + t->token_held = true; return; + } +} + +static void read_define_routine(struct analyser * a, struct name * q) { + struct node * p = new_node(a, c_define); + a->amongvar_needed = false; + if (q) { + check_name_type(a, q, 'R'); + if (q->definition != 0) error(a, e_redefined); + if (q->mode < 0) q->mode = a->mode; else + if (q->mode != a->mode) error2(a, e_declared_as_different_mode, q->mode); + } + p->name = q; + if (a->program == 0) a->program = p; else a->program_end->right = p; + a->program_end = p; + get_token(a, c_as); + p->left = read_C(a); + if (q) q->definition = p->left; + + if (a->substring != 0) { + error2(a, e_unresolved_substring, a->substring->line_number); + a->substring = 0; + } + p->amongvar_needed = a->amongvar_needed; +} + +static void read_define(struct analyser * a) { + if (get_token(a, c_name)) { + struct name * q = find_name(a); + int type; + if (q) { + type = q->type; + } else { + /* No declaration, so sniff next token - if it is 'as' then parse + * as a routine, otherwise as a grouping. + */ + if (read_token(a->tokeniser) == c_as) { + type = t_routine; + } else { + type = t_grouping; + } + a->tokeniser->token_held = true; + } + + if (type == t_grouping) { + read_define_grouping(a, q); + } else { + read_define_routine(a, q); + } + } +} + +static void read_backwardmode(struct analyser * a) { + int mode = a->mode; + a->mode = m_backward; + if (get_token(a, c_bra)) { + read_program_(a, c_ket); + check_token(a, c_ket); + } + a->mode = mode; +} + +static void read_program_(struct analyser * a, int terminator) { + struct tokeniser * t = a->tokeniser; + while (true) { + switch (read_token(t)) { + case c_strings: read_names(a, t_string); break; + case c_booleans: read_names(a, t_boolean); break; + case c_integers: read_names(a, t_integer); break; + case c_routines: read_names(a, t_routine); break; + case c_externals: read_names(a, t_external); break; + case c_groupings: read_names(a, t_grouping); break; + case c_define: read_define(a); break; + case c_backwardmode:read_backwardmode(a); break; + case c_ket: + if (terminator == c_ket) return; + /* fall through */ + default: + error(a, e_unexpected_token); break; + case -1: + if (terminator >= 0) omission_error(a, c_ket); + return; + } + } +} + +static void remove_dead_assignments(struct node * p, struct name * q) { + if (p->name == q) { + switch (p->type) { + case c_assignto: + case c_sliceto: + case c_mathassign: + case c_plusassign: + case c_minusassign: + case c_multiplyassign: + case c_divideassign: + case c_setmark: + case c_set: + case c_unset: + case c_dollar: + /* c_true is a no-op. */ + p->type = c_true; + break; + default: + /* There are no read accesses to this variable, so any + * references must be assignments. + */ + fprintf(stderr, "Unhandled type of dead assignment via %s\n", + name_of_token(p->type)); + exit(1); + } + } + if (p->AE) remove_dead_assignments(p->AE, q); + if (p->left) remove_dead_assignments(p->left, q); + if (p->aux) remove_dead_assignments(p->aux, q); + if (p->right) remove_dead_assignments(p->right, q); +} + +extern void read_program(struct analyser * a) { + read_program_(a, -1); + { + struct name * q = a->names; + while (q) { + switch (q->type) { + case t_external: case t_routine: + if (q->used && q->definition == 0) error4(a, q); + break; + case t_grouping: + if (q->used && q->grouping == 0) error4(a, q); + break; + } + q = q->next; + } + } + + if (a->tokeniser->error_count == 0) { + struct name * q = a->names; + struct name ** ptr = &(a->names); + while (q) { + if (!q->referenced) { + fprintf(stderr, "%s:%d: warning: %s '", + a->tokeniser->file, + q->declaration_line_number, + name_of_name_type(q->type)); + report_b(stderr, q->b); + if (q->type == t_routine || + q->type == t_external || + q->type == t_grouping) { + fprintf(stderr, "' declared but not defined\n"); + } else { + fprintf(stderr, "' defined but not used\n"); + q = q->next; + *ptr = q; + continue; + } + } else if (q->type == t_routine || q->type == t_grouping) { + /* It's OK to define a grouping but only use it to define other + * groupings. + */ + if (!q->used && !q->used_in_definition) { + int line_num; + if (q->type == t_routine) { + line_num = q->definition->line_number; + } else { + line_num = q->grouping->line_number; + } + fprintf(stderr, "%s:%d: warning: %s '", + a->tokeniser->file, + line_num, + name_of_name_type(q->type)); + report_b(stderr, q->b); + fprintf(stderr, "' defined but not used\n"); + } + } else if (q->type == t_external) { + /* Unused is OK. */ + } else if (!q->initialised) { + fprintf(stderr, "%s:%d: warning: %s '", + a->tokeniser->file, + q->declaration_line_number, + name_of_name_type(q->type)); + report_b(stderr, q->b); + fprintf(stderr, "' is never initialised\n"); + } else if (!q->value_used) { + fprintf(stderr, "%s:%d: warning: %s '", + a->tokeniser->file, + q->declaration_line_number, + name_of_name_type(q->type)); + report_b(stderr, q->b); + fprintf(stderr, "' is set but never used\n"); + remove_dead_assignments(a->program, q); + q = q->next; + *ptr = q; + continue; + } + ptr = &(q->next); + q = q->next; + } + + { + /* Now we've eliminated variables whose values are never used we + * can number the variables, which is used by some generators. + */ + int * name_count = a->name_count; + struct name * n; + for (n = a->names; n; n = n->next) { + n->count = name_count[n->type]++; + } + } + } +} + +extern struct analyser * create_analyser(struct tokeniser * t) { + NEW(analyser, a); + a->tokeniser = t; + a->nodes = 0; + a->names = 0; + a->literalstrings = 0; + a->program = 0; + a->amongs = 0; + a->among_count = 0; + a->groupings = 0; + a->mode = m_forward; + a->modifyable = true; + { int i; for (i = 0; i < t_size; i++) a->name_count[i] = 0; } + a->substring = 0; + a->int_limits_used = false; + return a; +} + +extern void close_analyser(struct analyser * a) { + { + struct node * q = a->nodes; + while (q) { + struct node * q_next = q->next; + FREE(q); + q = q_next; + } + } + { + struct name * q = a->names; + while (q) { + struct name * q_next = q->next; + lose_b(q->b); FREE(q); + q = q_next; + } + } + { + struct literalstring * q = a->literalstrings; + while (q) { + struct literalstring * q_next = q->next; + lose_b(q->b); FREE(q); + q = q_next; + } + } + { + struct among * q = a->amongs; + while (q) { + struct among * q_next = q->next; + FREE(q->b); + FREE(q->commands); + FREE(q); + q = q_next; + } + } + { + struct grouping * q = a->groupings; + while (q) { + struct grouping * q_next = q->next; + lose_b(q->b); FREE(q); + q = q_next; + } + } + FREE(a); +} diff --git a/contrib/snowball/compiler/driver.c b/contrib/snowball/compiler/driver.c new file mode 100644 index 0000000..587028f --- /dev/null +++ b/contrib/snowball/compiler/driver.c @@ -0,0 +1,574 @@ +#include <ctype.h> /* for toupper etc */ +#include <stdio.h> /* for fprintf etc */ +#include <stdlib.h> /* for free etc */ +#include <string.h> /* for strcmp */ +#include "header.h" + +#define DEFAULT_JAVA_PACKAGE "org.tartarus.snowball.ext" +#define DEFAULT_JAVA_BASE_CLASS "org.tartarus.snowball.SnowballProgram" +#define DEFAULT_JAVA_AMONG_CLASS "org.tartarus.snowball.Among" +#define DEFAULT_JAVA_STRING_CLASS "java.lang.StringBuilder" + +#define DEFAULT_GO_PACKAGE "snowball" +#define DEFAULT_GO_SNOWBALL_RUNTIME "github.com/snowballstem/snowball/go" + +#define DEFAULT_CS_NAMESPACE "Snowball" +#define DEFAULT_CS_BASE_CLASS "Stemmer" +#define DEFAULT_CS_AMONG_CLASS "Among" +#define DEFAULT_CS_STRING_CLASS "StringBuilder" + +#define DEFAULT_JS_BASE_CLASS "BaseStemmer" + +#define DEFAULT_PYTHON_BASE_CLASS "BaseStemmer" + +static int eq(const char * s1, const char * s2) { + return strcmp(s1, s2) == 0; +} + +__attribute__((noreturn)) +static void print_arglist(int exit_code) { + FILE * f = exit_code ? stderr : stdout; + fprintf(f, "Usage: snowball SOURCE_FILE... [OPTIONS]\n\n" + "Supported options:\n" + " -o[utput] file\n" + " -s[yntax]\n" + " -comments\n" +#ifndef DISABLE_JAVA + " -j[ava]\n" +#endif +#ifndef DISABLE_CSHARP + " -cs[harp]\n" +#endif + " -c++\n" +#ifndef DISABLE_PASCAL + " -pascal\n" +#endif +#ifndef DISABLE_PYTHON + " -py[thon]\n" +#endif +#ifndef DISABLE_JS + " -js\n" +#endif +#ifndef DISABLE_RUST + " -rust\n" +#endif +#ifndef DISABLE_GO + " -go\n" +#endif + " -w[idechars]\n" + " -u[tf8]\n" + " -n[ame] class name\n" + " -ep[refix] string\n" + " -vp[refix] string\n" + " -i[nclude] directory\n" + " -r[untime] path to runtime headers\n" + " -p[arentclassname] fully qualified parent class name\n" +#if !defined(DISABLE_JAVA) || !defined(DISABLE_CSHARP) + " -P[ackage] package name for stemmers\n" + " -S[tringclass] StringBuffer-compatible class\n" + " -a[mongclass] fully qualified name of the Among class\n" +#endif +#ifndef DISABLE_GO + " -gop[ackage] Go package name for stemmers\n" + " -gor[untime] Go snowball runtime package\n" +#endif + " --help display this help and exit\n" + " --version output version information and exit\n" + ); + exit(exit_code); +} + +static void check_lim(int i, int argc) { + if (i >= argc) { + fprintf(stderr, "argument list is one short\n"); + print_arglist(1); + } +} + +static FILE * get_output(symbol * b) { + char * s = b_to_s(b); + FILE * output = fopen(s, "w"); + if (output == 0) { + fprintf(stderr, "Can't open output %s\n", s); + exit(1); + } + free(s); + return output; +} + +static int read_options(struct options * o, int argc, char * argv[]) { + char * s; + int i = 1; + int new_argc = 1; + /* Note down the last option used to specify an explicit encoding so + * we can warn we ignored it for languages with a fixed encoding. + */ + const char * encoding_opt = NULL; + + /* set defaults: */ + + o->output_file = 0; + o->syntax_tree = false; + o->comments = false; + o->externals_prefix = NULL; + o->variables_prefix = 0; + o->runtime_path = 0; + o->parent_class_name = NULL; + o->string_class = NULL; + o->among_class = NULL; + o->package = NULL; + o->go_snowball_runtime = DEFAULT_GO_SNOWBALL_RUNTIME; + o->name = NULL; + o->make_lang = LANG_C; + o->includes = 0; + o->includes_end = 0; + o->encoding = ENC_SINGLEBYTE; + + /* read options: */ + + while (i < argc) { + s = argv[i++]; + if (s[0] != '-') { + /* Non-option argument - shuffle down. */ + argv[new_argc++] = s; + continue; + } + + { + if (eq(s, "-o") || eq(s, "-output")) { + check_lim(i, argc); + o->output_file = argv[i++]; + continue; + } + if (eq(s, "-n") || eq(s, "-name")) { + check_lim(i, argc); + o->name = argv[i++]; + continue; + } +#ifndef DISABLE_JS + if (eq(s, "-js")) { + o->make_lang = LANG_JAVASCRIPT; + continue; + } +#endif +#ifndef DISABLE_RUST + if (eq(s, "-rust")) { + o->make_lang = LANG_RUST; + continue; + } +#endif +#ifndef DISABLE_GO + if (eq(s, "-go")) { + o->make_lang = LANG_GO; + continue; + } +#endif +#ifndef DISABLE_JAVA + if (eq(s, "-j") || eq(s, "-java")) { + o->make_lang = LANG_JAVA; + continue; + } +#endif +#ifndef DISABLE_CSHARP + if (eq(s, "-cs") || eq(s, "-csharp")) { + o->make_lang = LANG_CSHARP; + continue; + } +#endif + if (eq(s, "-c++")) { + o->make_lang = LANG_CPLUSPLUS; + continue; + } +#ifndef DISABLE_PASCAL + if (eq(s, "-pascal")) { + o->make_lang = LANG_PASCAL; + continue; + } +#endif +#ifndef DISABLE_PYTHON + if (eq(s, "-py") || eq(s, "-python")) { + o->make_lang = LANG_PYTHON; + continue; + } +#endif + if (eq(s, "-w") || eq(s, "-widechars")) { + encoding_opt = s; + o->encoding = ENC_WIDECHARS; + continue; + } + if (eq(s, "-s") || eq(s, "-syntax")) { + o->syntax_tree = true; + continue; + } + if (eq(s, "-comments")) { + o->comments = true; + continue; + } + if (eq(s, "-ep") || eq(s, "-eprefix")) { + check_lim(i, argc); + o->externals_prefix = argv[i++]; + continue; + } + if (eq(s, "-vp") || eq(s, "-vprefix")) { + check_lim(i, argc); + o->variables_prefix = argv[i++]; + continue; + } + if (eq(s, "-i") || eq(s, "-include")) { + check_lim(i, argc); + + { + NEW(include, p); + symbol * b = add_s_to_b(0, argv[i++]); + b = add_s_to_b(b, "/"); + p->next = 0; p->b = b; + + if (o->includes == 0) o->includes = p; else + o->includes_end->next = p; + o->includes_end = p; + } + continue; + } + if (eq(s, "-r") || eq(s, "-runtime")) { + check_lim(i, argc); + o->runtime_path = argv[i++]; + continue; + } + if (eq(s, "-u") || eq(s, "-utf8")) { + encoding_opt = s; + o->encoding = ENC_UTF8; + continue; + } + if (eq(s, "-p") || eq(s, "-parentclassname")) { + check_lim(i, argc); + o->parent_class_name = argv[i++]; + continue; + } +#if !defined(DISABLE_JAVA) || !defined(DISABLE_CSHARP) + if (eq(s, "-P") || eq(s, "-Package")) { + check_lim(i, argc); + o->package = argv[i++]; + continue; + } + if (eq(s, "-S") || eq(s, "-stringclass")) { + check_lim(i, argc); + o->string_class = argv[i++]; + continue; + } + if (eq(s, "-a") || eq(s, "-amongclass")) { + check_lim(i, argc); + o->among_class = argv[i++]; + continue; + } +#endif +#ifndef DISABLE_GO + if (eq(s, "-gop") || eq(s, "-gopackage")) { + check_lim(i, argc); + o->package = argv[i++]; + continue; + } + if (eq(s, "-gor") || eq(s, "-goruntime")) { + check_lim(i, argc); + o->go_snowball_runtime = argv[i++]; + continue; + } +#endif + if (eq(s, "--help")) { + print_arglist(0); + } + + if (eq(s, "--version")) { + printf("Snowball compiler version " SNOWBALL_VERSION "\n"); + exit(0); + } + + fprintf(stderr, "'%s' misplaced\n", s); + print_arglist(1); + } + } + if (new_argc == 1) { + fprintf(stderr, "no source files specified\n"); + print_arglist(1); + } + argv[new_argc] = NULL; + + /* Set language-dependent defaults. */ + switch (o->make_lang) { + case LANG_C: + case LANG_CPLUSPLUS: + encoding_opt = NULL; + break; + case LANG_CSHARP: + o->encoding = ENC_WIDECHARS; + if (!o->parent_class_name) + o->parent_class_name = DEFAULT_CS_BASE_CLASS; + if (!o->string_class) + o->string_class = DEFAULT_CS_STRING_CLASS; + if (!o->among_class) + o->among_class = DEFAULT_CS_AMONG_CLASS; + if (!o->package) + o->package = DEFAULT_CS_NAMESPACE; + break; + case LANG_GO: + o->encoding = ENC_UTF8; + if (!o->package) + o->package = DEFAULT_GO_PACKAGE; + break; + case LANG_JAVA: + o->encoding = ENC_WIDECHARS; + if (!o->parent_class_name) + o->parent_class_name = DEFAULT_JAVA_BASE_CLASS; + if (!o->string_class) + o->string_class = DEFAULT_JAVA_STRING_CLASS; + if (!o->among_class) + o->among_class = DEFAULT_JAVA_AMONG_CLASS; + if (!o->package) + o->package = DEFAULT_JAVA_PACKAGE; + break; + case LANG_JAVASCRIPT: + o->encoding = ENC_WIDECHARS; + if (!o->parent_class_name) + o->parent_class_name = DEFAULT_JS_BASE_CLASS; + break; + case LANG_PYTHON: + o->encoding = ENC_WIDECHARS; + if (!o->parent_class_name) + o->parent_class_name = DEFAULT_PYTHON_BASE_CLASS; + break; + case LANG_RUST: + o->encoding = ENC_UTF8; + break; + default: + break; + } + + if (encoding_opt) { + fprintf(stderr, "warning: %s only meaningful for C and C++\n", + encoding_opt); + } + + if (o->make_lang != LANG_C && o->make_lang != LANG_CPLUSPLUS) { + if (o->runtime_path) { + fprintf(stderr, "warning: -r/-runtime only meaningful for C and C++\n"); + } + if (o->externals_prefix) { + fprintf(stderr, "warning: -ep/-eprefix only meaningful for C and C++\n"); + } + } + if (!o->externals_prefix) o->externals_prefix = ""; + + if (!o->name && o->output_file) { + /* Default class name to basename of output_file - this is the standard + * convention for at least Java and C#. + */ + const char * slash = strrchr(o->output_file, '/'); + size_t len; + const char * leaf = (slash == NULL) ? o->output_file : slash + 1; + + slash = strrchr(leaf, '\\'); + if (slash != NULL) leaf = slash + 1; + + { + const char * dot = strchr(leaf, '.'); + len = (dot == NULL) ? strlen(leaf) : (size_t)(dot - leaf); + } + + { + char * new_name = malloc(len + 1); + switch (o->make_lang) { + case LANG_CSHARP: + case LANG_PASCAL: + /* Upper case initial letter. */ + memcpy(new_name, leaf, len); + new_name[0] = toupper(new_name[0]); + break; + case LANG_JAVASCRIPT: + case LANG_PYTHON: { + /* Upper case initial letter and change each + * underscore+letter or hyphen+letter to an upper case + * letter. + */ + size_t i, j = 0; + int uc_next = true; + for (i = 0; i != len; ++i) { + unsigned char ch = leaf[i]; + if (ch == '_' || ch == '-') { + uc_next = true; + } else { + if (uc_next) { + new_name[j] = toupper(ch); + uc_next = false; + } else { + new_name[j] = ch; + } + ++j; + } + } + len = j; + break; + } + default: + /* Just copy. */ + memcpy(new_name, leaf, len); + break; + } + new_name[len] = '\0'; + o->name = new_name; + } + } + + return new_argc; +} + +extern int main(int argc, char * argv[]) { + + int i; + NEW(options, o); + argc = read_options(o, argc, argv); + { + char * file = argv[1]; + symbol * u = get_input(file); + if (u == 0) { + fprintf(stderr, "Can't open input %s\n", file); + exit(1); + } + { + struct tokeniser * t = create_tokeniser(u, file); + struct analyser * a = create_analyser(t); + struct input ** next_input_ptr = &(t->next); + a->encoding = t->encoding = o->encoding; + t->includes = o->includes; + /* If multiple source files are specified, set up the others to be + * read after the first in order, using the same mechanism as + * 'get' uses. */ + for (i = 2; i != argc; ++i) { + NEW(input, q); + file = argv[i]; + u = get_input(file); + if (u == 0) { + fprintf(stderr, "Can't open input %s\n", file); + exit(1); + } + q->p = u; + q->c = 0; + q->file = file; + q->file_needs_freeing = false; + q->line_number = 1; + *next_input_ptr = q; + next_input_ptr = &(q->next); + } + *next_input_ptr = NULL; + read_program(a); + if (t->error_count > 0) exit(1); + if (o->syntax_tree) print_program(a); + close_tokeniser(t); + if (!o->syntax_tree) { + struct generator * g; + + const char * s = o->output_file; + if (!s) { + fprintf(stderr, "Please include the -o option\n"); + print_arglist(1); + } + g = create_generator(a, o); + if (o->make_lang == LANG_C || o->make_lang == LANG_CPLUSPLUS) { + symbol * b = add_s_to_b(0, s); + b = add_s_to_b(b, ".h"); + o->output_h = get_output(b); + b[SIZE(b) - 1] = 'c'; + if (o->make_lang == LANG_CPLUSPLUS) { + b = add_s_to_b(b, "c"); + } + o->output_src = get_output(b); + lose_b(b); + + generate_program_c(g); + fclose(o->output_src); + fclose(o->output_h); + } +#ifndef DISABLE_JAVA + if (o->make_lang == LANG_JAVA) { + symbol * b = add_s_to_b(0, s); + b = add_s_to_b(b, ".java"); + o->output_src = get_output(b); + lose_b(b); + generate_program_java(g); + fclose(o->output_src); + } +#endif +#ifndef DISABLE_PASCAL + if (o->make_lang == LANG_PASCAL) { + symbol *b = add_s_to_b(0, s); + b = add_s_to_b(b, ".pas"); + o->output_src = get_output(b); + lose_b(b); + generate_program_pascal(g); + fclose(o->output_src); + } +#endif +#ifndef DISABLE_PYTHON + if (o->make_lang == LANG_PYTHON) { + symbol * b = add_s_to_b(0, s); + b = add_s_to_b(b, ".py"); + o->output_src = get_output(b); + lose_b(b); + generate_program_python(g); + fclose(o->output_src); + } +#endif +#ifndef DISABLE_JS + if (o->make_lang == LANG_JAVASCRIPT) { + symbol * b = add_s_to_b(0, s); + b = add_s_to_b(b, ".js"); + o->output_src = get_output(b); + lose_b(b); + generate_program_js(g); + fclose(o->output_src); + } +#endif +#ifndef DISABLE_CSHARP + if (o->make_lang == LANG_CSHARP) { + symbol * b = add_s_to_b(0, s); + b = add_s_to_b(b, ".cs"); + o->output_src = get_output(b); + lose_b(b); + generate_program_csharp(g); + fclose(o->output_src); + } +#endif +#ifndef DISABLE_RUST + if (o->make_lang == LANG_RUST) { + symbol * b = add_s_to_b(0, s); + b = add_s_to_b(b, ".rs"); + o->output_src = get_output(b); + lose_b(b); + generate_program_rust(g); + fclose(o->output_src); + } +#endif +#ifndef DISABLE_GO + if (o->make_lang == LANG_GO) { + symbol * b = add_s_to_b(0, s); + b = add_s_to_b(b, ".go"); + o->output_src = get_output(b); + lose_b(b); + generate_program_go(g); + fclose(o->output_src); + } +#endif + close_generator(g); + } + close_analyser(a); + } + lose_b(u); + } + { struct include * p = o->includes; + while (p) { + struct include * q = p->next; + lose_b(p->b); FREE(p); p = q; + } + } + FREE(o); + if (space_count) fprintf(stderr, "%d blocks unfreed\n", space_count); + return 0; +} diff --git a/contrib/snowball/compiler/generator.c b/contrib/snowball/compiler/generator.c new file mode 100644 index 0000000..eed86c1 --- /dev/null +++ b/contrib/snowball/compiler/generator.c @@ -0,0 +1,1725 @@ + +#include <limits.h> /* for INT_MAX */ +#include <stdio.h> /* for fprintf etc */ +#include <stdlib.h> /* for free etc */ +#include <string.h> /* for strlen */ +#include "header.h" + +/* Define this to get warning messages when optimisations can't be used. */ +/* #define OPTIMISATION_WARNINGS */ + +/* recursive use: */ + +static void generate(struct generator * g, struct node * p); + +static int new_label(struct generator * g) { + return g->next_label++; +} + +/* Write routines for simple entities */ + +/* Write a space if the preceding character was not whitespace */ +static void ws_opt_space(struct generator * g, const char * s) { + int ch = str_back(g->outbuf); + if (ch != ' ' && ch != '\n' && ch != '\t' && ch != -1) + write_char(g, ' '); + write_string(g, s); +} + +static void wi3(struct generator * g, int i) { + if (i < 100) write_char(g, ' '); + if (i < 10) write_char(g, ' '); + write_int(g, i); /* integer (width 3) */ +} + + +/* Write routines for items from the syntax tree */ + +static void write_varname(struct generator * g, struct name * p) { + + int ch = "SIIrxg"[p->type]; + switch (p->type) { + case t_external: + write_string(g, g->options->externals_prefix); break; + case t_string: + case t_boolean: + case t_integer: { + int count = p->count; + if (count < 0) { + fprintf(stderr, "Reference to optimised out variable "); + report_b(stderr, p->b); + fprintf(stderr, " attempted\n"); + exit(1); + } + if (p->type == t_boolean) { + /* We use a single array for booleans and integers, with the + * integers first. + */ + count += g->analyser->name_count[t_integer]; + } + write_char(g, ch); + write_char(g, '['); + write_int(g, count); + write_char(g, ']'); + return; + } + default: + write_char(g, ch); write_char(g, '_'); + } + write_b(g, p->b); +} + +static void write_varref(struct generator * g, struct name * p) { /* reference to variable */ + if (p->type < t_routine) write_string(g, "z->"); + write_varname(g, p); +} + +static void write_hexdigit(struct generator * g, int i) { + str_append_ch(g->outbuf, "0123456789ABCDEF"[i & 0xF]); /* hexchar */ +} + +static void write_hex(struct generator * g, int i) { + if (i >> 4) write_hex(g, i >> 4); + write_hexdigit(g, i); /* hex integer */ +} + +/* write character literal */ +static void wlitch(struct generator * g, int ch) { + if (32 <= ch && ch < 127) { + write_char(g, '\''); + if (ch == '\'' || ch == '\\') { + write_char(g, '\\'); + } + write_char(g, ch); + write_char(g, '\''); + } else { + write_string(g, "0x"); write_hex(g, ch); + } +} + +static void wlitarray(struct generator * g, symbol * p) { /* write literal array */ + + write_string(g, "{ "); + { + int i; + for (i = 0; i < SIZE(p); i++) { + wlitch(g, p[i]); + if (i < SIZE(p) - 1) write_string(g, ", "); + } + } + write_string(g, " }"); +} + +static void wlitref(struct generator * g, symbol * p) { /* write ref to literal array */ + + if (SIZE(p) == 0) { + write_char(g, '0'); + } else { + struct str * s = g->outbuf; + g->outbuf = g->declarations; + write_string(g, "static const symbol s_"); write_int(g, g->literalstring_count); write_string(g, "[] = "); + wlitarray(g, p); + write_string(g, ";\n"); + g->outbuf = s; + write_string(g, "s_"); write_int(g, g->literalstring_count); + g->literalstring_count++; + } +} + +static void write_margin(struct generator * g) { + int i; + for (i = 0; i < g->margin; i++) write_string(g, " "); +} + +void write_comment_content(struct generator * g, struct node * p) { + switch (p->type) { + case c_mathassign: + case c_plusassign: + case c_minusassign: + case c_multiplyassign: + case c_divideassign: + if (p->name) { + write_char(g, '$'); + write_b(g, p->name->b); + write_char(g, ' '); + } + write_string(g, name_of_token(p->type)); + write_string(g, " <integer expression>"); + break; + case c_eq: + case c_ne: + case c_gr: + case c_ge: + case c_ls: + case c_le: + write_string(g, "$(<integer expression> "); + write_string(g, name_of_token(p->type)); + write_string(g, " <integer expression>)"); + break; + default: + write_string(g, name_of_token(p->type)); + if (p->name) { + write_char(g, ' '); + write_b(g, p->name->b); + } + } + write_string(g, ", line "); + write_int(g, p->line_number); +} + +static void write_comment(struct generator * g, struct node * p) { + if (g->options->comments) { + ws_opt_space(g, "/* "); + write_comment_content(g, p); + write_string(g, " */"); + } + write_newline(g); +} + +static void wms(struct generator * g, const char * s) { + write_margin(g); write_string(g, s); } /* margin + string */ + +static void write_block_start(struct generator * g) { /* block start */ + wms(g, "{ "); + g->margin++; +} + +static void write_block_end(struct generator * g) { /* block end */ + + if (g->line_labelled == g->line_count) { wms(g, ";"); write_newline(g); } + g->margin--; + wms(g, "}"); write_newline(g); +} + +static void w(struct generator * g, const char * s); + +/* keep c */ +static void wk(struct generator * g, struct node * p, int keep_limit) { + ++g->keep_count; + if (p->mode == m_forward) { + write_string(g, "int c"); + write_int(g, g->keep_count); + write_string(g, " = z->c"); + if (keep_limit) { + write_string(g, ", mlimit"); + write_int(g, g->keep_count); + } + write_char(g, ';'); + } else { + write_string(g, "int m"); + write_int(g, g->keep_count); + write_string(g, " = z->l - z->c"); + if (keep_limit) { + write_string(g, ", mlimit"); + write_int(g, g->keep_count); + } + write_string(g, "; (void)m"); + write_int(g, g->keep_count); + write_char(g, ';'); + } +} + +static void wrestore(struct generator * g, struct node * p, int keep_token) { /* restore c */ + if (p->mode == m_forward) { + write_string(g, "z->c = c"); + } else { + write_string(g, "z->c = z->l - m"); + } + write_int(g, keep_token); write_char(g, ';'); +} + +static void wrestorelimit(struct generator * g, struct node * p, int keep_token) { /* restore limit */ + if (p->mode == m_forward) { + w(g, "z->l += mlimit"); + } else { + w(g, "z->lb = mlimit"); + } + write_int(g, keep_token); write_string(g, ";"); +} + +static void winc(struct generator * g, struct node * p) { /* increment c */ + write_string(g, p->mode == m_forward ? "z->c++;" : + "z->c--;"); +} + +static void wsetl(struct generator * g, int n) { + + g->margin--; + wms(g, "lab"); write_int(g, n); write_char(g, ':'); write_newline(g); + g->line_labelled = g->line_count; + g->margin++; +} + +static void wgotol(struct generator * g, int n) { + wms(g, "goto lab"); write_int(g, n); write_char(g, ';'); write_newline(g); +} + +static void write_failure(struct generator * g, struct node * p) { /* fail */ + if (g->failure_keep_count != 0) { + write_string(g, "{ "); + if (g->failure_keep_count > 0) { + wrestore(g, p, g->failure_keep_count); + } else { + wrestorelimit(g, p, -g->failure_keep_count); + } + write_char(g, ' '); + } + switch (g->failure_label) { + case x_return: + write_string(g, "return 0;"); + break; + default: + write_string(g, "goto lab"); + write_int(g, g->failure_label); + write_char(g, ';'); + g->label_used = 1; + } + if (g->failure_keep_count != 0) write_string(g, " }"); +} + + +/* if at limit fail */ +static void write_check_limit(struct generator * g, struct node * p) { + + write_string(g, p->mode == m_forward ? "if (z->c >= z->l) " : + "if (z->c <= z->lb) "); + write_failure(g, p); +} + +static void write_data_address(struct generator * g, struct node * p) { + symbol * b = p->literalstring; + if (b != 0) { + write_int(g, SIZE(b)); w(g, ", "); + wlitref(g, b); + } else { + write_varref(g, p->name); + } +} + +/* Formatted write. */ +static void writef(struct generator * g, const char * input, struct node * p) { + int i = 0; + int l = strlen(input); + + while (i < l) { + int ch = input[i++]; + if (ch != '~') { + write_char(g, ch); + continue; + } + switch (input[i++]) { + default: write_char(g, input[i - 1]); continue; + case 'C': write_comment(g, p); continue; + case 'k': wk(g, p, false); continue; + case 'K': wk(g, p, true); continue; + case 'i': winc(g, p); continue; + case 'l': write_check_limit(g, p); continue; + case 'f': write_failure(g, p); continue; + case 'M': write_margin(g); continue; + case 'N': write_newline(g); continue; + case '{': write_block_start(g); continue; + case '}': write_block_end(g); continue; + case 'S': write_string(g, g->S[input[i++] - '0']); continue; + case 'I': write_int(g, g->I[input[i++] - '0']); continue; + case 'J': wi3(g, g->I[input[i++] - '0']); continue; + case 'V': write_varref(g, g->V[input[i++] - '0']); continue; + case 'W': write_varname(g, g->V[input[i++] - '0']); continue; + case 'L': wlitref(g, g->L[input[i++] - '0']); continue; + case 'A': wlitarray(g, g->L[input[i++] - '0']); continue; + case 'c': wlitch(g, g->I[input[i++] - '0']); continue; + case 'a': write_data_address(g, p); continue; + case '+': g->margin++; continue; + case '-': g->margin--; continue; + case '$': /* insert_s, insert_v etc */ + write_char(g, p->literalstring == 0 ? 'v' : 's'); + continue; + case 'p': write_string(g, g->options->externals_prefix); continue; + } + } +} + +static void w(struct generator * g, const char * s) { + writef(g, s, 0); +} + +static void generate_AE(struct generator * g, struct node * p) { + const char * s; + switch (p->type) { + case c_name: + write_varref(g, p->name); break; + case c_number: + write_int(g, p->number); break; + case c_maxint: + write_string(g, "MAXINT"); break; + case c_minint: + write_string(g, "MININT"); break; + case c_neg: + write_char(g, '-'); generate_AE(g, p->right); break; + case c_multiply: + s = " * "; goto label0; + case c_plus: + s = " + "; goto label0; + case c_minus: + s = " - "; goto label0; + case c_divide: + s = " / "; + label0: + write_char(g, '('); generate_AE(g, p->left); + write_string(g, s); generate_AE(g, p->right); write_char(g, ')'); break; + case c_cursor: + w(g, "z->c"); break; + case c_limit: + w(g, p->mode == m_forward ? "z->l" : "z->lb"); break; + case c_len: + if (g->options->encoding == ENC_UTF8) { + w(g, "len_utf8(z->p)"); + break; + } + /* FALLTHRU */ + case c_size: + w(g, "SIZE(z->p)"); + break; + case c_lenof: + if (g->options->encoding == ENC_UTF8) { + g->V[0] = p->name; + w(g, "len_utf8(~V0)"); + break; + } + /* FALLTHRU */ + case c_sizeof: + g->V[0] = p->name; + w(g, "SIZE(~V0)"); + break; + } +} + +/* K_needed() tests to see if we really need to keep c. Not true when the + command does not touch the cursor. This and repeat_score() could be + elaborated almost indefinitely. +*/ + +static int K_needed_(struct generator * g, struct node * p, int call_depth) { + while (p) { + switch (p->type) { + case c_atlimit: + case c_do: + case c_dollar: + case c_leftslice: + case c_rightslice: + case c_mathassign: + case c_plusassign: + case c_minusassign: + case c_multiplyassign: + case c_divideassign: + case c_eq: + case c_ne: + case c_gr: + case c_ge: + case c_ls: + case c_le: + case c_sliceto: + case c_booltest: + case c_set: + case c_unset: + case c_true: + case c_false: + case c_debug: + break; + + case c_call: + /* Recursive functions aren't typical in snowball programs, so + * make the pessimistic assumption that keep is needed if we + * hit a generous limit on recursion. It's not likely to make + * a difference to any real world program, but means we won't + * recurse until we run out of stack for pathological cases. + */ + if (call_depth >= 100) return true; + if (K_needed_(g, p->name->definition, call_depth + 1)) + return true; + break; + + case c_bra: + if (K_needed_(g, p->left, call_depth)) return true; + break; + + default: return true; + } + p = p->right; + } + return false; +} + +extern int K_needed(struct generator * g, struct node * p) { + return K_needed_(g, p, 0); +} + +static int repeat_score(struct generator * g, struct node * p, int call_depth) { + int score = 0; + while (p) { + switch (p->type) { + case c_dollar: + case c_leftslice: + case c_rightslice: + case c_mathassign: + case c_plusassign: + case c_minusassign: + case c_multiplyassign: + case c_divideassign: + case c_eq: + case c_ne: + case c_gr: + case c_ge: + case c_ls: + case c_le: + case c_sliceto: /* case c_not: must not be included here! */ + case c_debug: + break; + + case c_call: + /* Recursive functions aren't typical in snowball programs, so + * make the pessimistic assumption that repeat requires cursor + * reinstatement if we hit a generous limit on recursion. It's + * not likely to make a difference to any real world program, + * but means we won't recurse until we run out of stack for + * pathological cases. + */ + if (call_depth >= 100) { + return 2; + } + score += repeat_score(g, p->name->definition, call_depth + 1); + if (score >= 2) + return score; + break; + + case c_bra: + score += repeat_score(g, p->left, call_depth); + if (score >= 2) + return score; + break; + + case c_name: + case c_literalstring: + case c_next: + case c_grouping: + case c_non: + case c_hop: + if (++score >= 2) + return score; + break; + + default: + return 2; + } + p = p->right; + } + return score; +} + +/* tests if an expression requires cursor reinstatement in a repeat */ + +extern int repeat_restore(struct generator * g, struct node * p) { + return repeat_score(g, p, 0) >= 2; +} + +static void generate_bra(struct generator * g, struct node * p) { + p = p->left; + while (p) { + generate(g, p); + p = p->right; + } +} + +static void generate_and(struct generator * g, struct node * p) { + int keep_c = 0; + if (K_needed(g, p->left)) { + writef(g, "~{~k~C", p); + keep_c = g->keep_count; + } else { + writef(g, "~M~C", p); + } + p = p->left; + while (p) { + generate(g, p); + if (keep_c && p->right != 0) { + w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N"); + } + p = p->right; + } + if (keep_c) w(g, "~}"); +} + +static void generate_or(struct generator * g, struct node * p) { + int keep_c = 0; + + int used = g->label_used; + int a0 = g->failure_label; + int a1 = g->failure_keep_count; + + int out_lab = new_label(g); + + if (K_needed(g, p->left)) { + writef(g, "~{~k~C", p); + keep_c = g->keep_count; + } else { + writef(g, "~M~C", p); + } + p = p->left; + g->failure_keep_count = 0; + while (p->right) { + g->failure_label = new_label(g); + g->label_used = 0; + generate(g, p); + wgotol(g, out_lab); + if (g->label_used) + wsetl(g, g->failure_label); + if (keep_c) { + w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N"); + } + p = p->right; + } + g->label_used = used; + g->failure_label = a0; + g->failure_keep_count = a1; + + generate(g, p); + if (keep_c) w(g, "~}"); + wsetl(g, out_lab); +} + +static void generate_backwards(struct generator * g, struct node * p) { + + writef(g, "~Mz->lb = z->c; z->c = z->l;~C~N", p); + generate(g, p->left); + w(g, "~Mz->c = z->lb;~N"); +} + + +static void generate_not(struct generator * g, struct node * p) { + int keep_c = 0; + + int used = g->label_used; + int a0 = g->failure_label; + int a1 = g->failure_keep_count; + + if (K_needed(g, p->left)) { + writef(g, "~{~k~C", p); + keep_c = g->keep_count; + } else { + writef(g, "~M~C", p); + } + + g->failure_label = new_label(g); + g->label_used = 0; + g->failure_keep_count = 0; + generate(g, p->left); + + { + int l = g->failure_label; + int u = g->label_used; + + g->label_used = used; + g->failure_label = a0; + g->failure_keep_count = a1; + + writef(g, "~M~f~N", p); + if (u) + wsetl(g, l); + } + if (keep_c) { + w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N~}"); + } +} + + +static void generate_try(struct generator * g, struct node * p) { + int keep_c = 0; + if (K_needed(g, p->left)) { + writef(g, "~{~k~C", p); + keep_c = g->keep_count; + } else { + writef(g, "~M~C", p); + } + g->failure_keep_count = keep_c; + + g->failure_label = new_label(g); + g->label_used = 0; + generate(g, p->left); + + if (g->label_used) + wsetl(g, g->failure_label); + + if (keep_c) w(g, "~}"); +} + +static void generate_set(struct generator * g, struct node * p) { + g->V[0] = p->name; writef(g, "~M~V0 = 1;~C", p); +} + +static void generate_unset(struct generator * g, struct node * p) { + g->V[0] = p->name; writef(g, "~M~V0 = 0;~C", p); +} + +static void generate_fail(struct generator * g, struct node * p) { + generate(g, p->left); + writef(g, "~M~f~C", p); +} + +/* generate_test() also implements 'reverse' */ + +static void generate_test(struct generator * g, struct node * p) { + int keep_c = 0; + if (K_needed(g, p->left)) { + keep_c = ++g->keep_count; + w(g, p->mode == m_forward ? "~{int c_test" : + "~{int m_test"); + write_int(g, keep_c); + w(g, p->mode == m_forward ? " = z->c;" : + " = z->l - z->c;"); + writef(g, "~C", p); + } else writef(g, "~M~C", p); + + generate(g, p->left); + + if (keep_c) { + w(g, p->mode == m_forward ? "~Mz->c = c_test" : + "~Mz->c = z->l - m_test"); + write_int(g, keep_c); + writef(g, ";~N~}", p); + } +} + +static void generate_do(struct generator * g, struct node * p) { + int keep_c = 0; + if (K_needed(g, p->left)) { + writef(g, "~{~k~C", p); + keep_c = g->keep_count; + } else { + writef(g, "~M~C", p); + } + + if (p->left->type == c_call) { + /* Optimise do <call> */ + g->V[0] = p->left->name; + writef(g, "~{int ret = ~V0(z);~C", p->left); + w(g, "~Mif (ret < 0) return ret;~N~}"); + } else { + g->failure_label = new_label(g); + g->label_used = 0; + g->failure_keep_count = 0; + generate(g, p->left); + + if (g->label_used) + wsetl(g, g->failure_label); + } + if (keep_c) { + w(g, "~M"); wrestore(g, p, keep_c); + w(g, "~N~}"); + } +} + +static void generate_next(struct generator * g, struct node * p) { + if (g->options->encoding == ENC_UTF8) { + if (p->mode == m_forward) + w(g, "~{int ret = skip_utf8(z->p, z->c, 0, z->l, 1"); + else + w(g, "~{int ret = skip_utf8(z->p, z->c, z->lb, 0, -1"); + writef(g, ");~N" + "~Mif (ret < 0) ~f~N" + "~Mz->c = ret;~C" + "~}", p); + } else + writef(g, "~M~l~N" + "~M~i~C", p); +} + +static void generate_GO_grouping(struct generator * g, struct node * p, int is_goto, int complement) { + + struct grouping * q = p->name->grouping; + g->S[0] = p->mode == m_forward ? "" : "_b"; + g->S[1] = complement ? "in" : "out"; + g->S[2] = g->options->encoding == ENC_UTF8 ? "_U" : ""; + g->V[0] = p->name; + g->I[0] = q->smallest_ch; + g->I[1] = q->largest_ch; + if (is_goto) { + writef(g, "~Mif (~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 1) < 0) ~f~C", p); + } else { + writef(g, "~{~C" + "~Mint ret = ~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 1);~N" + "~Mif (ret < 0) ~f~N", p); + if (p->mode == m_forward) + w(g, "~Mz->c += ret;~N"); + else + w(g, "~Mz->c -= ret;~N"); + w(g, "~}"); + } +} + +static void generate_GO(struct generator * g, struct node * p, int style) { + int keep_c = 0; + + int used = g->label_used; + int a0 = g->failure_label; + int a1 = g->failure_keep_count; + + if (p->left->type == c_grouping || p->left->type == c_non) { + /* Special case for "goto" or "gopast" when used on a grouping or an + * inverted grouping - the movement of c by the matching action is + * exactly what we want! */ +#ifdef OPTIMISATION_WARNINGS + printf("Optimising %s %s\n", style ? "goto" : "gopast", p->left->type == c_non ? "non" : "grouping"); +#endif + if (g->options->comments) { + writef(g, "~M~C", p); + } + generate_GO_grouping(g, p->left, style, p->left->type == c_non); + return; + } + + w(g, "~Mwhile(1) {"); writef(g, "~C~+", p); + + if (style == 1 || repeat_restore(g, p->left)) { + writef(g, "~M~k~N", p); + keep_c = g->keep_count; + } + + g->failure_label = new_label(g); + g->label_used = 0; + generate(g, p->left); + + if (style == 1) { + /* include for goto; omit for gopast */ + w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N"); + } + w(g, "~Mbreak;~N"); + if (g->label_used) + wsetl(g, g->failure_label); + if (keep_c) { + w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N"); + } + + g->label_used = used; + g->failure_label = a0; + g->failure_keep_count = a1; + +/* writef(g, "~M~l~N" + "~M~i~N", p); */ + generate_next(g, p); + w(g, "~}"); +} + +static void generate_loop(struct generator * g, struct node * p) { + w(g, "~{int i; for (i = "); generate_AE(g, p->AE); writef(g, "; i > 0; i--)~C" + "~{", p); + + generate(g, p->left); + + w(g, "~}" + "~}"); +} + +static void generate_repeat_or_atleast(struct generator * g, struct node * p, int atleast_case) { + int keep_c = 0; + if (atleast_case) { + writef(g, "~Mwhile(1) {~+~N", p); + } else { + writef(g, "~Mwhile(1) {~+~C", p); + } + + if (repeat_restore(g, p->left)) { + writef(g, "~M~k~N", p); + keep_c = g->keep_count; + } + + g->failure_label = new_label(g); + g->label_used = 0; + g->failure_keep_count = 0; + generate(g, p->left); + + if (atleast_case) w(g, "~Mi--;~N"); + + w(g, "~Mcontinue;~N"); + if (g->label_used) + wsetl(g, g->failure_label); + + if (keep_c) { + w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N"); + } + + w(g, "~Mbreak;~N" + "~}"); +} + +static void generate_repeat(struct generator * g, struct node * p) { + generate_repeat_or_atleast(g, p, false); +} + +static void generate_atleast(struct generator * g, struct node * p) { + w(g, "~{int i = "); generate_AE(g, p->AE); w(g, ";~C"); + { + int used = g->label_used; + int a0 = g->failure_label; + int a1 = g->failure_keep_count; + + generate_repeat_or_atleast(g, p, true); + + g->label_used = used; + g->failure_label = a0; + g->failure_keep_count = a1; + } + writef(g, "~Mif (i > 0) ~f~N" + "~}", p); +} + +static void generate_setmark(struct generator * g, struct node * p) { + g->V[0] = p->name; + writef(g, "~M~V0 = z->c;~C", p); +} + +static void generate_tomark(struct generator * g, struct node * p) { + g->S[0] = p->mode == m_forward ? ">" : "<"; + + w(g, "~Mif (z->c ~S0 "); generate_AE(g, p->AE); writef(g, ") ~f~N", p); + w(g, "~Mz->c = "); generate_AE(g, p->AE); writef(g, ";~C", p); +} + +static void generate_atmark(struct generator * g, struct node * p) { + + w(g, "~Mif (z->c != "); generate_AE(g, p->AE); writef(g, ") ~f~C", p); +} + +static void generate_hop(struct generator * g, struct node * p) { + g->S[0] = p->mode == m_forward ? "+" : "-"; + g->S[1] = p->mode == m_forward ? "0" : "z->lb"; + if (g->options->encoding == ENC_UTF8) { + w(g, "~{int ret = skip_utf8(z->p, z->c, ~S1, z->l, ~S0 "); + generate_AE(g, p->AE); writef(g, ");~C", p); + writef(g, "~Mif (ret < 0) ~f~N", p); + } else { + w(g, "~{int ret = z->c ~S0 "); + generate_AE(g, p->AE); writef(g, ";~C", p); + writef(g, "~Mif (~S1 > ret || ret > z->l) ~f~N", p); + } + writef(g, "~Mz->c = ret;~N" + "~}", p); +} + +static void generate_delete(struct generator * g, struct node * p) { + writef(g, "~{int ret = slice_del(z);~C", p); + writef(g, "~Mif (ret < 0) return ret;~N" + "~}", p); +} + +static void generate_tolimit(struct generator * g, struct node * p) { + g->S[0] = p->mode == m_forward ? "" : "b"; + writef(g, "~Mz->c = z->l~S0;~C", p); +} + +static void generate_atlimit(struct generator * g, struct node * p) { + g->S[0] = p->mode == m_forward ? "" : "b"; + g->S[1] = p->mode == m_forward ? "<" : ">"; + writef(g, "~Mif (z->c ~S1 z->l~S0) ~f~C", p); +} + +static void generate_leftslice(struct generator * g, struct node * p) { + g->S[0] = p->mode == m_forward ? "bra" : "ket"; + writef(g, "~Mz->~S0 = z->c;~C", p); +} + +static void generate_rightslice(struct generator * g, struct node * p) { + g->S[0] = p->mode == m_forward ? "ket" : "bra"; + writef(g, "~Mz->~S0 = z->c;~C", p); +} + +static void generate_assignto(struct generator * g, struct node * p) { + g->V[0] = p->name; + writef(g, "~M~V0 = assign_to(z, ~V0);~C" + "~Mif (~V0 == 0) return -1;~C", p); +} + +static void generate_sliceto(struct generator * g, struct node * p) { + g->V[0] = p->name; + writef(g, "~M~V0 = slice_to(z, ~V0);~C" + "~Mif (~V0 == 0) return -1;~N", p); +} + +static void generate_insert(struct generator * g, struct node * p, int style) { + + int keep_c = style == c_attach; + if (p->mode == m_backward) keep_c = !keep_c; + writef(g, "~{int ret;~N", p); + if (keep_c) w(g, "~{int saved_c = z->c;~N"); + writef(g, "~Mret = insert_~$(z, z->c, z->c, ~a);~C", p); + if (keep_c) w(g, "~Mz->c = saved_c;~N~}"); + writef(g, "~Mif (ret < 0) return ret;~N" + "~}", p); +} + +static void generate_assignfrom(struct generator * g, struct node * p) { + + int keep_c = p->mode == m_forward; /* like 'attach' */ + writef(g, "~{int ret;~N", p); + if (keep_c) writef(g, "~{int saved_c = z->c;~N", p); + w(g, "~Mret = "); + writef(g, keep_c ? "insert_~$(z, z->c, z->l, ~a);~C" : "insert_~$(z, z->lb, z->c, ~a);~C", p); + if (keep_c) w(g, "~Mz->c = saved_c;~N~}"); + writef(g, "~Mif (ret < 0) return ret;~N" + "~}", p); +} + +/* bugs marked <======= fixed 22/7/02. Similar fixes required for Java */ + +static void generate_slicefrom(struct generator * g, struct node * p) { + +/* w(g, "~Mslice_from_s(z, "); <============= bug! should be: */ + writef(g, "~{int ret = slice_from_~$(z, ~a);~C", p); + writef(g, "~Mif (ret < 0) return ret;~N" + "~}", p); +} + +static void generate_setlimit(struct generator * g, struct node * p) { + int keep_c; + if (p->left && p->left->type == c_tomark) { + /* Special case for: + * + * setlimit tomark AE for C + * + * All uses of setlimit in the current stemmers we ship follow this + * pattern, and by special-casing we can avoid having to save and + * restore c. + */ + struct node * q = p->left; + + ++g->keep_count; + writef(g, "~N~{int mlimit", p); + write_int(g, g->keep_count); + writef(g, ";~C", p); + keep_c = g->keep_count; + + g->S[0] = q->mode == m_forward ? ">" : "<"; + + w(g, "~Mif (z->c ~S0 "); generate_AE(g, q->AE); writef(g, ") ~f~N", q); + w(g, "~Mmlimit"); + write_int(g, keep_c); + if (p->mode == m_forward) { + w(g, " = z->l - z->c; z->l = "); + } else { + w(g, " = z->lb; z->lb = "); + } + generate_AE(g, q->AE); + w(g, ";~N"); + } else { + writef(g, "~{~K~C", p); + keep_c = g->keep_count; + generate(g, p->left); + + w(g, "~Mmlimit"); + write_int(g, keep_c); + if (p->mode == m_forward) + w(g, " = z->l - z->c; z->l = z->c;~N"); + else + w(g, " = z->lb; z->lb = z->c;~N"); + w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N"); + } + + g->failure_keep_count = -keep_c; + generate(g, p->aux); + w(g, "~M"); + wrestorelimit(g, p, -g->failure_keep_count); + w(g, "~N" + "~}"); +} + +/* dollar sets snowball up to operate on a string variable as if it were the + * current string */ +static void generate_dollar(struct generator * g, struct node * p) { + + int used = g->label_used; + int a0 = g->failure_label; + int a1 = g->failure_keep_count; + int keep_token; + g->failure_label = new_label(g); + g->label_used = 0; + g->failure_keep_count = 0; + + keep_token = ++g->keep_count; + g->I[0] = keep_token; + writef(g, "~{struct SN_env env~I0 = * z;~C", p); + g->V[0] = p->name; + /* Assume failure. */ + writef(g, "~Mint failure = 1;~N" + "~Mz->p = ~V0;~N" + "~Mz->lb = z->c = 0;~N" + "~Mz->l = SIZE(z->p);~N", p); + generate(g, p->left); + /* Mark success. */ + w(g, "~Mfailure = 0;~N"); + if (g->label_used) + wsetl(g, g->failure_label); + g->V[0] = p->name; /* necessary */ + + g->label_used = used; + g->failure_label = a0; + g->failure_keep_count = a1; + + g->I[0] = keep_token; + writef(g, "~M~V0 = z->p;~N" + "~M* z = env~I0;~N" + "~Mif (failure) ~f~N~}", p); +} + +static void generate_integer_assign(struct generator * g, struct node * p, char * s) { + + g->V[0] = p->name; + g->S[0] = s; + w(g, "~M~V0 ~S0 "); generate_AE(g, p->AE); writef(g, ";~C", p); +} + +static void generate_integer_test(struct generator * g, struct node * p, char * s) { + + w(g, "~Mif (!("); + generate_AE(g, p->left); + write_char(g, ' '); + write_string(g, s); + write_char(g, ' '); + generate_AE(g, p->AE); + writef(g, ")) ~f~C", p); +} + +static void generate_call(struct generator * g, struct node * p) { + + g->V[0] = p->name; + writef(g, "~{int ret = ~V0(z);~C", p); + if (g->failure_keep_count == 0 && g->failure_label == x_return) { + /* Combine the two tests in this special case for better optimisation + * and clearer generated code. */ + writef(g, "~Mif (ret <= 0) return ret;~N~}", p); + } else { + writef(g, "~Mif (ret == 0) ~f~N" + "~Mif (ret < 0) return ret;~N~}", p); + } +} + +static void generate_grouping(struct generator * g, struct node * p, int complement) { + + struct grouping * q = p->name->grouping; + g->S[0] = p->mode == m_forward ? "" : "_b"; + g->S[1] = complement ? "out" : "in"; + g->S[2] = g->options->encoding == ENC_UTF8 ? "_U" : ""; + g->V[0] = p->name; + g->I[0] = q->smallest_ch; + g->I[1] = q->largest_ch; + writef(g, "~Mif (~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 0)) ~f~C", p); +} + +static void generate_namedstring(struct generator * g, struct node * p) { + + g->S[0] = p->mode == m_forward ? "" : "_b"; + g->V[0] = p->name; + writef(g, "~Mif (!(eq_v~S0(z, ~V0))) ~f~C", p); +} + +static void generate_literalstring(struct generator * g, struct node * p) { + symbol * b = p->literalstring; + if (SIZE(b) == 1) { + /* It's quite common to compare with a single character literal string, + * so just inline the simpler code for this case rather than making a + * function call. In UTF-8 mode, only do this for the ASCII subset, + * since multi-byte characters are more complex to test against. + */ + if (g->options->encoding == ENC_UTF8 && *b >= 128) { + printf("single byte %d\n", *b); + exit(1); + } + g->I[0] = *b; + if (p->mode == m_forward) { + writef(g, "~Mif (z->c == z->l || z->p[z->c] != ~c0) ~f~C" + "~Mz->c++;~N", p); + } else { + writef(g, "~Mif (z->c <= z->lb || z->p[z->c - 1] != ~c0) ~f~C" + "~Mz->c--;~N", p); + } + } else { + g->S[0] = p->mode == m_forward ? "" : "_b"; + g->I[0] = SIZE(b); + g->L[0] = b; + + writef(g, "~Mif (!(eq_s~S0(z, ~I0, ~L0))) ~f~C", p); + } +} + +static void generate_define(struct generator * g, struct node * p) { + struct name * q = p->name; + g->next_label = 0; + + g->S[0] = q->type == t_routine ? "static" : "extern"; + g->V[0] = q; + + w(g, "~N~S0 int ~V0(struct SN_env * z) {"); + if (g->options->comments) { + write_string(g, p->mode == m_forward ? " /* forwardmode */" : " /* backwardmode */"); + } + w(g, "~N~+"); + if (p->amongvar_needed) w(g, "~Mint among_var;~N"); + g->failure_keep_count = 0; + g->failure_label = x_return; + g->label_used = 0; + g->keep_count = 0; + generate(g, p->left); + w(g, "~Mreturn 1;~N~}"); +} + +static void generate_substring(struct generator * g, struct node * p) { + + struct among * x = p->among; + int block = -1; + unsigned int bitmap = 0; + struct amongvec * among_cases = x->b; + int c; + int empty_case = -1; + int n_cases = 0; + symbol cases[2]; + int shortest_size = INT_MAX; + int shown_comment = 0; + + g->S[0] = p->mode == m_forward ? "" : "_b"; + g->I[0] = x->number; + g->I[1] = x->literalstring_count; + + /* In forward mode with non-ASCII UTF-8 characters, the first character + * of the string will often be the same, so instead look at the last + * common character position. + * + * In backward mode, we can't match if there are fewer characters before + * the current position than the minimum length. + */ + for (c = 0; c < x->literalstring_count; ++c) { + int size = among_cases[c].size; + if (size != 0 && size < shortest_size) { + shortest_size = size; + } + } + + for (c = 0; c < x->literalstring_count; ++c) { + symbol ch; + if (among_cases[c].size == 0) { + empty_case = c; + continue; + } + if (p->mode == m_forward) { + ch = among_cases[c].b[shortest_size - 1]; + } else { + ch = among_cases[c].b[among_cases[c].size - 1]; + } + if (n_cases == 0) { + block = ch >> 5; + } else if (ch >> 5 != block) { + block = -1; + if (n_cases > 2) break; + } + if (block == -1) { + if (n_cases > 0 && ch == cases[0]) continue; + if (n_cases < 2) { + cases[n_cases++] = ch; + } else if (ch != cases[1]) { + ++n_cases; + break; + } + } else { + if ((bitmap & (1u << (ch & 0x1f))) == 0) { + bitmap |= 1u << (ch & 0x1f); + if (n_cases < 2) + cases[n_cases] = ch; + ++n_cases; + } + } + } + + if (block != -1 || n_cases <= 2) { + char buf[64]; + g->I[2] = block; + g->I[3] = bitmap; + g->I[4] = shortest_size - 1; + if (p->mode == m_forward) { + sprintf(buf, "z->p[z->c + %d]", shortest_size - 1); + g->S[1] = buf; + if (shortest_size == 1) { + writef(g, "~Mif (z->c >= z->l", p); + } else { + writef(g, "~Mif (z->c + ~I4 >= z->l", p); + } + } else { + g->S[1] = "z->p[z->c - 1]"; + if (shortest_size == 1) { + writef(g, "~Mif (z->c <= z->lb", p); + } else { + writef(g, "~Mif (z->c - ~I4 <= z->lb", p); + } + } + if (n_cases == 0) { + /* We get this for the degenerate case: among { '' } + * This doesn't seem to be a useful construct, but it is + * syntactically valid. + */ + } else if (n_cases == 1) { + g->I[4] = cases[0]; + writef(g, " || ~S1 != ~I4", p); + } else if (n_cases == 2) { + g->I[4] = cases[0]; + g->I[5] = cases[1]; + writef(g, " || (~S1 != ~I4 && ~S1 != ~I5)", p); + } else { + writef(g, " || ~S1 >> 5 != ~I2 || !((~I3 >> (~S1 & 0x1f)) & 1)", p); + } + write_string(g, ") "); + if (empty_case != -1) { + /* If the among includes the empty string, it can never fail + * so not matching the bitmap means we match the empty string. + */ + g->I[4] = among_cases[empty_case].result; + writef(g, "among_var = ~I4; else~C", p); + } else { + writef(g, "~f~C", p); + } + shown_comment = 1; + } else { +#ifdef OPTIMISATION_WARNINGS + printf("Couldn't shortcut among %d\n", x->number); +#endif + } + + if (!x->amongvar_needed) { + writef(g, "~Mif (!(find_among~S0(z, a_~I0, ~I1))) ~f", p); + writef(g, shown_comment ? "~N" : "~C", p); + } else { + writef(g, "~Mamong_var = find_among~S0(z, a_~I0, ~I1);", p); + writef(g, shown_comment ? "~N" : "~C", p); + writef(g, "~Mif (!(among_var)) ~f~N", p); + } +} + +static void generate_among(struct generator * g, struct node * p) { + + struct among * x = p->among; + + if (x->substring == 0) generate_substring(g, p); + + if (x->starter != 0) generate(g, x->starter); + + if (x->command_count == 1 && x->nocommand_count == 0) { + /* Only one outcome ("no match" already handled). */ + generate(g, x->commands[0]); + } else if (x->command_count > 0) { + int i; + writef(g, "~Mswitch (among_var) {~C~+", p); + for (i = 1; i <= x->command_count; i++) { + g->I[0] = i; + w(g, "~Mcase ~I0:~N~+"); + generate(g, x->commands[i - 1]); + w(g, "~Mbreak;~N~-"); + } + w(g, "~}"); + } +} + +static void generate_booltest(struct generator * g, struct node * p) { + + g->V[0] = p->name; + writef(g, "~Mif (!(~V0)) ~f~C", p); +} + +static void generate_false(struct generator * g, struct node * p) { + + writef(g, "~M~f~C", p); +} + +static void generate_debug(struct generator * g, struct node * p) { + + g->I[0] = g->debug_count++; + g->I[1] = p->line_number; + writef(g, "~Mdebug(z, ~I0, ~I1);~C", p); + +} + +static void generate(struct generator * g, struct node * p) { + + int used = g->label_used; + int a0 = g->failure_label; + int a1 = g->failure_keep_count; + + switch (p->type) { + case c_define: generate_define(g, p); break; + case c_bra: generate_bra(g, p); break; + case c_and: generate_and(g, p); break; + case c_or: generate_or(g, p); break; + case c_backwards: generate_backwards(g, p); break; + case c_not: generate_not(g, p); break; + case c_set: generate_set(g, p); break; + case c_unset: generate_unset(g, p); break; + case c_try: generate_try(g, p); break; + case c_fail: generate_fail(g, p); break; + case c_reverse: + case c_test: generate_test(g, p); break; + case c_do: generate_do(g, p); break; + case c_goto: generate_GO(g, p, 1); break; + case c_gopast: generate_GO(g, p, 0); break; + case c_repeat: generate_repeat(g, p); break; + case c_loop: generate_loop(g, p); break; + case c_atleast: generate_atleast(g, p); break; + case c_setmark: generate_setmark(g, p); break; + case c_tomark: generate_tomark(g, p); break; + case c_atmark: generate_atmark(g, p); break; + case c_hop: generate_hop(g, p); break; + case c_delete: generate_delete(g, p); break; + case c_next: generate_next(g, p); break; + case c_tolimit: generate_tolimit(g, p); break; + case c_atlimit: generate_atlimit(g, p); break; + case c_leftslice: generate_leftslice(g, p); break; + case c_rightslice: generate_rightslice(g, p); break; + case c_assignto: generate_assignto(g, p); break; + case c_sliceto: generate_sliceto(g, p); break; + case c_assign: generate_assignfrom(g, p); break; + case c_insert: + case c_attach: generate_insert(g, p, p->type); break; + case c_slicefrom: generate_slicefrom(g, p); break; + case c_setlimit: generate_setlimit(g, p); break; + case c_dollar: generate_dollar(g, p); break; + case c_mathassign: generate_integer_assign(g, p, "="); break; + case c_plusassign: generate_integer_assign(g, p, "+="); break; + case c_minusassign: generate_integer_assign(g, p, "-="); break; + case c_multiplyassign:generate_integer_assign(g, p, "*="); break; + case c_divideassign: generate_integer_assign(g, p, "/="); break; + case c_eq: generate_integer_test(g, p, "=="); break; + case c_ne: generate_integer_test(g, p, "!="); break; + case c_gr: generate_integer_test(g, p, ">"); break; + case c_ge: generate_integer_test(g, p, ">="); break; + case c_ls: generate_integer_test(g, p, "<"); break; + case c_le: generate_integer_test(g, p, "<="); break; + case c_call: generate_call(g, p); break; + case c_grouping: generate_grouping(g, p, false); break; + case c_non: generate_grouping(g, p, true); break; + case c_name: generate_namedstring(g, p); break; + case c_literalstring: generate_literalstring(g, p); break; + case c_among: generate_among(g, p); break; + case c_substring: generate_substring(g, p); break; + case c_booltest: generate_booltest(g, p); break; + case c_false: generate_false(g, p); break; + case c_true: break; + case c_debug: generate_debug(g, p); break; + default: fprintf(stderr, "%d encountered\n", p->type); + exit(1); + } + + if (g->failure_label != a0) + g->label_used = used; + g->failure_label = a0; + g->failure_keep_count = a1; +} + +void write_generated_comment_content(struct generator * g) { + w(g, "Generated by Snowball " SNOWBALL_VERSION + " - https://snowballstem.org/"); +} + +void write_start_comment(struct generator * g, + const char * comment_start, + const char * comment_end) { + write_margin(g); + w(g, comment_start); + write_generated_comment_content(g); + if (comment_end) { + w(g, comment_end); + } + w(g, "~N~N"); +} + +static void generate_head(struct generator * g) { + + w(g, "#include \""); + if (g->options->runtime_path) { + write_string(g, g->options->runtime_path); + if (g->options->runtime_path[strlen(g->options->runtime_path) - 1] != '/') + write_char(g, '/'); + } + w(g, "header.h\"~N~N"); +} + +static void generate_routine_headers(struct generator * g) { + struct name * q; + for (q = g->analyser->names; q; q = q->next) { + g->V[0] = q; + switch (q->type) { + case t_routine: + w(g, "static int ~W0(struct SN_env * z);~N"); + break; + case t_external: + w(g, + "#ifdef __cplusplus~N" + "extern \"C\" {~N" + "#endif~N" + "extern int ~W0(struct SN_env * z);~N" + "#ifdef __cplusplus~N" + "}~N" + "#endif~N" + ); + break; + } + } +} + +static void generate_among_table(struct generator * g, struct among * x) { + + struct amongvec * v = x->b; + + g->I[0] = x->number; + { + int i; + for (i = 0; i < x->literalstring_count; i++) { + g->I[1] = i; + g->I[2] = v->size; + g->L[0] = v->b; + if (v->size) + w(g, "static const symbol s_~I0_~I1[~I2] = ~A0;~N"); + v++; + } + } + + g->I[1] = x->literalstring_count; + w(g, "~N~Mstatic const struct among a_~I0[~I1] =~N{~N"); + + v = x->b; + { + int i; + for (i = 0; i < x->literalstring_count; i++) { + g->I[1] = i; + g->I[2] = v->size; + g->I[3] = v->i; + g->I[4] = v->result; + g->S[0] = i < x->literalstring_count - 1 ? "," : ""; + + if (g->options->comments) { + w(g, "/*~J1 */ "); + } + w(g, "{ ~I2, "); + if (v->size == 0) { + w(g, "0,"); + } else { + w(g, "s_~I0_~I1,"); + } + w(g, " ~I3, ~I4, "); + if (v->function == 0) { + write_char(g, '0'); + } else { + write_varname(g, v->function); + } + w(g, "}~S0~N"); + v++; + } + } + w(g, "};~N~N"); +} + +static void generate_amongs(struct generator * g) { + struct among * x; + for (x = g->analyser->amongs; x; x = x->next) { + generate_among_table(g, x); + } +} + +static void set_bit(symbol * b, int i) { b[i/8] |= 1 << i%8; } + +static void generate_grouping_table(struct generator * g, struct grouping * q) { + + int range = q->largest_ch - q->smallest_ch + 1; + int size = (range + 7)/ 8; /* assume 8 bits per symbol */ + symbol * b = q->b; + symbol * map = create_b(size); + int i; + for (i = 0; i < size; i++) map[i] = 0; + + for (i = 0; i < SIZE(b); i++) set_bit(map, b[i] - q->smallest_ch); + + g->V[0] = q->name; + + w(g, "static const unsigned char ~V0[] = { "); + for (i = 0; i < size; i++) { + write_int(g, map[i]); + if (i < size - 1) w(g, ", "); + } + w(g, " };~N~N"); + lose_b(map); +} + +static void generate_groupings(struct generator * g) { + struct grouping * q; + for (q = g->analyser->groupings; q; q = q->next) { + if (q->name->used) + generate_grouping_table(g, q); + } +} + +static void generate_create(struct generator * g) { + + int * p = g->analyser->name_count; + g->I[0] = p[t_string]; + g->I[1] = p[t_integer] + p[t_boolean]; + w(g, "~N" + "extern struct SN_env * ~pcreate_env(void) { return SN_create_env(~I0, ~I1); }" + "~N"); +} + +static void generate_close(struct generator * g) { + + int * p = g->analyser->name_count; + g->I[0] = p[t_string]; + w(g, "~Nextern void ~pclose_env(struct SN_env * z) { SN_close_env(z, ~I0); }~N~N"); +} + +static void generate_create_and_close_templates(struct generator * g) { + w(g, "~N" + "extern struct SN_env * ~pcreate_env(void);~N" + "extern void ~pclose_env(struct SN_env * z);~N" + "~N"); +} + +static void generate_header_file(struct generator * g) { + + struct name * q; + const char * vp = g->options->variables_prefix; + g->S[0] = vp; + + w(g, "#ifdef __cplusplus~N" + "extern \"C\" {~N" + "#endif~N"); /* for C++ */ + + generate_create_and_close_templates(g); + for (q = g->analyser->names; q; q = q->next) { + g->V[0] = q; + switch (q->type) { + case t_external: + w(g, "extern int ~W0(struct SN_env * z);~N"); + break; + case t_string: + case t_integer: + case t_boolean: + if (vp) { + int count = q->count; + if (count < 0) { + /* Unused variables should get removed from `names`. */ + fprintf(stderr, "Optimised out variable "); + report_b(stderr, q->b); + fprintf(stderr, " still in names list\n"); + exit(1); + } + if (q->type == t_boolean) { + /* We use a single array for booleans and integers, + * with the integers first. + */ + count += g->analyser->name_count[t_integer]; + } + g->I[0] = count; + g->I[1] = "SIIrxg"[q->type]; + w(g, "#define ~S0"); + write_b(g, q->b); + w(g, " (~c1[~I0])~N"); + } + break; + } + } + + w(g, "~N" + "#ifdef __cplusplus~N" + "}~N" + "#endif~N"); /* for C++ */ + + w(g, "~N"); +} + +extern void generate_program_c(struct generator * g) { + + g->outbuf = str_new(); + write_start_comment(g, "/* ", " */"); + generate_head(g); + generate_routine_headers(g); + w(g, "#ifdef __cplusplus~N" + "extern \"C\" {~N" + "#endif~N" + "~N"); + generate_create_and_close_templates(g); + w(g, "~N" + "#ifdef __cplusplus~N" + "}~N" + "#endif~N"); + generate_amongs(g); + generate_groupings(g); + g->declarations = g->outbuf; + g->outbuf = str_new(); + g->literalstring_count = 0; + { + struct node * p = g->analyser->program; + while (p) { generate(g, p); p = p->right; } + } + generate_create(g); + generate_close(g); + output_str(g->options->output_src, g->declarations); + str_delete(g->declarations); + output_str(g->options->output_src, g->outbuf); + str_clear(g->outbuf); + + write_start_comment(g, "/* ", " */"); + generate_header_file(g); + output_str(g->options->output_h, g->outbuf); + str_delete(g->outbuf); +} + +/* Generator functions common to multiple languages. */ + +extern struct generator * create_generator(struct analyser * a, struct options * o) { + NEW(generator, g); + g->analyser = a; + g->options = o; + g->margin = 0; + g->debug_count = 0; + g->copy_from_count = 0; + g->line_count = 0; + g->line_labelled = 0; + g->failure_label = -1; + g->unreachable = false; +#ifndef DISABLE_PYTHON + g->max_label = 0; +#endif + return g; +} + +extern void close_generator(struct generator * g) { + FREE(g); +} + +/* Write routines for simple entities */ + +extern void write_char(struct generator * g, int ch) { + str_append_ch(g->outbuf, ch); /* character */ +} + +extern void write_newline(struct generator * g) { + str_append_ch(g->outbuf, '\n'); /* newline */ + g->line_count++; +} + +extern void write_string(struct generator * g, const char * s) { + str_append_string(g->outbuf, s); +} + +extern void write_int(struct generator * g, int i) { + str_append_int(g->outbuf, i); +} + +extern void write_b(struct generator * g, symbol * b) { + + str_append_b(g->outbuf, b); +} + +extern void write_str(struct generator * g, struct str * str) { + + str_append(g->outbuf, str); +} diff --git a/contrib/snowball/compiler/header.h b/contrib/snowball/compiler/header.h new file mode 100644 index 0000000..dadbee3 --- /dev/null +++ b/contrib/snowball/compiler/header.h @@ -0,0 +1,411 @@ +#include <stdio.h> + +#define SNOWBALL_VERSION "2.0.0" + +typedef unsigned char byte; +typedef unsigned short symbol; + +#define true 1 +#define false 0 + +#define MALLOC check_malloc +#define FREE check_free + +#define NEW(type, p) struct type * p = (struct type *) MALLOC(sizeof(struct type)) +#define NEWVEC(type, p, n) struct type * p = (struct type *) MALLOC(sizeof(struct type) * (n)) + +#define SIZE(p) ((int *)(p))[-1] +#define CAPACITY(p) ((int *)(p))[-2] + +extern symbol * create_b(int n); +extern void report_b(FILE * out, const symbol * p); +extern void lose_b(symbol * p); +extern symbol * increase_capacity(symbol * p, int n); +extern symbol * move_to_b(symbol * p, int n, const symbol * q); +extern symbol * add_to_b(symbol * p, int n, const symbol * q); +extern symbol * copy_b(const symbol * p); +extern char * b_to_s(const symbol * p); +extern symbol * add_s_to_b(symbol * p, const char * s); + +#define MOVE_TO_B(B, LIT) \ + move_to_b(B, sizeof(LIT) / sizeof(LIT[0]), LIT) + +struct str; /* defined in space.c */ + +extern struct str * str_new(void); +extern void str_delete(struct str * str); +extern void str_append(struct str * str, const struct str * add); +extern void str_append_ch(struct str * str, char add); +extern void str_append_b(struct str * str, const symbol * q); +extern void str_append_b_tail(struct str * str, const symbol * q, int skip); +extern void str_append_string(struct str * str, const char * s); +extern void str_append_int(struct str * str, int i); +extern void str_clear(struct str * str); +extern void str_assign(struct str * str, const char * s); +extern struct str * str_copy(const struct str * old); +extern symbol * str_data(const struct str * str); +extern int str_len(const struct str * str); +extern int str_back(const struct str *str); +extern int get_utf8(const symbol * p, int * slot); +extern int put_utf8(int ch, symbol * p); +extern void output_str(FILE * outfile, struct str * str); + +typedef enum { ENC_SINGLEBYTE, ENC_UTF8, ENC_WIDECHARS } enc; + +struct m_pair { + + struct m_pair * next; + symbol * name; + symbol * value; + +}; + +/* struct input must be a prefix of struct tokeniser. */ +struct input { + + struct input * next; + symbol * p; + int c; + char * file; + int file_needs_freeing; + int line_number; + +}; + +struct include { + + struct include * next; + symbol * b; + +}; + +enum token_codes { + +#include "syswords2.h" + + c_mathassign, + c_name, + c_number, + c_literalstring, + c_neg, + c_call, + c_grouping, + c_booltest, + + NUM_TOKEN_CODES +}; + +enum uplus_modes { + UPLUS_NONE, + UPLUS_DEFINED, + UPLUS_UNICODE +}; + +/* struct input must be a prefix of struct tokeniser. */ +struct tokeniser { + + struct input * next; + symbol * p; + int c; + char * file; + int file_needs_freeing; + int line_number; + symbol * b; + symbol * b2; + int number; + int m_start; + int m_end; + struct m_pair * m_pairs; + int get_depth; + int error_count; + int token; + int previous_token; + byte token_held; + enc encoding; + + int omission; + struct include * includes; + + /* Mode in which U+ has been used: + * UPLUS_NONE - not used yet + * UPLUS_DEFINED - stringdef U+xxxx .... + * UPLUS_UNICODE - {U+xxxx} used with implicit meaning + */ + int uplusmode; + + char token_disabled[NUM_TOKEN_CODES]; +}; + +extern symbol * get_input(const char * filename); +extern struct tokeniser * create_tokeniser(symbol * b, char * file); +extern int read_token(struct tokeniser * t); +extern const char * name_of_token(int code); +extern void disable_token(struct tokeniser * t, int code); +extern void close_tokeniser(struct tokeniser * t); + +extern int space_count; +extern void * check_malloc(int n); +extern void check_free(void * p); + +struct node; + +struct name { + + struct name * next; + symbol * b; + int type; /* t_string etc */ + int mode; /* )_ for routines, externals */ + struct node * definition; /* ) */ + int count; /* 0, 1, 2 for each type */ + struct grouping * grouping; /* for grouping names */ + byte referenced; + byte used_in_among; /* Function used in among? */ + byte value_used; /* (For variables) is its value ever used? */ + byte initialised; /* (For variables) is it ever initialised? */ + byte used_in_definition; /* (grouping) used in grouping definition? */ + struct node * used; /* First use, or NULL if not used */ + struct name * local_to; /* Local to one routine/external */ + int declaration_line_number;/* Line number of declaration */ + +}; + +struct literalstring { + + struct literalstring * next; + symbol * b; + +}; + +struct amongvec { + + symbol * b; /* the string giving the case */ + int size; /* - and its size */ + struct node * action; /* the corresponding action */ + int i; /* the amongvec index of the longest substring of b */ + int result; /* the numeric result for the case */ + int line_number; /* for diagnostics and stable sorting */ + struct name * function; + +}; + +struct among { + + struct among * next; + struct amongvec * b; /* pointer to the amongvec */ + int number; /* amongs are numbered 0, 1, 2 ... */ + int literalstring_count; /* in this among */ + int command_count; /* in this among */ + int nocommand_count; /* number of "no command" entries in this among */ + int function_count; /* in this among */ + int amongvar_needed; /* do we need to set among_var? */ + struct node * starter; /* i.e. among( (starter) 'string' ... ) */ + struct node * substring; /* i.e. substring ... among ( ... ) */ + struct node ** commands; /* array with command_count entries */ +}; + +struct grouping { + + struct grouping * next; + symbol * b; /* the characters of this group */ + int largest_ch; /* character with max code */ + int smallest_ch; /* character with min code */ + struct name * name; /* so g->name->grouping == g */ + int line_number; +}; + +struct node { + + struct node * next; + struct node * left; + struct node * aux; /* used in setlimit */ + struct among * among; /* used in among */ + struct node * right; + int type; + int mode; + struct node * AE; + struct name * name; + symbol * literalstring; + int number; + int line_number; + int amongvar_needed; /* used in routine definitions */ +}; + +enum name_types { + + t_size = 6, + + t_string = 0, t_boolean = 1, t_integer = 2, t_routine = 3, t_external = 4, + t_grouping = 5 + +/* If this list is extended, adjust wvn in generator.c */ +}; + +/* In name_count[i] below, remember that + type is + ----+---- + 0 | string + 1 | boolean + 2 | integer + 3 | routine + 4 | external + 5 | grouping +*/ + +struct analyser { + + struct tokeniser * tokeniser; + struct node * nodes; + struct name * names; + struct literalstring * literalstrings; + int mode; + byte modifyable; /* false inside reverse(...) */ + struct node * program; + struct node * program_end; + int name_count[t_size]; /* name_count[i] counts the number of names of type i */ + struct among * amongs; + struct among * amongs_end; + int among_count; + int amongvar_needed; /* used in reading routine definitions */ + struct grouping * groupings; + struct grouping * groupings_end; + struct node * substring; /* pending 'substring' in current routine definition */ + enc encoding; + byte int_limits_used; /* are maxint or minint used? */ +}; + +enum analyser_modes { + + m_forward = 0, m_backward /*, m_integer */ + +}; + +extern void print_program(struct analyser * a); +extern struct analyser * create_analyser(struct tokeniser * t); +extern void close_analyser(struct analyser * a); + +extern void read_program(struct analyser * a); + +struct generator { + + struct analyser * analyser; + struct options * options; + int unreachable; /* 0 if code can be reached, 1 if current code + * is unreachable. */ + int var_number; /* Number of next variable to use. */ + struct str * outbuf; /* temporary str to store output */ + struct str * declarations; /* str storing variable declarations */ + int next_label; +#ifndef DISABLE_PYTHON + int max_label; +#endif + int margin; + + /* if > 0, keep_count to restore in case of a failure; + * if < 0, the negated keep_count for the limit to restore in case of + * failure. */ + int failure_keep_count; +#if !defined(DISABLE_JAVA) && !defined(DISABLE_JS) && !defined(DISABLE_PYTHON) && !defined(DISABLE_CSHARP) + struct str * failure_str; /* This is used by some generators instead of failure_keep_count */ +#endif + + int label_used; /* Keep track of whether the failure label is used. */ + int failure_label; + int debug_count; + int copy_from_count; /* count of calls to copy_from() */ + + const char * S[10]; /* strings */ + symbol * B[10]; /* blocks */ + int I[10]; /* integers */ + struct name * V[5]; /* variables */ + symbol * L[5]; /* literals, used in formatted write */ + + int line_count; /* counts number of lines output */ + int line_labelled; /* in ISO C, will need extra ';' if it is a block end */ + int literalstring_count; + int keep_count; /* used to number keep/restore pairs to avoid compiler warnings + about shadowed variables */ +}; + +/* Special values for failure_label in struct generator. */ +enum special_labels { + x_return = -1 +}; + +struct options { + + /* for the command line: */ + + const char * output_file; + const char * name; + FILE * output_src; + FILE * output_h; + byte syntax_tree; + byte comments; + enc encoding; + enum { LANG_JAVA, LANG_C, LANG_CPLUSPLUS, LANG_CSHARP, LANG_PASCAL, LANG_PYTHON, LANG_JAVASCRIPT, LANG_RUST, LANG_GO } make_lang; + const char * externals_prefix; + const char * variables_prefix; + const char * runtime_path; + const char * parent_class_name; + const char * package; + const char * go_snowball_runtime; + const char * string_class; + const char * among_class; + struct include * includes; + struct include * includes_end; +}; + +/* Generator functions common to several backends. */ + +extern struct generator * create_generator(struct analyser * a, struct options * o); +extern void close_generator(struct generator * g); + +extern void write_char(struct generator * g, int ch); +extern void write_newline(struct generator * g); +extern void write_string(struct generator * g, const char * s); +extern void write_int(struct generator * g, int i); +extern void write_b(struct generator * g, symbol * b); +extern void write_str(struct generator * g, struct str * str); + +extern void write_comment_content(struct generator * g, struct node * p); +extern void write_generated_comment_content(struct generator * g); +extern void write_start_comment(struct generator * g, + const char * comment_start, + const char * comment_end); + +extern int K_needed(struct generator * g, struct node * p); +extern int repeat_restore(struct generator * g, struct node * p); + +/* Generator for C code. */ +extern void generate_program_c(struct generator * g); + +#ifndef DISABLE_JAVA +/* Generator for Java code. */ +extern void generate_program_java(struct generator * g); +#endif + +#ifndef DISABLE_CSHARP +/* Generator for C# code. */ +extern void generate_program_csharp(struct generator * g); +#endif + +#ifndef DISABLE_PASCAL +extern void generate_program_pascal(struct generator * g); +#endif + +#ifndef DISABLE_PYTHON +/* Generator for Python code. */ +extern void generate_program_python(struct generator * g); +#endif + +#ifndef DISABLE_JS +extern void generate_program_js(struct generator * g); +#endif + +#ifndef DISABLE_RUST +extern void generate_program_rust(struct generator * g); +#endif + +#ifndef DISABLE_GO +extern void generate_program_go(struct generator * g); +#endif diff --git a/contrib/snowball/compiler/space.c b/contrib/snowball/compiler/space.c new file mode 100644 index 0000000..5b05876 --- /dev/null +++ b/contrib/snowball/compiler/space.c @@ -0,0 +1,287 @@ + +#include <stdio.h> /* for printf */ +#include <stdlib.h> /* malloc, free */ +#include <string.h> /* memmove */ + +#include "header.h" + +#define HEAD 2*sizeof(int) +#define EXTENDER 40 + + +/* This modules provides a simple mechanism for arbitrary length writable + strings, called 'blocks'. They are 'symbol *' items rather than 'char *' + items however. + + The calls are: + + symbol * b = create_b(n); + - create an empty block b with room for n symbols + b = increase_capacity(b, n); + - increase the capacity of block b by n symbols (b may change) + b2 = copy_b(b) + - copy block b into b2 + lose_b(b); + - lose block b + b = move_to_b(b, n, p); + - set the data in b to be the n symbols at address p + b = add_to_b(b, n, p); + - add the n symbols at address p to the end of the data in b + SIZE(b) + - is the number of symbols in b + For example: + + symbol * b = create_b(0); + { int i; + char p[10]; + for (i = 0; i < 100; i++) { + sprintf(p, " %d", i); + add_s_to_b(b, p); + } + } + + and b contains " 0 1 2 ... 99" spaced out as symbols. +*/ + +/* For a block b, SIZE(b) is the number of symbols so far written into it, + CAPACITY(b) the total number it can contain, so SIZE(b) <= CAPACITY(b). + In fact blocks have 1 extra character over the promised capacity so + they can be zero terminated by 'b[SIZE(b)] = 0;' without fear of + overwriting. +*/ + +extern symbol * create_b(int n) { + symbol * p = (symbol *) (HEAD + (char *) MALLOC(HEAD + (n + 1) * sizeof(symbol))); + CAPACITY(p) = n; + SIZE(p) = 0; + return p; +} + +extern void report_b(FILE * out, const symbol * p) { + int i; + for (i = 0; i < SIZE(p); i++) { + if (p[i] > 255) { + printf("In report_b, can't convert p[%d] to char because it's 0x%02x\n", i, (int)p[i]); + exit(1); + } + putc(p[i], out); + } +} + +extern void output_str(FILE * outfile, struct str * str) { + report_b(outfile, str_data(str)); +} + +extern void lose_b(symbol * p) { + if (p == 0) return; + FREE((char *) p - HEAD); +} + +extern symbol * increase_capacity(symbol * p, int n) { + symbol * q = create_b(CAPACITY(p) + n + EXTENDER); + memmove(q, p, CAPACITY(p) * sizeof(symbol)); + SIZE(q) = SIZE(p); + lose_b(p); return q; +} + +extern symbol * move_to_b(symbol * p, int n, const symbol * q) { + int x = n - CAPACITY(p); + if (x > 0) p = increase_capacity(p, x); + memmove(p, q, n * sizeof(symbol)); SIZE(p) = n; return p; +} + +extern symbol * add_to_b(symbol * p, int n, const symbol * q) { + int x = SIZE(p) + n - CAPACITY(p); + if (x > 0) p = increase_capacity(p, x); + memmove(p + SIZE(p), q, n * sizeof(symbol)); SIZE(p) += n; return p; +} + +extern symbol * copy_b(const symbol * p) { + int n = SIZE(p); + symbol * q = create_b(n); + move_to_b(q, n, p); + return q; +} + +int space_count = 0; + +extern void * check_malloc(int n) { + space_count++; + return malloc(n); +} + +extern void check_free(void * p) { + space_count--; + free(p); +} + +/* To convert a block to a zero terminated string: */ + +extern char * b_to_s(const symbol * p) { + int n = SIZE(p); + char * s = (char *)malloc(n + 1); + { + int i; + for (i = 0; i < n; i++) { + if (p[i] > 255) { + printf("In b_to_s, can't convert p[%d] to char because it's 0x%02x\n", i, (int)p[i]); + exit(1); + } + s[i] = (char)p[i]; + } + } + s[n] = 0; + return s; +} + +/* To add a zero terminated string to a block. If p = 0 the + block is created. */ + +extern symbol * add_s_to_b(symbol * p, const char * s) { + int n = strlen(s); + int k; + if (p == 0) p = create_b(n); + k = SIZE(p); + { + int x = k + n - CAPACITY(p); + if (x > 0) p = increase_capacity(p, x); + } + { + int i; + for (i = 0; i < n; i++) p[i + k] = s[i]; + } + SIZE(p) += n; + return p; +} + +/* The next section defines string handling capabilities in terms + of the lower level block handling capabilities of space.c */ +/* -------------------------------------------------------------*/ + +struct str { + symbol * data; +}; + +/* Create a new string. */ +extern struct str * str_new(void) { + + struct str * output = (struct str *) malloc(sizeof(struct str)); + output->data = create_b(0); + return output; +} + +/* Delete a string. */ +extern void str_delete(struct str * str) { + + lose_b(str->data); + free(str); +} + +/* Append a str to this str. */ +extern void str_append(struct str * str, const struct str * add) { + + symbol * q = add->data; + str->data = add_to_b(str->data, SIZE(q), q); +} + +/* Append a character to this str. */ +extern void str_append_ch(struct str * str, char add) { + + symbol q[1]; + q[0] = add; + str->data = add_to_b(str->data, 1, q); +} + +/* Append a low level block to a str. */ +extern void str_append_b(struct str * str, const symbol * q) { + + str->data = add_to_b(str->data, SIZE(q), q); +} + +/* Append the tail of a low level block to a str. */ +extern void str_append_b_tail(struct str * str, const symbol * q, int skip) { + if (skip < 0 || skip >= SIZE(q)) return; + + str->data = add_to_b(str->data, SIZE(q) - skip, q + skip); +} + +/* Append a (char *, null terminated) string to a str. */ +extern void str_append_string(struct str * str, const char * s) { + + str->data = add_s_to_b(str->data, s); +} + +/* Append an integer to a str. */ +extern void str_append_int(struct str * str, int i) { + + char s[30]; + sprintf(s, "%d", i); + str_append_string(str, s); +} + +/* Clear a string */ +extern void str_clear(struct str * str) { + + SIZE(str->data) = 0; +} + +/* Set a string */ +extern void str_assign(struct str * str, const char * s) { + + str_clear(str); + str_append_string(str, s); +} + +/* Copy a string. */ +extern struct str * str_copy(const struct str * old) { + + struct str * newstr = str_new(); + str_append(newstr, old); + return newstr; +} + +/* Get the data stored in this str. */ +extern symbol * str_data(const struct str * str) { + + return str->data; +} + +/* Get the length of the str. */ +extern int str_len(const struct str * str) { + + return SIZE(str->data); +} + +/* Get the last character of the str. + * + * Or -1 if the string is empty. + */ +extern int str_back(const struct str *str) { + return SIZE(str->data) ? str->data[SIZE(str->data) - 1] : -1; +} + +extern int get_utf8(const symbol * p, int * slot) { + int b0, b1; + b0 = *p++; + if (b0 < 0xC0) { /* 1100 0000 */ + * slot = b0; return 1; + } + b1 = *p++; + if (b0 < 0xE0) { /* 1110 0000 */ + * slot = (b0 & 0x1F) << 6 | (b1 & 0x3F); return 2; + } + * slot = (b0 & 0xF) << 12 | (b1 & 0x3F) << 6 | (*p & 0x3F); return 3; +} + +extern int put_utf8(int ch, symbol * p) { + if (ch < 0x80) { + p[0] = ch; return 1; + } + if (ch < 0x800) { + p[0] = (ch >> 6) | 0xC0; + p[1] = (ch & 0x3F) | 0x80; return 2; + } + p[0] = (ch >> 12) | 0xE0; + p[1] = ((ch >> 6) & 0x3F) | 0x80; + p[2] = (ch & 0x3F) | 0x80; return 3; +} diff --git a/contrib/snowball/compiler/syswords.h b/contrib/snowball/compiler/syswords.h new file mode 100644 index 0000000..c401d3e --- /dev/null +++ b/contrib/snowball/compiler/syswords.h @@ -0,0 +1,86 @@ +static const struct system_word vocab[82+1] = { + { 0, (const byte *)"", 82+1}, + + { 1, (const byte *)"$", c_dollar }, + { 1, (const byte *)"(", c_bra }, + { 1, (const byte *)")", c_ket }, + { 1, (const byte *)"*", c_multiply }, + { 1, (const byte *)"+", c_plus }, + { 1, (const byte *)"-", c_minus }, + { 1, (const byte *)"/", c_divide }, + { 1, (const byte *)"<", c_ls }, + { 1, (const byte *)"=", c_assign }, + { 1, (const byte *)">", c_gr }, + { 1, (const byte *)"?", c_debug }, + { 1, (const byte *)"[", c_leftslice }, + { 1, (const byte *)"]", c_rightslice }, + { 2, (const byte *)"!=", c_ne }, + { 2, (const byte *)"*=", c_multiplyassign }, + { 2, (const byte *)"+=", c_plusassign }, + { 2, (const byte *)"-=", c_minusassign }, + { 2, (const byte *)"->", c_sliceto }, + { 2, (const byte *)"/*", c_comment2 }, + { 2, (const byte *)"//", c_comment1 }, + { 2, (const byte *)"/=", c_divideassign }, + { 2, (const byte *)"<+", c_insert }, + { 2, (const byte *)"<-", c_slicefrom }, + { 2, (const byte *)"<=", c_le }, + { 2, (const byte *)"==", c_eq }, + { 2, (const byte *)"=>", c_assignto }, + { 2, (const byte *)">=", c_ge }, + { 2, (const byte *)"as", c_as }, + { 2, (const byte *)"do", c_do }, + { 2, (const byte *)"or", c_or }, + { 3, (const byte *)"and", c_and }, + { 3, (const byte *)"for", c_for }, + { 3, (const byte *)"get", c_get }, + { 3, (const byte *)"hex", c_hex }, + { 3, (const byte *)"hop", c_hop }, + { 3, (const byte *)"len", c_len }, + { 3, (const byte *)"non", c_non }, + { 3, (const byte *)"not", c_not }, + { 3, (const byte *)"set", c_set }, + { 3, (const byte *)"try", c_try }, + { 4, (const byte *)"fail", c_fail }, + { 4, (const byte *)"goto", c_goto }, + { 4, (const byte *)"loop", c_loop }, + { 4, (const byte *)"next", c_next }, + { 4, (const byte *)"size", c_size }, + { 4, (const byte *)"test", c_test }, + { 4, (const byte *)"true", c_true }, + { 5, (const byte *)"among", c_among }, + { 5, (const byte *)"false", c_false }, + { 5, (const byte *)"lenof", c_lenof }, + { 5, (const byte *)"limit", c_limit }, + { 5, (const byte *)"unset", c_unset }, + { 6, (const byte *)"atmark", c_atmark }, + { 6, (const byte *)"attach", c_attach }, + { 6, (const byte *)"cursor", c_cursor }, + { 6, (const byte *)"define", c_define }, + { 6, (const byte *)"delete", c_delete }, + { 6, (const byte *)"gopast", c_gopast }, + { 6, (const byte *)"insert", c_insert }, + { 6, (const byte *)"maxint", c_maxint }, + { 6, (const byte *)"minint", c_minint }, + { 6, (const byte *)"repeat", c_repeat }, + { 6, (const byte *)"sizeof", c_sizeof }, + { 6, (const byte *)"tomark", c_tomark }, + { 7, (const byte *)"atleast", c_atleast }, + { 7, (const byte *)"atlimit", c_atlimit }, + { 7, (const byte *)"decimal", c_decimal }, + { 7, (const byte *)"reverse", c_reverse }, + { 7, (const byte *)"setmark", c_setmark }, + { 7, (const byte *)"strings", c_strings }, + { 7, (const byte *)"tolimit", c_tolimit }, + { 8, (const byte *)"booleans", c_booleans }, + { 8, (const byte *)"integers", c_integers }, + { 8, (const byte *)"routines", c_routines }, + { 8, (const byte *)"setlimit", c_setlimit }, + { 9, (const byte *)"backwards", c_backwards }, + { 9, (const byte *)"externals", c_externals }, + { 9, (const byte *)"groupings", c_groupings }, + { 9, (const byte *)"stringdef", c_stringdef }, + { 9, (const byte *)"substring", c_substring }, + { 12, (const byte *)"backwardmode", c_backwardmode }, + { 13, (const byte *)"stringescapes", c_stringescapes } +}; diff --git a/contrib/snowball/compiler/syswords2.h b/contrib/snowball/compiler/syswords2.h new file mode 100644 index 0000000..e853e32 --- /dev/null +++ b/contrib/snowball/compiler/syswords2.h @@ -0,0 +1,13 @@ + c_among = 4, c_and, c_as, c_assign, c_assignto, c_atleast, + c_atlimit, c_atmark, c_attach, c_backwardmode, c_backwards, + c_booleans, c_bra, c_comment1, c_comment2, c_cursor, c_debug, + c_decimal, c_define, c_delete, c_divide, c_divideassign, c_do, + c_dollar, c_eq, c_externals, c_fail, c_false, c_for, c_ge, c_get, + c_gopast, c_goto, c_gr, c_groupings, c_hex, c_hop, c_insert, + c_integers, c_ket, c_le, c_leftslice, c_len, c_lenof, c_limit, c_loop, + c_ls, c_maxint, c_minint, c_minus, c_minusassign, c_multiply, + c_multiplyassign, c_ne, c_next, c_non, c_not, c_or, c_plus, + c_plusassign, c_repeat, c_reverse, c_rightslice, c_routines, + c_set, c_setlimit, c_setmark, c_size, c_sizeof, c_slicefrom, + c_sliceto, c_stringdef, c_stringescapes, c_strings, c_substring, + c_test, c_tolimit, c_tomark, c_true, c_try, c_unset, diff --git a/contrib/snowball/compiler/tokeniser.c b/contrib/snowball/compiler/tokeniser.c new file mode 100644 index 0000000..e6c6386 --- /dev/null +++ b/contrib/snowball/compiler/tokeniser.c @@ -0,0 +1,567 @@ + +#include <stdio.h> /* stderr etc */ +#include <stdlib.h> /* malloc free */ +#include <string.h> /* strlen */ +#include <ctype.h> /* isalpha etc */ +#include "header.h" + +struct system_word { + int s_size; /* size of system word */ + const byte * s; /* pointer to the system word */ + int code; /* its internal code */ +}; + + +/* ASCII collating assumed in syswords.c */ + +#include "syswords.h" + +#define INITIAL_INPUT_BUFFER_SIZE 8192 + +static int hex_to_num(int ch); + +static int smaller(int a, int b) { return a < b ? a : b; } + +extern symbol * get_input(const char * filename) { + FILE * input = fopen(filename, "r"); + if (input == 0) { return 0; } + { + symbol * u = create_b(INITIAL_INPUT_BUFFER_SIZE); + int size = 0; + while (true) { + int ch = getc(input); + if (ch == EOF) break; + if (size >= CAPACITY(u)) u = increase_capacity(u, size); + u[size++] = ch; + } + fclose(input); + SIZE(u) = size; + return u; + } +} + +static void error(struct tokeniser * t, const char * s1, int n, symbol * p, const char * s2) { + if (t->error_count == 20) { fprintf(stderr, "... etc\n"); exit(1); } + fprintf(stderr, "%s:%d: ", t->file, t->line_number); + if (s1) fprintf(stderr, "%s", s1); + if (p) { + int i; + for (i = 0; i < n; i++) fprintf(stderr, "%c", p[i]); + } + if (s2) fprintf(stderr, "%s", s2); + fprintf(stderr, "\n"); + t->error_count++; +} + +static void error1(struct tokeniser * t, const char * s) { + error(t, s, 0,0, 0); +} + +static void error2(struct tokeniser * t, const char * s) { + error(t, "unexpected end of text after ", 0,0, s); +} + +static int compare_words(int m, symbol * p, int n, const byte * q) { + if (m != n) return m - n; + { + int i; for (i = 0; i < n; i++) { + int diff = p[i] - q[i]; + if (diff) return diff; + } + } + return 0; +} + +static int find_word(int n, symbol * p) { + int i = 0; int j = vocab->code; + do { + int k = i + (j - i)/2; + const struct system_word * w = vocab + k; + int diff = compare_words(n, p, w->s_size, w->s); + if (diff == 0) return w->code; + if (diff < 0) j = k; else i = k; + } while (j - i != 1); + return -1; +} + +static int get_number(int n, symbol * p) { + int x = 0; + int i; for (i = 0; i < n; i++) x = 10*x + p[i] - '0'; + return x; +} + +static int eq_s(struct tokeniser * t, const char * s) { + int l = strlen(s); + if (SIZE(t->p) - t->c < l) return false; + { + int i; + for (i = 0; i < l; i++) if (t->p[t->c + i] != s[i]) return false; + } + t->c += l; return true; +} + +static int white_space(struct tokeniser * t, int ch) { + switch (ch) { + case '\n': + t->line_number++; + /* fall through */ + case '\r': + case '\t': + case ' ': + return true; + } + return false; +} + +static symbol * find_in_m(struct tokeniser * t, int n, symbol * p) { + struct m_pair * q; + for (q = t->m_pairs; q; q = q->next) { + symbol * name = q->name; + if (n == SIZE(name) && memcmp(name, p, n * sizeof(symbol)) == 0) return q->value; + } + return 0; +} + +static int read_literal_string(struct tokeniser * t, int c) { + symbol * p = t->p; + int ch; + SIZE(t->b) = 0; + while (true) { + if (c >= SIZE(p)) { error2(t, "'"); return c; } + ch = p[c]; + if (ch == '\n') { error1(t, "string not terminated"); return c; } + c++; + if (ch == t->m_start) { + /* Inside insert characters. */ + int c0 = c; + int newlines = false; /* no newlines as yet */ + int black_found = false; /* no printing chars as yet */ + while (true) { + if (c >= SIZE(p)) { error2(t, "'"); return c; } + ch = p[c]; c++; + if (ch == t->m_end) break; + if (!white_space(t, ch)) black_found = true; + if (ch == '\n') newlines = true; + if (newlines && black_found) { + error1(t, "string not terminated"); + return c; + } + } + if (!newlines) { + int n = c - c0 - 1; /* macro size */ + int firstch = p[c0]; + symbol * q = find_in_m(t, n, p + c0); + if (q == 0) { + if (n == 1 && (firstch == '\'' || firstch == t->m_start)) + t->b = add_to_b(t->b, 1, p + c0); + else if (n >= 3 && firstch == 'U' && p[c0 + 1] == '+') { + int codepoint = 0; + int x; + if (t->uplusmode == UPLUS_DEFINED) { + /* See if found with xxxx upper-cased. */ + symbol * uc = create_b(n); + int i; + for (i = 0; i != n; ++i) { + uc[i] = toupper(p[c0 + i]); + } + q = find_in_m(t, n, uc); + lose_b(uc); + if (q != 0) { + t->b = add_to_b(t->b, SIZE(q), q); + continue; + } + error1(t, "Some U+xxxx stringdefs seen but not this one"); + } else { + t->uplusmode = UPLUS_UNICODE; + } + for (x = c0 + 2; x != c - 1; ++x) { + int hex = hex_to_num(p[x]); + if (hex < 0) { + error1(t, "Bad hex digit following U+"); + break; + } + codepoint = (codepoint << 4) | hex; + } + if (t->encoding == ENC_UTF8) { + if (codepoint < 0 || codepoint > 0x01ffff) { + error1(t, "character values exceed 0x01ffff"); + } + /* Ensure there's enough space for a max length + * UTF-8 sequence. */ + if (CAPACITY(t->b) < SIZE(t->b) + 3) { + t->b = increase_capacity(t->b, 3); + } + SIZE(t->b) += put_utf8(codepoint, t->b + SIZE(t->b)); + } else { + symbol sym; + if (t->encoding == ENC_SINGLEBYTE) { + /* Only ISO-8859-1 is handled this way - for + * other single-byte character sets you need + * stringdef all the U+xxxx codes you use + * like - e.g.: + * + * stringdef U+0171 hex 'FB' + */ + if (codepoint < 0 || codepoint > 0xff) { + error1(t, "character values exceed 256"); + } + } else { + if (codepoint < 0 || codepoint > 0xffff) { + error1(t, "character values exceed 64K"); + } + } + sym = codepoint; + t->b = add_to_b(t->b, 1, &sym); + } + } else + error(t, "string macro '", n, p + c0, "' undeclared"); + } else + t->b = add_to_b(t->b, SIZE(q), q); + } + } else { + if (ch == '\'') return c; + if (ch < 0 || ch >= 0x80) { + if (t->encoding != ENC_WIDECHARS) { + /* We don't really want people using non-ASCII literal + * strings, but historically it's worked for single-byte + * and UTF-8 if the source encoding matches what the + * generated stemmer works in and it seems unfair to just + * suddenly make this a hard error.` + */ + fprintf(stderr, + "%s:%d: warning: Non-ASCII literal strings aren't " + "portable - use stringdef instead\n", + t->file, t->line_number); + } else { + error1(t, "Non-ASCII literal strings aren't " + "portable - use stringdef instead"); + } + } + t->b = add_to_b(t->b, 1, p + c - 1); + } + } +} + +static int next_token(struct tokeniser * t) { + symbol * p = t->p; + int c = t->c; + int ch; + int code = -1; + while (true) { + if (c >= SIZE(p)) { t->c = c; return -1; } + ch = p[c]; + if (white_space(t, ch)) { c++; continue; } + if (isalpha(ch)) { + int c0 = c; + while (c < SIZE(p) && (isalnum(p[c]) || p[c] == '_')) c++; + code = find_word(c - c0, p + c0); + if (code < 0 || t->token_disabled[code]) { + t->b = move_to_b(t->b, c - c0, p + c0); + code = c_name; + } + } else + if (isdigit(ch)) { + int c0 = c; + while (c < SIZE(p) && isdigit(p[c])) c++; + t->number = get_number(c - c0, p + c0); + code = c_number; + } else + if (ch == '\'') { + c = read_literal_string(t, c + 1); + code = c_literalstring; + } else + { + int lim = smaller(2, SIZE(p) - c); + int i; + for (i = lim; i > 0; i--) { + code = find_word(i, p + c); + if (code >= 0) { c += i; break; } + } + } + if (code >= 0) { + t->c = c; + return code; + } + error(t, "'", 1, p + c, "' unknown"); + c++; + continue; + } +} + +static int next_char(struct tokeniser * t) { + if (t->c >= SIZE(t->p)) return -1; + return t->p[t->c++]; +} + +static int next_real_char(struct tokeniser * t) { + while (true) { + int ch = next_char(t); + if (!white_space(t, ch)) return ch; + } +} + +static void read_chars(struct tokeniser * t) { + int ch = next_real_char(t); + if (ch < 0) { error2(t, "stringdef"); return; } + { + int c0 = t->c-1; + while (true) { + ch = next_char(t); + if (white_space(t, ch) || ch < 0) break; + } + t->b2 = move_to_b(t->b2, t->c - c0 - 1, t->p + c0); + } +} + +static int decimal_to_num(int ch) { + if ('0' <= ch && ch <= '9') return ch - '0'; + return -1; +} + +static int hex_to_num(int ch) { + if ('0' <= ch && ch <= '9') return ch - '0'; + if ('a' <= ch && ch <= 'f') return ch - 'a' + 10; + if ('A' <= ch && ch <= 'F') return ch - 'A' + 10; + return -1; +} + +static void convert_numeric_string(struct tokeniser * t, symbol * p, int base) { + int c = 0; int d = 0; + while (true) { + while (c < SIZE(p) && p[c] == ' ') c++; + if (c == SIZE(p)) break; + { + int number = 0; + while (c != SIZE(p)) { + int ch = p[c]; + if (ch == ' ') break; + if (base == 10) { + ch = decimal_to_num(ch); + if (ch < 0) { + error1(t, "decimal string contains non-digits"); + return; + } + } else { + ch = hex_to_num(ch); + if (ch < 0) { + error1(t, "hex string contains non-hex characters"); + return; + } + } + number = base * number + ch; + c++; + } + if (t->encoding == ENC_SINGLEBYTE) { + if (number < 0 || number > 0xff) { + error1(t, "character values exceed 256"); + return; + } + } else { + if (number < 0 || number > 0xffff) { + error1(t, "character values exceed 64K"); + return; + } + } + if (t->encoding == ENC_UTF8) + d += put_utf8(number, p + d); + else + p[d++] = number; + } + } + SIZE(p) = d; +} + +extern int read_token(struct tokeniser * t) { + symbol * p = t->p; + int held = t->token_held; + t->token_held = false; + if (held) return t->token; + while (true) { + int code = next_token(t); + switch (code) { + case c_comment1: /* slash-slash comment */ + while (t->c < SIZE(p) && p[t->c] != '\n') t->c++; + continue; + case c_comment2: /* slash-star comment */ + while (true) { + if (t->c >= SIZE(p)) { + error1(t, "/* comment not terminated"); + t->token = -1; + return -1; + } + if (p[t->c] == '\n') t->line_number++; + if (eq_s(t, "*/")) break; + t->c++; + } + continue; + case c_stringescapes: { + int ch1 = next_real_char(t); + int ch2 = next_real_char(t); + if (ch2 < 0) { + error2(t, "stringescapes"); + continue; + } + if (ch1 == '\'') { + error1(t, "first stringescape cannot be '"); + continue; + } + t->m_start = ch1; + t->m_end = ch2; + continue; + } + case c_stringdef: { + int base = 0; + read_chars(t); + code = read_token(t); + if (code == c_hex) { base = 16; code = read_token(t); } else + if (code == c_decimal) { base = 10; code = read_token(t); } + if (code != c_literalstring) { + error1(t, "string omitted after stringdef"); + continue; + } + if (base > 0) convert_numeric_string(t, t->b, base); + { NEW(m_pair, q); + q->next = t->m_pairs; + q->name = copy_b(t->b2); + q->value = copy_b(t->b); + t->m_pairs = q; + if (t->uplusmode != UPLUS_DEFINED && + (SIZE(t->b2) >= 3 && t->b2[0] == 'U' && t->b2[1] == '+')) { + if (t->uplusmode == UPLUS_UNICODE) { + error1(t, "U+xxxx already used with implicit meaning"); + } else { + t->uplusmode = UPLUS_DEFINED; + } + } + } + continue; + } + case c_get: + code = read_token(t); + if (code != c_literalstring) { + error1(t, "string omitted after get"); continue; + } + t->get_depth++; + if (t->get_depth > 10) { + fprintf(stderr, "get directives go 10 deep. Looping?\n"); + exit(1); + } + { + NEW(input, q); + char * file = b_to_s(t->b); + symbol * u = get_input(file); + if (u == 0) { + struct include * r; + for (r = t->includes; r; r = r->next) { + symbol * b = copy_b(r->b); + b = add_to_b(b, SIZE(t->b), t->b); + free(file); + file = b_to_s(b); + u = get_input(file); + lose_b(b); + if (u != 0) break; + } + } + if (u == 0) { + error(t, "Can't get '", SIZE(t->b), t->b, "'"); + exit(1); + } + memmove(q, t, sizeof(struct input)); + t->next = q; + t->p = u; + t->c = 0; + t->file = file; + t->file_needs_freeing = true; + t->line_number = 1; + } + p = t->p; + continue; + case -1: + if (t->next) { + lose_b(p); + { + struct input * q = t->next; + memmove(t, q, sizeof(struct input)); p = t->p; + FREE(q); + } + t->get_depth--; + continue; + } + /* fall through */ + default: + t->previous_token = t->token; + t->token = code; + return code; + } + } +} + +extern const char * name_of_token(int code) { + int i; + for (i = 1; i < vocab->code; i++) + if ((vocab + i)->code == code) return (const char *)(vocab + i)->s; + switch (code) { + case c_mathassign: return "="; + case c_name: return "name"; + case c_number: return "number"; + case c_literalstring:return "literal"; + case c_neg: return "neg"; + case c_grouping: return "grouping"; + case c_call: return "call"; + case c_booltest: return "Boolean test"; + case -2: return "start of text"; + case -1: return "end of text"; + default: return "?"; + } +} + +extern void disable_token(struct tokeniser * t, int code) { + t->token_disabled[code] = 1; +} + +extern struct tokeniser * create_tokeniser(symbol * p, char * file) { + NEW(tokeniser, t); + t->next = 0; + t->p = p; + t->c = 0; + t->file = file; + t->file_needs_freeing = false; + t->line_number = 1; + t->b = create_b(0); + t->b2 = create_b(0); + t->m_start = -1; + t->m_pairs = 0; + t->get_depth = 0; + t->error_count = 0; + t->token_held = false; + t->token = -2; + t->previous_token = -2; + t->uplusmode = UPLUS_NONE; + memset(t->token_disabled, 0, sizeof(t->token_disabled)); + return t; +} + +extern void close_tokeniser(struct tokeniser * t) { + lose_b(t->b); + lose_b(t->b2); + { + struct m_pair * q = t->m_pairs; + while (q) { + struct m_pair * q_next = q->next; + lose_b(q->name); + lose_b(q->value); + FREE(q); + q = q_next; + } + } + { + struct input * q = t->next; + while (q) { + struct input * q_next = q->next; + FREE(q); + q = q_next; + } + } + if (t->file_needs_freeing) free(t->file); + FREE(t); +} |