Adding upstream version 2.0.16.upstream/2.0.16upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1124 insertions, 0 deletions

diff --git a/src/turtle_lexer.l b/src/turtle_lexer.l
new file mode 100644
index 0000000..8d0c53e
--- /dev/null
+++ b/src/turtle_lexer.l
@@ -0,0 +1,1124 @@
+/* -*- Mode: c; c-basic-offset: 2 -*-
+ *
+ * turtle_lexer.l - Raptor Turtle lexer - making tokens for turtle grammar generator
+ *
+ * Copyright (C) 2003-2013, David Beckett http://www.dajobe.org/
+ * Copyright (C) 2003-2005, University of Bristol, UK http://www.bristol.ac.uk/
+ * 
+ * This package is Free Software and part of Redland http://librdf.org/
+ * 
+ * It is licensed under the following three licenses as alternatives:
+ *   1. GNU Lesser General Public License (LGPL) V2.1 or any newer version
+ *   2. GNU General Public License (GPL) V2 or any newer version
+ *   3. Apache License, V2.0 or any newer version
+ * 
+ * You may not use this file except in compliance with at least one of
+ * the above three licenses.
+ * 
+ * See LICENSE.html or LICENSE.txt at the top of this package for the
+ * complete terms and further detail along with the license texts for
+ * the licenses in COPYING.LIB, COPYING and LICENSE-2.0.txt respectively.
+ * 
+ * 
+ * Turtle is defined in http://www.dajobe.org/2004/01/turtle/
+ *
+ * To generate the C files from this source, rather than use the
+ * shipped turtle_lexer.c/.h needs a patched version of flex 2.5.31 such
+ * as the one available in Debian GNU/Linux.   Details below
+ * near the %option descriptions.
+ *
+ */
+
+
+/* recognise 8-bits */
+%option 8bit
+%option warn nodefault
+
+/* all symbols prefixed by this */
+%option prefix="turtle_lexer_"
+
+/* This is not needed, flex is invoked -oturtle_lexer.c */
+/* %option outfile="turtle_lexer.c" */
+
+/* Emit a C header file for prototypes
+ * Only available in flex 2.5.13 or newer.
+ * It was renamed to header-file in flex 2.5.19
+ */
+%option header-file="turtle_lexer.h"
+
+/* Do not emit #include <unistd.h>
+ * Only available in flex 2.5.7 or newer.
+ * Broken in flex 2.5.31 without patches.
+ */
+%option nounistd
+
+/* Never interactive */
+/*  No isatty() check */
+%option never-interactive
+
+/* Batch scanner */
+%option batch
+
+/* Never use yyunput */
+%option nounput
+
+/* Supply our own alloc/realloc/free functions */
+%option noyyalloc noyyrealloc noyyfree
+
+/* Re-entrant scanner */
+%option reentrant
+
+%option extra-type="raptor_parser*"
+
+/* Makes yyget_lval() yyset_lval() and yylval appear */
+%option bison-bridge
+/* Makes yyget_lloc() yyset_lloc() and yylloc appear */
+/* %option bison-locations */
+
+  /* definitions */
+
+%{
+
+/* NOTE: These headers are NOT included here but are inserted by
+ * fix-flex since otherwise it appears far too late in the generated C
+ */
+
+/*
+#ifdef HAVE_CONFIG_H
+#include <raptor_config.h>
+#endif
+*/
+
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+#include <stdarg.h>
+#ifdef HAVE_ERRNO_H
+#include <errno.h>
+#endif
+#ifdef HAVE_STDLIB_H
+#include <stdlib.h>
+#endif
+#ifdef HAVE_SETJMP_H
+#include <setjmp.h>
+#endif
+
+#include "raptor2.h"
+#include "raptor_internal.h"
+
+#include <turtle_parser.h>
+#include <turtle_common.h>
+
+#define YYSTYPE TURTLE_PARSER_STYPE
+
+/* Prototypes */ 
+static unsigned char *turtle_copy_token(unsigned char *text, size_t len);
+static unsigned char *turtle_copy_string_token(raptor_parser* rdf_parser, unsigned char *text, size_t len, int delim);
+void turtle_lexer_syntax_error(void* ctx, const char *message, ...) RAPTOR_PRINTF_FORMAT(2, 3);
+
+#ifdef RAPTOR_DEBUG
+const char * turtle_token_print(raptor_world* world, int token, YYSTYPE *lval);
+#endif
+
+#ifdef __cplusplus
+#define INPUT_FN yyinput
+#else
+#define INPUT_FN input
+#endif
+
+
+#if FLEX_VERSION_DECIMAL < 20536
+/* debian flex 2.5.35-10.1 added these column header prototypes in
+ * re-entrant mode.  standard flex omits them
+ */
+void turtle_lexer_set_column(int column_no, yyscan_t yyscanner);
+int turtle_lexer_get_column(yyscan_t yyscanner);
+#endif
+
+static void turtle_lexer_cleanup(yyscan_t yyscanner);
+#undef yycleanup
+#define yycleanup turtle_lexer_cleanup
+
+#ifdef HAVE_SETJMP
+static jmp_buf turtle_lexer_fatal_error_longjmp_env;
+
+/* fatal error handler declaration */
+#define YY_FATAL_ERROR(msg) do {                                  \
+    turtle_lexer_error(yyscanner, RAPTOR_LOG_LEVEL_FATAL, msg);   \
+    longjmp(turtle_lexer_fatal_error_longjmp_env, 1);             \
+} while(0)
+#else
+#define YY_FATAL_ERROR(msg) do {                                   \
+    turtle_lexer_error(yyscanner, RAPTOR_LOG_LEVEL_FATAL, msg);    \
+    abort();                                                       \
+} while(0)
+#endif
+ 
+/* Remove the re-fill function since it should never be called */
+#define YY_INPUT(buf,result,max_size) { return YY_NULL; }
+
+static void turtle_lexer_error(yyscan_t yyscanner, raptor_log_level level, yyconst char *message, ...) RAPTOR_PRINTF_FORMAT(3, 4);
+
+/* Fatal error handler that returns EOF instead of abort()/longjmp()
+ * so that parser can clean up properly */
+#define YY_FATAL_ERROR_EOF(msg) do { \
+    turtle_lexer_error(yyscanner, RAPTOR_LOG_LEVEL_FATAL, "%s", msg);   \
+    yyterminate(); \
+} while(0)
+
+/* Out-of-memory reporting macro */
+#define TURTLE_LEXER_OOM() YY_FATAL_ERROR_EOF(turtle_lexer_oom_text)
+static char turtle_lexer_oom_text[]="turtle_lexer: Out of memory";
+
+/* Do not need input() to to read from stdin */
+#define YY_NO_INPUT 1
+
+#define YY_USER_ACTION				\
+  turtle_parser->consumed += yyleng;
+
+%}
+
+/* Tokens from Turtle 2013 spec - lex-ifyed to remove unicode ranges */
+PN_CHARS_BASE [A-Za-z\x80-\xff]
+PN_CHARS {PN_CHARS_BASE}|"_"|"-"|[0-9]
+BS_ESCAPES [-_~\.!$&\'()*+,;=/?#@%]
+HEX [0-9A-Fa-f]
+PLX "%"{HEX}{HEX})|("\\"{BS_ESCAPES}
+
+LANGTAG "@"[A-Za-z][-A-Z_a-z0-9]*
+
+/* flex: only 1 level of definition expansion so have to expand PLX */
+BN_LABEL  ({PN_CHARS_BASE}|"_"|[0-9])(({PN_CHARS}|".")*({PN_CHARS}))*
+PN_PREFIX ({PN_CHARS_BASE})(({PN_CHARS}|".")*({PN_CHARS}))*
+PN_LOCAL ({PN_CHARS_BASE}|"_"|[0-9]|":"|{PLX})(({PN_CHARS}|"."|":"|{PLX})*({PN_CHARS}|":"|{PLX}))*
+
+QNAME {PN_PREFIX}?":"{PN_LOCAL}?
+
+UCHAR "\\u"{HEX}{HEX}{HEX}{HEX}|"\\U"{HEX}{HEX}{HEX}{HEX}{HEX}{HEX}{HEX}{HEX}
+IRI "<"([^\x00-\x20<>\"{}\|^`\\]|{UCHAR})*">"
+
+INTEGER [-+]?[0-9]+
+DECIMAL [-+]?[0-9]*"."[0-9]+
+DOUBLE [-+]?([0-9]+"."[0-9]*{EXPONENT}|"."[0-9]+{EXPONENT}|[0-9]+{EXPONENT})
+EXPONENT [eE][+-]?[0-9]+
+
+
+%x PREF LONG_DLITERAL LONG_SLITERAL
+
+
+%%
+  /* rules */
+
+%{
+  raptor_parser *rdf_parser = yyextra;
+  raptor_turtle_parser* turtle_parser = (raptor_turtle_parser*)rdf_parser->context;
+
+#ifdef HAVE_SETJMP
+  if(setjmp(turtle_lexer_fatal_error_longjmp_env))
+    return 1;
+#endif
+%}
+    
+
+\r\n|\r|\n   { turtle_parser->lineno++; }
+ 
+[\ \t\v]+   { /* empty */ }
+
+
+"a" { return A; }
+
+"."       { return DOT; } 
+","       { return COMMA; } 
+";"       { return SEMICOLON; }
+"["       { return LEFT_SQUARE; }
+"]"       { return RIGHT_SQUARE; }
+"@prefix" { BEGIN(PREF); return PREFIX; }
+[Pp][Rr][Ee][Ff][Ii][Xx] { BEGIN(PREF);
+		return SPARQL_PREFIX; }
+"@base"   { return BASE; }
+[Bb][Aa][Ss][Ee] { return SPARQL_BASE; }
+"^^"      { return HAT; }
+"("       { return LEFT_ROUND; }
+")"       { return RIGHT_ROUND; }
+"{"       { return LEFT_CURLY; }
+"}"       { return RIGHT_CURLY; }
+"true"    { return TRUE_TOKEN; }
+"false"   { return FALSE_TOKEN; }
+
+
+\"([^\"\\\n\r]|\\[^\n\r])*\"   { yylval->string = turtle_copy_string_token(rdf_parser, (unsigned char*)yytext+1, yyleng-2, '"'); /* ' */
+                                if(!yylval->string)
+                                  yyterminate();
+
+                                return STRING_LITERAL; }
+
+\'([^\'\\\n\r]|\\[^\n\r])*\'   { yylval->string = turtle_copy_string_token(rdf_parser, (unsigned char*)yytext+1, yyleng-2, '"'); /* ' */
+                                if(!yylval->string)
+                                  yyterminate();
+
+                                return STRING_LITERAL; }
+
+\"\"\"				{ BEGIN(LONG_DLITERAL); 
+                                  turtle_parser->sb = raptor_new_stringbuffer();
+                                  if(!turtle_parser->sb)
+                                    TURTLE_LEXER_OOM();
+                          }
+
+<LONG_DLITERAL>\"\"\"			{
+		  size_t len;
+     
+		  BEGIN(INITIAL);
+                  len = raptor_stringbuffer_length(turtle_parser->sb);
+                  yylval->string = RAPTOR_MALLOC(unsigned char*, len + 1);
+                  if(!yylval->string)
+                    TURTLE_LEXER_OOM();
+                  raptor_stringbuffer_copy_to_string(turtle_parser->sb, (unsigned char*)yylval->string, len);
+                  yylval->string[len]='\0';
+
+                  raptor_free_stringbuffer(turtle_parser->sb);
+                  turtle_parser->sb = NULL;
+                  return STRING_LITERAL; }
+
+<LONG_DLITERAL>\"|(\\.|[^\"\\]|\n)*	{
+		  char *p;
+
+		  if(*yytext == EOF) {
+                    BEGIN(INITIAL);
+                    turtle_syntax_error(rdf_parser, "End of file in middle of literal");
+                    raptor_free_stringbuffer(turtle_parser->sb);
+                    turtle_parser->sb = NULL;
+                    return EOF;
+                  }
+
+		  for(p = yytext; *p; p++) {
+                    if(*p == '\n')
+                      turtle_parser->lineno++;
+                  }
+
+                  if(raptor_stringbuffer_append_turtle_string(turtle_parser->sb, (unsigned char*)yytext, yyleng, '"', (raptor_simple_message_handler)turtle_lexer_syntax_error, rdf_parser, 0)) { /* " */
+                    BEGIN(INITIAL);
+                    raptor_free_stringbuffer(turtle_parser->sb);
+                    turtle_parser->sb = NULL;
+                    YY_FATAL_ERROR_EOF("raptor_stringbuffer_append_turtle_string failed");
+                  }
+                  
+   }
+
+<LONG_DLITERAL>\\	{
+     		/* this should only happen if \ is at the end of the file so the Turtle doc is illegal anyway */
+                    BEGIN(INITIAL);
+                    raptor_free_stringbuffer(turtle_parser->sb);
+                    turtle_parser->sb = NULL;
+                    turtle_syntax_error(rdf_parser, "End of file in middle of \"\"\"literal\"\"\"");
+                    yyterminate();
+}
+
+<LONG_DLITERAL><<EOF>>     {
+                    BEGIN(INITIAL);
+                    raptor_free_stringbuffer(turtle_parser->sb);
+                    turtle_parser->sb = NULL;
+                    if(!turtle_parser->is_end) {
+                      /* next run will fix things, hopefully */
+                      return EOF;
+                    }
+                    /* otherwise abort */
+                    turtle_syntax_error(rdf_parser, "End of file in middle of \"\"\"literal\"\"\"");
+                    yyterminate();
+}
+    
+\'\'\'				{ BEGIN(LONG_SLITERAL); 
+                                  turtle_parser->sb = raptor_new_stringbuffer();
+                                  if(!turtle_parser->sb)
+                                    TURTLE_LEXER_OOM();
+                          }
+
+<LONG_SLITERAL>\'\'\'			{
+		  size_t len;
+     
+		  BEGIN(INITIAL);
+                  len = raptor_stringbuffer_length(turtle_parser->sb);
+                  yylval->string = RAPTOR_MALLOC(unsigned char*, len + 1);
+                  if(!yylval->string)
+                    TURTLE_LEXER_OOM();
+                  raptor_stringbuffer_copy_to_string(turtle_parser->sb, (unsigned char*)yylval->string, len);
+                  yylval->string[len]='\0';
+
+                  raptor_free_stringbuffer(turtle_parser->sb);
+                  turtle_parser->sb = NULL;
+                  return STRING_LITERAL; }
+
+<LONG_SLITERAL>\'|(\\.|[^\'\\]|\n)*	{
+		  char *p;
+
+		  if(*yytext == EOF) {
+                    BEGIN(INITIAL);
+                    turtle_syntax_error(rdf_parser, "End of file in middle of \'\'\'literal\'\'\'");
+                    raptor_free_stringbuffer(turtle_parser->sb);
+                    turtle_parser->sb = NULL;
+                    return EOF;
+                  }
+
+		  for(p = yytext; *p; p++) {
+                    if(*p == '\n')
+                      turtle_parser->lineno++;
+                  }
+
+                  if(raptor_stringbuffer_append_turtle_string(turtle_parser->sb, (unsigned char*)yytext, yyleng, '"', (raptor_simple_message_handler)turtle_lexer_syntax_error, rdf_parser, 0)) { /* " */
+                    BEGIN(INITIAL);
+                    raptor_free_stringbuffer(turtle_parser->sb);
+                    turtle_parser->sb = NULL;
+                    YY_FATAL_ERROR_EOF("raptor_stringbuffer_append_turtle_string failed");
+                  }
+                  
+   }
+
+<LONG_SLITERAL>\\	{
+     		/* this should only happen if \ is at the end of the file so the Turtle doc is illegal anyway */
+                    BEGIN(INITIAL);
+                    raptor_free_stringbuffer(turtle_parser->sb);
+                    turtle_parser->sb = NULL;
+                    turtle_syntax_error(rdf_parser, "End of file in middle of '''literal'''");
+                    yyterminate();
+}
+
+<LONG_SLITERAL><<EOF>>     {
+                    BEGIN(INITIAL);
+                    raptor_free_stringbuffer(turtle_parser->sb);
+                    turtle_parser->sb = NULL;
+                    if(!turtle_parser->is_end) {
+                      /* next run will fix things, hopefully */
+                      return EOF;
+                    }
+                    /* otherwise abort */
+                    turtle_syntax_error(rdf_parser, "End of file in middle of '''literal'''");
+                    yyterminate();
+}
+    
+"_:"{BN_LABEL}	{ yylval->string = turtle_copy_token((unsigned char*)yytext+2, yyleng-2);
+          if(!yylval->string)
+            YY_FATAL_ERROR_EOF("turtle_copy_token failed");
+                          return BLANK_LITERAL; }
+
+{QNAME}	{ yylval->uri = turtle_qname_to_uri(rdf_parser, (unsigned char*)yytext, yyleng);
+          if(!yylval->uri) {
+            turtle_lexer_error(yyscanner, RAPTOR_LOG_LEVEL_ERROR, "Failed to convert qname %s to URI", yytext);
+            yyterminate();
+          }
+
+                          return QNAME_LITERAL; }
+
+{DECIMAL}	{ yylval->string = turtle_copy_token((unsigned char*)yytext, yyleng);
+                  if(!yylval->string)
+                    YY_FATAL_ERROR_EOF("turtle_copy_token failed");
+                        return DECIMAL_LITERAL;
+}
+
+{DOUBLE} { yylval->string = turtle_copy_token((unsigned char*)yytext, yyleng);
+                  if(!yylval->string)
+                    YY_FATAL_ERROR_EOF("turtle_copy_token failed");
+                        return FLOATING_LITERAL;
+}
+
+{INTEGER}        { yylval->string = turtle_copy_token((unsigned char*)yytext, yyleng);
+                  if(!yylval->string)
+                    YY_FATAL_ERROR_EOF("turtle_copy_token failed");
+                          return INTEGER_LITERAL; }
+
+<PREF>[\ \t\v]+ { /* eat up leading whitespace */ }
+<PREF>{PN_PREFIX}":"	{ yylval->string=turtle_copy_token((unsigned char*)yytext, yyleng);
+                            if(!yylval->string)
+                              YY_FATAL_ERROR_EOF("turtle_copy_token failed");
+                          BEGIN(INITIAL);
+                          return IDENTIFIER; }
+<PREF>":"	{ BEGIN(INITIAL);
+		  yylval->string = turtle_copy_token((unsigned char*)yytext, 0);
+      if(!yylval->string)
+        YY_FATAL_ERROR_EOF("turtle_copy_token failed");
+                  return IDENTIFIER; }
+
+<PREF>(.|\n)	{ BEGIN(INITIAL);
+		  if(*yytext == EOF)
+                    return EOF;
+
+                  turtle_syntax_error(rdf_parser, "syntax error at '%c'", *yytext);
+                  yyterminate();  }
+
+
+{IRI}[\ \t\v\r\n]*("=")?[\ \t\v\r\n]*"{"   {
+                  raptor_stringbuffer* sb;
+                  unsigned char* uri_string;
+
+                  /* make length just the IRI */
+                  while(yytext[yyleng - 1] != '>')
+                    yyleng--;
+
+                  sb = raptor_new_stringbuffer();
+                  if(!sb)
+                    TURTLE_LEXER_OOM();
+
+                  /* start at yytext + 1 to skip '<' and operate over
+                   * length-2 bytes to skip '<' and '>'
+                   */
+                  if(raptor_stringbuffer_append_turtle_string(sb, (unsigned char*)yytext+1, yyleng-2, '>', (raptor_simple_message_handler)turtle_lexer_syntax_error, rdf_parser, 1)) {
+                    raptor_free_stringbuffer(sb);
+                    YY_FATAL_ERROR_EOF("raptor_stringbuffer_append_turtle_string failed");
+                  }
+                  uri_string = raptor_stringbuffer_as_string(sb);
+
+                  if(!*uri_string)
+                    yylval->uri = raptor_uri_copy(rdf_parser->base_uri);
+                  else
+                    yylval->uri = raptor_new_uri_relative_to_base(rdf_parser->world, rdf_parser->base_uri, uri_string);
+
+                  raptor_free_stringbuffer(sb);
+
+                  if(!yylval->uri)
+                    TURTLE_LEXER_OOM();
+                return GRAPH_NAME_LEFT_CURLY; }
+
+{QNAME}[\ \t\v\r\n]*("=")?[\ \t\v\r\n]*"{"   {
+               while(1) {
+                  int c = yytext[yyleng - 1];
+                  if(c == '{' || c == ' ' || c=='\t' || c == '\v' || c == '\n' ||
+                     c == '=') {
+                    yyleng--;
+                  } else
+                    break;
+                }
+                yytext[yyleng] = '\0';
+
+                yylval->uri = turtle_qname_to_uri(rdf_parser, (unsigned char*)yytext, yyleng);
+                if(!yylval->uri) {
+                  turtle_lexer_error(yyscanner, RAPTOR_LOG_LEVEL_ERROR, "Failed to convert qname %s to URI", yytext);
+                  yyterminate();
+                }
+
+                return GRAPH_NAME_LEFT_CURLY; }
+
+{IRI}   { if(yyleng == 2) 
+                  yylval->uri = raptor_uri_copy(rdf_parser->base_uri);
+                else {
+                  raptor_stringbuffer* sb;
+                  unsigned char* uri_string;
+
+                  yytext[yyleng-1]='\0';
+                  sb = raptor_new_stringbuffer();
+                  if(!sb)
+                    TURTLE_LEXER_OOM();
+                  if(raptor_stringbuffer_append_turtle_string(sb, (unsigned char*)yytext+1, yyleng-1, '>', (raptor_simple_message_handler)turtle_lexer_syntax_error, rdf_parser, 1)) {
+                    raptor_free_stringbuffer(sb);
+                    YY_FATAL_ERROR_EOF("raptor_stringbuffer_append_turtle_string failed");
+                  }
+                  uri_string = raptor_stringbuffer_as_string(sb);
+                  yylval->uri = raptor_new_uri_relative_to_base(rdf_parser->world, rdf_parser->base_uri, uri_string);
+                  if(!yylval->uri) {
+                    raptor_free_stringbuffer(sb);
+                    TURTLE_LEXER_OOM();
+                  }
+                  raptor_free_stringbuffer(sb);
+                }
+                return URI_LITERAL; }
+
+{LANGTAG}	{ yylval->string = turtle_copy_token((unsigned char*)yytext+1, yyleng-1);
+                  if(!yylval->string)
+                    YY_FATAL_ERROR_EOF("turtle_copy_token failed");
+                          return LANGTAG; }
+
+\#[^\r\n]*(\r\n|\r|\n)	{ /* # comment */
+		turtle_parser->lineno++;
+                }
+
+\#[^\r\n]*	{ /* # comment on the last line with no terminating newline */
+                }
+
+.         	{ if(*yytext == EOF)
+                    return EOF;
+
+                  turtle_syntax_error(rdf_parser, "syntax error at '%c'", *yytext);
+                  yyterminate();
+		}
+
+%%
+  /* user code */
+
+int
+yywrap (yyscan_t yyscanner) {
+  return 1;
+}
+
+
+static unsigned char *
+turtle_copy_token(unsigned char *text, size_t len)
+{
+  unsigned char *s;
+  if(!len)
+    len = strlen((const char*)text);
+  s = RAPTOR_MALLOC(unsigned char*, len + 1);
+  if(s) {
+    memcpy(s, text, len);
+    s[len] = '\0';
+  }
+  return s;
+}
+
+
+static unsigned char *
+turtle_copy_string_token(raptor_parser* rdf_parser, 
+                         unsigned char *string, size_t len, int delim)
+{
+  raptor_stringbuffer* sb = NULL;
+  int rc;
+  
+  if(len) {
+    sb = raptor_new_stringbuffer();
+    if(!sb)
+      return NULL;
+    
+    rc = raptor_stringbuffer_append_turtle_string(sb, string, len, delim,
+                                                  (raptor_simple_message_handler)turtle_lexer_syntax_error,
+                                                  rdf_parser, 0);
+    if(rc) {
+      raptor_free_stringbuffer(sb);
+      return NULL;
+    }
+
+    len = raptor_stringbuffer_length(sb);
+  }
+  
+  string = RAPTOR_MALLOC(unsigned char*, len + 1);
+  if(string) {
+    if(sb) 
+      raptor_stringbuffer_copy_to_string(sb, string, len+1);
+    string[len]='\0';
+  }
+
+  if(sb)
+    raptor_free_stringbuffer(sb);
+  
+  return string;
+}
+
+
+void
+turtle_lexer_syntax_error(void* ctx, const char *message, ...)
+{
+  raptor_parser* rdf_parser = (raptor_parser *)ctx;
+  raptor_turtle_parser* turtle_parser = (raptor_turtle_parser*)rdf_parser->context;
+  va_list arguments;
+  
+  rdf_parser->locator.line = turtle_parser->lineno;
+#ifdef RAPTOR_TURTLE_USE_ERROR_COLUMNS
+  rdf_parser->locator.column = turtle_lexer_get_column(yyscanner);
+#endif
+
+  va_start(arguments, message);
+  raptor_parser_log_error_varargs(((raptor_parser*)rdf_parser),
+                                  RAPTOR_LOG_LEVEL_ERROR, message, arguments);
+
+  va_end(arguments);
+}
+
+
+/*
+ * turtle_lexer_error:
+ * @yyscanner: scanner object
+ * @level: log level RAPTOR_LOG_LEVEL_FATAL otherwise error
+ * @message: erro message
+ *
+ * INTERNAL - replacement for the generated error handler.
+ */
+static void turtle_lexer_error(yyscan_t yyscanner,
+                               raptor_log_level level,
+                               yyconst char *message, ...)
+{
+  raptor_parser *rdf_parser = NULL;
+  va_list arguments;
+
+  va_start(arguments, message);
+
+  if(yyscanner)
+    rdf_parser = (raptor_parser*)turtle_lexer_get_extra(yyscanner);
+
+  /* This handles NULL rdf_parser properly */
+  raptor_parser_log_error_varargs(rdf_parser, level, message, arguments);
+
+  va_end(arguments);
+}
+
+
+/* Define LEXER_ALLOC_TRACKING to enable allocated memory tracking
+ * - fixes lexer memory leak when ensure_buffer_stack fails
+ */
+
+#ifdef LEXER_ALLOC_TRACKING
+typedef struct {
+  /* Number of void* slots allocated */
+  int lexer_allocs_size;
+  /* Allocted void* slots follow in memory after this header */
+} lexer_alloc_tracker_header;
+
+/* Initial alloc tracker slot array size - 2 seems to be enough for almost all cases */
+static const int initial_lexer_allocs_size = 2;
+#endif
+
+/*
+ * turtle_lexer_cleanup:
+ * @yyscanner:
+ *
+ * INTERNAL - Clean up unfreed lexer allocs if LEXER_ALLOC_TRACKING is enabled.
+ */
+static void turtle_lexer_cleanup(yyscan_t yyscanner)
+{
+#ifdef LEXER_ALLOC_TRACKING
+  raptor_parser *rdf_parser;
+  lexer_alloc_tracker_header *tracker;
+  void **lexer_allocs;
+  int i;
+
+  if(!yyscanner)
+    return;
+
+  rdf_parser = (raptor_parser *)turtle_lexer_get_extra(yyscanner);
+  if(!rdf_parser)
+    return;
+
+  tracker = (lexer_alloc_tracker_header *)rdf_parser->lexer_user_data;
+  if(!tracker)
+    return;
+  lexer_allocs = (void**)&tracker[1];
+
+  for(i = 0; i < tracker->lexer_allocs_size; ++i) {
+    if(lexer_allocs[i])
+      free(lexer_allocs[i]);
+    lexer_allocs[i] = NULL;
+  }
+  free(rdf_parser->lexer_user_data);
+  rdf_parser->lexer_user_data = NULL;
+#endif
+}
+
+
+/*
+ * turtle_lexer_alloc:
+ * @size
+ * @yyscanner
+ *
+ * INTERNAL - alloc replacement.
+ * Tracks allocated cells if LEXER_ALLOC_TRACKING is enabled.
+ */
+void *turtle_lexer_alloc(yy_size_t size, yyscan_t yyscanner)
+{
+#ifdef LEXER_ALLOC_TRACKING
+  raptor_parser *rdf_parser;
+  lexer_alloc_tracker_header *tracker;
+  void **lexer_allocs;
+  int i;
+  void *ptr;
+
+  /* yyscanner not initialized -> probably initializing yyscanner itself
+   * -> just malloc without tracking
+   */
+  if(!yyscanner)
+    return malloc(size);
+
+  rdf_parser = (raptor_parser *)turtle_lexer_get_extra(yyscanner);
+  if(!rdf_parser)
+    YY_FATAL_ERROR("lexer_alloc: yyscanner extra not initialized");
+
+  /* try to allocate tracker if it does not exist */
+  tracker = (lexer_alloc_tracker_header *)rdf_parser->lexer_user_data;
+  if(!tracker) {
+    /* allocate tracker header + array of void* slots */
+    tracker = (lexer_alloc_tracker_header*)calloc(1, sizeof(lexer_alloc_tracker_header)+initial_lexer_allocs_size*sizeof(void*));
+    if(!tracker)
+      YY_FATAL_ERROR("lexer_alloc: cannot allocate tracker");
+    tracker->lexer_allocs_size = initial_lexer_allocs_size;
+    rdf_parser->lexer_user_data = (void *)tracker;
+  }
+  lexer_allocs = (void**)&tracker[1];
+
+  /* allocate memory */
+  ptr = malloc(size);
+  
+  /* find a free slot for ptr */
+  for(i = 0; i < tracker->lexer_allocs_size; ++i) {
+    if(!lexer_allocs[i]) {
+      lexer_allocs[i] = ptr;
+      break;
+    }
+  }
+
+  /* no free slots -> grow tracker slot array */
+  if(i>=tracker->lexer_allocs_size) {
+    int j;
+    void **dest;
+    tracker = (lexer_alloc_tracker_header*)calloc(1, sizeof(lexer_alloc_tracker_header)+i*2*sizeof(void*));
+    if(!tracker) {
+      if(ptr)
+        free(ptr);
+      YY_FATAL_ERROR("lexer_alloc: cannot grow tracker");
+    }
+    tracker->lexer_allocs_size = i*2;
+    
+    /* copy data from old tracker */
+    dest = (void**)&tracker[1];
+    for(j = 0; j < i; ++j) {
+      dest[j] = lexer_allocs[j];
+    }
+    
+    /* set new item to first free slot */
+    dest[j] = ptr;
+
+    /* free old tracker and replace with new one */
+    free(rdf_parser->lexer_user_data);
+    rdf_parser->lexer_user_data = tracker;
+  }
+
+  return ptr;
+#else
+  return malloc(size);
+#endif
+}
+
+
+/*
+ * turtle_lexer_realloc:
+ *
+ * INTERNAL - realloc replacement
+ * Tracks allocated cells if LEXER_ALLOC_TRACKING is enabled.
+ */
+void *turtle_lexer_realloc(void *ptr, yy_size_t size, yyscan_t yyscanner)
+{
+#ifdef LEXER_ALLOC_TRACKING
+  raptor_parser *rdf_parser;
+  lexer_alloc_tracker_header *tracker;
+  void **lexer_allocs;
+  int i;
+  void *newptr;
+
+  if(!yyscanner)
+    YY_FATAL_ERROR("lexer_realloc: yyscanner not initialized");
+
+  rdf_parser = (raptor_parser *)turtle_lexer_get_extra(yyscanner);
+  if(!rdf_parser)
+    YY_FATAL_ERROR("lexer_realloc: yyscanner extra not initialized");
+
+  tracker = (lexer_alloc_tracker_header *)rdf_parser->lexer_user_data;
+  if(!tracker)
+    YY_FATAL_ERROR("lexer_realloc: no alloc tracker");
+  lexer_allocs = (void**)&tracker[1];
+
+  /* find the old slot for ptr */
+  for(i = 0; i < tracker->lexer_allocs_size; ++i) {
+    if(lexer_allocs[i] == ptr)
+      break;
+  }
+
+  /* no old slot -> error */  
+  if(i>=tracker->lexer_allocs_size)
+    YY_FATAL_ERROR("lexer_realloc: cell not in tracker");
+
+  /* realloc */
+  newptr = realloc((char*)ptr, size);
+
+  /* replace entry in tracker */
+  lexer_allocs[i] = newptr;
+
+  return newptr;
+#else
+  return realloc((char*)ptr, size);
+#endif
+}
+
+
+/*
+ * turtle_lexer_free:
+ *
+ * INTERNAL - free replacement.
+ * Checks for NULL pointer to be freed unlike the default lexer free function.
+ * Tracks allocated cells if LEXER_ALLOC_TRACKING is enabled.
+ */
+void turtle_lexer_free(void *ptr, yyscan_t yyscanner)
+{
+#ifdef LEXER_ALLOC_TRACKING
+  raptor_parser *rdf_parser;
+  lexer_alloc_tracker_header *tracker;
+  void **lexer_allocs;
+  int i;
+
+  /* do not free NULL */
+  if(!ptr)
+    return;
+
+  /* free ptr even if we would encounter an error */
+  free(ptr);
+
+  /* yyscanner is allocated with turtle_lexer_alloc() but it's never stored in the tracker
+   * - we need yyscanner to access the tracker */
+  if(!yyscanner || ptr == yyscanner)
+    return;
+
+  rdf_parser = (raptor_parser *)turtle_lexer_get_extra(yyscanner);
+  if(!rdf_parser)
+    return;
+
+  tracker = (lexer_alloc_tracker_header *)rdf_parser->lexer_user_data;
+  if(!tracker)
+    return;
+  lexer_allocs = (void**)&tracker[1];
+
+  /* find the slot for ptr */
+  for(i = 0; i < tracker->lexer_allocs_size; ++i) {
+    if(lexer_allocs[i] == ptr)
+      break;
+  }
+
+  /* no slot -> error */  
+  if(i>=tracker->lexer_allocs_size)
+    YY_FATAL_ERROR("lexer_free: cell not in tracker");
+
+  /* remove entry from tracker */
+  lexer_allocs[i] = NULL;
+#else
+  if(ptr)
+    free(ptr);
+#endif
+}
+
+
+#ifdef RAPTOR_DEBUG
+
+const char *
+turtle_token_print(raptor_world* world, int token, YYSTYPE *lval)
+{
+  #define TTP_DEBUG_BUFFER_SIZE 2048
+  static char buffer[TTP_DEBUG_BUFFER_SIZE];
+
+  if(!token)
+    return "<<EOF>>";
+  
+  switch(token) {
+    case PREFIX:
+      return "PREFIX";
+
+    case BASE:
+      return "BASE";
+
+    case A:
+      return "A";
+
+    case DOT:
+      return "DOT";
+
+    case COMMA:
+      return "COMMA";
+
+    case SEMICOLON:
+      return "SEMICOLON";
+
+    case LEFT_SQUARE:
+      return "LEFT_SQUARE";
+
+    case RIGHT_SQUARE:
+      return "RIGHT_SQUARE";
+
+    case HAT:
+      return "HAT";
+
+    case STRING_LITERAL:
+      snprintf(buffer, TTP_DEBUG_BUFFER_SIZE, "STRING_LITERAL(%s)",
+               lval->string);
+      return buffer;
+
+    case URI_LITERAL:
+      snprintf(buffer, TTP_DEBUG_BUFFER_SIZE, "URI_LITERAL(%s)", 
+               (lval->uri ? (char*)raptor_uri_as_string(lval->uri) : ""));
+      return buffer;
+
+    case BLANK_LITERAL:
+      snprintf(buffer, TTP_DEBUG_BUFFER_SIZE, "BLANK_LITERAL(%s)",
+               lval->string);
+      return buffer;
+
+    case QNAME_LITERAL:
+      snprintf(buffer, TTP_DEBUG_BUFFER_SIZE, "QNAME_LITERAL(%s)", 
+               (lval->uri ? (char*)raptor_uri_as_string(lval->uri) : ""));
+      return buffer;
+
+    case INTEGER_LITERAL:
+      snprintf(buffer, TTP_DEBUG_BUFFER_SIZE, "INTEGER_LITERAL(%s)",
+               lval->string);
+      return buffer;
+
+    case FLOATING_LITERAL:
+      snprintf(buffer, TTP_DEBUG_BUFFER_SIZE, "FLOATING_LITERAL(%s)",
+               lval->string);
+      return buffer;
+
+    case IDENTIFIER:
+      snprintf(buffer, TTP_DEBUG_BUFFER_SIZE, "IDENTIFIER(%s)", 
+               (lval->string ? (char*)lval->string : ""));
+      return buffer;
+
+    case LANGTAG:
+      snprintf(buffer, TTP_DEBUG_BUFFER_SIZE, "LANGTAG(%s)", 
+               (lval->string ? (char*)lval->string : ""));
+      return buffer;
+
+    case DECIMAL_LITERAL:
+      snprintf(buffer, TTP_DEBUG_BUFFER_SIZE, "DECIMAL_LITERAL(%s)",
+               lval->string);
+      return buffer;
+
+    case ERROR_TOKEN:
+      return "ERROR";
+
+    case LEFT_CURLY:
+      return "{";
+    
+    case RIGHT_CURLY:
+      return "}";
+
+    case GRAPH_NAME_LEFT_CURLY:
+      return "GRAPH_NAME {";
+
+   default:
+     RAPTOR_DEBUG2("UNKNOWN token %d - add a new case\n", token);
+     return "(UNKNOWN)";
+  }
+}
+#endif
+
+
+
+void
+turtle_token_free(raptor_world* world, int token, YYSTYPE *lval)
+{
+  if(!token)
+    return;
+
+  switch(token) {
+    case STRING_LITERAL:
+    case BLANK_LITERAL:
+    case IDENTIFIER:
+      if(lval->string)
+        RAPTOR_FREE(char*, lval->string);
+      break;
+
+    case URI_LITERAL:
+    case QNAME_LITERAL:
+      if(lval->uri)
+        raptor_free_uri(lval->uri);
+      break;
+    default:
+      break;
+  }
+}
+
+
+#ifdef STANDALONE
+
+#define FILE_READ_BUF_SIZE 4096
+
+int
+main(int argc, char *argv[]) 
+{
+  char *turtle_string = NULL;
+  raptor_parser rdf_parser;
+  raptor_turtle_parser turtle_parser;
+  yyscan_t scanner;
+  int token = EOF;
+  YYSTYPE lval;
+  const unsigned char *uri_string;
+  const char *filename = NULL;
+  char *buf = NULL;
+  size_t len;
+  raptor_world* world;
+  FILE *fh;
+  
+  world = raptor_new_world();
+  
+  if(argc > 1) {
+    filename = argv[1];
+    fh = fopen(filename, "r");
+    if(!fh) {
+      fprintf(stderr, "%s: Cannot open file %s - %s\n", argv[0], filename,
+              strerror(errno));
+      exit(1);
+    }
+  } else {
+    filename="<stdin>";
+    fh = (FILE*)stdin;
+  }
+
+  turtle_string = RAPTOR_CALLOC(char*, FILE_READ_BUF_SIZE, 1);
+  fread(turtle_string, FILE_READ_BUF_SIZE, 1, fh);
+  fclose(fh);
+
+  memset(&rdf_parser, 0, sizeof(rdf_parser));
+  memset(&turtle_parser, 0, sizeof(turtle_parser));
+
+  rdf_parser.world = world;
+
+  /* discard namespace errors - caused by not interpreting @prefix 
+   * and hence causing failed qname construction
+   */
+  raptor_namespaces_init(rdf_parser.world, &turtle_parser.namespaces, 0);
+
+  yylex_init(&turtle_parser.scanner);
+  scanner = turtle_parser.scanner;
+
+  len =  strlen(RAPTOR_GOOD_CAST(const char*, turtle_string));
+  buf =  RAPTOR_MALLOC(char*, len + 3);
+  memcpy(buf, turtle_string, len);
+  buf[len] =  ' ';
+  buf[len + 1] =  buf[len + 2] = '\0'; /* YY_END_OF_BUFFER_CHAR; */
+  (void)turtle_lexer__scan_buffer(buf, len + 3, scanner);
+
+  turtle_lexer_set_extra(&rdf_parser, scanner);
+
+  /* Initialise enough of the parser and locator to get error messages */
+  rdf_parser.context = &turtle_parser;
+  turtle_parser.lineno = 1;
+  rdf_parser.locator.file = filename;
+  rdf_parser.locator.column = -1;
+
+  uri_string = raptor_uri_filename_to_uri_string(filename);
+  rdf_parser.base_uri = raptor_new_uri(world, uri_string);
+  RAPTOR_FREE(char*, uri_string);
+  
+  while(1) {
+    memset(&lval, 0, sizeof(YYSTYPE));
+    if(turtle_lexer_get_text(scanner) != NULL)
+      printf("yyinput '%s'\n", turtle_lexer_get_text(scanner));
+    token = yylex(&lval, scanner);
+#ifdef RAPTOR_DEBUG
+    printf("token %s\n", turtle_token_print(world, token, &lval));
+#else
+    printf("token %d\n", token);
+#endif
+    turtle_token_free(world, token, &lval);
+    if(!token || token == EOF || token == ERROR_TOKEN)
+      break;
+  }
+
+  if(buf)
+    RAPTOR_FREE(char*, buf);
+  
+  yylex_destroy(scanner);
+
+  raptor_namespaces_clear(&turtle_parser.namespaces);
+  
+  raptor_free_uri(rdf_parser.base_uri);
+
+  RAPTOR_FREE(char*, turtle_string);
+
+  raptor_free_world(world);
+
+
+  if(token == ERROR_TOKEN)
+    return 1;
+ 
+  return 0;
+}
+#endif