sqlglot.tokens
1from __future__ import annotations 2 3import typing as t 4from enum import auto 5 6from sqlglot.helper import AutoName 7from sqlglot.trie import in_trie, new_trie 8 9 10class TokenType(AutoName): 11 L_PAREN = auto() 12 R_PAREN = auto() 13 L_BRACKET = auto() 14 R_BRACKET = auto() 15 L_BRACE = auto() 16 R_BRACE = auto() 17 COMMA = auto() 18 DOT = auto() 19 DASH = auto() 20 PLUS = auto() 21 COLON = auto() 22 DCOLON = auto() 23 SEMICOLON = auto() 24 STAR = auto() 25 BACKSLASH = auto() 26 SLASH = auto() 27 LT = auto() 28 LTE = auto() 29 GT = auto() 30 GTE = auto() 31 NOT = auto() 32 EQ = auto() 33 NEQ = auto() 34 NULLSAFE_EQ = auto() 35 AND = auto() 36 OR = auto() 37 AMP = auto() 38 DPIPE = auto() 39 PIPE = auto() 40 CARET = auto() 41 TILDA = auto() 42 ARROW = auto() 43 DARROW = auto() 44 FARROW = auto() 45 HASH = auto() 46 HASH_ARROW = auto() 47 DHASH_ARROW = auto() 48 LR_ARROW = auto() 49 LT_AT = auto() 50 AT_GT = auto() 51 DOLLAR = auto() 52 PARAMETER = auto() 53 SESSION_PARAMETER = auto() 54 DAMP = auto() 55 56 BLOCK_START = auto() 57 BLOCK_END = auto() 58 59 SPACE = auto() 60 BREAK = auto() 61 62 STRING = auto() 63 NUMBER = auto() 64 IDENTIFIER = auto() 65 DATABASE = auto() 66 COLUMN = auto() 67 COLUMN_DEF = auto() 68 SCHEMA = auto() 69 TABLE = auto() 70 VAR = auto() 71 BIT_STRING = auto() 72 HEX_STRING = auto() 73 BYTE_STRING = auto() 74 NATIONAL_STRING = auto() 75 RAW_STRING = auto() 76 77 # types 78 BIT = auto() 79 BOOLEAN = auto() 80 TINYINT = auto() 81 UTINYINT = auto() 82 SMALLINT = auto() 83 USMALLINT = auto() 84 INT = auto() 85 UINT = auto() 86 BIGINT = auto() 87 UBIGINT = auto() 88 INT128 = auto() 89 UINT128 = auto() 90 INT256 = auto() 91 UINT256 = auto() 92 FLOAT = auto() 93 DOUBLE = auto() 94 DECIMAL = auto() 95 BIGDECIMAL = auto() 96 CHAR = auto() 97 NCHAR = auto() 98 VARCHAR = auto() 99 NVARCHAR = auto() 100 TEXT = auto() 101 MEDIUMTEXT = auto() 102 LONGTEXT = auto() 103 MEDIUMBLOB = auto() 104 LONGBLOB = auto() 105 BINARY = auto() 106 VARBINARY = auto() 107 JSON = auto() 108 JSONB = auto() 109 TIME = auto() 110 TIMESTAMP = auto() 111 TIMESTAMPTZ = auto() 112 TIMESTAMPLTZ = auto() 113 DATETIME = auto() 114 DATETIME64 = auto() 115 DATE = auto() 116 UUID = auto() 117 GEOGRAPHY = auto() 118 NULLABLE = auto() 119 GEOMETRY = auto() 120 HLLSKETCH = auto() 121 HSTORE = auto() 122 SUPER = auto() 123 SERIAL = auto() 124 SMALLSERIAL = auto() 125 BIGSERIAL = auto() 126 XML = auto() 127 UNIQUEIDENTIFIER = auto() 128 MONEY = auto() 129 SMALLMONEY = auto() 130 ROWVERSION = auto() 131 IMAGE = auto() 132 VARIANT = auto() 133 OBJECT = auto() 134 INET = auto() 135 136 # keywords 137 ALIAS = auto() 138 ALTER = auto() 139 ALWAYS = auto() 140 ALL = auto() 141 ANTI = auto() 142 ANY = auto() 143 APPLY = auto() 144 ARRAY = auto() 145 ASC = auto() 146 ASOF = auto() 147 AUTO_INCREMENT = auto() 148 BEGIN = auto() 149 BETWEEN = auto() 150 CACHE = auto() 151 CASE = auto() 152 CHARACTER_SET = auto() 153 COLLATE = auto() 154 COMMAND = auto() 155 COMMENT = auto() 156 COMMIT = auto() 157 CONSTRAINT = auto() 158 CREATE = auto() 159 CROSS = auto() 160 CUBE = auto() 161 CURRENT_DATE = auto() 162 CURRENT_DATETIME = auto() 163 CURRENT_TIME = auto() 164 CURRENT_TIMESTAMP = auto() 165 CURRENT_USER = auto() 166 DEFAULT = auto() 167 DELETE = auto() 168 DESC = auto() 169 DESCRIBE = auto() 170 DISTINCT = auto() 171 DIV = auto() 172 DROP = auto() 173 ELSE = auto() 174 END = auto() 175 ESCAPE = auto() 176 EXCEPT = auto() 177 EXECUTE = auto() 178 EXISTS = auto() 179 FALSE = auto() 180 FETCH = auto() 181 FILTER = auto() 182 FINAL = auto() 183 FIRST = auto() 184 FOR = auto() 185 FOREIGN_KEY = auto() 186 FORMAT = auto() 187 FROM = auto() 188 FULL = auto() 189 FUNCTION = auto() 190 GLOB = auto() 191 GLOBAL = auto() 192 GROUP_BY = auto() 193 GROUPING_SETS = auto() 194 HAVING = auto() 195 HINT = auto() 196 IF = auto() 197 ILIKE = auto() 198 ILIKE_ANY = auto() 199 IN = auto() 200 INDEX = auto() 201 INNER = auto() 202 INSERT = auto() 203 INTERSECT = auto() 204 INTERVAL = auto() 205 INTO = auto() 206 INTRODUCER = auto() 207 IRLIKE = auto() 208 IS = auto() 209 ISNULL = auto() 210 JOIN = auto() 211 JOIN_MARKER = auto() 212 KEEP = auto() 213 LANGUAGE = auto() 214 LATERAL = auto() 215 LEFT = auto() 216 LIKE = auto() 217 LIKE_ANY = auto() 218 LIMIT = auto() 219 LOAD = auto() 220 LOCK = auto() 221 MAP = auto() 222 MATCH_RECOGNIZE = auto() 223 MERGE = auto() 224 MOD = auto() 225 NATURAL = auto() 226 NEXT = auto() 227 NEXT_VALUE_FOR = auto() 228 NOTNULL = auto() 229 NULL = auto() 230 OFFSET = auto() 231 ON = auto() 232 ORDER_BY = auto() 233 ORDERED = auto() 234 ORDINALITY = auto() 235 OUTER = auto() 236 OVER = auto() 237 OVERLAPS = auto() 238 OVERWRITE = auto() 239 PARTITION = auto() 240 PARTITION_BY = auto() 241 PERCENT = auto() 242 PIVOT = auto() 243 PLACEHOLDER = auto() 244 PRAGMA = auto() 245 PRIMARY_KEY = auto() 246 PROCEDURE = auto() 247 PROPERTIES = auto() 248 PSEUDO_TYPE = auto() 249 QUALIFY = auto() 250 QUOTE = auto() 251 RANGE = auto() 252 RECURSIVE = auto() 253 REPLACE = auto() 254 RETURNING = auto() 255 REFERENCES = auto() 256 RIGHT = auto() 257 RLIKE = auto() 258 ROLLBACK = auto() 259 ROLLUP = auto() 260 ROW = auto() 261 ROWS = auto() 262 SELECT = auto() 263 SEMI = auto() 264 SEPARATOR = auto() 265 SERDE_PROPERTIES = auto() 266 SET = auto() 267 SETTINGS = auto() 268 SHOW = auto() 269 SIMILAR_TO = auto() 270 SOME = auto() 271 STRUCT = auto() 272 TABLE_SAMPLE = auto() 273 TEMPORARY = auto() 274 TOP = auto() 275 THEN = auto() 276 TRUE = auto() 277 UNCACHE = auto() 278 UNION = auto() 279 UNNEST = auto() 280 UNPIVOT = auto() 281 UPDATE = auto() 282 USE = auto() 283 USING = auto() 284 VALUES = auto() 285 VIEW = auto() 286 VOLATILE = auto() 287 WHEN = auto() 288 WHERE = auto() 289 WINDOW = auto() 290 WITH = auto() 291 UNIQUE = auto() 292 293 294class Token: 295 __slots__ = ("token_type", "text", "line", "col", "start", "end", "comments") 296 297 @classmethod 298 def number(cls, number: int) -> Token: 299 """Returns a NUMBER token with `number` as its text.""" 300 return cls(TokenType.NUMBER, str(number)) 301 302 @classmethod 303 def string(cls, string: str) -> Token: 304 """Returns a STRING token with `string` as its text.""" 305 return cls(TokenType.STRING, string) 306 307 @classmethod 308 def identifier(cls, identifier: str) -> Token: 309 """Returns an IDENTIFIER token with `identifier` as its text.""" 310 return cls(TokenType.IDENTIFIER, identifier) 311 312 @classmethod 313 def var(cls, var: str) -> Token: 314 """Returns an VAR token with `var` as its text.""" 315 return cls(TokenType.VAR, var) 316 317 def __init__( 318 self, 319 token_type: TokenType, 320 text: str, 321 line: int = 1, 322 col: int = 1, 323 start: int = 0, 324 end: int = 0, 325 comments: t.List[str] = [], 326 ) -> None: 327 """Token initializer. 328 329 Args: 330 token_type: The TokenType Enum. 331 text: The text of the token. 332 line: The line that the token ends on. 333 col: The column that the token ends on. 334 start: The start index of the token. 335 end: The ending index of the token. 336 """ 337 self.token_type = token_type 338 self.text = text 339 self.line = line 340 self.col = col 341 self.start = start 342 self.end = end 343 self.comments = comments 344 345 def __repr__(self) -> str: 346 attributes = ", ".join(f"{k}: {getattr(self, k)}" for k in self.__slots__) 347 return f"<Token {attributes}>" 348 349 350class _Tokenizer(type): 351 def __new__(cls, clsname, bases, attrs): 352 klass = super().__new__(cls, clsname, bases, attrs) 353 354 def _convert_quotes(arr: t.List[str | t.Tuple[str, str]]) -> t.Dict[str, str]: 355 return dict( 356 (item, item) if isinstance(item, str) else (item[0], item[1]) for item in arr 357 ) 358 359 def _quotes_to_format( 360 token_type: TokenType, arr: t.List[str | t.Tuple[str, str]] 361 ) -> t.Dict[str, t.Tuple[str, TokenType]]: 362 return {k: (v, token_type) for k, v in _convert_quotes(arr).items()} 363 364 klass._QUOTES = _convert_quotes(klass.QUOTES) 365 klass._IDENTIFIERS = _convert_quotes(klass.IDENTIFIERS) 366 367 klass._FORMAT_STRINGS = { 368 **{ 369 p + s: (e, TokenType.NATIONAL_STRING) 370 for s, e in klass._QUOTES.items() 371 for p in ("n", "N") 372 }, 373 **_quotes_to_format(TokenType.BIT_STRING, klass.BIT_STRINGS), 374 **_quotes_to_format(TokenType.BYTE_STRING, klass.BYTE_STRINGS), 375 **_quotes_to_format(TokenType.HEX_STRING, klass.HEX_STRINGS), 376 **_quotes_to_format(TokenType.RAW_STRING, klass.RAW_STRINGS), 377 } 378 379 klass._STRING_ESCAPES = set(klass.STRING_ESCAPES) 380 klass._IDENTIFIER_ESCAPES = set(klass.IDENTIFIER_ESCAPES) 381 klass._COMMENTS = dict( 382 (comment, None) if isinstance(comment, str) else (comment[0], comment[1]) 383 for comment in klass.COMMENTS 384 ) 385 386 klass.KEYWORD_TRIE = new_trie( 387 key.upper() 388 for key in ( 389 *klass.KEYWORDS, 390 *klass._COMMENTS, 391 *klass._QUOTES, 392 *klass._FORMAT_STRINGS, 393 ) 394 if " " in key or any(single in key for single in klass.SINGLE_TOKENS) 395 ) 396 397 return klass 398 399 400class Tokenizer(metaclass=_Tokenizer): 401 SINGLE_TOKENS = { 402 "(": TokenType.L_PAREN, 403 ")": TokenType.R_PAREN, 404 "[": TokenType.L_BRACKET, 405 "]": TokenType.R_BRACKET, 406 "{": TokenType.L_BRACE, 407 "}": TokenType.R_BRACE, 408 "&": TokenType.AMP, 409 "^": TokenType.CARET, 410 ":": TokenType.COLON, 411 ",": TokenType.COMMA, 412 ".": TokenType.DOT, 413 "-": TokenType.DASH, 414 "=": TokenType.EQ, 415 ">": TokenType.GT, 416 "<": TokenType.LT, 417 "%": TokenType.MOD, 418 "!": TokenType.NOT, 419 "|": TokenType.PIPE, 420 "+": TokenType.PLUS, 421 ";": TokenType.SEMICOLON, 422 "/": TokenType.SLASH, 423 "\\": TokenType.BACKSLASH, 424 "*": TokenType.STAR, 425 "~": TokenType.TILDA, 426 "?": TokenType.PLACEHOLDER, 427 "@": TokenType.PARAMETER, 428 # used for breaking a var like x'y' but nothing else 429 # the token type doesn't matter 430 "'": TokenType.QUOTE, 431 "`": TokenType.IDENTIFIER, 432 '"': TokenType.IDENTIFIER, 433 "#": TokenType.HASH, 434 } 435 436 BIT_STRINGS: t.List[str | t.Tuple[str, str]] = [] 437 BYTE_STRINGS: t.List[str | t.Tuple[str, str]] = [] 438 HEX_STRINGS: t.List[str | t.Tuple[str, str]] = [] 439 RAW_STRINGS: t.List[str | t.Tuple[str, str]] = [] 440 IDENTIFIERS: t.List[str | t.Tuple[str, str]] = ['"'] 441 IDENTIFIER_ESCAPES = ['"'] 442 QUOTES: t.List[t.Tuple[str, str] | str] = ["'"] 443 STRING_ESCAPES = ["'"] 444 VAR_SINGLE_TOKENS: t.Set[str] = set() 445 446 _COMMENTS: t.Dict[str, str] = {} 447 _FORMAT_STRINGS: t.Dict[str, t.Tuple[str, TokenType]] = {} 448 _IDENTIFIERS: t.Dict[str, str] = {} 449 _IDENTIFIER_ESCAPES: t.Set[str] = set() 450 _QUOTES: t.Dict[str, str] = {} 451 _STRING_ESCAPES: t.Set[str] = set() 452 453 KEYWORDS: t.Dict[t.Optional[str], TokenType] = { 454 **{f"{{%{postfix}": TokenType.BLOCK_START for postfix in ("", "+", "-")}, 455 **{f"{prefix}%}}": TokenType.BLOCK_END for prefix in ("", "+", "-")}, 456 "{{+": TokenType.BLOCK_START, 457 "{{-": TokenType.BLOCK_START, 458 "+}}": TokenType.BLOCK_END, 459 "-}}": TokenType.BLOCK_END, 460 "/*+": TokenType.HINT, 461 "==": TokenType.EQ, 462 "::": TokenType.DCOLON, 463 "||": TokenType.DPIPE, 464 ">=": TokenType.GTE, 465 "<=": TokenType.LTE, 466 "<>": TokenType.NEQ, 467 "!=": TokenType.NEQ, 468 "<=>": TokenType.NULLSAFE_EQ, 469 "->": TokenType.ARROW, 470 "->>": TokenType.DARROW, 471 "=>": TokenType.FARROW, 472 "#>": TokenType.HASH_ARROW, 473 "#>>": TokenType.DHASH_ARROW, 474 "<->": TokenType.LR_ARROW, 475 "&&": TokenType.DAMP, 476 "ALL": TokenType.ALL, 477 "ALWAYS": TokenType.ALWAYS, 478 "AND": TokenType.AND, 479 "ANTI": TokenType.ANTI, 480 "ANY": TokenType.ANY, 481 "ASC": TokenType.ASC, 482 "AS": TokenType.ALIAS, 483 "AUTOINCREMENT": TokenType.AUTO_INCREMENT, 484 "AUTO_INCREMENT": TokenType.AUTO_INCREMENT, 485 "BEGIN": TokenType.BEGIN, 486 "BETWEEN": TokenType.BETWEEN, 487 "CACHE": TokenType.CACHE, 488 "UNCACHE": TokenType.UNCACHE, 489 "CASE": TokenType.CASE, 490 "CHARACTER SET": TokenType.CHARACTER_SET, 491 "COLLATE": TokenType.COLLATE, 492 "COLUMN": TokenType.COLUMN, 493 "COMMIT": TokenType.COMMIT, 494 "CONSTRAINT": TokenType.CONSTRAINT, 495 "CREATE": TokenType.CREATE, 496 "CROSS": TokenType.CROSS, 497 "CUBE": TokenType.CUBE, 498 "CURRENT_DATE": TokenType.CURRENT_DATE, 499 "CURRENT_TIME": TokenType.CURRENT_TIME, 500 "CURRENT_TIMESTAMP": TokenType.CURRENT_TIMESTAMP, 501 "CURRENT_USER": TokenType.CURRENT_USER, 502 "DATABASE": TokenType.DATABASE, 503 "DEFAULT": TokenType.DEFAULT, 504 "DELETE": TokenType.DELETE, 505 "DESC": TokenType.DESC, 506 "DESCRIBE": TokenType.DESCRIBE, 507 "DISTINCT": TokenType.DISTINCT, 508 "DIV": TokenType.DIV, 509 "DROP": TokenType.DROP, 510 "ELSE": TokenType.ELSE, 511 "END": TokenType.END, 512 "ESCAPE": TokenType.ESCAPE, 513 "EXCEPT": TokenType.EXCEPT, 514 "EXECUTE": TokenType.EXECUTE, 515 "EXISTS": TokenType.EXISTS, 516 "FALSE": TokenType.FALSE, 517 "FETCH": TokenType.FETCH, 518 "FILTER": TokenType.FILTER, 519 "FIRST": TokenType.FIRST, 520 "FULL": TokenType.FULL, 521 "FUNCTION": TokenType.FUNCTION, 522 "FOR": TokenType.FOR, 523 "FOREIGN KEY": TokenType.FOREIGN_KEY, 524 "FORMAT": TokenType.FORMAT, 525 "FROM": TokenType.FROM, 526 "GEOGRAPHY": TokenType.GEOGRAPHY, 527 "GEOMETRY": TokenType.GEOMETRY, 528 "GLOB": TokenType.GLOB, 529 "GROUP BY": TokenType.GROUP_BY, 530 "GROUPING SETS": TokenType.GROUPING_SETS, 531 "HAVING": TokenType.HAVING, 532 "IF": TokenType.IF, 533 "ILIKE": TokenType.ILIKE, 534 "IN": TokenType.IN, 535 "INDEX": TokenType.INDEX, 536 "INET": TokenType.INET, 537 "INNER": TokenType.INNER, 538 "INSERT": TokenType.INSERT, 539 "INTERVAL": TokenType.INTERVAL, 540 "INTERSECT": TokenType.INTERSECT, 541 "INTO": TokenType.INTO, 542 "IS": TokenType.IS, 543 "ISNULL": TokenType.ISNULL, 544 "JOIN": TokenType.JOIN, 545 "KEEP": TokenType.KEEP, 546 "LATERAL": TokenType.LATERAL, 547 "LEFT": TokenType.LEFT, 548 "LIKE": TokenType.LIKE, 549 "LIMIT": TokenType.LIMIT, 550 "LOAD": TokenType.LOAD, 551 "LOCK": TokenType.LOCK, 552 "MERGE": TokenType.MERGE, 553 "NATURAL": TokenType.NATURAL, 554 "NEXT": TokenType.NEXT, 555 "NEXT VALUE FOR": TokenType.NEXT_VALUE_FOR, 556 "NOT": TokenType.NOT, 557 "NOTNULL": TokenType.NOTNULL, 558 "NULL": TokenType.NULL, 559 "OBJECT": TokenType.OBJECT, 560 "OFFSET": TokenType.OFFSET, 561 "ON": TokenType.ON, 562 "OR": TokenType.OR, 563 "ORDER BY": TokenType.ORDER_BY, 564 "ORDINALITY": TokenType.ORDINALITY, 565 "OUTER": TokenType.OUTER, 566 "OVER": TokenType.OVER, 567 "OVERLAPS": TokenType.OVERLAPS, 568 "OVERWRITE": TokenType.OVERWRITE, 569 "PARTITION": TokenType.PARTITION, 570 "PARTITION BY": TokenType.PARTITION_BY, 571 "PARTITIONED BY": TokenType.PARTITION_BY, 572 "PARTITIONED_BY": TokenType.PARTITION_BY, 573 "PERCENT": TokenType.PERCENT, 574 "PIVOT": TokenType.PIVOT, 575 "PRAGMA": TokenType.PRAGMA, 576 "PRIMARY KEY": TokenType.PRIMARY_KEY, 577 "PROCEDURE": TokenType.PROCEDURE, 578 "QUALIFY": TokenType.QUALIFY, 579 "RANGE": TokenType.RANGE, 580 "RECURSIVE": TokenType.RECURSIVE, 581 "REGEXP": TokenType.RLIKE, 582 "REPLACE": TokenType.REPLACE, 583 "REFERENCES": TokenType.REFERENCES, 584 "RIGHT": TokenType.RIGHT, 585 "RLIKE": TokenType.RLIKE, 586 "ROLLBACK": TokenType.ROLLBACK, 587 "ROLLUP": TokenType.ROLLUP, 588 "ROW": TokenType.ROW, 589 "ROWS": TokenType.ROWS, 590 "SCHEMA": TokenType.SCHEMA, 591 "SELECT": TokenType.SELECT, 592 "SEMI": TokenType.SEMI, 593 "SET": TokenType.SET, 594 "SETTINGS": TokenType.SETTINGS, 595 "SHOW": TokenType.SHOW, 596 "SIMILAR TO": TokenType.SIMILAR_TO, 597 "SOME": TokenType.SOME, 598 "TABLE": TokenType.TABLE, 599 "TABLESAMPLE": TokenType.TABLE_SAMPLE, 600 "TEMP": TokenType.TEMPORARY, 601 "TEMPORARY": TokenType.TEMPORARY, 602 "THEN": TokenType.THEN, 603 "TRUE": TokenType.TRUE, 604 "UNION": TokenType.UNION, 605 "UNNEST": TokenType.UNNEST, 606 "UNPIVOT": TokenType.UNPIVOT, 607 "UPDATE": TokenType.UPDATE, 608 "USE": TokenType.USE, 609 "USING": TokenType.USING, 610 "UUID": TokenType.UUID, 611 "VALUES": TokenType.VALUES, 612 "VIEW": TokenType.VIEW, 613 "VOLATILE": TokenType.VOLATILE, 614 "WHEN": TokenType.WHEN, 615 "WHERE": TokenType.WHERE, 616 "WINDOW": TokenType.WINDOW, 617 "WITH": TokenType.WITH, 618 "APPLY": TokenType.APPLY, 619 "ARRAY": TokenType.ARRAY, 620 "BIT": TokenType.BIT, 621 "BOOL": TokenType.BOOLEAN, 622 "BOOLEAN": TokenType.BOOLEAN, 623 "BYTE": TokenType.TINYINT, 624 "TINYINT": TokenType.TINYINT, 625 "SHORT": TokenType.SMALLINT, 626 "SMALLINT": TokenType.SMALLINT, 627 "INT2": TokenType.SMALLINT, 628 "INTEGER": TokenType.INT, 629 "INT": TokenType.INT, 630 "INT4": TokenType.INT, 631 "LONG": TokenType.BIGINT, 632 "BIGINT": TokenType.BIGINT, 633 "INT8": TokenType.BIGINT, 634 "DEC": TokenType.DECIMAL, 635 "DECIMAL": TokenType.DECIMAL, 636 "BIGDECIMAL": TokenType.BIGDECIMAL, 637 "BIGNUMERIC": TokenType.BIGDECIMAL, 638 "MAP": TokenType.MAP, 639 "NULLABLE": TokenType.NULLABLE, 640 "NUMBER": TokenType.DECIMAL, 641 "NUMERIC": TokenType.DECIMAL, 642 "FIXED": TokenType.DECIMAL, 643 "REAL": TokenType.FLOAT, 644 "FLOAT": TokenType.FLOAT, 645 "FLOAT4": TokenType.FLOAT, 646 "FLOAT8": TokenType.DOUBLE, 647 "DOUBLE": TokenType.DOUBLE, 648 "DOUBLE PRECISION": TokenType.DOUBLE, 649 "JSON": TokenType.JSON, 650 "CHAR": TokenType.CHAR, 651 "CHARACTER": TokenType.CHAR, 652 "NCHAR": TokenType.NCHAR, 653 "VARCHAR": TokenType.VARCHAR, 654 "VARCHAR2": TokenType.VARCHAR, 655 "NVARCHAR": TokenType.NVARCHAR, 656 "NVARCHAR2": TokenType.NVARCHAR, 657 "STR": TokenType.TEXT, 658 "STRING": TokenType.TEXT, 659 "TEXT": TokenType.TEXT, 660 "CLOB": TokenType.TEXT, 661 "LONGVARCHAR": TokenType.TEXT, 662 "BINARY": TokenType.BINARY, 663 "BLOB": TokenType.VARBINARY, 664 "BYTEA": TokenType.VARBINARY, 665 "VARBINARY": TokenType.VARBINARY, 666 "TIME": TokenType.TIME, 667 "TIMESTAMP": TokenType.TIMESTAMP, 668 "TIMESTAMPTZ": TokenType.TIMESTAMPTZ, 669 "TIMESTAMPLTZ": TokenType.TIMESTAMPLTZ, 670 "DATE": TokenType.DATE, 671 "DATETIME": TokenType.DATETIME, 672 "UNIQUE": TokenType.UNIQUE, 673 "STRUCT": TokenType.STRUCT, 674 "VARIANT": TokenType.VARIANT, 675 "ALTER": TokenType.ALTER, 676 "ANALYZE": TokenType.COMMAND, 677 "CALL": TokenType.COMMAND, 678 "COMMENT": TokenType.COMMENT, 679 "COPY": TokenType.COMMAND, 680 "EXPLAIN": TokenType.COMMAND, 681 "GRANT": TokenType.COMMAND, 682 "OPTIMIZE": TokenType.COMMAND, 683 "PREPARE": TokenType.COMMAND, 684 "TRUNCATE": TokenType.COMMAND, 685 "VACUUM": TokenType.COMMAND, 686 } 687 688 WHITE_SPACE: t.Dict[t.Optional[str], TokenType] = { 689 " ": TokenType.SPACE, 690 "\t": TokenType.SPACE, 691 "\n": TokenType.BREAK, 692 "\r": TokenType.BREAK, 693 "\r\n": TokenType.BREAK, 694 } 695 696 COMMANDS = { 697 TokenType.COMMAND, 698 TokenType.EXECUTE, 699 TokenType.FETCH, 700 TokenType.SHOW, 701 } 702 703 COMMAND_PREFIX_TOKENS = {TokenType.SEMICOLON, TokenType.BEGIN} 704 705 # handle numeric literals like in hive (3L = BIGINT) 706 NUMERIC_LITERALS: t.Dict[str, str] = {} 707 ENCODE: t.Optional[str] = None 708 709 COMMENTS = ["--", ("/*", "*/"), ("{#", "#}")] 710 KEYWORD_TRIE: t.Dict = {} # autofilled 711 712 IDENTIFIER_CAN_START_WITH_DIGIT = False 713 714 __slots__ = ( 715 "sql", 716 "size", 717 "tokens", 718 "_start", 719 "_current", 720 "_line", 721 "_col", 722 "_comments", 723 "_char", 724 "_end", 725 "_peek", 726 "_prev_token_line", 727 ) 728 729 def __init__(self) -> None: 730 self.reset() 731 732 def reset(self) -> None: 733 self.sql = "" 734 self.size = 0 735 self.tokens: t.List[Token] = [] 736 self._start = 0 737 self._current = 0 738 self._line = 1 739 self._col = 0 740 self._comments: t.List[str] = [] 741 742 self._char = "" 743 self._end = False 744 self._peek = "" 745 self._prev_token_line = -1 746 747 def tokenize(self, sql: str) -> t.List[Token]: 748 """Returns a list of tokens corresponding to the SQL string `sql`.""" 749 self.reset() 750 self.sql = sql 751 self.size = len(sql) 752 753 try: 754 self._scan() 755 except Exception as e: 756 start = max(self._current - 50, 0) 757 end = min(self._current + 50, self.size - 1) 758 context = self.sql[start:end] 759 raise ValueError(f"Error tokenizing '{context}'") from e 760 761 return self.tokens 762 763 def _scan(self, until: t.Optional[t.Callable] = None) -> None: 764 while self.size and not self._end: 765 self._start = self._current 766 self._advance() 767 768 if self._char is None: 769 break 770 771 if self._char not in self.WHITE_SPACE: 772 if self._char.isdigit(): 773 self._scan_number() 774 elif self._char in self._IDENTIFIERS: 775 self._scan_identifier(self._IDENTIFIERS[self._char]) 776 else: 777 self._scan_keywords() 778 779 if until and until(): 780 break 781 782 if self.tokens and self._comments: 783 self.tokens[-1].comments.extend(self._comments) 784 785 def _chars(self, size: int) -> str: 786 if size == 1: 787 return self._char 788 789 start = self._current - 1 790 end = start + size 791 792 return self.sql[start:end] if end <= self.size else "" 793 794 def _advance(self, i: int = 1, alnum: bool = False) -> None: 795 if self.WHITE_SPACE.get(self._char) is TokenType.BREAK: 796 self._col = 1 797 self._line += 1 798 else: 799 self._col += i 800 801 self._current += i 802 self._end = self._current >= self.size 803 self._char = self.sql[self._current - 1] 804 self._peek = "" if self._end else self.sql[self._current] 805 806 if alnum and self._char.isalnum(): 807 # Here we use local variables instead of attributes for better performance 808 _col = self._col 809 _current = self._current 810 _end = self._end 811 _peek = self._peek 812 813 while _peek.isalnum(): 814 _col += 1 815 _current += 1 816 _end = _current >= self.size 817 _peek = "" if _end else self.sql[_current] 818 819 self._col = _col 820 self._current = _current 821 self._end = _end 822 self._peek = _peek 823 self._char = self.sql[_current - 1] 824 825 @property 826 def _text(self) -> str: 827 return self.sql[self._start : self._current] 828 829 def _add(self, token_type: TokenType, text: t.Optional[str] = None) -> None: 830 self._prev_token_line = self._line 831 self.tokens.append( 832 Token( 833 token_type, 834 text=self._text if text is None else text, 835 line=self._line, 836 col=self._col, 837 start=self._start, 838 end=self._current - 1, 839 comments=self._comments, 840 ) 841 ) 842 self._comments = [] 843 844 # If we have either a semicolon or a begin token before the command's token, we'll parse 845 # whatever follows the command's token as a string 846 if ( 847 token_type in self.COMMANDS 848 and self._peek != ";" 849 and (len(self.tokens) == 1 or self.tokens[-2].token_type in self.COMMAND_PREFIX_TOKENS) 850 ): 851 start = self._current 852 tokens = len(self.tokens) 853 self._scan(lambda: self._peek == ";") 854 self.tokens = self.tokens[:tokens] 855 text = self.sql[start : self._current].strip() 856 if text: 857 self._add(TokenType.STRING, text) 858 859 def _scan_keywords(self) -> None: 860 size = 0 861 word = None 862 chars = self._text 863 char = chars 864 prev_space = False 865 skip = False 866 trie = self.KEYWORD_TRIE 867 single_token = char in self.SINGLE_TOKENS 868 869 while chars: 870 if skip: 871 result = 1 872 else: 873 result, trie = in_trie(trie, char.upper()) 874 875 if result == 0: 876 break 877 if result == 2: 878 word = chars 879 880 size += 1 881 end = self._current - 1 + size 882 883 if end < self.size: 884 char = self.sql[end] 885 single_token = single_token or char in self.SINGLE_TOKENS 886 is_space = char in self.WHITE_SPACE 887 888 if not is_space or not prev_space: 889 if is_space: 890 char = " " 891 chars += char 892 prev_space = is_space 893 skip = False 894 else: 895 skip = True 896 else: 897 char = "" 898 chars = " " 899 900 word = None if not single_token and chars[-1] not in self.WHITE_SPACE else word 901 902 if not word: 903 if self._char in self.SINGLE_TOKENS: 904 self._add(self.SINGLE_TOKENS[self._char], text=self._char) 905 return 906 self._scan_var() 907 return 908 909 if self._scan_string(word): 910 return 911 if self._scan_comment(word): 912 return 913 914 self._advance(size - 1) 915 word = word.upper() 916 self._add(self.KEYWORDS[word], text=word) 917 918 def _scan_comment(self, comment_start: str) -> bool: 919 if comment_start not in self._COMMENTS: 920 return False 921 922 comment_start_line = self._line 923 comment_start_size = len(comment_start) 924 comment_end = self._COMMENTS[comment_start] 925 926 if comment_end: 927 # Skip the comment's start delimiter 928 self._advance(comment_start_size) 929 930 comment_end_size = len(comment_end) 931 while not self._end and self._chars(comment_end_size) != comment_end: 932 self._advance(alnum=True) 933 934 self._comments.append(self._text[comment_start_size : -comment_end_size + 1]) 935 self._advance(comment_end_size - 1) 936 else: 937 while not self._end and not self.WHITE_SPACE.get(self._peek) is TokenType.BREAK: 938 self._advance(alnum=True) 939 self._comments.append(self._text[comment_start_size:]) 940 941 # Leading comment is attached to the succeeding token, whilst trailing comment to the preceding. 942 # Multiple consecutive comments are preserved by appending them to the current comments list. 943 if comment_start_line == self._prev_token_line: 944 self.tokens[-1].comments.extend(self._comments) 945 self._comments = [] 946 self._prev_token_line = self._line 947 948 return True 949 950 def _scan_number(self) -> None: 951 if self._char == "0": 952 peek = self._peek.upper() 953 if peek == "B": 954 return self._scan_bits() if self.BIT_STRINGS else self._add(TokenType.NUMBER) 955 elif peek == "X": 956 return self._scan_hex() if self.HEX_STRINGS else self._add(TokenType.NUMBER) 957 958 decimal = False 959 scientific = 0 960 961 while True: 962 if self._peek.isdigit(): 963 self._advance() 964 elif self._peek == "." and not decimal: 965 decimal = True 966 self._advance() 967 elif self._peek in ("-", "+") and scientific == 1: 968 scientific += 1 969 self._advance() 970 elif self._peek.upper() == "E" and not scientific: 971 scientific += 1 972 self._advance() 973 elif self._peek.isidentifier(): 974 number_text = self._text 975 literal = "" 976 977 while self._peek.strip() and self._peek not in self.SINGLE_TOKENS: 978 literal += self._peek.upper() 979 self._advance() 980 981 token_type = self.KEYWORDS.get(self.NUMERIC_LITERALS.get(literal)) 982 983 if token_type: 984 self._add(TokenType.NUMBER, number_text) 985 self._add(TokenType.DCOLON, "::") 986 return self._add(token_type, literal) 987 elif self.IDENTIFIER_CAN_START_WITH_DIGIT: 988 return self._add(TokenType.VAR) 989 990 self._add(TokenType.NUMBER, number_text) 991 return self._advance(-len(literal)) 992 else: 993 return self._add(TokenType.NUMBER) 994 995 def _scan_bits(self) -> None: 996 self._advance() 997 value = self._extract_value() 998 try: 999 # If `value` can't be converted to a binary, fallback to tokenizing it as an identifier 1000 int(value, 2) 1001 self._add(TokenType.BIT_STRING, value[2:]) # Drop the 0b 1002 except ValueError: 1003 self._add(TokenType.IDENTIFIER) 1004 1005 def _scan_hex(self) -> None: 1006 self._advance() 1007 value = self._extract_value() 1008 try: 1009 # If `value` can't be converted to a hex, fallback to tokenizing it as an identifier 1010 int(value, 16) 1011 self._add(TokenType.HEX_STRING, value[2:]) # Drop the 0x 1012 except ValueError: 1013 self._add(TokenType.IDENTIFIER) 1014 1015 def _extract_value(self) -> str: 1016 while True: 1017 char = self._peek.strip() 1018 if char and char not in self.SINGLE_TOKENS: 1019 self._advance(alnum=True) 1020 else: 1021 break 1022 1023 return self._text 1024 1025 def _scan_string(self, start: str) -> bool: 1026 base = None 1027 token_type = TokenType.STRING 1028 1029 if start in self._QUOTES: 1030 end = self._QUOTES[start] 1031 elif start in self._FORMAT_STRINGS: 1032 end, token_type = self._FORMAT_STRINGS[start] 1033 1034 if token_type == TokenType.HEX_STRING: 1035 base = 16 1036 elif token_type == TokenType.BIT_STRING: 1037 base = 2 1038 else: 1039 return False 1040 1041 self._advance(len(start)) 1042 text = self._extract_string(end) 1043 1044 if base: 1045 try: 1046 int(text, base) 1047 except: 1048 raise RuntimeError( 1049 f"Numeric string contains invalid characters from {self._line}:{self._start}" 1050 ) 1051 else: 1052 text = text.encode(self.ENCODE).decode(self.ENCODE) if self.ENCODE else text 1053 1054 self._add(token_type, text) 1055 return True 1056 1057 def _scan_identifier(self, identifier_end: str) -> None: 1058 self._advance() 1059 text = self._extract_string(identifier_end, self._IDENTIFIER_ESCAPES) 1060 self._add(TokenType.IDENTIFIER, text) 1061 1062 def _scan_var(self) -> None: 1063 while True: 1064 char = self._peek.strip() 1065 if char and (char in self.VAR_SINGLE_TOKENS or char not in self.SINGLE_TOKENS): 1066 self._advance(alnum=True) 1067 else: 1068 break 1069 1070 self._add( 1071 TokenType.VAR 1072 if self.tokens and self.tokens[-1].token_type == TokenType.PARAMETER 1073 else self.KEYWORDS.get(self._text.upper(), TokenType.VAR) 1074 ) 1075 1076 def _extract_string(self, delimiter: str, escapes=None) -> str: 1077 text = "" 1078 delim_size = len(delimiter) 1079 escapes = self._STRING_ESCAPES if escapes is None else escapes 1080 1081 while True: 1082 if self._char in escapes and (self._peek == delimiter or self._peek in escapes): 1083 if self._peek == delimiter: 1084 text += self._peek 1085 else: 1086 text += self._char + self._peek 1087 1088 if self._current + 1 < self.size: 1089 self._advance(2) 1090 else: 1091 raise RuntimeError(f"Missing {delimiter} from {self._line}:{self._current}") 1092 else: 1093 if self._chars(delim_size) == delimiter: 1094 if delim_size > 1: 1095 self._advance(delim_size - 1) 1096 break 1097 1098 if self._end: 1099 raise RuntimeError(f"Missing {delimiter} from {self._line}:{self._start}") 1100 1101 current = self._current - 1 1102 self._advance(alnum=True) 1103 text += self.sql[current : self._current - 1] 1104 1105 return text
11class TokenType(AutoName): 12 L_PAREN = auto() 13 R_PAREN = auto() 14 L_BRACKET = auto() 15 R_BRACKET = auto() 16 L_BRACE = auto() 17 R_BRACE = auto() 18 COMMA = auto() 19 DOT = auto() 20 DASH = auto() 21 PLUS = auto() 22 COLON = auto() 23 DCOLON = auto() 24 SEMICOLON = auto() 25 STAR = auto() 26 BACKSLASH = auto() 27 SLASH = auto() 28 LT = auto() 29 LTE = auto() 30 GT = auto() 31 GTE = auto() 32 NOT = auto() 33 EQ = auto() 34 NEQ = auto() 35 NULLSAFE_EQ = auto() 36 AND = auto() 37 OR = auto() 38 AMP = auto() 39 DPIPE = auto() 40 PIPE = auto() 41 CARET = auto() 42 TILDA = auto() 43 ARROW = auto() 44 DARROW = auto() 45 FARROW = auto() 46 HASH = auto() 47 HASH_ARROW = auto() 48 DHASH_ARROW = auto() 49 LR_ARROW = auto() 50 LT_AT = auto() 51 AT_GT = auto() 52 DOLLAR = auto() 53 PARAMETER = auto() 54 SESSION_PARAMETER = auto() 55 DAMP = auto() 56 57 BLOCK_START = auto() 58 BLOCK_END = auto() 59 60 SPACE = auto() 61 BREAK = auto() 62 63 STRING = auto() 64 NUMBER = auto() 65 IDENTIFIER = auto() 66 DATABASE = auto() 67 COLUMN = auto() 68 COLUMN_DEF = auto() 69 SCHEMA = auto() 70 TABLE = auto() 71 VAR = auto() 72 BIT_STRING = auto() 73 HEX_STRING = auto() 74 BYTE_STRING = auto() 75 NATIONAL_STRING = auto() 76 RAW_STRING = auto() 77 78 # types 79 BIT = auto() 80 BOOLEAN = auto() 81 TINYINT = auto() 82 UTINYINT = auto() 83 SMALLINT = auto() 84 USMALLINT = auto() 85 INT = auto() 86 UINT = auto() 87 BIGINT = auto() 88 UBIGINT = auto() 89 INT128 = auto() 90 UINT128 = auto() 91 INT256 = auto() 92 UINT256 = auto() 93 FLOAT = auto() 94 DOUBLE = auto() 95 DECIMAL = auto() 96 BIGDECIMAL = auto() 97 CHAR = auto() 98 NCHAR = auto() 99 VARCHAR = auto() 100 NVARCHAR = auto() 101 TEXT = auto() 102 MEDIUMTEXT = auto() 103 LONGTEXT = auto() 104 MEDIUMBLOB = auto() 105 LONGBLOB = auto() 106 BINARY = auto() 107 VARBINARY = auto() 108 JSON = auto() 109 JSONB = auto() 110 TIME = auto() 111 TIMESTAMP = auto() 112 TIMESTAMPTZ = auto() 113 TIMESTAMPLTZ = auto() 114 DATETIME = auto() 115 DATETIME64 = auto() 116 DATE = auto() 117 UUID = auto() 118 GEOGRAPHY = auto() 119 NULLABLE = auto() 120 GEOMETRY = auto() 121 HLLSKETCH = auto() 122 HSTORE = auto() 123 SUPER = auto() 124 SERIAL = auto() 125 SMALLSERIAL = auto() 126 BIGSERIAL = auto() 127 XML = auto() 128 UNIQUEIDENTIFIER = auto() 129 MONEY = auto() 130 SMALLMONEY = auto() 131 ROWVERSION = auto() 132 IMAGE = auto() 133 VARIANT = auto() 134 OBJECT = auto() 135 INET = auto() 136 137 # keywords 138 ALIAS = auto() 139 ALTER = auto() 140 ALWAYS = auto() 141 ALL = auto() 142 ANTI = auto() 143 ANY = auto() 144 APPLY = auto() 145 ARRAY = auto() 146 ASC = auto() 147 ASOF = auto() 148 AUTO_INCREMENT = auto() 149 BEGIN = auto() 150 BETWEEN = auto() 151 CACHE = auto() 152 CASE = auto() 153 CHARACTER_SET = auto() 154 COLLATE = auto() 155 COMMAND = auto() 156 COMMENT = auto() 157 COMMIT = auto() 158 CONSTRAINT = auto() 159 CREATE = auto() 160 CROSS = auto() 161 CUBE = auto() 162 CURRENT_DATE = auto() 163 CURRENT_DATETIME = auto() 164 CURRENT_TIME = auto() 165 CURRENT_TIMESTAMP = auto() 166 CURRENT_USER = auto() 167 DEFAULT = auto() 168 DELETE = auto() 169 DESC = auto() 170 DESCRIBE = auto() 171 DISTINCT = auto() 172 DIV = auto() 173 DROP = auto() 174 ELSE = auto() 175 END = auto() 176 ESCAPE = auto() 177 EXCEPT = auto() 178 EXECUTE = auto() 179 EXISTS = auto() 180 FALSE = auto() 181 FETCH = auto() 182 FILTER = auto() 183 FINAL = auto() 184 FIRST = auto() 185 FOR = auto() 186 FOREIGN_KEY = auto() 187 FORMAT = auto() 188 FROM = auto() 189 FULL = auto() 190 FUNCTION = auto() 191 GLOB = auto() 192 GLOBAL = auto() 193 GROUP_BY = auto() 194 GROUPING_SETS = auto() 195 HAVING = auto() 196 HINT = auto() 197 IF = auto() 198 ILIKE = auto() 199 ILIKE_ANY = auto() 200 IN = auto() 201 INDEX = auto() 202 INNER = auto() 203 INSERT = auto() 204 INTERSECT = auto() 205 INTERVAL = auto() 206 INTO = auto() 207 INTRODUCER = auto() 208 IRLIKE = auto() 209 IS = auto() 210 ISNULL = auto() 211 JOIN = auto() 212 JOIN_MARKER = auto() 213 KEEP = auto() 214 LANGUAGE = auto() 215 LATERAL = auto() 216 LEFT = auto() 217 LIKE = auto() 218 LIKE_ANY = auto() 219 LIMIT = auto() 220 LOAD = auto() 221 LOCK = auto() 222 MAP = auto() 223 MATCH_RECOGNIZE = auto() 224 MERGE = auto() 225 MOD = auto() 226 NATURAL = auto() 227 NEXT = auto() 228 NEXT_VALUE_FOR = auto() 229 NOTNULL = auto() 230 NULL = auto() 231 OFFSET = auto() 232 ON = auto() 233 ORDER_BY = auto() 234 ORDERED = auto() 235 ORDINALITY = auto() 236 OUTER = auto() 237 OVER = auto() 238 OVERLAPS = auto() 239 OVERWRITE = auto() 240 PARTITION = auto() 241 PARTITION_BY = auto() 242 PERCENT = auto() 243 PIVOT = auto() 244 PLACEHOLDER = auto() 245 PRAGMA = auto() 246 PRIMARY_KEY = auto() 247 PROCEDURE = auto() 248 PROPERTIES = auto() 249 PSEUDO_TYPE = auto() 250 QUALIFY = auto() 251 QUOTE = auto() 252 RANGE = auto() 253 RECURSIVE = auto() 254 REPLACE = auto() 255 RETURNING = auto() 256 REFERENCES = auto() 257 RIGHT = auto() 258 RLIKE = auto() 259 ROLLBACK = auto() 260 ROLLUP = auto() 261 ROW = auto() 262 ROWS = auto() 263 SELECT = auto() 264 SEMI = auto() 265 SEPARATOR = auto() 266 SERDE_PROPERTIES = auto() 267 SET = auto() 268 SETTINGS = auto() 269 SHOW = auto() 270 SIMILAR_TO = auto() 271 SOME = auto() 272 STRUCT = auto() 273 TABLE_SAMPLE = auto() 274 TEMPORARY = auto() 275 TOP = auto() 276 THEN = auto() 277 TRUE = auto() 278 UNCACHE = auto() 279 UNION = auto() 280 UNNEST = auto() 281 UNPIVOT = auto() 282 UPDATE = auto() 283 USE = auto() 284 USING = auto() 285 VALUES = auto() 286 VIEW = auto() 287 VOLATILE = auto() 288 WHEN = auto() 289 WHERE = auto() 290 WINDOW = auto() 291 WITH = auto() 292 UNIQUE = auto()
An enumeration.
L_PAREN =
<TokenType.L_PAREN: 'L_PAREN'>
R_PAREN =
<TokenType.R_PAREN: 'R_PAREN'>
L_BRACKET =
<TokenType.L_BRACKET: 'L_BRACKET'>
R_BRACKET =
<TokenType.R_BRACKET: 'R_BRACKET'>
L_BRACE =
<TokenType.L_BRACE: 'L_BRACE'>
R_BRACE =
<TokenType.R_BRACE: 'R_BRACE'>
COMMA =
<TokenType.COMMA: 'COMMA'>
DOT =
<TokenType.DOT: 'DOT'>
DASH =
<TokenType.DASH: 'DASH'>
PLUS =
<TokenType.PLUS: 'PLUS'>
COLON =
<TokenType.COLON: 'COLON'>
DCOLON =
<TokenType.DCOLON: 'DCOLON'>
SEMICOLON =
<TokenType.SEMICOLON: 'SEMICOLON'>
STAR =
<TokenType.STAR: 'STAR'>
BACKSLASH =
<TokenType.BACKSLASH: 'BACKSLASH'>
SLASH =
<TokenType.SLASH: 'SLASH'>
LT =
<TokenType.LT: 'LT'>
LTE =
<TokenType.LTE: 'LTE'>
GT =
<TokenType.GT: 'GT'>
GTE =
<TokenType.GTE: 'GTE'>
NOT =
<TokenType.NOT: 'NOT'>
EQ =
<TokenType.EQ: 'EQ'>
NEQ =
<TokenType.NEQ: 'NEQ'>
NULLSAFE_EQ =
<TokenType.NULLSAFE_EQ: 'NULLSAFE_EQ'>
AND =
<TokenType.AND: 'AND'>
OR =
<TokenType.OR: 'OR'>
AMP =
<TokenType.AMP: 'AMP'>
DPIPE =
<TokenType.DPIPE: 'DPIPE'>
PIPE =
<TokenType.PIPE: 'PIPE'>
CARET =
<TokenType.CARET: 'CARET'>
TILDA =
<TokenType.TILDA: 'TILDA'>
ARROW =
<TokenType.ARROW: 'ARROW'>
DARROW =
<TokenType.DARROW: 'DARROW'>
FARROW =
<TokenType.FARROW: 'FARROW'>
HASH =
<TokenType.HASH: 'HASH'>
HASH_ARROW =
<TokenType.HASH_ARROW: 'HASH_ARROW'>
DHASH_ARROW =
<TokenType.DHASH_ARROW: 'DHASH_ARROW'>
LR_ARROW =
<TokenType.LR_ARROW: 'LR_ARROW'>
LT_AT =
<TokenType.LT_AT: 'LT_AT'>
AT_GT =
<TokenType.AT_GT: 'AT_GT'>
DOLLAR =
<TokenType.DOLLAR: 'DOLLAR'>
PARAMETER =
<TokenType.PARAMETER: 'PARAMETER'>
SESSION_PARAMETER =
<TokenType.SESSION_PARAMETER: 'SESSION_PARAMETER'>
DAMP =
<TokenType.DAMP: 'DAMP'>
BLOCK_START =
<TokenType.BLOCK_START: 'BLOCK_START'>
BLOCK_END =
<TokenType.BLOCK_END: 'BLOCK_END'>
SPACE =
<TokenType.SPACE: 'SPACE'>
BREAK =
<TokenType.BREAK: 'BREAK'>
STRING =
<TokenType.STRING: 'STRING'>
NUMBER =
<TokenType.NUMBER: 'NUMBER'>
IDENTIFIER =
<TokenType.IDENTIFIER: 'IDENTIFIER'>
DATABASE =
<TokenType.DATABASE: 'DATABASE'>
COLUMN =
<TokenType.COLUMN: 'COLUMN'>
COLUMN_DEF =
<TokenType.COLUMN_DEF: 'COLUMN_DEF'>
SCHEMA =
<TokenType.SCHEMA: 'SCHEMA'>
TABLE =
<TokenType.TABLE: 'TABLE'>
VAR =
<TokenType.VAR: 'VAR'>
BIT_STRING =
<TokenType.BIT_STRING: 'BIT_STRING'>
HEX_STRING =
<TokenType.HEX_STRING: 'HEX_STRING'>
BYTE_STRING =
<TokenType.BYTE_STRING: 'BYTE_STRING'>
NATIONAL_STRING =
<TokenType.NATIONAL_STRING: 'NATIONAL_STRING'>
RAW_STRING =
<TokenType.RAW_STRING: 'RAW_STRING'>
BIT =
<TokenType.BIT: 'BIT'>
BOOLEAN =
<TokenType.BOOLEAN: 'BOOLEAN'>
TINYINT =
<TokenType.TINYINT: 'TINYINT'>
UTINYINT =
<TokenType.UTINYINT: 'UTINYINT'>
SMALLINT =
<TokenType.SMALLINT: 'SMALLINT'>
USMALLINT =
<TokenType.USMALLINT: 'USMALLINT'>
INT =
<TokenType.INT: 'INT'>
UINT =
<TokenType.UINT: 'UINT'>
BIGINT =
<TokenType.BIGINT: 'BIGINT'>
UBIGINT =
<TokenType.UBIGINT: 'UBIGINT'>
INT128 =
<TokenType.INT128: 'INT128'>
UINT128 =
<TokenType.UINT128: 'UINT128'>
INT256 =
<TokenType.INT256: 'INT256'>
UINT256 =
<TokenType.UINT256: 'UINT256'>
FLOAT =
<TokenType.FLOAT: 'FLOAT'>
DOUBLE =
<TokenType.DOUBLE: 'DOUBLE'>
DECIMAL =
<TokenType.DECIMAL: 'DECIMAL'>
BIGDECIMAL =
<TokenType.BIGDECIMAL: 'BIGDECIMAL'>
CHAR =
<TokenType.CHAR: 'CHAR'>
NCHAR =
<TokenType.NCHAR: 'NCHAR'>
VARCHAR =
<TokenType.VARCHAR: 'VARCHAR'>
NVARCHAR =
<TokenType.NVARCHAR: 'NVARCHAR'>
TEXT =
<TokenType.TEXT: 'TEXT'>
MEDIUMTEXT =
<TokenType.MEDIUMTEXT: 'MEDIUMTEXT'>
LONGTEXT =
<TokenType.LONGTEXT: 'LONGTEXT'>
MEDIUMBLOB =
<TokenType.MEDIUMBLOB: 'MEDIUMBLOB'>
LONGBLOB =
<TokenType.LONGBLOB: 'LONGBLOB'>
BINARY =
<TokenType.BINARY: 'BINARY'>
VARBINARY =
<TokenType.VARBINARY: 'VARBINARY'>
JSON =
<TokenType.JSON: 'JSON'>
JSONB =
<TokenType.JSONB: 'JSONB'>
TIME =
<TokenType.TIME: 'TIME'>
TIMESTAMP =
<TokenType.TIMESTAMP: 'TIMESTAMP'>
TIMESTAMPTZ =
<TokenType.TIMESTAMPTZ: 'TIMESTAMPTZ'>
TIMESTAMPLTZ =
<TokenType.TIMESTAMPLTZ: 'TIMESTAMPLTZ'>
DATETIME =
<TokenType.DATETIME: 'DATETIME'>
DATETIME64 =
<TokenType.DATETIME64: 'DATETIME64'>
DATE =
<TokenType.DATE: 'DATE'>
UUID =
<TokenType.UUID: 'UUID'>
GEOGRAPHY =
<TokenType.GEOGRAPHY: 'GEOGRAPHY'>
NULLABLE =
<TokenType.NULLABLE: 'NULLABLE'>
GEOMETRY =
<TokenType.GEOMETRY: 'GEOMETRY'>
HLLSKETCH =
<TokenType.HLLSKETCH: 'HLLSKETCH'>
HSTORE =
<TokenType.HSTORE: 'HSTORE'>
SUPER =
<TokenType.SUPER: 'SUPER'>
SERIAL =
<TokenType.SERIAL: 'SERIAL'>
SMALLSERIAL =
<TokenType.SMALLSERIAL: 'SMALLSERIAL'>
BIGSERIAL =
<TokenType.BIGSERIAL: 'BIGSERIAL'>
XML =
<TokenType.XML: 'XML'>
UNIQUEIDENTIFIER =
<TokenType.UNIQUEIDENTIFIER: 'UNIQUEIDENTIFIER'>
MONEY =
<TokenType.MONEY: 'MONEY'>
SMALLMONEY =
<TokenType.SMALLMONEY: 'SMALLMONEY'>
ROWVERSION =
<TokenType.ROWVERSION: 'ROWVERSION'>
IMAGE =
<TokenType.IMAGE: 'IMAGE'>
VARIANT =
<TokenType.VARIANT: 'VARIANT'>
OBJECT =
<TokenType.OBJECT: 'OBJECT'>
INET =
<TokenType.INET: 'INET'>
ALIAS =
<TokenType.ALIAS: 'ALIAS'>
ALTER =
<TokenType.ALTER: 'ALTER'>
ALWAYS =
<TokenType.ALWAYS: 'ALWAYS'>
ALL =
<TokenType.ALL: 'ALL'>
ANTI =
<TokenType.ANTI: 'ANTI'>
ANY =
<TokenType.ANY: 'ANY'>
APPLY =
<TokenType.APPLY: 'APPLY'>
ARRAY =
<TokenType.ARRAY: 'ARRAY'>
ASC =
<TokenType.ASC: 'ASC'>
ASOF =
<TokenType.ASOF: 'ASOF'>
AUTO_INCREMENT =
<TokenType.AUTO_INCREMENT: 'AUTO_INCREMENT'>
BEGIN =
<TokenType.BEGIN: 'BEGIN'>
BETWEEN =
<TokenType.BETWEEN: 'BETWEEN'>
CACHE =
<TokenType.CACHE: 'CACHE'>
CASE =
<TokenType.CASE: 'CASE'>
CHARACTER_SET =
<TokenType.CHARACTER_SET: 'CHARACTER_SET'>
COLLATE =
<TokenType.COLLATE: 'COLLATE'>
COMMAND =
<TokenType.COMMAND: 'COMMAND'>
COMMENT =
<TokenType.COMMENT: 'COMMENT'>
COMMIT =
<TokenType.COMMIT: 'COMMIT'>
CONSTRAINT =
<TokenType.CONSTRAINT: 'CONSTRAINT'>
CREATE =
<TokenType.CREATE: 'CREATE'>
CROSS =
<TokenType.CROSS: 'CROSS'>
CUBE =
<TokenType.CUBE: 'CUBE'>
CURRENT_DATE =
<TokenType.CURRENT_DATE: 'CURRENT_DATE'>
CURRENT_DATETIME =
<TokenType.CURRENT_DATETIME: 'CURRENT_DATETIME'>
CURRENT_TIME =
<TokenType.CURRENT_TIME: 'CURRENT_TIME'>
CURRENT_TIMESTAMP =
<TokenType.CURRENT_TIMESTAMP: 'CURRENT_TIMESTAMP'>
CURRENT_USER =
<TokenType.CURRENT_USER: 'CURRENT_USER'>
DEFAULT =
<TokenType.DEFAULT: 'DEFAULT'>
DELETE =
<TokenType.DELETE: 'DELETE'>
DESC =
<TokenType.DESC: 'DESC'>
DESCRIBE =
<TokenType.DESCRIBE: 'DESCRIBE'>
DISTINCT =
<TokenType.DISTINCT: 'DISTINCT'>
DIV =
<TokenType.DIV: 'DIV'>
DROP =
<TokenType.DROP: 'DROP'>
ELSE =
<TokenType.ELSE: 'ELSE'>
END =
<TokenType.END: 'END'>
ESCAPE =
<TokenType.ESCAPE: 'ESCAPE'>
EXCEPT =
<TokenType.EXCEPT: 'EXCEPT'>
EXECUTE =
<TokenType.EXECUTE: 'EXECUTE'>
EXISTS =
<TokenType.EXISTS: 'EXISTS'>
FALSE =
<TokenType.FALSE: 'FALSE'>
FETCH =
<TokenType.FETCH: 'FETCH'>
FILTER =
<TokenType.FILTER: 'FILTER'>
FINAL =
<TokenType.FINAL: 'FINAL'>
FIRST =
<TokenType.FIRST: 'FIRST'>
FOR =
<TokenType.FOR: 'FOR'>
FOREIGN_KEY =
<TokenType.FOREIGN_KEY: 'FOREIGN_KEY'>
FORMAT =
<TokenType.FORMAT: 'FORMAT'>
FROM =
<TokenType.FROM: 'FROM'>
FULL =
<TokenType.FULL: 'FULL'>
FUNCTION =
<TokenType.FUNCTION: 'FUNCTION'>
GLOB =
<TokenType.GLOB: 'GLOB'>
GLOBAL =
<TokenType.GLOBAL: 'GLOBAL'>
GROUP_BY =
<TokenType.GROUP_BY: 'GROUP_BY'>
GROUPING_SETS =
<TokenType.GROUPING_SETS: 'GROUPING_SETS'>
HAVING =
<TokenType.HAVING: 'HAVING'>
HINT =
<TokenType.HINT: 'HINT'>
IF =
<TokenType.IF: 'IF'>
ILIKE =
<TokenType.ILIKE: 'ILIKE'>
ILIKE_ANY =
<TokenType.ILIKE_ANY: 'ILIKE_ANY'>
IN =
<TokenType.IN: 'IN'>
INDEX =
<TokenType.INDEX: 'INDEX'>
INNER =
<TokenType.INNER: 'INNER'>
INSERT =
<TokenType.INSERT: 'INSERT'>
INTERSECT =
<TokenType.INTERSECT: 'INTERSECT'>
INTERVAL =
<TokenType.INTERVAL: 'INTERVAL'>
INTO =
<TokenType.INTO: 'INTO'>
INTRODUCER =
<TokenType.INTRODUCER: 'INTRODUCER'>
IRLIKE =
<TokenType.IRLIKE: 'IRLIKE'>
IS =
<TokenType.IS: 'IS'>
ISNULL =
<TokenType.ISNULL: 'ISNULL'>
JOIN =
<TokenType.JOIN: 'JOIN'>
JOIN_MARKER =
<TokenType.JOIN_MARKER: 'JOIN_MARKER'>
KEEP =
<TokenType.KEEP: 'KEEP'>
LANGUAGE =
<TokenType.LANGUAGE: 'LANGUAGE'>
LATERAL =
<TokenType.LATERAL: 'LATERAL'>
LEFT =
<TokenType.LEFT: 'LEFT'>
LIKE =
<TokenType.LIKE: 'LIKE'>
LIKE_ANY =
<TokenType.LIKE_ANY: 'LIKE_ANY'>
LIMIT =
<TokenType.LIMIT: 'LIMIT'>
LOAD =
<TokenType.LOAD: 'LOAD'>
LOCK =
<TokenType.LOCK: 'LOCK'>
MAP =
<TokenType.MAP: 'MAP'>
MATCH_RECOGNIZE =
<TokenType.MATCH_RECOGNIZE: 'MATCH_RECOGNIZE'>
MERGE =
<TokenType.MERGE: 'MERGE'>
MOD =
<TokenType.MOD: 'MOD'>
NATURAL =
<TokenType.NATURAL: 'NATURAL'>
NEXT =
<TokenType.NEXT: 'NEXT'>
NEXT_VALUE_FOR =
<TokenType.NEXT_VALUE_FOR: 'NEXT_VALUE_FOR'>
NOTNULL =
<TokenType.NOTNULL: 'NOTNULL'>
NULL =
<TokenType.NULL: 'NULL'>
OFFSET =
<TokenType.OFFSET: 'OFFSET'>
ON =
<TokenType.ON: 'ON'>
ORDER_BY =
<TokenType.ORDER_BY: 'ORDER_BY'>
ORDERED =
<TokenType.ORDERED: 'ORDERED'>
ORDINALITY =
<TokenType.ORDINALITY: 'ORDINALITY'>
OUTER =
<TokenType.OUTER: 'OUTER'>
OVER =
<TokenType.OVER: 'OVER'>
OVERLAPS =
<TokenType.OVERLAPS: 'OVERLAPS'>
OVERWRITE =
<TokenType.OVERWRITE: 'OVERWRITE'>
PARTITION =
<TokenType.PARTITION: 'PARTITION'>
PARTITION_BY =
<TokenType.PARTITION_BY: 'PARTITION_BY'>
PERCENT =
<TokenType.PERCENT: 'PERCENT'>
PIVOT =
<TokenType.PIVOT: 'PIVOT'>
PLACEHOLDER =
<TokenType.PLACEHOLDER: 'PLACEHOLDER'>
PRAGMA =
<TokenType.PRAGMA: 'PRAGMA'>
PRIMARY_KEY =
<TokenType.PRIMARY_KEY: 'PRIMARY_KEY'>
PROCEDURE =
<TokenType.PROCEDURE: 'PROCEDURE'>
PROPERTIES =
<TokenType.PROPERTIES: 'PROPERTIES'>
PSEUDO_TYPE =
<TokenType.PSEUDO_TYPE: 'PSEUDO_TYPE'>
QUALIFY =
<TokenType.QUALIFY: 'QUALIFY'>
QUOTE =
<TokenType.QUOTE: 'QUOTE'>
RANGE =
<TokenType.RANGE: 'RANGE'>
RECURSIVE =
<TokenType.RECURSIVE: 'RECURSIVE'>
REPLACE =
<TokenType.REPLACE: 'REPLACE'>
RETURNING =
<TokenType.RETURNING: 'RETURNING'>
REFERENCES =
<TokenType.REFERENCES: 'REFERENCES'>
RIGHT =
<TokenType.RIGHT: 'RIGHT'>
RLIKE =
<TokenType.RLIKE: 'RLIKE'>
ROLLBACK =
<TokenType.ROLLBACK: 'ROLLBACK'>
ROLLUP =
<TokenType.ROLLUP: 'ROLLUP'>
ROW =
<TokenType.ROW: 'ROW'>
ROWS =
<TokenType.ROWS: 'ROWS'>
SELECT =
<TokenType.SELECT: 'SELECT'>
SEMI =
<TokenType.SEMI: 'SEMI'>
SEPARATOR =
<TokenType.SEPARATOR: 'SEPARATOR'>
SERDE_PROPERTIES =
<TokenType.SERDE_PROPERTIES: 'SERDE_PROPERTIES'>
SET =
<TokenType.SET: 'SET'>
SETTINGS =
<TokenType.SETTINGS: 'SETTINGS'>
SHOW =
<TokenType.SHOW: 'SHOW'>
SIMILAR_TO =
<TokenType.SIMILAR_TO: 'SIMILAR_TO'>
SOME =
<TokenType.SOME: 'SOME'>
STRUCT =
<TokenType.STRUCT: 'STRUCT'>
TABLE_SAMPLE =
<TokenType.TABLE_SAMPLE: 'TABLE_SAMPLE'>
TEMPORARY =
<TokenType.TEMPORARY: 'TEMPORARY'>
TOP =
<TokenType.TOP: 'TOP'>
THEN =
<TokenType.THEN: 'THEN'>
TRUE =
<TokenType.TRUE: 'TRUE'>
UNCACHE =
<TokenType.UNCACHE: 'UNCACHE'>
UNION =
<TokenType.UNION: 'UNION'>
UNNEST =
<TokenType.UNNEST: 'UNNEST'>
UNPIVOT =
<TokenType.UNPIVOT: 'UNPIVOT'>
UPDATE =
<TokenType.UPDATE: 'UPDATE'>
USE =
<TokenType.USE: 'USE'>
USING =
<TokenType.USING: 'USING'>
VALUES =
<TokenType.VALUES: 'VALUES'>
VIEW =
<TokenType.VIEW: 'VIEW'>
VOLATILE =
<TokenType.VOLATILE: 'VOLATILE'>
WHEN =
<TokenType.WHEN: 'WHEN'>
WHERE =
<TokenType.WHERE: 'WHERE'>
WINDOW =
<TokenType.WINDOW: 'WINDOW'>
WITH =
<TokenType.WITH: 'WITH'>
UNIQUE =
<TokenType.UNIQUE: 'UNIQUE'>
Inherited Members
- enum.Enum
- name
- value
class
Token:
295class Token: 296 __slots__ = ("token_type", "text", "line", "col", "start", "end", "comments") 297 298 @classmethod 299 def number(cls, number: int) -> Token: 300 """Returns a NUMBER token with `number` as its text.""" 301 return cls(TokenType.NUMBER, str(number)) 302 303 @classmethod 304 def string(cls, string: str) -> Token: 305 """Returns a STRING token with `string` as its text.""" 306 return cls(TokenType.STRING, string) 307 308 @classmethod 309 def identifier(cls, identifier: str) -> Token: 310 """Returns an IDENTIFIER token with `identifier` as its text.""" 311 return cls(TokenType.IDENTIFIER, identifier) 312 313 @classmethod 314 def var(cls, var: str) -> Token: 315 """Returns an VAR token with `var` as its text.""" 316 return cls(TokenType.VAR, var) 317 318 def __init__( 319 self, 320 token_type: TokenType, 321 text: str, 322 line: int = 1, 323 col: int = 1, 324 start: int = 0, 325 end: int = 0, 326 comments: t.List[str] = [], 327 ) -> None: 328 """Token initializer. 329 330 Args: 331 token_type: The TokenType Enum. 332 text: The text of the token. 333 line: The line that the token ends on. 334 col: The column that the token ends on. 335 start: The start index of the token. 336 end: The ending index of the token. 337 """ 338 self.token_type = token_type 339 self.text = text 340 self.line = line 341 self.col = col 342 self.start = start 343 self.end = end 344 self.comments = comments 345 346 def __repr__(self) -> str: 347 attributes = ", ".join(f"{k}: {getattr(self, k)}" for k in self.__slots__) 348 return f"<Token {attributes}>"
Token( token_type: sqlglot.tokens.TokenType, text: str, line: int = 1, col: int = 1, start: int = 0, end: int = 0, comments: List[str] = [])
318 def __init__( 319 self, 320 token_type: TokenType, 321 text: str, 322 line: int = 1, 323 col: int = 1, 324 start: int = 0, 325 end: int = 0, 326 comments: t.List[str] = [], 327 ) -> None: 328 """Token initializer. 329 330 Args: 331 token_type: The TokenType Enum. 332 text: The text of the token. 333 line: The line that the token ends on. 334 col: The column that the token ends on. 335 start: The start index of the token. 336 end: The ending index of the token. 337 """ 338 self.token_type = token_type 339 self.text = text 340 self.line = line 341 self.col = col 342 self.start = start 343 self.end = end 344 self.comments = comments
Token initializer.
Arguments:
- token_type: The TokenType Enum.
- text: The text of the token.
- line: The line that the token ends on.
- col: The column that the token ends on.
- start: The start index of the token.
- end: The ending index of the token.
298 @classmethod 299 def number(cls, number: int) -> Token: 300 """Returns a NUMBER token with `number` as its text.""" 301 return cls(TokenType.NUMBER, str(number))
Returns a NUMBER token with number
as its text.
303 @classmethod 304 def string(cls, string: str) -> Token: 305 """Returns a STRING token with `string` as its text.""" 306 return cls(TokenType.STRING, string)
Returns a STRING token with string
as its text.
308 @classmethod 309 def identifier(cls, identifier: str) -> Token: 310 """Returns an IDENTIFIER token with `identifier` as its text.""" 311 return cls(TokenType.IDENTIFIER, identifier)
Returns an IDENTIFIER token with identifier
as its text.
class
Tokenizer:
401class Tokenizer(metaclass=_Tokenizer): 402 SINGLE_TOKENS = { 403 "(": TokenType.L_PAREN, 404 ")": TokenType.R_PAREN, 405 "[": TokenType.L_BRACKET, 406 "]": TokenType.R_BRACKET, 407 "{": TokenType.L_BRACE, 408 "}": TokenType.R_BRACE, 409 "&": TokenType.AMP, 410 "^": TokenType.CARET, 411 ":": TokenType.COLON, 412 ",": TokenType.COMMA, 413 ".": TokenType.DOT, 414 "-": TokenType.DASH, 415 "=": TokenType.EQ, 416 ">": TokenType.GT, 417 "<": TokenType.LT, 418 "%": TokenType.MOD, 419 "!": TokenType.NOT, 420 "|": TokenType.PIPE, 421 "+": TokenType.PLUS, 422 ";": TokenType.SEMICOLON, 423 "/": TokenType.SLASH, 424 "\\": TokenType.BACKSLASH, 425 "*": TokenType.STAR, 426 "~": TokenType.TILDA, 427 "?": TokenType.PLACEHOLDER, 428 "@": TokenType.PARAMETER, 429 # used for breaking a var like x'y' but nothing else 430 # the token type doesn't matter 431 "'": TokenType.QUOTE, 432 "`": TokenType.IDENTIFIER, 433 '"': TokenType.IDENTIFIER, 434 "#": TokenType.HASH, 435 } 436 437 BIT_STRINGS: t.List[str | t.Tuple[str, str]] = [] 438 BYTE_STRINGS: t.List[str | t.Tuple[str, str]] = [] 439 HEX_STRINGS: t.List[str | t.Tuple[str, str]] = [] 440 RAW_STRINGS: t.List[str | t.Tuple[str, str]] = [] 441 IDENTIFIERS: t.List[str | t.Tuple[str, str]] = ['"'] 442 IDENTIFIER_ESCAPES = ['"'] 443 QUOTES: t.List[t.Tuple[str, str] | str] = ["'"] 444 STRING_ESCAPES = ["'"] 445 VAR_SINGLE_TOKENS: t.Set[str] = set() 446 447 _COMMENTS: t.Dict[str, str] = {} 448 _FORMAT_STRINGS: t.Dict[str, t.Tuple[str, TokenType]] = {} 449 _IDENTIFIERS: t.Dict[str, str] = {} 450 _IDENTIFIER_ESCAPES: t.Set[str] = set() 451 _QUOTES: t.Dict[str, str] = {} 452 _STRING_ESCAPES: t.Set[str] = set() 453 454 KEYWORDS: t.Dict[t.Optional[str], TokenType] = { 455 **{f"{{%{postfix}": TokenType.BLOCK_START for postfix in ("", "+", "-")}, 456 **{f"{prefix}%}}": TokenType.BLOCK_END for prefix in ("", "+", "-")}, 457 "{{+": TokenType.BLOCK_START, 458 "{{-": TokenType.BLOCK_START, 459 "+}}": TokenType.BLOCK_END, 460 "-}}": TokenType.BLOCK_END, 461 "/*+": TokenType.HINT, 462 "==": TokenType.EQ, 463 "::": TokenType.DCOLON, 464 "||": TokenType.DPIPE, 465 ">=": TokenType.GTE, 466 "<=": TokenType.LTE, 467 "<>": TokenType.NEQ, 468 "!=": TokenType.NEQ, 469 "<=>": TokenType.NULLSAFE_EQ, 470 "->": TokenType.ARROW, 471 "->>": TokenType.DARROW, 472 "=>": TokenType.FARROW, 473 "#>": TokenType.HASH_ARROW, 474 "#>>": TokenType.DHASH_ARROW, 475 "<->": TokenType.LR_ARROW, 476 "&&": TokenType.DAMP, 477 "ALL": TokenType.ALL, 478 "ALWAYS": TokenType.ALWAYS, 479 "AND": TokenType.AND, 480 "ANTI": TokenType.ANTI, 481 "ANY": TokenType.ANY, 482 "ASC": TokenType.ASC, 483 "AS": TokenType.ALIAS, 484 "AUTOINCREMENT": TokenType.AUTO_INCREMENT, 485 "AUTO_INCREMENT": TokenType.AUTO_INCREMENT, 486 "BEGIN": TokenType.BEGIN, 487 "BETWEEN": TokenType.BETWEEN, 488 "CACHE": TokenType.CACHE, 489 "UNCACHE": TokenType.UNCACHE, 490 "CASE": TokenType.CASE, 491 "CHARACTER SET": TokenType.CHARACTER_SET, 492 "COLLATE": TokenType.COLLATE, 493 "COLUMN": TokenType.COLUMN, 494 "COMMIT": TokenType.COMMIT, 495 "CONSTRAINT": TokenType.CONSTRAINT, 496 "CREATE": TokenType.CREATE, 497 "CROSS": TokenType.CROSS, 498 "CUBE": TokenType.CUBE, 499 "CURRENT_DATE": TokenType.CURRENT_DATE, 500 "CURRENT_TIME": TokenType.CURRENT_TIME, 501 "CURRENT_TIMESTAMP": TokenType.CURRENT_TIMESTAMP, 502 "CURRENT_USER": TokenType.CURRENT_USER, 503 "DATABASE": TokenType.DATABASE, 504 "DEFAULT": TokenType.DEFAULT, 505 "DELETE": TokenType.DELETE, 506 "DESC": TokenType.DESC, 507 "DESCRIBE": TokenType.DESCRIBE, 508 "DISTINCT": TokenType.DISTINCT, 509 "DIV": TokenType.DIV, 510 "DROP": TokenType.DROP, 511 "ELSE": TokenType.ELSE, 512 "END": TokenType.END, 513 "ESCAPE": TokenType.ESCAPE, 514 "EXCEPT": TokenType.EXCEPT, 515 "EXECUTE": TokenType.EXECUTE, 516 "EXISTS": TokenType.EXISTS, 517 "FALSE": TokenType.FALSE, 518 "FETCH": TokenType.FETCH, 519 "FILTER": TokenType.FILTER, 520 "FIRST": TokenType.FIRST, 521 "FULL": TokenType.FULL, 522 "FUNCTION": TokenType.FUNCTION, 523 "FOR": TokenType.FOR, 524 "FOREIGN KEY": TokenType.FOREIGN_KEY, 525 "FORMAT": TokenType.FORMAT, 526 "FROM": TokenType.FROM, 527 "GEOGRAPHY": TokenType.GEOGRAPHY, 528 "GEOMETRY": TokenType.GEOMETRY, 529 "GLOB": TokenType.GLOB, 530 "GROUP BY": TokenType.GROUP_BY, 531 "GROUPING SETS": TokenType.GROUPING_SETS, 532 "HAVING": TokenType.HAVING, 533 "IF": TokenType.IF, 534 "ILIKE": TokenType.ILIKE, 535 "IN": TokenType.IN, 536 "INDEX": TokenType.INDEX, 537 "INET": TokenType.INET, 538 "INNER": TokenType.INNER, 539 "INSERT": TokenType.INSERT, 540 "INTERVAL": TokenType.INTERVAL, 541 "INTERSECT": TokenType.INTERSECT, 542 "INTO": TokenType.INTO, 543 "IS": TokenType.IS, 544 "ISNULL": TokenType.ISNULL, 545 "JOIN": TokenType.JOIN, 546 "KEEP": TokenType.KEEP, 547 "LATERAL": TokenType.LATERAL, 548 "LEFT": TokenType.LEFT, 549 "LIKE": TokenType.LIKE, 550 "LIMIT": TokenType.LIMIT, 551 "LOAD": TokenType.LOAD, 552 "LOCK": TokenType.LOCK, 553 "MERGE": TokenType.MERGE, 554 "NATURAL": TokenType.NATURAL, 555 "NEXT": TokenType.NEXT, 556 "NEXT VALUE FOR": TokenType.NEXT_VALUE_FOR, 557 "NOT": TokenType.NOT, 558 "NOTNULL": TokenType.NOTNULL, 559 "NULL": TokenType.NULL, 560 "OBJECT": TokenType.OBJECT, 561 "OFFSET": TokenType.OFFSET, 562 "ON": TokenType.ON, 563 "OR": TokenType.OR, 564 "ORDER BY": TokenType.ORDER_BY, 565 "ORDINALITY": TokenType.ORDINALITY, 566 "OUTER": TokenType.OUTER, 567 "OVER": TokenType.OVER, 568 "OVERLAPS": TokenType.OVERLAPS, 569 "OVERWRITE": TokenType.OVERWRITE, 570 "PARTITION": TokenType.PARTITION, 571 "PARTITION BY": TokenType.PARTITION_BY, 572 "PARTITIONED BY": TokenType.PARTITION_BY, 573 "PARTITIONED_BY": TokenType.PARTITION_BY, 574 "PERCENT": TokenType.PERCENT, 575 "PIVOT": TokenType.PIVOT, 576 "PRAGMA": TokenType.PRAGMA, 577 "PRIMARY KEY": TokenType.PRIMARY_KEY, 578 "PROCEDURE": TokenType.PROCEDURE, 579 "QUALIFY": TokenType.QUALIFY, 580 "RANGE": TokenType.RANGE, 581 "RECURSIVE": TokenType.RECURSIVE, 582 "REGEXP": TokenType.RLIKE, 583 "REPLACE": TokenType.REPLACE, 584 "REFERENCES": TokenType.REFERENCES, 585 "RIGHT": TokenType.RIGHT, 586 "RLIKE": TokenType.RLIKE, 587 "ROLLBACK": TokenType.ROLLBACK, 588 "ROLLUP": TokenType.ROLLUP, 589 "ROW": TokenType.ROW, 590 "ROWS": TokenType.ROWS, 591 "SCHEMA": TokenType.SCHEMA, 592 "SELECT": TokenType.SELECT, 593 "SEMI": TokenType.SEMI, 594 "SET": TokenType.SET, 595 "SETTINGS": TokenType.SETTINGS, 596 "SHOW": TokenType.SHOW, 597 "SIMILAR TO": TokenType.SIMILAR_TO, 598 "SOME": TokenType.SOME, 599 "TABLE": TokenType.TABLE, 600 "TABLESAMPLE": TokenType.TABLE_SAMPLE, 601 "TEMP": TokenType.TEMPORARY, 602 "TEMPORARY": TokenType.TEMPORARY, 603 "THEN": TokenType.THEN, 604 "TRUE": TokenType.TRUE, 605 "UNION": TokenType.UNION, 606 "UNNEST": TokenType.UNNEST, 607 "UNPIVOT": TokenType.UNPIVOT, 608 "UPDATE": TokenType.UPDATE, 609 "USE": TokenType.USE, 610 "USING": TokenType.USING, 611 "UUID": TokenType.UUID, 612 "VALUES": TokenType.VALUES, 613 "VIEW": TokenType.VIEW, 614 "VOLATILE": TokenType.VOLATILE, 615 "WHEN": TokenType.WHEN, 616 "WHERE": TokenType.WHERE, 617 "WINDOW": TokenType.WINDOW, 618 "WITH": TokenType.WITH, 619 "APPLY": TokenType.APPLY, 620 "ARRAY": TokenType.ARRAY, 621 "BIT": TokenType.BIT, 622 "BOOL": TokenType.BOOLEAN, 623 "BOOLEAN": TokenType.BOOLEAN, 624 "BYTE": TokenType.TINYINT, 625 "TINYINT": TokenType.TINYINT, 626 "SHORT": TokenType.SMALLINT, 627 "SMALLINT": TokenType.SMALLINT, 628 "INT2": TokenType.SMALLINT, 629 "INTEGER": TokenType.INT, 630 "INT": TokenType.INT, 631 "INT4": TokenType.INT, 632 "LONG": TokenType.BIGINT, 633 "BIGINT": TokenType.BIGINT, 634 "INT8": TokenType.BIGINT, 635 "DEC": TokenType.DECIMAL, 636 "DECIMAL": TokenType.DECIMAL, 637 "BIGDECIMAL": TokenType.BIGDECIMAL, 638 "BIGNUMERIC": TokenType.BIGDECIMAL, 639 "MAP": TokenType.MAP, 640 "NULLABLE": TokenType.NULLABLE, 641 "NUMBER": TokenType.DECIMAL, 642 "NUMERIC": TokenType.DECIMAL, 643 "FIXED": TokenType.DECIMAL, 644 "REAL": TokenType.FLOAT, 645 "FLOAT": TokenType.FLOAT, 646 "FLOAT4": TokenType.FLOAT, 647 "FLOAT8": TokenType.DOUBLE, 648 "DOUBLE": TokenType.DOUBLE, 649 "DOUBLE PRECISION": TokenType.DOUBLE, 650 "JSON": TokenType.JSON, 651 "CHAR": TokenType.CHAR, 652 "CHARACTER": TokenType.CHAR, 653 "NCHAR": TokenType.NCHAR, 654 "VARCHAR": TokenType.VARCHAR, 655 "VARCHAR2": TokenType.VARCHAR, 656 "NVARCHAR": TokenType.NVARCHAR, 657 "NVARCHAR2": TokenType.NVARCHAR, 658 "STR": TokenType.TEXT, 659 "STRING": TokenType.TEXT, 660 "TEXT": TokenType.TEXT, 661 "CLOB": TokenType.TEXT, 662 "LONGVARCHAR": TokenType.TEXT, 663 "BINARY": TokenType.BINARY, 664 "BLOB": TokenType.VARBINARY, 665 "BYTEA": TokenType.VARBINARY, 666 "VARBINARY": TokenType.VARBINARY, 667 "TIME": TokenType.TIME, 668 "TIMESTAMP": TokenType.TIMESTAMP, 669 "TIMESTAMPTZ": TokenType.TIMESTAMPTZ, 670 "TIMESTAMPLTZ": TokenType.TIMESTAMPLTZ, 671 "DATE": TokenType.DATE, 672 "DATETIME": TokenType.DATETIME, 673 "UNIQUE": TokenType.UNIQUE, 674 "STRUCT": TokenType.STRUCT, 675 "VARIANT": TokenType.VARIANT, 676 "ALTER": TokenType.ALTER, 677 "ANALYZE": TokenType.COMMAND, 678 "CALL": TokenType.COMMAND, 679 "COMMENT": TokenType.COMMENT, 680 "COPY": TokenType.COMMAND, 681 "EXPLAIN": TokenType.COMMAND, 682 "GRANT": TokenType.COMMAND, 683 "OPTIMIZE": TokenType.COMMAND, 684 "PREPARE": TokenType.COMMAND, 685 "TRUNCATE": TokenType.COMMAND, 686 "VACUUM": TokenType.COMMAND, 687 } 688 689 WHITE_SPACE: t.Dict[t.Optional[str], TokenType] = { 690 " ": TokenType.SPACE, 691 "\t": TokenType.SPACE, 692 "\n": TokenType.BREAK, 693 "\r": TokenType.BREAK, 694 "\r\n": TokenType.BREAK, 695 } 696 697 COMMANDS = { 698 TokenType.COMMAND, 699 TokenType.EXECUTE, 700 TokenType.FETCH, 701 TokenType.SHOW, 702 } 703 704 COMMAND_PREFIX_TOKENS = {TokenType.SEMICOLON, TokenType.BEGIN} 705 706 # handle numeric literals like in hive (3L = BIGINT) 707 NUMERIC_LITERALS: t.Dict[str, str] = {} 708 ENCODE: t.Optional[str] = None 709 710 COMMENTS = ["--", ("/*", "*/"), ("{#", "#}")] 711 KEYWORD_TRIE: t.Dict = {} # autofilled 712 713 IDENTIFIER_CAN_START_WITH_DIGIT = False 714 715 __slots__ = ( 716 "sql", 717 "size", 718 "tokens", 719 "_start", 720 "_current", 721 "_line", 722 "_col", 723 "_comments", 724 "_char", 725 "_end", 726 "_peek", 727 "_prev_token_line", 728 ) 729 730 def __init__(self) -> None: 731 self.reset() 732 733 def reset(self) -> None: 734 self.sql = "" 735 self.size = 0 736 self.tokens: t.List[Token] = [] 737 self._start = 0 738 self._current = 0 739 self._line = 1 740 self._col = 0 741 self._comments: t.List[str] = [] 742 743 self._char = "" 744 self._end = False 745 self._peek = "" 746 self._prev_token_line = -1 747 748 def tokenize(self, sql: str) -> t.List[Token]: 749 """Returns a list of tokens corresponding to the SQL string `sql`.""" 750 self.reset() 751 self.sql = sql 752 self.size = len(sql) 753 754 try: 755 self._scan() 756 except Exception as e: 757 start = max(self._current - 50, 0) 758 end = min(self._current + 50, self.size - 1) 759 context = self.sql[start:end] 760 raise ValueError(f"Error tokenizing '{context}'") from e 761 762 return self.tokens 763 764 def _scan(self, until: t.Optional[t.Callable] = None) -> None: 765 while self.size and not self._end: 766 self._start = self._current 767 self._advance() 768 769 if self._char is None: 770 break 771 772 if self._char not in self.WHITE_SPACE: 773 if self._char.isdigit(): 774 self._scan_number() 775 elif self._char in self._IDENTIFIERS: 776 self._scan_identifier(self._IDENTIFIERS[self._char]) 777 else: 778 self._scan_keywords() 779 780 if until and until(): 781 break 782 783 if self.tokens and self._comments: 784 self.tokens[-1].comments.extend(self._comments) 785 786 def _chars(self, size: int) -> str: 787 if size == 1: 788 return self._char 789 790 start = self._current - 1 791 end = start + size 792 793 return self.sql[start:end] if end <= self.size else "" 794 795 def _advance(self, i: int = 1, alnum: bool = False) -> None: 796 if self.WHITE_SPACE.get(self._char) is TokenType.BREAK: 797 self._col = 1 798 self._line += 1 799 else: 800 self._col += i 801 802 self._current += i 803 self._end = self._current >= self.size 804 self._char = self.sql[self._current - 1] 805 self._peek = "" if self._end else self.sql[self._current] 806 807 if alnum and self._char.isalnum(): 808 # Here we use local variables instead of attributes for better performance 809 _col = self._col 810 _current = self._current 811 _end = self._end 812 _peek = self._peek 813 814 while _peek.isalnum(): 815 _col += 1 816 _current += 1 817 _end = _current >= self.size 818 _peek = "" if _end else self.sql[_current] 819 820 self._col = _col 821 self._current = _current 822 self._end = _end 823 self._peek = _peek 824 self._char = self.sql[_current - 1] 825 826 @property 827 def _text(self) -> str: 828 return self.sql[self._start : self._current] 829 830 def _add(self, token_type: TokenType, text: t.Optional[str] = None) -> None: 831 self._prev_token_line = self._line 832 self.tokens.append( 833 Token( 834 token_type, 835 text=self._text if text is None else text, 836 line=self._line, 837 col=self._col, 838 start=self._start, 839 end=self._current - 1, 840 comments=self._comments, 841 ) 842 ) 843 self._comments = [] 844 845 # If we have either a semicolon or a begin token before the command's token, we'll parse 846 # whatever follows the command's token as a string 847 if ( 848 token_type in self.COMMANDS 849 and self._peek != ";" 850 and (len(self.tokens) == 1 or self.tokens[-2].token_type in self.COMMAND_PREFIX_TOKENS) 851 ): 852 start = self._current 853 tokens = len(self.tokens) 854 self._scan(lambda: self._peek == ";") 855 self.tokens = self.tokens[:tokens] 856 text = self.sql[start : self._current].strip() 857 if text: 858 self._add(TokenType.STRING, text) 859 860 def _scan_keywords(self) -> None: 861 size = 0 862 word = None 863 chars = self._text 864 char = chars 865 prev_space = False 866 skip = False 867 trie = self.KEYWORD_TRIE 868 single_token = char in self.SINGLE_TOKENS 869 870 while chars: 871 if skip: 872 result = 1 873 else: 874 result, trie = in_trie(trie, char.upper()) 875 876 if result == 0: 877 break 878 if result == 2: 879 word = chars 880 881 size += 1 882 end = self._current - 1 + size 883 884 if end < self.size: 885 char = self.sql[end] 886 single_token = single_token or char in self.SINGLE_TOKENS 887 is_space = char in self.WHITE_SPACE 888 889 if not is_space or not prev_space: 890 if is_space: 891 char = " " 892 chars += char 893 prev_space = is_space 894 skip = False 895 else: 896 skip = True 897 else: 898 char = "" 899 chars = " " 900 901 word = None if not single_token and chars[-1] not in self.WHITE_SPACE else word 902 903 if not word: 904 if self._char in self.SINGLE_TOKENS: 905 self._add(self.SINGLE_TOKENS[self._char], text=self._char) 906 return 907 self._scan_var() 908 return 909 910 if self._scan_string(word): 911 return 912 if self._scan_comment(word): 913 return 914 915 self._advance(size - 1) 916 word = word.upper() 917 self._add(self.KEYWORDS[word], text=word) 918 919 def _scan_comment(self, comment_start: str) -> bool: 920 if comment_start not in self._COMMENTS: 921 return False 922 923 comment_start_line = self._line 924 comment_start_size = len(comment_start) 925 comment_end = self._COMMENTS[comment_start] 926 927 if comment_end: 928 # Skip the comment's start delimiter 929 self._advance(comment_start_size) 930 931 comment_end_size = len(comment_end) 932 while not self._end and self._chars(comment_end_size) != comment_end: 933 self._advance(alnum=True) 934 935 self._comments.append(self._text[comment_start_size : -comment_end_size + 1]) 936 self._advance(comment_end_size - 1) 937 else: 938 while not self._end and not self.WHITE_SPACE.get(self._peek) is TokenType.BREAK: 939 self._advance(alnum=True) 940 self._comments.append(self._text[comment_start_size:]) 941 942 # Leading comment is attached to the succeeding token, whilst trailing comment to the preceding. 943 # Multiple consecutive comments are preserved by appending them to the current comments list. 944 if comment_start_line == self._prev_token_line: 945 self.tokens[-1].comments.extend(self._comments) 946 self._comments = [] 947 self._prev_token_line = self._line 948 949 return True 950 951 def _scan_number(self) -> None: 952 if self._char == "0": 953 peek = self._peek.upper() 954 if peek == "B": 955 return self._scan_bits() if self.BIT_STRINGS else self._add(TokenType.NUMBER) 956 elif peek == "X": 957 return self._scan_hex() if self.HEX_STRINGS else self._add(TokenType.NUMBER) 958 959 decimal = False 960 scientific = 0 961 962 while True: 963 if self._peek.isdigit(): 964 self._advance() 965 elif self._peek == "." and not decimal: 966 decimal = True 967 self._advance() 968 elif self._peek in ("-", "+") and scientific == 1: 969 scientific += 1 970 self._advance() 971 elif self._peek.upper() == "E" and not scientific: 972 scientific += 1 973 self._advance() 974 elif self._peek.isidentifier(): 975 number_text = self._text 976 literal = "" 977 978 while self._peek.strip() and self._peek not in self.SINGLE_TOKENS: 979 literal += self._peek.upper() 980 self._advance() 981 982 token_type = self.KEYWORDS.get(self.NUMERIC_LITERALS.get(literal)) 983 984 if token_type: 985 self._add(TokenType.NUMBER, number_text) 986 self._add(TokenType.DCOLON, "::") 987 return self._add(token_type, literal) 988 elif self.IDENTIFIER_CAN_START_WITH_DIGIT: 989 return self._add(TokenType.VAR) 990 991 self._add(TokenType.NUMBER, number_text) 992 return self._advance(-len(literal)) 993 else: 994 return self._add(TokenType.NUMBER) 995 996 def _scan_bits(self) -> None: 997 self._advance() 998 value = self._extract_value() 999 try: 1000 # If `value` can't be converted to a binary, fallback to tokenizing it as an identifier 1001 int(value, 2) 1002 self._add(TokenType.BIT_STRING, value[2:]) # Drop the 0b 1003 except ValueError: 1004 self._add(TokenType.IDENTIFIER) 1005 1006 def _scan_hex(self) -> None: 1007 self._advance() 1008 value = self._extract_value() 1009 try: 1010 # If `value` can't be converted to a hex, fallback to tokenizing it as an identifier 1011 int(value, 16) 1012 self._add(TokenType.HEX_STRING, value[2:]) # Drop the 0x 1013 except ValueError: 1014 self._add(TokenType.IDENTIFIER) 1015 1016 def _extract_value(self) -> str: 1017 while True: 1018 char = self._peek.strip() 1019 if char and char not in self.SINGLE_TOKENS: 1020 self._advance(alnum=True) 1021 else: 1022 break 1023 1024 return self._text 1025 1026 def _scan_string(self, start: str) -> bool: 1027 base = None 1028 token_type = TokenType.STRING 1029 1030 if start in self._QUOTES: 1031 end = self._QUOTES[start] 1032 elif start in self._FORMAT_STRINGS: 1033 end, token_type = self._FORMAT_STRINGS[start] 1034 1035 if token_type == TokenType.HEX_STRING: 1036 base = 16 1037 elif token_type == TokenType.BIT_STRING: 1038 base = 2 1039 else: 1040 return False 1041 1042 self._advance(len(start)) 1043 text = self._extract_string(end) 1044 1045 if base: 1046 try: 1047 int(text, base) 1048 except: 1049 raise RuntimeError( 1050 f"Numeric string contains invalid characters from {self._line}:{self._start}" 1051 ) 1052 else: 1053 text = text.encode(self.ENCODE).decode(self.ENCODE) if self.ENCODE else text 1054 1055 self._add(token_type, text) 1056 return True 1057 1058 def _scan_identifier(self, identifier_end: str) -> None: 1059 self._advance() 1060 text = self._extract_string(identifier_end, self._IDENTIFIER_ESCAPES) 1061 self._add(TokenType.IDENTIFIER, text) 1062 1063 def _scan_var(self) -> None: 1064 while True: 1065 char = self._peek.strip() 1066 if char and (char in self.VAR_SINGLE_TOKENS or char not in self.SINGLE_TOKENS): 1067 self._advance(alnum=True) 1068 else: 1069 break 1070 1071 self._add( 1072 TokenType.VAR 1073 if self.tokens and self.tokens[-1].token_type == TokenType.PARAMETER 1074 else self.KEYWORDS.get(self._text.upper(), TokenType.VAR) 1075 ) 1076 1077 def _extract_string(self, delimiter: str, escapes=None) -> str: 1078 text = "" 1079 delim_size = len(delimiter) 1080 escapes = self._STRING_ESCAPES if escapes is None else escapes 1081 1082 while True: 1083 if self._char in escapes and (self._peek == delimiter or self._peek in escapes): 1084 if self._peek == delimiter: 1085 text += self._peek 1086 else: 1087 text += self._char + self._peek 1088 1089 if self._current + 1 < self.size: 1090 self._advance(2) 1091 else: 1092 raise RuntimeError(f"Missing {delimiter} from {self._line}:{self._current}") 1093 else: 1094 if self._chars(delim_size) == delimiter: 1095 if delim_size > 1: 1096 self._advance(delim_size - 1) 1097 break 1098 1099 if self._end: 1100 raise RuntimeError(f"Missing {delimiter} from {self._line}:{self._start}") 1101 1102 current = self._current - 1 1103 self._advance(alnum=True) 1104 text += self.sql[current : self._current - 1] 1105 1106 return text
def
reset(self) -> None:
733 def reset(self) -> None: 734 self.sql = "" 735 self.size = 0 736 self.tokens: t.List[Token] = [] 737 self._start = 0 738 self._current = 0 739 self._line = 1 740 self._col = 0 741 self._comments: t.List[str] = [] 742 743 self._char = "" 744 self._end = False 745 self._peek = "" 746 self._prev_token_line = -1
748 def tokenize(self, sql: str) -> t.List[Token]: 749 """Returns a list of tokens corresponding to the SQL string `sql`.""" 750 self.reset() 751 self.sql = sql 752 self.size = len(sql) 753 754 try: 755 self._scan() 756 except Exception as e: 757 start = max(self._current - 50, 0) 758 end = min(self._current + 50, self.size - 1) 759 context = self.sql[start:end] 760 raise ValueError(f"Error tokenizing '{context}'") from e 761 762 return self.tokens
Returns a list of tokens corresponding to the SQL string sql
.