sqlglot.tokens
1from __future__ import annotations 2 3import typing as t 4from enum import auto 5 6from sqlglot.helper import AutoName 7from sqlglot.trie import in_trie, new_trie 8 9 10class TokenType(AutoName): 11 L_PAREN = auto() 12 R_PAREN = auto() 13 L_BRACKET = auto() 14 R_BRACKET = auto() 15 L_BRACE = auto() 16 R_BRACE = auto() 17 COMMA = auto() 18 DOT = auto() 19 DASH = auto() 20 PLUS = auto() 21 COLON = auto() 22 DCOLON = auto() 23 SEMICOLON = auto() 24 STAR = auto() 25 BACKSLASH = auto() 26 SLASH = auto() 27 LT = auto() 28 LTE = auto() 29 GT = auto() 30 GTE = auto() 31 NOT = auto() 32 EQ = auto() 33 NEQ = auto() 34 NULLSAFE_EQ = auto() 35 AND = auto() 36 OR = auto() 37 AMP = auto() 38 DPIPE = auto() 39 PIPE = auto() 40 CARET = auto() 41 TILDA = auto() 42 ARROW = auto() 43 DARROW = auto() 44 FARROW = auto() 45 HASH = auto() 46 HASH_ARROW = auto() 47 DHASH_ARROW = auto() 48 LR_ARROW = auto() 49 DOLLAR = auto() 50 PARAMETER = auto() 51 SESSION_PARAMETER = auto() 52 NATIONAL = auto() 53 54 BLOCK_START = auto() 55 BLOCK_END = auto() 56 57 SPACE = auto() 58 BREAK = auto() 59 60 STRING = auto() 61 NUMBER = auto() 62 IDENTIFIER = auto() 63 DATABASE = auto() 64 COLUMN = auto() 65 COLUMN_DEF = auto() 66 SCHEMA = auto() 67 TABLE = auto() 68 VAR = auto() 69 BIT_STRING = auto() 70 HEX_STRING = auto() 71 BYTE_STRING = auto() 72 73 # types 74 BOOLEAN = auto() 75 TINYINT = auto() 76 SMALLINT = auto() 77 INT = auto() 78 BIGINT = auto() 79 FLOAT = auto() 80 DOUBLE = auto() 81 DECIMAL = auto() 82 CHAR = auto() 83 NCHAR = auto() 84 VARCHAR = auto() 85 NVARCHAR = auto() 86 TEXT = auto() 87 MEDIUMTEXT = auto() 88 LONGTEXT = auto() 89 MEDIUMBLOB = auto() 90 LONGBLOB = auto() 91 BINARY = auto() 92 VARBINARY = auto() 93 JSON = auto() 94 JSONB = auto() 95 TIME = auto() 96 TIMESTAMP = auto() 97 TIMESTAMPTZ = auto() 98 TIMESTAMPLTZ = auto() 99 DATETIME = auto() 100 DATE = auto() 101 UUID = auto() 102 GEOGRAPHY = auto() 103 NULLABLE = auto() 104 GEOMETRY = auto() 105 HLLSKETCH = auto() 106 HSTORE = auto() 107 SUPER = auto() 108 SERIAL = auto() 109 SMALLSERIAL = auto() 110 BIGSERIAL = auto() 111 XML = auto() 112 UNIQUEIDENTIFIER = auto() 113 MONEY = auto() 114 SMALLMONEY = auto() 115 ROWVERSION = auto() 116 IMAGE = auto() 117 VARIANT = auto() 118 OBJECT = auto() 119 INET = auto() 120 121 # keywords 122 ALIAS = auto() 123 ALTER = auto() 124 ALWAYS = auto() 125 ALL = auto() 126 ANTI = auto() 127 ANY = auto() 128 APPLY = auto() 129 ARRAY = auto() 130 ASC = auto() 131 ASOF = auto() 132 AT_TIME_ZONE = auto() 133 AUTO_INCREMENT = auto() 134 BEGIN = auto() 135 BETWEEN = auto() 136 BOTH = auto() 137 BUCKET = auto() 138 BY_DEFAULT = auto() 139 CACHE = auto() 140 CASCADE = auto() 141 CASE = auto() 142 CHARACTER_SET = auto() 143 CLUSTER_BY = auto() 144 COLLATE = auto() 145 COMMAND = auto() 146 COMMENT = auto() 147 COMMIT = auto() 148 COMPOUND = auto() 149 CONSTRAINT = auto() 150 CREATE = auto() 151 CROSS = auto() 152 CUBE = auto() 153 CURRENT_DATE = auto() 154 CURRENT_DATETIME = auto() 155 CURRENT_ROW = auto() 156 CURRENT_TIME = auto() 157 CURRENT_TIMESTAMP = auto() 158 DEFAULT = auto() 159 DELETE = auto() 160 DESC = auto() 161 DESCRIBE = auto() 162 DISTINCT = auto() 163 DISTINCT_FROM = auto() 164 DISTRIBUTE_BY = auto() 165 DIV = auto() 166 DROP = auto() 167 ELSE = auto() 168 END = auto() 169 ESCAPE = auto() 170 EXCEPT = auto() 171 EXECUTE = auto() 172 EXISTS = auto() 173 FALSE = auto() 174 FETCH = auto() 175 FILTER = auto() 176 FINAL = auto() 177 FIRST = auto() 178 FOLLOWING = auto() 179 FOR = auto() 180 FOREIGN_KEY = auto() 181 FORMAT = auto() 182 FROM = auto() 183 FULL = auto() 184 FUNCTION = auto() 185 GLOB = auto() 186 GLOBAL = auto() 187 GROUP_BY = auto() 188 GROUPING_SETS = auto() 189 HAVING = auto() 190 HINT = auto() 191 IF = auto() 192 IGNORE_NULLS = auto() 193 ILIKE = auto() 194 ILIKE_ANY = auto() 195 IN = auto() 196 INDEX = auto() 197 INNER = auto() 198 INSERT = auto() 199 INTERSECT = auto() 200 INTERVAL = auto() 201 INTO = auto() 202 INTRODUCER = auto() 203 IRLIKE = auto() 204 IS = auto() 205 ISNULL = auto() 206 JOIN = auto() 207 JOIN_MARKER = auto() 208 LANGUAGE = auto() 209 LATERAL = auto() 210 LAZY = auto() 211 LEADING = auto() 212 LEFT = auto() 213 LIKE = auto() 214 LIKE_ANY = auto() 215 LIMIT = auto() 216 LOAD_DATA = auto() 217 LOCAL = auto() 218 MAP = auto() 219 MATCH_RECOGNIZE = auto() 220 MATERIALIZED = auto() 221 MERGE = auto() 222 MOD = auto() 223 NATURAL = auto() 224 NEXT = auto() 225 NO_ACTION = auto() 226 NOTNULL = auto() 227 NULL = auto() 228 NULLS_FIRST = auto() 229 NULLS_LAST = auto() 230 OFFSET = auto() 231 ON = auto() 232 ONLY = auto() 233 OPTIONS = auto() 234 ORDER_BY = auto() 235 ORDERED = auto() 236 ORDINALITY = auto() 237 OUTER = auto() 238 OUT_OF = auto() 239 OVER = auto() 240 OVERLAPS = auto() 241 OVERWRITE = auto() 242 PARTITION = auto() 243 PARTITION_BY = auto() 244 PERCENT = auto() 245 PIVOT = auto() 246 PLACEHOLDER = auto() 247 PRECEDING = auto() 248 PRIMARY_KEY = auto() 249 PROCEDURE = auto() 250 PROPERTIES = auto() 251 PSEUDO_TYPE = auto() 252 QUALIFY = auto() 253 QUOTE = auto() 254 RANGE = auto() 255 RECURSIVE = auto() 256 REPLACE = auto() 257 RESPECT_NULLS = auto() 258 RETURNING = auto() 259 REFERENCES = auto() 260 RIGHT = auto() 261 RLIKE = auto() 262 ROLLBACK = auto() 263 ROLLUP = auto() 264 ROW = auto() 265 ROWS = auto() 266 SEED = auto() 267 SELECT = auto() 268 SEMI = auto() 269 SEPARATOR = auto() 270 SERDE_PROPERTIES = auto() 271 SET = auto() 272 SHOW = auto() 273 SIMILAR_TO = auto() 274 SOME = auto() 275 SORTKEY = auto() 276 SORT_BY = auto() 277 STRUCT = auto() 278 TABLE_SAMPLE = auto() 279 TEMPORARY = auto() 280 TOP = auto() 281 THEN = auto() 282 TRAILING = auto() 283 TRUE = auto() 284 UNBOUNDED = auto() 285 UNCACHE = auto() 286 UNION = auto() 287 UNLOGGED = auto() 288 UNNEST = auto() 289 UNPIVOT = auto() 290 UPDATE = auto() 291 USE = auto() 292 USING = auto() 293 VALUES = auto() 294 VIEW = auto() 295 VOLATILE = auto() 296 WHEN = auto() 297 WHERE = auto() 298 WINDOW = auto() 299 WITH = auto() 300 WITH_TIME_ZONE = auto() 301 WITH_LOCAL_TIME_ZONE = auto() 302 WITHIN_GROUP = auto() 303 WITHOUT_TIME_ZONE = auto() 304 UNIQUE = auto() 305 306 307class Token: 308 __slots__ = ("token_type", "text", "line", "col", "comments") 309 310 @classmethod 311 def number(cls, number: int) -> Token: 312 """Returns a NUMBER token with `number` as its text.""" 313 return cls(TokenType.NUMBER, str(number)) 314 315 @classmethod 316 def string(cls, string: str) -> Token: 317 """Returns a STRING token with `string` as its text.""" 318 return cls(TokenType.STRING, string) 319 320 @classmethod 321 def identifier(cls, identifier: str) -> Token: 322 """Returns an IDENTIFIER token with `identifier` as its text.""" 323 return cls(TokenType.IDENTIFIER, identifier) 324 325 @classmethod 326 def var(cls, var: str) -> Token: 327 """Returns an VAR token with `var` as its text.""" 328 return cls(TokenType.VAR, var) 329 330 def __init__( 331 self, 332 token_type: TokenType, 333 text: str, 334 line: int = 1, 335 col: int = 1, 336 comments: t.List[str] = [], 337 ) -> None: 338 self.token_type = token_type 339 self.text = text 340 self.line = line 341 self.col = max(col - len(text), 1) 342 self.comments = comments 343 344 def __repr__(self) -> str: 345 attributes = ", ".join(f"{k}: {getattr(self, k)}" for k in self.__slots__) 346 return f"<Token {attributes}>" 347 348 349class _Tokenizer(type): 350 def __new__(cls, clsname, bases, attrs): # type: ignore 351 klass = super().__new__(cls, clsname, bases, attrs) 352 353 klass._QUOTES = { 354 f"{prefix}{s}": e 355 for s, e in cls._delimeter_list_to_dict(klass.QUOTES).items() 356 for prefix in (("",) if s[0].isalpha() else ("", "n", "N")) 357 } 358 klass._BIT_STRINGS = cls._delimeter_list_to_dict(klass.BIT_STRINGS) 359 klass._HEX_STRINGS = cls._delimeter_list_to_dict(klass.HEX_STRINGS) 360 klass._BYTE_STRINGS = cls._delimeter_list_to_dict(klass.BYTE_STRINGS) 361 klass._IDENTIFIERS = cls._delimeter_list_to_dict(klass.IDENTIFIERS) 362 klass._STRING_ESCAPES = set(klass.STRING_ESCAPES) 363 klass._IDENTIFIER_ESCAPES = set(klass.IDENTIFIER_ESCAPES) 364 klass._COMMENTS = dict( 365 (comment, None) if isinstance(comment, str) else (comment[0], comment[1]) 366 for comment in klass.COMMENTS 367 ) 368 369 klass.KEYWORD_TRIE = new_trie( 370 key.upper() 371 for key in { 372 **klass.KEYWORDS, 373 **{comment: TokenType.COMMENT for comment in klass._COMMENTS}, 374 **{quote: TokenType.QUOTE for quote in klass._QUOTES}, 375 **{bit_string: TokenType.BIT_STRING for bit_string in klass._BIT_STRINGS}, 376 **{hex_string: TokenType.HEX_STRING for hex_string in klass._HEX_STRINGS}, 377 **{byte_string: TokenType.BYTE_STRING for byte_string in klass._BYTE_STRINGS}, 378 } 379 if " " in key or any(single in key for single in klass.SINGLE_TOKENS) 380 ) 381 382 return klass 383 384 @staticmethod 385 def _delimeter_list_to_dict(list: t.List[str | t.Tuple[str, str]]) -> t.Dict[str, str]: 386 return dict((item, item) if isinstance(item, str) else (item[0], item[1]) for item in list) 387 388 389class Tokenizer(metaclass=_Tokenizer): 390 SINGLE_TOKENS = { 391 "(": TokenType.L_PAREN, 392 ")": TokenType.R_PAREN, 393 "[": TokenType.L_BRACKET, 394 "]": TokenType.R_BRACKET, 395 "{": TokenType.L_BRACE, 396 "}": TokenType.R_BRACE, 397 "&": TokenType.AMP, 398 "^": TokenType.CARET, 399 ":": TokenType.COLON, 400 ",": TokenType.COMMA, 401 ".": TokenType.DOT, 402 "-": TokenType.DASH, 403 "=": TokenType.EQ, 404 ">": TokenType.GT, 405 "<": TokenType.LT, 406 "%": TokenType.MOD, 407 "!": TokenType.NOT, 408 "|": TokenType.PIPE, 409 "+": TokenType.PLUS, 410 ";": TokenType.SEMICOLON, 411 "/": TokenType.SLASH, 412 "\\": TokenType.BACKSLASH, 413 "*": TokenType.STAR, 414 "~": TokenType.TILDA, 415 "?": TokenType.PLACEHOLDER, 416 "@": TokenType.PARAMETER, 417 # used for breaking a var like x'y' but nothing else 418 # the token type doesn't matter 419 "'": TokenType.QUOTE, 420 "`": TokenType.IDENTIFIER, 421 '"': TokenType.IDENTIFIER, 422 "#": TokenType.HASH, 423 } 424 425 QUOTES: t.List[t.Tuple[str, str] | str] = ["'"] 426 427 BIT_STRINGS: t.List[str | t.Tuple[str, str]] = [] 428 429 HEX_STRINGS: t.List[str | t.Tuple[str, str]] = [] 430 431 BYTE_STRINGS: t.List[str | t.Tuple[str, str]] = [] 432 433 IDENTIFIERS: t.List[str | t.Tuple[str, str]] = ['"'] 434 435 STRING_ESCAPES = ["'"] 436 437 _STRING_ESCAPES: t.Set[str] = set() 438 439 IDENTIFIER_ESCAPES = ['"'] 440 441 _IDENTIFIER_ESCAPES: t.Set[str] = set() 442 443 KEYWORDS = { 444 **{f"{{%{postfix}": TokenType.BLOCK_START for postfix in ("", "+", "-")}, 445 **{f"{prefix}%}}": TokenType.BLOCK_END for prefix in ("", "+", "-")}, 446 "{{+": TokenType.BLOCK_START, 447 "{{-": TokenType.BLOCK_START, 448 "+}}": TokenType.BLOCK_END, 449 "-}}": TokenType.BLOCK_END, 450 "/*+": TokenType.HINT, 451 "==": TokenType.EQ, 452 "::": TokenType.DCOLON, 453 "||": TokenType.DPIPE, 454 ">=": TokenType.GTE, 455 "<=": TokenType.LTE, 456 "<>": TokenType.NEQ, 457 "!=": TokenType.NEQ, 458 "<=>": TokenType.NULLSAFE_EQ, 459 "->": TokenType.ARROW, 460 "->>": TokenType.DARROW, 461 "=>": TokenType.FARROW, 462 "#>": TokenType.HASH_ARROW, 463 "#>>": TokenType.DHASH_ARROW, 464 "<->": TokenType.LR_ARROW, 465 "ALL": TokenType.ALL, 466 "ALWAYS": TokenType.ALWAYS, 467 "AND": TokenType.AND, 468 "ANTI": TokenType.ANTI, 469 "ANY": TokenType.ANY, 470 "ASC": TokenType.ASC, 471 "AS": TokenType.ALIAS, 472 "AT TIME ZONE": TokenType.AT_TIME_ZONE, 473 "AUTOINCREMENT": TokenType.AUTO_INCREMENT, 474 "AUTO_INCREMENT": TokenType.AUTO_INCREMENT, 475 "BEGIN": TokenType.BEGIN, 476 "BETWEEN": TokenType.BETWEEN, 477 "BOTH": TokenType.BOTH, 478 "BUCKET": TokenType.BUCKET, 479 "BY DEFAULT": TokenType.BY_DEFAULT, 480 "CACHE": TokenType.CACHE, 481 "UNCACHE": TokenType.UNCACHE, 482 "CASE": TokenType.CASE, 483 "CASCADE": TokenType.CASCADE, 484 "CHARACTER SET": TokenType.CHARACTER_SET, 485 "CLUSTER BY": TokenType.CLUSTER_BY, 486 "COLLATE": TokenType.COLLATE, 487 "COLUMN": TokenType.COLUMN, 488 "COMMIT": TokenType.COMMIT, 489 "COMPOUND": TokenType.COMPOUND, 490 "CONSTRAINT": TokenType.CONSTRAINT, 491 "CREATE": TokenType.CREATE, 492 "CROSS": TokenType.CROSS, 493 "CUBE": TokenType.CUBE, 494 "CURRENT_DATE": TokenType.CURRENT_DATE, 495 "CURRENT ROW": TokenType.CURRENT_ROW, 496 "CURRENT_TIMESTAMP": TokenType.CURRENT_TIMESTAMP, 497 "DATABASE": TokenType.DATABASE, 498 "DEFAULT": TokenType.DEFAULT, 499 "DELETE": TokenType.DELETE, 500 "DESC": TokenType.DESC, 501 "DESCRIBE": TokenType.DESCRIBE, 502 "DISTINCT": TokenType.DISTINCT, 503 "DISTINCT FROM": TokenType.DISTINCT_FROM, 504 "DISTRIBUTE BY": TokenType.DISTRIBUTE_BY, 505 "DIV": TokenType.DIV, 506 "DROP": TokenType.DROP, 507 "ELSE": TokenType.ELSE, 508 "END": TokenType.END, 509 "ESCAPE": TokenType.ESCAPE, 510 "EXCEPT": TokenType.EXCEPT, 511 "EXECUTE": TokenType.EXECUTE, 512 "EXISTS": TokenType.EXISTS, 513 "FALSE": TokenType.FALSE, 514 "FETCH": TokenType.FETCH, 515 "FILTER": TokenType.FILTER, 516 "FIRST": TokenType.FIRST, 517 "FULL": TokenType.FULL, 518 "FUNCTION": TokenType.FUNCTION, 519 "FOLLOWING": TokenType.FOLLOWING, 520 "FOR": TokenType.FOR, 521 "FOREIGN KEY": TokenType.FOREIGN_KEY, 522 "FORMAT": TokenType.FORMAT, 523 "FROM": TokenType.FROM, 524 "GLOB": TokenType.GLOB, 525 "GROUP BY": TokenType.GROUP_BY, 526 "GROUPING SETS": TokenType.GROUPING_SETS, 527 "HAVING": TokenType.HAVING, 528 "IF": TokenType.IF, 529 "ILIKE": TokenType.ILIKE, 530 "IGNORE NULLS": TokenType.IGNORE_NULLS, 531 "IN": TokenType.IN, 532 "INDEX": TokenType.INDEX, 533 "INET": TokenType.INET, 534 "INNER": TokenType.INNER, 535 "INSERT": TokenType.INSERT, 536 "INTERVAL": TokenType.INTERVAL, 537 "INTERSECT": TokenType.INTERSECT, 538 "INTO": TokenType.INTO, 539 "IS": TokenType.IS, 540 "ISNULL": TokenType.ISNULL, 541 "JOIN": TokenType.JOIN, 542 "LATERAL": TokenType.LATERAL, 543 "LAZY": TokenType.LAZY, 544 "LEADING": TokenType.LEADING, 545 "LEFT": TokenType.LEFT, 546 "LIKE": TokenType.LIKE, 547 "LIMIT": TokenType.LIMIT, 548 "LOAD DATA": TokenType.LOAD_DATA, 549 "LOCAL": TokenType.LOCAL, 550 "MATERIALIZED": TokenType.MATERIALIZED, 551 "MERGE": TokenType.MERGE, 552 "NATURAL": TokenType.NATURAL, 553 "NEXT": TokenType.NEXT, 554 "NO ACTION": TokenType.NO_ACTION, 555 "NOT": TokenType.NOT, 556 "NOTNULL": TokenType.NOTNULL, 557 "NULL": TokenType.NULL, 558 "NULLS FIRST": TokenType.NULLS_FIRST, 559 "NULLS LAST": TokenType.NULLS_LAST, 560 "OBJECT": TokenType.OBJECT, 561 "OFFSET": TokenType.OFFSET, 562 "ON": TokenType.ON, 563 "ONLY": TokenType.ONLY, 564 "OPTIONS": TokenType.OPTIONS, 565 "OR": TokenType.OR, 566 "ORDER BY": TokenType.ORDER_BY, 567 "ORDINALITY": TokenType.ORDINALITY, 568 "OUTER": TokenType.OUTER, 569 "OUT OF": TokenType.OUT_OF, 570 "OVER": TokenType.OVER, 571 "OVERLAPS": TokenType.OVERLAPS, 572 "OVERWRITE": TokenType.OVERWRITE, 573 "PARTITION": TokenType.PARTITION, 574 "PARTITION BY": TokenType.PARTITION_BY, 575 "PARTITIONED BY": TokenType.PARTITION_BY, 576 "PARTITIONED_BY": TokenType.PARTITION_BY, 577 "PERCENT": TokenType.PERCENT, 578 "PIVOT": TokenType.PIVOT, 579 "PRECEDING": TokenType.PRECEDING, 580 "PRIMARY KEY": TokenType.PRIMARY_KEY, 581 "PROCEDURE": TokenType.PROCEDURE, 582 "QUALIFY": TokenType.QUALIFY, 583 "RANGE": TokenType.RANGE, 584 "RECURSIVE": TokenType.RECURSIVE, 585 "REGEXP": TokenType.RLIKE, 586 "REPLACE": TokenType.REPLACE, 587 "RESPECT NULLS": TokenType.RESPECT_NULLS, 588 "REFERENCES": TokenType.REFERENCES, 589 "RIGHT": TokenType.RIGHT, 590 "RLIKE": TokenType.RLIKE, 591 "ROLLBACK": TokenType.ROLLBACK, 592 "ROLLUP": TokenType.ROLLUP, 593 "ROW": TokenType.ROW, 594 "ROWS": TokenType.ROWS, 595 "SCHEMA": TokenType.SCHEMA, 596 "SEED": TokenType.SEED, 597 "SELECT": TokenType.SELECT, 598 "SEMI": TokenType.SEMI, 599 "SET": TokenType.SET, 600 "SHOW": TokenType.SHOW, 601 "SIMILAR TO": TokenType.SIMILAR_TO, 602 "SOME": TokenType.SOME, 603 "SORTKEY": TokenType.SORTKEY, 604 "SORT BY": TokenType.SORT_BY, 605 "TABLE": TokenType.TABLE, 606 "TABLESAMPLE": TokenType.TABLE_SAMPLE, 607 "TEMP": TokenType.TEMPORARY, 608 "TEMPORARY": TokenType.TEMPORARY, 609 "THEN": TokenType.THEN, 610 "TRUE": TokenType.TRUE, 611 "TRAILING": TokenType.TRAILING, 612 "UNBOUNDED": TokenType.UNBOUNDED, 613 "UNION": TokenType.UNION, 614 "UNLOGGED": TokenType.UNLOGGED, 615 "UNNEST": TokenType.UNNEST, 616 "UNPIVOT": TokenType.UNPIVOT, 617 "UPDATE": TokenType.UPDATE, 618 "USE": TokenType.USE, 619 "USING": TokenType.USING, 620 "VALUES": TokenType.VALUES, 621 "VIEW": TokenType.VIEW, 622 "VOLATILE": TokenType.VOLATILE, 623 "WHEN": TokenType.WHEN, 624 "WHERE": TokenType.WHERE, 625 "WINDOW": TokenType.WINDOW, 626 "WITH": TokenType.WITH, 627 "WITH TIME ZONE": TokenType.WITH_TIME_ZONE, 628 "WITH LOCAL TIME ZONE": TokenType.WITH_LOCAL_TIME_ZONE, 629 "WITHIN GROUP": TokenType.WITHIN_GROUP, 630 "WITHOUT TIME ZONE": TokenType.WITHOUT_TIME_ZONE, 631 "APPLY": TokenType.APPLY, 632 "ARRAY": TokenType.ARRAY, 633 "BOOL": TokenType.BOOLEAN, 634 "BOOLEAN": TokenType.BOOLEAN, 635 "BYTE": TokenType.TINYINT, 636 "TINYINT": TokenType.TINYINT, 637 "SHORT": TokenType.SMALLINT, 638 "SMALLINT": TokenType.SMALLINT, 639 "INT2": TokenType.SMALLINT, 640 "INTEGER": TokenType.INT, 641 "INT": TokenType.INT, 642 "INT4": TokenType.INT, 643 "LONG": TokenType.BIGINT, 644 "BIGINT": TokenType.BIGINT, 645 "INT8": TokenType.BIGINT, 646 "DECIMAL": TokenType.DECIMAL, 647 "MAP": TokenType.MAP, 648 "NULLABLE": TokenType.NULLABLE, 649 "NUMBER": TokenType.DECIMAL, 650 "NUMERIC": TokenType.DECIMAL, 651 "FIXED": TokenType.DECIMAL, 652 "REAL": TokenType.FLOAT, 653 "FLOAT": TokenType.FLOAT, 654 "FLOAT4": TokenType.FLOAT, 655 "FLOAT8": TokenType.DOUBLE, 656 "DOUBLE": TokenType.DOUBLE, 657 "DOUBLE PRECISION": TokenType.DOUBLE, 658 "JSON": TokenType.JSON, 659 "CHAR": TokenType.CHAR, 660 "CHARACTER": TokenType.CHAR, 661 "NCHAR": TokenType.NCHAR, 662 "VARCHAR": TokenType.VARCHAR, 663 "VARCHAR2": TokenType.VARCHAR, 664 "NVARCHAR": TokenType.NVARCHAR, 665 "NVARCHAR2": TokenType.NVARCHAR, 666 "STR": TokenType.TEXT, 667 "STRING": TokenType.TEXT, 668 "TEXT": TokenType.TEXT, 669 "CLOB": TokenType.TEXT, 670 "LONGVARCHAR": TokenType.TEXT, 671 "BINARY": TokenType.BINARY, 672 "BLOB": TokenType.VARBINARY, 673 "BYTEA": TokenType.VARBINARY, 674 "VARBINARY": TokenType.VARBINARY, 675 "TIME": TokenType.TIME, 676 "TIMESTAMP": TokenType.TIMESTAMP, 677 "TIMESTAMPTZ": TokenType.TIMESTAMPTZ, 678 "TIMESTAMPLTZ": TokenType.TIMESTAMPLTZ, 679 "DATE": TokenType.DATE, 680 "DATETIME": TokenType.DATETIME, 681 "UNIQUE": TokenType.UNIQUE, 682 "STRUCT": TokenType.STRUCT, 683 "VARIANT": TokenType.VARIANT, 684 "ALTER": TokenType.ALTER, 685 "ALTER AGGREGATE": TokenType.COMMAND, 686 "ALTER DEFAULT": TokenType.COMMAND, 687 "ALTER DOMAIN": TokenType.COMMAND, 688 "ALTER ROLE": TokenType.COMMAND, 689 "ALTER RULE": TokenType.COMMAND, 690 "ALTER SEQUENCE": TokenType.COMMAND, 691 "ALTER TYPE": TokenType.COMMAND, 692 "ALTER USER": TokenType.COMMAND, 693 "ALTER VIEW": TokenType.COMMAND, 694 "ANALYZE": TokenType.COMMAND, 695 "CALL": TokenType.COMMAND, 696 "COMMENT": TokenType.COMMENT, 697 "COPY": TokenType.COMMAND, 698 "EXPLAIN": TokenType.COMMAND, 699 "GRANT": TokenType.COMMAND, 700 "OPTIMIZE": TokenType.COMMAND, 701 "PREPARE": TokenType.COMMAND, 702 "TRUNCATE": TokenType.COMMAND, 703 "VACUUM": TokenType.COMMAND, 704 } 705 706 WHITE_SPACE: t.Dict[str, TokenType] = { 707 " ": TokenType.SPACE, 708 "\t": TokenType.SPACE, 709 "\n": TokenType.BREAK, 710 "\r": TokenType.BREAK, 711 "\r\n": TokenType.BREAK, 712 } 713 714 COMMANDS = { 715 TokenType.COMMAND, 716 TokenType.EXECUTE, 717 TokenType.FETCH, 718 TokenType.SET, 719 TokenType.SHOW, 720 } 721 722 COMMAND_PREFIX_TOKENS = {TokenType.SEMICOLON, TokenType.BEGIN} 723 724 # handle numeric literals like in hive (3L = BIGINT) 725 NUMERIC_LITERALS: t.Dict[str, str] = {} 726 ENCODE: t.Optional[str] = None 727 728 COMMENTS = ["--", ("/*", "*/"), ("{#", "#}")] 729 KEYWORD_TRIE = None # autofilled 730 731 IDENTIFIER_CAN_START_WITH_DIGIT = False 732 733 __slots__ = ( 734 "sql", 735 "size", 736 "tokens", 737 "_start", 738 "_current", 739 "_line", 740 "_col", 741 "_comments", 742 "_char", 743 "_end", 744 "_peek", 745 "_prev_token_line", 746 "_prev_token_comments", 747 "_prev_token_type", 748 ) 749 750 def __init__(self) -> None: 751 self.reset() 752 753 def reset(self) -> None: 754 self.sql = "" 755 self.size = 0 756 self.tokens: t.List[Token] = [] 757 self._start = 0 758 self._current = 0 759 self._line = 1 760 self._col = 1 761 self._comments: t.List[str] = [] 762 763 self._char = None 764 self._end = None 765 self._peek = None 766 self._prev_token_line = -1 767 self._prev_token_comments: t.List[str] = [] 768 self._prev_token_type = None 769 770 def tokenize(self, sql: str) -> t.List[Token]: 771 """Returns a list of tokens corresponding to the SQL string `sql`.""" 772 self.reset() 773 self.sql = sql 774 self.size = len(sql) 775 self._scan() 776 return self.tokens 777 778 def _scan(self, until: t.Optional[t.Callable] = None) -> None: 779 while self.size and not self._end: 780 self._start = self._current 781 self._advance() 782 783 if self._char is None: 784 break 785 786 if self._char not in self.WHITE_SPACE: 787 if self._char.isdigit(): 788 self._scan_number() 789 elif self._char in self._IDENTIFIERS: 790 self._scan_identifier(self._IDENTIFIERS[self._char]) 791 else: 792 self._scan_keywords() 793 794 if until and until(): 795 break 796 797 def _chars(self, size: int) -> str: 798 if size == 1: 799 return self._char # type: ignore 800 start = self._current - 1 801 end = start + size 802 if end <= self.size: 803 return self.sql[start:end] 804 return "" 805 806 def _line_break(self, char: t.Optional[str]) -> bool: 807 return self.WHITE_SPACE.get(char) == TokenType.BREAK # type: ignore 808 809 def _advance(self, i: int = 1) -> None: 810 if self._line_break(self._char): 811 self._set_new_line() 812 813 self._col += i 814 self._current += i 815 self._end = self._current >= self.size # type: ignore 816 self._char = self.sql[self._current - 1] # type: ignore 817 self._peek = self.sql[self._current] if self._current < self.size else "" # type: ignore 818 819 def _set_new_line(self) -> None: 820 self._col = 1 821 self._line += 1 822 823 @property 824 def _text(self) -> str: 825 return self.sql[self._start : self._current] 826 827 def _add(self, token_type: TokenType, text: t.Optional[str] = None) -> None: 828 self._prev_token_line = self._line 829 self._prev_token_comments = self._comments 830 self._prev_token_type = token_type # type: ignore 831 self.tokens.append( 832 Token( 833 token_type, 834 self._text if text is None else text, 835 self._line, 836 self._col, 837 self._comments, 838 ) 839 ) 840 self._comments = [] 841 842 # If we have either a semicolon or a begin token before the command's token, we'll parse 843 # whatever follows the command's token as a string 844 if token_type in self.COMMANDS and ( 845 len(self.tokens) == 1 or self.tokens[-2].token_type in self.COMMAND_PREFIX_TOKENS 846 ): 847 start = self._current 848 tokens = len(self.tokens) 849 self._scan(lambda: self._peek == ";") 850 self.tokens = self.tokens[:tokens] 851 text = self.sql[start : self._current].strip() 852 if text: 853 self._add(TokenType.STRING, text) 854 855 def _scan_keywords(self) -> None: 856 size = 0 857 word = None 858 chars: t.Optional[str] = self._text 859 char = chars 860 prev_space = False 861 skip = False 862 trie = self.KEYWORD_TRIE 863 864 while chars: 865 if skip: 866 result = 1 867 else: 868 result, trie = in_trie(trie, char.upper()) # type: ignore 869 870 if result == 0: 871 break 872 if result == 2: 873 word = chars 874 size += 1 875 end = self._current - 1 + size 876 877 if end < self.size: 878 char = self.sql[end] 879 is_space = char in self.WHITE_SPACE 880 881 if not is_space or not prev_space: 882 if is_space: 883 char = " " 884 chars += char 885 prev_space = is_space 886 skip = False 887 else: 888 skip = True 889 else: 890 chars = None 891 892 if not word: 893 if self._char in self.SINGLE_TOKENS: 894 self._add(self.SINGLE_TOKENS[self._char]) # type: ignore 895 return 896 self._scan_var() 897 return 898 899 if self._scan_string(word): 900 return 901 if self._scan_formatted_string(word): 902 return 903 if self._scan_comment(word): 904 return 905 906 self._advance(size - 1) 907 self._add(self.KEYWORDS[word.upper()]) 908 909 def _scan_comment(self, comment_start: str) -> bool: 910 if comment_start not in self._COMMENTS: # type: ignore 911 return False 912 913 comment_start_line = self._line 914 comment_start_size = len(comment_start) 915 comment_end = self._COMMENTS[comment_start] # type: ignore 916 917 if comment_end: 918 comment_end_size = len(comment_end) 919 920 while not self._end and self._chars(comment_end_size) != comment_end: 921 self._advance() 922 923 self._comments.append(self._text[comment_start_size : -comment_end_size + 1]) # type: ignore 924 self._advance(comment_end_size - 1) 925 else: 926 while not self._end and not self._line_break(self._peek): 927 self._advance() 928 self._comments.append(self._text[comment_start_size:]) # type: ignore 929 930 # Leading comment is attached to the succeeding token, whilst trailing comment to the preceding. 931 # Multiple consecutive comments are preserved by appending them to the current comments list. 932 if comment_start_line == self._prev_token_line: 933 self.tokens[-1].comments.extend(self._comments) 934 self._comments = [] 935 self._prev_token_line = self._line 936 937 return True 938 939 def _scan_number(self) -> None: 940 if self._char == "0": 941 peek = self._peek.upper() # type: ignore 942 if peek == "B": 943 return self._scan_bits() 944 elif peek == "X": 945 return self._scan_hex() 946 947 decimal = False 948 scientific = 0 949 950 while True: 951 if self._peek.isdigit(): # type: ignore 952 self._advance() 953 elif self._peek == "." and not decimal: 954 decimal = True 955 self._advance() 956 elif self._peek in ("-", "+") and scientific == 1: 957 scientific += 1 958 self._advance() 959 elif self._peek.upper() == "E" and not scientific: # type: ignore 960 scientific += 1 961 self._advance() 962 elif self._peek.isidentifier(): # type: ignore 963 number_text = self._text 964 literal = [] 965 966 while self._peek.strip() and self._peek not in self.SINGLE_TOKENS: # type: ignore 967 literal.append(self._peek.upper()) # type: ignore 968 self._advance() 969 970 literal = "".join(literal) # type: ignore 971 token_type = self.KEYWORDS.get(self.NUMERIC_LITERALS.get(literal)) # type: ignore 972 973 if token_type: 974 self._add(TokenType.NUMBER, number_text) 975 self._add(TokenType.DCOLON, "::") 976 return self._add(token_type, literal) # type: ignore 977 elif self.IDENTIFIER_CAN_START_WITH_DIGIT: 978 return self._add(TokenType.VAR) 979 980 self._add(TokenType.NUMBER, number_text) 981 return self._advance(-len(literal)) 982 else: 983 return self._add(TokenType.NUMBER) 984 985 def _scan_bits(self) -> None: 986 self._advance() 987 value = self._extract_value() 988 try: 989 self._add(TokenType.BIT_STRING, f"{int(value, 2)}") 990 except ValueError: 991 self._add(TokenType.IDENTIFIER) 992 993 def _scan_hex(self) -> None: 994 self._advance() 995 value = self._extract_value() 996 try: 997 self._add(TokenType.HEX_STRING, f"{int(value, 16)}") 998 except ValueError: 999 self._add(TokenType.IDENTIFIER) 1000 1001 def _extract_value(self) -> str: 1002 while True: 1003 char = self._peek.strip() # type: ignore 1004 if char and char not in self.SINGLE_TOKENS: 1005 self._advance() 1006 else: 1007 break 1008 1009 return self._text 1010 1011 def _scan_string(self, quote: str) -> bool: 1012 quote_end = self._QUOTES.get(quote) # type: ignore 1013 if quote_end is None: 1014 return False 1015 1016 self._advance(len(quote)) 1017 text = self._extract_string(quote_end) 1018 text = text.encode(self.ENCODE).decode(self.ENCODE) if self.ENCODE else text # type: ignore 1019 self._add(TokenType.NATIONAL if quote[0].upper() == "N" else TokenType.STRING, text) 1020 return True 1021 1022 # X'1234, b'0110', E'\\\\\' etc. 1023 def _scan_formatted_string(self, string_start: str) -> bool: 1024 if string_start in self._HEX_STRINGS: # type: ignore 1025 delimiters = self._HEX_STRINGS # type: ignore 1026 token_type = TokenType.HEX_STRING 1027 base = 16 1028 elif string_start in self._BIT_STRINGS: # type: ignore 1029 delimiters = self._BIT_STRINGS # type: ignore 1030 token_type = TokenType.BIT_STRING 1031 base = 2 1032 elif string_start in self._BYTE_STRINGS: # type: ignore 1033 delimiters = self._BYTE_STRINGS # type: ignore 1034 token_type = TokenType.BYTE_STRING 1035 base = None 1036 else: 1037 return False 1038 1039 self._advance(len(string_start)) 1040 string_end = delimiters.get(string_start) 1041 text = self._extract_string(string_end) 1042 1043 if base is None: 1044 self._add(token_type, text) 1045 else: 1046 try: 1047 self._add(token_type, f"{int(text, base)}") 1048 except: 1049 raise RuntimeError( 1050 f"Numeric string contains invalid characters from {self._line}:{self._start}" 1051 ) 1052 1053 return True 1054 1055 def _scan_identifier(self, identifier_end: str) -> None: 1056 text = "" 1057 identifier_end_is_escape = identifier_end in self._IDENTIFIER_ESCAPES 1058 1059 while True: 1060 if self._end: 1061 raise RuntimeError(f"Missing {identifier_end} from {self._line}:{self._start}") 1062 1063 self._advance() 1064 if self._char == identifier_end: 1065 if identifier_end_is_escape and self._peek == identifier_end: 1066 text += identifier_end # type: ignore 1067 self._advance() 1068 continue 1069 1070 break 1071 1072 text += self._char # type: ignore 1073 1074 self._add(TokenType.IDENTIFIER, text) 1075 1076 def _scan_var(self) -> None: 1077 while True: 1078 char = self._peek.strip() # type: ignore 1079 if char and char not in self.SINGLE_TOKENS: 1080 self._advance() 1081 else: 1082 break 1083 self._add( 1084 TokenType.VAR 1085 if self._prev_token_type == TokenType.PARAMETER 1086 else self.KEYWORDS.get(self._text.upper(), TokenType.VAR) 1087 ) 1088 1089 def _extract_string(self, delimiter: str) -> str: 1090 text = "" 1091 delim_size = len(delimiter) 1092 1093 while True: 1094 if self._char in self._STRING_ESCAPES and ( 1095 self._peek == delimiter or self._peek in self._STRING_ESCAPES 1096 ): 1097 if self._peek == delimiter: 1098 text += self._peek # type: ignore 1099 else: 1100 text += self._char + self._peek # type: ignore 1101 1102 if self._current + 1 < self.size: 1103 self._advance(2) 1104 else: 1105 raise RuntimeError(f"Missing {delimiter} from {self._line}:{self._current}") 1106 else: 1107 if self._chars(delim_size) == delimiter: 1108 if delim_size > 1: 1109 self._advance(delim_size - 1) 1110 break 1111 1112 if self._end: 1113 raise RuntimeError(f"Missing {delimiter} from {self._line}:{self._start}") 1114 text += self._char # type: ignore 1115 self._advance() 1116 1117 return text
11class TokenType(AutoName): 12 L_PAREN = auto() 13 R_PAREN = auto() 14 L_BRACKET = auto() 15 R_BRACKET = auto() 16 L_BRACE = auto() 17 R_BRACE = auto() 18 COMMA = auto() 19 DOT = auto() 20 DASH = auto() 21 PLUS = auto() 22 COLON = auto() 23 DCOLON = auto() 24 SEMICOLON = auto() 25 STAR = auto() 26 BACKSLASH = auto() 27 SLASH = auto() 28 LT = auto() 29 LTE = auto() 30 GT = auto() 31 GTE = auto() 32 NOT = auto() 33 EQ = auto() 34 NEQ = auto() 35 NULLSAFE_EQ = auto() 36 AND = auto() 37 OR = auto() 38 AMP = auto() 39 DPIPE = auto() 40 PIPE = auto() 41 CARET = auto() 42 TILDA = auto() 43 ARROW = auto() 44 DARROW = auto() 45 FARROW = auto() 46 HASH = auto() 47 HASH_ARROW = auto() 48 DHASH_ARROW = auto() 49 LR_ARROW = auto() 50 DOLLAR = auto() 51 PARAMETER = auto() 52 SESSION_PARAMETER = auto() 53 NATIONAL = auto() 54 55 BLOCK_START = auto() 56 BLOCK_END = auto() 57 58 SPACE = auto() 59 BREAK = auto() 60 61 STRING = auto() 62 NUMBER = auto() 63 IDENTIFIER = auto() 64 DATABASE = auto() 65 COLUMN = auto() 66 COLUMN_DEF = auto() 67 SCHEMA = auto() 68 TABLE = auto() 69 VAR = auto() 70 BIT_STRING = auto() 71 HEX_STRING = auto() 72 BYTE_STRING = auto() 73 74 # types 75 BOOLEAN = auto() 76 TINYINT = auto() 77 SMALLINT = auto() 78 INT = auto() 79 BIGINT = auto() 80 FLOAT = auto() 81 DOUBLE = auto() 82 DECIMAL = auto() 83 CHAR = auto() 84 NCHAR = auto() 85 VARCHAR = auto() 86 NVARCHAR = auto() 87 TEXT = auto() 88 MEDIUMTEXT = auto() 89 LONGTEXT = auto() 90 MEDIUMBLOB = auto() 91 LONGBLOB = auto() 92 BINARY = auto() 93 VARBINARY = auto() 94 JSON = auto() 95 JSONB = auto() 96 TIME = auto() 97 TIMESTAMP = auto() 98 TIMESTAMPTZ = auto() 99 TIMESTAMPLTZ = auto() 100 DATETIME = auto() 101 DATE = auto() 102 UUID = auto() 103 GEOGRAPHY = auto() 104 NULLABLE = auto() 105 GEOMETRY = auto() 106 HLLSKETCH = auto() 107 HSTORE = auto() 108 SUPER = auto() 109 SERIAL = auto() 110 SMALLSERIAL = auto() 111 BIGSERIAL = auto() 112 XML = auto() 113 UNIQUEIDENTIFIER = auto() 114 MONEY = auto() 115 SMALLMONEY = auto() 116 ROWVERSION = auto() 117 IMAGE = auto() 118 VARIANT = auto() 119 OBJECT = auto() 120 INET = auto() 121 122 # keywords 123 ALIAS = auto() 124 ALTER = auto() 125 ALWAYS = auto() 126 ALL = auto() 127 ANTI = auto() 128 ANY = auto() 129 APPLY = auto() 130 ARRAY = auto() 131 ASC = auto() 132 ASOF = auto() 133 AT_TIME_ZONE = auto() 134 AUTO_INCREMENT = auto() 135 BEGIN = auto() 136 BETWEEN = auto() 137 BOTH = auto() 138 BUCKET = auto() 139 BY_DEFAULT = auto() 140 CACHE = auto() 141 CASCADE = auto() 142 CASE = auto() 143 CHARACTER_SET = auto() 144 CLUSTER_BY = auto() 145 COLLATE = auto() 146 COMMAND = auto() 147 COMMENT = auto() 148 COMMIT = auto() 149 COMPOUND = auto() 150 CONSTRAINT = auto() 151 CREATE = auto() 152 CROSS = auto() 153 CUBE = auto() 154 CURRENT_DATE = auto() 155 CURRENT_DATETIME = auto() 156 CURRENT_ROW = auto() 157 CURRENT_TIME = auto() 158 CURRENT_TIMESTAMP = auto() 159 DEFAULT = auto() 160 DELETE = auto() 161 DESC = auto() 162 DESCRIBE = auto() 163 DISTINCT = auto() 164 DISTINCT_FROM = auto() 165 DISTRIBUTE_BY = auto() 166 DIV = auto() 167 DROP = auto() 168 ELSE = auto() 169 END = auto() 170 ESCAPE = auto() 171 EXCEPT = auto() 172 EXECUTE = auto() 173 EXISTS = auto() 174 FALSE = auto() 175 FETCH = auto() 176 FILTER = auto() 177 FINAL = auto() 178 FIRST = auto() 179 FOLLOWING = auto() 180 FOR = auto() 181 FOREIGN_KEY = auto() 182 FORMAT = auto() 183 FROM = auto() 184 FULL = auto() 185 FUNCTION = auto() 186 GLOB = auto() 187 GLOBAL = auto() 188 GROUP_BY = auto() 189 GROUPING_SETS = auto() 190 HAVING = auto() 191 HINT = auto() 192 IF = auto() 193 IGNORE_NULLS = auto() 194 ILIKE = auto() 195 ILIKE_ANY = auto() 196 IN = auto() 197 INDEX = auto() 198 INNER = auto() 199 INSERT = auto() 200 INTERSECT = auto() 201 INTERVAL = auto() 202 INTO = auto() 203 INTRODUCER = auto() 204 IRLIKE = auto() 205 IS = auto() 206 ISNULL = auto() 207 JOIN = auto() 208 JOIN_MARKER = auto() 209 LANGUAGE = auto() 210 LATERAL = auto() 211 LAZY = auto() 212 LEADING = auto() 213 LEFT = auto() 214 LIKE = auto() 215 LIKE_ANY = auto() 216 LIMIT = auto() 217 LOAD_DATA = auto() 218 LOCAL = auto() 219 MAP = auto() 220 MATCH_RECOGNIZE = auto() 221 MATERIALIZED = auto() 222 MERGE = auto() 223 MOD = auto() 224 NATURAL = auto() 225 NEXT = auto() 226 NO_ACTION = auto() 227 NOTNULL = auto() 228 NULL = auto() 229 NULLS_FIRST = auto() 230 NULLS_LAST = auto() 231 OFFSET = auto() 232 ON = auto() 233 ONLY = auto() 234 OPTIONS = auto() 235 ORDER_BY = auto() 236 ORDERED = auto() 237 ORDINALITY = auto() 238 OUTER = auto() 239 OUT_OF = auto() 240 OVER = auto() 241 OVERLAPS = auto() 242 OVERWRITE = auto() 243 PARTITION = auto() 244 PARTITION_BY = auto() 245 PERCENT = auto() 246 PIVOT = auto() 247 PLACEHOLDER = auto() 248 PRECEDING = auto() 249 PRIMARY_KEY = auto() 250 PROCEDURE = auto() 251 PROPERTIES = auto() 252 PSEUDO_TYPE = auto() 253 QUALIFY = auto() 254 QUOTE = auto() 255 RANGE = auto() 256 RECURSIVE = auto() 257 REPLACE = auto() 258 RESPECT_NULLS = auto() 259 RETURNING = auto() 260 REFERENCES = auto() 261 RIGHT = auto() 262 RLIKE = auto() 263 ROLLBACK = auto() 264 ROLLUP = auto() 265 ROW = auto() 266 ROWS = auto() 267 SEED = auto() 268 SELECT = auto() 269 SEMI = auto() 270 SEPARATOR = auto() 271 SERDE_PROPERTIES = auto() 272 SET = auto() 273 SHOW = auto() 274 SIMILAR_TO = auto() 275 SOME = auto() 276 SORTKEY = auto() 277 SORT_BY = auto() 278 STRUCT = auto() 279 TABLE_SAMPLE = auto() 280 TEMPORARY = auto() 281 TOP = auto() 282 THEN = auto() 283 TRAILING = auto() 284 TRUE = auto() 285 UNBOUNDED = auto() 286 UNCACHE = auto() 287 UNION = auto() 288 UNLOGGED = auto() 289 UNNEST = auto() 290 UNPIVOT = auto() 291 UPDATE = auto() 292 USE = auto() 293 USING = auto() 294 VALUES = auto() 295 VIEW = auto() 296 VOLATILE = auto() 297 WHEN = auto() 298 WHERE = auto() 299 WINDOW = auto() 300 WITH = auto() 301 WITH_TIME_ZONE = auto() 302 WITH_LOCAL_TIME_ZONE = auto() 303 WITHIN_GROUP = auto() 304 WITHOUT_TIME_ZONE = auto() 305 UNIQUE = auto()
An enumeration.
L_PAREN =
<TokenType.L_PAREN: 'L_PAREN'>
R_PAREN =
<TokenType.R_PAREN: 'R_PAREN'>
L_BRACKET =
<TokenType.L_BRACKET: 'L_BRACKET'>
R_BRACKET =
<TokenType.R_BRACKET: 'R_BRACKET'>
L_BRACE =
<TokenType.L_BRACE: 'L_BRACE'>
R_BRACE =
<TokenType.R_BRACE: 'R_BRACE'>
COMMA =
<TokenType.COMMA: 'COMMA'>
DOT =
<TokenType.DOT: 'DOT'>
DASH =
<TokenType.DASH: 'DASH'>
PLUS =
<TokenType.PLUS: 'PLUS'>
COLON =
<TokenType.COLON: 'COLON'>
DCOLON =
<TokenType.DCOLON: 'DCOLON'>
SEMICOLON =
<TokenType.SEMICOLON: 'SEMICOLON'>
STAR =
<TokenType.STAR: 'STAR'>
BACKSLASH =
<TokenType.BACKSLASH: 'BACKSLASH'>
SLASH =
<TokenType.SLASH: 'SLASH'>
LT =
<TokenType.LT: 'LT'>
LTE =
<TokenType.LTE: 'LTE'>
GT =
<TokenType.GT: 'GT'>
GTE =
<TokenType.GTE: 'GTE'>
NOT =
<TokenType.NOT: 'NOT'>
EQ =
<TokenType.EQ: 'EQ'>
NEQ =
<TokenType.NEQ: 'NEQ'>
NULLSAFE_EQ =
<TokenType.NULLSAFE_EQ: 'NULLSAFE_EQ'>
AND =
<TokenType.AND: 'AND'>
OR =
<TokenType.OR: 'OR'>
AMP =
<TokenType.AMP: 'AMP'>
DPIPE =
<TokenType.DPIPE: 'DPIPE'>
PIPE =
<TokenType.PIPE: 'PIPE'>
CARET =
<TokenType.CARET: 'CARET'>
TILDA =
<TokenType.TILDA: 'TILDA'>
ARROW =
<TokenType.ARROW: 'ARROW'>
DARROW =
<TokenType.DARROW: 'DARROW'>
FARROW =
<TokenType.FARROW: 'FARROW'>
HASH =
<TokenType.HASH: 'HASH'>
HASH_ARROW =
<TokenType.HASH_ARROW: 'HASH_ARROW'>
DHASH_ARROW =
<TokenType.DHASH_ARROW: 'DHASH_ARROW'>
LR_ARROW =
<TokenType.LR_ARROW: 'LR_ARROW'>
DOLLAR =
<TokenType.DOLLAR: 'DOLLAR'>
PARAMETER =
<TokenType.PARAMETER: 'PARAMETER'>
SESSION_PARAMETER =
<TokenType.SESSION_PARAMETER: 'SESSION_PARAMETER'>
NATIONAL =
<TokenType.NATIONAL: 'NATIONAL'>
BLOCK_START =
<TokenType.BLOCK_START: 'BLOCK_START'>
BLOCK_END =
<TokenType.BLOCK_END: 'BLOCK_END'>
SPACE =
<TokenType.SPACE: 'SPACE'>
BREAK =
<TokenType.BREAK: 'BREAK'>
STRING =
<TokenType.STRING: 'STRING'>
NUMBER =
<TokenType.NUMBER: 'NUMBER'>
IDENTIFIER =
<TokenType.IDENTIFIER: 'IDENTIFIER'>
DATABASE =
<TokenType.DATABASE: 'DATABASE'>
COLUMN =
<TokenType.COLUMN: 'COLUMN'>
COLUMN_DEF =
<TokenType.COLUMN_DEF: 'COLUMN_DEF'>
SCHEMA =
<TokenType.SCHEMA: 'SCHEMA'>
TABLE =
<TokenType.TABLE: 'TABLE'>
VAR =
<TokenType.VAR: 'VAR'>
BIT_STRING =
<TokenType.BIT_STRING: 'BIT_STRING'>
HEX_STRING =
<TokenType.HEX_STRING: 'HEX_STRING'>
BYTE_STRING =
<TokenType.BYTE_STRING: 'BYTE_STRING'>
BOOLEAN =
<TokenType.BOOLEAN: 'BOOLEAN'>
TINYINT =
<TokenType.TINYINT: 'TINYINT'>
SMALLINT =
<TokenType.SMALLINT: 'SMALLINT'>
INT =
<TokenType.INT: 'INT'>
BIGINT =
<TokenType.BIGINT: 'BIGINT'>
FLOAT =
<TokenType.FLOAT: 'FLOAT'>
DOUBLE =
<TokenType.DOUBLE: 'DOUBLE'>
DECIMAL =
<TokenType.DECIMAL: 'DECIMAL'>
CHAR =
<TokenType.CHAR: 'CHAR'>
NCHAR =
<TokenType.NCHAR: 'NCHAR'>
VARCHAR =
<TokenType.VARCHAR: 'VARCHAR'>
NVARCHAR =
<TokenType.NVARCHAR: 'NVARCHAR'>
TEXT =
<TokenType.TEXT: 'TEXT'>
MEDIUMTEXT =
<TokenType.MEDIUMTEXT: 'MEDIUMTEXT'>
LONGTEXT =
<TokenType.LONGTEXT: 'LONGTEXT'>
MEDIUMBLOB =
<TokenType.MEDIUMBLOB: 'MEDIUMBLOB'>
LONGBLOB =
<TokenType.LONGBLOB: 'LONGBLOB'>
BINARY =
<TokenType.BINARY: 'BINARY'>
VARBINARY =
<TokenType.VARBINARY: 'VARBINARY'>
JSON =
<TokenType.JSON: 'JSON'>
JSONB =
<TokenType.JSONB: 'JSONB'>
TIME =
<TokenType.TIME: 'TIME'>
TIMESTAMP =
<TokenType.TIMESTAMP: 'TIMESTAMP'>
TIMESTAMPTZ =
<TokenType.TIMESTAMPTZ: 'TIMESTAMPTZ'>
TIMESTAMPLTZ =
<TokenType.TIMESTAMPLTZ: 'TIMESTAMPLTZ'>
DATETIME =
<TokenType.DATETIME: 'DATETIME'>
DATE =
<TokenType.DATE: 'DATE'>
UUID =
<TokenType.UUID: 'UUID'>
GEOGRAPHY =
<TokenType.GEOGRAPHY: 'GEOGRAPHY'>
NULLABLE =
<TokenType.NULLABLE: 'NULLABLE'>
GEOMETRY =
<TokenType.GEOMETRY: 'GEOMETRY'>
HLLSKETCH =
<TokenType.HLLSKETCH: 'HLLSKETCH'>
HSTORE =
<TokenType.HSTORE: 'HSTORE'>
SUPER =
<TokenType.SUPER: 'SUPER'>
SERIAL =
<TokenType.SERIAL: 'SERIAL'>
SMALLSERIAL =
<TokenType.SMALLSERIAL: 'SMALLSERIAL'>
BIGSERIAL =
<TokenType.BIGSERIAL: 'BIGSERIAL'>
XML =
<TokenType.XML: 'XML'>
UNIQUEIDENTIFIER =
<TokenType.UNIQUEIDENTIFIER: 'UNIQUEIDENTIFIER'>
MONEY =
<TokenType.MONEY: 'MONEY'>
SMALLMONEY =
<TokenType.SMALLMONEY: 'SMALLMONEY'>
ROWVERSION =
<TokenType.ROWVERSION: 'ROWVERSION'>
IMAGE =
<TokenType.IMAGE: 'IMAGE'>
VARIANT =
<TokenType.VARIANT: 'VARIANT'>
OBJECT =
<TokenType.OBJECT: 'OBJECT'>
INET =
<TokenType.INET: 'INET'>
ALIAS =
<TokenType.ALIAS: 'ALIAS'>
ALTER =
<TokenType.ALTER: 'ALTER'>
ALWAYS =
<TokenType.ALWAYS: 'ALWAYS'>
ALL =
<TokenType.ALL: 'ALL'>
ANTI =
<TokenType.ANTI: 'ANTI'>
ANY =
<TokenType.ANY: 'ANY'>
APPLY =
<TokenType.APPLY: 'APPLY'>
ARRAY =
<TokenType.ARRAY: 'ARRAY'>
ASC =
<TokenType.ASC: 'ASC'>
ASOF =
<TokenType.ASOF: 'ASOF'>
AT_TIME_ZONE =
<TokenType.AT_TIME_ZONE: 'AT_TIME_ZONE'>
AUTO_INCREMENT =
<TokenType.AUTO_INCREMENT: 'AUTO_INCREMENT'>
BEGIN =
<TokenType.BEGIN: 'BEGIN'>
BETWEEN =
<TokenType.BETWEEN: 'BETWEEN'>
BOTH =
<TokenType.BOTH: 'BOTH'>
BUCKET =
<TokenType.BUCKET: 'BUCKET'>
BY_DEFAULT =
<TokenType.BY_DEFAULT: 'BY_DEFAULT'>
CACHE =
<TokenType.CACHE: 'CACHE'>
CASCADE =
<TokenType.CASCADE: 'CASCADE'>
CASE =
<TokenType.CASE: 'CASE'>
CHARACTER_SET =
<TokenType.CHARACTER_SET: 'CHARACTER_SET'>
CLUSTER_BY =
<TokenType.CLUSTER_BY: 'CLUSTER_BY'>
COLLATE =
<TokenType.COLLATE: 'COLLATE'>
COMMAND =
<TokenType.COMMAND: 'COMMAND'>
COMMENT =
<TokenType.COMMENT: 'COMMENT'>
COMMIT =
<TokenType.COMMIT: 'COMMIT'>
COMPOUND =
<TokenType.COMPOUND: 'COMPOUND'>
CONSTRAINT =
<TokenType.CONSTRAINT: 'CONSTRAINT'>
CREATE =
<TokenType.CREATE: 'CREATE'>
CROSS =
<TokenType.CROSS: 'CROSS'>
CUBE =
<TokenType.CUBE: 'CUBE'>
CURRENT_DATE =
<TokenType.CURRENT_DATE: 'CURRENT_DATE'>
CURRENT_DATETIME =
<TokenType.CURRENT_DATETIME: 'CURRENT_DATETIME'>
CURRENT_ROW =
<TokenType.CURRENT_ROW: 'CURRENT_ROW'>
CURRENT_TIME =
<TokenType.CURRENT_TIME: 'CURRENT_TIME'>
CURRENT_TIMESTAMP =
<TokenType.CURRENT_TIMESTAMP: 'CURRENT_TIMESTAMP'>
DEFAULT =
<TokenType.DEFAULT: 'DEFAULT'>
DELETE =
<TokenType.DELETE: 'DELETE'>
DESC =
<TokenType.DESC: 'DESC'>
DESCRIBE =
<TokenType.DESCRIBE: 'DESCRIBE'>
DISTINCT =
<TokenType.DISTINCT: 'DISTINCT'>
DISTINCT_FROM =
<TokenType.DISTINCT_FROM: 'DISTINCT_FROM'>
DISTRIBUTE_BY =
<TokenType.DISTRIBUTE_BY: 'DISTRIBUTE_BY'>
DIV =
<TokenType.DIV: 'DIV'>
DROP =
<TokenType.DROP: 'DROP'>
ELSE =
<TokenType.ELSE: 'ELSE'>
END =
<TokenType.END: 'END'>
ESCAPE =
<TokenType.ESCAPE: 'ESCAPE'>
EXCEPT =
<TokenType.EXCEPT: 'EXCEPT'>
EXECUTE =
<TokenType.EXECUTE: 'EXECUTE'>
EXISTS =
<TokenType.EXISTS: 'EXISTS'>
FALSE =
<TokenType.FALSE: 'FALSE'>
FETCH =
<TokenType.FETCH: 'FETCH'>
FILTER =
<TokenType.FILTER: 'FILTER'>
FINAL =
<TokenType.FINAL: 'FINAL'>
FIRST =
<TokenType.FIRST: 'FIRST'>
FOLLOWING =
<TokenType.FOLLOWING: 'FOLLOWING'>
FOR =
<TokenType.FOR: 'FOR'>
FOREIGN_KEY =
<TokenType.FOREIGN_KEY: 'FOREIGN_KEY'>
FORMAT =
<TokenType.FORMAT: 'FORMAT'>
FROM =
<TokenType.FROM: 'FROM'>
FULL =
<TokenType.FULL: 'FULL'>
FUNCTION =
<TokenType.FUNCTION: 'FUNCTION'>
GLOB =
<TokenType.GLOB: 'GLOB'>
GLOBAL =
<TokenType.GLOBAL: 'GLOBAL'>
GROUP_BY =
<TokenType.GROUP_BY: 'GROUP_BY'>
GROUPING_SETS =
<TokenType.GROUPING_SETS: 'GROUPING_SETS'>
HAVING =
<TokenType.HAVING: 'HAVING'>
HINT =
<TokenType.HINT: 'HINT'>
IF =
<TokenType.IF: 'IF'>
IGNORE_NULLS =
<TokenType.IGNORE_NULLS: 'IGNORE_NULLS'>
ILIKE =
<TokenType.ILIKE: 'ILIKE'>
ILIKE_ANY =
<TokenType.ILIKE_ANY: 'ILIKE_ANY'>
IN =
<TokenType.IN: 'IN'>
INDEX =
<TokenType.INDEX: 'INDEX'>
INNER =
<TokenType.INNER: 'INNER'>
INSERT =
<TokenType.INSERT: 'INSERT'>
INTERSECT =
<TokenType.INTERSECT: 'INTERSECT'>
INTERVAL =
<TokenType.INTERVAL: 'INTERVAL'>
INTO =
<TokenType.INTO: 'INTO'>
INTRODUCER =
<TokenType.INTRODUCER: 'INTRODUCER'>
IRLIKE =
<TokenType.IRLIKE: 'IRLIKE'>
IS =
<TokenType.IS: 'IS'>
ISNULL =
<TokenType.ISNULL: 'ISNULL'>
JOIN =
<TokenType.JOIN: 'JOIN'>
JOIN_MARKER =
<TokenType.JOIN_MARKER: 'JOIN_MARKER'>
LANGUAGE =
<TokenType.LANGUAGE: 'LANGUAGE'>
LATERAL =
<TokenType.LATERAL: 'LATERAL'>
LAZY =
<TokenType.LAZY: 'LAZY'>
LEADING =
<TokenType.LEADING: 'LEADING'>
LEFT =
<TokenType.LEFT: 'LEFT'>
LIKE =
<TokenType.LIKE: 'LIKE'>
LIKE_ANY =
<TokenType.LIKE_ANY: 'LIKE_ANY'>
LIMIT =
<TokenType.LIMIT: 'LIMIT'>
LOAD_DATA =
<TokenType.LOAD_DATA: 'LOAD_DATA'>
LOCAL =
<TokenType.LOCAL: 'LOCAL'>
MAP =
<TokenType.MAP: 'MAP'>
MATCH_RECOGNIZE =
<TokenType.MATCH_RECOGNIZE: 'MATCH_RECOGNIZE'>
MATERIALIZED =
<TokenType.MATERIALIZED: 'MATERIALIZED'>
MERGE =
<TokenType.MERGE: 'MERGE'>
MOD =
<TokenType.MOD: 'MOD'>
NATURAL =
<TokenType.NATURAL: 'NATURAL'>
NEXT =
<TokenType.NEXT: 'NEXT'>
NO_ACTION =
<TokenType.NO_ACTION: 'NO_ACTION'>
NOTNULL =
<TokenType.NOTNULL: 'NOTNULL'>
NULL =
<TokenType.NULL: 'NULL'>
NULLS_FIRST =
<TokenType.NULLS_FIRST: 'NULLS_FIRST'>
NULLS_LAST =
<TokenType.NULLS_LAST: 'NULLS_LAST'>
OFFSET =
<TokenType.OFFSET: 'OFFSET'>
ON =
<TokenType.ON: 'ON'>
ONLY =
<TokenType.ONLY: 'ONLY'>
OPTIONS =
<TokenType.OPTIONS: 'OPTIONS'>
ORDER_BY =
<TokenType.ORDER_BY: 'ORDER_BY'>
ORDERED =
<TokenType.ORDERED: 'ORDERED'>
ORDINALITY =
<TokenType.ORDINALITY: 'ORDINALITY'>
OUTER =
<TokenType.OUTER: 'OUTER'>
OUT_OF =
<TokenType.OUT_OF: 'OUT_OF'>
OVER =
<TokenType.OVER: 'OVER'>
OVERLAPS =
<TokenType.OVERLAPS: 'OVERLAPS'>
OVERWRITE =
<TokenType.OVERWRITE: 'OVERWRITE'>
PARTITION =
<TokenType.PARTITION: 'PARTITION'>
PARTITION_BY =
<TokenType.PARTITION_BY: 'PARTITION_BY'>
PERCENT =
<TokenType.PERCENT: 'PERCENT'>
PIVOT =
<TokenType.PIVOT: 'PIVOT'>
PLACEHOLDER =
<TokenType.PLACEHOLDER: 'PLACEHOLDER'>
PRECEDING =
<TokenType.PRECEDING: 'PRECEDING'>
PRIMARY_KEY =
<TokenType.PRIMARY_KEY: 'PRIMARY_KEY'>
PROCEDURE =
<TokenType.PROCEDURE: 'PROCEDURE'>
PROPERTIES =
<TokenType.PROPERTIES: 'PROPERTIES'>
PSEUDO_TYPE =
<TokenType.PSEUDO_TYPE: 'PSEUDO_TYPE'>
QUALIFY =
<TokenType.QUALIFY: 'QUALIFY'>
QUOTE =
<TokenType.QUOTE: 'QUOTE'>
RANGE =
<TokenType.RANGE: 'RANGE'>
RECURSIVE =
<TokenType.RECURSIVE: 'RECURSIVE'>
REPLACE =
<TokenType.REPLACE: 'REPLACE'>
RESPECT_NULLS =
<TokenType.RESPECT_NULLS: 'RESPECT_NULLS'>
RETURNING =
<TokenType.RETURNING: 'RETURNING'>
REFERENCES =
<TokenType.REFERENCES: 'REFERENCES'>
RIGHT =
<TokenType.RIGHT: 'RIGHT'>
RLIKE =
<TokenType.RLIKE: 'RLIKE'>
ROLLBACK =
<TokenType.ROLLBACK: 'ROLLBACK'>
ROLLUP =
<TokenType.ROLLUP: 'ROLLUP'>
ROW =
<TokenType.ROW: 'ROW'>
ROWS =
<TokenType.ROWS: 'ROWS'>
SEED =
<TokenType.SEED: 'SEED'>
SELECT =
<TokenType.SELECT: 'SELECT'>
SEMI =
<TokenType.SEMI: 'SEMI'>
SEPARATOR =
<TokenType.SEPARATOR: 'SEPARATOR'>
SERDE_PROPERTIES =
<TokenType.SERDE_PROPERTIES: 'SERDE_PROPERTIES'>
SET =
<TokenType.SET: 'SET'>
SHOW =
<TokenType.SHOW: 'SHOW'>
SIMILAR_TO =
<TokenType.SIMILAR_TO: 'SIMILAR_TO'>
SOME =
<TokenType.SOME: 'SOME'>
SORTKEY =
<TokenType.SORTKEY: 'SORTKEY'>
SORT_BY =
<TokenType.SORT_BY: 'SORT_BY'>
STRUCT =
<TokenType.STRUCT: 'STRUCT'>
TABLE_SAMPLE =
<TokenType.TABLE_SAMPLE: 'TABLE_SAMPLE'>
TEMPORARY =
<TokenType.TEMPORARY: 'TEMPORARY'>
TOP =
<TokenType.TOP: 'TOP'>
THEN =
<TokenType.THEN: 'THEN'>
TRAILING =
<TokenType.TRAILING: 'TRAILING'>
TRUE =
<TokenType.TRUE: 'TRUE'>
UNBOUNDED =
<TokenType.UNBOUNDED: 'UNBOUNDED'>
UNCACHE =
<TokenType.UNCACHE: 'UNCACHE'>
UNION =
<TokenType.UNION: 'UNION'>
UNLOGGED =
<TokenType.UNLOGGED: 'UNLOGGED'>
UNNEST =
<TokenType.UNNEST: 'UNNEST'>
UNPIVOT =
<TokenType.UNPIVOT: 'UNPIVOT'>
UPDATE =
<TokenType.UPDATE: 'UPDATE'>
USE =
<TokenType.USE: 'USE'>
USING =
<TokenType.USING: 'USING'>
VALUES =
<TokenType.VALUES: 'VALUES'>
VIEW =
<TokenType.VIEW: 'VIEW'>
VOLATILE =
<TokenType.VOLATILE: 'VOLATILE'>
WHEN =
<TokenType.WHEN: 'WHEN'>
WHERE =
<TokenType.WHERE: 'WHERE'>
WINDOW =
<TokenType.WINDOW: 'WINDOW'>
WITH =
<TokenType.WITH: 'WITH'>
WITH_TIME_ZONE =
<TokenType.WITH_TIME_ZONE: 'WITH_TIME_ZONE'>
WITH_LOCAL_TIME_ZONE =
<TokenType.WITH_LOCAL_TIME_ZONE: 'WITH_LOCAL_TIME_ZONE'>
WITHIN_GROUP =
<TokenType.WITHIN_GROUP: 'WITHIN_GROUP'>
WITHOUT_TIME_ZONE =
<TokenType.WITHOUT_TIME_ZONE: 'WITHOUT_TIME_ZONE'>
UNIQUE =
<TokenType.UNIQUE: 'UNIQUE'>
Inherited Members
- enum.Enum
- name
- value
class
Token:
308class Token: 309 __slots__ = ("token_type", "text", "line", "col", "comments") 310 311 @classmethod 312 def number(cls, number: int) -> Token: 313 """Returns a NUMBER token with `number` as its text.""" 314 return cls(TokenType.NUMBER, str(number)) 315 316 @classmethod 317 def string(cls, string: str) -> Token: 318 """Returns a STRING token with `string` as its text.""" 319 return cls(TokenType.STRING, string) 320 321 @classmethod 322 def identifier(cls, identifier: str) -> Token: 323 """Returns an IDENTIFIER token with `identifier` as its text.""" 324 return cls(TokenType.IDENTIFIER, identifier) 325 326 @classmethod 327 def var(cls, var: str) -> Token: 328 """Returns an VAR token with `var` as its text.""" 329 return cls(TokenType.VAR, var) 330 331 def __init__( 332 self, 333 token_type: TokenType, 334 text: str, 335 line: int = 1, 336 col: int = 1, 337 comments: t.List[str] = [], 338 ) -> None: 339 self.token_type = token_type 340 self.text = text 341 self.line = line 342 self.col = max(col - len(text), 1) 343 self.comments = comments 344 345 def __repr__(self) -> str: 346 attributes = ", ".join(f"{k}: {getattr(self, k)}" for k in self.__slots__) 347 return f"<Token {attributes}>"
Token( token_type: sqlglot.tokens.TokenType, text: str, line: int = 1, col: int = 1, comments: List[str] = [])
311 @classmethod 312 def number(cls, number: int) -> Token: 313 """Returns a NUMBER token with `number` as its text.""" 314 return cls(TokenType.NUMBER, str(number))
Returns a NUMBER token with number
as its text.
316 @classmethod 317 def string(cls, string: str) -> Token: 318 """Returns a STRING token with `string` as its text.""" 319 return cls(TokenType.STRING, string)
Returns a STRING token with string
as its text.
321 @classmethod 322 def identifier(cls, identifier: str) -> Token: 323 """Returns an IDENTIFIER token with `identifier` as its text.""" 324 return cls(TokenType.IDENTIFIER, identifier)
Returns an IDENTIFIER token with identifier
as its text.
class
Tokenizer:
390class Tokenizer(metaclass=_Tokenizer): 391 SINGLE_TOKENS = { 392 "(": TokenType.L_PAREN, 393 ")": TokenType.R_PAREN, 394 "[": TokenType.L_BRACKET, 395 "]": TokenType.R_BRACKET, 396 "{": TokenType.L_BRACE, 397 "}": TokenType.R_BRACE, 398 "&": TokenType.AMP, 399 "^": TokenType.CARET, 400 ":": TokenType.COLON, 401 ",": TokenType.COMMA, 402 ".": TokenType.DOT, 403 "-": TokenType.DASH, 404 "=": TokenType.EQ, 405 ">": TokenType.GT, 406 "<": TokenType.LT, 407 "%": TokenType.MOD, 408 "!": TokenType.NOT, 409 "|": TokenType.PIPE, 410 "+": TokenType.PLUS, 411 ";": TokenType.SEMICOLON, 412 "/": TokenType.SLASH, 413 "\\": TokenType.BACKSLASH, 414 "*": TokenType.STAR, 415 "~": TokenType.TILDA, 416 "?": TokenType.PLACEHOLDER, 417 "@": TokenType.PARAMETER, 418 # used for breaking a var like x'y' but nothing else 419 # the token type doesn't matter 420 "'": TokenType.QUOTE, 421 "`": TokenType.IDENTIFIER, 422 '"': TokenType.IDENTIFIER, 423 "#": TokenType.HASH, 424 } 425 426 QUOTES: t.List[t.Tuple[str, str] | str] = ["'"] 427 428 BIT_STRINGS: t.List[str | t.Tuple[str, str]] = [] 429 430 HEX_STRINGS: t.List[str | t.Tuple[str, str]] = [] 431 432 BYTE_STRINGS: t.List[str | t.Tuple[str, str]] = [] 433 434 IDENTIFIERS: t.List[str | t.Tuple[str, str]] = ['"'] 435 436 STRING_ESCAPES = ["'"] 437 438 _STRING_ESCAPES: t.Set[str] = set() 439 440 IDENTIFIER_ESCAPES = ['"'] 441 442 _IDENTIFIER_ESCAPES: t.Set[str] = set() 443 444 KEYWORDS = { 445 **{f"{{%{postfix}": TokenType.BLOCK_START for postfix in ("", "+", "-")}, 446 **{f"{prefix}%}}": TokenType.BLOCK_END for prefix in ("", "+", "-")}, 447 "{{+": TokenType.BLOCK_START, 448 "{{-": TokenType.BLOCK_START, 449 "+}}": TokenType.BLOCK_END, 450 "-}}": TokenType.BLOCK_END, 451 "/*+": TokenType.HINT, 452 "==": TokenType.EQ, 453 "::": TokenType.DCOLON, 454 "||": TokenType.DPIPE, 455 ">=": TokenType.GTE, 456 "<=": TokenType.LTE, 457 "<>": TokenType.NEQ, 458 "!=": TokenType.NEQ, 459 "<=>": TokenType.NULLSAFE_EQ, 460 "->": TokenType.ARROW, 461 "->>": TokenType.DARROW, 462 "=>": TokenType.FARROW, 463 "#>": TokenType.HASH_ARROW, 464 "#>>": TokenType.DHASH_ARROW, 465 "<->": TokenType.LR_ARROW, 466 "ALL": TokenType.ALL, 467 "ALWAYS": TokenType.ALWAYS, 468 "AND": TokenType.AND, 469 "ANTI": TokenType.ANTI, 470 "ANY": TokenType.ANY, 471 "ASC": TokenType.ASC, 472 "AS": TokenType.ALIAS, 473 "AT TIME ZONE": TokenType.AT_TIME_ZONE, 474 "AUTOINCREMENT": TokenType.AUTO_INCREMENT, 475 "AUTO_INCREMENT": TokenType.AUTO_INCREMENT, 476 "BEGIN": TokenType.BEGIN, 477 "BETWEEN": TokenType.BETWEEN, 478 "BOTH": TokenType.BOTH, 479 "BUCKET": TokenType.BUCKET, 480 "BY DEFAULT": TokenType.BY_DEFAULT, 481 "CACHE": TokenType.CACHE, 482 "UNCACHE": TokenType.UNCACHE, 483 "CASE": TokenType.CASE, 484 "CASCADE": TokenType.CASCADE, 485 "CHARACTER SET": TokenType.CHARACTER_SET, 486 "CLUSTER BY": TokenType.CLUSTER_BY, 487 "COLLATE": TokenType.COLLATE, 488 "COLUMN": TokenType.COLUMN, 489 "COMMIT": TokenType.COMMIT, 490 "COMPOUND": TokenType.COMPOUND, 491 "CONSTRAINT": TokenType.CONSTRAINT, 492 "CREATE": TokenType.CREATE, 493 "CROSS": TokenType.CROSS, 494 "CUBE": TokenType.CUBE, 495 "CURRENT_DATE": TokenType.CURRENT_DATE, 496 "CURRENT ROW": TokenType.CURRENT_ROW, 497 "CURRENT_TIMESTAMP": TokenType.CURRENT_TIMESTAMP, 498 "DATABASE": TokenType.DATABASE, 499 "DEFAULT": TokenType.DEFAULT, 500 "DELETE": TokenType.DELETE, 501 "DESC": TokenType.DESC, 502 "DESCRIBE": TokenType.DESCRIBE, 503 "DISTINCT": TokenType.DISTINCT, 504 "DISTINCT FROM": TokenType.DISTINCT_FROM, 505 "DISTRIBUTE BY": TokenType.DISTRIBUTE_BY, 506 "DIV": TokenType.DIV, 507 "DROP": TokenType.DROP, 508 "ELSE": TokenType.ELSE, 509 "END": TokenType.END, 510 "ESCAPE": TokenType.ESCAPE, 511 "EXCEPT": TokenType.EXCEPT, 512 "EXECUTE": TokenType.EXECUTE, 513 "EXISTS": TokenType.EXISTS, 514 "FALSE": TokenType.FALSE, 515 "FETCH": TokenType.FETCH, 516 "FILTER": TokenType.FILTER, 517 "FIRST": TokenType.FIRST, 518 "FULL": TokenType.FULL, 519 "FUNCTION": TokenType.FUNCTION, 520 "FOLLOWING": TokenType.FOLLOWING, 521 "FOR": TokenType.FOR, 522 "FOREIGN KEY": TokenType.FOREIGN_KEY, 523 "FORMAT": TokenType.FORMAT, 524 "FROM": TokenType.FROM, 525 "GLOB": TokenType.GLOB, 526 "GROUP BY": TokenType.GROUP_BY, 527 "GROUPING SETS": TokenType.GROUPING_SETS, 528 "HAVING": TokenType.HAVING, 529 "IF": TokenType.IF, 530 "ILIKE": TokenType.ILIKE, 531 "IGNORE NULLS": TokenType.IGNORE_NULLS, 532 "IN": TokenType.IN, 533 "INDEX": TokenType.INDEX, 534 "INET": TokenType.INET, 535 "INNER": TokenType.INNER, 536 "INSERT": TokenType.INSERT, 537 "INTERVAL": TokenType.INTERVAL, 538 "INTERSECT": TokenType.INTERSECT, 539 "INTO": TokenType.INTO, 540 "IS": TokenType.IS, 541 "ISNULL": TokenType.ISNULL, 542 "JOIN": TokenType.JOIN, 543 "LATERAL": TokenType.LATERAL, 544 "LAZY": TokenType.LAZY, 545 "LEADING": TokenType.LEADING, 546 "LEFT": TokenType.LEFT, 547 "LIKE": TokenType.LIKE, 548 "LIMIT": TokenType.LIMIT, 549 "LOAD DATA": TokenType.LOAD_DATA, 550 "LOCAL": TokenType.LOCAL, 551 "MATERIALIZED": TokenType.MATERIALIZED, 552 "MERGE": TokenType.MERGE, 553 "NATURAL": TokenType.NATURAL, 554 "NEXT": TokenType.NEXT, 555 "NO ACTION": TokenType.NO_ACTION, 556 "NOT": TokenType.NOT, 557 "NOTNULL": TokenType.NOTNULL, 558 "NULL": TokenType.NULL, 559 "NULLS FIRST": TokenType.NULLS_FIRST, 560 "NULLS LAST": TokenType.NULLS_LAST, 561 "OBJECT": TokenType.OBJECT, 562 "OFFSET": TokenType.OFFSET, 563 "ON": TokenType.ON, 564 "ONLY": TokenType.ONLY, 565 "OPTIONS": TokenType.OPTIONS, 566 "OR": TokenType.OR, 567 "ORDER BY": TokenType.ORDER_BY, 568 "ORDINALITY": TokenType.ORDINALITY, 569 "OUTER": TokenType.OUTER, 570 "OUT OF": TokenType.OUT_OF, 571 "OVER": TokenType.OVER, 572 "OVERLAPS": TokenType.OVERLAPS, 573 "OVERWRITE": TokenType.OVERWRITE, 574 "PARTITION": TokenType.PARTITION, 575 "PARTITION BY": TokenType.PARTITION_BY, 576 "PARTITIONED BY": TokenType.PARTITION_BY, 577 "PARTITIONED_BY": TokenType.PARTITION_BY, 578 "PERCENT": TokenType.PERCENT, 579 "PIVOT": TokenType.PIVOT, 580 "PRECEDING": TokenType.PRECEDING, 581 "PRIMARY KEY": TokenType.PRIMARY_KEY, 582 "PROCEDURE": TokenType.PROCEDURE, 583 "QUALIFY": TokenType.QUALIFY, 584 "RANGE": TokenType.RANGE, 585 "RECURSIVE": TokenType.RECURSIVE, 586 "REGEXP": TokenType.RLIKE, 587 "REPLACE": TokenType.REPLACE, 588 "RESPECT NULLS": TokenType.RESPECT_NULLS, 589 "REFERENCES": TokenType.REFERENCES, 590 "RIGHT": TokenType.RIGHT, 591 "RLIKE": TokenType.RLIKE, 592 "ROLLBACK": TokenType.ROLLBACK, 593 "ROLLUP": TokenType.ROLLUP, 594 "ROW": TokenType.ROW, 595 "ROWS": TokenType.ROWS, 596 "SCHEMA": TokenType.SCHEMA, 597 "SEED": TokenType.SEED, 598 "SELECT": TokenType.SELECT, 599 "SEMI": TokenType.SEMI, 600 "SET": TokenType.SET, 601 "SHOW": TokenType.SHOW, 602 "SIMILAR TO": TokenType.SIMILAR_TO, 603 "SOME": TokenType.SOME, 604 "SORTKEY": TokenType.SORTKEY, 605 "SORT BY": TokenType.SORT_BY, 606 "TABLE": TokenType.TABLE, 607 "TABLESAMPLE": TokenType.TABLE_SAMPLE, 608 "TEMP": TokenType.TEMPORARY, 609 "TEMPORARY": TokenType.TEMPORARY, 610 "THEN": TokenType.THEN, 611 "TRUE": TokenType.TRUE, 612 "TRAILING": TokenType.TRAILING, 613 "UNBOUNDED": TokenType.UNBOUNDED, 614 "UNION": TokenType.UNION, 615 "UNLOGGED": TokenType.UNLOGGED, 616 "UNNEST": TokenType.UNNEST, 617 "UNPIVOT": TokenType.UNPIVOT, 618 "UPDATE": TokenType.UPDATE, 619 "USE": TokenType.USE, 620 "USING": TokenType.USING, 621 "VALUES": TokenType.VALUES, 622 "VIEW": TokenType.VIEW, 623 "VOLATILE": TokenType.VOLATILE, 624 "WHEN": TokenType.WHEN, 625 "WHERE": TokenType.WHERE, 626 "WINDOW": TokenType.WINDOW, 627 "WITH": TokenType.WITH, 628 "WITH TIME ZONE": TokenType.WITH_TIME_ZONE, 629 "WITH LOCAL TIME ZONE": TokenType.WITH_LOCAL_TIME_ZONE, 630 "WITHIN GROUP": TokenType.WITHIN_GROUP, 631 "WITHOUT TIME ZONE": TokenType.WITHOUT_TIME_ZONE, 632 "APPLY": TokenType.APPLY, 633 "ARRAY": TokenType.ARRAY, 634 "BOOL": TokenType.BOOLEAN, 635 "BOOLEAN": TokenType.BOOLEAN, 636 "BYTE": TokenType.TINYINT, 637 "TINYINT": TokenType.TINYINT, 638 "SHORT": TokenType.SMALLINT, 639 "SMALLINT": TokenType.SMALLINT, 640 "INT2": TokenType.SMALLINT, 641 "INTEGER": TokenType.INT, 642 "INT": TokenType.INT, 643 "INT4": TokenType.INT, 644 "LONG": TokenType.BIGINT, 645 "BIGINT": TokenType.BIGINT, 646 "INT8": TokenType.BIGINT, 647 "DECIMAL": TokenType.DECIMAL, 648 "MAP": TokenType.MAP, 649 "NULLABLE": TokenType.NULLABLE, 650 "NUMBER": TokenType.DECIMAL, 651 "NUMERIC": TokenType.DECIMAL, 652 "FIXED": TokenType.DECIMAL, 653 "REAL": TokenType.FLOAT, 654 "FLOAT": TokenType.FLOAT, 655 "FLOAT4": TokenType.FLOAT, 656 "FLOAT8": TokenType.DOUBLE, 657 "DOUBLE": TokenType.DOUBLE, 658 "DOUBLE PRECISION": TokenType.DOUBLE, 659 "JSON": TokenType.JSON, 660 "CHAR": TokenType.CHAR, 661 "CHARACTER": TokenType.CHAR, 662 "NCHAR": TokenType.NCHAR, 663 "VARCHAR": TokenType.VARCHAR, 664 "VARCHAR2": TokenType.VARCHAR, 665 "NVARCHAR": TokenType.NVARCHAR, 666 "NVARCHAR2": TokenType.NVARCHAR, 667 "STR": TokenType.TEXT, 668 "STRING": TokenType.TEXT, 669 "TEXT": TokenType.TEXT, 670 "CLOB": TokenType.TEXT, 671 "LONGVARCHAR": TokenType.TEXT, 672 "BINARY": TokenType.BINARY, 673 "BLOB": TokenType.VARBINARY, 674 "BYTEA": TokenType.VARBINARY, 675 "VARBINARY": TokenType.VARBINARY, 676 "TIME": TokenType.TIME, 677 "TIMESTAMP": TokenType.TIMESTAMP, 678 "TIMESTAMPTZ": TokenType.TIMESTAMPTZ, 679 "TIMESTAMPLTZ": TokenType.TIMESTAMPLTZ, 680 "DATE": TokenType.DATE, 681 "DATETIME": TokenType.DATETIME, 682 "UNIQUE": TokenType.UNIQUE, 683 "STRUCT": TokenType.STRUCT, 684 "VARIANT": TokenType.VARIANT, 685 "ALTER": TokenType.ALTER, 686 "ALTER AGGREGATE": TokenType.COMMAND, 687 "ALTER DEFAULT": TokenType.COMMAND, 688 "ALTER DOMAIN": TokenType.COMMAND, 689 "ALTER ROLE": TokenType.COMMAND, 690 "ALTER RULE": TokenType.COMMAND, 691 "ALTER SEQUENCE": TokenType.COMMAND, 692 "ALTER TYPE": TokenType.COMMAND, 693 "ALTER USER": TokenType.COMMAND, 694 "ALTER VIEW": TokenType.COMMAND, 695 "ANALYZE": TokenType.COMMAND, 696 "CALL": TokenType.COMMAND, 697 "COMMENT": TokenType.COMMENT, 698 "COPY": TokenType.COMMAND, 699 "EXPLAIN": TokenType.COMMAND, 700 "GRANT": TokenType.COMMAND, 701 "OPTIMIZE": TokenType.COMMAND, 702 "PREPARE": TokenType.COMMAND, 703 "TRUNCATE": TokenType.COMMAND, 704 "VACUUM": TokenType.COMMAND, 705 } 706 707 WHITE_SPACE: t.Dict[str, TokenType] = { 708 " ": TokenType.SPACE, 709 "\t": TokenType.SPACE, 710 "\n": TokenType.BREAK, 711 "\r": TokenType.BREAK, 712 "\r\n": TokenType.BREAK, 713 } 714 715 COMMANDS = { 716 TokenType.COMMAND, 717 TokenType.EXECUTE, 718 TokenType.FETCH, 719 TokenType.SET, 720 TokenType.SHOW, 721 } 722 723 COMMAND_PREFIX_TOKENS = {TokenType.SEMICOLON, TokenType.BEGIN} 724 725 # handle numeric literals like in hive (3L = BIGINT) 726 NUMERIC_LITERALS: t.Dict[str, str] = {} 727 ENCODE: t.Optional[str] = None 728 729 COMMENTS = ["--", ("/*", "*/"), ("{#", "#}")] 730 KEYWORD_TRIE = None # autofilled 731 732 IDENTIFIER_CAN_START_WITH_DIGIT = False 733 734 __slots__ = ( 735 "sql", 736 "size", 737 "tokens", 738 "_start", 739 "_current", 740 "_line", 741 "_col", 742 "_comments", 743 "_char", 744 "_end", 745 "_peek", 746 "_prev_token_line", 747 "_prev_token_comments", 748 "_prev_token_type", 749 ) 750 751 def __init__(self) -> None: 752 self.reset() 753 754 def reset(self) -> None: 755 self.sql = "" 756 self.size = 0 757 self.tokens: t.List[Token] = [] 758 self._start = 0 759 self._current = 0 760 self._line = 1 761 self._col = 1 762 self._comments: t.List[str] = [] 763 764 self._char = None 765 self._end = None 766 self._peek = None 767 self._prev_token_line = -1 768 self._prev_token_comments: t.List[str] = [] 769 self._prev_token_type = None 770 771 def tokenize(self, sql: str) -> t.List[Token]: 772 """Returns a list of tokens corresponding to the SQL string `sql`.""" 773 self.reset() 774 self.sql = sql 775 self.size = len(sql) 776 self._scan() 777 return self.tokens 778 779 def _scan(self, until: t.Optional[t.Callable] = None) -> None: 780 while self.size and not self._end: 781 self._start = self._current 782 self._advance() 783 784 if self._char is None: 785 break 786 787 if self._char not in self.WHITE_SPACE: 788 if self._char.isdigit(): 789 self._scan_number() 790 elif self._char in self._IDENTIFIERS: 791 self._scan_identifier(self._IDENTIFIERS[self._char]) 792 else: 793 self._scan_keywords() 794 795 if until and until(): 796 break 797 798 def _chars(self, size: int) -> str: 799 if size == 1: 800 return self._char # type: ignore 801 start = self._current - 1 802 end = start + size 803 if end <= self.size: 804 return self.sql[start:end] 805 return "" 806 807 def _line_break(self, char: t.Optional[str]) -> bool: 808 return self.WHITE_SPACE.get(char) == TokenType.BREAK # type: ignore 809 810 def _advance(self, i: int = 1) -> None: 811 if self._line_break(self._char): 812 self._set_new_line() 813 814 self._col += i 815 self._current += i 816 self._end = self._current >= self.size # type: ignore 817 self._char = self.sql[self._current - 1] # type: ignore 818 self._peek = self.sql[self._current] if self._current < self.size else "" # type: ignore 819 820 def _set_new_line(self) -> None: 821 self._col = 1 822 self._line += 1 823 824 @property 825 def _text(self) -> str: 826 return self.sql[self._start : self._current] 827 828 def _add(self, token_type: TokenType, text: t.Optional[str] = None) -> None: 829 self._prev_token_line = self._line 830 self._prev_token_comments = self._comments 831 self._prev_token_type = token_type # type: ignore 832 self.tokens.append( 833 Token( 834 token_type, 835 self._text if text is None else text, 836 self._line, 837 self._col, 838 self._comments, 839 ) 840 ) 841 self._comments = [] 842 843 # If we have either a semicolon or a begin token before the command's token, we'll parse 844 # whatever follows the command's token as a string 845 if token_type in self.COMMANDS and ( 846 len(self.tokens) == 1 or self.tokens[-2].token_type in self.COMMAND_PREFIX_TOKENS 847 ): 848 start = self._current 849 tokens = len(self.tokens) 850 self._scan(lambda: self._peek == ";") 851 self.tokens = self.tokens[:tokens] 852 text = self.sql[start : self._current].strip() 853 if text: 854 self._add(TokenType.STRING, text) 855 856 def _scan_keywords(self) -> None: 857 size = 0 858 word = None 859 chars: t.Optional[str] = self._text 860 char = chars 861 prev_space = False 862 skip = False 863 trie = self.KEYWORD_TRIE 864 865 while chars: 866 if skip: 867 result = 1 868 else: 869 result, trie = in_trie(trie, char.upper()) # type: ignore 870 871 if result == 0: 872 break 873 if result == 2: 874 word = chars 875 size += 1 876 end = self._current - 1 + size 877 878 if end < self.size: 879 char = self.sql[end] 880 is_space = char in self.WHITE_SPACE 881 882 if not is_space or not prev_space: 883 if is_space: 884 char = " " 885 chars += char 886 prev_space = is_space 887 skip = False 888 else: 889 skip = True 890 else: 891 chars = None 892 893 if not word: 894 if self._char in self.SINGLE_TOKENS: 895 self._add(self.SINGLE_TOKENS[self._char]) # type: ignore 896 return 897 self._scan_var() 898 return 899 900 if self._scan_string(word): 901 return 902 if self._scan_formatted_string(word): 903 return 904 if self._scan_comment(word): 905 return 906 907 self._advance(size - 1) 908 self._add(self.KEYWORDS[word.upper()]) 909 910 def _scan_comment(self, comment_start: str) -> bool: 911 if comment_start not in self._COMMENTS: # type: ignore 912 return False 913 914 comment_start_line = self._line 915 comment_start_size = len(comment_start) 916 comment_end = self._COMMENTS[comment_start] # type: ignore 917 918 if comment_end: 919 comment_end_size = len(comment_end) 920 921 while not self._end and self._chars(comment_end_size) != comment_end: 922 self._advance() 923 924 self._comments.append(self._text[comment_start_size : -comment_end_size + 1]) # type: ignore 925 self._advance(comment_end_size - 1) 926 else: 927 while not self._end and not self._line_break(self._peek): 928 self._advance() 929 self._comments.append(self._text[comment_start_size:]) # type: ignore 930 931 # Leading comment is attached to the succeeding token, whilst trailing comment to the preceding. 932 # Multiple consecutive comments are preserved by appending them to the current comments list. 933 if comment_start_line == self._prev_token_line: 934 self.tokens[-1].comments.extend(self._comments) 935 self._comments = [] 936 self._prev_token_line = self._line 937 938 return True 939 940 def _scan_number(self) -> None: 941 if self._char == "0": 942 peek = self._peek.upper() # type: ignore 943 if peek == "B": 944 return self._scan_bits() 945 elif peek == "X": 946 return self._scan_hex() 947 948 decimal = False 949 scientific = 0 950 951 while True: 952 if self._peek.isdigit(): # type: ignore 953 self._advance() 954 elif self._peek == "." and not decimal: 955 decimal = True 956 self._advance() 957 elif self._peek in ("-", "+") and scientific == 1: 958 scientific += 1 959 self._advance() 960 elif self._peek.upper() == "E" and not scientific: # type: ignore 961 scientific += 1 962 self._advance() 963 elif self._peek.isidentifier(): # type: ignore 964 number_text = self._text 965 literal = [] 966 967 while self._peek.strip() and self._peek not in self.SINGLE_TOKENS: # type: ignore 968 literal.append(self._peek.upper()) # type: ignore 969 self._advance() 970 971 literal = "".join(literal) # type: ignore 972 token_type = self.KEYWORDS.get(self.NUMERIC_LITERALS.get(literal)) # type: ignore 973 974 if token_type: 975 self._add(TokenType.NUMBER, number_text) 976 self._add(TokenType.DCOLON, "::") 977 return self._add(token_type, literal) # type: ignore 978 elif self.IDENTIFIER_CAN_START_WITH_DIGIT: 979 return self._add(TokenType.VAR) 980 981 self._add(TokenType.NUMBER, number_text) 982 return self._advance(-len(literal)) 983 else: 984 return self._add(TokenType.NUMBER) 985 986 def _scan_bits(self) -> None: 987 self._advance() 988 value = self._extract_value() 989 try: 990 self._add(TokenType.BIT_STRING, f"{int(value, 2)}") 991 except ValueError: 992 self._add(TokenType.IDENTIFIER) 993 994 def _scan_hex(self) -> None: 995 self._advance() 996 value = self._extract_value() 997 try: 998 self._add(TokenType.HEX_STRING, f"{int(value, 16)}") 999 except ValueError: 1000 self._add(TokenType.IDENTIFIER) 1001 1002 def _extract_value(self) -> str: 1003 while True: 1004 char = self._peek.strip() # type: ignore 1005 if char and char not in self.SINGLE_TOKENS: 1006 self._advance() 1007 else: 1008 break 1009 1010 return self._text 1011 1012 def _scan_string(self, quote: str) -> bool: 1013 quote_end = self._QUOTES.get(quote) # type: ignore 1014 if quote_end is None: 1015 return False 1016 1017 self._advance(len(quote)) 1018 text = self._extract_string(quote_end) 1019 text = text.encode(self.ENCODE).decode(self.ENCODE) if self.ENCODE else text # type: ignore 1020 self._add(TokenType.NATIONAL if quote[0].upper() == "N" else TokenType.STRING, text) 1021 return True 1022 1023 # X'1234, b'0110', E'\\\\\' etc. 1024 def _scan_formatted_string(self, string_start: str) -> bool: 1025 if string_start in self._HEX_STRINGS: # type: ignore 1026 delimiters = self._HEX_STRINGS # type: ignore 1027 token_type = TokenType.HEX_STRING 1028 base = 16 1029 elif string_start in self._BIT_STRINGS: # type: ignore 1030 delimiters = self._BIT_STRINGS # type: ignore 1031 token_type = TokenType.BIT_STRING 1032 base = 2 1033 elif string_start in self._BYTE_STRINGS: # type: ignore 1034 delimiters = self._BYTE_STRINGS # type: ignore 1035 token_type = TokenType.BYTE_STRING 1036 base = None 1037 else: 1038 return False 1039 1040 self._advance(len(string_start)) 1041 string_end = delimiters.get(string_start) 1042 text = self._extract_string(string_end) 1043 1044 if base is None: 1045 self._add(token_type, text) 1046 else: 1047 try: 1048 self._add(token_type, f"{int(text, base)}") 1049 except: 1050 raise RuntimeError( 1051 f"Numeric string contains invalid characters from {self._line}:{self._start}" 1052 ) 1053 1054 return True 1055 1056 def _scan_identifier(self, identifier_end: str) -> None: 1057 text = "" 1058 identifier_end_is_escape = identifier_end in self._IDENTIFIER_ESCAPES 1059 1060 while True: 1061 if self._end: 1062 raise RuntimeError(f"Missing {identifier_end} from {self._line}:{self._start}") 1063 1064 self._advance() 1065 if self._char == identifier_end: 1066 if identifier_end_is_escape and self._peek == identifier_end: 1067 text += identifier_end # type: ignore 1068 self._advance() 1069 continue 1070 1071 break 1072 1073 text += self._char # type: ignore 1074 1075 self._add(TokenType.IDENTIFIER, text) 1076 1077 def _scan_var(self) -> None: 1078 while True: 1079 char = self._peek.strip() # type: ignore 1080 if char and char not in self.SINGLE_TOKENS: 1081 self._advance() 1082 else: 1083 break 1084 self._add( 1085 TokenType.VAR 1086 if self._prev_token_type == TokenType.PARAMETER 1087 else self.KEYWORDS.get(self._text.upper(), TokenType.VAR) 1088 ) 1089 1090 def _extract_string(self, delimiter: str) -> str: 1091 text = "" 1092 delim_size = len(delimiter) 1093 1094 while True: 1095 if self._char in self._STRING_ESCAPES and ( 1096 self._peek == delimiter or self._peek in self._STRING_ESCAPES 1097 ): 1098 if self._peek == delimiter: 1099 text += self._peek # type: ignore 1100 else: 1101 text += self._char + self._peek # type: ignore 1102 1103 if self._current + 1 < self.size: 1104 self._advance(2) 1105 else: 1106 raise RuntimeError(f"Missing {delimiter} from {self._line}:{self._current}") 1107 else: 1108 if self._chars(delim_size) == delimiter: 1109 if delim_size > 1: 1110 self._advance(delim_size - 1) 1111 break 1112 1113 if self._end: 1114 raise RuntimeError(f"Missing {delimiter} from {self._line}:{self._start}") 1115 text += self._char # type: ignore 1116 self._advance() 1117 1118 return text
def
reset(self) -> None:
754 def reset(self) -> None: 755 self.sql = "" 756 self.size = 0 757 self.tokens: t.List[Token] = [] 758 self._start = 0 759 self._current = 0 760 self._line = 1 761 self._col = 1 762 self._comments: t.List[str] = [] 763 764 self._char = None 765 self._end = None 766 self._peek = None 767 self._prev_token_line = -1 768 self._prev_token_comments: t.List[str] = [] 769 self._prev_token_type = None