diff options
Diffstat (limited to 'sqlglot/tokens.py')
| -rw-r--r-- | sqlglot/tokens.py | 45 |
1 files changed, 40 insertions, 5 deletions
diff --git a/sqlglot/tokens.py b/sqlglot/tokens.py index ad329d2..a30ec24 100644 --- a/sqlglot/tokens.py +++ b/sqlglot/tokens.py @@ -113,6 +113,18 @@ class TokenType(AutoName): DATETIME = auto() DATETIME64 = auto() DATE = auto() + INT4RANGE = auto() + INT4MULTIRANGE = auto() + INT8RANGE = auto() + INT8MULTIRANGE = auto() + NUMRANGE = auto() + NUMMULTIRANGE = auto() + TSRANGE = auto() + TSMULTIRANGE = auto() + TSTZRANGE = auto() + TSTZMULTIRANGE = auto() + DATERANGE = auto() + DATEMULTIRANGE = auto() UUID = auto() GEOGRAPHY = auto() NULLABLE = auto() @@ -167,6 +179,7 @@ class TokenType(AutoName): DELETE = auto() DESC = auto() DESCRIBE = auto() + DICTIONARY = auto() DISTINCT = auto() DIV = auto() DROP = auto() @@ -480,6 +493,7 @@ class Tokenizer(metaclass=_Tokenizer): "ANY": TokenType.ANY, "ASC": TokenType.ASC, "AS": TokenType.ALIAS, + "ASOF": TokenType.ASOF, "AUTOINCREMENT": TokenType.AUTO_INCREMENT, "AUTO_INCREMENT": TokenType.AUTO_INCREMENT, "BEGIN": TokenType.BEGIN, @@ -669,6 +683,18 @@ class Tokenizer(metaclass=_Tokenizer): "TIMESTAMPLTZ": TokenType.TIMESTAMPLTZ, "DATE": TokenType.DATE, "DATETIME": TokenType.DATETIME, + "INT4RANGE": TokenType.INT4RANGE, + "INT4MULTIRANGE": TokenType.INT4MULTIRANGE, + "INT8RANGE": TokenType.INT8RANGE, + "INT8MULTIRANGE": TokenType.INT8MULTIRANGE, + "NUMRANGE": TokenType.NUMRANGE, + "NUMMULTIRANGE": TokenType.NUMMULTIRANGE, + "TSRANGE": TokenType.TSRANGE, + "TSMULTIRANGE": TokenType.TSMULTIRANGE, + "TSTZRANGE": TokenType.TSTZRANGE, + "TSTZMULTIRANGE": TokenType.TSTZMULTIRANGE, + "DATERANGE": TokenType.DATERANGE, + "DATEMULTIRANGE": TokenType.DATEMULTIRANGE, "UNIQUE": TokenType.UNIQUE, "STRUCT": TokenType.STRUCT, "VARIANT": TokenType.VARIANT, @@ -709,8 +735,6 @@ class Tokenizer(metaclass=_Tokenizer): COMMENTS = ["--", ("/*", "*/"), ("{#", "#}")] KEYWORD_TRIE: t.Dict = {} # autofilled - IDENTIFIER_CAN_START_WITH_DIGIT = False - __slots__ = ( "sql", "size", @@ -724,6 +748,7 @@ class Tokenizer(metaclass=_Tokenizer): "_end", "_peek", "_prev_token_line", + "identifiers_can_start_with_digit", ) def __init__(self) -> None: @@ -826,6 +851,12 @@ class Tokenizer(metaclass=_Tokenizer): def _text(self) -> str: return self.sql[self._start : self._current] + def peek(self, i: int = 0) -> str: + i = self._current + i + if i < self.size: + return self.sql[i] + return "" + def _add(self, token_type: TokenType, text: t.Optional[str] = None) -> None: self._prev_token_line = self._line self.tokens.append( @@ -962,8 +993,12 @@ class Tokenizer(metaclass=_Tokenizer): if self._peek.isdigit(): self._advance() elif self._peek == "." and not decimal: - decimal = True - self._advance() + after = self.peek(1) + if after.isdigit() or not after.strip(): + decimal = True + self._advance() + else: + return self._add(TokenType.VAR) elif self._peek in ("-", "+") and scientific == 1: scientific += 1 self._advance() @@ -984,7 +1019,7 @@ class Tokenizer(metaclass=_Tokenizer): self._add(TokenType.NUMBER, number_text) self._add(TokenType.DCOLON, "::") return self._add(token_type, literal) - elif self.IDENTIFIER_CAN_START_WITH_DIGIT: + elif self.identifiers_can_start_with_digit: # type: ignore return self._add(TokenType.VAR) self._add(TokenType.NUMBER, number_text) |