From 5b1ac5070c43c40a2b5bbc991198b0dddf45dc75 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 8 Mar 2023 08:22:15 +0100 Subject: Merging upstream version 11.3.3. Signed-off-by: Daniel Baumann --- sqlglot/tokens.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'sqlglot/tokens.py') diff --git a/sqlglot/tokens.py b/sqlglot/tokens.py index 7a23803..5f4b77d 100644 --- a/sqlglot/tokens.py +++ b/sqlglot/tokens.py @@ -855,11 +855,12 @@ class Tokenizer(metaclass=_Tokenizer): def _scan_keywords(self) -> None: size = 0 word = None - chars: t.Optional[str] = self._text + chars = self._text char = chars prev_space = False skip = False trie = self.KEYWORD_TRIE + single_token = char in self.SINGLE_TOKENS while chars: if skip: @@ -876,6 +877,7 @@ class Tokenizer(metaclass=_Tokenizer): if end < self.size: char = self.sql[end] + single_token = single_token or char in self.SINGLE_TOKENS is_space = char in self.WHITE_SPACE if not is_space or not prev_space: @@ -887,7 +889,9 @@ class Tokenizer(metaclass=_Tokenizer): else: skip = True else: - chars = None + chars = " " + + word = None if not single_token and chars[-1] not in self.WHITE_SPACE else word if not word: if self._char in self.SINGLE_TOKENS: -- cgit v1.2.3