diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2023-03-08 07:22:15 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2023-03-08 07:22:15 +0000 |
commit | 5b1ac5070c43c40a2b5bbc991198b0dddf45dc75 (patch) | |
tree | ed329138d5e8e5c9d5164b5c853d6f40a116f4d6 /sqlglot/tokens.py | |
parent | Releasing debian version 11.3.0-1. (diff) | |
download | sqlglot-5b1ac5070c43c40a2b5bbc991198b0dddf45dc75.tar.xz sqlglot-5b1ac5070c43c40a2b5bbc991198b0dddf45dc75.zip |
Merging upstream version 11.3.3.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'sqlglot/tokens.py')
-rw-r--r-- | sqlglot/tokens.py | 8 |
1 files changed, 6 insertions, 2 deletions
diff --git a/sqlglot/tokens.py b/sqlglot/tokens.py index 7a23803..5f4b77d 100644 --- a/sqlglot/tokens.py +++ b/sqlglot/tokens.py @@ -855,11 +855,12 @@ class Tokenizer(metaclass=_Tokenizer): def _scan_keywords(self) -> None: size = 0 word = None - chars: t.Optional[str] = self._text + chars = self._text char = chars prev_space = False skip = False trie = self.KEYWORD_TRIE + single_token = char in self.SINGLE_TOKENS while chars: if skip: @@ -876,6 +877,7 @@ class Tokenizer(metaclass=_Tokenizer): if end < self.size: char = self.sql[end] + single_token = single_token or char in self.SINGLE_TOKENS is_space = char in self.WHITE_SPACE if not is_space or not prev_space: @@ -887,7 +889,9 @@ class Tokenizer(metaclass=_Tokenizer): else: skip = True else: - chars = None + chars = " " + + word = None if not single_token and chars[-1] not in self.WHITE_SPACE else word if not word: if self._char in self.SINGLE_TOKENS: |