summaryrefslogtreecommitdiffstats
path: root/tests/test_tokens.py
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2023-05-03 09:12:24 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2023-05-03 09:12:24 +0000
commit98d5537435b2951b36c45f1fda667fa27c165794 (patch)
treed26b4dfa6cf91847100fe10a94a04dcc2ad36a86 /tests/test_tokens.py
parentAdding upstream version 11.5.2. (diff)
downloadsqlglot-98d5537435b2951b36c45f1fda667fa27c165794.tar.xz
sqlglot-98d5537435b2951b36c45f1fda667fa27c165794.zip
Adding upstream version 11.7.1.upstream/11.7.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'tests/test_tokens.py')
-rw-r--r--tests/test_tokens.py18
1 files changed, 13 insertions, 5 deletions
diff --git a/tests/test_tokens.py b/tests/test_tokens.py
index 8481f4d..987c60b 100644
--- a/tests/test_tokens.py
+++ b/tests/test_tokens.py
@@ -14,6 +14,7 @@ class TestTokens(unittest.TestCase):
("foo", []),
("foo /*comment 1*/ /*comment 2*/", ["comment 1", "comment 2"]),
("foo\n-- comment", [" comment"]),
+ ("1 /*/2 */", ["/2 "]),
]
for sql, comment in sql_comment:
@@ -22,14 +23,17 @@ class TestTokens(unittest.TestCase):
def test_token_line(self):
tokens = Tokenizer().tokenize(
"""SELECT /*
- line break
- */
- 'x
- y',
- x"""
+line break
+*/
+'x
+ y',
+x"""
)
+ self.assertEqual(tokens[1].line, 5)
+ self.assertEqual(tokens[1].col, 3)
self.assertEqual(tokens[-1].line, 6)
+ self.assertEqual(tokens[-1].col, 1)
def test_command(self):
tokens = Tokenizer().tokenize("SHOW;")
@@ -46,6 +50,10 @@ class TestTokens(unittest.TestCase):
self.assertEqual(tokens[2].token_type, TokenType.SHOW)
self.assertEqual(tokens[3].token_type, TokenType.SEMICOLON)
+ def test_error_msg(self):
+ with self.assertRaisesRegex(ValueError, "Error tokenizing 'select.*"):
+ Tokenizer().tokenize("select /*")
+
def test_jinja(self):
tokenizer = Tokenizer()