summaryrefslogtreecommitdiffstats
path: root/tests/test_tokens.py
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2023-06-02 23:59:11 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2023-06-02 23:59:11 +0000
commitcaea5267cb8e1fea3702adbdf6f68fd37d13b3b7 (patch)
treef06f1da1ab3b6906beca1c3c7222d28ff00766ac /tests/test_tokens.py
parentAdding upstream version 12.2.0. (diff)
downloadsqlglot-caea5267cb8e1fea3702adbdf6f68fd37d13b3b7.tar.xz
sqlglot-caea5267cb8e1fea3702adbdf6f68fd37d13b3b7.zip
Adding upstream version 15.0.0.upstream/15.0.0
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'tests/test_tokens.py')
-rw-r--r--tests/test_tokens.py21
1 files changed, 17 insertions, 4 deletions
diff --git a/tests/test_tokens.py b/tests/test_tokens.py
index f70d70e..30af34f 100644
--- a/tests/test_tokens.py
+++ b/tests/test_tokens.py
@@ -20,7 +20,7 @@ class TestTokens(unittest.TestCase):
for sql, comment in sql_comment:
self.assertEqual(tokenizer.tokenize(sql)[0].comments, comment)
- def test_token_line(self):
+ def test_token_line_col(self):
tokens = Tokenizer().tokenize(
"""SELECT /*
line break
@@ -30,10 +30,23 @@ line break
x"""
)
+ self.assertEqual(tokens[0].line, 1)
+ self.assertEqual(tokens[0].col, 6)
self.assertEqual(tokens[1].line, 5)
self.assertEqual(tokens[1].col, 3)
- self.assertEqual(tokens[-1].line, 6)
- self.assertEqual(tokens[-1].col, 1)
+ self.assertEqual(tokens[2].line, 5)
+ self.assertEqual(tokens[2].col, 4)
+ self.assertEqual(tokens[3].line, 6)
+ self.assertEqual(tokens[3].col, 1)
+
+ tokens = Tokenizer().tokenize("SELECT .")
+
+ self.assertEqual(tokens[1].line, 1)
+ self.assertEqual(tokens[1].col, 8)
+
+ self.assertEqual(Tokenizer().tokenize("'''abc'")[0].start, 0)
+ self.assertEqual(Tokenizer().tokenize("'''abc'")[0].end, 6)
+ self.assertEqual(Tokenizer().tokenize("'abc'")[0].start, 0)
def test_command(self):
tokens = Tokenizer().tokenize("SHOW;")
@@ -51,7 +64,7 @@ x"""
self.assertEqual(tokens[3].token_type, TokenType.SEMICOLON)
def test_error_msg(self):
- with self.assertRaisesRegex(ValueError, "Error tokenizing 'select.*"):
+ with self.assertRaisesRegex(ValueError, "Error tokenizing 'select /'"):
Tokenizer().tokenize("select /*")
def test_jinja(self):