summaryrefslogtreecommitdiffstats
path: root/tests/test_tokens.py
diff options
context:
space:
mode:
authorDaniel Baumann <mail@daniel-baumann.ch>2023-12-10 10:45:55 +0000
committerDaniel Baumann <mail@daniel-baumann.ch>2023-12-10 10:45:55 +0000
commit02df6cdb000c8dbf739abda2af321a4f90d1b059 (patch)
tree2fc1daf848082ff67a11e60025cac260e3c318b2 /tests/test_tokens.py
parentAdding upstream version 19.0.1. (diff)
downloadsqlglot-02df6cdb000c8dbf739abda2af321a4f90d1b059.tar.xz
sqlglot-02df6cdb000c8dbf739abda2af321a4f90d1b059.zip
Adding upstream version 20.1.0.upstream/20.1.0
Signed-off-by: Daniel Baumann <mail@daniel-baumann.ch>
Diffstat (limited to 'tests/test_tokens.py')
-rw-r--r--tests/test_tokens.py8
1 files changed, 8 insertions, 0 deletions
diff --git a/tests/test_tokens.py b/tests/test_tokens.py
index f4d3858..b97f54a 100644
--- a/tests/test_tokens.py
+++ b/tests/test_tokens.py
@@ -13,6 +13,7 @@ class TestTokens(unittest.TestCase):
(" group bys ", 2),
("group by)", 2),
("group bys)", 3),
+ ("group \r", 1),
):
tokens = Tokenizer().tokenize(string)
self.assertTrue("GROUP" in tokens[0].text.upper())
@@ -63,6 +64,13 @@ x"""
self.assertEqual(Tokenizer().tokenize("'''abc'")[0].end, 6)
self.assertEqual(Tokenizer().tokenize("'abc'")[0].start, 0)
+ tokens = Tokenizer().tokenize("SELECT\r\n 1,\r\n 2")
+
+ self.assertEqual(tokens[0].line, 1)
+ self.assertEqual(tokens[1].line, 2)
+ self.assertEqual(tokens[2].line, 2)
+ self.assertEqual(tokens[3].line, 3)
+
def test_command(self):
tokens = Tokenizer().tokenize("SHOW;")
self.assertEqual(tokens[0].token_type, TokenType.SHOW)