summaryrefslogtreecommitdiffstats
path: root/tests/test_tokens.py
diff options
context:
space:
mode:
Diffstat (limited to 'tests/test_tokens.py')
-rw-r--r--tests/test_tokens.py14
1 files changed, 14 insertions, 0 deletions
diff --git a/tests/test_tokens.py b/tests/test_tokens.py
index b97f54a..970c1ac 100644
--- a/tests/test_tokens.py
+++ b/tests/test_tokens.py
@@ -71,6 +71,20 @@ x"""
self.assertEqual(tokens[2].line, 2)
self.assertEqual(tokens[3].line, 3)
+ def test_crlf(self):
+ tokens = Tokenizer().tokenize("SELECT a\r\nFROM b")
+ tokens = [(token.token_type, token.text) for token in tokens]
+
+ self.assertEqual(
+ tokens,
+ [
+ (TokenType.SELECT, "SELECT"),
+ (TokenType.VAR, "a"),
+ (TokenType.FROM, "FROM"),
+ (TokenType.VAR, "b"),
+ ],
+ )
+
def test_command(self):
tokens = Tokenizer().tokenize("SHOW;")
self.assertEqual(tokens[0].token_type, TokenType.SHOW)