Adding upstream version 20.3.0.upstream/20.3.0

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 14 insertions, 0 deletions

diff --git a/tests/test_tokens.py b/tests/test_tokens.py
index b97f54a..970c1ac 100644
--- a/tests/test_tokens.py
+++ b/tests/test_tokens.py
@@ -71,6 +71,20 @@ x"""
         self.assertEqual(tokens[2].line, 2)
         self.assertEqual(tokens[3].line, 3)
 
+    def test_crlf(self):
+        tokens = Tokenizer().tokenize("SELECT a\r\nFROM b")
+        tokens = [(token.token_type, token.text) for token in tokens]
+
+        self.assertEqual(
+            tokens,
+            [
+                (TokenType.SELECT, "SELECT"),
+                (TokenType.VAR, "a"),
+                (TokenType.FROM, "FROM"),
+                (TokenType.VAR, "b"),
+            ],
+        )
+
     def test_command(self):
         tokens = Tokenizer().tokenize("SHOW;")
         self.assertEqual(tokens[0].token_type, TokenType.SHOW)