Merging upstream version 10.0.1.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 18 insertions, 0 deletions

diff --git a/tests/test_tokens.py b/tests/test_tokens.py
new file mode 100644
index 0000000..943c2b0
--- /dev/null
+++ b/tests/test_tokens.py
@@ -0,0 +1,18 @@
+import unittest
+
+from sqlglot.tokens import Tokenizer
+
+
+class TestTokens(unittest.TestCase):
+    def test_comment_attachment(self):
+        tokenizer = Tokenizer()
+        sql_comment = [
+            ("/*comment*/ foo", "comment"),
+            ("/*comment*/ foo --test", "comment"),
+            ("--comment\nfoo --test", "comment"),
+            ("foo --comment", "comment"),
+            ("foo", None),
+        ]
+
+        for sql, comment in sql_comment:
+            self.assertEqual(tokenizer.tokenize(sql)[0].comment, comment)