summaryrefslogtreecommitdiffstats
path: root/tests/test_tokens.py
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--tests/test_tokens.py18
1 files changed, 18 insertions, 0 deletions
diff --git a/tests/test_tokens.py b/tests/test_tokens.py
new file mode 100644
index 0000000..943c2b0
--- /dev/null
+++ b/tests/test_tokens.py
@@ -0,0 +1,18 @@
+import unittest
+
+from sqlglot.tokens import Tokenizer
+
+
+class TestTokens(unittest.TestCase):
+ def test_comment_attachment(self):
+ tokenizer = Tokenizer()
+ sql_comment = [
+ ("/*comment*/ foo", "comment"),
+ ("/*comment*/ foo --test", "comment"),
+ ("--comment\nfoo --test", "comment"),
+ ("foo --comment", "comment"),
+ ("foo", None),
+ ]
+
+ for sql, comment in sql_comment:
+ self.assertEqual(tokenizer.tokenize(sql)[0].comment, comment)