diff options
Diffstat (limited to 'tests/test_tokens.py')
-rw-r--r-- | tests/test_tokens.py | 11 |
1 files changed, 11 insertions, 0 deletions
diff --git a/tests/test_tokens.py b/tests/test_tokens.py index 987c60b..f70d70e 100644 --- a/tests/test_tokens.py +++ b/tests/test_tokens.py @@ -102,3 +102,14 @@ x""" (TokenType.SEMICOLON, ";"), ], ) + + tokens = tokenizer.tokenize("""'{{ var('x') }}'""") + tokens = [(token.token_type, token.text) for token in tokens] + self.assertEqual( + tokens, + [ + (TokenType.STRING, "{{ var("), + (TokenType.VAR, "x"), + (TokenType.STRING, ") }}"), + ], + ) |