1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
|
import bz2
import gzip
from pygments import highlight
from pygments.formatters import HtmlFormatter, RawTokenFormatter
from pygments.lexers import PythonLexer, RawTokenLexer
def test_raw_token():
code = "2 + α"
raw = highlight(code, PythonLexer(), RawTokenFormatter())
html = highlight(code, PythonLexer(), HtmlFormatter())
assert highlight(raw, RawTokenLexer(), RawTokenFormatter()) == raw
assert highlight(raw, RawTokenLexer(), HtmlFormatter()) == html
assert highlight(raw.decode(), RawTokenLexer(), HtmlFormatter()) == html
raw_gz = highlight(code, PythonLexer(), RawTokenFormatter(compress="gz"))
assert gzip.decompress(raw_gz) == raw
assert highlight(raw_gz, RawTokenLexer(compress="gz"), RawTokenFormatter()) == raw
assert (
highlight(
raw_gz.decode("latin1"), RawTokenLexer(compress="gz"), RawTokenFormatter()
)
== raw
)
raw_bz2 = highlight(code, PythonLexer(), RawTokenFormatter(compress="bz2"))
assert bz2.decompress(raw_bz2) == raw
assert highlight(raw_bz2, RawTokenLexer(compress="bz2"), RawTokenFormatter()) == raw
assert (
highlight(
raw_bz2.decode("latin1"), RawTokenLexer(compress="bz2"), RawTokenFormatter()
)
== raw
)
def test_invalid_raw_token():
# These should not throw exceptions.
assert (
highlight("Tolkien", RawTokenLexer(), RawTokenFormatter())
== b"Token.Error\t'Tolkien\\n'\n"
)
assert (
highlight("Tolkien\t'x'", RawTokenLexer(), RawTokenFormatter())
== b"Token\t'x'\n"
)
assert (
highlight("Token.Text\t42", RawTokenLexer(), RawTokenFormatter())
== b"Token.Error\t'Token.Text\\t42\\n'\n"
)
assert (
highlight("Token.Text\t'", RawTokenLexer(), RawTokenFormatter())
== b'Token.Error\t"Token.Text\\t\'\\n"\n'
)
assert (
highlight("Token.Text\t'α'", RawTokenLexer(), RawTokenFormatter())
== b"Token.Text\t'\\u03b1'\n"
)
assert (
highlight("Token.Text\tu'α'", RawTokenLexer(), RawTokenFormatter())
== b"Token.Text\t'\\u03b1'\n"
)
assert (
highlight(b"Token.Text\t'\xff'", RawTokenLexer(), RawTokenFormatter())
== b"Token.Text\t'\\xff'\n"
)
|
