1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
|
<?php
/**
* This file is part of FPDI
*
* @package setasign\Fpdi
* @copyright Copyright (c) 2020 Setasign GmbH & Co. KG (https://www.setasign.com)
* @license http://opensource.org/licenses/mit-license The MIT License
*/
namespace setasign\Fpdi\PdfParser;
/**
* A tokenizer class.
*/
class Tokenizer
{
/**
* @var StreamReader
*/
protected $streamReader;
/**
* A token stack.
*
* @var string[]
*/
protected $stack = [];
/**
* Tokenizer constructor.
*
* @param StreamReader $streamReader
*/
public function __construct(StreamReader $streamReader)
{
$this->streamReader = $streamReader;
}
/**
* Get the stream reader instance.
*
* @return StreamReader
*/
public function getStreamReader()
{
return $this->streamReader;
}
/**
* Clear the token stack.
*/
public function clearStack()
{
$this->stack = [];
}
/**
* Push a token onto the stack.
*
* @param string $token
*/
public function pushStack($token)
{
$this->stack[] = $token;
}
/**
* Get next token.
*
* @return bool|string
*/
public function getNextToken()
{
$token = \array_pop($this->stack);
if ($token !== null) {
return $token;
}
if (($byte = $this->streamReader->readByte()) === false) {
return false;
}
if (\in_array($byte, ["\x20", "\x0A", "\x0D", "\x0C", "\x09", "\x00"], true)) {
if ($this->leapWhiteSpaces() === false) {
return false;
}
$byte = $this->streamReader->readByte();
}
switch ($byte) {
case '/':
case '[':
case ']':
case '(':
case ')':
case '{':
case '}':
case '<':
case '>':
return $byte;
case '%':
$this->streamReader->readLine();
return $this->getNextToken();
}
/* This way is faster than checking single bytes.
*/
$bufferOffset = $this->streamReader->getOffset();
do {
$lastBuffer = $this->streamReader->getBuffer(false);
$pos = \strcspn(
$lastBuffer,
"\x00\x09\x0A\x0C\x0D\x20()<>[]{}/%",
$bufferOffset
);
} while (
// Break the loop if a delimiter or white space char is matched
// in the current buffer or increase the buffers length
$lastBuffer !== false &&
(
$bufferOffset + $pos === \strlen($lastBuffer) &&
$this->streamReader->increaseLength()
)
);
$result = \substr($lastBuffer, $bufferOffset - 1, $pos + 1);
$this->streamReader->setOffset($bufferOffset + $pos);
return $result;
}
/**
* Leap white spaces.
*
* @return boolean
*/
public function leapWhiteSpaces()
{
do {
if (!$this->streamReader->ensureContent()) {
return false;
}
$buffer = $this->streamReader->getBuffer(false);
$matches = \strspn($buffer, "\x20\x0A\x0C\x0D\x09\x00", $this->streamReader->getOffset());
if ($matches > 0) {
$this->streamReader->addOffset($matches);
}
} while ($this->streamReader->getOffset() >= $this->streamReader->getBufferLength());
return true;
}
}
|
