diff options
Diffstat (limited to 'docs/sqlglot/tokens.html')
| -rw-r--r-- | docs/sqlglot/tokens.html | 852 |
1 files changed, 427 insertions, 425 deletions
diff --git a/docs/sqlglot/tokens.html b/docs/sqlglot/tokens.html index 5ec83ff..4ea9bdf 100644 --- a/docs/sqlglot/tokens.html +++ b/docs/sqlglot/tokens.html @@ -3,7 +3,7 @@ <head> <meta charset="utf-8"> <meta name="viewport" content="width=device-width, initial-scale=1"> - <meta name="generator" content="pdoc 13.0.1"/> + <meta name="generator" content="pdoc 13.1.0"/> <title>sqlglot.tokens API documentation</title> <style>/*! * Bootstrap Reboot v5.0.0 (https://getbootstrap.com/) * Copyright 2011-2021 The Bootstrap Authors * Copyright 2011-2021 Twitter, Inc. * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE) * Forked from Normalize.css, licensed MIT (https://github.com/necolas/normalize.css/blob/master/LICENSE.md) */*,::after,::before{box-sizing:border-box}@media (prefers-reduced-motion:no-preference){:root{scroll-behavior:smooth}}body{margin:0;font-family:system-ui,-apple-system,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans","Liberation Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-size:1rem;font-weight:400;line-height:1.5;color:#212529;background-color:#fff;-webkit-text-size-adjust:100%;-webkit-tap-highlight-color:transparent}hr{margin:1rem 0;color:inherit;background-color:currentColor;border:0;opacity:.25}hr:not([size]){height:1px}h1,h2,h3,h4,h5,h6{margin-top:0;margin-bottom:.5rem;font-weight:500;line-height:1.2}h1{font-size:calc(1.375rem + 1.5vw)}@media (min-width:1200px){h1{font-size:2.5rem}}h2{font-size:calc(1.325rem + .9vw)}@media (min-width:1200px){h2{font-size:2rem}}h3{font-size:calc(1.3rem + .6vw)}@media (min-width:1200px){h3{font-size:1.75rem}}h4{font-size:calc(1.275rem + .3vw)}@media (min-width:1200px){h4{font-size:1.5rem}}h5{font-size:1.25rem}h6{font-size:1rem}p{margin-top:0;margin-bottom:1rem}abbr[data-bs-original-title],abbr[title]{-webkit-text-decoration:underline dotted;text-decoration:underline dotted;cursor:help;-webkit-text-decoration-skip-ink:none;text-decoration-skip-ink:none}address{margin-bottom:1rem;font-style:normal;line-height:inherit}ol,ul{padding-left:2rem}dl,ol,ul{margin-top:0;margin-bottom:1rem}ol ol,ol ul,ul ol,ul ul{margin-bottom:0}dt{font-weight:700}dd{margin-bottom:.5rem;margin-left:0}blockquote{margin:0 0 1rem}b,strong{font-weight:bolder}small{font-size:.875em}mark{padding:.2em;background-color:#fcf8e3}sub,sup{position:relative;font-size:.75em;line-height:0;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}a{color:#0d6efd;text-decoration:underline}a:hover{color:#0a58ca}a:not([href]):not([class]),a:not([href]):not([class]):hover{color:inherit;text-decoration:none}code,kbd,pre,samp{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;font-size:1em;direction:ltr;unicode-bidi:bidi-override}pre{display:block;margin-top:0;margin-bottom:1rem;overflow:auto;font-size:.875em}pre code{font-size:inherit;color:inherit;word-break:normal}code{font-size:.875em;color:#d63384;word-wrap:break-word}a>code{color:inherit}kbd{padding:.2rem .4rem;font-size:.875em;color:#fff;background-color:#212529;border-radius:.2rem}kbd kbd{padding:0;font-size:1em;font-weight:700}figure{margin:0 0 1rem}img,svg{vertical-align:middle}table{caption-side:bottom;border-collapse:collapse}caption{padding-top:.5rem;padding-bottom:.5rem;color:#6c757d;text-align:left}th{text-align:inherit;text-align:-webkit-match-parent}tbody,td,tfoot,th,thead,tr{border-color:inherit;border-style:solid;border-width:0}label{display:inline-block}button{border-radius:0}button:focus:not(:focus-visible){outline:0}button,input,optgroup,select,textarea{margin:0;font-family:inherit;font-size:inherit;line-height:inherit}button,select{text-transform:none}[role=button]{cursor:pointer}select{word-wrap:normal}select:disabled{opacity:1}[list]::-webkit-calendar-picker-indicator{display:none}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button}[type=button]:not(:disabled),[type=reset]:not(:disabled),[type=submit]:not(:disabled),button:not(:disabled){cursor:pointer}::-moz-focus-inner{padding:0;border-style:none}textarea{resize:vertical}fieldset{min-width:0;padding:0;margin:0;border:0}legend{float:left;width:100%;padding:0;margin-bottom:.5rem;font-size:calc(1.275rem + .3vw);line-height:inherit}@media (min-width:1200px){legend{font-size:1.5rem}}legend+*{clear:left}::-webkit-datetime-edit-day-field,::-webkit-datetime-edit-fields-wrapper,::-webkit-datetime-edit-hour-field,::-webkit-datetime-edit-minute,::-webkit-datetime-edit-month-field,::-webkit-datetime-edit-text,::-webkit-datetime-edit-year-field{padding:0}::-webkit-inner-spin-button{height:auto}[type=search]{outline-offset:-2px;-webkit-appearance:textfield}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-color-swatch-wrapper{padding:0}::file-selector-button{font:inherit}::-webkit-file-upload-button{font:inherit;-webkit-appearance:button}output{display:inline-block}iframe{border:0}summary{display:list-item;cursor:pointer}progress{vertical-align:baseline}[hidden]{display:none!important}</style> @@ -1890,7 +1890,7 @@ </span><span id="L-908"><a href="#L-908"><span class="linenos"> 908</span></a> </span><span id="L-909"><a href="#L-909"><span class="linenos"> 909</span></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">word</span><span class="p">:</span> </span><span id="L-910"><a href="#L-910"><span class="linenos"> 910</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">SINGLE_TOKENS</span><span class="p">:</span> -</span><span id="L-911"><a href="#L-911"><span class="linenos"> 911</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">SINGLE_TOKENS</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_char</span><span class="p">])</span> <span class="c1"># type: ignore</span> +</span><span id="L-911"><a href="#L-911"><span class="linenos"> 911</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">SINGLE_TOKENS</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_char</span><span class="p">],</span> <span class="n">text</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_char</span><span class="p">)</span> <span class="c1"># type: ignore</span> </span><span id="L-912"><a href="#L-912"><span class="linenos"> 912</span></a> <span class="k">return</span> </span><span id="L-913"><a href="#L-913"><span class="linenos"> 913</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_scan_var</span><span class="p">()</span> </span><span id="L-914"><a href="#L-914"><span class="linenos"> 914</span></a> <span class="k">return</span> @@ -1903,217 +1903,218 @@ </span><span id="L-921"><a href="#L-921"><span class="linenos"> 921</span></a> <span class="k">return</span> </span><span id="L-922"><a href="#L-922"><span class="linenos"> 922</span></a> </span><span id="L-923"><a href="#L-923"><span class="linenos"> 923</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="n">size</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> -</span><span id="L-924"><a href="#L-924"><span class="linenos"> 924</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">KEYWORDS</span><span class="p">[</span><span class="n">word</span><span class="o">.</span><span class="n">upper</span><span class="p">()])</span> -</span><span id="L-925"><a href="#L-925"><span class="linenos"> 925</span></a> -</span><span id="L-926"><a href="#L-926"><span class="linenos"> 926</span></a> <span class="k">def</span> <span class="nf">_scan_comment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">comment_start</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> -</span><span id="L-927"><a href="#L-927"><span class="linenos"> 927</span></a> <span class="k">if</span> <span class="n">comment_start</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_COMMENTS</span><span class="p">:</span> <span class="c1"># type: ignore</span> -</span><span id="L-928"><a href="#L-928"><span class="linenos"> 928</span></a> <span class="k">return</span> <span class="kc">False</span> -</span><span id="L-929"><a href="#L-929"><span class="linenos"> 929</span></a> -</span><span id="L-930"><a href="#L-930"><span class="linenos"> 930</span></a> <span class="n">comment_start_line</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_line</span> -</span><span id="L-931"><a href="#L-931"><span class="linenos"> 931</span></a> <span class="n">comment_start_size</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">comment_start</span><span class="p">)</span> -</span><span id="L-932"><a href="#L-932"><span class="linenos"> 932</span></a> <span class="n">comment_end</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_COMMENTS</span><span class="p">[</span><span class="n">comment_start</span><span class="p">]</span> <span class="c1"># type: ignore</span> -</span><span id="L-933"><a href="#L-933"><span class="linenos"> 933</span></a> -</span><span id="L-934"><a href="#L-934"><span class="linenos"> 934</span></a> <span class="k">if</span> <span class="n">comment_end</span><span class="p">:</span> -</span><span id="L-935"><a href="#L-935"><span class="linenos"> 935</span></a> <span class="n">comment_end_size</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">comment_end</span><span class="p">)</span> -</span><span id="L-936"><a href="#L-936"><span class="linenos"> 936</span></a> -</span><span id="L-937"><a href="#L-937"><span class="linenos"> 937</span></a> <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_end</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_chars</span><span class="p">(</span><span class="n">comment_end_size</span><span class="p">)</span> <span class="o">!=</span> <span class="n">comment_end</span><span class="p">:</span> -</span><span id="L-938"><a href="#L-938"><span class="linenos"> 938</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="L-939"><a href="#L-939"><span class="linenos"> 939</span></a> -</span><span id="L-940"><a href="#L-940"><span class="linenos"> 940</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_comments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_text</span><span class="p">[</span><span class="n">comment_start_size</span> <span class="p">:</span> <span class="o">-</span><span class="n">comment_end_size</span> <span class="o">+</span> <span class="mi">1</span><span class="p">])</span> <span class="c1"># type: ignore</span> -</span><span id="L-941"><a href="#L-941"><span class="linenos"> 941</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="n">comment_end_size</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> -</span><span id="L-942"><a href="#L-942"><span class="linenos"> 942</span></a> <span class="k">else</span><span class="p">:</span> -</span><span id="L-943"><a href="#L-943"><span class="linenos"> 943</span></a> <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_end</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">WHITE_SPACE</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="p">)</span> <span class="ow">is</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">BREAK</span><span class="p">:</span> -</span><span id="L-944"><a href="#L-944"><span class="linenos"> 944</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="L-945"><a href="#L-945"><span class="linenos"> 945</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_comments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_text</span><span class="p">[</span><span class="n">comment_start_size</span><span class="p">:])</span> <span class="c1"># type: ignore</span> -</span><span id="L-946"><a href="#L-946"><span class="linenos"> 946</span></a> -</span><span id="L-947"><a href="#L-947"><span class="linenos"> 947</span></a> <span class="c1"># Leading comment is attached to the succeeding token, whilst trailing comment to the preceding.</span> -</span><span id="L-948"><a href="#L-948"><span class="linenos"> 948</span></a> <span class="c1"># Multiple consecutive comments are preserved by appending them to the current comments list.</span> -</span><span id="L-949"><a href="#L-949"><span class="linenos"> 949</span></a> <span class="k">if</span> <span class="n">comment_start_line</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prev_token_line</span><span class="p">:</span> -</span><span id="L-950"><a href="#L-950"><span class="linenos"> 950</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">tokens</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">comments</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_comments</span><span class="p">)</span> -</span><span id="L-951"><a href="#L-951"><span class="linenos"> 951</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_comments</span> <span class="o">=</span> <span class="p">[]</span> -</span><span id="L-952"><a href="#L-952"><span class="linenos"> 952</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_prev_token_line</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_line</span> -</span><span id="L-953"><a href="#L-953"><span class="linenos"> 953</span></a> -</span><span id="L-954"><a href="#L-954"><span class="linenos"> 954</span></a> <span class="k">return</span> <span class="kc">True</span> -</span><span id="L-955"><a href="#L-955"><span class="linenos"> 955</span></a> -</span><span id="L-956"><a href="#L-956"><span class="linenos"> 956</span></a> <span class="k">def</span> <span class="nf">_scan_number</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> -</span><span id="L-957"><a href="#L-957"><span class="linenos"> 957</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="o">==</span> <span class="s2">"0"</span><span class="p">:</span> -</span><span id="L-958"><a href="#L-958"><span class="linenos"> 958</span></a> <span class="n">peek</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span> <span class="c1"># type: ignore</span> -</span><span id="L-959"><a href="#L-959"><span class="linenos"> 959</span></a> <span class="k">if</span> <span class="n">peek</span> <span class="o">==</span> <span class="s2">"B"</span><span class="p">:</span> -</span><span id="L-960"><a href="#L-960"><span class="linenos"> 960</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_scan_bits</span><span class="p">()</span> -</span><span id="L-961"><a href="#L-961"><span class="linenos"> 961</span></a> <span class="k">elif</span> <span class="n">peek</span> <span class="o">==</span> <span class="s2">"X"</span><span class="p">:</span> -</span><span id="L-962"><a href="#L-962"><span class="linenos"> 962</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_scan_hex</span><span class="p">()</span> -</span><span id="L-963"><a href="#L-963"><span class="linenos"> 963</span></a> -</span><span id="L-964"><a href="#L-964"><span class="linenos"> 964</span></a> <span class="n">decimal</span> <span class="o">=</span> <span class="kc">False</span> -</span><span id="L-965"><a href="#L-965"><span class="linenos"> 965</span></a> <span class="n">scientific</span> <span class="o">=</span> <span class="mi">0</span> -</span><span id="L-966"><a href="#L-966"><span class="linenos"> 966</span></a> -</span><span id="L-967"><a href="#L-967"><span class="linenos"> 967</span></a> <span class="k">while</span> <span class="kc">True</span><span class="p">:</span> -</span><span id="L-968"><a href="#L-968"><span class="linenos"> 968</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">isdigit</span><span class="p">():</span> <span class="c1"># type: ignore</span> -</span><span id="L-969"><a href="#L-969"><span class="linenos"> 969</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="L-970"><a href="#L-970"><span class="linenos"> 970</span></a> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="o">==</span> <span class="s2">"."</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">decimal</span><span class="p">:</span> -</span><span id="L-971"><a href="#L-971"><span class="linenos"> 971</span></a> <span class="n">decimal</span> <span class="o">=</span> <span class="kc">True</span> -</span><span id="L-972"><a href="#L-972"><span class="linenos"> 972</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="L-973"><a href="#L-973"><span class="linenos"> 973</span></a> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"-"</span><span class="p">,</span> <span class="s2">"+"</span><span class="p">)</span> <span class="ow">and</span> <span class="n">scientific</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span> -</span><span id="L-974"><a href="#L-974"><span class="linenos"> 974</span></a> <span class="n">scientific</span> <span class="o">+=</span> <span class="mi">1</span> -</span><span id="L-975"><a href="#L-975"><span class="linenos"> 975</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="L-976"><a href="#L-976"><span class="linenos"> 976</span></a> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"E"</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">scientific</span><span class="p">:</span> <span class="c1"># type: ignore</span> -</span><span id="L-977"><a href="#L-977"><span class="linenos"> 977</span></a> <span class="n">scientific</span> <span class="o">+=</span> <span class="mi">1</span> -</span><span id="L-978"><a href="#L-978"><span class="linenos"> 978</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="L-979"><a href="#L-979"><span class="linenos"> 979</span></a> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">isidentifier</span><span class="p">():</span> <span class="c1"># type: ignore</span> -</span><span id="L-980"><a href="#L-980"><span class="linenos"> 980</span></a> <span class="n">number_text</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_text</span> -</span><span id="L-981"><a href="#L-981"><span class="linenos"> 981</span></a> <span class="n">literal</span> <span class="o">=</span> <span class="p">[]</span> -</span><span id="L-982"><a href="#L-982"><span class="linenos"> 982</span></a> -</span><span id="L-983"><a href="#L-983"><span class="linenos"> 983</span></a> <span class="k">while</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">SINGLE_TOKENS</span><span class="p">:</span> <span class="c1"># type: ignore</span> -</span><span id="L-984"><a href="#L-984"><span class="linenos"> 984</span></a> <span class="n">literal</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">upper</span><span class="p">())</span> <span class="c1"># type: ignore</span> -</span><span id="L-985"><a href="#L-985"><span class="linenos"> 985</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="L-986"><a href="#L-986"><span class="linenos"> 986</span></a> -</span><span id="L-987"><a href="#L-987"><span class="linenos"> 987</span></a> <span class="n">literal</span> <span class="o">=</span> <span class="s2">""</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">literal</span><span class="p">)</span> <span class="c1"># type: ignore</span> -</span><span id="L-988"><a href="#L-988"><span class="linenos"> 988</span></a> <span class="n">token_type</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">KEYWORDS</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">NUMERIC_LITERALS</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">literal</span><span class="p">))</span> <span class="c1"># type: ignore</span> -</span><span id="L-989"><a href="#L-989"><span class="linenos"> 989</span></a> -</span><span id="L-990"><a href="#L-990"><span class="linenos"> 990</span></a> <span class="k">if</span> <span class="n">token_type</span><span class="p">:</span> -</span><span id="L-991"><a href="#L-991"><span class="linenos"> 991</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">NUMBER</span><span class="p">,</span> <span class="n">number_text</span><span class="p">)</span> -</span><span id="L-992"><a href="#L-992"><span class="linenos"> 992</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">DCOLON</span><span class="p">,</span> <span class="s2">"::"</span><span class="p">)</span> -</span><span id="L-993"><a href="#L-993"><span class="linenos"> 993</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">token_type</span><span class="p">,</span> <span class="n">literal</span><span class="p">)</span> <span class="c1"># type: ignore</span> -</span><span id="L-994"><a href="#L-994"><span class="linenos"> 994</span></a> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">IDENTIFIER_CAN_START_WITH_DIGIT</span><span class="p">:</span> -</span><span id="L-995"><a href="#L-995"><span class="linenos"> 995</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">VAR</span><span class="p">)</span> -</span><span id="L-996"><a href="#L-996"><span class="linenos"> 996</span></a> -</span><span id="L-997"><a href="#L-997"><span class="linenos"> 997</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">NUMBER</span><span class="p">,</span> <span class="n">number_text</span><span class="p">)</span> -</span><span id="L-998"><a href="#L-998"><span class="linenos"> 998</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="o">-</span><span class="nb">len</span><span class="p">(</span><span class="n">literal</span><span class="p">))</span> -</span><span id="L-999"><a href="#L-999"><span class="linenos"> 999</span></a> <span class="k">else</span><span class="p">:</span> -</span><span id="L-1000"><a href="#L-1000"><span class="linenos">1000</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">NUMBER</span><span class="p">)</span> -</span><span id="L-1001"><a href="#L-1001"><span class="linenos">1001</span></a> -</span><span id="L-1002"><a href="#L-1002"><span class="linenos">1002</span></a> <span class="k">def</span> <span class="nf">_scan_bits</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> -</span><span id="L-1003"><a href="#L-1003"><span class="linenos">1003</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="L-1004"><a href="#L-1004"><span class="linenos">1004</span></a> <span class="n">value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extract_value</span><span class="p">()</span> -</span><span id="L-1005"><a href="#L-1005"><span class="linenos">1005</span></a> <span class="k">try</span><span class="p">:</span> -</span><span id="L-1006"><a href="#L-1006"><span class="linenos">1006</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">BIT_STRING</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="nb">int</span><span class="p">(</span><span class="n">value</span><span class="p">,</span><span class="w"> </span><span class="mi">2</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> -</span><span id="L-1007"><a href="#L-1007"><span class="linenos">1007</span></a> <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span> -</span><span id="L-1008"><a href="#L-1008"><span class="linenos">1008</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">IDENTIFIER</span><span class="p">)</span> -</span><span id="L-1009"><a href="#L-1009"><span class="linenos">1009</span></a> -</span><span id="L-1010"><a href="#L-1010"><span class="linenos">1010</span></a> <span class="k">def</span> <span class="nf">_scan_hex</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> -</span><span id="L-1011"><a href="#L-1011"><span class="linenos">1011</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="L-1012"><a href="#L-1012"><span class="linenos">1012</span></a> <span class="n">value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extract_value</span><span class="p">()</span> -</span><span id="L-1013"><a href="#L-1013"><span class="linenos">1013</span></a> <span class="k">try</span><span class="p">:</span> -</span><span id="L-1014"><a href="#L-1014"><span class="linenos">1014</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">HEX_STRING</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="nb">int</span><span class="p">(</span><span class="n">value</span><span class="p">,</span><span class="w"> </span><span class="mi">16</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> -</span><span id="L-1015"><a href="#L-1015"><span class="linenos">1015</span></a> <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span> -</span><span id="L-1016"><a href="#L-1016"><span class="linenos">1016</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">IDENTIFIER</span><span class="p">)</span> -</span><span id="L-1017"><a href="#L-1017"><span class="linenos">1017</span></a> -</span><span id="L-1018"><a href="#L-1018"><span class="linenos">1018</span></a> <span class="k">def</span> <span class="nf">_extract_value</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> -</span><span id="L-1019"><a href="#L-1019"><span class="linenos">1019</span></a> <span class="k">while</span> <span class="kc">True</span><span class="p">:</span> -</span><span id="L-1020"><a href="#L-1020"><span class="linenos">1020</span></a> <span class="n">char</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="c1"># type: ignore</span> -</span><span id="L-1021"><a href="#L-1021"><span class="linenos">1021</span></a> <span class="k">if</span> <span class="n">char</span> <span class="ow">and</span> <span class="n">char</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">SINGLE_TOKENS</span><span class="p">:</span> -</span><span id="L-1022"><a href="#L-1022"><span class="linenos">1022</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="L-1023"><a href="#L-1023"><span class="linenos">1023</span></a> <span class="k">else</span><span class="p">:</span> -</span><span id="L-1024"><a href="#L-1024"><span class="linenos">1024</span></a> <span class="k">break</span> -</span><span id="L-1025"><a href="#L-1025"><span class="linenos">1025</span></a> -</span><span id="L-1026"><a href="#L-1026"><span class="linenos">1026</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_text</span> -</span><span id="L-1027"><a href="#L-1027"><span class="linenos">1027</span></a> -</span><span id="L-1028"><a href="#L-1028"><span class="linenos">1028</span></a> <span class="k">def</span> <span class="nf">_scan_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">quote</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> -</span><span id="L-1029"><a href="#L-1029"><span class="linenos">1029</span></a> <span class="n">quote_end</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_QUOTES</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">quote</span><span class="p">)</span> <span class="c1"># type: ignore</span> -</span><span id="L-1030"><a href="#L-1030"><span class="linenos">1030</span></a> <span class="k">if</span> <span class="n">quote_end</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> -</span><span id="L-1031"><a href="#L-1031"><span class="linenos">1031</span></a> <span class="k">return</span> <span class="kc">False</span> -</span><span id="L-1032"><a href="#L-1032"><span class="linenos">1032</span></a> -</span><span id="L-1033"><a href="#L-1033"><span class="linenos">1033</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">quote</span><span class="p">))</span> -</span><span id="L-1034"><a href="#L-1034"><span class="linenos">1034</span></a> <span class="n">text</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extract_string</span><span class="p">(</span><span class="n">quote_end</span><span class="p">)</span> -</span><span id="L-1035"><a href="#L-1035"><span class="linenos">1035</span></a> <span class="n">text</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ENCODE</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ENCODE</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">ENCODE</span> <span class="k">else</span> <span class="n">text</span> <span class="c1"># type: ignore</span> -</span><span id="L-1036"><a href="#L-1036"><span class="linenos">1036</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">NATIONAL</span> <span class="k">if</span> <span class="n">quote</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"N"</span> <span class="k">else</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">STRING</span><span class="p">,</span> <span class="n">text</span><span class="p">)</span> -</span><span id="L-1037"><a href="#L-1037"><span class="linenos">1037</span></a> <span class="k">return</span> <span class="kc">True</span> -</span><span id="L-1038"><a href="#L-1038"><span class="linenos">1038</span></a> -</span><span id="L-1039"><a href="#L-1039"><span class="linenos">1039</span></a> <span class="c1"># X'1234, b'0110', E'\\\\\' etc.</span> -</span><span id="L-1040"><a href="#L-1040"><span class="linenos">1040</span></a> <span class="k">def</span> <span class="nf">_scan_formatted_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">string_start</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> -</span><span id="L-1041"><a href="#L-1041"><span class="linenos">1041</span></a> <span class="k">if</span> <span class="n">string_start</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_HEX_STRINGS</span><span class="p">:</span> <span class="c1"># type: ignore</span> -</span><span id="L-1042"><a href="#L-1042"><span class="linenos">1042</span></a> <span class="n">delimiters</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_HEX_STRINGS</span> <span class="c1"># type: ignore</span> -</span><span id="L-1043"><a href="#L-1043"><span class="linenos">1043</span></a> <span class="n">token_type</span> <span class="o">=</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">HEX_STRING</span> -</span><span id="L-1044"><a href="#L-1044"><span class="linenos">1044</span></a> <span class="n">base</span> <span class="o">=</span> <span class="mi">16</span> -</span><span id="L-1045"><a href="#L-1045"><span class="linenos">1045</span></a> <span class="k">elif</span> <span class="n">string_start</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_BIT_STRINGS</span><span class="p">:</span> <span class="c1"># type: ignore</span> -</span><span id="L-1046"><a href="#L-1046"><span class="linenos">1046</span></a> <span class="n">delimiters</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_BIT_STRINGS</span> <span class="c1"># type: ignore</span> -</span><span id="L-1047"><a href="#L-1047"><span class="linenos">1047</span></a> <span class="n">token_type</span> <span class="o">=</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">BIT_STRING</span> -</span><span id="L-1048"><a href="#L-1048"><span class="linenos">1048</span></a> <span class="n">base</span> <span class="o">=</span> <span class="mi">2</span> -</span><span id="L-1049"><a href="#L-1049"><span class="linenos">1049</span></a> <span class="k">elif</span> <span class="n">string_start</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_BYTE_STRINGS</span><span class="p">:</span> <span class="c1"># type: ignore</span> -</span><span id="L-1050"><a href="#L-1050"><span class="linenos">1050</span></a> <span class="n">delimiters</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_BYTE_STRINGS</span> <span class="c1"># type: ignore</span> -</span><span id="L-1051"><a href="#L-1051"><span class="linenos">1051</span></a> <span class="n">token_type</span> <span class="o">=</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">BYTE_STRING</span> -</span><span id="L-1052"><a href="#L-1052"><span class="linenos">1052</span></a> <span class="n">base</span> <span class="o">=</span> <span class="kc">None</span> -</span><span id="L-1053"><a href="#L-1053"><span class="linenos">1053</span></a> <span class="k">else</span><span class="p">:</span> -</span><span id="L-1054"><a href="#L-1054"><span class="linenos">1054</span></a> <span class="k">return</span> <span class="kc">False</span> -</span><span id="L-1055"><a href="#L-1055"><span class="linenos">1055</span></a> -</span><span id="L-1056"><a href="#L-1056"><span class="linenos">1056</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">string_start</span><span class="p">))</span> -</span><span id="L-1057"><a href="#L-1057"><span class="linenos">1057</span></a> <span class="n">string_end</span> <span class="o">=</span> <span class="n">delimiters</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">string_start</span><span class="p">)</span> -</span><span id="L-1058"><a href="#L-1058"><span class="linenos">1058</span></a> <span class="n">text</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extract_string</span><span class="p">(</span><span class="n">string_end</span><span class="p">)</span> -</span><span id="L-1059"><a href="#L-1059"><span class="linenos">1059</span></a> -</span><span id="L-1060"><a href="#L-1060"><span class="linenos">1060</span></a> <span class="k">if</span> <span class="n">base</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> -</span><span id="L-1061"><a href="#L-1061"><span class="linenos">1061</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">token_type</span><span class="p">,</span> <span class="n">text</span><span class="p">)</span> -</span><span id="L-1062"><a href="#L-1062"><span class="linenos">1062</span></a> <span class="k">else</span><span class="p">:</span> -</span><span id="L-1063"><a href="#L-1063"><span class="linenos">1063</span></a> <span class="k">try</span><span class="p">:</span> -</span><span id="L-1064"><a href="#L-1064"><span class="linenos">1064</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">token_type</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="nb">int</span><span class="p">(</span><span class="n">text</span><span class="p">,</span><span class="w"> </span><span class="n">base</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> -</span><span id="L-1065"><a href="#L-1065"><span class="linenos">1065</span></a> <span class="k">except</span><span class="p">:</span> -</span><span id="L-1066"><a href="#L-1066"><span class="linenos">1066</span></a> <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span> -</span><span id="L-1067"><a href="#L-1067"><span class="linenos">1067</span></a> <span class="sa">f</span><span class="s2">"Numeric string contains invalid characters from </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_line</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_start</span><span class="si">}</span><span class="s2">"</span> -</span><span id="L-1068"><a href="#L-1068"><span class="linenos">1068</span></a> <span class="p">)</span> -</span><span id="L-1069"><a href="#L-1069"><span class="linenos">1069</span></a> -</span><span id="L-1070"><a href="#L-1070"><span class="linenos">1070</span></a> <span class="k">return</span> <span class="kc">True</span> -</span><span id="L-1071"><a href="#L-1071"><span class="linenos">1071</span></a> -</span><span id="L-1072"><a href="#L-1072"><span class="linenos">1072</span></a> <span class="k">def</span> <span class="nf">_scan_identifier</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">identifier_end</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> -</span><span id="L-1073"><a href="#L-1073"><span class="linenos">1073</span></a> <span class="n">text</span> <span class="o">=</span> <span class="s2">""</span> -</span><span id="L-1074"><a href="#L-1074"><span class="linenos">1074</span></a> <span class="n">identifier_end_is_escape</span> <span class="o">=</span> <span class="n">identifier_end</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_IDENTIFIER_ESCAPES</span> -</span><span id="L-1075"><a href="#L-1075"><span class="linenos">1075</span></a> -</span><span id="L-1076"><a href="#L-1076"><span class="linenos">1076</span></a> <span class="k">while</span> <span class="kc">True</span><span class="p">:</span> -</span><span id="L-1077"><a href="#L-1077"><span class="linenos">1077</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_end</span><span class="p">:</span> -</span><span id="L-1078"><a href="#L-1078"><span class="linenos">1078</span></a> <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Missing </span><span class="si">{</span><span class="n">identifier_end</span><span class="si">}</span><span class="s2"> from </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_line</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_start</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> -</span><span id="L-1079"><a href="#L-1079"><span class="linenos">1079</span></a> -</span><span id="L-1080"><a href="#L-1080"><span class="linenos">1080</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="L-1081"><a href="#L-1081"><span class="linenos">1081</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="o">==</span> <span class="n">identifier_end</span><span class="p">:</span> -</span><span id="L-1082"><a href="#L-1082"><span class="linenos">1082</span></a> <span class="k">if</span> <span class="n">identifier_end_is_escape</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="o">==</span> <span class="n">identifier_end</span><span class="p">:</span> -</span><span id="L-1083"><a href="#L-1083"><span class="linenos">1083</span></a> <span class="n">text</span> <span class="o">+=</span> <span class="n">identifier_end</span> <span class="c1"># type: ignore</span> -</span><span id="L-1084"><a href="#L-1084"><span class="linenos">1084</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="L-1085"><a href="#L-1085"><span class="linenos">1085</span></a> <span class="k">continue</span> -</span><span id="L-1086"><a href="#L-1086"><span class="linenos">1086</span></a> -</span><span id="L-1087"><a href="#L-1087"><span class="linenos">1087</span></a> <span class="k">break</span> -</span><span id="L-1088"><a href="#L-1088"><span class="linenos">1088</span></a> -</span><span id="L-1089"><a href="#L-1089"><span class="linenos">1089</span></a> <span class="n">text</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="c1"># type: ignore</span> -</span><span id="L-1090"><a href="#L-1090"><span class="linenos">1090</span></a> -</span><span id="L-1091"><a href="#L-1091"><span class="linenos">1091</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">IDENTIFIER</span><span class="p">,</span> <span class="n">text</span><span class="p">)</span> -</span><span id="L-1092"><a href="#L-1092"><span class="linenos">1092</span></a> -</span><span id="L-1093"><a href="#L-1093"><span class="linenos">1093</span></a> <span class="k">def</span> <span class="nf">_scan_var</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> -</span><span id="L-1094"><a href="#L-1094"><span class="linenos">1094</span></a> <span class="k">while</span> <span class="kc">True</span><span class="p">:</span> -</span><span id="L-1095"><a href="#L-1095"><span class="linenos">1095</span></a> <span class="n">char</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="c1"># type: ignore</span> -</span><span id="L-1096"><a href="#L-1096"><span class="linenos">1096</span></a> <span class="k">if</span> <span class="n">char</span> <span class="ow">and</span> <span class="n">char</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">SINGLE_TOKENS</span><span class="p">:</span> -</span><span id="L-1097"><a href="#L-1097"><span class="linenos">1097</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="L-1098"><a href="#L-1098"><span class="linenos">1098</span></a> <span class="k">else</span><span class="p">:</span> -</span><span id="L-1099"><a href="#L-1099"><span class="linenos">1099</span></a> <span class="k">break</span> -</span><span id="L-1100"><a href="#L-1100"><span class="linenos">1100</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span> -</span><span id="L-1101"><a href="#L-1101"><span class="linenos">1101</span></a> <span class="n">TokenType</span><span class="o">.</span><span class="n">VAR</span> -</span><span id="L-1102"><a href="#L-1102"><span class="linenos">1102</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prev_token_type</span> <span class="o">==</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">PARAMETER</span> -</span><span id="L-1103"><a href="#L-1103"><span class="linenos">1103</span></a> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">KEYWORDS</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_text</span><span class="o">.</span><span class="n">upper</span><span class="p">(),</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">VAR</span><span class="p">)</span> -</span><span id="L-1104"><a href="#L-1104"><span class="linenos">1104</span></a> <span class="p">)</span> -</span><span id="L-1105"><a href="#L-1105"><span class="linenos">1105</span></a> -</span><span id="L-1106"><a href="#L-1106"><span class="linenos">1106</span></a> <span class="k">def</span> <span class="nf">_extract_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">delimiter</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> -</span><span id="L-1107"><a href="#L-1107"><span class="linenos">1107</span></a> <span class="n">text</span> <span class="o">=</span> <span class="s2">""</span> -</span><span id="L-1108"><a href="#L-1108"><span class="linenos">1108</span></a> <span class="n">delim_size</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">delimiter</span><span class="p">)</span> -</span><span id="L-1109"><a href="#L-1109"><span class="linenos">1109</span></a> -</span><span id="L-1110"><a href="#L-1110"><span class="linenos">1110</span></a> <span class="k">while</span> <span class="kc">True</span><span class="p">:</span> -</span><span id="L-1111"><a href="#L-1111"><span class="linenos">1111</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_STRING_ESCAPES</span> <span class="ow">and</span> <span class="p">(</span> -</span><span id="L-1112"><a href="#L-1112"><span class="linenos">1112</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="o">==</span> <span class="n">delimiter</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_STRING_ESCAPES</span> -</span><span id="L-1113"><a href="#L-1113"><span class="linenos">1113</span></a> <span class="p">):</span> -</span><span id="L-1114"><a href="#L-1114"><span class="linenos">1114</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="o">==</span> <span class="n">delimiter</span><span class="p">:</span> -</span><span id="L-1115"><a href="#L-1115"><span class="linenos">1115</span></a> <span class="n">text</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="c1"># type: ignore</span> -</span><span id="L-1116"><a href="#L-1116"><span class="linenos">1116</span></a> <span class="k">else</span><span class="p">:</span> -</span><span id="L-1117"><a href="#L-1117"><span class="linenos">1117</span></a> <span class="n">text</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="c1"># type: ignore</span> -</span><span id="L-1118"><a href="#L-1118"><span class="linenos">1118</span></a> -</span><span id="L-1119"><a href="#L-1119"><span class="linenos">1119</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_current</span> <span class="o">+</span> <span class="mi">1</span> <span class="o"><</span> <span class="bp">self</span><span class="o">.</span><span class="n">size</span><span class="p">:</span> -</span><span id="L-1120"><a href="#L-1120"><span class="linenos">1120</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span> -</span><span id="L-1121"><a href="#L-1121"><span class="linenos">1121</span></a> <span class="k">else</span><span class="p">:</span> -</span><span id="L-1122"><a href="#L-1122"><span class="linenos">1122</span></a> <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Missing </span><span class="si">{</span><span class="n">delimiter</span><span class="si">}</span><span class="s2"> from </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_line</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_current</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> -</span><span id="L-1123"><a href="#L-1123"><span class="linenos">1123</span></a> <span class="k">else</span><span class="p">:</span> -</span><span id="L-1124"><a href="#L-1124"><span class="linenos">1124</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_chars</span><span class="p">(</span><span class="n">delim_size</span><span class="p">)</span> <span class="o">==</span> <span class="n">delimiter</span><span class="p">:</span> -</span><span id="L-1125"><a href="#L-1125"><span class="linenos">1125</span></a> <span class="k">if</span> <span class="n">delim_size</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span> -</span><span id="L-1126"><a href="#L-1126"><span class="linenos">1126</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="n">delim_size</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> -</span><span id="L-1127"><a href="#L-1127"><span class="linenos">1127</span></a> <span class="k">break</span> -</span><span id="L-1128"><a href="#L-1128"><span class="linenos">1128</span></a> -</span><span id="L-1129"><a href="#L-1129"><span class="linenos">1129</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_end</span><span class="p">:</span> -</span><span id="L-1130"><a href="#L-1130"><span class="linenos">1130</span></a> <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Missing </span><span class="si">{</span><span class="n">delimiter</span><span class="si">}</span><span class="s2"> from </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_line</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_start</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> -</span><span id="L-1131"><a href="#L-1131"><span class="linenos">1131</span></a> <span class="n">text</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="c1"># type: ignore</span> -</span><span id="L-1132"><a href="#L-1132"><span class="linenos">1132</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="L-1133"><a href="#L-1133"><span class="linenos">1133</span></a> -</span><span id="L-1134"><a href="#L-1134"><span class="linenos">1134</span></a> <span class="k">return</span> <span class="n">text</span> +</span><span id="L-924"><a href="#L-924"><span class="linenos"> 924</span></a> <span class="n">word</span> <span class="o">=</span> <span class="n">word</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span> +</span><span id="L-925"><a href="#L-925"><span class="linenos"> 925</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">KEYWORDS</span><span class="p">[</span><span class="n">word</span><span class="p">],</span> <span class="n">text</span><span class="o">=</span><span class="n">word</span><span class="p">)</span> +</span><span id="L-926"><a href="#L-926"><span class="linenos"> 926</span></a> +</span><span id="L-927"><a href="#L-927"><span class="linenos"> 927</span></a> <span class="k">def</span> <span class="nf">_scan_comment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">comment_start</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> +</span><span id="L-928"><a href="#L-928"><span class="linenos"> 928</span></a> <span class="k">if</span> <span class="n">comment_start</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_COMMENTS</span><span class="p">:</span> <span class="c1"># type: ignore</span> +</span><span id="L-929"><a href="#L-929"><span class="linenos"> 929</span></a> <span class="k">return</span> <span class="kc">False</span> +</span><span id="L-930"><a href="#L-930"><span class="linenos"> 930</span></a> +</span><span id="L-931"><a href="#L-931"><span class="linenos"> 931</span></a> <span class="n">comment_start_line</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_line</span> +</span><span id="L-932"><a href="#L-932"><span class="linenos"> 932</span></a> <span class="n">comment_start_size</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">comment_start</span><span class="p">)</span> +</span><span id="L-933"><a href="#L-933"><span class="linenos"> 933</span></a> <span class="n">comment_end</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_COMMENTS</span><span class="p">[</span><span class="n">comment_start</span><span class="p">]</span> <span class="c1"># type: ignore</span> +</span><span id="L-934"><a href="#L-934"><span class="linenos"> 934</span></a> +</span><span id="L-935"><a href="#L-935"><span class="linenos"> 935</span></a> <span class="k">if</span> <span class="n">comment_end</span><span class="p">:</span> +</span><span id="L-936"><a href="#L-936"><span class="linenos"> 936</span></a> <span class="n">comment_end_size</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">comment_end</span><span class="p">)</span> +</span><span id="L-937"><a href="#L-937"><span class="linenos"> 937</span></a> +</span><span id="L-938"><a href="#L-938"><span class="linenos"> 938</span></a> <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_end</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_chars</span><span class="p">(</span><span class="n">comment_end_size</span><span class="p">)</span> <span class="o">!=</span> <span class="n">comment_end</span><span class="p">:</span> +</span><span id="L-939"><a href="#L-939"><span class="linenos"> 939</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="L-940"><a href="#L-940"><span class="linenos"> 940</span></a> +</span><span id="L-941"><a href="#L-941"><span class="linenos"> 941</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_comments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_text</span><span class="p">[</span><span class="n">comment_start_size</span> <span class="p">:</span> <span class="o">-</span><span class="n">comment_end_size</span> <span class="o">+</span> <span class="mi">1</span><span class="p">])</span> <span class="c1"># type: ignore</span> +</span><span id="L-942"><a href="#L-942"><span class="linenos"> 942</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="n">comment_end_size</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> +</span><span id="L-943"><a href="#L-943"><span class="linenos"> 943</span></a> <span class="k">else</span><span class="p">:</span> +</span><span id="L-944"><a href="#L-944"><span class="linenos"> 944</span></a> <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_end</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">WHITE_SPACE</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="p">)</span> <span class="ow">is</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">BREAK</span><span class="p">:</span> +</span><span id="L-945"><a href="#L-945"><span class="linenos"> 945</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="L-946"><a href="#L-946"><span class="linenos"> 946</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_comments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_text</span><span class="p">[</span><span class="n">comment_start_size</span><span class="p">:])</span> <span class="c1"># type: ignore</span> +</span><span id="L-947"><a href="#L-947"><span class="linenos"> 947</span></a> +</span><span id="L-948"><a href="#L-948"><span class="linenos"> 948</span></a> <span class="c1"># Leading comment is attached to the succeeding token, whilst trailing comment to the preceding.</span> +</span><span id="L-949"><a href="#L-949"><span class="linenos"> 949</span></a> <span class="c1"># Multiple consecutive comments are preserved by appending them to the current comments list.</span> +</span><span id="L-950"><a href="#L-950"><span class="linenos"> 950</span></a> <span class="k">if</span> <span class="n">comment_start_line</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prev_token_line</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">_end</span><span class="p">:</span> +</span><span id="L-951"><a href="#L-951"><span class="linenos"> 951</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">tokens</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">comments</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_comments</span><span class="p">)</span> +</span><span id="L-952"><a href="#L-952"><span class="linenos"> 952</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_comments</span> <span class="o">=</span> <span class="p">[]</span> +</span><span id="L-953"><a href="#L-953"><span class="linenos"> 953</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_prev_token_line</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_line</span> +</span><span id="L-954"><a href="#L-954"><span class="linenos"> 954</span></a> +</span><span id="L-955"><a href="#L-955"><span class="linenos"> 955</span></a> <span class="k">return</span> <span class="kc">True</span> +</span><span id="L-956"><a href="#L-956"><span class="linenos"> 956</span></a> +</span><span id="L-957"><a href="#L-957"><span class="linenos"> 957</span></a> <span class="k">def</span> <span class="nf">_scan_number</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> +</span><span id="L-958"><a href="#L-958"><span class="linenos"> 958</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="o">==</span> <span class="s2">"0"</span><span class="p">:</span> +</span><span id="L-959"><a href="#L-959"><span class="linenos"> 959</span></a> <span class="n">peek</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span> <span class="c1"># type: ignore</span> +</span><span id="L-960"><a href="#L-960"><span class="linenos"> 960</span></a> <span class="k">if</span> <span class="n">peek</span> <span class="o">==</span> <span class="s2">"B"</span><span class="p">:</span> +</span><span id="L-961"><a href="#L-961"><span class="linenos"> 961</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_scan_bits</span><span class="p">()</span> +</span><span id="L-962"><a href="#L-962"><span class="linenos"> 962</span></a> <span class="k">elif</span> <span class="n">peek</span> <span class="o">==</span> <span class="s2">"X"</span><span class="p">:</span> +</span><span id="L-963"><a href="#L-963"><span class="linenos"> 963</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_scan_hex</span><span class="p">()</span> +</span><span id="L-964"><a href="#L-964"><span class="linenos"> 964</span></a> +</span><span id="L-965"><a href="#L-965"><span class="linenos"> 965</span></a> <span class="n">decimal</span> <span class="o">=</span> <span class="kc">False</span> +</span><span id="L-966"><a href="#L-966"><span class="linenos"> 966</span></a> <span class="n">scientific</span> <span class="o">=</span> <span class="mi">0</span> +</span><span id="L-967"><a href="#L-967"><span class="linenos"> 967</span></a> +</span><span id="L-968"><a href="#L-968"><span class="linenos"> 968</span></a> <span class="k">while</span> <span class="kc">True</span><span class="p">:</span> +</span><span id="L-969"><a href="#L-969"><span class="linenos"> 969</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">isdigit</span><span class="p">():</span> <span class="c1"># type: ignore</span> +</span><span id="L-970"><a href="#L-970"><span class="linenos"> 970</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="L-971"><a href="#L-971"><span class="linenos"> 971</span></a> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="o">==</span> <span class="s2">"."</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">decimal</span><span class="p">:</span> +</span><span id="L-972"><a href="#L-972"><span class="linenos"> 972</span></a> <span class="n">decimal</span> <span class="o">=</span> <span class="kc">True</span> +</span><span id="L-973"><a href="#L-973"><span class="linenos"> 973</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="L-974"><a href="#L-974"><span class="linenos"> 974</span></a> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"-"</span><span class="p">,</span> <span class="s2">"+"</span><span class="p">)</span> <span class="ow">and</span> <span class="n">scientific</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span> +</span><span id="L-975"><a href="#L-975"><span class="linenos"> 975</span></a> <span class="n">scientific</span> <span class="o">+=</span> <span class="mi">1</span> +</span><span id="L-976"><a href="#L-976"><span class="linenos"> 976</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="L-977"><a href="#L-977"><span class="linenos"> 977</span></a> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"E"</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">scientific</span><span class="p">:</span> <span class="c1"># type: ignore</span> +</span><span id="L-978"><a href="#L-978"><span class="linenos"> 978</span></a> <span class="n">scientific</span> <span class="o">+=</span> <span class="mi">1</span> +</span><span id="L-979"><a href="#L-979"><span class="linenos"> 979</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="L-980"><a href="#L-980"><span class="linenos"> 980</span></a> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">isidentifier</span><span class="p">():</span> <span class="c1"># type: ignore</span> +</span><span id="L-981"><a href="#L-981"><span class="linenos"> 981</span></a> <span class="n">number_text</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_text</span> +</span><span id="L-982"><a href="#L-982"><span class="linenos"> 982</span></a> <span class="n">literal</span> <span class="o">=</span> <span class="p">[]</span> +</span><span id="L-983"><a href="#L-983"><span class="linenos"> 983</span></a> +</span><span id="L-984"><a href="#L-984"><span class="linenos"> 984</span></a> <span class="k">while</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">SINGLE_TOKENS</span><span class="p">:</span> <span class="c1"># type: ignore</span> +</span><span id="L-985"><a href="#L-985"><span class="linenos"> 985</span></a> <span class="n">literal</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">upper</span><span class="p">())</span> <span class="c1"># type: ignore</span> +</span><span id="L-986"><a href="#L-986"><span class="linenos"> 986</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="L-987"><a href="#L-987"><span class="linenos"> 987</span></a> +</span><span id="L-988"><a href="#L-988"><span class="linenos"> 988</span></a> <span class="n">literal</span> <span class="o">=</span> <span class="s2">""</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">literal</span><span class="p">)</span> <span class="c1"># type: ignore</span> +</span><span id="L-989"><a href="#L-989"><span class="linenos"> 989</span></a> <span class="n">token_type</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">KEYWORDS</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">NUMERIC_LITERALS</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">literal</span><span class="p">))</span> <span class="c1"># type: ignore</span> +</span><span id="L-990"><a href="#L-990"><span class="linenos"> 990</span></a> +</span><span id="L-991"><a href="#L-991"><span class="linenos"> 991</span></a> <span class="k">if</span> <span class="n">token_type</span><span class="p">:</span> +</span><span id="L-992"><a href="#L-992"><span class="linenos"> 992</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">NUMBER</span><span class="p">,</span> <span class="n">number_text</span><span class="p">)</span> +</span><span id="L-993"><a href="#L-993"><span class="linenos"> 993</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">DCOLON</span><span class="p">,</span> <span class="s2">"::"</span><span class="p">)</span> +</span><span id="L-994"><a href="#L-994"><span class="linenos"> 994</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">token_type</span><span class="p">,</span> <span class="n">literal</span><span class="p">)</span> <span class="c1"># type: ignore</span> +</span><span id="L-995"><a href="#L-995"><span class="linenos"> 995</span></a> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">IDENTIFIER_CAN_START_WITH_DIGIT</span><span class="p">:</span> +</span><span id="L-996"><a href="#L-996"><span class="linenos"> 996</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">VAR</span><span class="p">)</span> +</span><span id="L-997"><a href="#L-997"><span class="linenos"> 997</span></a> +</span><span id="L-998"><a href="#L-998"><span class="linenos"> 998</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">NUMBER</span><span class="p">,</span> <span class="n">number_text</span><span class="p">)</span> +</span><span id="L-999"><a href="#L-999"><span class="linenos"> 999</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="o">-</span><span class="nb">len</span><span class="p">(</span><span class="n">literal</span><span class="p">))</span> +</span><span id="L-1000"><a href="#L-1000"><span class="linenos">1000</span></a> <span class="k">else</span><span class="p">:</span> +</span><span id="L-1001"><a href="#L-1001"><span class="linenos">1001</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">NUMBER</span><span class="p">)</span> +</span><span id="L-1002"><a href="#L-1002"><span class="linenos">1002</span></a> +</span><span id="L-1003"><a href="#L-1003"><span class="linenos">1003</span></a> <span class="k">def</span> <span class="nf">_scan_bits</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> +</span><span id="L-1004"><a href="#L-1004"><span class="linenos">1004</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="L-1005"><a href="#L-1005"><span class="linenos">1005</span></a> <span class="n">value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extract_value</span><span class="p">()</span> +</span><span id="L-1006"><a href="#L-1006"><span class="linenos">1006</span></a> <span class="k">try</span><span class="p">:</span> +</span><span id="L-1007"><a href="#L-1007"><span class="linenos">1007</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">BIT_STRING</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="nb">int</span><span class="p">(</span><span class="n">value</span><span class="p">,</span><span class="w"> </span><span class="mi">2</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> +</span><span id="L-1008"><a href="#L-1008"><span class="linenos">1008</span></a> <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span> +</span><span id="L-1009"><a href="#L-1009"><span class="linenos">1009</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">IDENTIFIER</span><span class="p">)</span> +</span><span id="L-1010"><a href="#L-1010"><span class="linenos">1010</span></a> +</span><span id="L-1011"><a href="#L-1011"><span class="linenos">1011</span></a> <span class="k">def</span> <span class="nf">_scan_hex</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> +</span><span id="L-1012"><a href="#L-1012"><span class="linenos">1012</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="L-1013"><a href="#L-1013"><span class="linenos">1013</span></a> <span class="n">value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extract_value</span><span class="p">()</span> +</span><span id="L-1014"><a href="#L-1014"><span class="linenos">1014</span></a> <span class="k">try</span><span class="p">:</span> +</span><span id="L-1015"><a href="#L-1015"><span class="linenos">1015</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">HEX_STRING</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="nb">int</span><span class="p">(</span><span class="n">value</span><span class="p">,</span><span class="w"> </span><span class="mi">16</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> +</span><span id="L-1016"><a href="#L-1016"><span class="linenos">1016</span></a> <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span> +</span><span id="L-1017"><a href="#L-1017"><span class="linenos">1017</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">IDENTIFIER</span><span class="p">)</span> +</span><span id="L-1018"><a href="#L-1018"><span class="linenos">1018</span></a> +</span><span id="L-1019"><a href="#L-1019"><span class="linenos">1019</span></a> <span class="k">def</span> <span class="nf">_extract_value</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> +</span><span id="L-1020"><a href="#L-1020"><span class="linenos">1020</span></a> <span class="k">while</span> <span class="kc">True</span><span class="p">:</span> +</span><span id="L-1021"><a href="#L-1021"><span class="linenos">1021</span></a> <span class="n">char</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="c1"># type: ignore</span> +</span><span id="L-1022"><a href="#L-1022"><span class="linenos">1022</span></a> <span class="k">if</span> <span class="n">char</span> <span class="ow">and</span> <span class="n">char</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">SINGLE_TOKENS</span><span class="p">:</span> +</span><span id="L-1023"><a href="#L-1023"><span class="linenos">1023</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="L-1024"><a href="#L-1024"><span class="linenos">1024</span></a> <span class="k">else</span><span class="p">:</span> +</span><span id="L-1025"><a href="#L-1025"><span class="linenos">1025</span></a> <span class="k">break</span> +</span><span id="L-1026"><a href="#L-1026"><span class="linenos">1026</span></a> +</span><span id="L-1027"><a href="#L-1027"><span class="linenos">1027</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_text</span> +</span><span id="L-1028"><a href="#L-1028"><span class="linenos">1028</span></a> +</span><span id="L-1029"><a href="#L-1029"><span class="linenos">1029</span></a> <span class="k">def</span> <span class="nf">_scan_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">quote</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> +</span><span id="L-1030"><a href="#L-1030"><span class="linenos">1030</span></a> <span class="n">quote_end</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_QUOTES</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">quote</span><span class="p">)</span> <span class="c1"># type: ignore</span> +</span><span id="L-1031"><a href="#L-1031"><span class="linenos">1031</span></a> <span class="k">if</span> <span class="n">quote_end</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> +</span><span id="L-1032"><a href="#L-1032"><span class="linenos">1032</span></a> <span class="k">return</span> <span class="kc">False</span> +</span><span id="L-1033"><a href="#L-1033"><span class="linenos">1033</span></a> +</span><span id="L-1034"><a href="#L-1034"><span class="linenos">1034</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">quote</span><span class="p">))</span> +</span><span id="L-1035"><a href="#L-1035"><span class="linenos">1035</span></a> <span class="n">text</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extract_string</span><span class="p">(</span><span class="n">quote_end</span><span class="p">)</span> +</span><span id="L-1036"><a href="#L-1036"><span class="linenos">1036</span></a> <span class="n">text</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ENCODE</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ENCODE</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">ENCODE</span> <span class="k">else</span> <span class="n">text</span> <span class="c1"># type: ignore</span> +</span><span id="L-1037"><a href="#L-1037"><span class="linenos">1037</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">NATIONAL</span> <span class="k">if</span> <span class="n">quote</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"N"</span> <span class="k">else</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">STRING</span><span class="p">,</span> <span class="n">text</span><span class="p">)</span> +</span><span id="L-1038"><a href="#L-1038"><span class="linenos">1038</span></a> <span class="k">return</span> <span class="kc">True</span> +</span><span id="L-1039"><a href="#L-1039"><span class="linenos">1039</span></a> +</span><span id="L-1040"><a href="#L-1040"><span class="linenos">1040</span></a> <span class="c1"># X'1234, b'0110', E'\\\\\' etc.</span> +</span><span id="L-1041"><a href="#L-1041"><span class="linenos">1041</span></a> <span class="k">def</span> <span class="nf">_scan_formatted_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">string_start</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> +</span><span id="L-1042"><a href="#L-1042"><span class="linenos">1042</span></a> <span class="k">if</span> <span class="n">string_start</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_HEX_STRINGS</span><span class="p">:</span> <span class="c1"># type: ignore</span> +</span><span id="L-1043"><a href="#L-1043"><span class="linenos">1043</span></a> <span class="n">delimiters</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_HEX_STRINGS</span> <span class="c1"># type: ignore</span> +</span><span id="L-1044"><a href="#L-1044"><span class="linenos">1044</span></a> <span class="n">token_type</span> <span class="o">=</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">HEX_STRING</span> +</span><span id="L-1045"><a href="#L-1045"><span class="linenos">1045</span></a> <span class="n">base</span> <span class="o">=</span> <span class="mi">16</span> +</span><span id="L-1046"><a href="#L-1046"><span class="linenos">1046</span></a> <span class="k">elif</span> <span class="n">string_start</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_BIT_STRINGS</span><span class="p">:</span> <span class="c1"># type: ignore</span> +</span><span id="L-1047"><a href="#L-1047"><span class="linenos">1047</span></a> <span class="n">delimiters</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_BIT_STRINGS</span> <span class="c1"># type: ignore</span> +</span><span id="L-1048"><a href="#L-1048"><span class="linenos">1048</span></a> <span class="n">token_type</span> <span class="o">=</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">BIT_STRING</span> +</span><span id="L-1049"><a href="#L-1049"><span class="linenos">1049</span></a> <span class="n">base</span> <span class="o">=</span> <span class="mi">2</span> +</span><span id="L-1050"><a href="#L-1050"><span class="linenos">1050</span></a> <span class="k">elif</span> <span class="n">string_start</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_BYTE_STRINGS</span><span class="p">:</span> <span class="c1"># type: ignore</span> +</span><span id="L-1051"><a href="#L-1051"><span class="linenos">1051</span></a> <span class="n">delimiters</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_BYTE_STRINGS</span> <span class="c1"># type: ignore</span> +</span><span id="L-1052"><a href="#L-1052"><span class="linenos">1052</span></a> <span class="n">token_type</span> <span class="o">=</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">BYTE_STRING</span> +</span><span id="L-1053"><a href="#L-1053"><span class="linenos">1053</span></a> <span class="n">base</span> <span class="o">=</span> <span class="kc">None</span> +</span><span id="L-1054"><a href="#L-1054"><span class="linenos">1054</span></a> <span class="k">else</span><span class="p">:</span> +</span><span id="L-1055"><a href="#L-1055"><span class="linenos">1055</span></a> <span class="k">return</span> <span class="kc">False</span> +</span><span id="L-1056"><a href="#L-1056"><span class="linenos">1056</span></a> +</span><span id="L-1057"><a href="#L-1057"><span class="linenos">1057</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">string_start</span><span class="p">))</span> +</span><span id="L-1058"><a href="#L-1058"><span class="linenos">1058</span></a> <span class="n">string_end</span> <span class="o">=</span> <span class="n">delimiters</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">string_start</span><span class="p">)</span> +</span><span id="L-1059"><a href="#L-1059"><span class="linenos">1059</span></a> <span class="n">text</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extract_string</span><span class="p">(</span><span class="n">string_end</span><span class="p">)</span> +</span><span id="L-1060"><a href="#L-1060"><span class="linenos">1060</span></a> +</span><span id="L-1061"><a href="#L-1061"><span class="linenos">1061</span></a> <span class="k">if</span> <span class="n">base</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> +</span><span id="L-1062"><a href="#L-1062"><span class="linenos">1062</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">token_type</span><span class="p">,</span> <span class="n">text</span><span class="p">)</span> +</span><span id="L-1063"><a href="#L-1063"><span class="linenos">1063</span></a> <span class="k">else</span><span class="p">:</span> +</span><span id="L-1064"><a href="#L-1064"><span class="linenos">1064</span></a> <span class="k">try</span><span class="p">:</span> +</span><span id="L-1065"><a href="#L-1065"><span class="linenos">1065</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">token_type</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="nb">int</span><span class="p">(</span><span class="n">text</span><span class="p">,</span><span class="w"> </span><span class="n">base</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> +</span><span id="L-1066"><a href="#L-1066"><span class="linenos">1066</span></a> <span class="k">except</span><span class="p">:</span> +</span><span id="L-1067"><a href="#L-1067"><span class="linenos">1067</span></a> <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span> +</span><span id="L-1068"><a href="#L-1068"><span class="linenos">1068</span></a> <span class="sa">f</span><span class="s2">"Numeric string contains invalid characters from </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_line</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_start</span><span class="si">}</span><span class="s2">"</span> +</span><span id="L-1069"><a href="#L-1069"><span class="linenos">1069</span></a> <span class="p">)</span> +</span><span id="L-1070"><a href="#L-1070"><span class="linenos">1070</span></a> +</span><span id="L-1071"><a href="#L-1071"><span class="linenos">1071</span></a> <span class="k">return</span> <span class="kc">True</span> +</span><span id="L-1072"><a href="#L-1072"><span class="linenos">1072</span></a> +</span><span id="L-1073"><a href="#L-1073"><span class="linenos">1073</span></a> <span class="k">def</span> <span class="nf">_scan_identifier</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">identifier_end</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> +</span><span id="L-1074"><a href="#L-1074"><span class="linenos">1074</span></a> <span class="n">text</span> <span class="o">=</span> <span class="s2">""</span> +</span><span id="L-1075"><a href="#L-1075"><span class="linenos">1075</span></a> <span class="n">identifier_end_is_escape</span> <span class="o">=</span> <span class="n">identifier_end</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_IDENTIFIER_ESCAPES</span> +</span><span id="L-1076"><a href="#L-1076"><span class="linenos">1076</span></a> +</span><span id="L-1077"><a href="#L-1077"><span class="linenos">1077</span></a> <span class="k">while</span> <span class="kc">True</span><span class="p">:</span> +</span><span id="L-1078"><a href="#L-1078"><span class="linenos">1078</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_end</span><span class="p">:</span> +</span><span id="L-1079"><a href="#L-1079"><span class="linenos">1079</span></a> <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Missing </span><span class="si">{</span><span class="n">identifier_end</span><span class="si">}</span><span class="s2"> from </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_line</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_start</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> +</span><span id="L-1080"><a href="#L-1080"><span class="linenos">1080</span></a> +</span><span id="L-1081"><a href="#L-1081"><span class="linenos">1081</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="L-1082"><a href="#L-1082"><span class="linenos">1082</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="o">==</span> <span class="n">identifier_end</span><span class="p">:</span> +</span><span id="L-1083"><a href="#L-1083"><span class="linenos">1083</span></a> <span class="k">if</span> <span class="n">identifier_end_is_escape</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="o">==</span> <span class="n">identifier_end</span><span class="p">:</span> +</span><span id="L-1084"><a href="#L-1084"><span class="linenos">1084</span></a> <span class="n">text</span> <span class="o">+=</span> <span class="n">identifier_end</span> <span class="c1"># type: ignore</span> +</span><span id="L-1085"><a href="#L-1085"><span class="linenos">1085</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="L-1086"><a href="#L-1086"><span class="linenos">1086</span></a> <span class="k">continue</span> +</span><span id="L-1087"><a href="#L-1087"><span class="linenos">1087</span></a> +</span><span id="L-1088"><a href="#L-1088"><span class="linenos">1088</span></a> <span class="k">break</span> +</span><span id="L-1089"><a href="#L-1089"><span class="linenos">1089</span></a> +</span><span id="L-1090"><a href="#L-1090"><span class="linenos">1090</span></a> <span class="n">text</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="c1"># type: ignore</span> +</span><span id="L-1091"><a href="#L-1091"><span class="linenos">1091</span></a> +</span><span id="L-1092"><a href="#L-1092"><span class="linenos">1092</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">IDENTIFIER</span><span class="p">,</span> <span class="n">text</span><span class="p">)</span> +</span><span id="L-1093"><a href="#L-1093"><span class="linenos">1093</span></a> +</span><span id="L-1094"><a href="#L-1094"><span class="linenos">1094</span></a> <span class="k">def</span> <span class="nf">_scan_var</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> +</span><span id="L-1095"><a href="#L-1095"><span class="linenos">1095</span></a> <span class="k">while</span> <span class="kc">True</span><span class="p">:</span> +</span><span id="L-1096"><a href="#L-1096"><span class="linenos">1096</span></a> <span class="n">char</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="c1"># type: ignore</span> +</span><span id="L-1097"><a href="#L-1097"><span class="linenos">1097</span></a> <span class="k">if</span> <span class="n">char</span> <span class="ow">and</span> <span class="n">char</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">SINGLE_TOKENS</span><span class="p">:</span> +</span><span id="L-1098"><a href="#L-1098"><span class="linenos">1098</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="L-1099"><a href="#L-1099"><span class="linenos">1099</span></a> <span class="k">else</span><span class="p">:</span> +</span><span id="L-1100"><a href="#L-1100"><span class="linenos">1100</span></a> <span class="k">break</span> +</span><span id="L-1101"><a href="#L-1101"><span class="linenos">1101</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span> +</span><span id="L-1102"><a href="#L-1102"><span class="linenos">1102</span></a> <span class="n">TokenType</span><span class="o">.</span><span class="n">VAR</span> +</span><span id="L-1103"><a href="#L-1103"><span class="linenos">1103</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prev_token_type</span> <span class="o">==</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">PARAMETER</span> +</span><span id="L-1104"><a href="#L-1104"><span class="linenos">1104</span></a> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">KEYWORDS</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_text</span><span class="o">.</span><span class="n">upper</span><span class="p">(),</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">VAR</span><span class="p">)</span> +</span><span id="L-1105"><a href="#L-1105"><span class="linenos">1105</span></a> <span class="p">)</span> +</span><span id="L-1106"><a href="#L-1106"><span class="linenos">1106</span></a> +</span><span id="L-1107"><a href="#L-1107"><span class="linenos">1107</span></a> <span class="k">def</span> <span class="nf">_extract_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">delimiter</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> +</span><span id="L-1108"><a href="#L-1108"><span class="linenos">1108</span></a> <span class="n">text</span> <span class="o">=</span> <span class="s2">""</span> +</span><span id="L-1109"><a href="#L-1109"><span class="linenos">1109</span></a> <span class="n">delim_size</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">delimiter</span><span class="p">)</span> +</span><span id="L-1110"><a href="#L-1110"><span class="linenos">1110</span></a> +</span><span id="L-1111"><a href="#L-1111"><span class="linenos">1111</span></a> <span class="k">while</span> <span class="kc">True</span><span class="p">:</span> +</span><span id="L-1112"><a href="#L-1112"><span class="linenos">1112</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_STRING_ESCAPES</span> <span class="ow">and</span> <span class="p">(</span> +</span><span id="L-1113"><a href="#L-1113"><span class="linenos">1113</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="o">==</span> <span class="n">delimiter</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_STRING_ESCAPES</span> +</span><span id="L-1114"><a href="#L-1114"><span class="linenos">1114</span></a> <span class="p">):</span> +</span><span id="L-1115"><a href="#L-1115"><span class="linenos">1115</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="o">==</span> <span class="n">delimiter</span><span class="p">:</span> +</span><span id="L-1116"><a href="#L-1116"><span class="linenos">1116</span></a> <span class="n">text</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="c1"># type: ignore</span> +</span><span id="L-1117"><a href="#L-1117"><span class="linenos">1117</span></a> <span class="k">else</span><span class="p">:</span> +</span><span id="L-1118"><a href="#L-1118"><span class="linenos">1118</span></a> <span class="n">text</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="c1"># type: ignore</span> +</span><span id="L-1119"><a href="#L-1119"><span class="linenos">1119</span></a> +</span><span id="L-1120"><a href="#L-1120"><span class="linenos">1120</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_current</span> <span class="o">+</span> <span class="mi">1</span> <span class="o"><</span> <span class="bp">self</span><span class="o">.</span><span class="n">size</span><span class="p">:</span> +</span><span id="L-1121"><a href="#L-1121"><span class="linenos">1121</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span> +</span><span id="L-1122"><a href="#L-1122"><span class="linenos">1122</span></a> <span class="k">else</span><span class="p">:</span> +</span><span id="L-1123"><a href="#L-1123"><span class="linenos">1123</span></a> <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Missing </span><span class="si">{</span><span class="n">delimiter</span><span class="si">}</span><span class="s2"> from </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_line</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_current</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> +</span><span id="L-1124"><a href="#L-1124"><span class="linenos">1124</span></a> <span class="k">else</span><span class="p">:</span> +</span><span id="L-1125"><a href="#L-1125"><span class="linenos">1125</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_chars</span><span class="p">(</span><span class="n">delim_size</span><span class="p">)</span> <span class="o">==</span> <span class="n">delimiter</span><span class="p">:</span> +</span><span id="L-1126"><a href="#L-1126"><span class="linenos">1126</span></a> <span class="k">if</span> <span class="n">delim_size</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span> +</span><span id="L-1127"><a href="#L-1127"><span class="linenos">1127</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="n">delim_size</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> +</span><span id="L-1128"><a href="#L-1128"><span class="linenos">1128</span></a> <span class="k">break</span> +</span><span id="L-1129"><a href="#L-1129"><span class="linenos">1129</span></a> +</span><span id="L-1130"><a href="#L-1130"><span class="linenos">1130</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_end</span><span class="p">:</span> +</span><span id="L-1131"><a href="#L-1131"><span class="linenos">1131</span></a> <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Missing </span><span class="si">{</span><span class="n">delimiter</span><span class="si">}</span><span class="s2"> from </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_line</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_start</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> +</span><span id="L-1132"><a href="#L-1132"><span class="linenos">1132</span></a> <span class="n">text</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="c1"># type: ignore</span> +</span><span id="L-1133"><a href="#L-1133"><span class="linenos">1133</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="L-1134"><a href="#L-1134"><span class="linenos">1134</span></a> +</span><span id="L-1135"><a href="#L-1135"><span class="linenos">1135</span></a> <span class="k">return</span> <span class="n">text</span> </span></pre></div> @@ -6710,7 +6711,7 @@ </span><span id="Tokenizer-909"><a href="#Tokenizer-909"><span class="linenos"> 909</span></a> </span><span id="Tokenizer-910"><a href="#Tokenizer-910"><span class="linenos"> 910</span></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">word</span><span class="p">:</span> </span><span id="Tokenizer-911"><a href="#Tokenizer-911"><span class="linenos"> 911</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">SINGLE_TOKENS</span><span class="p">:</span> -</span><span id="Tokenizer-912"><a href="#Tokenizer-912"><span class="linenos"> 912</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">SINGLE_TOKENS</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_char</span><span class="p">])</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-912"><a href="#Tokenizer-912"><span class="linenos"> 912</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">SINGLE_TOKENS</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_char</span><span class="p">],</span> <span class="n">text</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_char</span><span class="p">)</span> <span class="c1"># type: ignore</span> </span><span id="Tokenizer-913"><a href="#Tokenizer-913"><span class="linenos"> 913</span></a> <span class="k">return</span> </span><span id="Tokenizer-914"><a href="#Tokenizer-914"><span class="linenos"> 914</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_scan_var</span><span class="p">()</span> </span><span id="Tokenizer-915"><a href="#Tokenizer-915"><span class="linenos"> 915</span></a> <span class="k">return</span> @@ -6723,217 +6724,218 @@ </span><span id="Tokenizer-922"><a href="#Tokenizer-922"><span class="linenos"> 922</span></a> <span class="k">return</span> </span><span id="Tokenizer-923"><a href="#Tokenizer-923"><span class="linenos"> 923</span></a> </span><span id="Tokenizer-924"><a href="#Tokenizer-924"><span class="linenos"> 924</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="n">size</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> -</span><span id="Tokenizer-925"><a href="#Tokenizer-925"><span class="linenos"> 925</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">KEYWORDS</span><span class="p">[</span><span class="n">word</span><span class="o">.</span><span class="n">upper</span><span class="p">()])</span> -</span><span id="Tokenizer-926"><a href="#Tokenizer-926"><span class="linenos"> 926</span></a> -</span><span id="Tokenizer-927"><a href="#Tokenizer-927"><span class="linenos"> 927</span></a> <span class="k">def</span> <span class="nf">_scan_comment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">comment_start</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> -</span><span id="Tokenizer-928"><a href="#Tokenizer-928"><span class="linenos"> 928</span></a> <span class="k">if</span> <span class="n">comment_start</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_COMMENTS</span><span class="p">:</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-929"><a href="#Tokenizer-929"><span class="linenos"> 929</span></a> <span class="k">return</span> <span class="kc">False</span> -</span><span id="Tokenizer-930"><a href="#Tokenizer-930"><span class="linenos"> 930</span></a> -</span><span id="Tokenizer-931"><a href="#Tokenizer-931"><span class="linenos"> 931</span></a> <span class="n">comment_start_line</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_line</span> -</span><span id="Tokenizer-932"><a href="#Tokenizer-932"><span class="linenos"> 932</span></a> <span class="n">comment_start_size</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">comment_start</span><span class="p">)</span> -</span><span id="Tokenizer-933"><a href="#Tokenizer-933"><span class="linenos"> 933</span></a> <span class="n">comment_end</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_COMMENTS</span><span class="p">[</span><span class="n">comment_start</span><span class="p">]</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-934"><a href="#Tokenizer-934"><span class="linenos"> 934</span></a> -</span><span id="Tokenizer-935"><a href="#Tokenizer-935"><span class="linenos"> 935</span></a> <span class="k">if</span> <span class="n">comment_end</span><span class="p">:</span> -</span><span id="Tokenizer-936"><a href="#Tokenizer-936"><span class="linenos"> 936</span></a> <span class="n">comment_end_size</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">comment_end</span><span class="p">)</span> -</span><span id="Tokenizer-937"><a href="#Tokenizer-937"><span class="linenos"> 937</span></a> -</span><span id="Tokenizer-938"><a href="#Tokenizer-938"><span class="linenos"> 938</span></a> <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_end</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_chars</span><span class="p">(</span><span class="n">comment_end_size</span><span class="p">)</span> <span class="o">!=</span> <span class="n">comment_end</span><span class="p">:</span> -</span><span id="Tokenizer-939"><a href="#Tokenizer-939"><span class="linenos"> 939</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="Tokenizer-940"><a href="#Tokenizer-940"><span class="linenos"> 940</span></a> -</span><span id="Tokenizer-941"><a href="#Tokenizer-941"><span class="linenos"> 941</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_comments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_text</span><span class="p">[</span><span class="n">comment_start_size</span> <span class="p">:</span> <span class="o">-</span><span class="n">comment_end_size</span> <span class="o">+</span> <span class="mi">1</span><span class="p">])</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-942"><a href="#Tokenizer-942"><span class="linenos"> 942</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="n">comment_end_size</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> -</span><span id="Tokenizer-943"><a href="#Tokenizer-943"><span class="linenos"> 943</span></a> <span class="k">else</span><span class="p">:</span> -</span><span id="Tokenizer-944"><a href="#Tokenizer-944"><span class="linenos"> 944</span></a> <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_end</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">WHITE_SPACE</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="p">)</span> <span class="ow">is</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">BREAK</span><span class="p">:</span> -</span><span id="Tokenizer-945"><a href="#Tokenizer-945"><span class="linenos"> 945</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="Tokenizer-946"><a href="#Tokenizer-946"><span class="linenos"> 946</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_comments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_text</span><span class="p">[</span><span class="n">comment_start_size</span><span class="p">:])</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-947"><a href="#Tokenizer-947"><span class="linenos"> 947</span></a> -</span><span id="Tokenizer-948"><a href="#Tokenizer-948"><span class="linenos"> 948</span></a> <span class="c1"># Leading comment is attached to the succeeding token, whilst trailing comment to the preceding.</span> -</span><span id="Tokenizer-949"><a href="#Tokenizer-949"><span class="linenos"> 949</span></a> <span class="c1"># Multiple consecutive comments are preserved by appending them to the current comments list.</span> -</span><span id="Tokenizer-950"><a href="#Tokenizer-950"><span class="linenos"> 950</span></a> <span class="k">if</span> <span class="n">comment_start_line</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prev_token_line</span><span class="p">:</span> -</span><span id="Tokenizer-951"><a href="#Tokenizer-951"><span class="linenos"> 951</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">tokens</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">comments</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_comments</span><span class="p">)</span> -</span><span id="Tokenizer-952"><a href="#Tokenizer-952"><span class="linenos"> 952</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_comments</span> <span class="o">=</span> <span class="p">[]</span> -</span><span id="Tokenizer-953"><a href="#Tokenizer-953"><span class="linenos"> 953</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_prev_token_line</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_line</span> -</span><span id="Tokenizer-954"><a href="#Tokenizer-954"><span class="linenos"> 954</span></a> -</span><span id="Tokenizer-955"><a href="#Tokenizer-955"><span class="linenos"> 955</span></a> <span class="k">return</span> <span class="kc">True</span> -</span><span id="Tokenizer-956"><a href="#Tokenizer-956"><span class="linenos"> 956</span></a> -</span><span id="Tokenizer-957"><a href="#Tokenizer-957"><span class="linenos"> 957</span></a> <span class="k">def</span> <span class="nf">_scan_number</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> -</span><span id="Tokenizer-958"><a href="#Tokenizer-958"><span class="linenos"> 958</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="o">==</span> <span class="s2">"0"</span><span class="p">:</span> -</span><span id="Tokenizer-959"><a href="#Tokenizer-959"><span class="linenos"> 959</span></a> <span class="n">peek</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-960"><a href="#Tokenizer-960"><span class="linenos"> 960</span></a> <span class="k">if</span> <span class="n">peek</span> <span class="o">==</span> <span class="s2">"B"</span><span class="p">:</span> -</span><span id="Tokenizer-961"><a href="#Tokenizer-961"><span class="linenos"> 961</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_scan_bits</span><span class="p">()</span> -</span><span id="Tokenizer-962"><a href="#Tokenizer-962"><span class="linenos"> 962</span></a> <span class="k">elif</span> <span class="n">peek</span> <span class="o">==</span> <span class="s2">"X"</span><span class="p">:</span> -</span><span id="Tokenizer-963"><a href="#Tokenizer-963"><span class="linenos"> 963</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_scan_hex</span><span class="p">()</span> -</span><span id="Tokenizer-964"><a href="#Tokenizer-964"><span class="linenos"> 964</span></a> -</span><span id="Tokenizer-965"><a href="#Tokenizer-965"><span class="linenos"> 965</span></a> <span class="n">decimal</span> <span class="o">=</span> <span class="kc">False</span> -</span><span id="Tokenizer-966"><a href="#Tokenizer-966"><span class="linenos"> 966</span></a> <span class="n">scientific</span> <span class="o">=</span> <span class="mi">0</span> -</span><span id="Tokenizer-967"><a href="#Tokenizer-967"><span class="linenos"> 967</span></a> -</span><span id="Tokenizer-968"><a href="#Tokenizer-968"><span class="linenos"> 968</span></a> <span class="k">while</span> <span class="kc">True</span><span class="p">:</span> -</span><span id="Tokenizer-969"><a href="#Tokenizer-969"><span class="linenos"> 969</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">isdigit</span><span class="p">():</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-970"><a href="#Tokenizer-970"><span class="linenos"> 970</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="Tokenizer-971"><a href="#Tokenizer-971"><span class="linenos"> 971</span></a> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="o">==</span> <span class="s2">"."</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">decimal</span><span class="p">:</span> -</span><span id="Tokenizer-972"><a href="#Tokenizer-972"><span class="linenos"> 972</span></a> <span class="n">decimal</span> <span class="o">=</span> <span class="kc">True</span> -</span><span id="Tokenizer-973"><a href="#Tokenizer-973"><span class="linenos"> 973</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="Tokenizer-974"><a href="#Tokenizer-974"><span class="linenos"> 974</span></a> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"-"</span><span class="p">,</span> <span class="s2">"+"</span><span class="p">)</span> <span class="ow">and</span> <span class="n">scientific</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span> -</span><span id="Tokenizer-975"><a href="#Tokenizer-975"><span class="linenos"> 975</span></a> <span class="n">scientific</span> <span class="o">+=</span> <span class="mi">1</span> -</span><span id="Tokenizer-976"><a href="#Tokenizer-976"><span class="linenos"> 976</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="Tokenizer-977"><a href="#Tokenizer-977"><span class="linenos"> 977</span></a> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"E"</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">scientific</span><span class="p">:</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-978"><a href="#Tokenizer-978"><span class="linenos"> 978</span></a> <span class="n">scientific</span> <span class="o">+=</span> <span class="mi">1</span> -</span><span id="Tokenizer-979"><a href="#Tokenizer-979"><span class="linenos"> 979</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="Tokenizer-980"><a href="#Tokenizer-980"><span class="linenos"> 980</span></a> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">isidentifier</span><span class="p">():</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-981"><a href="#Tokenizer-981"><span class="linenos"> 981</span></a> <span class="n">number_text</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_text</span> -</span><span id="Tokenizer-982"><a href="#Tokenizer-982"><span class="linenos"> 982</span></a> <span class="n">literal</span> <span class="o">=</span> <span class="p">[]</span> -</span><span id="Tokenizer-983"><a href="#Tokenizer-983"><span class="linenos"> 983</span></a> -</span><span id="Tokenizer-984"><a href="#Tokenizer-984"><span class="linenos"> 984</span></a> <span class="k">while</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">SINGLE_TOKENS</span><span class="p">:</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-985"><a href="#Tokenizer-985"><span class="linenos"> 985</span></a> <span class="n">literal</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">upper</span><span class="p">())</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-986"><a href="#Tokenizer-986"><span class="linenos"> 986</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="Tokenizer-987"><a href="#Tokenizer-987"><span class="linenos"> 987</span></a> -</span><span id="Tokenizer-988"><a href="#Tokenizer-988"><span class="linenos"> 988</span></a> <span class="n">literal</span> <span class="o">=</span> <span class="s2">""</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">literal</span><span class="p">)</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-989"><a href="#Tokenizer-989"><span class="linenos"> 989</span></a> <span class="n">token_type</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">KEYWORDS</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">NUMERIC_LITERALS</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">literal</span><span class="p">))</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-990"><a href="#Tokenizer-990"><span class="linenos"> 990</span></a> -</span><span id="Tokenizer-991"><a href="#Tokenizer-991"><span class="linenos"> 991</span></a> <span class="k">if</span> <span class="n">token_type</span><span class="p">:</span> -</span><span id="Tokenizer-992"><a href="#Tokenizer-992"><span class="linenos"> 992</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">NUMBER</span><span class="p">,</span> <span class="n">number_text</span><span class="p">)</span> -</span><span id="Tokenizer-993"><a href="#Tokenizer-993"><span class="linenos"> 993</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">DCOLON</span><span class="p">,</span> <span class="s2">"::"</span><span class="p">)</span> -</span><span id="Tokenizer-994"><a href="#Tokenizer-994"><span class="linenos"> 994</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">token_type</span><span class="p">,</span> <span class="n">literal</span><span class="p">)</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-995"><a href="#Tokenizer-995"><span class="linenos"> 995</span></a> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">IDENTIFIER_CAN_START_WITH_DIGIT</span><span class="p">:</span> -</span><span id="Tokenizer-996"><a href="#Tokenizer-996"><span class="linenos"> 996</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">VAR</span><span class="p">)</span> -</span><span id="Tokenizer-997"><a href="#Tokenizer-997"><span class="linenos"> 997</span></a> -</span><span id="Tokenizer-998"><a href="#Tokenizer-998"><span class="linenos"> 998</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">NUMBER</span><span class="p">,</span> <span class="n">number_text</span><span class="p">)</span> -</span><span id="Tokenizer-999"><a href="#Tokenizer-999"><span class="linenos"> 999</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="o">-</span><span class="nb">len</span><span class="p">(</span><span class="n">literal</span><span class="p">))</span> -</span><span id="Tokenizer-1000"><a href="#Tokenizer-1000"><span class="linenos">1000</span></a> <span class="k">else</span><span class="p">:</span> -</span><span id="Tokenizer-1001"><a href="#Tokenizer-1001"><span class="linenos">1001</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">NUMBER</span><span class="p">)</span> -</span><span id="Tokenizer-1002"><a href="#Tokenizer-1002"><span class="linenos">1002</span></a> -</span><span id="Tokenizer-1003"><a href="#Tokenizer-1003"><span class="linenos">1003</span></a> <span class="k">def</span> <span class="nf">_scan_bits</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> -</span><span id="Tokenizer-1004"><a href="#Tokenizer-1004"><span class="linenos">1004</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="Tokenizer-1005"><a href="#Tokenizer-1005"><span class="linenos">1005</span></a> <span class="n">value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extract_value</span><span class="p">()</span> -</span><span id="Tokenizer-1006"><a href="#Tokenizer-1006"><span class="linenos">1006</span></a> <span class="k">try</span><span class="p">:</span> -</span><span id="Tokenizer-1007"><a href="#Tokenizer-1007"><span class="linenos">1007</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">BIT_STRING</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="nb">int</span><span class="p">(</span><span class="n">value</span><span class="p">,</span><span class="w"> </span><span class="mi">2</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> -</span><span id="Tokenizer-1008"><a href="#Tokenizer-1008"><span class="linenos">1008</span></a> <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span> -</span><span id="Tokenizer-1009"><a href="#Tokenizer-1009"><span class="linenos">1009</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">IDENTIFIER</span><span class="p">)</span> -</span><span id="Tokenizer-1010"><a href="#Tokenizer-1010"><span class="linenos">1010</span></a> -</span><span id="Tokenizer-1011"><a href="#Tokenizer-1011"><span class="linenos">1011</span></a> <span class="k">def</span> <span class="nf">_scan_hex</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> -</span><span id="Tokenizer-1012"><a href="#Tokenizer-1012"><span class="linenos">1012</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="Tokenizer-1013"><a href="#Tokenizer-1013"><span class="linenos">1013</span></a> <span class="n">value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extract_value</span><span class="p">()</span> -</span><span id="Tokenizer-1014"><a href="#Tokenizer-1014"><span class="linenos">1014</span></a> <span class="k">try</span><span class="p">:</span> -</span><span id="Tokenizer-1015"><a href="#Tokenizer-1015"><span class="linenos">1015</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">HEX_STRING</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="nb">int</span><span class="p">(</span><span class="n">value</span><span class="p">,</span><span class="w"> </span><span class="mi">16</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> -</span><span id="Tokenizer-1016"><a href="#Tokenizer-1016"><span class="linenos">1016</span></a> <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span> -</span><span id="Tokenizer-1017"><a href="#Tokenizer-1017"><span class="linenos">1017</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">IDENTIFIER</span><span class="p">)</span> -</span><span id="Tokenizer-1018"><a href="#Tokenizer-1018"><span class="linenos">1018</span></a> -</span><span id="Tokenizer-1019"><a href="#Tokenizer-1019"><span class="linenos">1019</span></a> <span class="k">def</span> <span class="nf">_extract_value</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> -</span><span id="Tokenizer-1020"><a href="#Tokenizer-1020"><span class="linenos">1020</span></a> <span class="k">while</span> <span class="kc">True</span><span class="p">:</span> -</span><span id="Tokenizer-1021"><a href="#Tokenizer-1021"><span class="linenos">1021</span></a> <span class="n">char</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-1022"><a href="#Tokenizer-1022"><span class="linenos">1022</span></a> <span class="k">if</span> <span class="n">char</span> <span class="ow">and</span> <span class="n">char</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">SINGLE_TOKENS</span><span class="p">:</span> -</span><span id="Tokenizer-1023"><a href="#Tokenizer-1023"><span class="linenos">1023</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="Tokenizer-1024"><a href="#Tokenizer-1024"><span class="linenos">1024</span></a> <span class="k">else</span><span class="p">:</span> -</span><span id="Tokenizer-1025"><a href="#Tokenizer-1025"><span class="linenos">1025</span></a> <span class="k">break</span> -</span><span id="Tokenizer-1026"><a href="#Tokenizer-1026"><span class="linenos">1026</span></a> -</span><span id="Tokenizer-1027"><a href="#Tokenizer-1027"><span class="linenos">1027</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_text</span> -</span><span id="Tokenizer-1028"><a href="#Tokenizer-1028"><span class="linenos">1028</span></a> -</span><span id="Tokenizer-1029"><a href="#Tokenizer-1029"><span class="linenos">1029</span></a> <span class="k">def</span> <span class="nf">_scan_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">quote</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> -</span><span id="Tokenizer-1030"><a href="#Tokenizer-1030"><span class="linenos">1030</span></a> <span class="n">quote_end</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_QUOTES</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">quote</span><span class="p">)</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-1031"><a href="#Tokenizer-1031"><span class="linenos">1031</span></a> <span class="k">if</span> <span class="n">quote_end</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> -</span><span id="Tokenizer-1032"><a href="#Tokenizer-1032"><span class="linenos">1032</span></a> <span class="k">return</span> <span class="kc">False</span> -</span><span id="Tokenizer-1033"><a href="#Tokenizer-1033"><span class="linenos">1033</span></a> -</span><span id="Tokenizer-1034"><a href="#Tokenizer-1034"><span class="linenos">1034</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">quote</span><span class="p">))</span> -</span><span id="Tokenizer-1035"><a href="#Tokenizer-1035"><span class="linenos">1035</span></a> <span class="n">text</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extract_string</span><span class="p">(</span><span class="n">quote_end</span><span class="p">)</span> -</span><span id="Tokenizer-1036"><a href="#Tokenizer-1036"><span class="linenos">1036</span></a> <span class="n">text</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ENCODE</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ENCODE</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">ENCODE</span> <span class="k">else</span> <span class="n">text</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-1037"><a href="#Tokenizer-1037"><span class="linenos">1037</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">NATIONAL</span> <span class="k">if</span> <span class="n">quote</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"N"</span> <span class="k">else</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">STRING</span><span class="p">,</span> <span class="n">text</span><span class="p">)</span> -</span><span id="Tokenizer-1038"><a href="#Tokenizer-1038"><span class="linenos">1038</span></a> <span class="k">return</span> <span class="kc">True</span> -</span><span id="Tokenizer-1039"><a href="#Tokenizer-1039"><span class="linenos">1039</span></a> -</span><span id="Tokenizer-1040"><a href="#Tokenizer-1040"><span class="linenos">1040</span></a> <span class="c1"># X'1234, b'0110', E'\\\\\' etc.</span> -</span><span id="Tokenizer-1041"><a href="#Tokenizer-1041"><span class="linenos">1041</span></a> <span class="k">def</span> <span class="nf">_scan_formatted_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">string_start</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> -</span><span id="Tokenizer-1042"><a href="#Tokenizer-1042"><span class="linenos">1042</span></a> <span class="k">if</span> <span class="n">string_start</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_HEX_STRINGS</span><span class="p">:</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-1043"><a href="#Tokenizer-1043"><span class="linenos">1043</span></a> <span class="n">delimiters</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_HEX_STRINGS</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-1044"><a href="#Tokenizer-1044"><span class="linenos">1044</span></a> <span class="n">token_type</span> <span class="o">=</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">HEX_STRING</span> -</span><span id="Tokenizer-1045"><a href="#Tokenizer-1045"><span class="linenos">1045</span></a> <span class="n">base</span> <span class="o">=</span> <span class="mi">16</span> -</span><span id="Tokenizer-1046"><a href="#Tokenizer-1046"><span class="linenos">1046</span></a> <span class="k">elif</span> <span class="n">string_start</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_BIT_STRINGS</span><span class="p">:</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-1047"><a href="#Tokenizer-1047"><span class="linenos">1047</span></a> <span class="n">delimiters</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_BIT_STRINGS</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-1048"><a href="#Tokenizer-1048"><span class="linenos">1048</span></a> <span class="n">token_type</span> <span class="o">=</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">BIT_STRING</span> -</span><span id="Tokenizer-1049"><a href="#Tokenizer-1049"><span class="linenos">1049</span></a> <span class="n">base</span> <span class="o">=</span> <span class="mi">2</span> -</span><span id="Tokenizer-1050"><a href="#Tokenizer-1050"><span class="linenos">1050</span></a> <span class="k">elif</span> <span class="n">string_start</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_BYTE_STRINGS</span><span class="p">:</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-1051"><a href="#Tokenizer-1051"><span class="linenos">1051</span></a> <span class="n">delimiters</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_BYTE_STRINGS</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-1052"><a href="#Tokenizer-1052"><span class="linenos">1052</span></a> <span class="n">token_type</span> <span class="o">=</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">BYTE_STRING</span> -</span><span id="Tokenizer-1053"><a href="#Tokenizer-1053"><span class="linenos">1053</span></a> <span class="n">base</span> <span class="o">=</span> <span class="kc">None</span> -</span><span id="Tokenizer-1054"><a href="#Tokenizer-1054"><span class="linenos">1054</span></a> <span class="k">else</span><span class="p">:</span> -</span><span id="Tokenizer-1055"><a href="#Tokenizer-1055"><span class="linenos">1055</span></a> <span class="k">return</span> <span class="kc">False</span> -</span><span id="Tokenizer-1056"><a href="#Tokenizer-1056"><span class="linenos">1056</span></a> -</span><span id="Tokenizer-1057"><a href="#Tokenizer-1057"><span class="linenos">1057</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">string_start</span><span class="p">))</span> -</span><span id="Tokenizer-1058"><a href="#Tokenizer-1058"><span class="linenos">1058</span></a> <span class="n">string_end</span> <span class="o">=</span> <span class="n">delimiters</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">string_start</span><span class="p">)</span> -</span><span id="Tokenizer-1059"><a href="#Tokenizer-1059"><span class="linenos">1059</span></a> <span class="n">text</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extract_string</span><span class="p">(</span><span class="n">string_end</span><span class="p">)</span> -</span><span id="Tokenizer-1060"><a href="#Tokenizer-1060"><span class="linenos">1060</span></a> -</span><span id="Tokenizer-1061"><a href="#Tokenizer-1061"><span class="linenos">1061</span></a> <span class="k">if</span> <span class="n">base</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> -</span><span id="Tokenizer-1062"><a href="#Tokenizer-1062"><span class="linenos">1062</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">token_type</span><span class="p">,</span> <span class="n">text</span><span class="p">)</span> -</span><span id="Tokenizer-1063"><a href="#Tokenizer-1063"><span class="linenos">1063</span></a> <span class="k">else</span><span class="p">:</span> -</span><span id="Tokenizer-1064"><a href="#Tokenizer-1064"><span class="linenos">1064</span></a> <span class="k">try</span><span class="p">:</span> -</span><span id="Tokenizer-1065"><a href="#Tokenizer-1065"><span class="linenos">1065</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">token_type</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="nb">int</span><span class="p">(</span><span class="n">text</span><span class="p">,</span><span class="w"> </span><span class="n">base</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> -</span><span id="Tokenizer-1066"><a href="#Tokenizer-1066"><span class="linenos">1066</span></a> <span class="k">except</span><span class="p">:</span> -</span><span id="Tokenizer-1067"><a href="#Tokenizer-1067"><span class="linenos">1067</span></a> <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span> -</span><span id="Tokenizer-1068"><a href="#Tokenizer-1068"><span class="linenos">1068</span></a> <span class="sa">f</span><span class="s2">"Numeric string contains invalid characters from </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_line</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_start</span><span class="si">}</span><span class="s2">"</span> -</span><span id="Tokenizer-1069"><a href="#Tokenizer-1069"><span class="linenos">1069</span></a> <span class="p">)</span> -</span><span id="Tokenizer-1070"><a href="#Tokenizer-1070"><span class="linenos">1070</span></a> -</span><span id="Tokenizer-1071"><a href="#Tokenizer-1071"><span class="linenos">1071</span></a> <span class="k">return</span> <span class="kc">True</span> -</span><span id="Tokenizer-1072"><a href="#Tokenizer-1072"><span class="linenos">1072</span></a> -</span><span id="Tokenizer-1073"><a href="#Tokenizer-1073"><span class="linenos">1073</span></a> <span class="k">def</span> <span class="nf">_scan_identifier</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">identifier_end</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> -</span><span id="Tokenizer-1074"><a href="#Tokenizer-1074"><span class="linenos">1074</span></a> <span class="n">text</span> <span class="o">=</span> <span class="s2">""</span> -</span><span id="Tokenizer-1075"><a href="#Tokenizer-1075"><span class="linenos">1075</span></a> <span class="n">identifier_end_is_escape</span> <span class="o">=</span> <span class="n">identifier_end</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_IDENTIFIER_ESCAPES</span> -</span><span id="Tokenizer-1076"><a href="#Tokenizer-1076"><span class="linenos">1076</span></a> -</span><span id="Tokenizer-1077"><a href="#Tokenizer-1077"><span class="linenos">1077</span></a> <span class="k">while</span> <span class="kc">True</span><span class="p">:</span> -</span><span id="Tokenizer-1078"><a href="#Tokenizer-1078"><span class="linenos">1078</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_end</span><span class="p">:</span> -</span><span id="Tokenizer-1079"><a href="#Tokenizer-1079"><span class="linenos">1079</span></a> <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Missing </span><span class="si">{</span><span class="n">identifier_end</span><span class="si">}</span><span class="s2"> from </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_line</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_start</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> -</span><span id="Tokenizer-1080"><a href="#Tokenizer-1080"><span class="linenos">1080</span></a> -</span><span id="Tokenizer-1081"><a href="#Tokenizer-1081"><span class="linenos">1081</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="Tokenizer-1082"><a href="#Tokenizer-1082"><span class="linenos">1082</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="o">==</span> <span class="n">identifier_end</span><span class="p">:</span> -</span><span id="Tokenizer-1083"><a href="#Tokenizer-1083"><span class="linenos">1083</span></a> <span class="k">if</span> <span class="n">identifier_end_is_escape</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="o">==</span> <span class="n">identifier_end</span><span class="p">:</span> -</span><span id="Tokenizer-1084"><a href="#Tokenizer-1084"><span class="linenos">1084</span></a> <span class="n">text</span> <span class="o">+=</span> <span class="n">identifier_end</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-1085"><a href="#Tokenizer-1085"><span class="linenos">1085</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="Tokenizer-1086"><a href="#Tokenizer-1086"><span class="linenos">1086</span></a> <span class="k">continue</span> -</span><span id="Tokenizer-1087"><a href="#Tokenizer-1087"><span class="linenos">1087</span></a> -</span><span id="Tokenizer-1088"><a href="#Tokenizer-1088"><span class="linenos">1088</span></a> <span class="k">break</span> -</span><span id="Tokenizer-1089"><a href="#Tokenizer-1089"><span class="linenos">1089</span></a> -</span><span id="Tokenizer-1090"><a href="#Tokenizer-1090"><span class="linenos">1090</span></a> <span class="n">text</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-1091"><a href="#Tokenizer-1091"><span class="linenos">1091</span></a> -</span><span id="Tokenizer-1092"><a href="#Tokenizer-1092"><span class="linenos">1092</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">IDENTIFIER</span><span class="p">,</span> <span class="n">text</span><span class="p">)</span> -</span><span id="Tokenizer-1093"><a href="#Tokenizer-1093"><span class="linenos">1093</span></a> -</span><span id="Tokenizer-1094"><a href="#Tokenizer-1094"><span class="linenos">1094</span></a> <span class="k">def</span> <span class="nf">_scan_var</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> -</span><span id="Tokenizer-1095"><a href="#Tokenizer-1095"><span class="linenos">1095</span></a> <span class="k">while</span> <span class="kc">True</span><span class="p">:</span> -</span><span id="Tokenizer-1096"><a href="#Tokenizer-1096"><span class="linenos">1096</span></a> <span class="n">char</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-1097"><a href="#Tokenizer-1097"><span class="linenos">1097</span></a> <span class="k">if</span> <span class="n">char</span> <span class="ow">and</span> <span class="n">char</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">SINGLE_TOKENS</span><span class="p">:</span> -</span><span id="Tokenizer-1098"><a href="#Tokenizer-1098"><span class="linenos">1098</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="Tokenizer-1099"><a href="#Tokenizer-1099"><span class="linenos">1099</span></a> <span class="k">else</span><span class="p">:</span> -</span><span id="Tokenizer-1100"><a href="#Tokenizer-1100"><span class="linenos">1100</span></a> <span class="k">break</span> -</span><span id="Tokenizer-1101"><a href="#Tokenizer-1101"><span class="linenos">1101</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span> -</span><span id="Tokenizer-1102"><a href="#Tokenizer-1102"><span class="linenos">1102</span></a> <span class="n">TokenType</span><span class="o">.</span><span class="n">VAR</span> -</span><span id="Tokenizer-1103"><a href="#Tokenizer-1103"><span class="linenos">1103</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prev_token_type</span> <span class="o">==</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">PARAMETER</span> -</span><span id="Tokenizer-1104"><a href="#Tokenizer-1104"><span class="linenos">1104</span></a> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">KEYWORDS</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_text</span><span class="o">.</span><span class="n">upper</span><span class="p">(),</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">VAR</span><span class="p">)</span> -</span><span id="Tokenizer-1105"><a href="#Tokenizer-1105"><span class="linenos">1105</span></a> <span class="p">)</span> -</span><span id="Tokenizer-1106"><a href="#Tokenizer-1106"><span class="linenos">1106</span></a> -</span><span id="Tokenizer-1107"><a href="#Tokenizer-1107"><span class="linenos">1107</span></a> <span class="k">def</span> <span class="nf">_extract_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">delimiter</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> -</span><span id="Tokenizer-1108"><a href="#Tokenizer-1108"><span class="linenos">1108</span></a> <span class="n">text</span> <span class="o">=</span> <span class="s2">""</span> -</span><span id="Tokenizer-1109"><a href="#Tokenizer-1109"><span class="linenos">1109</span></a> <span class="n">delim_size</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">delimiter</span><span class="p">)</span> -</span><span id="Tokenizer-1110"><a href="#Tokenizer-1110"><span class="linenos">1110</span></a> -</span><span id="Tokenizer-1111"><a href="#Tokenizer-1111"><span class="linenos">1111</span></a> <span class="k">while</span> <span class="kc">True</span><span class="p">:</span> -</span><span id="Tokenizer-1112"><a href="#Tokenizer-1112"><span class="linenos">1112</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_STRING_ESCAPES</span> <span class="ow">and</span> <span class="p">(</span> -</span><span id="Tokenizer-1113"><a href="#Tokenizer-1113"><span class="linenos">1113</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="o">==</span> <span class="n">delimiter</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_STRING_ESCAPES</span> -</span><span id="Tokenizer-1114"><a href="#Tokenizer-1114"><span class="linenos">1114</span></a> <span class="p">):</span> -</span><span id="Tokenizer-1115"><a href="#Tokenizer-1115"><span class="linenos">1115</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="o">==</span> <span class="n">delimiter</span><span class="p">:</span> -</span><span id="Tokenizer-1116"><a href="#Tokenizer-1116"><span class="linenos">1116</span></a> <span class="n">text</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-1117"><a href="#Tokenizer-1117"><span class="linenos">1117</span></a> <span class="k">else</span><span class="p">:</span> -</span><span id="Tokenizer-1118"><a href="#Tokenizer-1118"><span class="linenos">1118</span></a> <span class="n">text</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-1119"><a href="#Tokenizer-1119"><span class="linenos">1119</span></a> -</span><span id="Tokenizer-1120"><a href="#Tokenizer-1120"><span class="linenos">1120</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_current</span> <span class="o">+</span> <span class="mi">1</span> <span class="o"><</span> <span class="bp">self</span><span class="o">.</span><span class="n">size</span><span class="p">:</span> -</span><span id="Tokenizer-1121"><a href="#Tokenizer-1121"><span class="linenos">1121</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span> -</span><span id="Tokenizer-1122"><a href="#Tokenizer-1122"><span class="linenos">1122</span></a> <span class="k">else</span><span class="p">:</span> -</span><span id="Tokenizer-1123"><a href="#Tokenizer-1123"><span class="linenos">1123</span></a> <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Missing </span><span class="si">{</span><span class="n">delimiter</span><span class="si">}</span><span class="s2"> from </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_line</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_current</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> -</span><span id="Tokenizer-1124"><a href="#Tokenizer-1124"><span class="linenos">1124</span></a> <span class="k">else</span><span class="p">:</span> -</span><span id="Tokenizer-1125"><a href="#Tokenizer-1125"><span class="linenos">1125</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_chars</span><span class="p">(</span><span class="n">delim_size</span><span class="p">)</span> <span class="o">==</span> <span class="n">delimiter</span><span class="p">:</span> -</span><span id="Tokenizer-1126"><a href="#Tokenizer-1126"><span class="linenos">1126</span></a> <span class="k">if</span> <span class="n">delim_size</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span> -</span><span id="Tokenizer-1127"><a href="#Tokenizer-1127"><span class="linenos">1127</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="n">delim_size</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> -</span><span id="Tokenizer-1128"><a href="#Tokenizer-1128"><span class="linenos">1128</span></a> <span class="k">break</span> -</span><span id="Tokenizer-1129"><a href="#Tokenizer-1129"><span class="linenos">1129</span></a> -</span><span id="Tokenizer-1130"><a href="#Tokenizer-1130"><span class="linenos">1130</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_end</span><span class="p">:</span> -</span><span id="Tokenizer-1131"><a href="#Tokenizer-1131"><span class="linenos">1131</span></a> <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Missing </span><span class="si">{</span><span class="n">delimiter</span><span class="si">}</span><span class="s2"> from </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_line</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_start</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> -</span><span id="Tokenizer-1132"><a href="#Tokenizer-1132"><span class="linenos">1132</span></a> <span class="n">text</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="c1"># type: ignore</span> -</span><span id="Tokenizer-1133"><a href="#Tokenizer-1133"><span class="linenos">1133</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> -</span><span id="Tokenizer-1134"><a href="#Tokenizer-1134"><span class="linenos">1134</span></a> -</span><span id="Tokenizer-1135"><a href="#Tokenizer-1135"><span class="linenos">1135</span></a> <span class="k">return</span> <span class="n">text</span> +</span><span id="Tokenizer-925"><a href="#Tokenizer-925"><span class="linenos"> 925</span></a> <span class="n">word</span> <span class="o">=</span> <span class="n">word</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span> +</span><span id="Tokenizer-926"><a href="#Tokenizer-926"><span class="linenos"> 926</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">KEYWORDS</span><span class="p">[</span><span class="n">word</span><span class="p">],</span> <span class="n">text</span><span class="o">=</span><span class="n">word</span><span class="p">)</span> +</span><span id="Tokenizer-927"><a href="#Tokenizer-927"><span class="linenos"> 927</span></a> +</span><span id="Tokenizer-928"><a href="#Tokenizer-928"><span class="linenos"> 928</span></a> <span class="k">def</span> <span class="nf">_scan_comment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">comment_start</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> +</span><span id="Tokenizer-929"><a href="#Tokenizer-929"><span class="linenos"> 929</span></a> <span class="k">if</span> <span class="n">comment_start</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_COMMENTS</span><span class="p">:</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-930"><a href="#Tokenizer-930"><span class="linenos"> 930</span></a> <span class="k">return</span> <span class="kc">False</span> +</span><span id="Tokenizer-931"><a href="#Tokenizer-931"><span class="linenos"> 931</span></a> +</span><span id="Tokenizer-932"><a href="#Tokenizer-932"><span class="linenos"> 932</span></a> <span class="n">comment_start_line</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_line</span> +</span><span id="Tokenizer-933"><a href="#Tokenizer-933"><span class="linenos"> 933</span></a> <span class="n">comment_start_size</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">comment_start</span><span class="p">)</span> +</span><span id="Tokenizer-934"><a href="#Tokenizer-934"><span class="linenos"> 934</span></a> <span class="n">comment_end</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_COMMENTS</span><span class="p">[</span><span class="n">comment_start</span><span class="p">]</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-935"><a href="#Tokenizer-935"><span class="linenos"> 935</span></a> +</span><span id="Tokenizer-936"><a href="#Tokenizer-936"><span class="linenos"> 936</span></a> <span class="k">if</span> <span class="n">comment_end</span><span class="p">:</span> +</span><span id="Tokenizer-937"><a href="#Tokenizer-937"><span class="linenos"> 937</span></a> <span class="n">comment_end_size</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">comment_end</span><span class="p">)</span> +</span><span id="Tokenizer-938"><a href="#Tokenizer-938"><span class="linenos"> 938</span></a> +</span><span id="Tokenizer-939"><a href="#Tokenizer-939"><span class="linenos"> 939</span></a> <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_end</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_chars</span><span class="p">(</span><span class="n">comment_end_size</span><span class="p">)</span> <span class="o">!=</span> <span class="n">comment_end</span><span class="p">:</span> +</span><span id="Tokenizer-940"><a href="#Tokenizer-940"><span class="linenos"> 940</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="Tokenizer-941"><a href="#Tokenizer-941"><span class="linenos"> 941</span></a> +</span><span id="Tokenizer-942"><a href="#Tokenizer-942"><span class="linenos"> 942</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_comments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_text</span><span class="p">[</span><span class="n">comment_start_size</span> <span class="p">:</span> <span class="o">-</span><span class="n">comment_end_size</span> <span class="o">+</span> <span class="mi">1</span><span class="p">])</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-943"><a href="#Tokenizer-943"><span class="linenos"> 943</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="n">comment_end_size</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> +</span><span id="Tokenizer-944"><a href="#Tokenizer-944"><span class="linenos"> 944</span></a> <span class="k">else</span><span class="p">:</span> +</span><span id="Tokenizer-945"><a href="#Tokenizer-945"><span class="linenos"> 945</span></a> <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_end</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">WHITE_SPACE</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="p">)</span> <span class="ow">is</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">BREAK</span><span class="p">:</span> +</span><span id="Tokenizer-946"><a href="#Tokenizer-946"><span class="linenos"> 946</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="Tokenizer-947"><a href="#Tokenizer-947"><span class="linenos"> 947</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_comments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_text</span><span class="p">[</span><span class="n">comment_start_size</span><span class="p">:])</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-948"><a href="#Tokenizer-948"><span class="linenos"> 948</span></a> +</span><span id="Tokenizer-949"><a href="#Tokenizer-949"><span class="linenos"> 949</span></a> <span class="c1"># Leading comment is attached to the succeeding token, whilst trailing comment to the preceding.</span> +</span><span id="Tokenizer-950"><a href="#Tokenizer-950"><span class="linenos"> 950</span></a> <span class="c1"># Multiple consecutive comments are preserved by appending them to the current comments list.</span> +</span><span id="Tokenizer-951"><a href="#Tokenizer-951"><span class="linenos"> 951</span></a> <span class="k">if</span> <span class="n">comment_start_line</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prev_token_line</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">_end</span><span class="p">:</span> +</span><span id="Tokenizer-952"><a href="#Tokenizer-952"><span class="linenos"> 952</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">tokens</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">comments</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_comments</span><span class="p">)</span> +</span><span id="Tokenizer-953"><a href="#Tokenizer-953"><span class="linenos"> 953</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_comments</span> <span class="o">=</span> <span class="p">[]</span> +</span><span id="Tokenizer-954"><a href="#Tokenizer-954"><span class="linenos"> 954</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_prev_token_line</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_line</span> +</span><span id="Tokenizer-955"><a href="#Tokenizer-955"><span class="linenos"> 955</span></a> +</span><span id="Tokenizer-956"><a href="#Tokenizer-956"><span class="linenos"> 956</span></a> <span class="k">return</span> <span class="kc">True</span> +</span><span id="Tokenizer-957"><a href="#Tokenizer-957"><span class="linenos"> 957</span></a> +</span><span id="Tokenizer-958"><a href="#Tokenizer-958"><span class="linenos"> 958</span></a> <span class="k">def</span> <span class="nf">_scan_number</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> +</span><span id="Tokenizer-959"><a href="#Tokenizer-959"><span class="linenos"> 959</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="o">==</span> <span class="s2">"0"</span><span class="p">:</span> +</span><span id="Tokenizer-960"><a href="#Tokenizer-960"><span class="linenos"> 960</span></a> <span class="n">peek</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-961"><a href="#Tokenizer-961"><span class="linenos"> 961</span></a> <span class="k">if</span> <span class="n">peek</span> <span class="o">==</span> <span class="s2">"B"</span><span class="p">:</span> +</span><span id="Tokenizer-962"><a href="#Tokenizer-962"><span class="linenos"> 962</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_scan_bits</span><span class="p">()</span> +</span><span id="Tokenizer-963"><a href="#Tokenizer-963"><span class="linenos"> 963</span></a> <span class="k">elif</span> <span class="n">peek</span> <span class="o">==</span> <span class="s2">"X"</span><span class="p">:</span> +</span><span id="Tokenizer-964"><a href="#Tokenizer-964"><span class="linenos"> 964</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_scan_hex</span><span class="p">()</span> +</span><span id="Tokenizer-965"><a href="#Tokenizer-965"><span class="linenos"> 965</span></a> +</span><span id="Tokenizer-966"><a href="#Tokenizer-966"><span class="linenos"> 966</span></a> <span class="n">decimal</span> <span class="o">=</span> <span class="kc">False</span> +</span><span id="Tokenizer-967"><a href="#Tokenizer-967"><span class="linenos"> 967</span></a> <span class="n">scientific</span> <span class="o">=</span> <span class="mi">0</span> +</span><span id="Tokenizer-968"><a href="#Tokenizer-968"><span class="linenos"> 968</span></a> +</span><span id="Tokenizer-969"><a href="#Tokenizer-969"><span class="linenos"> 969</span></a> <span class="k">while</span> <span class="kc">True</span><span class="p">:</span> +</span><span id="Tokenizer-970"><a href="#Tokenizer-970"><span class="linenos"> 970</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">isdigit</span><span class="p">():</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-971"><a href="#Tokenizer-971"><span class="linenos"> 971</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="Tokenizer-972"><a href="#Tokenizer-972"><span class="linenos"> 972</span></a> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="o">==</span> <span class="s2">"."</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">decimal</span><span class="p">:</span> +</span><span id="Tokenizer-973"><a href="#Tokenizer-973"><span class="linenos"> 973</span></a> <span class="n">decimal</span> <span class="o">=</span> <span class="kc">True</span> +</span><span id="Tokenizer-974"><a href="#Tokenizer-974"><span class="linenos"> 974</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="Tokenizer-975"><a href="#Tokenizer-975"><span class="linenos"> 975</span></a> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"-"</span><span class="p">,</span> <span class="s2">"+"</span><span class="p">)</span> <span class="ow">and</span> <span class="n">scientific</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span> +</span><span id="Tokenizer-976"><a href="#Tokenizer-976"><span class="linenos"> 976</span></a> <span class="n">scientific</span> <span class="o">+=</span> <span class="mi">1</span> +</span><span id="Tokenizer-977"><a href="#Tokenizer-977"><span class="linenos"> 977</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="Tokenizer-978"><a href="#Tokenizer-978"><span class="linenos"> 978</span></a> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"E"</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">scientific</span><span class="p">:</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-979"><a href="#Tokenizer-979"><span class="linenos"> 979</span></a> <span class="n">scientific</span> <span class="o">+=</span> <span class="mi">1</span> +</span><span id="Tokenizer-980"><a href="#Tokenizer-980"><span class="linenos"> 980</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="Tokenizer-981"><a href="#Tokenizer-981"><span class="linenos"> 981</span></a> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">isidentifier</span><span class="p">():</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-982"><a href="#Tokenizer-982"><span class="linenos"> 982</span></a> <span class="n">number_text</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_text</span> +</span><span id="Tokenizer-983"><a href="#Tokenizer-983"><span class="linenos"> 983</span></a> <span class="n">literal</span> <span class="o">=</span> <span class="p">[]</span> +</span><span id="Tokenizer-984"><a href="#Tokenizer-984"><span class="linenos"> 984</span></a> +</span><span id="Tokenizer-985"><a href="#Tokenizer-985"><span class="linenos"> 985</span></a> <span class="k">while</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">SINGLE_TOKENS</span><span class="p">:</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-986"><a href="#Tokenizer-986"><span class="linenos"> 986</span></a> <span class="n">literal</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">upper</span><span class="p">())</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-987"><a href="#Tokenizer-987"><span class="linenos"> 987</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="Tokenizer-988"><a href="#Tokenizer-988"><span class="linenos"> 988</span></a> +</span><span id="Tokenizer-989"><a href="#Tokenizer-989"><span class="linenos"> 989</span></a> <span class="n">literal</span> <span class="o">=</span> <span class="s2">""</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">literal</span><span class="p">)</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-990"><a href="#Tokenizer-990"><span class="linenos"> 990</span></a> <span class="n">token_type</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">KEYWORDS</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">NUMERIC_LITERALS</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">literal</span><span class="p">))</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-991"><a href="#Tokenizer-991"><span class="linenos"> 991</span></a> +</span><span id="Tokenizer-992"><a href="#Tokenizer-992"><span class="linenos"> 992</span></a> <span class="k">if</span> <span class="n">token_type</span><span class="p">:</span> +</span><span id="Tokenizer-993"><a href="#Tokenizer-993"><span class="linenos"> 993</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">NUMBER</span><span class="p">,</span> <span class="n">number_text</span><span class="p">)</span> +</span><span id="Tokenizer-994"><a href="#Tokenizer-994"><span class="linenos"> 994</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">DCOLON</span><span class="p">,</span> <span class="s2">"::"</span><span class="p">)</span> +</span><span id="Tokenizer-995"><a href="#Tokenizer-995"><span class="linenos"> 995</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">token_type</span><span class="p">,</span> <span class="n">literal</span><span class="p">)</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-996"><a href="#Tokenizer-996"><span class="linenos"> 996</span></a> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">IDENTIFIER_CAN_START_WITH_DIGIT</span><span class="p">:</span> +</span><span id="Tokenizer-997"><a href="#Tokenizer-997"><span class="linenos"> 997</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">VAR</span><span class="p">)</span> +</span><span id="Tokenizer-998"><a href="#Tokenizer-998"><span class="linenos"> 998</span></a> +</span><span id="Tokenizer-999"><a href="#Tokenizer-999"><span class="linenos"> 999</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">NUMBER</span><span class="p">,</span> <span class="n">number_text</span><span class="p">)</span> +</span><span id="Tokenizer-1000"><a href="#Tokenizer-1000"><span class="linenos">1000</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="o">-</span><span class="nb">len</span><span class="p">(</span><span class="n">literal</span><span class="p">))</span> +</span><span id="Tokenizer-1001"><a href="#Tokenizer-1001"><span class="linenos">1001</span></a> <span class="k">else</span><span class="p">:</span> +</span><span id="Tokenizer-1002"><a href="#Tokenizer-1002"><span class="linenos">1002</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">NUMBER</span><span class="p">)</span> +</span><span id="Tokenizer-1003"><a href="#Tokenizer-1003"><span class="linenos">1003</span></a> +</span><span id="Tokenizer-1004"><a href="#Tokenizer-1004"><span class="linenos">1004</span></a> <span class="k">def</span> <span class="nf">_scan_bits</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> +</span><span id="Tokenizer-1005"><a href="#Tokenizer-1005"><span class="linenos">1005</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="Tokenizer-1006"><a href="#Tokenizer-1006"><span class="linenos">1006</span></a> <span class="n">value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extract_value</span><span class="p">()</span> +</span><span id="Tokenizer-1007"><a href="#Tokenizer-1007"><span class="linenos">1007</span></a> <span class="k">try</span><span class="p">:</span> +</span><span id="Tokenizer-1008"><a href="#Tokenizer-1008"><span class="linenos">1008</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">BIT_STRING</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="nb">int</span><span class="p">(</span><span class="n">value</span><span class="p">,</span><span class="w"> </span><span class="mi">2</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> +</span><span id="Tokenizer-1009"><a href="#Tokenizer-1009"><span class="linenos">1009</span></a> <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span> +</span><span id="Tokenizer-1010"><a href="#Tokenizer-1010"><span class="linenos">1010</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">IDENTIFIER</span><span class="p">)</span> +</span><span id="Tokenizer-1011"><a href="#Tokenizer-1011"><span class="linenos">1011</span></a> +</span><span id="Tokenizer-1012"><a href="#Tokenizer-1012"><span class="linenos">1012</span></a> <span class="k">def</span> <span class="nf">_scan_hex</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> +</span><span id="Tokenizer-1013"><a href="#Tokenizer-1013"><span class="linenos">1013</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="Tokenizer-1014"><a href="#Tokenizer-1014"><span class="linenos">1014</span></a> <span class="n">value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extract_value</span><span class="p">()</span> +</span><span id="Tokenizer-1015"><a href="#Tokenizer-1015"><span class="linenos">1015</span></a> <span class="k">try</span><span class="p">:</span> +</span><span id="Tokenizer-1016"><a href="#Tokenizer-1016"><span class="linenos">1016</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">HEX_STRING</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="nb">int</span><span class="p">(</span><span class="n">value</span><span class="p">,</span><span class="w"> </span><span class="mi">16</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> +</span><span id="Tokenizer-1017"><a href="#Tokenizer-1017"><span class="linenos">1017</span></a> <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span> +</span><span id="Tokenizer-1018"><a href="#Tokenizer-1018"><span class="linenos">1018</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">IDENTIFIER</span><span class="p">)</span> +</span><span id="Tokenizer-1019"><a href="#Tokenizer-1019"><span class="linenos">1019</span></a> +</span><span id="Tokenizer-1020"><a href="#Tokenizer-1020"><span class="linenos">1020</span></a> <span class="k">def</span> <span class="nf">_extract_value</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> +</span><span id="Tokenizer-1021"><a href="#Tokenizer-1021"><span class="linenos">1021</span></a> <span class="k">while</span> <span class="kc">True</span><span class="p">:</span> +</span><span id="Tokenizer-1022"><a href="#Tokenizer-1022"><span class="linenos">1022</span></a> <span class="n">char</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-1023"><a href="#Tokenizer-1023"><span class="linenos">1023</span></a> <span class="k">if</span> <span class="n">char</span> <span class="ow">and</span> <span class="n">char</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">SINGLE_TOKENS</span><span class="p">:</span> +</span><span id="Tokenizer-1024"><a href="#Tokenizer-1024"><span class="linenos">1024</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="Tokenizer-1025"><a href="#Tokenizer-1025"><span class="linenos">1025</span></a> <span class="k">else</span><span class="p">:</span> +</span><span id="Tokenizer-1026"><a href="#Tokenizer-1026"><span class="linenos">1026</span></a> <span class="k">break</span> +</span><span id="Tokenizer-1027"><a href="#Tokenizer-1027"><span class="linenos">1027</span></a> +</span><span id="Tokenizer-1028"><a href="#Tokenizer-1028"><span class="linenos">1028</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_text</span> +</span><span id="Tokenizer-1029"><a href="#Tokenizer-1029"><span class="linenos">1029</span></a> +</span><span id="Tokenizer-1030"><a href="#Tokenizer-1030"><span class="linenos">1030</span></a> <span class="k">def</span> <span class="nf">_scan_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">quote</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> +</span><span id="Tokenizer-1031"><a href="#Tokenizer-1031"><span class="linenos">1031</span></a> <span class="n">quote_end</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_QUOTES</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">quote</span><span class="p">)</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-1032"><a href="#Tokenizer-1032"><span class="linenos">1032</span></a> <span class="k">if</span> <span class="n">quote_end</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> +</span><span id="Tokenizer-1033"><a href="#Tokenizer-1033"><span class="linenos">1033</span></a> <span class="k">return</span> <span class="kc">False</span> +</span><span id="Tokenizer-1034"><a href="#Tokenizer-1034"><span class="linenos">1034</span></a> +</span><span id="Tokenizer-1035"><a href="#Tokenizer-1035"><span class="linenos">1035</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">quote</span><span class="p">))</span> +</span><span id="Tokenizer-1036"><a href="#Tokenizer-1036"><span class="linenos">1036</span></a> <span class="n">text</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extract_string</span><span class="p">(</span><span class="n">quote_end</span><span class="p">)</span> +</span><span id="Tokenizer-1037"><a href="#Tokenizer-1037"><span class="linenos">1037</span></a> <span class="n">text</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ENCODE</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ENCODE</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">ENCODE</span> <span class="k">else</span> <span class="n">text</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-1038"><a href="#Tokenizer-1038"><span class="linenos">1038</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">NATIONAL</span> <span class="k">if</span> <span class="n">quote</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"N"</span> <span class="k">else</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">STRING</span><span class="p">,</span> <span class="n">text</span><span class="p">)</span> +</span><span id="Tokenizer-1039"><a href="#Tokenizer-1039"><span class="linenos">1039</span></a> <span class="k">return</span> <span class="kc">True</span> +</span><span id="Tokenizer-1040"><a href="#Tokenizer-1040"><span class="linenos">1040</span></a> +</span><span id="Tokenizer-1041"><a href="#Tokenizer-1041"><span class="linenos">1041</span></a> <span class="c1"># X'1234, b'0110', E'\\\\\' etc.</span> +</span><span id="Tokenizer-1042"><a href="#Tokenizer-1042"><span class="linenos">1042</span></a> <span class="k">def</span> <span class="nf">_scan_formatted_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">string_start</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> +</span><span id="Tokenizer-1043"><a href="#Tokenizer-1043"><span class="linenos">1043</span></a> <span class="k">if</span> <span class="n">string_start</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_HEX_STRINGS</span><span class="p">:</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-1044"><a href="#Tokenizer-1044"><span class="linenos">1044</span></a> <span class="n">delimiters</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_HEX_STRINGS</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-1045"><a href="#Tokenizer-1045"><span class="linenos">1045</span></a> <span class="n">token_type</span> <span class="o">=</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">HEX_STRING</span> +</span><span id="Tokenizer-1046"><a href="#Tokenizer-1046"><span class="linenos">1046</span></a> <span class="n">base</span> <span class="o">=</span> <span class="mi">16</span> +</span><span id="Tokenizer-1047"><a href="#Tokenizer-1047"><span class="linenos">1047</span></a> <span class="k">elif</span> <span class="n">string_start</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_BIT_STRINGS</span><span class="p">:</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-1048"><a href="#Tokenizer-1048"><span class="linenos">1048</span></a> <span class="n">delimiters</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_BIT_STRINGS</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-1049"><a href="#Tokenizer-1049"><span class="linenos">1049</span></a> <span class="n">token_type</span> <span class="o">=</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">BIT_STRING</span> +</span><span id="Tokenizer-1050"><a href="#Tokenizer-1050"><span class="linenos">1050</span></a> <span class="n">base</span> <span class="o">=</span> <span class="mi">2</span> +</span><span id="Tokenizer-1051"><a href="#Tokenizer-1051"><span class="linenos">1051</span></a> <span class="k">elif</span> <span class="n">string_start</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_BYTE_STRINGS</span><span class="p">:</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-1052"><a href="#Tokenizer-1052"><span class="linenos">1052</span></a> <span class="n">delimiters</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_BYTE_STRINGS</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-1053"><a href="#Tokenizer-1053"><span class="linenos">1053</span></a> <span class="n">token_type</span> <span class="o">=</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">BYTE_STRING</span> +</span><span id="Tokenizer-1054"><a href="#Tokenizer-1054"><span class="linenos">1054</span></a> <span class="n">base</span> <span class="o">=</span> <span class="kc">None</span> +</span><span id="Tokenizer-1055"><a href="#Tokenizer-1055"><span class="linenos">1055</span></a> <span class="k">else</span><span class="p">:</span> +</span><span id="Tokenizer-1056"><a href="#Tokenizer-1056"><span class="linenos">1056</span></a> <span class="k">return</span> <span class="kc">False</span> +</span><span id="Tokenizer-1057"><a href="#Tokenizer-1057"><span class="linenos">1057</span></a> +</span><span id="Tokenizer-1058"><a href="#Tokenizer-1058"><span class="linenos">1058</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">string_start</span><span class="p">))</span> +</span><span id="Tokenizer-1059"><a href="#Tokenizer-1059"><span class="linenos">1059</span></a> <span class="n">string_end</span> <span class="o">=</span> <span class="n">delimiters</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">string_start</span><span class="p">)</span> +</span><span id="Tokenizer-1060"><a href="#Tokenizer-1060"><span class="linenos">1060</span></a> <span class="n">text</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extract_string</span><span class="p">(</span><span class="n">string_end</span><span class="p">)</span> +</span><span id="Tokenizer-1061"><a href="#Tokenizer-1061"><span class="linenos">1061</span></a> +</span><span id="Tokenizer-1062"><a href="#Tokenizer-1062"><span class="linenos">1062</span></a> <span class="k">if</span> <span class="n">base</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> +</span><span id="Tokenizer-1063"><a href="#Tokenizer-1063"><span class="linenos">1063</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">token_type</span><span class="p">,</span> <span class="n">text</span><span class="p">)</span> +</span><span id="Tokenizer-1064"><a href="#Tokenizer-1064"><span class="linenos">1064</span></a> <span class="k">else</span><span class="p">:</span> +</span><span id="Tokenizer-1065"><a href="#Tokenizer-1065"><span class="linenos">1065</span></a> <span class="k">try</span><span class="p">:</span> +</span><span id="Tokenizer-1066"><a href="#Tokenizer-1066"><span class="linenos">1066</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">token_type</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="nb">int</span><span class="p">(</span><span class="n">text</span><span class="p">,</span><span class="w"> </span><span class="n">base</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> +</span><span id="Tokenizer-1067"><a href="#Tokenizer-1067"><span class="linenos">1067</span></a> <span class="k">except</span><span class="p">:</span> +</span><span id="Tokenizer-1068"><a href="#Tokenizer-1068"><span class="linenos">1068</span></a> <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span> +</span><span id="Tokenizer-1069"><a href="#Tokenizer-1069"><span class="linenos">1069</span></a> <span class="sa">f</span><span class="s2">"Numeric string contains invalid characters from </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_line</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_start</span><span class="si">}</span><span class="s2">"</span> +</span><span id="Tokenizer-1070"><a href="#Tokenizer-1070"><span class="linenos">1070</span></a> <span class="p">)</span> +</span><span id="Tokenizer-1071"><a href="#Tokenizer-1071"><span class="linenos">1071</span></a> +</span><span id="Tokenizer-1072"><a href="#Tokenizer-1072"><span class="linenos">1072</span></a> <span class="k">return</span> <span class="kc">True</span> +</span><span id="Tokenizer-1073"><a href="#Tokenizer-1073"><span class="linenos">1073</span></a> +</span><span id="Tokenizer-1074"><a href="#Tokenizer-1074"><span class="linenos">1074</span></a> <span class="k">def</span> <span class="nf">_scan_identifier</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">identifier_end</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> +</span><span id="Tokenizer-1075"><a href="#Tokenizer-1075"><span class="linenos">1075</span></a> <span class="n">text</span> <span class="o">=</span> <span class="s2">""</span> +</span><span id="Tokenizer-1076"><a href="#Tokenizer-1076"><span class="linenos">1076</span></a> <span class="n">identifier_end_is_escape</span> <span class="o">=</span> <span class="n">identifier_end</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_IDENTIFIER_ESCAPES</span> +</span><span id="Tokenizer-1077"><a href="#Tokenizer-1077"><span class="linenos">1077</span></a> +</span><span id="Tokenizer-1078"><a href="#Tokenizer-1078"><span class="linenos">1078</span></a> <span class="k">while</span> <span class="kc">True</span><span class="p">:</span> +</span><span id="Tokenizer-1079"><a href="#Tokenizer-1079"><span class="linenos">1079</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_end</span><span class="p">:</span> +</span><span id="Tokenizer-1080"><a href="#Tokenizer-1080"><span class="linenos">1080</span></a> <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Missing </span><span class="si">{</span><span class="n">identifier_end</span><span class="si">}</span><span class="s2"> from </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_line</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_start</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> +</span><span id="Tokenizer-1081"><a href="#Tokenizer-1081"><span class="linenos">1081</span></a> +</span><span id="Tokenizer-1082"><a href="#Tokenizer-1082"><span class="linenos">1082</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="Tokenizer-1083"><a href="#Tokenizer-1083"><span class="linenos">1083</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="o">==</span> <span class="n">identifier_end</span><span class="p">:</span> +</span><span id="Tokenizer-1084"><a href="#Tokenizer-1084"><span class="linenos">1084</span></a> <span class="k">if</span> <span class="n">identifier_end_is_escape</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="o">==</span> <span class="n">identifier_end</span><span class="p">:</span> +</span><span id="Tokenizer-1085"><a href="#Tokenizer-1085"><span class="linenos">1085</span></a> <span class="n">text</span> <span class="o">+=</span> <span class="n">identifier_end</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-1086"><a href="#Tokenizer-1086"><span class="linenos">1086</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="Tokenizer-1087"><a href="#Tokenizer-1087"><span class="linenos">1087</span></a> <span class="k">continue</span> +</span><span id="Tokenizer-1088"><a href="#Tokenizer-1088"><span class="linenos">1088</span></a> +</span><span id="Tokenizer-1089"><a href="#Tokenizer-1089"><span class="linenos">1089</span></a> <span class="k">break</span> +</span><span id="Tokenizer-1090"><a href="#Tokenizer-1090"><span class="linenos">1090</span></a> +</span><span id="Tokenizer-1091"><a href="#Tokenizer-1091"><span class="linenos">1091</span></a> <span class="n">text</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-1092"><a href="#Tokenizer-1092"><span class="linenos">1092</span></a> +</span><span id="Tokenizer-1093"><a href="#Tokenizer-1093"><span class="linenos">1093</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span><span class="n">TokenType</span><span class="o">.</span><span class="n">IDENTIFIER</span><span class="p">,</span> <span class="n">text</span><span class="p">)</span> +</span><span id="Tokenizer-1094"><a href="#Tokenizer-1094"><span class="linenos">1094</span></a> +</span><span id="Tokenizer-1095"><a href="#Tokenizer-1095"><span class="linenos">1095</span></a> <span class="k">def</span> <span class="nf">_scan_var</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> +</span><span id="Tokenizer-1096"><a href="#Tokenizer-1096"><span class="linenos">1096</span></a> <span class="k">while</span> <span class="kc">True</span><span class="p">:</span> +</span><span id="Tokenizer-1097"><a href="#Tokenizer-1097"><span class="linenos">1097</span></a> <span class="n">char</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-1098"><a href="#Tokenizer-1098"><span class="linenos">1098</span></a> <span class="k">if</span> <span class="n">char</span> <span class="ow">and</span> <span class="n">char</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">SINGLE_TOKENS</span><span class="p">:</span> +</span><span id="Tokenizer-1099"><a href="#Tokenizer-1099"><span class="linenos">1099</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="Tokenizer-1100"><a href="#Tokenizer-1100"><span class="linenos">1100</span></a> <span class="k">else</span><span class="p">:</span> +</span><span id="Tokenizer-1101"><a href="#Tokenizer-1101"><span class="linenos">1101</span></a> <span class="k">break</span> +</span><span id="Tokenizer-1102"><a href="#Tokenizer-1102"><span class="linenos">1102</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_add</span><span class="p">(</span> +</span><span id="Tokenizer-1103"><a href="#Tokenizer-1103"><span class="linenos">1103</span></a> <span class="n">TokenType</span><span class="o">.</span><span class="n">VAR</span> +</span><span id="Tokenizer-1104"><a href="#Tokenizer-1104"><span class="linenos">1104</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prev_token_type</span> <span class="o">==</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">PARAMETER</span> +</span><span id="Tokenizer-1105"><a href="#Tokenizer-1105"><span class="linenos">1105</span></a> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">KEYWORDS</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_text</span><span class="o">.</span><span class="n">upper</span><span class="p">(),</span> <span class="n">TokenType</span><span class="o">.</span><span class="n">VAR</span><span class="p">)</span> +</span><span id="Tokenizer-1106"><a href="#Tokenizer-1106"><span class="linenos">1106</span></a> <span class="p">)</span> +</span><span id="Tokenizer-1107"><a href="#Tokenizer-1107"><span class="linenos">1107</span></a> +</span><span id="Tokenizer-1108"><a href="#Tokenizer-1108"><span class="linenos">1108</span></a> <span class="k">def</span> <span class="nf">_extract_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">delimiter</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> +</span><span id="Tokenizer-1109"><a href="#Tokenizer-1109"><span class="linenos">1109</span></a> <span class="n">text</span> <span class="o">=</span> <span class="s2">""</span> +</span><span id="Tokenizer-1110"><a href="#Tokenizer-1110"><span class="linenos">1110</span></a> <span class="n">delim_size</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">delimiter</span><span class="p">)</span> +</span><span id="Tokenizer-1111"><a href="#Tokenizer-1111"><span class="linenos">1111</span></a> +</span><span id="Tokenizer-1112"><a href="#Tokenizer-1112"><span class="linenos">1112</span></a> <span class="k">while</span> <span class="kc">True</span><span class="p">:</span> +</span><span id="Tokenizer-1113"><a href="#Tokenizer-1113"><span class="linenos">1113</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_STRING_ESCAPES</span> <span class="ow">and</span> <span class="p">(</span> +</span><span id="Tokenizer-1114"><a href="#Tokenizer-1114"><span class="linenos">1114</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="o">==</span> <span class="n">delimiter</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_STRING_ESCAPES</span> +</span><span id="Tokenizer-1115"><a href="#Tokenizer-1115"><span class="linenos">1115</span></a> <span class="p">):</span> +</span><span id="Tokenizer-1116"><a href="#Tokenizer-1116"><span class="linenos">1116</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="o">==</span> <span class="n">delimiter</span><span class="p">:</span> +</span><span id="Tokenizer-1117"><a href="#Tokenizer-1117"><span class="linenos">1117</span></a> <span class="n">text</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-1118"><a href="#Tokenizer-1118"><span class="linenos">1118</span></a> <span class="k">else</span><span class="p">:</span> +</span><span id="Tokenizer-1119"><a href="#Tokenizer-1119"><span class="linenos">1119</span></a> <span class="n">text</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">_peek</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-1120"><a href="#Tokenizer-1120"><span class="linenos">1120</span></a> +</span><span id="Tokenizer-1121"><a href="#Tokenizer-1121"><span class="linenos">1121</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_current</span> <span class="o">+</span> <span class="mi">1</span> <span class="o"><</span> <span class="bp">self</span><span class="o">.</span><span class="n">size</span><span class="p">:</span> +</span><span id="Tokenizer-1122"><a href="#Tokenizer-1122"><span class="linenos">1122</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span> +</span><span id="Tokenizer-1123"><a href="#Tokenizer-1123"><span class="linenos">1123</span></a> <span class="k">else</span><span class="p">:</span> +</span><span id="Tokenizer-1124"><a href="#Tokenizer-1124"><span class="linenos">1124</span></a> <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Missing </span><span class="si">{</span><span class="n">delimiter</span><span class="si">}</span><span class="s2"> from </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_line</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_current</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> +</span><span id="Tokenizer-1125"><a href="#Tokenizer-1125"><span class="linenos">1125</span></a> <span class="k">else</span><span class="p">:</span> +</span><span id="Tokenizer-1126"><a href="#Tokenizer-1126"><span class="linenos">1126</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_chars</span><span class="p">(</span><span class="n">delim_size</span><span class="p">)</span> <span class="o">==</span> <span class="n">delimiter</span><span class="p">:</span> +</span><span id="Tokenizer-1127"><a href="#Tokenizer-1127"><span class="linenos">1127</span></a> <span class="k">if</span> <span class="n">delim_size</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span> +</span><span id="Tokenizer-1128"><a href="#Tokenizer-1128"><span class="linenos">1128</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">(</span><span class="n">delim_size</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> +</span><span id="Tokenizer-1129"><a href="#Tokenizer-1129"><span class="linenos">1129</span></a> <span class="k">break</span> +</span><span id="Tokenizer-1130"><a href="#Tokenizer-1130"><span class="linenos">1130</span></a> +</span><span id="Tokenizer-1131"><a href="#Tokenizer-1131"><span class="linenos">1131</span></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_end</span><span class="p">:</span> +</span><span id="Tokenizer-1132"><a href="#Tokenizer-1132"><span class="linenos">1132</span></a> <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Missing </span><span class="si">{</span><span class="n">delimiter</span><span class="si">}</span><span class="s2"> from </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_line</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_start</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> +</span><span id="Tokenizer-1133"><a href="#Tokenizer-1133"><span class="linenos">1133</span></a> <span class="n">text</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_char</span> <span class="c1"># type: ignore</span> +</span><span id="Tokenizer-1134"><a href="#Tokenizer-1134"><span class="linenos">1134</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_advance</span><span class="p">()</span> +</span><span id="Tokenizer-1135"><a href="#Tokenizer-1135"><span class="linenos">1135</span></a> +</span><span id="Tokenizer-1136"><a href="#Tokenizer-1136"><span class="linenos">1136</span></a> <span class="k">return</span> <span class="n">text</span> </span></pre></div> |