From 1f403ad2197fc7442409f434ee574f3e6b46fb73 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sat, 4 May 2024 13:33:32 +0200 Subject: Adding upstream version 2.14.0+dfsg. Signed-off-by: Daniel Baumann --- .coveragerc | 3 + .dockerignore | 16 + .gitattributes | 1 + .github/workflows/build.yaml | 72 + .github/workflows/docs.yaml | 40 + .gitignore | 21 + AUTHORS | 264 + CHANGES | 2134 ++++++++ Contributing.md | 167 + LICENSE | 25 + MANIFEST.in | 5 + Makefile | 68 + README.rst | 93 + description.rst | 18 + doc/Makefile | 163 + doc/_static/demo-worker.js | 74 + doc/_static/demo.css | 89 + doc/_static/demo.js | 200 + doc/_static/favicon.ico | Bin 0 -> 16958 bytes doc/_static/github.png | Bin 0 -> 1127 bytes doc/_static/logo_new.png | Bin 0 -> 40944 bytes doc/_static/logo_only.png | Bin 0 -> 16424 bytes doc/_static/spinner.gif | Bin 0 -> 10771 bytes doc/_templates/demo.html | 97 + doc/_templates/demo_sidebar.html | 1 + doc/_templates/docssidebar.html | 3 + doc/_templates/index_with_try.html | 0 doc/_templates/indexsidebar.html | 18 + doc/_templates/styles.html | 55 + doc/_themes/pygments14/layout.html | 101 + doc/_themes/pygments14/localtoc.html | 17 + doc/_themes/pygments14/relations.html | 25 + doc/_themes/pygments14/static/bodybg.png | Bin 0 -> 51903 bytes doc/_themes/pygments14/static/docbg.png | Bin 0 -> 61296 bytes doc/_themes/pygments14/static/listitem.png | Bin 0 -> 207 bytes doc/_themes/pygments14/static/logo.png | Bin 0 -> 26933 bytes doc/_themes/pygments14/static/pocoo.png | Bin 0 -> 2154 bytes doc/_themes/pygments14/static/pygments14.css_t | 422 ++ doc/_themes/pygments14/theme.conf | 17 + doc/conf.py | 291 ++ doc/docs/api.rst | 360 ++ doc/docs/authors.rst | 4 + doc/docs/changelog.rst | 1 + doc/docs/cmdline.rst | 218 + doc/docs/filterdevelopment.rst | 75 + doc/docs/filters.rst | 48 + doc/docs/formatterdevelopment.rst | 169 + doc/docs/formatters.rst | 48 + doc/docs/index.rst | 64 + doc/docs/integrate.rst | 40 + doc/docs/java.rst | 70 + doc/docs/lexerdevelopment.rst | 748 +++ doc/docs/lexers.rst | 69 + doc/docs/moinmoin.rst | 39 + doc/docs/plugins.rst | 122 + doc/docs/quickstart.rst | 205 + doc/docs/rstdirective.rst | 22 + doc/docs/security.rst | 31 + doc/docs/styledevelopment.rst | 96 + doc/docs/styles.rst | 157 + doc/docs/terminal-sessions.rst | 46 + doc/docs/tokens.rst | 376 ++ doc/docs/unicode.rst | 58 + doc/download.rst | 39 + doc/examples/example.py | 14 + doc/faq.rst | 142 + doc/index.rst | 47 + doc/languages.rst | 18 + doc/make.bat | 190 + doc/pygmentize.1 | 112 + doc/pyodide/Dockerfile | 20 + doc/pyodide/meta.yaml | 8 + doc/styles.rst | 5 + external/autopygmentize | 145 + external/lasso-builtins-generator-9.lasso | 162 + external/lilypond-builtins-generator.ly | 391 ++ external/markdown-processor.py | 66 + external/moin-parser.py | 111 + external/pygments.bashcomp | 38 + external/rst-directive.py | 81 + external/scheme-builtins-generator.scm | 116 + pygments/__init__.py | 82 + pygments/__main__.py | 17 + pygments/cmdline.py | 668 +++ pygments/console.py | 70 + pygments/filter.py | 71 + pygments/filters/__init__.py | 940 ++++ pygments/formatter.py | 94 + pygments/formatters/__init__.py | 142 + pygments/formatters/_mapping.py | 23 + pygments/formatters/bbcode.py | 108 + pygments/formatters/groff.py | 170 + pygments/formatters/html.py | 991 ++++ pygments/formatters/img.py | 645 +++ pygments/formatters/irc.py | 154 + pygments/formatters/latex.py | 521 ++ pygments/formatters/other.py | 161 + pygments/formatters/pangomarkup.py | 83 + pygments/formatters/rtf.py | 146 + pygments/formatters/svg.py | 188 + pygments/formatters/terminal.py | 127 + pygments/formatters/terminal256.py | 338 ++ pygments/lexer.py | 883 ++++ pygments/lexers/__init__.py | 334 ++ pygments/lexers/_ada_builtins.py | 103 + pygments/lexers/_asy_builtins.py | 1644 ++++++ pygments/lexers/_cl_builtins.py | 231 + pygments/lexers/_cocoa_builtins.py | 75 + pygments/lexers/_csound_builtins.py | 1780 +++++++ pygments/lexers/_css_builtins.py | 558 ++ pygments/lexers/_julia_builtins.py | 411 ++ pygments/lexers/_lasso_builtins.py | 5326 ++++++++++++++++++++ pygments/lexers/_lilypond_builtins.py | 4886 ++++++++++++++++++ pygments/lexers/_lua_builtins.py | 285 ++ pygments/lexers/_mapping.py | 553 ++ pygments/lexers/_mql_builtins.py | 1171 +++++ pygments/lexers/_mysql_builtins.py | 1335 +++++ pygments/lexers/_openedge_builtins.py | 2600 ++++++++++ pygments/lexers/_php_builtins.py | 3325 ++++++++++++ pygments/lexers/_postgres_builtins.py | 684 +++ pygments/lexers/_qlik_builtins.py | 666 +++ pygments/lexers/_scheme_builtins.py | 1609 ++++++ pygments/lexers/_scilab_builtins.py | 3093 ++++++++++++ pygments/lexers/_sourcemod_builtins.py | 1151 +++++ pygments/lexers/_stan_builtins.py | 648 +++ pygments/lexers/_stata_builtins.py | 457 ++ pygments/lexers/_tsql_builtins.py | 1003 ++++ pygments/lexers/_usd_builtins.py | 112 + pygments/lexers/_vbscript_builtins.py | 279 + pygments/lexers/_vim_builtins.py | 1938 +++++++ pygments/lexers/actionscript.py | 245 + pygments/lexers/ada.py | 144 + pygments/lexers/agile.py | 23 + pygments/lexers/algebra.py | 302 ++ pygments/lexers/ambient.py | 76 + pygments/lexers/amdgpu.py | 53 + pygments/lexers/ampl.py | 88 + pygments/lexers/apdlexer.py | 447 ++ pygments/lexers/apl.py | 104 + pygments/lexers/archetype.py | 319 ++ pygments/lexers/arrow.py | 117 + pygments/lexers/arturo.py | 250 + pygments/lexers/asc.py | 55 + pygments/lexers/asm.py | 1037 ++++ pygments/lexers/automation.py | 381 ++ pygments/lexers/bare.py | 102 + pygments/lexers/basic.py | 665 +++ pygments/lexers/bdd.py | 58 + pygments/lexers/berry.py | 99 + pygments/lexers/bibtex.py | 159 + pygments/lexers/boa.py | 97 + pygments/lexers/business.py | 626 +++ pygments/lexers/c_cpp.py | 409 ++ pygments/lexers/c_like.py | 666 +++ pygments/lexers/capnproto.py | 75 + pygments/lexers/cddl.py | 173 + pygments/lexers/chapel.py | 136 + pygments/lexers/clean.py | 179 + pygments/lexers/comal.py | 80 + pygments/lexers/compiled.py | 34 + pygments/lexers/configs.py | 1174 +++++ pygments/lexers/console.py | 114 + pygments/lexers/cplint.py | 44 + pygments/lexers/crystal.py | 365 ++ pygments/lexers/csound.py | 468 ++ pygments/lexers/css.py | 602 +++ pygments/lexers/d.py | 258 + pygments/lexers/dalvik.py | 127 + pygments/lexers/data.py | 767 +++ pygments/lexers/devicetree.py | 109 + pygments/lexers/diff.py | 165 + pygments/lexers/dotnet.py | 729 +++ pygments/lexers/dsls.py | 981 ++++ pygments/lexers/dylan.py | 287 ++ pygments/lexers/ecl.py | 145 + pygments/lexers/eiffel.py | 69 + pygments/lexers/elm.py | 124 + pygments/lexers/elpi.py | 165 + pygments/lexers/email.py | 132 + pygments/lexers/erlang.py | 528 ++ pygments/lexers/esoteric.py | 301 ++ pygments/lexers/ezhil.py | 77 + pygments/lexers/factor.py | 364 ++ pygments/lexers/fantom.py | 251 + pygments/lexers/felix.py | 276 + pygments/lexers/fift.py | 67 + pygments/lexers/floscript.py | 82 + pygments/lexers/forth.py | 179 + pygments/lexers/fortran.py | 213 + pygments/lexers/foxpro.py | 427 ++ pygments/lexers/freefem.py | 894 ++++ pygments/lexers/func.py | 108 + pygments/lexers/functional.py | 20 + pygments/lexers/futhark.py | 106 + pygments/lexers/gcodelexer.py | 35 + pygments/lexers/gdscript.py | 188 + pygments/lexers/go.py | 98 + pygments/lexers/grammar_notation.py | 265 + pygments/lexers/graph.py | 105 + pygments/lexers/graphics.py | 797 +++ pygments/lexers/graphviz.py | 59 + pygments/lexers/gsql.py | 104 + pygments/lexers/haskell.py | 871 ++++ pygments/lexers/haxe.py | 937 ++++ pygments/lexers/hdl.py | 465 ++ pygments/lexers/hexdump.py | 102 + pygments/lexers/html.py | 605 +++ pygments/lexers/idl.py | 285 ++ pygments/lexers/igor.py | 420 ++ pygments/lexers/inferno.py | 96 + pygments/lexers/installers.py | 327 ++ pygments/lexers/int_fiction.py | 1382 +++++ pygments/lexers/iolang.py | 62 + pygments/lexers/j.py | 152 + pygments/lexers/javascript.py | 1588 ++++++ pygments/lexers/jmespath.py | 68 + pygments/lexers/jslt.py | 95 + pygments/lexers/jsonnet.py | 168 + pygments/lexers/julia.py | 294 ++ pygments/lexers/jvm.py | 1820 +++++++ pygments/lexers/kuin.py | 333 ++ pygments/lexers/lilypond.py | 226 + pygments/lexers/lisp.py | 2838 +++++++++++ pygments/lexers/macaulay2.py | 1739 +++++++ pygments/lexers/make.py | 209 + pygments/lexers/markup.py | 765 +++ pygments/lexers/math.py | 20 + pygments/lexers/matlab.py | 3308 ++++++++++++ pygments/lexers/maxima.py | 85 + pygments/lexers/meson.py | 140 + pygments/lexers/mime.py | 210 + pygments/lexers/minecraft.py | 394 ++ pygments/lexers/mips.py | 128 + pygments/lexers/ml.py | 960 ++++ pygments/lexers/modeling.py | 369 ++ pygments/lexers/modula2.py | 1580 ++++++ pygments/lexers/monte.py | 204 + pygments/lexers/mosel.py | 447 ++ pygments/lexers/ncl.py | 893 ++++ pygments/lexers/nimrod.py | 200 + pygments/lexers/nit.py | 64 + pygments/lexers/nix.py | 135 + pygments/lexers/oberon.py | 120 + pygments/lexers/objective.py | 505 ++ pygments/lexers/ooc.py | 85 + pygments/lexers/other.py | 40 + pygments/lexers/parasail.py | 79 + pygments/lexers/parsers.py | 801 +++ pygments/lexers/pascal.py | 641 +++ pygments/lexers/pawn.py | 202 + pygments/lexers/perl.py | 733 +++ pygments/lexers/phix.py | 364 ++ pygments/lexers/php.py | 319 ++ pygments/lexers/pointless.py | 71 + pygments/lexers/pony.py | 93 + pygments/lexers/praat.py | 304 ++ pygments/lexers/procfile.py | 42 + pygments/lexers/prolog.py | 304 ++ pygments/lexers/promql.py | 175 + pygments/lexers/python.py | 1204 +++++ pygments/lexers/q.py | 188 + pygments/lexers/qlik.py | 117 + pygments/lexers/qvt.py | 151 + pygments/lexers/r.py | 190 + pygments/lexers/rdf.py | 462 ++ pygments/lexers/rebol.py | 430 ++ pygments/lexers/resource.py | 84 + pygments/lexers/ride.py | 139 + pygments/lexers/rita.py | 43 + pygments/lexers/rnc.py | 67 + pygments/lexers/roboconf.py | 81 + pygments/lexers/robotframework.py | 552 ++ pygments/lexers/ruby.py | 523 ++ pygments/lexers/rust.py | 223 + pygments/lexers/sas.py | 227 + pygments/lexers/savi.py | 170 + pygments/lexers/scdoc.py | 79 + pygments/lexers/scripting.py | 1286 +++++ pygments/lexers/sgf.py | 60 + pygments/lexers/shell.py | 918 ++++ pygments/lexers/sieve.py | 78 + pygments/lexers/slash.py | 184 + pygments/lexers/smalltalk.py | 196 + pygments/lexers/smithy.py | 78 + pygments/lexers/smv.py | 78 + pygments/lexers/snobol.py | 82 + pygments/lexers/solidity.py | 87 + pygments/lexers/sophia.py | 103 + pygments/lexers/special.py | 116 + pygments/lexers/spice.py | 71 + pygments/lexers/sql.py | 838 +++ pygments/lexers/srcinfo.py | 62 + pygments/lexers/stata.py | 171 + pygments/lexers/supercollider.py | 95 + pygments/lexers/tal.py | 74 + pygments/lexers/tcl.py | 149 + pygments/lexers/teal.py | 89 + pygments/lexers/templates.py | 2300 +++++++++ pygments/lexers/teraterm.py | 326 ++ pygments/lexers/testing.py | 210 + pygments/lexers/text.py | 26 + pygments/lexers/textedit.py | 202 + pygments/lexers/textfmts.py | 431 ++ pygments/lexers/theorem.py | 484 ++ pygments/lexers/thingsdb.py | 116 + pygments/lexers/tlb.py | 57 + pygments/lexers/tnt.py | 271 + pygments/lexers/trafficscript.py | 51 + pygments/lexers/typoscript.py | 217 + pygments/lexers/ul4.py | 267 + pygments/lexers/unicon.py | 411 ++ pygments/lexers/urbi.py | 145 + pygments/lexers/usd.py | 90 + pygments/lexers/varnish.py | 189 + pygments/lexers/verification.py | 114 + pygments/lexers/web.py | 23 + pygments/lexers/webassembly.py | 120 + pygments/lexers/webidl.py | 299 ++ pygments/lexers/webmisc.py | 1010 ++++ pygments/lexers/whiley.py | 116 + pygments/lexers/wowtoc.py | 120 + pygments/lexers/wren.py | 99 + pygments/lexers/x10.py | 67 + pygments/lexers/xorg.py | 37 + pygments/lexers/yang.py | 104 + pygments/lexers/zig.py | 124 + pygments/modeline.py | 43 + pygments/plugin.py | 88 + pygments/regexopt.py | 91 + pygments/scanner.py | 104 + pygments/sphinxext.py | 217 + pygments/style.py | 197 + pygments/styles/__init__.py | 97 + pygments/styles/abap.py | 28 + pygments/styles/algol.py | 61 + pygments/styles/algol_nu.py | 61 + pygments/styles/arduino.py | 96 + pygments/styles/autumn.py | 62 + pygments/styles/borland.py | 48 + pygments/styles/bw.py | 47 + pygments/styles/colorful.py | 78 + pygments/styles/default.py | 71 + pygments/styles/dracula.py | 102 + pygments/styles/emacs.py | 70 + pygments/styles/friendly.py | 71 + pygments/styles/friendly_grayscale.py | 75 + pygments/styles/fruity.py | 41 + pygments/styles/gh_dark.py | 107 + pygments/styles/gruvbox.py | 109 + pygments/styles/igor.py | 27 + pygments/styles/inkpot.py | 67 + pygments/styles/lilypond.py | 56 + pygments/styles/lovelace.py | 94 + pygments/styles/manni.py | 74 + pygments/styles/material.py | 117 + pygments/styles/monokai.py | 106 + pygments/styles/murphy.py | 77 + pygments/styles/native.py | 65 + pygments/styles/nord.py | 150 + pygments/styles/onedark.py | 59 + pygments/styles/paraiso_dark.py | 119 + pygments/styles/paraiso_light.py | 119 + pygments/styles/pastie.py | 72 + pygments/styles/perldoc.py | 67 + pygments/styles/rainbow_dash.py | 88 + pygments/styles/rrt.py | 33 + pygments/styles/sas.py | 41 + pygments/styles/solarized.py | 136 + pygments/styles/staroffice.py | 26 + pygments/styles/stata_dark.py | 38 + pygments/styles/stata_light.py | 37 + pygments/styles/tango.py | 139 + pygments/styles/trac.py | 60 + pygments/styles/vim.py | 61 + pygments/styles/vs.py | 36 + pygments/styles/xcode.py | 48 + pygments/styles/zenburn.py | 78 + pygments/token.py | 213 + pygments/unistring.py | 153 + pygments/util.py | 308 ++ pyproject.toml | 6 + requirements.txt | 7 + scripts/check_crlf.py | 32 + scripts/check_repeated_token.py | 77 + scripts/check_sources.py | 201 + scripts/check_whitespace_token.py | 52 + scripts/count_token_references.py | 270 + scripts/debug_lexer.py | 306 ++ scripts/detect_missing_analyse_text.py | 48 + scripts/gen_mapfiles.py | 53 + scripts/get_css_properties.py | 33 + scripts/get_vimkw.py | 72 + scripts/pylintrc | 301 ++ scripts/release-checklist | 24 + scripts/update_contrasts.py | 21 + scripts/utility.py | 69 + scripts/vim2pygments.py | 932 ++++ setup.cfg | 54 + setup.py | 4 + tests/__init__.py | 7 + tests/conftest.py | 127 + tests/contrast/min_contrasts.json | 50 + tests/contrast/test_contrasts.py | 101 + tests/dtds/HTML4-f.dtd | 37 + tests/dtds/HTML4-s.dtd | 869 ++++ tests/dtds/HTML4.dcl | 88 + tests/dtds/HTML4.dtd | 1092 ++++ tests/dtds/HTML4.soc | 9 + tests/dtds/HTMLlat1.ent | 195 + tests/dtds/HTMLspec.ent | 77 + tests/dtds/HTMLsym.ent | 241 + ...s_step_1_start_1_special_0_anchor_filename.html | 4 + ...step_1_start_1_special_0_anchor_nofilename.html | 4 + ...step_1_start_1_special_0_noanchor_filename.html | 4 + ...ep_1_start_1_special_0_noanchor_nofilename.html | 4 + ...s_step_1_start_1_special_3_anchor_filename.html | 4 + ...step_1_start_1_special_3_anchor_nofilename.html | 4 + ...step_1_start_1_special_3_noanchor_filename.html | 4 + ...ep_1_start_1_special_3_noanchor_nofilename.html | 4 + ...s_step_1_start_8_special_0_anchor_filename.html | 4 + ...step_1_start_8_special_0_anchor_nofilename.html | 4 + ...step_1_start_8_special_0_noanchor_filename.html | 4 + ...ep_1_start_8_special_0_noanchor_nofilename.html | 4 + ...s_step_1_start_8_special_3_anchor_filename.html | 4 + ...step_1_start_8_special_3_anchor_nofilename.html | 4 + ...step_1_start_8_special_3_noanchor_filename.html | 4 + ...ep_1_start_8_special_3_noanchor_nofilename.html | 4 + ...s_step_2_start_1_special_0_anchor_filename.html | 4 + ...step_2_start_1_special_0_anchor_nofilename.html | 4 + ...step_2_start_1_special_0_noanchor_filename.html | 4 + ...ep_2_start_1_special_0_noanchor_nofilename.html | 4 + ...s_step_2_start_1_special_3_anchor_filename.html | 4 + ...step_2_start_1_special_3_anchor_nofilename.html | 4 + ...step_2_start_1_special_3_noanchor_filename.html | 4 + ...ep_2_start_1_special_3_noanchor_nofilename.html | 4 + ...s_step_2_start_8_special_0_anchor_filename.html | 4 + ...step_2_start_8_special_0_anchor_nofilename.html | 4 + ...step_2_start_8_special_0_noanchor_filename.html | 4 + ...ep_2_start_8_special_0_noanchor_nofilename.html | 4 + ...s_step_2_start_8_special_3_anchor_filename.html | 4 + ...step_2_start_8_special_3_anchor_nofilename.html | 4 + ...step_2_start_8_special_3_noanchor_filename.html | 4 + ...ep_2_start_8_special_3_noanchor_nofilename.html | 4 + ...s_step_1_start_1_special_0_anchor_filename.html | 4 + ...step_1_start_1_special_0_anchor_nofilename.html | 4 + ...step_1_start_1_special_0_noanchor_filename.html | 4 + ...ep_1_start_1_special_0_noanchor_nofilename.html | 4 + ...s_step_1_start_1_special_3_anchor_filename.html | 4 + ...step_1_start_1_special_3_anchor_nofilename.html | 4 + ...step_1_start_1_special_3_noanchor_filename.html | 4 + ...ep_1_start_1_special_3_noanchor_nofilename.html | 4 + ...s_step_1_start_8_special_0_anchor_filename.html | 4 + ...step_1_start_8_special_0_anchor_nofilename.html | 4 + ...step_1_start_8_special_0_noanchor_filename.html | 4 + ...ep_1_start_8_special_0_noanchor_nofilename.html | 4 + ...s_step_1_start_8_special_3_anchor_filename.html | 4 + ...step_1_start_8_special_3_anchor_nofilename.html | 4 + ...step_1_start_8_special_3_noanchor_filename.html | 4 + ...ep_1_start_8_special_3_noanchor_nofilename.html | 4 + ...s_step_2_start_1_special_0_anchor_filename.html | 4 + ...step_2_start_1_special_0_anchor_nofilename.html | 4 + ...step_2_start_1_special_0_noanchor_filename.html | 4 + ...ep_2_start_1_special_0_noanchor_nofilename.html | 4 + ...s_step_2_start_1_special_3_anchor_filename.html | 4 + ...step_2_start_1_special_3_anchor_nofilename.html | 4 + ...step_2_start_1_special_3_noanchor_filename.html | 4 + ...ep_2_start_1_special_3_noanchor_nofilename.html | 4 + ...s_step_2_start_8_special_0_anchor_filename.html | 4 + ...step_2_start_8_special_0_anchor_nofilename.html | 4 + ...step_2_start_8_special_0_noanchor_filename.html | 4 + ...ep_2_start_8_special_0_noanchor_nofilename.html | 4 + ...s_step_2_start_8_special_3_anchor_filename.html | 4 + ...step_2_start_8_special_3_anchor_nofilename.html | 4 + ...step_2_start_8_special_3_noanchor_filename.html | 4 + ...ep_2_start_8_special_3_noanchor_nofilename.html | 4 + ...s_step_1_start_1_special_0_anchor_filename.html | 6 + ...step_1_start_1_special_0_anchor_nofilename.html | 6 + ...step_1_start_1_special_0_noanchor_filename.html | 6 + ...ep_1_start_1_special_0_noanchor_nofilename.html | 6 + ...s_step_1_start_1_special_3_anchor_filename.html | 6 + ...step_1_start_1_special_3_anchor_nofilename.html | 6 + ...step_1_start_1_special_3_noanchor_filename.html | 6 + ...ep_1_start_1_special_3_noanchor_nofilename.html | 6 + ...s_step_1_start_8_special_0_anchor_filename.html | 6 + ...step_1_start_8_special_0_anchor_nofilename.html | 6 + ...step_1_start_8_special_0_noanchor_filename.html | 6 + ...ep_1_start_8_special_0_noanchor_nofilename.html | 6 + ...s_step_1_start_8_special_3_anchor_filename.html | 6 + ...step_1_start_8_special_3_anchor_nofilename.html | 6 + ...step_1_start_8_special_3_noanchor_filename.html | 6 + ...ep_1_start_8_special_3_noanchor_nofilename.html | 6 + ...s_step_2_start_1_special_0_anchor_filename.html | 6 + ...step_2_start_1_special_0_anchor_nofilename.html | 6 + ...step_2_start_1_special_0_noanchor_filename.html | 6 + ...ep_2_start_1_special_0_noanchor_nofilename.html | 6 + ...s_step_2_start_1_special_3_anchor_filename.html | 6 + ...step_2_start_1_special_3_anchor_nofilename.html | 6 + ...step_2_start_1_special_3_noanchor_filename.html | 6 + ...ep_2_start_1_special_3_noanchor_nofilename.html | 6 + ...s_step_2_start_8_special_0_anchor_filename.html | 6 + ...step_2_start_8_special_0_anchor_nofilename.html | 6 + ...step_2_start_8_special_0_noanchor_filename.html | 6 + ...ep_2_start_8_special_0_noanchor_nofilename.html | 6 + ...s_step_2_start_8_special_3_anchor_filename.html | 6 + ...step_2_start_8_special_3_anchor_nofilename.html | 6 + ...step_2_start_8_special_3_noanchor_filename.html | 6 + ...ep_2_start_8_special_3_noanchor_nofilename.html | 6 + ...s_step_1_start_1_special_0_anchor_filename.html | 6 + ...step_1_start_1_special_0_anchor_nofilename.html | 6 + ...step_1_start_1_special_0_noanchor_filename.html | 6 + ...ep_1_start_1_special_0_noanchor_nofilename.html | 6 + ...s_step_1_start_1_special_3_anchor_filename.html | 6 + ...step_1_start_1_special_3_anchor_nofilename.html | 6 + ...step_1_start_1_special_3_noanchor_filename.html | 6 + ...ep_1_start_1_special_3_noanchor_nofilename.html | 6 + ...s_step_1_start_8_special_0_anchor_filename.html | 6 + ...step_1_start_8_special_0_anchor_nofilename.html | 6 + ...step_1_start_8_special_0_noanchor_filename.html | 6 + ...ep_1_start_8_special_0_noanchor_nofilename.html | 6 + ...s_step_1_start_8_special_3_anchor_filename.html | 6 + ...step_1_start_8_special_3_anchor_nofilename.html | 6 + ...step_1_start_8_special_3_noanchor_filename.html | 6 + ...ep_1_start_8_special_3_noanchor_nofilename.html | 6 + ...s_step_2_start_1_special_0_anchor_filename.html | 6 + ...step_2_start_1_special_0_anchor_nofilename.html | 6 + ...step_2_start_1_special_0_noanchor_filename.html | 6 + ...ep_2_start_1_special_0_noanchor_nofilename.html | 6 + ...s_step_2_start_1_special_3_anchor_filename.html | 6 + ...step_2_start_1_special_3_anchor_nofilename.html | 6 + ...step_2_start_1_special_3_noanchor_filename.html | 6 + ...ep_2_start_1_special_3_noanchor_nofilename.html | 6 + ...s_step_2_start_8_special_0_anchor_filename.html | 6 + ...step_2_start_8_special_0_anchor_nofilename.html | 6 + ...step_2_start_8_special_0_noanchor_filename.html | 6 + ...ep_2_start_8_special_0_noanchor_nofilename.html | 6 + ...s_step_2_start_8_special_3_anchor_filename.html | 6 + ...step_2_start_8_special_3_anchor_nofilename.html | 6 + ...step_2_start_8_special_3_noanchor_filename.html | 6 + ...ep_2_start_8_special_3_noanchor_nofilename.html | 6 + .../snippets/apacheconf/test_directive_no_args.txt | 12 + .../apacheconf/test_fix_lock_absolute_path.txt | 8 + tests/snippets/apacheconf/test_include_globs.txt | 8 + ...test_malformed_scoped_directive_closing_tag.txt | 19 + .../apacheconf/test_multi_include_globs.txt | 8 + .../apacheconf/test_multi_include_globs_root.txt | 8 + .../apacheconf/test_multiline_argument.txt | 20 + .../snippets/apacheconf/test_multiline_comment.txt | 12 + .../apacheconf/test_normal_scoped_directive.txt | 14 + tests/snippets/apl/test_leading_underscore.txt | 26 + tests/snippets/asm/test_cpuid.txt | 9 + tests/snippets/bibtex/test_basic_bst.txt | 54 + tests/snippets/bibtex/test_comment.txt | 7 + tests/snippets/bibtex/test_entry.txt | 63 + tests/snippets/bibtex/test_mismatched_brace.txt | 10 + tests/snippets/bibtex/test_missing_body.txt | 10 + tests/snippets/bibtex/test_preamble.txt | 11 + tests/snippets/bibtex/test_string.txt | 15 + tests/snippets/c/test_comment_end.txt | 31 + tests/snippets/c/test_function_comments.txt | 409 ++ tests/snippets/c/test_label.txt | 31 + .../c/test_label_followed_by_statement.txt | 35 + tests/snippets/c/test_label_space_before_colon.txt | 32 + tests/snippets/c/test_numbers.txt | 20 + tests/snippets/c/test_preproc_file.txt | 17 + tests/snippets/c/test_preproc_file2.txt | 17 + tests/snippets/c/test_preproc_file3.txt | 18 + tests/snippets/c/test_preproc_file4.txt | 13 + tests/snippets/c/test_preproc_file5.txt | 19 + .../snippets/c/test_string_resembling_decl_end.txt | 41 + tests/snippets/c/test_switch.txt | 56 + .../snippets/c/test_switch_space_before_colon.txt | 58 + tests/snippets/cfm/test_basic_comment.txt | 8 + tests/snippets/cfm/test_nested_comment.txt | 12 + .../coffeescript/test_beware_infinite_loop.txt | 14 + tests/snippets/coffeescript/test_mixed_slashes.txt | 13 + tests/snippets/conftest.py | 32 + tests/snippets/console/fake_ps2_prompt.txt | 14 + tests/snippets/console/prompt_in_output.txt | 21 + tests/snippets/console/ps2_prompt.txt | 15 + .../snippets/console/test_comment_after_prompt.txt | 6 + .../console/test_newline_in_echo_no_ps2.txt | 16 + .../snippets/console/test_newline_in_echo_ps2.txt | 16 + .../snippets/console/test_newline_in_ls_no_ps2.txt | 16 + tests/snippets/console/test_newline_in_ls_ps2.txt | 16 + tests/snippets/console/test_virtualenv.txt | 11 + tests/snippets/coq/test_unicode.txt | 15 + tests/snippets/cpp/test_good_comment.txt | 6 + tests/snippets/cpp/test_open_comment.txt | 5 + tests/snippets/cpp/test_unicode_identifiers.txt | 146 + tests/snippets/crystal/test_annotation.txt | 16 + tests/snippets/crystal/test_array_access.txt | 11 + tests/snippets/crystal/test_chars.txt | 25 + .../snippets/crystal/test_constant_and_module.txt | 14 + .../snippets/crystal/test_escaped_bracestring.txt | 19 + .../crystal/test_escaped_interpolation.txt | 9 + .../crystal/test_interpolation_nested_curly.txt | 56 + tests/snippets/crystal/test_lib.txt | 58 + tests/snippets/crystal/test_macro.txt | 76 + tests/snippets/crystal/test_operator_methods.txt | 18 + tests/snippets/crystal/test_percent_strings.txt | 41 + .../crystal/test_percent_strings_special.txt | 31 + tests/snippets/crystal/test_pseudo_builtins.txt | 20 + tests/snippets/crystal/test_pseudo_keywords.txt | 50 + tests/snippets/crystal/test_range_syntax1.txt | 8 + tests/snippets/crystal/test_range_syntax2.txt | 10 + tests/snippets/csound/test_braced_strings.txt | 11 + tests/snippets/csound/test_comments.txt | 16 + tests/snippets/csound/test_escape_sequences.txt | 122 + .../test_function_like_macro_definitions.txt | 44 + .../snippets/csound/test_function_like_macros.txt | 40 + .../csound/test_global_value_identifiers.txt | 30 + tests/snippets/csound/test_goto_statements.txt | 176 + tests/snippets/csound/test_include_directives.txt | 14 + .../snippets/csound/test_includestr_directives.txt | 11 + tests/snippets/csound/test_instrument_blocks.txt | 42 + tests/snippets/csound/test_keywords.txt | 62 + tests/snippets/csound/test_labels.txt | 13 + .../csound/test_macro_preprocessor_directives.txt | 20 + tests/snippets/csound/test_name.txt | 9 + tests/snippets/csound/test_numbers.txt | 52 + .../csound/test_object_like_macro_definitions.txt | 30 + tests/snippets/csound/test_operators.txt | 114 + .../csound/test_other_preprocessor_directives.txt | 26 + .../test_printks_and_prints_escape_sequences.txt | 290 ++ tests/snippets/csound/test_quoted_strings.txt | 9 + .../snippets/csound/test_user_defined_opcodes.txt | 24 + tests/snippets/doscon/test_gt_only.txt | 11 + .../elpi/test_catastrophic_backtracking.txt | 6 + tests/snippets/elpi/test_chr.txt | 54 + tests/snippets/elpi/test_clause.txt | 67 + tests/snippets/elpi/test_namespace.txt | 35 + tests/snippets/elpi/test_pred.txt | 60 + tests/snippets/elpi/test_type.txt | 112 + tests/snippets/ezhil/test_function.txt | 100 + tests/snippets/ezhil/test_gcd_expr.txt | 21 + tests/snippets/ezhil/test_if_statement.txt | 28 + tests/snippets/ezhil/test_sum.txt | 8 + tests/snippets/fortran/test_string_cataback.txt | 112 + tests/snippets/gas/test_comments.txt | 29 + tests/snippets/gdscript/test_comment.txt | 6 + tests/snippets/gdscript/test_export_array.txt | 17 + .../snippets/gdscript/test_function_with_types.txt | 33 + tests/snippets/gdscript/test_inner_class.txt | 20 + tests/snippets/gdscript/test_multiline_string.txt | 8 + tests/snippets/gdscript/test_signal.txt | 15 + tests/snippets/gdscript/test_simple_function.txt | 22 + .../test_variable_declaration_and_assigment.txt | 12 + tests/snippets/haskell/test_promoted_names.txt | 10 + ...multiline-comment-catastrophic-backtracking.txt | 34 + .../http/test_application_calendar_xml.txt | 28 + tests/snippets/http/test_application_xml.txt | 28 + tests/snippets/http/test_http_status_line.txt | 12 + ...test_http_status_line_without_reason_phrase.txt | 10 + ..._status_line_without_reason_phrase_rfc_7230.txt | 11 + tests/snippets/idris/test_compiler_directive.txt | 20 + tests/snippets/idris/test_reserved_word.txt | 29 + tests/snippets/ini/test_indented_entries_1.txt | 16 + tests/snippets/ini/test_indented_entries_2.txt | 20 + tests/snippets/ini/test_indented_entries_3.txt | 20 + tests/snippets/j/test_deal_operator.txt | 8 + tests/snippets/j/test_deal_operator_fixed_seed.txt | 9 + tests/snippets/java/test_default.txt | 36 + tests/snippets/java/test_enhanced_for.txt | 22 + tests/snippets/java/test_multiline_string.txt | 185 + tests/snippets/java/test_multiline_string_only.txt | 46 + tests/snippets/java/test_numeric_literals.txt | 34 + tests/snippets/java/test_record.txt | 67 + tests/snippets/js/super.txt | 72 + tests/snippets/jslt/test_sample.txt | 83 + tests/snippets/json/test_basic.txt | 30 + tests/snippets/json/test_basic_bare.txt | 23 + tests/snippets/julia-repl/test_repl.txt | 51 + tests/snippets/julia/test_keywords.txt | 101 + tests/snippets/julia/test_macros.txt | 56 + tests/snippets/julia/test_names.txt | 148 + tests/snippets/julia/test_numbers.txt | 261 + tests/snippets/julia/test_operators.txt | 172 + tests/snippets/julia/test_strings.txt | 225 + tests/snippets/julia/test_symbols.txt | 78 + tests/snippets/julia/test_types.txt | 196 + tests/snippets/julia/test_unicode.txt | 37 + .../test_can_cope_generics_in_destructuring.txt | 27 + ...t_can_cope_with_backtick_names_in_functions.txt | 8 + ...pe_with_commas_and_dashes_in_backtick_Names.txt | 8 + .../kotlin/test_can_cope_with_destructuring.txt | 16 + .../kotlin/test_can_cope_with_generics.txt | 34 + tests/snippets/kotlin/test_modifier_keyword.txt | 18 + .../test_should_cope_with_multiline_comments.txt | 12 + .../snippets/kotlin/test_string_interpolation.txt | 35 + tests/snippets/less/test_single_line_comments.txt | 21 + .../snippets/mason/test_handles_tags_correctly.txt | 69 + .../matlab/test_classes_with_properties.txt | 105 + tests/snippets/matlab/test_command_mode.txt | 12 + .../matlab/test_comment_after_continuation.txt | 25 + tests/snippets/matlab/test_dot_operator.txt | 10 + .../matlab/test_keywords_ended_by_newline.txt | 36 + tests/snippets/matlab/test_line_continuation.txt | 25 + .../test_multiple_spaces_variable_assignment.txt | 13 + .../snippets/matlab/test_one_space_assignment.txt | 13 + .../matlab/test_operator_multiple_space.txt | 13 + tests/snippets/matlab/test_single_line.txt | 18 + .../matlabsession/test_wrong_continuation.txt | 18 + tests/snippets/mcfunction/commenting.txt | 173 + tests/snippets/mcfunction/coordinates.txt | 188 + tests/snippets/mcfunction/data.txt | 120 + tests/snippets/mcfunction/difficult_1.txt | 56 + tests/snippets/mcfunction/multiline.txt | 108 + tests/snippets/mcfunction/selectors.txt | 73 + tests/snippets/mcfunction/simple.txt | 92 + tests/snippets/md/test_bold_fenced_by_asterisk.txt | 15 + .../snippets/md/test_bold_fenced_by_underscore.txt | 15 + tests/snippets/md/test_bulleted_list_1.txt | 14 + tests/snippets/md/test_bulleted_list_2.txt | 14 + tests/snippets/md/test_bulleted_list_3.txt | 14 + tests/snippets/md/test_bulleted_list_4.txt | 19 + .../md/test_code_block_fenced_by_backticks.txt | 15 + .../snippets/md/test_code_block_with_language.txt | 16 + tests/snippets/md/test_escape_italics.txt | 23 + tests/snippets/md/test_inline_code.txt | 36 + tests/snippets/md/test_inline_code_after_block.txt | 19 + tests/snippets/md/test_inline_code_in_list.txt | 26 + tests/snippets/md/test_invalid_bold.txt | 31 + tests/snippets/md/test_invalid_italics.txt | 31 + tests/snippets/md/test_italics_and_bold.txt | 21 + .../md/test_italics_fenced_by_asterisk.txt | 15 + .../md/test_italics_fenced_by_underscore.txt | 15 + tests/snippets/md/test_italics_no_multiline.txt | 10 + tests/snippets/md/test_links.txt | 23 + tests/snippets/md/test_mentions.txt | 10 + tests/snippets/md/test_numbered_list.txt | 14 + tests/snippets/md/test_quote.txt | 10 + tests/snippets/md/test_reference_style_links.txt | 18 + tests/snippets/md/test_strikethrough.txt | 9 + tests/snippets/md/test_task_list.txt | 34 + tests/snippets/md/test_topics.txt | 10 + tests/snippets/mips/deprecated_substrings.txt | 34 + tests/snippets/mips/keyword_substrings.txt | 254 + tests/snippets/mips/variable_substrings.txt | 102 + tests/snippets/nasm/checkid.txt | 32 + .../objectivec/test_literal_number_bool.txt | 7 + .../test_literal_number_bool_expression.txt | 9 + .../objectivec/test_literal_number_expression.txt | 11 + .../objectivec/test_literal_number_int.txt | 9 + .../test_literal_number_nested_expression.txt | 15 + tests/snippets/objectivec/test_module_import.txt | 9 + tests/snippets/octave/test_multilinecomment.txt | 27 + tests/snippets/omg-idl/annotation_named_params.txt | 27 + tests/snippets/omg-idl/enumerators.txt | 18 + tests/snippets/peg/test_basic.txt | 17 + tests/snippets/peg/test_modified_strings.txt | 21 + tests/snippets/peg/test_operators.txt | 29 + tests/snippets/php/test_backslashes_in_strings.txt | 28 + tests/snippets/php/test_string_escaping_run.txt | 16 + .../snippets/powershell/test_colon_punctuation.txt | 35 + .../snippets/powershell/test_remoting_session.txt | 19 + tests/snippets/powershell/test_session.txt | 28 + .../snippets/praat/test_broken_unquoted_string.txt | 16 + tests/snippets/praat/test_function_call.txt | 20 + tests/snippets/praat/test_inline_if.txt | 27 + ...interpolated_indexed_numeric_with_precision.txt | 6 + ...t_interpolated_local_numeric_with_precision.txt | 6 + .../praat/test_interpolated_numeric_hash.txt | 6 + .../praat/test_interpolated_numeric_indexed.txt | 6 + .../test_interpolated_numeric_with_precision.txt | 6 + .../praat/test_interpolated_string_hash.txt | 6 + .../praat/test_interpolated_string_indexed.txt | 6 + .../snippets/praat/test_interpolation_boundary.txt | 14 + tests/snippets/praat/test_numeric_assignment.txt | 11 + tests/snippets/praat/test_string_assignment.txt | 12 + .../snippets/praat/test_string_escaped_quotes.txt | 13 + .../promql/test_complex_exp_single_quotes.txt | 35 + .../promql/test_expression_and_comment.txt | 15 + tests/snippets/promql/test_function_delta.txt | 19 + tests/snippets/promql/test_function_multi_line.txt | 80 + .../test_function_multi_line_with_offset.txt | 87 + .../promql/test_function_sum_with_args.txt | 19 + .../test_matching_operator_no_regex_match.txt | 16 + tests/snippets/promql/test_metric.txt | 6 + .../promql/test_metric_multiple_labels.txt | 19 + .../test_metric_multiple_labels_with_spaces.txt | 22 + tests/snippets/promql/test_metric_one_label.txt | 13 + tests/snippets/properties/test_comments.txt | 12 + .../properties/test_escaped_space_in_key.txt | 10 + .../properties/test_escaped_space_in_value.txt | 10 + tests/snippets/properties/test_just_key.txt | 6 + .../properties/test_just_key_with_space.txt | 6 + .../test_leading_whitespace_comments.txt | 6 + .../properties/test_space_delimited_kv_pair.txt | 8 + tests/snippets/pwsh-session/test_continuation.txt | 124 + tests/snippets/python/test_bytes_escape_codes.txt | 24 + tests/snippets/python/test_floats.txt | 75 + tests/snippets/python/test_fstring_01a.txt | 25 + tests/snippets/python/test_fstring_01b.txt | 25 + tests/snippets/python/test_fstring_02a.txt | 13 + tests/snippets/python/test_fstring_02b.txt | 13 + tests/snippets/python/test_fstring_03a.txt | 14 + tests/snippets/python/test_fstring_03b.txt | 14 + tests/snippets/python/test_fstring_04a.txt | 13 + tests/snippets/python/test_fstring_04b.txt | 13 + tests/snippets/python/test_fstring_05a.txt | 16 + tests/snippets/python/test_fstring_05b.txt | 16 + tests/snippets/python/test_fstring_06a.txt | 16 + tests/snippets/python/test_fstring_06b.txt | 16 + tests/snippets/python/test_fstring_07a.txt | 17 + tests/snippets/python/test_fstring_07b.txt | 17 + tests/snippets/python/test_fstring_08a.txt | 15 + tests/snippets/python/test_fstring_08b.txt | 15 + tests/snippets/python/test_fstring_09a.txt | 14 + tests/snippets/python/test_fstring_09b.txt | 14 + tests/snippets/python/test_fstring_10a.txt | 18 + tests/snippets/python/test_fstring_10b.txt | 18 + tests/snippets/python/test_fstring_11a.txt | 18 + tests/snippets/python/test_fstring_11b.txt | 18 + tests/snippets/python/test_fstring_12a.txt | 16 + tests/snippets/python/test_fstring_12b.txt | 16 + tests/snippets/python/test_fstring_13a.txt | 17 + tests/snippets/python/test_fstring_13b.txt | 17 + tests/snippets/python/test_fstring_14a.txt | 20 + tests/snippets/python/test_fstring_14b.txt | 20 + tests/snippets/python/test_fstring_15a.txt | 42 + tests/snippets/python/test_fstring_15b.txt | 42 + tests/snippets/python/test_fstring_16a.txt | 18 + tests/snippets/python/test_fstring_16b.txt | 18 + tests/snippets/python/test_fstring_17a.txt | 14 + tests/snippets/python/test_fstring_17b.txt | 14 + tests/snippets/python/test_fstring_18a.txt | 25 + tests/snippets/python/test_fstring_18b.txt | 25 + tests/snippets/python/test_fstring_19a.txt | 46 + tests/snippets/python/test_fstring_19b.txt | 46 + tests/snippets/python/test_fstring_20a.txt | 17 + tests/snippets/python/test_fstring_20b.txt | 17 + tests/snippets/python/test_fstring_21a.txt | 15 + tests/snippets/python/test_fstring_21b.txt | 15 + tests/snippets/python/test_fstring_22a.txt | 14 + tests/snippets/python/test_fstring_22b.txt | 14 + tests/snippets/python/test_fstring_23a.txt | 11 + tests/snippets/python/test_fstring_23b.txt | 11 + tests/snippets/python/test_fstring_24a.txt | 23 + tests/snippets/python/test_fstring_24b.txt | 23 + tests/snippets/python/test_fstring_25a.txt | 24 + tests/snippets/python/test_fstring_25b.txt | 24 + tests/snippets/python/test_fstring_26a.txt | 20 + tests/snippets/python/test_fstring_26b.txt | 20 + tests/snippets/python/test_fstring_27a.txt | 11 + tests/snippets/python/test_fstring_27b.txt | 11 + tests/snippets/python/test_fstring_28a.txt | 11 + tests/snippets/python/test_fstring_28b.txt | 11 + tests/snippets/python/test_fstring_29a.txt | 15 + tests/snippets/python/test_fstring_29b.txt | 15 + tests/snippets/python/test_fstring_30a.txt | 16 + tests/snippets/python/test_fstring_30b.txt | 16 + tests/snippets/python/test_fstring_31a.txt | 15 + tests/snippets/python/test_fstring_31b.txt | 15 + tests/snippets/python/test_fstring_32a.txt | 15 + tests/snippets/python/test_fstring_32b.txt | 15 + tests/snippets/python/test_fstring_33a.txt | 15 + tests/snippets/python/test_fstring_33b.txt | 15 + tests/snippets/python/test_fstring_34a.txt | 20 + tests/snippets/python/test_fstring_34b.txt | 20 + tests/snippets/python/test_fstring_35a.txt | 15 + tests/snippets/python/test_fstring_35b.txt | 15 + tests/snippets/python/test_fstring_36a.txt | 16 + tests/snippets/python/test_fstring_36b.txt | 16 + tests/snippets/python/test_needs_name.txt | 55 + tests/snippets/python/test_pep_515.txt | 28 + tests/snippets/python/test_raw_fstring.txt | 46 + tests/snippets/python/test_string_escape_codes.txt | 20 + tests/snippets/python/test_walrus_operator.txt | 21 + tests/snippets/python2/test_cls_builtin.txt | 34 + .../snippets/qbasic/test_keywords_with_dollar.txt | 22 + tests/snippets/r/test_call.txt | 12 + tests/snippets/r/test_custom_operator.txt | 10 + tests/snippets/r/test_dot_indexing.txt | 9 + tests/snippets/r/test_dot_name.txt | 10 + tests/snippets/r/test_indexing.txt | 9 + tests/snippets/r/test_name1.txt | 6 + tests/snippets/r/test_name2.txt | 8 + tests/snippets/r/test_name3.txt | 8 + tests/snippets/ruby/test_escaped_bracestring.txt | 19 + .../ruby/test_interpolation_nested_curly.txt | 56 + tests/snippets/ruby/test_operator_methods.txt | 9 + tests/snippets/ruby/test_range_syntax1.txt | 8 + tests/snippets/ruby/test_range_syntax2.txt | 8 + tests/snippets/ruby/test_range_syntax3.txt | 10 + tests/snippets/rust/test_attribute.txt | 12 + tests/snippets/rust/test_break.txt | 39 + tests/snippets/rust/test_rawstrings.txt | 117 + .../scala/test_colon_colon_function_name.txt | 33 + tests/snippets/scala/test_default_parameter.txt | 37 + tests/snippets/scala/test_end_val.txt | 8 + tests/snippets/scala/test_end_valx.txt | 8 + tests/snippets/scala/test_float_with_exponents.txt | 12 + .../snippets/scala/test_function_operator_name.txt | 18 + tests/snippets/scala/test_import_path.txt | 12 + .../scala/test_invalid_symbol_and_invalid_char.txt | 8 + tests/snippets/scala/test_open_soft_keyword.txt | 12 + tests/snippets/scala/test_package_name.txt | 11 + tests/snippets/scala/test_prepend_operator.txt | 10 + tests/snippets/scala/test_qualified_name.txt | 10 + tests/snippets/scala/test_qualified_name_class.txt | 10 + tests/snippets/scala/test_script_header.txt | 6 + .../snippets/scala/test_symbol_followed_by_op.txt | 7 + .../scala/test_symbol_name_ending_with_star.txt | 6 + tests/snippets/scala/test_underscore_name.txt | 12 + tests/snippets/scheme/keywords.txt | 43 + tests/snippets/scheme/numbers.txt | 169 + tests/snippets/scheme/strings.txt | 85 + tests/snippets/shell/test_array_nums.txt | 14 + .../shell/test_curly_no_escape_and_quotes.txt | 15 + tests/snippets/shell/test_curly_with_escape.txt | 13 + tests/snippets/shell/test_end_of_line_nums.txt | 15 + tests/snippets/shell/test_parsed_single.txt | 8 + tests/snippets/shell/test_short_variable_names.txt | 26 + .../test_prefixed_name_starting_with_number.txt | 8 + tests/snippets/smarty/test_nested_curly.txt | 18 + tests/snippets/snbt/json.txt | 43 + tests/snippets/snbt/literals.txt | 41 + tests/snippets/snbt/multiline.txt | 56 + tests/snippets/snbt/nesting.txt | 39 + tests/snippets/snbt/quoted_keys.txt | 29 + tests/snippets/systemverilog/test_basic.txt | 157 + tests/snippets/systemverilog/test_classes.txt | 89 + tests/snippets/systemverilog/test_numbers.txt | 158 + tests/snippets/systemverilog/test_operators.txt | 213 + tests/snippets/tcl/test_comma_and_at.txt | 131 + tests/snippets/tcl/test_vars.txt | 17 + tests/snippets/teal/test_comments.txt | 28 + tests/snippets/teal/test_literals.txt | 28 + tests/snippets/teal/test_strings.txt | 15 + tests/snippets/terraform/test_attributes.txt | 155 + tests/snippets/terraform/test_backend.txt | 44 + tests/snippets/terraform/test_comment.txt | 64 + tests/snippets/terraform/test_functions.txt | 56 + tests/snippets/terraform/test_heredoc.txt | 65 + tests/snippets/terraform/test_module.txt | 32 + tests/snippets/terraform/test_resource.txt | 211 + tests/snippets/terraform/test_types.txt | 94 + .../terraform/test_variable_declaration.txt | 41 + tests/snippets/terraform/test_variable_read.txt | 23 + .../test_prefixed_name_starting_with_number.txt | 8 + .../typescript/test_function_definition.txt | 18 + tests/snippets/unixconfig/etc_group.txt | 45 + tests/snippets/unixconfig/etc_passwd.txt | 86 + tests/snippets/unixconfig/etc_shadow.txt | 74 + tests/snippets/usd/test_attribute.txt | 174 + tests/snippets/usd/test_composition_arcs.txt | 101 + tests/snippets/usd/test_metadata.txt | 36 + tests/snippets/usd/test_numbers.txt | 21 + tests/snippets/usd/test_outer_match_at_sign.txt | 14 + tests/snippets/usd/test_outer_match_double.txt | 12 + tests/snippets/usd/test_outer_match_single.txt | 12 + tests/snippets/usd/test_string_multiple_line.txt | 20 + tests/snippets/usd/test_string_priority.txt | 10 + tests/snippets/usd/test_string_single_line.txt | 6 + tests/snippets/vbscript/test_floats.txt | 34 + tests/snippets/vbscript/test_floats_multiple.txt | 7 + tests/snippets/vbscript/test_integers.txt | 14 + tests/snippets/vbscript/test_invalid_character.txt | 10 + tests/snippets/vbscript/test_names.txt | 18 + .../snippets/vbscript/test_reject_almost_float.txt | 7 + .../snippets/vbscript/test_unterminated_string.txt | 7 + ...align_and_offset_accept_hexadecimal_numbers.txt | 14 + .../snippets/wat/test_comment_with_open_paren.txt | 10 + tests/snippets/wat/test_comment_with_semicolon.txt | 10 + tests/snippets/wat/test_i32_const_is_builtin.txt | 6 + tests/snippets/wat/test_multiline_comment.txt | 11 + tests/snippets/wat/test_nested_comment.txt | 14 + tests/snippets/wat/test_string_byte_escape.txt | 9 + tests/snippets/wat/test_string_with_escape.txt | 9 + tests/snippets/wat/test_variable_name_pattern.txt | 6 + tests/snippets/whiley/test_whiley_operator.txt | 10 + tests/snippets/wren/lonely-paren.txt | 10 + ...multiline-comment-catastrophic-backtracking.txt | 56 + tests/snippets/yaml/test_yaml.txt | 13 + tests/snippets/yaml/test_yaml_colon_in_key.txt | 11 + .../yaml/test_yaml_colon_in_key_double.txt | 11 + .../snippets/yaml/test_yaml_colon_in_key_start.txt | 11 + tests/snippets/yang/test_float_value.txt | 11 + tests/snippets/yang/test_integer_value.txt | 11 + tests/snippets/yang/test_namespace_1.txt | 11 + tests/snippets/yang/test_namespace_2.txt | 13 + tests/snippets/yang/test_revision_date.txt | 11 + tests/snippets/yang/test_string_value.txt | 11 + tests/support/empty.py | 0 tests/support/html_formatter.py | 5 + tests/support/python_lexer.py | 11 + tests/support/structural_diff.py | 37 + tests/support/tags | 36 + tests/test_basic_api.py | 351 ++ tests/test_cmdline.py | 324 ++ tests/test_coffeescript.py | 52 + tests/test_crystal.py | 80 + tests/test_data.py | 285 ++ tests/test_devicetree_lexer.py | 32 + tests/test_func.py | 44 + tests/test_groff_formatter.py | 40 + tests/test_guess.py | 184 + tests/test_html_formatter.py | 271 + tests/test_html_formatter_linenos_elements.py | 63 + tests/test_html_lexer.py | 131 + tests/test_inherit.py | 101 + tests/test_irc_formatter.py | 30 + tests/test_java.py | 40 + tests/test_javascript.py | 84 + tests/test_latex_formatter.py | 107 + tests/test_markdown_lexer.py | 178 + tests/test_modeline.py | 20 + tests/test_mysql.py | 273 + tests/test_pangomarkup_formatter.py | 44 + tests/test_perllexer.py | 190 + tests/test_procfile.py | 40 + tests/test_raw_token.py | 68 + tests/test_regexlexer.py | 65 + tests/test_regexopt.py | 102 + tests/test_robotframework_lexer.py | 38 + tests/test_rtf_formatter.py | 107 + tests/test_ruby.py | 54 + tests/test_sql.py | 115 + tests/test_templates.py | 130 + tests/test_terminal_formatter.py | 100 + tests/test_thingsdb.py | 36 + tests/test_tnt.py | 226 + tests/test_token.py | 51 + tests/test_unistring.py | 45 + tests/test_usd.py | 64 + tests/test_using_api.py | 39 + tests/test_util.py | 189 + tests/test_words.py | 366 ++ tox.ini | 15 + 1029 files changed, 149790 insertions(+) create mode 100644 .coveragerc create mode 100644 .dockerignore create mode 100644 .gitattributes create mode 100644 .github/workflows/build.yaml create mode 100644 .github/workflows/docs.yaml create mode 100644 .gitignore create mode 100644 AUTHORS create mode 100644 CHANGES create mode 100644 Contributing.md create mode 100644 LICENSE create mode 100644 MANIFEST.in create mode 100644 Makefile create mode 100644 README.rst create mode 100644 description.rst create mode 100644 doc/Makefile create mode 100644 doc/_static/demo-worker.js create mode 100644 doc/_static/demo.css create mode 100644 doc/_static/demo.js create mode 100644 doc/_static/favicon.ico create mode 100644 doc/_static/github.png create mode 100644 doc/_static/logo_new.png create mode 100644 doc/_static/logo_only.png create mode 100644 doc/_static/spinner.gif create mode 100644 doc/_templates/demo.html create mode 100644 doc/_templates/demo_sidebar.html create mode 100644 doc/_templates/docssidebar.html create mode 100644 doc/_templates/index_with_try.html create mode 100644 doc/_templates/indexsidebar.html create mode 100644 doc/_templates/styles.html create mode 100644 doc/_themes/pygments14/layout.html create mode 100644 doc/_themes/pygments14/localtoc.html create mode 100644 doc/_themes/pygments14/relations.html create mode 100644 doc/_themes/pygments14/static/bodybg.png create mode 100644 doc/_themes/pygments14/static/docbg.png create mode 100644 doc/_themes/pygments14/static/listitem.png create mode 100644 doc/_themes/pygments14/static/logo.png create mode 100644 doc/_themes/pygments14/static/pocoo.png create mode 100644 doc/_themes/pygments14/static/pygments14.css_t create mode 100644 doc/_themes/pygments14/theme.conf create mode 100644 doc/conf.py create mode 100644 doc/docs/api.rst create mode 100644 doc/docs/authors.rst create mode 100644 doc/docs/changelog.rst create mode 100644 doc/docs/cmdline.rst create mode 100644 doc/docs/filterdevelopment.rst create mode 100644 doc/docs/filters.rst create mode 100644 doc/docs/formatterdevelopment.rst create mode 100644 doc/docs/formatters.rst create mode 100644 doc/docs/index.rst create mode 100644 doc/docs/integrate.rst create mode 100644 doc/docs/java.rst create mode 100644 doc/docs/lexerdevelopment.rst create mode 100644 doc/docs/lexers.rst create mode 100644 doc/docs/moinmoin.rst create mode 100644 doc/docs/plugins.rst create mode 100644 doc/docs/quickstart.rst create mode 100644 doc/docs/rstdirective.rst create mode 100644 doc/docs/security.rst create mode 100644 doc/docs/styledevelopment.rst create mode 100644 doc/docs/styles.rst create mode 100644 doc/docs/terminal-sessions.rst create mode 100644 doc/docs/tokens.rst create mode 100644 doc/docs/unicode.rst create mode 100644 doc/download.rst create mode 100644 doc/examples/example.py create mode 100644 doc/faq.rst create mode 100644 doc/index.rst create mode 100644 doc/languages.rst create mode 100644 doc/make.bat create mode 100644 doc/pygmentize.1 create mode 100644 doc/pyodide/Dockerfile create mode 100644 doc/pyodide/meta.yaml create mode 100644 doc/styles.rst create mode 100755 external/autopygmentize create mode 100755 external/lasso-builtins-generator-9.lasso create mode 100644 external/lilypond-builtins-generator.ly create mode 100644 external/markdown-processor.py create mode 100644 external/moin-parser.py create mode 100644 external/pygments.bashcomp create mode 100644 external/rst-directive.py create mode 100644 external/scheme-builtins-generator.scm create mode 100644 pygments/__init__.py create mode 100644 pygments/__main__.py create mode 100644 pygments/cmdline.py create mode 100644 pygments/console.py create mode 100644 pygments/filter.py create mode 100644 pygments/filters/__init__.py create mode 100644 pygments/formatter.py create mode 100644 pygments/formatters/__init__.py create mode 100755 pygments/formatters/_mapping.py create mode 100644 pygments/formatters/bbcode.py create mode 100644 pygments/formatters/groff.py create mode 100644 pygments/formatters/html.py create mode 100644 pygments/formatters/img.py create mode 100644 pygments/formatters/irc.py create mode 100644 pygments/formatters/latex.py create mode 100644 pygments/formatters/other.py create mode 100644 pygments/formatters/pangomarkup.py create mode 100644 pygments/formatters/rtf.py create mode 100644 pygments/formatters/svg.py create mode 100644 pygments/formatters/terminal.py create mode 100644 pygments/formatters/terminal256.py create mode 100644 pygments/lexer.py create mode 100644 pygments/lexers/__init__.py create mode 100644 pygments/lexers/_ada_builtins.py create mode 100644 pygments/lexers/_asy_builtins.py create mode 100644 pygments/lexers/_cl_builtins.py create mode 100644 pygments/lexers/_cocoa_builtins.py create mode 100644 pygments/lexers/_csound_builtins.py create mode 100644 pygments/lexers/_css_builtins.py create mode 100644 pygments/lexers/_julia_builtins.py create mode 100644 pygments/lexers/_lasso_builtins.py create mode 100644 pygments/lexers/_lilypond_builtins.py create mode 100644 pygments/lexers/_lua_builtins.py create mode 100644 pygments/lexers/_mapping.py create mode 100644 pygments/lexers/_mql_builtins.py create mode 100644 pygments/lexers/_mysql_builtins.py create mode 100644 pygments/lexers/_openedge_builtins.py create mode 100644 pygments/lexers/_php_builtins.py create mode 100644 pygments/lexers/_postgres_builtins.py create mode 100644 pygments/lexers/_qlik_builtins.py create mode 100644 pygments/lexers/_scheme_builtins.py create mode 100644 pygments/lexers/_scilab_builtins.py create mode 100644 pygments/lexers/_sourcemod_builtins.py create mode 100644 pygments/lexers/_stan_builtins.py create mode 100644 pygments/lexers/_stata_builtins.py create mode 100644 pygments/lexers/_tsql_builtins.py create mode 100644 pygments/lexers/_usd_builtins.py create mode 100644 pygments/lexers/_vbscript_builtins.py create mode 100644 pygments/lexers/_vim_builtins.py create mode 100644 pygments/lexers/actionscript.py create mode 100644 pygments/lexers/ada.py create mode 100644 pygments/lexers/agile.py create mode 100644 pygments/lexers/algebra.py create mode 100644 pygments/lexers/ambient.py create mode 100644 pygments/lexers/amdgpu.py create mode 100644 pygments/lexers/ampl.py create mode 100644 pygments/lexers/apdlexer.py create mode 100644 pygments/lexers/apl.py create mode 100644 pygments/lexers/archetype.py create mode 100644 pygments/lexers/arrow.py create mode 100644 pygments/lexers/arturo.py create mode 100644 pygments/lexers/asc.py create mode 100644 pygments/lexers/asm.py create mode 100644 pygments/lexers/automation.py create mode 100644 pygments/lexers/bare.py create mode 100644 pygments/lexers/basic.py create mode 100644 pygments/lexers/bdd.py create mode 100644 pygments/lexers/berry.py create mode 100644 pygments/lexers/bibtex.py create mode 100644 pygments/lexers/boa.py create mode 100644 pygments/lexers/business.py create mode 100644 pygments/lexers/c_cpp.py create mode 100644 pygments/lexers/c_like.py create mode 100644 pygments/lexers/capnproto.py create mode 100644 pygments/lexers/cddl.py create mode 100644 pygments/lexers/chapel.py create mode 100644 pygments/lexers/clean.py create mode 100644 pygments/lexers/comal.py create mode 100644 pygments/lexers/compiled.py create mode 100644 pygments/lexers/configs.py create mode 100644 pygments/lexers/console.py create mode 100644 pygments/lexers/cplint.py create mode 100644 pygments/lexers/crystal.py create mode 100644 pygments/lexers/csound.py create mode 100644 pygments/lexers/css.py create mode 100644 pygments/lexers/d.py create mode 100644 pygments/lexers/dalvik.py create mode 100644 pygments/lexers/data.py create mode 100644 pygments/lexers/devicetree.py create mode 100644 pygments/lexers/diff.py create mode 100644 pygments/lexers/dotnet.py create mode 100644 pygments/lexers/dsls.py create mode 100644 pygments/lexers/dylan.py create mode 100644 pygments/lexers/ecl.py create mode 100644 pygments/lexers/eiffel.py create mode 100644 pygments/lexers/elm.py create mode 100644 pygments/lexers/elpi.py create mode 100644 pygments/lexers/email.py create mode 100644 pygments/lexers/erlang.py create mode 100644 pygments/lexers/esoteric.py create mode 100644 pygments/lexers/ezhil.py create mode 100644 pygments/lexers/factor.py create mode 100644 pygments/lexers/fantom.py create mode 100644 pygments/lexers/felix.py create mode 100644 pygments/lexers/fift.py create mode 100644 pygments/lexers/floscript.py create mode 100644 pygments/lexers/forth.py create mode 100644 pygments/lexers/fortran.py create mode 100644 pygments/lexers/foxpro.py create mode 100644 pygments/lexers/freefem.py create mode 100644 pygments/lexers/func.py create mode 100644 pygments/lexers/functional.py create mode 100644 pygments/lexers/futhark.py create mode 100644 pygments/lexers/gcodelexer.py create mode 100644 pygments/lexers/gdscript.py create mode 100644 pygments/lexers/go.py create mode 100644 pygments/lexers/grammar_notation.py create mode 100644 pygments/lexers/graph.py create mode 100644 pygments/lexers/graphics.py create mode 100644 pygments/lexers/graphviz.py create mode 100755 pygments/lexers/gsql.py create mode 100644 pygments/lexers/haskell.py create mode 100644 pygments/lexers/haxe.py create mode 100644 pygments/lexers/hdl.py create mode 100644 pygments/lexers/hexdump.py create mode 100644 pygments/lexers/html.py create mode 100644 pygments/lexers/idl.py create mode 100644 pygments/lexers/igor.py create mode 100644 pygments/lexers/inferno.py create mode 100644 pygments/lexers/installers.py create mode 100644 pygments/lexers/int_fiction.py create mode 100644 pygments/lexers/iolang.py create mode 100644 pygments/lexers/j.py create mode 100644 pygments/lexers/javascript.py create mode 100644 pygments/lexers/jmespath.py create mode 100644 pygments/lexers/jslt.py create mode 100644 pygments/lexers/jsonnet.py create mode 100644 pygments/lexers/julia.py create mode 100644 pygments/lexers/jvm.py create mode 100644 pygments/lexers/kuin.py create mode 100644 pygments/lexers/lilypond.py create mode 100644 pygments/lexers/lisp.py create mode 100644 pygments/lexers/macaulay2.py create mode 100644 pygments/lexers/make.py create mode 100644 pygments/lexers/markup.py create mode 100644 pygments/lexers/math.py create mode 100644 pygments/lexers/matlab.py create mode 100644 pygments/lexers/maxima.py create mode 100644 pygments/lexers/meson.py create mode 100644 pygments/lexers/mime.py create mode 100644 pygments/lexers/minecraft.py create mode 100644 pygments/lexers/mips.py create mode 100644 pygments/lexers/ml.py create mode 100644 pygments/lexers/modeling.py create mode 100644 pygments/lexers/modula2.py create mode 100644 pygments/lexers/monte.py create mode 100644 pygments/lexers/mosel.py create mode 100644 pygments/lexers/ncl.py create mode 100644 pygments/lexers/nimrod.py create mode 100644 pygments/lexers/nit.py create mode 100644 pygments/lexers/nix.py create mode 100644 pygments/lexers/oberon.py create mode 100644 pygments/lexers/objective.py create mode 100644 pygments/lexers/ooc.py create mode 100644 pygments/lexers/other.py create mode 100644 pygments/lexers/parasail.py create mode 100644 pygments/lexers/parsers.py create mode 100644 pygments/lexers/pascal.py create mode 100644 pygments/lexers/pawn.py create mode 100644 pygments/lexers/perl.py create mode 100644 pygments/lexers/phix.py create mode 100644 pygments/lexers/php.py create mode 100644 pygments/lexers/pointless.py create mode 100644 pygments/lexers/pony.py create mode 100644 pygments/lexers/praat.py create mode 100644 pygments/lexers/procfile.py create mode 100644 pygments/lexers/prolog.py create mode 100644 pygments/lexers/promql.py create mode 100644 pygments/lexers/python.py create mode 100644 pygments/lexers/q.py create mode 100644 pygments/lexers/qlik.py create mode 100644 pygments/lexers/qvt.py create mode 100644 pygments/lexers/r.py create mode 100644 pygments/lexers/rdf.py create mode 100644 pygments/lexers/rebol.py create mode 100644 pygments/lexers/resource.py create mode 100644 pygments/lexers/ride.py create mode 100644 pygments/lexers/rita.py create mode 100644 pygments/lexers/rnc.py create mode 100644 pygments/lexers/roboconf.py create mode 100644 pygments/lexers/robotframework.py create mode 100644 pygments/lexers/ruby.py create mode 100644 pygments/lexers/rust.py create mode 100644 pygments/lexers/sas.py create mode 100644 pygments/lexers/savi.py create mode 100644 pygments/lexers/scdoc.py create mode 100644 pygments/lexers/scripting.py create mode 100644 pygments/lexers/sgf.py create mode 100644 pygments/lexers/shell.py create mode 100644 pygments/lexers/sieve.py create mode 100644 pygments/lexers/slash.py create mode 100644 pygments/lexers/smalltalk.py create mode 100644 pygments/lexers/smithy.py create mode 100644 pygments/lexers/smv.py create mode 100644 pygments/lexers/snobol.py create mode 100644 pygments/lexers/solidity.py create mode 100644 pygments/lexers/sophia.py create mode 100644 pygments/lexers/special.py create mode 100644 pygments/lexers/spice.py create mode 100644 pygments/lexers/sql.py create mode 100644 pygments/lexers/srcinfo.py create mode 100644 pygments/lexers/stata.py create mode 100644 pygments/lexers/supercollider.py create mode 100644 pygments/lexers/tal.py create mode 100644 pygments/lexers/tcl.py create mode 100644 pygments/lexers/teal.py create mode 100644 pygments/lexers/templates.py create mode 100644 pygments/lexers/teraterm.py create mode 100644 pygments/lexers/testing.py create mode 100644 pygments/lexers/text.py create mode 100644 pygments/lexers/textedit.py create mode 100644 pygments/lexers/textfmts.py create mode 100644 pygments/lexers/theorem.py create mode 100644 pygments/lexers/thingsdb.py create mode 100644 pygments/lexers/tlb.py create mode 100644 pygments/lexers/tnt.py create mode 100644 pygments/lexers/trafficscript.py create mode 100644 pygments/lexers/typoscript.py create mode 100644 pygments/lexers/ul4.py create mode 100644 pygments/lexers/unicon.py create mode 100644 pygments/lexers/urbi.py create mode 100644 pygments/lexers/usd.py create mode 100644 pygments/lexers/varnish.py create mode 100644 pygments/lexers/verification.py create mode 100644 pygments/lexers/web.py create mode 100644 pygments/lexers/webassembly.py create mode 100644 pygments/lexers/webidl.py create mode 100644 pygments/lexers/webmisc.py create mode 100644 pygments/lexers/whiley.py create mode 100644 pygments/lexers/wowtoc.py create mode 100644 pygments/lexers/wren.py create mode 100644 pygments/lexers/x10.py create mode 100644 pygments/lexers/xorg.py create mode 100644 pygments/lexers/yang.py create mode 100644 pygments/lexers/zig.py create mode 100644 pygments/modeline.py create mode 100644 pygments/plugin.py create mode 100644 pygments/regexopt.py create mode 100644 pygments/scanner.py create mode 100644 pygments/sphinxext.py create mode 100644 pygments/style.py create mode 100644 pygments/styles/__init__.py create mode 100644 pygments/styles/abap.py create mode 100644 pygments/styles/algol.py create mode 100644 pygments/styles/algol_nu.py create mode 100644 pygments/styles/arduino.py create mode 100644 pygments/styles/autumn.py create mode 100644 pygments/styles/borland.py create mode 100644 pygments/styles/bw.py create mode 100644 pygments/styles/colorful.py create mode 100644 pygments/styles/default.py create mode 100644 pygments/styles/dracula.py create mode 100644 pygments/styles/emacs.py create mode 100644 pygments/styles/friendly.py create mode 100644 pygments/styles/friendly_grayscale.py create mode 100644 pygments/styles/fruity.py create mode 100644 pygments/styles/gh_dark.py create mode 100644 pygments/styles/gruvbox.py create mode 100644 pygments/styles/igor.py create mode 100644 pygments/styles/inkpot.py create mode 100644 pygments/styles/lilypond.py create mode 100644 pygments/styles/lovelace.py create mode 100644 pygments/styles/manni.py create mode 100644 pygments/styles/material.py create mode 100644 pygments/styles/monokai.py create mode 100644 pygments/styles/murphy.py create mode 100644 pygments/styles/native.py create mode 100644 pygments/styles/nord.py create mode 100644 pygments/styles/onedark.py create mode 100644 pygments/styles/paraiso_dark.py create mode 100644 pygments/styles/paraiso_light.py create mode 100644 pygments/styles/pastie.py create mode 100644 pygments/styles/perldoc.py create mode 100644 pygments/styles/rainbow_dash.py create mode 100644 pygments/styles/rrt.py create mode 100644 pygments/styles/sas.py create mode 100644 pygments/styles/solarized.py create mode 100644 pygments/styles/staroffice.py create mode 100644 pygments/styles/stata_dark.py create mode 100644 pygments/styles/stata_light.py create mode 100644 pygments/styles/tango.py create mode 100644 pygments/styles/trac.py create mode 100644 pygments/styles/vim.py create mode 100644 pygments/styles/vs.py create mode 100644 pygments/styles/xcode.py create mode 100644 pygments/styles/zenburn.py create mode 100644 pygments/token.py create mode 100644 pygments/unistring.py create mode 100644 pygments/util.py create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 scripts/check_crlf.py create mode 100755 scripts/check_repeated_token.py create mode 100755 scripts/check_sources.py create mode 100644 scripts/check_whitespace_token.py create mode 100755 scripts/count_token_references.py create mode 100755 scripts/debug_lexer.py create mode 100644 scripts/detect_missing_analyse_text.py create mode 100644 scripts/gen_mapfiles.py create mode 100644 scripts/get_css_properties.py create mode 100644 scripts/get_vimkw.py create mode 100644 scripts/pylintrc create mode 100644 scripts/release-checklist create mode 100755 scripts/update_contrasts.py create mode 100644 scripts/utility.py create mode 100755 scripts/vim2pygments.py create mode 100644 setup.cfg create mode 100755 setup.py create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/contrast/min_contrasts.json create mode 100644 tests/contrast/test_contrasts.py create mode 100644 tests/dtds/HTML4-f.dtd create mode 100644 tests/dtds/HTML4-s.dtd create mode 100644 tests/dtds/HTML4.dcl create mode 100644 tests/dtds/HTML4.dtd create mode 100644 tests/dtds/HTML4.soc create mode 100644 tests/dtds/HTMLlat1.ent create mode 100644 tests/dtds/HTMLspec.ent create mode 100644 tests/dtds/HTMLsym.ent create mode 100644 tests/html_linenos_expected_output/inline_cls_step_1_start_1_special_0_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_1_start_1_special_0_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_1_start_1_special_0_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_1_start_1_special_0_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_1_start_1_special_3_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_1_start_1_special_3_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_1_start_1_special_3_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_1_start_1_special_3_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_1_start_8_special_0_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_1_start_8_special_0_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_1_start_8_special_0_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_1_start_8_special_0_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_1_start_8_special_3_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_1_start_8_special_3_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_1_start_8_special_3_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_1_start_8_special_3_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_2_start_1_special_0_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_2_start_1_special_0_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_2_start_1_special_0_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_2_start_1_special_0_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_2_start_1_special_3_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_2_start_1_special_3_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_2_start_1_special_3_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_2_start_1_special_3_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_2_start_8_special_0_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_2_start_8_special_0_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_2_start_8_special_0_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_2_start_8_special_0_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_2_start_8_special_3_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_2_start_8_special_3_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_2_start_8_special_3_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_cls_step_2_start_8_special_3_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_1_start_1_special_0_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_1_start_1_special_0_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_1_start_1_special_0_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_1_start_1_special_0_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_1_start_1_special_3_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_1_start_1_special_3_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_1_start_1_special_3_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_1_start_1_special_3_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_1_start_8_special_0_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_1_start_8_special_0_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_1_start_8_special_0_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_1_start_8_special_0_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_1_start_8_special_3_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_1_start_8_special_3_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_1_start_8_special_3_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_1_start_8_special_3_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_2_start_1_special_0_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_2_start_1_special_0_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_2_start_1_special_0_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_2_start_1_special_0_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_2_start_1_special_3_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_2_start_1_special_3_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_2_start_1_special_3_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_2_start_1_special_3_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_2_start_8_special_0_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_2_start_8_special_0_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_2_start_8_special_0_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_2_start_8_special_0_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_2_start_8_special_3_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_2_start_8_special_3_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_2_start_8_special_3_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/inline_nocls_step_2_start_8_special_3_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_1_start_1_special_0_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_1_start_1_special_0_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_1_start_1_special_0_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_1_start_1_special_0_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_1_start_1_special_3_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_1_start_1_special_3_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_1_start_1_special_3_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_1_start_1_special_3_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_1_start_8_special_0_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_1_start_8_special_0_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_1_start_8_special_0_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_1_start_8_special_0_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_1_start_8_special_3_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_1_start_8_special_3_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_1_start_8_special_3_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_1_start_8_special_3_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_2_start_1_special_0_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_2_start_1_special_0_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_2_start_1_special_0_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_2_start_1_special_0_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_2_start_1_special_3_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_2_start_1_special_3_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_2_start_1_special_3_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_2_start_1_special_3_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_2_start_8_special_0_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_2_start_8_special_0_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_2_start_8_special_0_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_2_start_8_special_0_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_2_start_8_special_3_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_2_start_8_special_3_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_2_start_8_special_3_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_cls_step_2_start_8_special_3_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_1_start_1_special_0_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_1_start_1_special_0_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_1_start_1_special_0_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_1_start_1_special_0_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_1_start_1_special_3_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_1_start_1_special_3_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_1_start_1_special_3_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_1_start_1_special_3_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_1_start_8_special_0_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_1_start_8_special_0_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_1_start_8_special_0_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_1_start_8_special_0_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_1_start_8_special_3_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_1_start_8_special_3_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_1_start_8_special_3_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_1_start_8_special_3_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_2_start_1_special_0_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_2_start_1_special_0_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_2_start_1_special_0_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_2_start_1_special_0_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_2_start_1_special_3_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_2_start_1_special_3_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_2_start_1_special_3_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_2_start_1_special_3_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_2_start_8_special_0_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_2_start_8_special_0_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_2_start_8_special_0_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_2_start_8_special_0_noanchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_2_start_8_special_3_anchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_2_start_8_special_3_anchor_nofilename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_2_start_8_special_3_noanchor_filename.html create mode 100644 tests/html_linenos_expected_output/table_nocls_step_2_start_8_special_3_noanchor_nofilename.html create mode 100644 tests/snippets/apacheconf/test_directive_no_args.txt create mode 100644 tests/snippets/apacheconf/test_fix_lock_absolute_path.txt create mode 100644 tests/snippets/apacheconf/test_include_globs.txt create mode 100644 tests/snippets/apacheconf/test_malformed_scoped_directive_closing_tag.txt create mode 100644 tests/snippets/apacheconf/test_multi_include_globs.txt create mode 100644 tests/snippets/apacheconf/test_multi_include_globs_root.txt create mode 100644 tests/snippets/apacheconf/test_multiline_argument.txt create mode 100644 tests/snippets/apacheconf/test_multiline_comment.txt create mode 100644 tests/snippets/apacheconf/test_normal_scoped_directive.txt create mode 100644 tests/snippets/apl/test_leading_underscore.txt create mode 100644 tests/snippets/asm/test_cpuid.txt create mode 100644 tests/snippets/bibtex/test_basic_bst.txt create mode 100644 tests/snippets/bibtex/test_comment.txt create mode 100644 tests/snippets/bibtex/test_entry.txt create mode 100644 tests/snippets/bibtex/test_mismatched_brace.txt create mode 100644 tests/snippets/bibtex/test_missing_body.txt create mode 100644 tests/snippets/bibtex/test_preamble.txt create mode 100644 tests/snippets/bibtex/test_string.txt create mode 100644 tests/snippets/c/test_comment_end.txt create mode 100644 tests/snippets/c/test_function_comments.txt create mode 100644 tests/snippets/c/test_label.txt create mode 100644 tests/snippets/c/test_label_followed_by_statement.txt create mode 100644 tests/snippets/c/test_label_space_before_colon.txt create mode 100644 tests/snippets/c/test_numbers.txt create mode 100644 tests/snippets/c/test_preproc_file.txt create mode 100644 tests/snippets/c/test_preproc_file2.txt create mode 100644 tests/snippets/c/test_preproc_file3.txt create mode 100644 tests/snippets/c/test_preproc_file4.txt create mode 100644 tests/snippets/c/test_preproc_file5.txt create mode 100644 tests/snippets/c/test_string_resembling_decl_end.txt create mode 100644 tests/snippets/c/test_switch.txt create mode 100644 tests/snippets/c/test_switch_space_before_colon.txt create mode 100644 tests/snippets/cfm/test_basic_comment.txt create mode 100644 tests/snippets/cfm/test_nested_comment.txt create mode 100644 tests/snippets/coffeescript/test_beware_infinite_loop.txt create mode 100644 tests/snippets/coffeescript/test_mixed_slashes.txt create mode 100644 tests/snippets/conftest.py create mode 100644 tests/snippets/console/fake_ps2_prompt.txt create mode 100644 tests/snippets/console/prompt_in_output.txt create mode 100644 tests/snippets/console/ps2_prompt.txt create mode 100644 tests/snippets/console/test_comment_after_prompt.txt create mode 100644 tests/snippets/console/test_newline_in_echo_no_ps2.txt create mode 100644 tests/snippets/console/test_newline_in_echo_ps2.txt create mode 100644 tests/snippets/console/test_newline_in_ls_no_ps2.txt create mode 100644 tests/snippets/console/test_newline_in_ls_ps2.txt create mode 100644 tests/snippets/console/test_virtualenv.txt create mode 100644 tests/snippets/coq/test_unicode.txt create mode 100644 tests/snippets/cpp/test_good_comment.txt create mode 100644 tests/snippets/cpp/test_open_comment.txt create mode 100644 tests/snippets/cpp/test_unicode_identifiers.txt create mode 100644 tests/snippets/crystal/test_annotation.txt create mode 100644 tests/snippets/crystal/test_array_access.txt create mode 100644 tests/snippets/crystal/test_chars.txt create mode 100644 tests/snippets/crystal/test_constant_and_module.txt create mode 100644 tests/snippets/crystal/test_escaped_bracestring.txt create mode 100644 tests/snippets/crystal/test_escaped_interpolation.txt create mode 100644 tests/snippets/crystal/test_interpolation_nested_curly.txt create mode 100644 tests/snippets/crystal/test_lib.txt create mode 100644 tests/snippets/crystal/test_macro.txt create mode 100644 tests/snippets/crystal/test_operator_methods.txt create mode 100644 tests/snippets/crystal/test_percent_strings.txt create mode 100644 tests/snippets/crystal/test_percent_strings_special.txt create mode 100644 tests/snippets/crystal/test_pseudo_builtins.txt create mode 100644 tests/snippets/crystal/test_pseudo_keywords.txt create mode 100644 tests/snippets/crystal/test_range_syntax1.txt create mode 100644 tests/snippets/crystal/test_range_syntax2.txt create mode 100644 tests/snippets/csound/test_braced_strings.txt create mode 100644 tests/snippets/csound/test_comments.txt create mode 100644 tests/snippets/csound/test_escape_sequences.txt create mode 100644 tests/snippets/csound/test_function_like_macro_definitions.txt create mode 100644 tests/snippets/csound/test_function_like_macros.txt create mode 100644 tests/snippets/csound/test_global_value_identifiers.txt create mode 100644 tests/snippets/csound/test_goto_statements.txt create mode 100644 tests/snippets/csound/test_include_directives.txt create mode 100644 tests/snippets/csound/test_includestr_directives.txt create mode 100644 tests/snippets/csound/test_instrument_blocks.txt create mode 100644 tests/snippets/csound/test_keywords.txt create mode 100644 tests/snippets/csound/test_labels.txt create mode 100644 tests/snippets/csound/test_macro_preprocessor_directives.txt create mode 100644 tests/snippets/csound/test_name.txt create mode 100644 tests/snippets/csound/test_numbers.txt create mode 100644 tests/snippets/csound/test_object_like_macro_definitions.txt create mode 100644 tests/snippets/csound/test_operators.txt create mode 100644 tests/snippets/csound/test_other_preprocessor_directives.txt create mode 100644 tests/snippets/csound/test_printks_and_prints_escape_sequences.txt create mode 100644 tests/snippets/csound/test_quoted_strings.txt create mode 100644 tests/snippets/csound/test_user_defined_opcodes.txt create mode 100644 tests/snippets/doscon/test_gt_only.txt create mode 100644 tests/snippets/elpi/test_catastrophic_backtracking.txt create mode 100644 tests/snippets/elpi/test_chr.txt create mode 100644 tests/snippets/elpi/test_clause.txt create mode 100644 tests/snippets/elpi/test_namespace.txt create mode 100644 tests/snippets/elpi/test_pred.txt create mode 100644 tests/snippets/elpi/test_type.txt create mode 100644 tests/snippets/ezhil/test_function.txt create mode 100644 tests/snippets/ezhil/test_gcd_expr.txt create mode 100644 tests/snippets/ezhil/test_if_statement.txt create mode 100644 tests/snippets/ezhil/test_sum.txt create mode 100644 tests/snippets/fortran/test_string_cataback.txt create mode 100644 tests/snippets/gas/test_comments.txt create mode 100644 tests/snippets/gdscript/test_comment.txt create mode 100644 tests/snippets/gdscript/test_export_array.txt create mode 100644 tests/snippets/gdscript/test_function_with_types.txt create mode 100644 tests/snippets/gdscript/test_inner_class.txt create mode 100644 tests/snippets/gdscript/test_multiline_string.txt create mode 100644 tests/snippets/gdscript/test_signal.txt create mode 100644 tests/snippets/gdscript/test_simple_function.txt create mode 100644 tests/snippets/gdscript/test_variable_declaration_and_assigment.txt create mode 100644 tests/snippets/haskell/test_promoted_names.txt create mode 100644 tests/snippets/html/multiline-comment-catastrophic-backtracking.txt create mode 100644 tests/snippets/http/test_application_calendar_xml.txt create mode 100644 tests/snippets/http/test_application_xml.txt create mode 100644 tests/snippets/http/test_http_status_line.txt create mode 100644 tests/snippets/http/test_http_status_line_without_reason_phrase.txt create mode 100644 tests/snippets/http/test_http_status_line_without_reason_phrase_rfc_7230.txt create mode 100644 tests/snippets/idris/test_compiler_directive.txt create mode 100644 tests/snippets/idris/test_reserved_word.txt create mode 100644 tests/snippets/ini/test_indented_entries_1.txt create mode 100644 tests/snippets/ini/test_indented_entries_2.txt create mode 100644 tests/snippets/ini/test_indented_entries_3.txt create mode 100644 tests/snippets/j/test_deal_operator.txt create mode 100644 tests/snippets/j/test_deal_operator_fixed_seed.txt create mode 100644 tests/snippets/java/test_default.txt create mode 100644 tests/snippets/java/test_enhanced_for.txt create mode 100644 tests/snippets/java/test_multiline_string.txt create mode 100644 tests/snippets/java/test_multiline_string_only.txt create mode 100644 tests/snippets/java/test_numeric_literals.txt create mode 100644 tests/snippets/java/test_record.txt create mode 100644 tests/snippets/js/super.txt create mode 100644 tests/snippets/jslt/test_sample.txt create mode 100644 tests/snippets/json/test_basic.txt create mode 100644 tests/snippets/json/test_basic_bare.txt create mode 100644 tests/snippets/julia-repl/test_repl.txt create mode 100644 tests/snippets/julia/test_keywords.txt create mode 100644 tests/snippets/julia/test_macros.txt create mode 100644 tests/snippets/julia/test_names.txt create mode 100644 tests/snippets/julia/test_numbers.txt create mode 100644 tests/snippets/julia/test_operators.txt create mode 100644 tests/snippets/julia/test_strings.txt create mode 100644 tests/snippets/julia/test_symbols.txt create mode 100644 tests/snippets/julia/test_types.txt create mode 100644 tests/snippets/julia/test_unicode.txt create mode 100644 tests/snippets/kotlin/test_can_cope_generics_in_destructuring.txt create mode 100644 tests/snippets/kotlin/test_can_cope_with_backtick_names_in_functions.txt create mode 100644 tests/snippets/kotlin/test_can_cope_with_commas_and_dashes_in_backtick_Names.txt create mode 100644 tests/snippets/kotlin/test_can_cope_with_destructuring.txt create mode 100644 tests/snippets/kotlin/test_can_cope_with_generics.txt create mode 100644 tests/snippets/kotlin/test_modifier_keyword.txt create mode 100644 tests/snippets/kotlin/test_should_cope_with_multiline_comments.txt create mode 100644 tests/snippets/kotlin/test_string_interpolation.txt create mode 100644 tests/snippets/less/test_single_line_comments.txt create mode 100644 tests/snippets/mason/test_handles_tags_correctly.txt create mode 100644 tests/snippets/matlab/test_classes_with_properties.txt create mode 100644 tests/snippets/matlab/test_command_mode.txt create mode 100644 tests/snippets/matlab/test_comment_after_continuation.txt create mode 100644 tests/snippets/matlab/test_dot_operator.txt create mode 100644 tests/snippets/matlab/test_keywords_ended_by_newline.txt create mode 100644 tests/snippets/matlab/test_line_continuation.txt create mode 100644 tests/snippets/matlab/test_multiple_spaces_variable_assignment.txt create mode 100644 tests/snippets/matlab/test_one_space_assignment.txt create mode 100644 tests/snippets/matlab/test_operator_multiple_space.txt create mode 100644 tests/snippets/matlab/test_single_line.txt create mode 100644 tests/snippets/matlabsession/test_wrong_continuation.txt create mode 100644 tests/snippets/mcfunction/commenting.txt create mode 100644 tests/snippets/mcfunction/coordinates.txt create mode 100644 tests/snippets/mcfunction/data.txt create mode 100644 tests/snippets/mcfunction/difficult_1.txt create mode 100644 tests/snippets/mcfunction/multiline.txt create mode 100644 tests/snippets/mcfunction/selectors.txt create mode 100644 tests/snippets/mcfunction/simple.txt create mode 100644 tests/snippets/md/test_bold_fenced_by_asterisk.txt create mode 100644 tests/snippets/md/test_bold_fenced_by_underscore.txt create mode 100644 tests/snippets/md/test_bulleted_list_1.txt create mode 100644 tests/snippets/md/test_bulleted_list_2.txt create mode 100644 tests/snippets/md/test_bulleted_list_3.txt create mode 100644 tests/snippets/md/test_bulleted_list_4.txt create mode 100644 tests/snippets/md/test_code_block_fenced_by_backticks.txt create mode 100644 tests/snippets/md/test_code_block_with_language.txt create mode 100644 tests/snippets/md/test_escape_italics.txt create mode 100644 tests/snippets/md/test_inline_code.txt create mode 100644 tests/snippets/md/test_inline_code_after_block.txt create mode 100644 tests/snippets/md/test_inline_code_in_list.txt create mode 100644 tests/snippets/md/test_invalid_bold.txt create mode 100644 tests/snippets/md/test_invalid_italics.txt create mode 100644 tests/snippets/md/test_italics_and_bold.txt create mode 100644 tests/snippets/md/test_italics_fenced_by_asterisk.txt create mode 100644 tests/snippets/md/test_italics_fenced_by_underscore.txt create mode 100644 tests/snippets/md/test_italics_no_multiline.txt create mode 100644 tests/snippets/md/test_links.txt create mode 100644 tests/snippets/md/test_mentions.txt create mode 100644 tests/snippets/md/test_numbered_list.txt create mode 100644 tests/snippets/md/test_quote.txt create mode 100644 tests/snippets/md/test_reference_style_links.txt create mode 100644 tests/snippets/md/test_strikethrough.txt create mode 100644 tests/snippets/md/test_task_list.txt create mode 100644 tests/snippets/md/test_topics.txt create mode 100644 tests/snippets/mips/deprecated_substrings.txt create mode 100644 tests/snippets/mips/keyword_substrings.txt create mode 100644 tests/snippets/mips/variable_substrings.txt create mode 100644 tests/snippets/nasm/checkid.txt create mode 100644 tests/snippets/objectivec/test_literal_number_bool.txt create mode 100644 tests/snippets/objectivec/test_literal_number_bool_expression.txt create mode 100644 tests/snippets/objectivec/test_literal_number_expression.txt create mode 100644 tests/snippets/objectivec/test_literal_number_int.txt create mode 100644 tests/snippets/objectivec/test_literal_number_nested_expression.txt create mode 100644 tests/snippets/objectivec/test_module_import.txt create mode 100644 tests/snippets/octave/test_multilinecomment.txt create mode 100644 tests/snippets/omg-idl/annotation_named_params.txt create mode 100644 tests/snippets/omg-idl/enumerators.txt create mode 100644 tests/snippets/peg/test_basic.txt create mode 100644 tests/snippets/peg/test_modified_strings.txt create mode 100644 tests/snippets/peg/test_operators.txt create mode 100644 tests/snippets/php/test_backslashes_in_strings.txt create mode 100644 tests/snippets/php/test_string_escaping_run.txt create mode 100644 tests/snippets/powershell/test_colon_punctuation.txt create mode 100644 tests/snippets/powershell/test_remoting_session.txt create mode 100644 tests/snippets/powershell/test_session.txt create mode 100644 tests/snippets/praat/test_broken_unquoted_string.txt create mode 100644 tests/snippets/praat/test_function_call.txt create mode 100644 tests/snippets/praat/test_inline_if.txt create mode 100644 tests/snippets/praat/test_interpolated_indexed_numeric_with_precision.txt create mode 100644 tests/snippets/praat/test_interpolated_local_numeric_with_precision.txt create mode 100644 tests/snippets/praat/test_interpolated_numeric_hash.txt create mode 100644 tests/snippets/praat/test_interpolated_numeric_indexed.txt create mode 100644 tests/snippets/praat/test_interpolated_numeric_with_precision.txt create mode 100644 tests/snippets/praat/test_interpolated_string_hash.txt create mode 100644 tests/snippets/praat/test_interpolated_string_indexed.txt create mode 100644 tests/snippets/praat/test_interpolation_boundary.txt create mode 100644 tests/snippets/praat/test_numeric_assignment.txt create mode 100644 tests/snippets/praat/test_string_assignment.txt create mode 100644 tests/snippets/praat/test_string_escaped_quotes.txt create mode 100644 tests/snippets/promql/test_complex_exp_single_quotes.txt create mode 100644 tests/snippets/promql/test_expression_and_comment.txt create mode 100644 tests/snippets/promql/test_function_delta.txt create mode 100644 tests/snippets/promql/test_function_multi_line.txt create mode 100644 tests/snippets/promql/test_function_multi_line_with_offset.txt create mode 100644 tests/snippets/promql/test_function_sum_with_args.txt create mode 100644 tests/snippets/promql/test_matching_operator_no_regex_match.txt create mode 100644 tests/snippets/promql/test_metric.txt create mode 100644 tests/snippets/promql/test_metric_multiple_labels.txt create mode 100644 tests/snippets/promql/test_metric_multiple_labels_with_spaces.txt create mode 100644 tests/snippets/promql/test_metric_one_label.txt create mode 100644 tests/snippets/properties/test_comments.txt create mode 100644 tests/snippets/properties/test_escaped_space_in_key.txt create mode 100644 tests/snippets/properties/test_escaped_space_in_value.txt create mode 100644 tests/snippets/properties/test_just_key.txt create mode 100644 tests/snippets/properties/test_just_key_with_space.txt create mode 100644 tests/snippets/properties/test_leading_whitespace_comments.txt create mode 100644 tests/snippets/properties/test_space_delimited_kv_pair.txt create mode 100644 tests/snippets/pwsh-session/test_continuation.txt create mode 100644 tests/snippets/python/test_bytes_escape_codes.txt create mode 100644 tests/snippets/python/test_floats.txt create mode 100644 tests/snippets/python/test_fstring_01a.txt create mode 100644 tests/snippets/python/test_fstring_01b.txt create mode 100644 tests/snippets/python/test_fstring_02a.txt create mode 100644 tests/snippets/python/test_fstring_02b.txt create mode 100644 tests/snippets/python/test_fstring_03a.txt create mode 100644 tests/snippets/python/test_fstring_03b.txt create mode 100644 tests/snippets/python/test_fstring_04a.txt create mode 100644 tests/snippets/python/test_fstring_04b.txt create mode 100644 tests/snippets/python/test_fstring_05a.txt create mode 100644 tests/snippets/python/test_fstring_05b.txt create mode 100644 tests/snippets/python/test_fstring_06a.txt create mode 100644 tests/snippets/python/test_fstring_06b.txt create mode 100644 tests/snippets/python/test_fstring_07a.txt create mode 100644 tests/snippets/python/test_fstring_07b.txt create mode 100644 tests/snippets/python/test_fstring_08a.txt create mode 100644 tests/snippets/python/test_fstring_08b.txt create mode 100644 tests/snippets/python/test_fstring_09a.txt create mode 100644 tests/snippets/python/test_fstring_09b.txt create mode 100644 tests/snippets/python/test_fstring_10a.txt create mode 100644 tests/snippets/python/test_fstring_10b.txt create mode 100644 tests/snippets/python/test_fstring_11a.txt create mode 100644 tests/snippets/python/test_fstring_11b.txt create mode 100644 tests/snippets/python/test_fstring_12a.txt create mode 100644 tests/snippets/python/test_fstring_12b.txt create mode 100644 tests/snippets/python/test_fstring_13a.txt create mode 100644 tests/snippets/python/test_fstring_13b.txt create mode 100644 tests/snippets/python/test_fstring_14a.txt create mode 100644 tests/snippets/python/test_fstring_14b.txt create mode 100644 tests/snippets/python/test_fstring_15a.txt create mode 100644 tests/snippets/python/test_fstring_15b.txt create mode 100644 tests/snippets/python/test_fstring_16a.txt create mode 100644 tests/snippets/python/test_fstring_16b.txt create mode 100644 tests/snippets/python/test_fstring_17a.txt create mode 100644 tests/snippets/python/test_fstring_17b.txt create mode 100644 tests/snippets/python/test_fstring_18a.txt create mode 100644 tests/snippets/python/test_fstring_18b.txt create mode 100644 tests/snippets/python/test_fstring_19a.txt create mode 100644 tests/snippets/python/test_fstring_19b.txt create mode 100644 tests/snippets/python/test_fstring_20a.txt create mode 100644 tests/snippets/python/test_fstring_20b.txt create mode 100644 tests/snippets/python/test_fstring_21a.txt create mode 100644 tests/snippets/python/test_fstring_21b.txt create mode 100644 tests/snippets/python/test_fstring_22a.txt create mode 100644 tests/snippets/python/test_fstring_22b.txt create mode 100644 tests/snippets/python/test_fstring_23a.txt create mode 100644 tests/snippets/python/test_fstring_23b.txt create mode 100644 tests/snippets/python/test_fstring_24a.txt create mode 100644 tests/snippets/python/test_fstring_24b.txt create mode 100644 tests/snippets/python/test_fstring_25a.txt create mode 100644 tests/snippets/python/test_fstring_25b.txt create mode 100644 tests/snippets/python/test_fstring_26a.txt create mode 100644 tests/snippets/python/test_fstring_26b.txt create mode 100644 tests/snippets/python/test_fstring_27a.txt create mode 100644 tests/snippets/python/test_fstring_27b.txt create mode 100644 tests/snippets/python/test_fstring_28a.txt create mode 100644 tests/snippets/python/test_fstring_28b.txt create mode 100644 tests/snippets/python/test_fstring_29a.txt create mode 100644 tests/snippets/python/test_fstring_29b.txt create mode 100644 tests/snippets/python/test_fstring_30a.txt create mode 100644 tests/snippets/python/test_fstring_30b.txt create mode 100644 tests/snippets/python/test_fstring_31a.txt create mode 100644 tests/snippets/python/test_fstring_31b.txt create mode 100644 tests/snippets/python/test_fstring_32a.txt create mode 100644 tests/snippets/python/test_fstring_32b.txt create mode 100644 tests/snippets/python/test_fstring_33a.txt create mode 100644 tests/snippets/python/test_fstring_33b.txt create mode 100644 tests/snippets/python/test_fstring_34a.txt create mode 100644 tests/snippets/python/test_fstring_34b.txt create mode 100644 tests/snippets/python/test_fstring_35a.txt create mode 100644 tests/snippets/python/test_fstring_35b.txt create mode 100644 tests/snippets/python/test_fstring_36a.txt create mode 100644 tests/snippets/python/test_fstring_36b.txt create mode 100644 tests/snippets/python/test_needs_name.txt create mode 100644 tests/snippets/python/test_pep_515.txt create mode 100644 tests/snippets/python/test_raw_fstring.txt create mode 100644 tests/snippets/python/test_string_escape_codes.txt create mode 100644 tests/snippets/python/test_walrus_operator.txt create mode 100644 tests/snippets/python2/test_cls_builtin.txt create mode 100644 tests/snippets/qbasic/test_keywords_with_dollar.txt create mode 100644 tests/snippets/r/test_call.txt create mode 100644 tests/snippets/r/test_custom_operator.txt create mode 100644 tests/snippets/r/test_dot_indexing.txt create mode 100644 tests/snippets/r/test_dot_name.txt create mode 100644 tests/snippets/r/test_indexing.txt create mode 100644 tests/snippets/r/test_name1.txt create mode 100644 tests/snippets/r/test_name2.txt create mode 100644 tests/snippets/r/test_name3.txt create mode 100644 tests/snippets/ruby/test_escaped_bracestring.txt create mode 100644 tests/snippets/ruby/test_interpolation_nested_curly.txt create mode 100644 tests/snippets/ruby/test_operator_methods.txt create mode 100644 tests/snippets/ruby/test_range_syntax1.txt create mode 100644 tests/snippets/ruby/test_range_syntax2.txt create mode 100644 tests/snippets/ruby/test_range_syntax3.txt create mode 100644 tests/snippets/rust/test_attribute.txt create mode 100644 tests/snippets/rust/test_break.txt create mode 100644 tests/snippets/rust/test_rawstrings.txt create mode 100644 tests/snippets/scala/test_colon_colon_function_name.txt create mode 100644 tests/snippets/scala/test_default_parameter.txt create mode 100644 tests/snippets/scala/test_end_val.txt create mode 100644 tests/snippets/scala/test_end_valx.txt create mode 100644 tests/snippets/scala/test_float_with_exponents.txt create mode 100644 tests/snippets/scala/test_function_operator_name.txt create mode 100644 tests/snippets/scala/test_import_path.txt create mode 100644 tests/snippets/scala/test_invalid_symbol_and_invalid_char.txt create mode 100644 tests/snippets/scala/test_open_soft_keyword.txt create mode 100644 tests/snippets/scala/test_package_name.txt create mode 100644 tests/snippets/scala/test_prepend_operator.txt create mode 100644 tests/snippets/scala/test_qualified_name.txt create mode 100644 tests/snippets/scala/test_qualified_name_class.txt create mode 100644 tests/snippets/scala/test_script_header.txt create mode 100644 tests/snippets/scala/test_symbol_followed_by_op.txt create mode 100644 tests/snippets/scala/test_symbol_name_ending_with_star.txt create mode 100644 tests/snippets/scala/test_underscore_name.txt create mode 100644 tests/snippets/scheme/keywords.txt create mode 100644 tests/snippets/scheme/numbers.txt create mode 100644 tests/snippets/scheme/strings.txt create mode 100644 tests/snippets/shell/test_array_nums.txt create mode 100644 tests/snippets/shell/test_curly_no_escape_and_quotes.txt create mode 100644 tests/snippets/shell/test_curly_with_escape.txt create mode 100644 tests/snippets/shell/test_end_of_line_nums.txt create mode 100644 tests/snippets/shell/test_parsed_single.txt create mode 100644 tests/snippets/shell/test_short_variable_names.txt create mode 100644 tests/snippets/shexc/test_prefixed_name_starting_with_number.txt create mode 100644 tests/snippets/smarty/test_nested_curly.txt create mode 100644 tests/snippets/snbt/json.txt create mode 100644 tests/snippets/snbt/literals.txt create mode 100644 tests/snippets/snbt/multiline.txt create mode 100644 tests/snippets/snbt/nesting.txt create mode 100644 tests/snippets/snbt/quoted_keys.txt create mode 100644 tests/snippets/systemverilog/test_basic.txt create mode 100644 tests/snippets/systemverilog/test_classes.txt create mode 100644 tests/snippets/systemverilog/test_numbers.txt create mode 100644 tests/snippets/systemverilog/test_operators.txt create mode 100644 tests/snippets/tcl/test_comma_and_at.txt create mode 100644 tests/snippets/tcl/test_vars.txt create mode 100644 tests/snippets/teal/test_comments.txt create mode 100644 tests/snippets/teal/test_literals.txt create mode 100644 tests/snippets/teal/test_strings.txt create mode 100644 tests/snippets/terraform/test_attributes.txt create mode 100644 tests/snippets/terraform/test_backend.txt create mode 100644 tests/snippets/terraform/test_comment.txt create mode 100644 tests/snippets/terraform/test_functions.txt create mode 100644 tests/snippets/terraform/test_heredoc.txt create mode 100644 tests/snippets/terraform/test_module.txt create mode 100644 tests/snippets/terraform/test_resource.txt create mode 100644 tests/snippets/terraform/test_types.txt create mode 100644 tests/snippets/terraform/test_variable_declaration.txt create mode 100644 tests/snippets/terraform/test_variable_read.txt create mode 100644 tests/snippets/turtle/test_prefixed_name_starting_with_number.txt create mode 100644 tests/snippets/typescript/test_function_definition.txt create mode 100644 tests/snippets/unixconfig/etc_group.txt create mode 100644 tests/snippets/unixconfig/etc_passwd.txt create mode 100644 tests/snippets/unixconfig/etc_shadow.txt create mode 100644 tests/snippets/usd/test_attribute.txt create mode 100644 tests/snippets/usd/test_composition_arcs.txt create mode 100644 tests/snippets/usd/test_metadata.txt create mode 100644 tests/snippets/usd/test_numbers.txt create mode 100644 tests/snippets/usd/test_outer_match_at_sign.txt create mode 100644 tests/snippets/usd/test_outer_match_double.txt create mode 100644 tests/snippets/usd/test_outer_match_single.txt create mode 100644 tests/snippets/usd/test_string_multiple_line.txt create mode 100644 tests/snippets/usd/test_string_priority.txt create mode 100644 tests/snippets/usd/test_string_single_line.txt create mode 100644 tests/snippets/vbscript/test_floats.txt create mode 100644 tests/snippets/vbscript/test_floats_multiple.txt create mode 100644 tests/snippets/vbscript/test_integers.txt create mode 100644 tests/snippets/vbscript/test_invalid_character.txt create mode 100644 tests/snippets/vbscript/test_names.txt create mode 100644 tests/snippets/vbscript/test_reject_almost_float.txt create mode 100644 tests/snippets/vbscript/test_unterminated_string.txt create mode 100644 tests/snippets/wat/test_align_and_offset_accept_hexadecimal_numbers.txt create mode 100644 tests/snippets/wat/test_comment_with_open_paren.txt create mode 100644 tests/snippets/wat/test_comment_with_semicolon.txt create mode 100644 tests/snippets/wat/test_i32_const_is_builtin.txt create mode 100644 tests/snippets/wat/test_multiline_comment.txt create mode 100644 tests/snippets/wat/test_nested_comment.txt create mode 100644 tests/snippets/wat/test_string_byte_escape.txt create mode 100644 tests/snippets/wat/test_string_with_escape.txt create mode 100644 tests/snippets/wat/test_variable_name_pattern.txt create mode 100644 tests/snippets/whiley/test_whiley_operator.txt create mode 100644 tests/snippets/wren/lonely-paren.txt create mode 100644 tests/snippets/xml/multiline-comment-catastrophic-backtracking.txt create mode 100644 tests/snippets/yaml/test_yaml.txt create mode 100644 tests/snippets/yaml/test_yaml_colon_in_key.txt create mode 100644 tests/snippets/yaml/test_yaml_colon_in_key_double.txt create mode 100644 tests/snippets/yaml/test_yaml_colon_in_key_start.txt create mode 100644 tests/snippets/yang/test_float_value.txt create mode 100644 tests/snippets/yang/test_integer_value.txt create mode 100644 tests/snippets/yang/test_namespace_1.txt create mode 100644 tests/snippets/yang/test_namespace_2.txt create mode 100644 tests/snippets/yang/test_revision_date.txt create mode 100644 tests/snippets/yang/test_string_value.txt create mode 100644 tests/support/empty.py create mode 100644 tests/support/html_formatter.py create mode 100644 tests/support/python_lexer.py create mode 100644 tests/support/structural_diff.py create mode 100644 tests/support/tags create mode 100644 tests/test_basic_api.py create mode 100644 tests/test_cmdline.py create mode 100644 tests/test_coffeescript.py create mode 100644 tests/test_crystal.py create mode 100644 tests/test_data.py create mode 100644 tests/test_devicetree_lexer.py create mode 100644 tests/test_func.py create mode 100644 tests/test_groff_formatter.py create mode 100644 tests/test_guess.py create mode 100644 tests/test_html_formatter.py create mode 100644 tests/test_html_formatter_linenos_elements.py create mode 100644 tests/test_html_lexer.py create mode 100644 tests/test_inherit.py create mode 100644 tests/test_irc_formatter.py create mode 100644 tests/test_java.py create mode 100644 tests/test_javascript.py create mode 100644 tests/test_latex_formatter.py create mode 100644 tests/test_markdown_lexer.py create mode 100644 tests/test_modeline.py create mode 100644 tests/test_mysql.py create mode 100644 tests/test_pangomarkup_formatter.py create mode 100644 tests/test_perllexer.py create mode 100644 tests/test_procfile.py create mode 100644 tests/test_raw_token.py create mode 100644 tests/test_regexlexer.py create mode 100644 tests/test_regexopt.py create mode 100644 tests/test_robotframework_lexer.py create mode 100644 tests/test_rtf_formatter.py create mode 100644 tests/test_ruby.py create mode 100644 tests/test_sql.py create mode 100644 tests/test_templates.py create mode 100644 tests/test_terminal_formatter.py create mode 100644 tests/test_thingsdb.py create mode 100644 tests/test_tnt.py create mode 100644 tests/test_token.py create mode 100644 tests/test_unistring.py create mode 100755 tests/test_usd.py create mode 100644 tests/test_using_api.py create mode 100644 tests/test_util.py create mode 100644 tests/test_words.py create mode 100644 tox.ini diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..a6c6adb --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[run] +include = + pygments/* diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e08d9e1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +doc +tests +Pygments.egg-info +TAGS +build +dist +htmlcov +venv +**/__pycache__ +.* +*.rst +*.egg +*.pyo +.*.sw[op] + +!/doc/pyodide/meta.yaml diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..44a295f --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +tests/examplefiles/*/*.output linguist-generated diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..2d26dfb --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,72 @@ +name: Pygments + +on: [push, pull_request] + +permissions: + contents: read # to fetch code (actions/checkout) + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] + exclude: + - os: ubuntu-latest + python-version: "3.6" + include: + - os: ubuntu-20.04 + python-version: "3.6" + max-parallel: 4 + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install package + run: | + pip install -r requirements.txt + pip install . + - name: Test package + run: pytest -W error + + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + - name: Run make check + run: make check + - name: Fail if the basic checks failed + run: make check + if: runner.os == 'Linux' + + check-mapfiles: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + - name: Regenerate mapfiles + run: make mapfiles + - name: Fail if mapfiles changed + run: | + if git ls-files -m | grep mapping; then + echo 'Please run "make mapfiles" and add the changes to a commit.' + exit 1 + fi + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: Check out regexlint + run: git clone https://github.com/pygments/regexlint + - name: Run regexlint + run: make regexlint REGEXLINT=`pwd`/regexlint diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 0000000..41adf4e --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,40 @@ +name: Docs + +on: + push: + branches: + - master + +permissions: {} +jobs: + build: + permissions: + contents: write # to push pages branch (peaceiris/actions-gh-pages) + + runs-on: ubuntu-latest + steps: + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.10" + - name: Checkout Pygments + uses: actions/checkout@v2 + - name: Install Sphinx & WCAG contrast ratio + run: pip install Sphinx wcag-contrast-ratio + - name: Create Pyodide WASM package + run: cd doc && make pyodide + - name: Sphinx build + run: | + cd doc + WEBSITE_BUILD=1 make dirhtml + touch _build/dirhtml/.nojekyll + echo -e 'pygments.org\nwww.pygments.org' > _build/dirhtml/CNAME + echo 'Automated deployment of docs for GitHub pages.' > _build/dirhtml/README + - name: Deploy to repo + if: github.repository_owner == 'pygments' + uses: peaceiris/actions-gh-pages@v3 + with: + deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }} + external_repository: pygments/pygments.github.io + publish_branch: master + publish_dir: ./doc/_build/dirhtml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cbfddbe --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +*.egg +*.pyc +*.pyo +.*.sw[op] +/.pytest_cache/ +/.idea/ +/.project +/.tags +/.tox/ +/.cache/ +/Pygments.egg-info/* +/TAGS +/build/* +/dist/* +/doc/_build +/.coverage +/htmlcov +/.vscode +venv/ +.venv/ +.DS_Store diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..18a32c6 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,264 @@ +Pygments is written and maintained by Georg Brandl . + +Major developers are Tim Hatch and Armin Ronacher +. + +Other contributors, listed alphabetically, are: + +* Sam Aaron -- Ioke lexer +* Jean Abou Samra -- LilyPond lexer +* João Abecasis -- JSLT lexer +* Ali Afshar -- image formatter +* Thomas Aglassinger -- Easytrieve, JCL, Rexx, Transact-SQL and VBScript + lexers +* Muthiah Annamalai -- Ezhil lexer +* Kumar Appaiah -- Debian control lexer +* Andreas Amann -- AppleScript lexer +* Timothy Armstrong -- Dart lexer fixes +* Jeffrey Arnold -- R/S, Rd, BUGS, Jags, and Stan lexers +* Eiríkr Åsheim -- Uxntal lexer +* Jeremy Ashkenas -- CoffeeScript lexer +* José Joaquín Atria -- Praat lexer +* Stefan Matthias Aust -- Smalltalk lexer +* Lucas Bajolet -- Nit lexer +* Ben Bangert -- Mako lexers +* Max Battcher -- Darcs patch lexer +* Thomas Baruchel -- APL lexer +* Tim Baumann -- (Literate) Agda lexer +* Paul Baumgart, 280 North, Inc. -- Objective-J lexer +* Michael Bayer -- Myghty lexers +* Thomas Beale -- Archetype lexers +* John Benediktsson -- Factor lexer +* Trevor Bergeron -- mIRC formatter +* Vincent Bernat -- LessCSS lexer +* Christopher Bertels -- Fancy lexer +* Sébastien Bigaret -- QVT Operational lexer +* Jarrett Billingsley -- MiniD lexer +* Adam Blinkinsop -- Haskell, Redcode lexers +* Stéphane Blondon -- Procfile, SGF and Sieve lexers +* Frits van Bommel -- assembler lexers +* Pierre Bourdon -- bugfixes +* Martijn Braam -- Kernel log lexer, BARE lexer +* Matthias Bussonnier -- ANSI style handling for terminal-256 formatter +* chebee7i -- Python traceback lexer improvements +* Hiram Chirino -- Scaml and Jade lexers +* Mauricio Caceres -- SAS and Stata lexers. +* Ian Cooper -- VGL lexer +* David Corbett -- Inform, Jasmin, JSGF, Snowball, and TADS 3 lexers +* Leaf Corcoran -- MoonScript lexer +* Christopher Creutzig -- MuPAD lexer +* Daniël W. Crompton -- Pike lexer +* Pete Curry -- bugfixes +* Bryan Davis -- EBNF lexer +* Bruno Deferrari -- Shen lexer +* Walter Dörwald -- UL4 lexer +* Luke Drummond -- Meson lexer +* Giedrius Dubinskas -- HTML formatter improvements +* Owen Durni -- Haxe lexer +* Alexander Dutton, Oxford University Computing Services -- SPARQL lexer +* James Edwards -- Terraform lexer +* Nick Efford -- Python 3 lexer +* Sven Efftinge -- Xtend lexer +* Artem Egorkine -- terminal256 formatter +* Matthew Fernandez -- CAmkES lexer +* Paweł Fertyk -- GDScript lexer, HTML formatter improvements +* Michael Ficarra -- CPSA lexer +* James H. Fisher -- PostScript lexer +* William S. Fulton -- SWIG lexer +* Carlos Galdino -- Elixir and Elixir Console lexers +* Michael Galloy -- IDL lexer +* Naveen Garg -- Autohotkey lexer +* Simon Garnotel -- FreeFem++ lexer +* Laurent Gautier -- R/S lexer +* Alex Gaynor -- PyPy log lexer +* Richard Gerkin -- Igor Pro lexer +* Alain Gilbert -- TypeScript lexer +* Alex Gilding -- BlitzBasic lexer +* GitHub, Inc -- DASM16, Augeas, TOML, and Slash lexers +* Bertrand Goetzmann -- Groovy lexer +* Krzysiek Goj -- Scala lexer +* Rostyslav Golda -- FloScript lexer +* Andrey Golovizin -- BibTeX lexers +* Matt Good -- Genshi, Cheetah lexers +* Michał Górny -- vim modeline support +* Alex Gosse -- TrafficScript lexer +* Patrick Gotthardt -- PHP namespaces support +* Hubert Gruniaux -- C and C++ lexer improvements +* Olivier Guibe -- Asymptote lexer +* Phil Hagelberg -- Fennel lexer +* Florian Hahn -- Boogie lexer +* Martin Harriman -- SNOBOL lexer +* Matthew Harrison -- SVG formatter +* Steven Hazel -- Tcl lexer +* Dan Michael Heggø -- Turtle lexer +* Aslak Hellesøy -- Gherkin lexer +* Greg Hendershott -- Racket lexer +* Justin Hendrick -- ParaSail lexer +* Jordi Gutiérrez Hermoso -- Octave lexer +* David Hess, Fish Software, Inc. -- Objective-J lexer +* Ken Hilton -- Typographic Number Theory and Arrow lexers +* Varun Hiremath -- Debian control lexer +* Rob Hoelz -- Perl 6 lexer +* Doug Hogan -- Mscgen lexer +* Ben Hollis -- Mason lexer +* Max Horn -- GAP lexer +* Fred Hornsey -- OMG IDL Lexer +* Alastair Houghton -- Lexer inheritance facility +* Tim Howard -- BlitzMax lexer +* Dustin Howett -- Logos lexer +* Ivan Inozemtsev -- Fantom lexer +* Hiroaki Itoh -- Shell console rewrite, Lexers for PowerShell session, + MSDOS session, BC, WDiff +* Brian R. Jackson -- Tea lexer +* Christian Jann -- ShellSession lexer +* Dennis Kaarsemaker -- sources.list lexer +* Dmitri Kabak -- Inferno Limbo lexer +* Igor Kalnitsky -- vhdl lexer +* Colin Kennedy - USD lexer +* Alexander Kit -- MaskJS lexer +* Pekka Klärck -- Robot Framework lexer +* Gerwin Klein -- Isabelle lexer +* Eric Knibbe -- Lasso lexer +* Stepan Koltsov -- Clay lexer +* Oliver Kopp - Friendly grayscale style +* Adam Koprowski -- Opa lexer +* Benjamin Kowarsch -- Modula-2 lexer +* Domen Kožar -- Nix lexer +* Oleh Krekel -- Emacs Lisp lexer +* Alexander Kriegisch -- Kconfig and AspectJ lexers +* Marek Kubica -- Scheme lexer +* Jochen Kupperschmidt -- Markdown processor +* Gerd Kurzbach -- Modelica lexer +* Jon Larimer, Google Inc. -- Smali lexer +* Olov Lassus -- Dart lexer +* Matt Layman -- TAP lexer +* Kristian Lyngstøl -- Varnish lexers +* Sylvestre Ledru -- Scilab lexer +* Chee Sing Lee -- Flatline lexer +* Mark Lee -- Vala lexer +* Pete Lomax -- Phix lexer +* Valentin Lorentz -- C++ lexer improvements +* Ben Mabey -- Gherkin lexer +* Angus MacArthur -- QML lexer +* Louis Mandel -- X10 lexer +* Louis Marchand -- Eiffel lexer +* Simone Margaritelli -- Hybris lexer +* Tim Martin - World of Warcraft TOC lexer +* Kirk McDonald -- D lexer +* Gordon McGregor -- SystemVerilog lexer +* Stephen McKamey -- Duel/JBST lexer +* Brian McKenna -- F# lexer +* Charles McLaughlin -- Puppet lexer +* Kurt McKee -- Tera Term macro lexer, PostgreSQL updates, MySQL overhaul, JSON lexer +* Joe Eli McIlvain -- Savi lexer +* Lukas Meuser -- BBCode formatter, Lua lexer +* Cat Miller -- Pig lexer +* Paul Miller -- LiveScript lexer +* Hong Minhee -- HTTP lexer +* Michael Mior -- Awk lexer +* Bruce Mitchener -- Dylan lexer rewrite +* Reuben Morais -- SourcePawn lexer +* Jon Morton -- Rust lexer +* Paulo Moura -- Logtalk lexer +* Mher Movsisyan -- DTD lexer +* Dejan Muhamedagic -- Crmsh lexer +* Ana Nelson -- Ragel, ANTLR, R console lexers +* Kurt Neufeld -- Markdown lexer +* Nam T. Nguyen -- Monokai style +* Jesper Noehr -- HTML formatter "anchorlinenos" +* Mike Nolta -- Julia lexer +* Avery Nortonsmith -- Pointless lexer +* Jonas Obrist -- BBCode lexer +* Edward O'Callaghan -- Cryptol lexer +* David Oliva -- Rebol lexer +* Pat Pannuto -- nesC lexer +* Jon Parise -- Protocol buffers and Thrift lexers +* Benjamin Peterson -- Test suite refactoring +* Ronny Pfannschmidt -- BBCode lexer +* Dominik Picheta -- Nimrod lexer +* Andrew Pinkham -- RTF Formatter Refactoring +* Clément Prévost -- UrbiScript lexer +* Tanner Prynn -- cmdline -x option and loading lexers from files +* Oleh Prypin -- Crystal lexer (based on Ruby lexer) +* Nick Psaris -- K and Q lexers +* Xidorn Quan -- Web IDL lexer +* Elias Rabel -- Fortran fixed form lexer +* raichoo -- Idris lexer +* Daniel Ramirez -- GDScript lexer +* Kashif Rasul -- CUDA lexer +* Nathan Reed -- HLSL lexer +* Justin Reidy -- MXML lexer +* Norman Richards -- JSON lexer +* Corey Richardson -- Rust lexer updates +* Fabrizio Riguzzi -- cplint leder +* Lubomir Rintel -- GoodData MAQL and CL lexers +* Andre Roberge -- Tango style +* Georg Rollinger -- HSAIL lexer +* Michiel Roos -- TypoScript lexer +* Konrad Rudolph -- LaTeX formatter enhancements +* Mario Ruggier -- Evoque lexers +* Miikka Salminen -- Lovelace style, Hexdump lexer, lexer enhancements +* Stou Sandalski -- NumPy, FORTRAN, tcsh and XSLT lexers +* Matteo Sasso -- Common Lisp lexer +* Joe Schafer -- Ada lexer +* Max Schillinger -- TiddlyWiki5 lexer +* Ken Schutte -- Matlab lexers +* René Schwaiger -- Rainbow Dash style +* Sebastian Schweizer -- Whiley lexer +* Tassilo Schweyer -- Io, MOOCode lexers +* Pablo Seminario -- PromQL lexer +* Ted Shaw -- AutoIt lexer +* Joerg Sieker -- ABAP lexer +* Robert Simmons -- Standard ML lexer +* Kirill Simonov -- YAML lexer +* Corbin Simpson -- Monte lexer +* Ville Skyttä -- ASCII armored lexer +* Alexander Smishlajev -- Visual FoxPro lexer +* Steve Spigarelli -- XQuery lexer +* Jerome St-Louis -- eC lexer +* Camil Staps -- Clean and NuSMV lexers; Solarized style +* James Strachan -- Kotlin lexer +* Tom Stuart -- Treetop lexer +* Colin Sullivan -- SuperCollider lexer +* Ben Swift -- Extempore lexer +* tatt61880 -- Kuin lexer +* Edoardo Tenani -- Arduino lexer +* Tiberius Teng -- default style overhaul +* Jeremy Thurgood -- Erlang, Squid config lexers +* Brian Tiffin -- OpenCOBOL lexer +* Bob Tolbert -- Hy lexer +* Doug Torrance -- Macaulay2 lexer +* Matthias Trute -- Forth lexer +* Tuoa Spi T4 -- Bdd lexer +* Erick Tryzelaar -- Felix lexer +* Alexander Udalov -- Kotlin lexer improvements +* Thomas Van Doren -- Chapel lexer +* Daniele Varrazzo -- PostgreSQL lexers +* Abe Voelker -- OpenEdge ABL lexer +* Pepijn de Vos -- HTML formatter CTags support +* Matthias Vallentin -- Bro lexer +* Benoît Vinot -- AMPL lexer +* Linh Vu Hong -- RSL lexer +* Immanuel Washington -- Smithy lexer +* Nathan Weizenbaum -- Haml and Sass lexers +* Nathan Whetsell -- Csound lexers +* Dietmar Winkler -- Modelica lexer +* Nils Winter -- Smalltalk lexer +* Davy Wybiral -- Clojure lexer +* Whitney Young -- ObjectiveC lexer +* Diego Zamboni -- CFengine3 lexer +* Enrique Zamudio -- Ceylon lexer +* Alex Zimin -- Nemerle lexer +* Rob Zimmerman -- Kal lexer +* Vincent Zurczak -- Roboconf lexer +* Hubert Gruniaux -- C and C++ lexer improvements +* Thomas Symalla -- AMDGPU Lexer +* 15b3 -- Image Formatter improvements +* Fabian Neumann -- CDDL lexer +* Thomas Duboucher -- CDDL lexer +* Philipp Imhof -- Pango Markup formatter +* Thomas Voss -- Sed lexer +* Martin Fischer -- WCAG contrast testing +* Marc Auberer -- Spice lexer + +Many thanks for all contributions! diff --git a/CHANGES b/CHANGES new file mode 100644 index 0000000..2aa54fa --- /dev/null +++ b/CHANGES @@ -0,0 +1,2134 @@ +Pygments changelog +================== + +Pull request numbers before 2.4.2 are not linked as they refer to the now defunct Bitbucket project. + +Version 2.14.0 +-------------- +(released January 1st, 2023) + +- Added lexers: + + * Arturo (#2259) + * GAP session (#2211) + * Fift (#2249) + * func (#2232) + * Jsonnet (#2239) + * Minecraft schema (#2276) + * MIPS (#2228) + * Phix (#2222) + * Portugol (#2300) + * TL-b (#2247) + * World of Warcraft TOC format (#2244, #2245) + * Wren (#2271) + +- Updated lexers: + + * Abap: Update keywords (#2281) + * Alloy: Update for Alloy 6 (#1963) + * C family (C, C++ and many others): + + - Fix an issue where a chunk would be wrongly recognized as a function + definition due to braces in comments (#2210) + - Improve parantheses handling for function definitions (#2207, #2208) + + * C#: Fix number and operator recognition (#2256, #2257) + * CSound: Updated builtins (#2268) + * F#: Add ``.fsx`` file extension (#2282) + * gas (GNU assembler): recognize braces as punctuation (#2230) + * HTTP: Add `CONNECT` keyword (#2242) + * Inform 6: Fix lexing of properties and doubles (#2214) + * INI: Allow comments that are not their own line (#2217, #2161) + * Java properties: Fix issue with whitespace-delimited keys, support + comments starting with `!` and escapes, no longer support undocumented + `;` and `//` comments (#2241) + * LilyPond: Improve heuristics, add ``\maxima`` duration (#2283) + * LLVM: Add opaque pointer type (#2269) + * Macaulay2: Update keywords (#2305) + * Minecraft-related lexers (SNB and Minecraft function) moved to + ``pygments.lexers.minecraft`` (#2276) + * Nim: General improvements (#1970) + * Nix: Fix single quotes inside indented strings (#2289) + * Objective J: Fix catastrophic backtracking (#2225) + * NASM: Add support for SSE/AVX/AVX-512 registers as well as 'rel' + and 'abs' address operators (#2212) + * Powershell: + + - Add ``local:`` keyword (#2254) + - Allow continuations without markers (#2262, #2263) + + * Solidity: Add boolean operators (#2292) + * Spice: Add ``enum`` keyword and fix a bug regarding binary, + hexadecimal and octal number tokens (#2227) + * YAML: Accept colons in key names (#2277) + +- Fix `make mapfiles` when Pygments is not installed in editable mode + (#2223) + +- Support more filetypes and compression types in `autopygmentize` (#2219) +- Merge consecutive tokens in Autohotkey, Clay (#2248) +- Add ``.nasm`` as a recognized file type for NASM (#2280) +- Add ``*Spec.hs`` as a recognized file type for ``HSpec`` (#2308) +- Add ``*.pyi`` (for typing stub files) as a recognized file type for + Python (#2331) +- The HTML lexer no longer emits empty spans for whitespace (#2304) +- Fix ``IRCFormatter`` inserting linenumbers incorrectly (#2270) + +Version 2.13.0 +-------------- +(released August 15th, 2022) + +- Added lexers: + + * COMAL-80 (#2180) + * JMESPath (#2174, #2175, #2179, #2182) + * Sql+Jinja (#2148) + +- Updated lexers: + + * Ada: support Ada 2022 (#2121); disable recognition of namespaces + because it disturbs lexing of aspects (#2125) + * Agda: allow straight quotes in module names (#2163) + * C family (C, C++ and many others): allow comments between + elements of function headers, e.g. between the arguments and + the opening brace for the body (#1891) + * C++: Resolve several cases of ``Error`` tokens (#2207, #2208) + * Coq: Add some common keywords, improve recognition of ``Set`` + and qualified identifiers (#2158) + * F*: Allow C-style comments anywhere in a line + * Fortran: Fix catastrophic backtracking with backslashes in strings + (#2194) + * Go: add support for generics (#2167) + * Inform: Update for version 6.40 (#2190) + * Isabelle: recognize cartouches (#2089) + * Java: support multiline strings aka. text blocks (#2132) + * Kotlin: Add ``value`` modifier (#2142) + * LilyPond: Add some missing builtins + * Macaulay2: Update builtins (#2139) + * Matlab session: fix traceback when a line continuation ellipsis + appears in the output (#2166) + * .NET: Add aliases for LibreOffice Basic, OpenOfficeBasic and + StarOffice Basic (#2170) + * Nim: Use ``Name.Builtin`` instead of ``Keyword.Type`` (#2136) + * PHP: fix `\"$var\"` inside strings (#2105) + * Python: only recognize ``\N``, ``\u`` and ``\U`` escape sequences + in string literals, but not in bytes literals where they are + not supported (#2204) + * Tcl: support ``${name}`` variables (#2145) + * Terraform: Accept leading whitespace for `<<` heredoc + delimiters (#2162) + * Teraterm: Various improvements (#2165) + * Spice: add support for the recently added features including more + builtin functions and bin, oct, hex number formats (#2206) + +- Added styles: + + * GitHub dark (#2192) + * StarOffice (#2168) + * Nord (`nord` and `nord-darker`; #2189, #1799, #1678) + +- Pygments now tries to use the ``importlib.metadata`` module to + discover plugins instead of the slower ``pkg_resources`` (#2155). In + particular, this largely speeds up the ``pygmentize`` script when + the lexer is not specified. + + ``importlib.metadata`` is only available in the Python standard + library since Python 3.8. For older versions, there exists an + ``importlib_metadata`` backport on PyPI. For this reason, Pygments + now defines a packaging extra ``plugins``, which adds a requirement + on ``importlib_metadata`` if the Python version is older than + 3.8. Thus, in order to install Pygments with optimal plugin + support even for old Python versions, you should do:: + + pip install pygments[plugins] + + Pygments still falls back on ``pkg_resources`` if neither + ``importlib.metadata`` nor ``importlib_metadata`` is found, but it + will be slower. + +- Silently ignore ``BrokenPipeError`` in the command-line interface + (#2193). +- The ``HtmlFormatter`` now uses the ``linespans`` attribute for + ``anchorlinenos`` if the ``lineanchors`` attribute is unset (#2026). +- The ``highlight``, ``lex`` and ``format`` functions no longer + wrongly report "argument must be a lexer/formatter instance, not a + class" in some cases where this is not the actual problem (#2123). +- Fix warnings in doc build (#2124). +- The ``codetagify`` filter now recognizes ``FIXME`` tags by default (#2150). +- The ``pygmentize`` command now recognizes if the ``COLORTERM`` + environment variable is set to a value indicating that true-color + support is available. In that case, it uses the ``TerminalTrueColorFormatter`` + by default (#2160) +- Remove redundant caches for filename patterns (#2153) +- Use new non-deprecated Pillow API for text bounding box in ``ImageFormatter`` + (#2198) +- Remove ``default_style`` (#930, #2183) +- Stop treating ``DeprecationWarnings`` as errors in the unit tests (#2196) + +Version 2.12.0 +-------------- +(released April 24th, 2022) + +- Added lexers: + + * Berry (#2070) + * Cplint (#2045) + * Macaulay2 (#1791) + * MCFunction (#2107) + * Minecraft (#2107) + * Qlik (#1925) + * ``UnixConfigLexer`` for "colon-separated" config files, like ``/etc/passwd`` (#2112) + * Uxntal (#2086) + * K and Q (#2073) + +- Updated lexers: + + * Agda: Update keyword list (#2017) + * C family: Fix identifiers after ``case`` statements (#2084) + * Clojure: Highlight ratios (#2042) + * Csound: Update to 6.17 (#2064) + * CSS: Update the list of properties (#2113) + * Elpi: + + - Fix catastrophic backtracking (#2053, #2061) + - Fix handling of ``->`` (#2028) + + * Futhark: Add missing tokens (#2118) + * Gherkin: Add ``But`` (#2046) + * Inform6: Update to 6.36 (#2050) + * Jinja2: add ``.xxx.j2`` and ``.xxx.jinja2`` to relevant lexers + (for ``xxx`` = ``html``, ``xml``, etc.) (#2103) + * JSON: Support C comments in JSON (#2049). Note: This doesn't mean the JSON parser now supports JSONC or JSON5 proper, just that it doesn't error out when seeing a ``/* */`` or ``//`` style comment. If you need proper comment handling, consider using the ``JavaScript`` lexer. + * LilyPond: + + - Fix incorrect lexing of names containing a built-in (#2071) + - Fix properties containing dashes (#2099) + + * PHP: Update builtin function and keyword list (#2054, #2056) + * Python: highlight ``EncodingWarning`` (#2106) + * Savi: fix highlighting for underscore/private identifiers, + add string interpolation (#2102); fix nested type name highlighting + (#2110) + * Scheme: Various improvements (#2060) + * Spice: Update the keyword list, add new types (#2063, #2067) + * Terraform: + + - Support non-idiomatic comments (#2065, #2066) + - Fix class name lexing (#2097) + +- Add ``plugins`` argument to ``get_all_lexers()``. +- Bump minimal Python version to 3.6 (#2059) +- Fix multiple lexers marking whitespace as ``Text`` (#2025) +- Remove various redundant uses of ``re.UNICODE`` (#2058) +- Associate ``.resource`` with the Robot framework (#2047) +- Associate ``.cljc`` with Clojure (#2043) +- Associate ``.tpp`` with C++ (#2031) +- Remove traces of Python 2 from the documentation (#2039) +- The ``native`` style was updated to meet the WCAG AAA contrast guidelines (#2038) +- Fix various typos (#2030) +- Fix ``Groff`` formatter not inheriting token styles correctly (#2024) +- Various improvements to the CI (#2036) +- The Ada lexer has been moved to a separate file (#2117) +- When ``linenos=table`` is used, the ```` itself is now wrapped with a ``
`` tag instead of placing it inside the ``
` is emitted before the + table content which contains a single `
`` cell (#632.) With this change, the output matches the documented behavior. + +.. note:: + + If you have subclassed ``HtmlFormatter.wrap``, you may have to adjust the logic. + + +Version 2.11.2 +-------------- +(released January 6th, 2022) + +- Updated lexers: + + * C-family: Fix incorrect handling of labels (#2022, #1996, #1182) + * Java: Fixed an issue with ``record`` keywords result in ``Error`` tokens in some cases (#2016, #2018) + +- Fix links to line numbers not working correctly (#2014) +- Remove ``underline`` from ``Whitespace`` style in the ``Tango`` theme (#2020) +- Fix ``IRC`` and ``Terminal256`` formatters not backtracking correctly for custom token types, resulting in some unstyled tokens (#1986) + + +Version 2.11.1 +-------------- +(released December 31st, 2021) + +- Updated lexers: + + * C-family: Handle return types with multiple tokens (e.g. ``unsigned int``) (#2008) + * JSON: Fix a regression which caused whitespace before ``:`` to result in ``Error`` tokens (#2010) + * SPICE: Various improvements (#2009) + + +Version 2.11.0 +-------------- +(released December 30th, 2021) + +- Added lexers: + + * BDD (#1803) + * Elpi (#1894) + * LilyPond (#1845, #1968, #1971, #2001). This comes with a custom style as well. + * Maxima (#1885) + * Rita (#1541, #2003) + * Savi (#1863) + * Sed (#1935) + * Sophia contracts (#1974) + * Spice (#1980) + * ``.SRCINFO`` (#1951) + +- Updated lexers: + + * ABNF: Allow one-character rules (#1804) + * Assembly: Fix incorrect token endings (#1895, #1961) + * Bibtex: Distinguish between ``comment`` and ``commentary`` (#1899, #1806) + * C family: Support unicode identifiers (#1848) + * CDDL: Fix slow lexing speed (#1959) + * Debian control: Add missing fields (#1946) + * Devicetree: Recognize hexadecimal addresses for nodes (#1949) + * GDScript: Add ``void`` data type (#1948) + * GSQL + + - Fix comment handling (#2002) + - Fix catastrophic backtracking (#2006) + + * HTML, XML: Improve comment handling (#1896) + * Java: Add ``yield`` (#1941) and sealed classes/record (#1902) + * Makefiles (#1860, #1898) + * objdump-nasm: Improve handling of ``--no-show-raw-insn`` dumps (#1981) + * Prolog: Support escaped ``\`` inside quoted strings (#1479) + * Python: + + - Support ``~`` in tracebacks (#2004) + - Support the pattern matching keywords (#1797, #1994) + + * RobotFramework: Improve empty brace handling (#1921, #1922) + * Terraform + + - Add the 'set' type (#1909) + - Support heredocs (#1909) + +- Added styles: + + * Dracula (#1796) + * Friendly Grayscale (#1040, #1273) + * LilyPond (#1845) -- to be used for the ``LilyPond`` language. + * One-Dark (#1924, #1979) + +.. note:: + + All of the new styles unfortunately do not conform to WCAG recommendations. + +- There is new infrastructure in place to improve style accessibility. The default style has been updated to conform to WCAG recommendations. All styles are now checked for sufficient contrast by default to prevent regressions. (#1919, #1937, #1938, #1940) +- Clean up unused imports (#1887) +- Fix multiple lexers producing repeated single-character tokens +- Fix multiple lexers marking whitespace as ``Text`` (#1237, #1905, #1908, #1914, #1911, #1923, #1939, #1957, #1978) +- Remove duplicated assignments in the Paraiso style (#1934) +- ``pygmentize`` supports JSON output for the various list functions now, making it easier to consume them from scripts. (#1437, #1890) +- Use the ``shell`` lexer for ``kshrc`` files (#1947) +- Use the ``ruby`` lexer for ``Vagrantfile`` files (#1936) +- Use the C lexer for ``.xbm`` and ``.xpm`` files (#1802) +- Add a ``groff`` formatter (#1873) +- Update documentation (#1928) +- Line anchors now link to themselves (#1973) +- Add official support for Python 3.10 (#1917) +- Fix several missing colors in dark styles: Gruvbox dark, Monokai, Rrt, Sas, Strata dark (#1955) +- Associate more file types with ``man`` pages +- The ``HtmlFormatter`` can now emit tooltips for each token to ease debugging of lexers (#1822) +- Add ``f90`` as an alias for ``fortran`` (#2000) + + +Version 2.10.0 +-------------- +(released August 15th, 2021) + +- Added lexers: + + * ASC armored files (#1807) + * GSQL (#1809, #1866) + * Javascript REPL (#1825) + * procfile (#1808) + * Smithy (#1878, #1879) + +- Updated lexers: + + * C-family: Fix preprocessor token issues (#1830) + * C# (#1573, #1869) + * CSound (#1837) + * Fennel (#1862) + * JavaScript (#1741, #1814) + * LLVM (#1824) + * Python (#1852) + * Rust + + - Fix lexing of "break" and "continue" (#1843) + - Improve attribute handling (#1813) + + * Scala: Add support for the ``\`` operator (#1857) + * Swift (#1767, #1842) + * Tcl: Allow ``,`` and ``@`` in strings (#1834, #1742) + * TOML (#1870, #1872) + +- Fix assert statements in TNT lexer. +- Token types across all lexers have been unified (using the most common token + type name) (#1816, #1819) +- Improve Jasmin min score analysis (#1619) +- Add new alias for Go files (#1827) +- Fix multi-line console highlighting (#1833) +- Add a new trivial lexer which outputs everything as `Text.Generic.Output` (#1835, #1836) +- Use the ``.ini`` lexer for ``systemd`` files (#1849) +- Fix a ``FutureWarning`` related to ``words()`` (#1854) +- ``pwsh`` is now recognized as an alias for PowerShell (#1876) + + +Version 2.9.0 +------------- +(released May 3rd, 2021) + +- Added lexers: + + * APDL, gcode (#1714) + * Kuin (#1300) + * NestedText (#1578) + * OMG IDL (#1595) + * TEAL (#1671) + * ThingsDB (#1295) + * WebAssembly (#1416, #1564) + +- Updated lexers: + + * AMDGPU (#1717, #1775) + * APL (#1747) + * C/C++: Improve namespace handling (#1722, #1561, #1719, #1746) + * Chapel (#1743) + * Coq (#1721) + * Cython (#853) + * DeviceTree (#1755) + * Groovy (#1765) + * Julia (#1715) + * Octave: Allow multiline and block-percent comments (#1726) + * PowerShell: Improve lexing of ``:`` (#1682, #1758) + * PromQL (#1783) + * Python: Improve float parsing (#1768, #1740) + * Rust (#1061) + * Scala: Rewrite to support Scala3 (#1694, #1035, #1121) + * Terraform: Support 0.14 syntax (#1756) + * Velocity: Detect multi-line patterns (#1776) + +- Add Pango formatter (#1727) +- Autopygmentize uses ``file`` first instead of ``pygments -N`` (#1786) +- Fix links (#1716) +- Fix issue with LaTeX formatter and ``minted`` (#1734, #1735, #1736, #1737) +- Improve alias order (#1780) +- Improve line number colors (#1779, #1778) +- Fix CTag related issue (#1724) +- Recognize ``.leex`` as Elixir templates +- Fix incorrect variable being accessed (#1748) + +- Updated `filename` handling in HTML formatter if `linenos='table'` (#1757) + + * Previously the filename would be emitted within the `` holding the + code, but outside the `
`. This would invariably break the alignment
+    with line numbers.
+  * Now if `filename` is specified, a separate `
` with `colspan=2` so it + spans both the line number and code columns. The filename is still + within `...` so any existing styles + should still apply, although the CSS path may need to change. + * For an example of the new output format see + `table_cls_step_1_start_1_special_0_noanchor_filename.html` + in the `tests/html_linenos_expected_output/` directory. + * For more details and discussion see the issue + https://github.com/pygments/pygments/issues/1757 + +- Added styles: + + * Gruvbox light+dark (#1763) + + +Version 2.8.0 +------------- +(released February 14, 2021) + +- Added lexers: + + * AMDGPU (#1626) + * CDDL (#1379, #1239) + * Futhark (#1691) + * Graphviz/DOT (#1657, #731) + +- Updated lexers: + + * AutoIt: Support single quoted strings (#1667, #1663) + * C/C++ & related: Fix mishandling ``*/`` (#1695) + * Cocoa: Add builtin types (#1703) + * Console (#1672) + * Eiffel: Fix performance issues (#1658) + * Fortran: Improve combined keyword detection (#1677, #1188) + * J: Fix operator ``?`` lexing (#1700, #1149) + * JavaScript/TypeScript: Fix escapes in backtick strings (#1679, #1686) + * Kotlin: Improve string interpolation, modifier keyword handling, and various small issues (#1699) + * LESS: Support single-line comments (#1046) + * Matlab: + + - Add support for class properties (#1466) + - Update builtin functions (#1705) + - Various cleanups (#1673) + + * OpenEdge (#1696) + * Python: Improve handling of raw f-strings (#1681, #1683) + * Ruby: Better method name handling (#1531) + * Stata: Updated keywords (#1470) + +- Added styles: + + * Material (#1662) + * Zenburn (#1659) + +- The `pygmentize` script now uses `argparse`, all options should work + as before + +- Add `pygmentize -C` option to guess a lexer from content + +- With this release, Pygments moves to a new internal testing system (#1649.) + See ``Contributing.md`` for details. The main advantage of this new change + is a much better test coverage of all existing example lexers. It also makes + it much easier to add new test snippets. +- Make guessing prefer Python 3 lexer +- Do not guess MIME or SQL without reason +- Changed setuptools to use a declarative config through ``setup.cfg``. + Building Pygments now requires setuptools 39.2+. +- Add markdown to MarkdownLexer aliases (#1687) +- Change line number handling + + * In ```` based output, the ``td.linenos`` element will have either a + ``normal`` or ``special`` class attached. Previously, only ``special`` line + numbers got a class. This prevents styles from getting applied twice - + once via ``
``, once via ````. This also means
+    that ``td.linenos pre`` is no longer styled, instead, use
+    ``td.linenos .normal`` and ``td.linenos .special``.
+  * In the "inline" style, the DOM element order was changed. The line number
+    is added first, then the line is wrapped is wrapped by the highlighter.
+    This fixes lines not being fully highlighted.
+  * The visual output for inline and non-inline line numbers & highlighting,
+    as well as class-based and inline styling is now consistent.
+  * Line number styles are set to ``background-color: transparent`` and
+    ``color: inherit`` by default. This works much better with dark styles
+    which don't have colors set for line numbers.
+
+- Remove "raw" alias from RawTokenLexer, so that it cannot be
+  selected by alias.
+- Fix RawTokenLexer to work in Python 3 and handle exceptions.
+- Add prompt colors to the Solarized theme (#1529)
+- Image formatter supports background colors now (#1374)
+- Add support for anchors in conjunction with inline line numbers (#1591)
+- Modernize the codebase using ``pyupgrade`` (#1622)
+- Add support for line numbers to the ``terminal256`` formatter (#1674, #1653)
+- Improve ``analyze_text`` logic for ``ECL`` (#1610)
+- Improve ``analyze_text`` logic for ``CBM Basic V2`` (#1607)
+- Improve LaTeX formatter (#1708, #1709)
+
+
+Version 2.7.4
+-------------
+(released January 12, 2021)
+
+- Updated lexers:
+
+  - Apache configurations: Improve handling of malformed tags (#1656)
+  - CSS: Add support for variables (#1633, #1666)
+  - Crystal (#1650, #1670)
+  - Coq (#1648)
+  - Fortran: Add missing keywords (#1635, #1665)
+  - Ini (#1624)
+  - JavaScript and variants (#1647 -- missing regex flags, #1651)
+  - Markdown (#1623, #1617)
+  - Shell
+
+    - Lex trailing whitespace as part of the prompt (#1645)
+    - Add missing ``in`` keyword (#1652)
+
+  - SQL - Fix keywords (#1668)
+  - Typescript: Fix incorrect punctuation handling (#1510, #1511)
+
+- Fix infinite loop in SML lexer (#1625), `CVE-2021-20270 `_
+- Fix backtracking string regexes in JavaScript/TypeScript, Modula2
+  and many other lexers (#1637) `CVE-2021-27291 `_
+- Limit recursion with nesting Ruby heredocs (#1638)
+- Fix a few inefficient regexes for guessing lexers
+- Fix the raw token lexer handling of Unicode (#1616)
+- Revert a private API change in the HTML formatter (#1655) --
+  please note that private APIs remain subject to change!
+- Fix several exponential/cubic-complexity regexes found by
+  Ben Caller/Doyensec (#1675)
+- Fix incorrect MATLAB example (#1582)
+
+Thanks to Google's OSS-Fuzz project for finding many of these bugs.
+
+
+Version 2.7.3
+-------------
+(released December 6, 2020)
+
+- Updated lexers:
+
+  * Ada (#1581)
+  * HTML (#1615, #1614)
+  * Java (#1594, #1586)
+  * JavaScript (#1605, #1589, #1588)
+  * JSON (#1569 -- this is a complete rewrite)
+  * Lean (#1601)
+  * LLVM (#1612)
+  * Mason (#1592)
+  * MySQL (#1555, #1551)
+  * Rust (#1608)
+  * Turtle (#1590, #1553)
+
+- Deprecated JsonBareObjectLexer, which is now identical to JsonLexer (#1600)
+- The ``ImgFormatter`` now calculates the exact character width, which fixes some issues with overlapping text (#1213, #1611)
+- Documentation fixes (#1609, #1599, #1598)
+- Fixed duplicated Juttle language alias (#1604, #1606)
+- Added support for Kotlin scripts (#1587)
+- Removed CSS rule which forced margin to 0
+
+
+Version 2.7.2
+-------------
+(released October 24, 2020)
+
+- Updated lexers:
+
+  * Latex (#1517, #1516)
+  * LLVM (#1565)
+  * SPARQL (#1559)
+
+- Fix Python console/traceback lexer problems with custom exceptions without messages (#1548)
+- Allow loading ttc fonts on Mac/image formatter (#1223)
+- Improve ``analyze_text`` across a variety of lexers (#1549)
+- Remove CSS rule which forced the vertical padding to 0 for line numbers (#1583, #1579)
+- Fix ``TNTLexer`` crashing on unexpected EOL (#1568, #1570)
+- ``regexlint`` can be now run locally as part of ``tox`` tests (#1557)
+- Fix typos (#1550, #1562)
+- Add Python 3.9 as a supported version (#1554)
+
+
+Version 2.7.1
+-------------
+(released September 16, 2020)
+
+- Fixed a regression in the JSON lexer (#1544)
+
+
+Version 2.7.0
+-------------
+(released September 12, 2020)
+
+- Added lexers:
+
+  * Arrow (#1481, #1499)
+  * BARE (#1488)
+  * Devicetree (#1434)
+  * F* (#1409)
+  * GDScript (#1457)
+  * Pointless (#1494)
+  * PromQL (#1506)
+  * PsySH (#1438)
+  * Singularity (#1285)
+  * TiddlyWiki5 (#1390)
+  * TNT (#1414)
+  * YANG (#1408, #1428)
+
+- Updated lexers:
+
+  * APL (#1503)
+  * C++ (#1350, which also fixes: #1222, #996, #906, #828, #1162, #1166,
+    #1396)
+  * Chapel (#1423)
+  * CMake (#1491)
+  * CSound (#1509)
+  * Cython (#1507)
+  * Dart (#1449)
+  * Fennel (#1535)
+  * Fortran (#1442)
+  * GAS (#1530)
+  * HTTP (#1432, #1520, #1521)
+  * Inform 6 (#1461)
+  * Javascript (#1533)
+  * JSON (#1065, #1528)
+  * Lean (#1415)
+  * Matlab (#1399)
+  * Markdown (#1492, #1495)
+  * MySQL (#975, #1063, #1453, #1527)
+  * NASM (#1465)
+  * Nim (#1426)
+  * PostgreSQL (#1513)
+  * PowerShell (#1398, #1497)
+  * Protobuf (#1505)
+  * Robot (#1480)
+  * SQL (#1402)
+  * SystemVerilog (#1436, #1452, #1454, #1460, #1462, #1463, #1464, #1471, #1496, #1504)
+  * TeraTerm (#1337)
+  * XML (#1502)
+
+- Added a new filter for math symbols (#1406)
+- The Kconfig lexer will match Kconfig derivative names now (#1458)
+- Improved HTML formatter output (#1500)
+- ``.markdown`` is now recognized as an extension for Markdown files (#1476)
+- Fixed line number colors for Solarized (#1477, #1356)
+- Improvements to exception handling (#1478)
+- Improvements to tests (#1532, #1533, #1539)
+- Various code cleanups (#1536, #1537, #1538)
+
+
+Version 2.6.1
+-------------
+(released March 8, 2020)
+
+- This release fixes a packaging issue. No functional changes.
+
+
+Version 2.6
+-----------
+(released March 8, 2020)
+
+- Running Pygments on Python 2.x is no longer supported.
+  (The Python 2 lexer still exists.)
+
+- Added lexers:
+
+  * Linux kernel logs (#1310)
+  * LLVM MIR (#1361)
+  * MiniScript (#1397)
+  * Mosel (#1287, #1326)
+  * Parsing Expression Grammar (#1336)
+  * ReasonML (#1386)
+  * Ride (#1319, #1321)
+  * Sieve (#1257)
+  * USD (#1290)
+  * WebIDL (#1309)
+
+- Updated lexers:
+
+  * Apache2 (#1378)
+  * Chapel (#1357)
+  * CSound (#1383)
+  * D (#1375, #1362)
+  * Haskell (#1347, #1177)
+  * Idris (#1360)
+  * Perl6/Raku lexer (#1344)
+  * Python3 (#1382, #1385)
+  * Rust: Updated lexer to cover more builtins (mostly macros) and miscellaneous
+    new syntax (#1320)
+  * SQL: Add temporal support keywords (#1402)
+
+- The 256-color/true-color terminal formatters now support the italic attribute
+  in styles (#1288)
+- Support HTTP 2/3 header (#1308)
+- Support missing reason in HTTP header (#1322)
+- Boogie/Silver: support line continuations and triggers, move contract keywords
+  to separate category (#1299)
+- GAS: support C-style comments (#1291)
+- Fix names in S lexer (#1330, #1333)
+- Fix numeric literals in Ada (#1334)
+- Recognize ``.mjs`` files as Javascript (#1392)
+- Recognize ``.eex`` files as Elixir (#1387)
+- Fix ``re.MULTILINE`` usage (#1388)
+- Recognize ``pipenv`` and ``poetry`` dependency & lock files (PR#1376)
+- Improve font search on Windows (#1247)
+- Remove unused script block (#1401)
+
+
+Version 2.5.2
+-------------
+(released November 29, 2019)
+
+- Fix incompatibility with some setuptools versions (PR#1316)
+
+- Fix lexing of ReST field lists (PR#1279)
+- Fix lexing of Matlab keywords as field names (PR#1282)
+- Recognize double-quoted strings in Matlab (PR#1278)
+- Avoid slow backtracking in Vim lexer (PR#1312)
+- Fix Scala highlighting of types (PR#1315)
+- Highlight field lists more consistently in ReST (PR#1279)
+- Fix highlighting Matlab keywords in field names (PR#1282)
+- Recognize Matlab double quoted strings (PR#1278)
+- Add some Terraform keywords
+- Update Modelica lexer to 3.4
+- Update Crystal examples
+
+
+Version 2.5.1
+-------------
+(released November 26, 2019)
+
+- This release fixes a packaging issue. No functional changes.
+
+
+Version 2.5.0
+-------------
+(released November 26, 2019)
+
+- Added lexers:
+
+  * Email (PR#1246)
+  * Erlang, Elixir shells (PR#823, #1521)
+  * Notmuch (PR#1264)
+  * `Scdoc `_ (PR#1268)
+  * `Solidity `_ (#1214)
+  * `Zeek `_ (new name for Bro) (PR#1269)
+  * `Zig `_ (PR#820)
+
+- Updated lexers:
+
+  * Apache2 Configuration (PR#1251)
+  * Bash sessions (#1253)
+  * CSound (PR#1250)
+  * Dart
+  * Dockerfile
+  * Emacs Lisp
+  * Handlebars (PR#773)
+  * Java (#1101, #987)
+  * Logtalk (PR#1261)
+  * Matlab (PR#1271)
+  * Praat (PR#1277)
+  * Python3 (PR#1255, PR#1400)
+  * Ruby
+  * YAML (#1528)
+  * Velocity
+
+- Added styles:
+
+  * Inkpot (PR#1276)
+
+- The ``PythonLexer`` class is now an alias for the former ``Python3Lexer``.
+  The old ``PythonLexer`` is available as ``Python2Lexer``.  Same change has
+  been done for the ``PythonTracebackLexer``.  The ``python3`` option for
+  the ``PythonConsoleLexer`` is now true by default.
+
+- Bump ``NasmLexer`` priority over ``TasmLexer`` for ``.asm`` files
+  (fixes #1326)
+- Default font in the ``ImageFormatter`` has been updated (#928, PR#1245)
+- Test suite switched to py.test, removed nose dependency (#1490)
+- Reduce ``TeraTerm`` lexer score -- it used to match nearly all languages
+  (#1256)
+- Treat ``Skylark``/``Starlark`` files as Python files (PR#1259)
+- Image formatter: actually respect ``line_number_separator`` option
+
+- Add LICENSE file to wheel builds
+- Agda: fix lambda highlighting
+- Dart: support ``@`` annotations
+- Dockerfile: accept ``FROM ... AS`` syntax
+- Emacs Lisp: add more string functions
+- GAS: accept registers in directive arguments
+- Java: make structural punctuation (braces, parens, colon, comma) ``Punctuation``, not ``Operator`` (#987)
+- Java: support ``var`` contextual keyword (#1101)
+- Matlab: Fix recognition of ``function`` keyword (PR#1271)
+- Python: recognize ``.jy`` filenames (#976)
+- Python: recognize ``f`` string prefix (#1156)
+- Ruby: support squiggly heredocs
+- Shell sessions: recognize Virtualenv prompt (PR#1266)
+- Velocity: support silent reference syntax
+
+
+Version 2.4.2
+-------------
+(released May 28, 2019)
+
+- Fix encoding error when guessing lexer with given ``encoding`` option
+  (#1438)
+
+
+Version 2.4.1
+-------------
+(released May 24, 2019)
+
+- Updated lexers:
+
+  * Coq (#1430)
+  * MSDOS Session (PR#734)
+  * NASM (#1517)
+  * Objective-C (PR#813, #1508)
+  * Prolog (#1511)
+  * TypeScript (#1515)
+
+- Support CSS variables in stylesheets (PR#814, #1356)
+- Fix F# lexer name (PR#709)
+- Fix ``TerminalFormatter`` using bold for bright text (#1480)
+
+
+Version 2.4.0
+-------------
+(released May 8, 2019)
+
+- Added lexers:
+
+  * Augeas (PR#807)
+  * BBC Basic (PR#806)
+  * Boa (PR#756)
+  * Charm++ CI (PR#788)
+  * DASM16 (PR#807)
+  * FloScript (PR#750)
+  * FreeFem++ (PR#785)
+  * Hspec (PR#790)
+  * Pony (PR#627)
+  * SGF (PR#780)
+  * Slash (PR#807)
+  * Slurm (PR#760)
+  * Tera Term Language (PR#749)
+  * TOML (PR#807)
+  * Unicon (PR#731)
+  * VBScript (PR#673)
+
+- Updated lexers:
+
+  * Apache2 (PR#766)
+  * Cypher (PR#746)
+  * LLVM (PR#792)
+  * Makefiles (PR#766)
+  * PHP (#1482)
+  * Rust
+  * SQL (PR#672)
+  * Stan (PR#774)
+  * Stata (PR#800)
+  * Terraform (PR#787)
+  * YAML
+
+- Add solarized style (PR#708)
+- Add support for Markdown reference-style links (PR#753)
+- Add license information to generated HTML/CSS files (#1496)
+- Change ANSI color names (PR#777)
+- Fix catastrophic backtracking in the bash lexer (#1494)
+- Fix documentation failing to build using Sphinx 2.0 (#1501)
+- Fix incorrect links in the Lisp and R lexer documentation (PR#775)
+- Fix rare unicode errors on Python 2.7 (PR#798, #1492)
+- Fix lexers popping from an empty stack (#1506)
+- TypoScript uses ``.typoscript`` now (#1498)
+- Updated Trove classifiers and ``pip`` requirements (PR#799)
+
+
+
+Version 2.3.1
+-------------
+(released Dec 16, 2018)
+
+- Updated lexers:
+
+  * ASM (PR#784)
+  * Chapel (PR#735)
+  * Clean (PR#621)
+  * CSound (PR#684)
+  * Elm (PR#744)
+  * Fortran (PR#747)
+  * GLSL (PR#740)
+  * Haskell (PR#745)
+  * Hy (PR#754)
+  * Igor Pro (PR#764)
+  * PowerShell (PR#705)
+  * Python (PR#720, #1299, PR#715)
+  * SLexer (PR#680)
+  * YAML (PR#762, PR#724)
+
+- Fix invalid string escape sequences
+- Fix `FutureWarning` introduced by regex changes in Python 3.7
+
+
+Version 2.3.0
+-------------
+(released Nov 25, 2018)
+
+- Added lexers:
+
+  * Fennel (PR#783)
+  * HLSL (PR#675)
+
+- Updated lexers:
+
+  * Dockerfile (PR#714)
+
+- Minimum Python versions changed to 2.7 and 3.5
+- Added support for Python 3.7 generator changes (PR#772)
+- Fix incorrect token type in SCSS for single-quote strings (#1322)
+- Use `terminal256` formatter if `TERM` contains `256` (PR#666)
+- Fix incorrect handling of GitHub style fences in Markdown (PR#741, #1389)
+- Fix `%a` not being highlighted in Python3 strings (PR#727)
+
+
+Version 2.2.0
+-------------
+(released Jan 22, 2017)
+
+- Added lexers:
+
+  * AMPL
+  * TypoScript (#1173)
+  * Varnish config (PR#554)
+  * Clean (PR#503)
+  * WDiff (PR#513)
+  * Flatline (PR#551)
+  * Silver (PR#537)
+  * HSAIL (PR#518)
+  * JSGF (PR#546)
+  * NCAR command language (PR#536)
+  * Extempore (PR#530)
+  * Cap'n Proto (PR#595)
+  * Whiley (PR#573)
+  * Monte (PR#592)
+  * Crystal (PR#576)
+  * Snowball (PR#589)
+  * CapDL (PR#579)
+  * NuSMV (PR#564)
+  * SAS, Stata (PR#593)
+
+- Added the ability to load lexer and formatter classes directly from files
+  with the `-x` command line option and the `lexers.load_lexer_from_file()`
+  and `formatters.load_formatter_from_file()` functions. (PR#559)
+
+- Added `lexers.find_lexer_class_by_name()`. (#1203)
+
+- Added new token types and lexing for magic methods and variables in Python
+  and PHP.
+
+- Added a new token type for string affixes and lexing for them in Python, C++
+  and Postgresql lexers.
+
+- Added a new token type for heredoc (and similar) string delimiters and
+  lexing for them in C++, Perl, PHP, Postgresql and Ruby lexers.
+
+- Styles can now define colors with ANSI colors for use in the 256-color
+  terminal formatter. (PR#531)
+
+- Improved the CSS lexer. (#1083, #1130)
+
+- Added "Rainbow Dash" style. (PR#623)
+
+- Delay loading `pkg_resources`, which takes a long while to import. (PR#690)
+
+
+Version 2.1.3
+-------------
+(released Mar 2, 2016)
+
+- Fixed regression in Bash lexer (PR#563)
+
+
+Version 2.1.2
+-------------
+(released Feb 29, 2016)
+
+- Fixed Python 3 regression in image formatter (#1215)
+- Fixed regression in Bash lexer (PR#562)
+
+
+Version 2.1.1
+-------------
+(released Feb 14, 2016)
+
+- Fixed Jython compatibility (#1205)
+- Fixed HTML formatter output with leading empty lines (#1111)
+- Added a mapping table for LaTeX encodings and added utf8 (#1152)
+- Fixed image formatter font searching on Macs (#1188)
+- Fixed deepcopy-ing of Token instances (#1168)
+- Fixed Julia string interpolation (#1170)
+- Fixed statefulness of HttpLexer between get_tokens calls
+- Many smaller fixes to various lexers
+
+
+Version 2.1
+-----------
+(released Jan 17, 2016)
+
+- Added lexers:
+
+  * Emacs Lisp (PR#431)
+  * Arduino (PR#442)
+  * Modula-2 with multi-dialect support (#1090)
+  * Fortran fixed format (PR#213)
+  * Archetype Definition language (PR#483)
+  * Terraform (PR#432)
+  * Jcl, Easytrieve (PR#208)
+  * ParaSail (PR#381)
+  * Boogie (PR#420)
+  * Turtle (PR#425)
+  * Fish Shell (PR#422)
+  * Roboconf (PR#449)
+  * Test Anything Protocol (PR#428)
+  * Shen (PR#385)
+  * Component Pascal (PR#437)
+  * SuperCollider (PR#472)
+  * Shell consoles (Tcsh, PowerShell, MSDOS) (PR#479)
+  * Elm and J (PR#452)
+  * Crmsh (PR#440)
+  * Praat (PR#492)
+  * CSound (PR#494)
+  * Ezhil (PR#443)
+  * Thrift (PR#469)
+  * QVT Operational (PR#204)
+  * Hexdump (PR#508)
+  * CAmkES Configuration (PR#462)
+
+- Added styles:
+
+  * Lovelace (PR#456)
+  * Algol and Algol-nu (#1090)
+
+- Added formatters:
+
+  * IRC (PR#458)
+  * True color (24-bit) terminal ANSI sequences (#1142)
+    (formatter alias: "16m")
+
+- New "filename" option for HTML formatter (PR#527).
+
+- Improved performance of the HTML formatter for long lines (PR#504).
+
+- Updated autopygmentize script (PR#445).
+
+- Fixed style inheritance for non-standard token types in HTML output.
+
+- Added support for async/await to Python 3 lexer.
+
+- Rewrote linenos option for TerminalFormatter (it's better, but slightly
+  different output than before) (#1147).
+
+- Javascript lexer now supports most of ES6 (#1100).
+
+- Cocoa builtins updated for iOS 8.1 (PR#433).
+
+- Combined BashSessionLexer and ShellSessionLexer, new version should support
+  the prompt styles of either.
+
+- Added option to pygmentize to show a full traceback on exceptions.
+
+- Fixed incomplete output on Windows and Python 3 (e.g. when using iPython
+  Notebook) (#1153).
+
+- Allowed more traceback styles in Python console lexer (PR#253).
+
+- Added decorators to TypeScript (PR#509).
+
+- Fix highlighting of certain IRC logs formats (#1076).
+
+
+Version 2.0.2
+-------------
+(released Jan 20, 2015)
+
+- Fix Python tracebacks getting duplicated in the console lexer (#1068).
+
+- Backquote-delimited identifiers are now recognized in F# (#1062).
+
+
+Version 2.0.1
+-------------
+(released Nov 10, 2014)
+
+- Fix an encoding issue when using ``pygmentize`` with the ``-o`` option.
+
+
+Version 2.0
+-----------
+(released Nov 9, 2014)
+
+- Default lexer encoding is now "guess", i.e. UTF-8 / Locale / Latin1 is
+  tried in that order.
+
+- Major update to Swift lexer (PR#410).
+
+- Multiple fixes to lexer guessing in conflicting cases:
+
+  * recognize HTML5 by doctype
+  * recognize XML by XML declaration
+  * don't recognize C/C++ as SystemVerilog
+
+- Simplified regexes and builtin lists.
+
+
+Version 2.0rc1
+--------------
+(released Oct 16, 2014)
+
+- Dropped Python 2.4 and 2.5 compatibility.  This is in favor of single-source
+  compatibility between Python 2.6, 2.7 and 3.3+.
+
+- New website and documentation based on Sphinx (finally!)
+
+- Lexers added:
+
+  * APL (#969)
+  * Agda and Literate Agda (PR#203)
+  * Alloy (PR#355)
+  * AmbientTalk
+  * BlitzBasic (PR#197)
+  * ChaiScript (PR#24)
+  * Chapel (PR#256)
+  * Cirru (PR#275)
+  * Clay (PR#184)
+  * ColdFusion CFC (PR#283)
+  * Cryptol and Literate Cryptol (PR#344)
+  * Cypher (PR#257)
+  * Docker config files
+  * EBNF (PR#193)
+  * Eiffel (PR#273)
+  * GAP (PR#311)
+  * Golo (PR#309)
+  * Handlebars (PR#186)
+  * Hy (PR#238)
+  * Idris and Literate Idris (PR#210)
+  * Igor Pro (PR#172)
+  * Inform 6/7 (PR#281)
+  * Intel objdump (PR#279)
+  * Isabelle (PR#386)
+  * Jasmin (PR#349)
+  * JSON-LD (PR#289)
+  * Kal (PR#233)
+  * Lean (PR#399)
+  * LSL (PR#296)
+  * Limbo (PR#291)
+  * Liquid (#977)
+  * MQL (PR#285)
+  * MaskJS (PR#280)
+  * Mozilla preprocessors
+  * Mathematica (PR#245)
+  * NesC (PR#166)
+  * Nit (PR#375)
+  * Nix (PR#267)
+  * Pan
+  * Pawn (PR#211)
+  * Perl 6 (PR#181)
+  * Pig (PR#304)
+  * Pike (PR#237)
+  * QBasic (PR#182)
+  * Red (PR#341)
+  * ResourceBundle (#1038)
+  * Rexx (PR#199)
+  * Rql (PR#251)
+  * Rsl
+  * SPARQL (PR#78)
+  * Slim (PR#366)
+  * Swift (PR#371)
+  * Swig (PR#168)
+  * TADS 3 (PR#407)
+  * Todo.txt todo lists
+  * Twig (PR#404)
+
+- Added a helper to "optimize" regular expressions that match one of many
+  literal words; this can save 20% and more lexing time with lexers that
+  highlight many keywords or builtins.
+
+- New styles: "xcode" and "igor", similar to the default highlighting of
+  the respective IDEs.
+
+- The command-line "pygmentize" tool now tries a little harder to find the
+  correct encoding for files and the terminal (#979).
+
+- Added "inencoding" option for lexers to override "encoding" analogous
+  to "outencoding" (#800).
+
+- Added line-by-line "streaming" mode for pygmentize with the "-s" option.
+  (PR#165)  Only fully works for lexers that have no constructs spanning
+  lines!
+
+- Added an "envname" option to the LaTeX formatter to select a replacement
+  verbatim environment (PR#235).
+
+- Updated the Makefile lexer to yield a little more useful highlighting.
+
+- Lexer aliases passed to ``get_lexer_by_name()`` are now case-insensitive.
+
+- File name matching in lexers and formatters will now use a regex cache
+  for speed (PR#205).
+
+- Pygments will now recognize "vim" modelines when guessing the lexer for
+  a file based on content (PR#118).
+
+- Major restructure of the ``pygments.lexers`` module namespace.  There are now
+  many more modules with less lexers per module.  Old modules are still around
+  and re-export the lexers they previously contained.
+
+- The NameHighlightFilter now works with any Name.* token type (#790).
+
+- Python 3 lexer: add new exceptions from PEP 3151.
+
+- Opa lexer: add new keywords (PR#170).
+
+- Julia lexer: add keywords and underscore-separated number
+  literals (PR#176).
+
+- Lasso lexer: fix method highlighting, update builtins. Fix
+  guessing so that plain XML isn't always taken as Lasso (PR#163).
+
+- Objective C/C++ lexers: allow "@" prefixing any expression (#871).
+
+- Ruby lexer: fix lexing of Name::Space tokens (#860) and of symbols
+  in hashes (#873).
+
+- Stan lexer: update for version 2.4.0 of the language (PR#162, PR#255, PR#377).
+
+- JavaScript lexer: add the "yield" keyword (PR#196).
+
+- HTTP lexer: support for PATCH method (PR#190).
+
+- Koka lexer: update to newest language spec (PR#201).
+
+- Haxe lexer: rewrite and support for Haxe 3 (PR#174).
+
+- Prolog lexer: add different kinds of numeric literals (#864).
+
+- F# lexer: rewrite with newest spec for F# 3.0 (#842), fix a bug with
+  dotted chains (#948).
+
+- Kotlin lexer: general update (PR#271).
+
+- Rebol lexer: fix comment detection and analyse_text (PR#261).
+
+- LLVM lexer: update keywords to v3.4 (PR#258).
+
+- PHP lexer: add new keywords and binary literals (PR#222).
+
+- external/markdown-processor.py updated to newest python-markdown (PR#221).
+
+- CSS lexer: some highlighting order fixes (PR#231).
+
+- Ceylon lexer: fix parsing of nested multiline comments (#915).
+
+- C family lexers: fix parsing of indented preprocessor directives (#944).
+
+- Rust lexer: update to 0.9 language version (PR#270, PR#388).
+
+- Elixir lexer: update to 0.15 language version (PR#392).
+
+- Fix swallowing incomplete tracebacks in Python console lexer (#874).
+
+
+Version 1.6
+-----------
+(released Feb 3, 2013)
+
+- Lexers added:
+
+  * Dylan console (PR#149)
+  * Logos (PR#150)
+  * Shell sessions (PR#158)
+
+- Fix guessed lexers not receiving lexer options (#838).
+
+- Fix unquoted HTML attribute lexing in Opa (#841).
+
+- Fixes to the Dart lexer (PR#160).
+
+
+Version 1.6rc1
+--------------
+(released Jan 9, 2013)
+
+- Lexers added:
+
+  * AspectJ (PR#90)
+  * AutoIt (PR#122)
+  * BUGS-like languages (PR#89)
+  * Ceylon (PR#86)
+  * Croc (new name for MiniD)
+  * CUDA (PR#75)
+  * Dg (PR#116)
+  * IDL (PR#115)
+  * Jags (PR#89)
+  * Julia (PR#61)
+  * Kconfig (#711)
+  * Lasso (PR#95, PR#113)
+  * LiveScript (PR#84)
+  * Monkey (PR#117)
+  * Mscgen (PR#80)
+  * NSIS scripts (PR#136)
+  * OpenCOBOL (PR#72)
+  * QML (PR#123)
+  * Puppet (PR#133)
+  * Racket (PR#94)
+  * Rdoc (PR#99)
+  * Robot Framework (PR#137)
+  * RPM spec files (PR#124)
+  * Rust (PR#67)
+  * Smali (Dalvik assembly)
+  * SourcePawn (PR#39)
+  * Stan (PR#89)
+  * Treetop (PR#125)
+  * TypeScript (PR#114)
+  * VGL (PR#12)
+  * Visual FoxPro (#762)
+  * Windows Registry (#819)
+  * Xtend (PR#68)
+
+- The HTML formatter now supports linking to tags using CTags files, when the
+  python-ctags package is installed (PR#87).
+
+- The HTML formatter now has a "linespans" option that wraps every line in a
+   tag with a specific id (PR#82).
+
+- When deriving a lexer from another lexer with token definitions, definitions
+  for states not in the child lexer are now inherited.  If you override a state
+  in the child lexer, an "inherit" keyword has been added to insert the base
+  state at that position (PR#141).
+
+- The C family lexers now inherit token definitions from a common base class,
+  removing code duplication (PR#141).
+
+- Use "colorama" on Windows for console color output (PR#142).
+
+- Fix Template Haskell highlighting (PR#63).
+
+- Fix some S/R lexer errors (PR#91).
+
+- Fix a bug in the Prolog lexer with names that start with 'is' (#810).
+
+- Rewrite Dylan lexer, add Dylan LID lexer (PR#147).
+
+- Add a Java quickstart document (PR#146).
+
+- Add a "external/autopygmentize" file that can be used as .lessfilter (#802).
+
+
+Version 1.5
+-----------
+(codename Zeitdilatation, released Mar 10, 2012)
+
+- Lexers added:
+
+  * Awk (#630)
+  * Fancy (#633)
+  * PyPy Log
+  * eC
+  * Nimrod
+  * Nemerle (#667)
+  * F# (#353)
+  * Groovy (#501)
+  * PostgreSQL (#660)
+  * DTD
+  * Gosu (#634)
+  * Octave (PR#22)
+  * Standard ML (PR#14)
+  * CFengine3 (#601)
+  * Opa (PR#37)
+  * HTTP sessions (PR#42)
+  * JSON (PR#31)
+  * SNOBOL (PR#30)
+  * MoonScript (PR#43)
+  * ECL (PR#29)
+  * Urbiscript (PR#17)
+  * OpenEdge ABL (PR#27)
+  * SystemVerilog (PR#35)
+  * Coq (#734)
+  * PowerShell (#654)
+  * Dart (#715)
+  * Fantom (PR#36)
+  * Bro (PR#5)
+  * NewLISP (PR#26)
+  * VHDL (PR#45)
+  * Scilab (#740)
+  * Elixir (PR#57)
+  * Tea (PR#56)
+  * Kotlin (PR#58)
+
+- Fix Python 3 terminal highlighting with pygmentize (#691).
+
+- In the LaTeX formatter, escape special &, < and > chars (#648).
+
+- In the LaTeX formatter, fix display problems for styles with token
+  background colors (#670).
+
+- Enhancements to the Squid conf lexer (#664).
+
+- Several fixes to the reStructuredText lexer (#636).
+
+- Recognize methods in the ObjC lexer (#638).
+
+- Fix Lua "class" highlighting: it does not have classes (#665).
+
+- Fix degenerate regex in Scala lexer (#671) and highlighting bugs (#713, 708).
+
+- Fix number pattern order in Ocaml lexer (#647).
+
+- Fix generic type highlighting in ActionScript 3 (#666).
+
+- Fixes to the Clojure lexer (PR#9).
+
+- Fix degenerate regex in Nemerle lexer (#706).
+
+- Fix infinite looping in CoffeeScript lexer (#729).
+
+- Fix crashes and analysis with ObjectiveC lexer (#693, #696).
+
+- Add some Fortran 2003 keywords.
+
+- Fix Boo string regexes (#679).
+
+- Add "rrt" style (#727).
+
+- Fix infinite looping in Darcs Patch lexer.
+
+- Lots of misc fixes to character-eating bugs and ordering problems in many
+  different lexers.
+
+
+Version 1.4
+-----------
+(codename Unschärfe, released Jan 03, 2011)
+
+- Lexers added:
+
+  * Factor (#520)
+  * PostScript (#486)
+  * Verilog (#491)
+  * BlitzMax Basic (#478)
+  * Ioke (#465)
+  * Java properties, split out of the INI lexer (#445)
+  * Scss (#509)
+  * Duel/JBST
+  * XQuery (#617)
+  * Mason (#615)
+  * GoodData (#609)
+  * SSP (#473)
+  * Autohotkey (#417)
+  * Google Protocol Buffers
+  * Hybris (#506)
+
+- Do not fail in analyse_text methods (#618).
+
+- Performance improvements in the HTML formatter (#523).
+
+- With the ``noclasses`` option in the HTML formatter, some styles
+  present in the stylesheet were not added as inline styles.
+
+- Four fixes to the Lua lexer (#480, #481, #482, #497).
+
+- More context-sensitive Gherkin lexer with support for more i18n translations.
+
+- Support new OO keywords in Matlab lexer (#521).
+
+- Small fix in the CoffeeScript lexer (#519).
+
+- A bugfix for backslashes in ocaml strings (#499).
+
+- Fix unicode/raw docstrings in the Python lexer (#489).
+
+- Allow PIL to work without PIL.pth (#502).
+
+- Allow seconds as a unit in CSS (#496).
+
+- Support ``application/javascript`` as a JavaScript mime type (#504).
+
+- Support `Offload `_ C++ Extensions as
+  keywords in the C++ lexer (#484).
+
+- Escape more characters in LaTeX output (#505).
+
+- Update Haml/Sass lexers to version 3 (#509).
+
+- Small PHP lexer string escaping fix (#515).
+
+- Support comments before preprocessor directives, and unsigned/
+  long long literals in C/C++ (#613, #616).
+
+- Support line continuations in the INI lexer (#494).
+
+- Fix lexing of Dylan string and char literals (#628).
+
+- Fix class/procedure name highlighting in VB.NET lexer (#624).
+
+
+Version 1.3.1
+-------------
+(bugfix release, released Mar 05, 2010)
+
+- The ``pygmentize`` script was missing from the distribution.
+
+
+Version 1.3
+-----------
+(codename Schneeglöckchen, released Mar 01, 2010)
+
+- Added the ``ensurenl`` lexer option, which can be used to suppress the
+  automatic addition of a newline to the lexer input.
+
+- Lexers added:
+
+  * Ada
+  * Coldfusion
+  * Modula-2
+  * Haxe
+  * R console
+  * Objective-J
+  * Haml and Sass
+  * CoffeeScript
+
+- Enhanced reStructuredText highlighting.
+
+- Added support for PHP 5.3 namespaces in the PHP lexer.
+
+- Added a bash completion script for `pygmentize`, to the external/
+  directory (#466).
+
+- Fixed a bug in `do_insertions()` used for multi-lexer languages.
+
+- Fixed a Ruby regex highlighting bug (#476).
+
+- Fixed regex highlighting bugs in Perl lexer (#258).
+
+- Add small enhancements to the C lexer (#467) and Bash lexer (#469).
+
+- Small fixes for the Tcl, Debian control file, Nginx config,
+  Smalltalk, Objective-C, Clojure, Lua lexers.
+
+- Gherkin lexer: Fixed single apostrophe bug and added new i18n keywords.
+
+
+Version 1.2.2
+-------------
+(bugfix release, released Jan 02, 2010)
+
+* Removed a backwards incompatibility in the LaTeX formatter that caused
+  Sphinx to produce invalid commands when writing LaTeX output (#463).
+
+* Fixed a forever-backtracking regex in the BashLexer (#462).
+
+
+Version 1.2.1
+-------------
+(bugfix release, released Jan 02, 2010)
+
+* Fixed mishandling of an ellipsis in place of the frames in a Python
+  console traceback, resulting in clobbered output.
+
+
+Version 1.2
+-----------
+(codename Neujahr, released Jan 01, 2010)
+
+- Dropped Python 2.3 compatibility.
+
+- Lexers added:
+
+  * Asymptote
+  * Go
+  * Gherkin (Cucumber)
+  * CMake
+  * Ooc
+  * Coldfusion
+  * Haxe
+  * R console
+
+- Added options for rendering LaTeX in source code comments in the
+  LaTeX formatter (#461).
+
+- Updated the Logtalk lexer.
+
+- Added `line_number_start` option to image formatter (#456).
+
+- Added `hl_lines` and `hl_color` options to image formatter (#457).
+
+- Fixed the HtmlFormatter's handling of noclasses=True to not output any
+  classes (#427).
+
+- Added the Monokai style (#453).
+
+- Fixed LLVM lexer identifier syntax and added new keywords (#442).
+
+- Fixed the PythonTracebackLexer to handle non-traceback data in header or
+  trailer, and support more partial tracebacks that start on line 2 (#437).
+
+- Fixed the CLexer to not highlight ternary statements as labels.
+
+- Fixed lexing of some Ruby quoting peculiarities (#460).
+
+- A few ASM lexer fixes (#450).
+
+
+Version 1.1.1
+-------------
+(bugfix release, released Sep 15, 2009)
+
+- Fixed the BBCode lexer (#435).
+
+- Added support for new Jinja2 keywords.
+
+- Fixed test suite failures.
+
+- Added Gentoo-specific suffixes to Bash lexer.
+
+
+Version 1.1
+-----------
+(codename Brillouin, released Sep 11, 2009)
+
+- Ported Pygments to Python 3.  This needed a few changes in the way
+  encodings are handled; they may affect corner cases when used with
+  Python 2 as well.
+
+- Lexers added:
+
+  * Antlr/Ragel, thanks to Ana Nelson
+  * (Ba)sh shell
+  * Erlang shell
+  * GLSL
+  * Prolog
+  * Evoque
+  * Modelica
+  * Rebol
+  * MXML
+  * Cython
+  * ABAP
+  * ASP.net (VB/C#)
+  * Vala
+  * Newspeak
+
+- Fixed the LaTeX formatter's output so that output generated for one style
+  can be used with the style definitions of another (#384).
+
+- Added "anchorlinenos" and "noclobber_cssfile" (#396) options to HTML
+  formatter.
+
+- Support multiline strings in Lua lexer.
+
+- Rewrite of the JavaScript lexer by Pumbaa80 to better support regular
+  expression literals (#403).
+
+- When pygmentize is asked to highlight a file for which multiple lexers
+  match the filename, use the analyse_text guessing engine to determine the
+  winner (#355).
+
+- Fixed minor bugs in the JavaScript lexer (#383), the Matlab lexer (#378),
+  the Scala lexer (#392), the INI lexer (#391), the Clojure lexer (#387)
+  and the AS3 lexer (#389).
+
+- Fixed three Perl heredoc lexing bugs (#379, #400, #422).
+
+- Fixed a bug in the image formatter which misdetected lines (#380).
+
+- Fixed bugs lexing extended Ruby strings and regexes.
+
+- Fixed a bug when lexing git diffs.
+
+- Fixed a bug lexing the empty commit in the PHP lexer (#405).
+
+- Fixed a bug causing Python numbers to be mishighlighted as floats (#397).
+
+- Fixed a bug when backslashes are used in odd locations in Python (#395).
+
+- Fixed various bugs in Matlab and S-Plus lexers, thanks to Winston Chang (#410,
+  #411, #413, #414) and fmarc (#419).
+
+- Fixed a bug in Haskell single-line comment detection (#426).
+
+- Added new-style reStructuredText directive for docutils 0.5+ (#428).
+
+
+Version 1.0
+-----------
+(codename Dreiundzwanzig, released Nov 23, 2008)
+
+- Don't use join(splitlines()) when converting newlines to ``\n``,
+  because that doesn't keep all newlines at the end when the
+  ``stripnl`` lexer option is False.
+
+- Added ``-N`` option to command-line interface to get a lexer name
+  for a given filename.
+
+- Added Tango style, written by Andre Roberge for the Crunchy project.
+
+- Added Python3TracebackLexer and ``python3`` option to
+  PythonConsoleLexer.
+
+- Fixed a few bugs in the Haskell lexer.
+
+- Fixed PythonTracebackLexer to be able to recognize SyntaxError and
+  KeyboardInterrupt (#360).
+
+- Provide one formatter class per image format, so that surprises like::
+
+    pygmentize -f gif -o foo.gif foo.py
+
+  creating a PNG file are avoided.
+
+- Actually use the `font_size` option of the image formatter.
+
+- Fixed numpy lexer that it doesn't listen for `*.py` any longer.
+
+- Fixed HTML formatter so that text options can be Unicode
+  strings (#371).
+
+- Unified Diff lexer supports the "udiff" alias now.
+
+- Fixed a few issues in Scala lexer (#367).
+
+- RubyConsoleLexer now supports simple prompt mode (#363).
+
+- JavascriptLexer is smarter about what constitutes a regex (#356).
+
+- Add Applescript lexer, thanks to Andreas Amann (#330).
+
+- Make the codetags more strict about matching words (#368).
+
+- NginxConfLexer is a little more accurate on mimetypes and
+  variables (#370).
+
+
+Version 0.11.1
+--------------
+(released Aug 24, 2008)
+
+- Fixed a Jython compatibility issue in pygments.unistring (#358).
+
+
+Version 0.11
+------------
+(codename Straußenei, released Aug 23, 2008)
+
+Many thanks go to Tim Hatch for writing or integrating most of the bug
+fixes and new features.
+
+- Lexers added:
+
+  * Nasm-style assembly language, thanks to delroth
+  * YAML, thanks to Kirill Simonov
+  * ActionScript 3, thanks to Pierre Bourdon
+  * Cheetah/Spitfire templates, thanks to Matt Good
+  * Lighttpd config files
+  * Nginx config files
+  * Gnuplot plotting scripts
+  * Clojure
+  * POV-Ray scene files
+  * Sqlite3 interactive console sessions
+  * Scala source files, thanks to Krzysiek Goj
+
+- Lexers improved:
+
+  * C lexer highlights standard library functions now and supports C99
+    types.
+  * Bash lexer now correctly highlights heredocs without preceding
+    whitespace.
+  * Vim lexer now highlights hex colors properly and knows a couple
+    more keywords.
+  * Irc logs lexer now handles xchat's default time format (#340) and
+    correctly highlights lines ending in ``>``.
+  * Support more delimiters for perl regular expressions (#258).
+  * ObjectiveC lexer now supports 2.0 features.
+
+- Added "Visual Studio" style.
+
+- Updated markdown processor to Markdown 1.7.
+
+- Support roman/sans/mono style defs and use them in the LaTeX
+  formatter.
+
+- The RawTokenFormatter is no longer registered to ``*.raw`` and it's
+  documented that tokenization with this lexer may raise exceptions.
+
+- New option ``hl_lines`` to HTML formatter, to highlight certain
+  lines.
+
+- New option ``prestyles`` to HTML formatter.
+
+- New option *-g* to pygmentize, to allow lexer guessing based on
+  filetext (can be slowish, so file extensions are still checked
+  first).
+
+- ``guess_lexer()`` now makes its decision much faster due to a cache
+  of whether data is xml-like (a check which is used in several
+  versions of ``analyse_text()``.  Several lexers also have more
+  accurate ``analyse_text()`` now.
+
+
+Version 0.10
+------------
+(codename Malzeug, released May 06, 2008)
+
+- Lexers added:
+
+  * Io
+  * Smalltalk
+  * Darcs patches
+  * Tcl
+  * Matlab
+  * Matlab sessions
+  * FORTRAN
+  * XSLT
+  * tcsh
+  * NumPy
+  * Python 3
+  * S, S-plus, R statistics languages
+  * Logtalk
+
+- In the LatexFormatter, the *commandprefix* option is now by default
+  'PY' instead of 'C', since the latter resulted in several collisions
+  with other packages.  Also, the special meaning of the *arg*
+  argument to ``get_style_defs()`` was removed.
+
+- Added ImageFormatter, to format code as PNG, JPG, GIF or BMP.
+  (Needs the Python Imaging Library.)
+
+- Support doc comments in the PHP lexer.
+
+- Handle format specifications in the Perl lexer.
+
+- Fix comment handling in the Batch lexer.
+
+- Add more file name extensions for the C++, INI and XML lexers.
+
+- Fixes in the IRC and MuPad lexers.
+
+- Fix function and interface name highlighting in the Java lexer.
+
+- Fix at-rule handling in the CSS lexer.
+
+- Handle KeyboardInterrupts gracefully in pygmentize.
+
+- Added BlackWhiteStyle.
+
+- Bash lexer now correctly highlights math, does not require
+  whitespace after semicolons, and correctly highlights boolean
+  operators.
+
+- Makefile lexer is now capable of handling BSD and GNU make syntax.
+
+
+Version 0.9
+-----------
+(codename Herbstzeitlose, released Oct 14, 2007)
+
+- Lexers added:
+
+  * Erlang
+  * ActionScript
+  * Literate Haskell
+  * Common Lisp
+  * Various assembly languages
+  * Gettext catalogs
+  * Squid configuration
+  * Debian control files
+  * MySQL-style SQL
+  * MOOCode
+
+- Lexers improved:
+
+  * Greatly improved the Haskell and OCaml lexers.
+  * Improved the Bash lexer's handling of nested constructs.
+  * The C# and Java lexers exhibited abysmal performance with some
+    input code; this should now be fixed.
+  * The IRC logs lexer is now able to colorize weechat logs too.
+  * The Lua lexer now recognizes multi-line comments.
+  * Fixed bugs in the D and MiniD lexer.
+
+- The encoding handling of the command line mode (pygmentize) was
+  enhanced. You shouldn't get UnicodeErrors from it anymore if you
+  don't give an encoding option.
+
+- Added a ``-P`` option to the command line mode which can be used to
+  give options whose values contain commas or equals signs.
+
+- Added 256-color terminal formatter.
+
+- Added an experimental SVG formatter.
+
+- Added the ``lineanchors`` option to the HTML formatter, thanks to
+  Ian Charnas for the idea.
+
+- Gave the line numbers table a CSS class in the HTML formatter.
+
+- Added a Vim 7-like style.
+
+
+Version 0.8.1
+-------------
+(released Jun 27, 2007)
+
+- Fixed POD highlighting in the Ruby lexer.
+
+- Fixed Unicode class and namespace name highlighting in the C# lexer.
+
+- Fixed Unicode string prefix highlighting in the Python lexer.
+
+- Fixed a bug in the D and MiniD lexers.
+
+- Fixed the included MoinMoin parser.
+
+
+Version 0.8
+-----------
+(codename Maikäfer, released May 30, 2007)
+
+- Lexers added:
+
+  * Haskell, thanks to Adam Blinkinsop
+  * Redcode, thanks to Adam Blinkinsop
+  * D, thanks to Kirk McDonald
+  * MuPad, thanks to Christopher Creutzig
+  * MiniD, thanks to Jarrett Billingsley
+  * Vim Script, by Tim Hatch
+
+- The HTML formatter now has a second line-numbers mode in which it
+  will just integrate the numbers in the same ``
`` tag as the
+  code.
+
+- The `CSharpLexer` now is Unicode-aware, which means that it has an
+  option that can be set so that it correctly lexes Unicode
+  identifiers allowed by the C# specs.
+
+- Added a `RaiseOnErrorTokenFilter` that raises an exception when the
+  lexer generates an error token, and a `VisibleWhitespaceFilter` that
+  converts whitespace (spaces, tabs, newlines) into visible
+  characters.
+
+- Fixed the `do_insertions()` helper function to yield correct
+  indices.
+
+- The ReST lexer now automatically highlights source code blocks in
+  ".. sourcecode:: language" and ".. code:: language" directive
+  blocks.
+
+- Improved the default style (thanks to Tiberius Teng). The old
+  default is still available as the "emacs" style (which was an alias
+  before).
+
+- The `get_style_defs` method of HTML formatters now uses the
+  `cssclass` option as the default selector if it was given.
+
+- Improved the ReST and Bash lexers a bit.
+
+- Fixed a few bugs in the Makefile and Bash lexers, thanks to Tim
+  Hatch.
+
+- Fixed a bug in the command line code that disallowed ``-O`` options
+  when using the ``-S`` option.
+
+- Fixed a bug in the `RawTokenFormatter`.
+
+
+Version 0.7.1
+-------------
+(released Feb 15, 2007)
+
+- Fixed little highlighting bugs in the Python, Java, Scheme and
+  Apache Config lexers.
+
+- Updated the included manpage.
+
+- Included a built version of the documentation in the source tarball.
+
+
+Version 0.7
+-----------
+(codename Faschingskrapfn, released Feb 14, 2007)
+
+- Added a MoinMoin parser that uses Pygments. With it, you get
+  Pygments highlighting in Moin Wiki pages.
+
+- Changed the exception raised if no suitable lexer, formatter etc. is
+  found in one of the `get_*_by_*` functions to a custom exception,
+  `pygments.util.ClassNotFound`. It is, however, a subclass of
+  `ValueError` in order to retain backwards compatibility.
+
+- Added a `-H` command line option which can be used to get the
+  docstring of a lexer, formatter or filter.
+
+- Made the handling of lexers and formatters more consistent. The
+  aliases and filename patterns of formatters are now attributes on
+  them.
+
+- Added an OCaml lexer, thanks to Adam Blinkinsop.
+
+- Made the HTML formatter more flexible, and easily subclassable in
+  order to make it easy to implement custom wrappers, e.g. alternate
+  line number markup. See the documentation.
+
+- Added an `outencoding` option to all formatters, making it possible
+  to override the `encoding` (which is used by lexers and formatters)
+  when using the command line interface. Also, if using the terminal
+  formatter and the output file is a terminal and has an encoding
+  attribute, use it if no encoding is given.
+
+- Made it possible to just drop style modules into the `styles`
+  subpackage of the Pygments installation.
+
+- Added a "state" keyword argument to the `using` helper.
+
+- Added a `commandprefix` option to the `LatexFormatter` which allows
+  to control how the command names are constructed.
+
+- Added quite a few new lexers, thanks to Tim Hatch:
+
+  * Java Server Pages
+  * Windows batch files
+  * Trac Wiki markup
+  * Python tracebacks
+  * ReStructuredText
+  * Dylan
+  * and the Befunge esoteric programming language (yay!)
+
+- Added Mako lexers by Ben Bangert.
+
+- Added "fruity" style, another dark background originally vim-based
+  theme.
+
+- Added sources.list lexer by Dennis Kaarsemaker.
+
+- Added token stream filters, and a pygmentize option to use them.
+
+- Changed behavior of `in` Operator for tokens.
+
+- Added mimetypes for all lexers.
+
+- Fixed some problems lexing Python strings.
+
+- Fixed tickets: #167, #178, #179, #180, #185, #201.
+
+
+Version 0.6
+-----------
+(codename Zimtstern, released Dec 20, 2006)
+
+- Added option for the HTML formatter to write the CSS to an external
+  file in "full document" mode.
+
+- Added RTF formatter.
+
+- Added Bash and Apache configuration lexers (thanks to Tim Hatch).
+
+- Improved guessing methods for various lexers.
+
+- Added `@media` support to CSS lexer (thanks to Tim Hatch).
+
+- Added a Groff lexer (thanks to Tim Hatch).
+
+- License change to BSD.
+
+- Added lexers for the Myghty template language.
+
+- Added a Scheme lexer (thanks to Marek Kubica).
+
+- Added some functions to iterate over existing lexers, formatters and
+  lexers.
+
+- The HtmlFormatter's `get_style_defs()` can now take a list as an
+  argument to generate CSS with multiple prefixes.
+
+- Support for guessing input encoding added.
+
+- Encoding support added: all processing is now done with Unicode
+  strings, input and output are converted from and optionally to byte
+  strings (see the ``encoding`` option of lexers and formatters).
+
+- Some improvements in the C(++) lexers handling comments and line
+  continuations.
+
+
+Version 0.5.1
+-------------
+(released Oct 30, 2006)
+
+- Fixed traceback in ``pygmentize -L`` (thanks to Piotr Ozarowski).
+
+
+Version 0.5
+-----------
+(codename PyKleur, released Oct 30, 2006)
+
+- Initial public release.
diff --git a/Contributing.md b/Contributing.md
new file mode 100644
index 0000000..93da428
--- /dev/null
+++ b/Contributing.md
@@ -0,0 +1,167 @@
+Licensing
+=========
+
+The code is distributed under the BSD 2-clause license. Contributors making pull
+requests must agree that they are able and willing to put their contributions
+under that license.
+
+Goals & non-goals of Pygments
+=============================
+
+Python support
+--------------
+
+Pygments supports all supported Python versions as per the [Python Developer's Guide](https://devguide.python.org/versions/). Additionally, the default Python version of the latest stable version of RHEL, Ubuntu LTS, and Debian are supported, even if they're officially EOL. Supporting other end-of-life versions is a non-goal of Pygments.
+
+Validation
+----------
+
+Pygments does not attempt to validate the input. Accepting code that is not legal for a given language is acceptable if it simplifies the codebase and does not result in surprising behavior. For instance, in C89, accepting `//` based comments would be fine because de-facto all compilers supported it, and having a separate lexer for it would not be worth it.
+
+Contribution checklist
+======================
+
+* Check the documentation for how to write
+  [a new lexer](https://pygments.org/docs/lexerdevelopment/),
+  [a new formatter](https://pygments.org/docs/formatterdevelopment/) or
+  [a new filter](https://pygments.org/docs/filterdevelopment/)
+
+* Make sure to add a test for your new functionality, and where applicable,
+  write documentation.
+
+* When writing rules, try to merge simple rules. For instance, combine:
+
+  ```python
+  _PUNCTUATION = [
+    (r"\(", token.Punctuation),
+    (r"\)", token.Punctuation),
+    (r"\[", token.Punctuation),
+    (r"\]", token.Punctuation),
+    ("{", token.Punctuation),
+    ("}", token.Punctuation),
+  ]
+  ```
+
+  into:
+
+  ```python
+  (r"[\(\)\[\]{}]", token.Punctuation)
+  ```
+
+* Be careful with ``.*``. This matches greedily as much as it can. For instance,
+  a rule like ``@.*@`` will match the whole string ``@first@ second @third@``,
+  instead of matching ``@first@`` and ``@second@``. You can use ``@.*?@`` in
+  this case to stop early. The ``?`` tries to match _as few times_ as possible.
+
+* Beware of so-called "catastrophic backtracking".  As a first example, consider
+  the regular expression ``(A+)*C``.  This is equivalent to ``A*B`` regarding
+  what it matches, but *non*-matches will take very long.  This is because
+  of the way the regular expression engine works.  Suppose you feed it 50
+  'A's, and a 'C' at the end.  It first matches the 'A's greedily in ``A+``,
+  but finds that it cannot match the end since 'B' is not the same as 'C'.
+  Then it backtracks, removing one 'A' from the first ``A+`` and trying to
+  match the rest as another ``(A+)*``.  This fails again, so it backtracks
+  further left in the input string, etc.  In effect, it tries all combinations
+
+  ```
+  (AAAAAAAAAAAAAAAAA)
+  (AAAAAAAAAAAAAAAA)(A)
+  (AAAAAAAAAAAAAAA)(AA)
+  (AAAAAAAAAAAAAAA)(A)(A)
+  (AAAAAAAAAAAAAA)(AAA)
+  (AAAAAAAAAAAAAA)(AA)(A)
+  ...
+  ```
+
+  Thus, the matching has exponential complexity.  In a lexer, the
+  effect is that Pygments will seemingly hang when parsing invalid
+  input.
+
+  ```python
+  >>> import re
+  >>> re.match('(A+)*B', 'A'*50 + 'C') # hangs
+  ```
+
+  As a more subtle and real-life example, here is a badly written
+  regular expression to match strings:
+
+  ```python
+  r'"(\\?.)*?"'
+  ```
+
+  If the ending quote is missing, the regular expression engine will
+  find that it cannot match at the end, and try to backtrack with less
+  matches in the ``*?``.  When it finds a backslash, as it has already
+  tried the possibility ``\\.``, it tries ``.`` (recognizing it as a
+  simple character without meaning), which leads to the same
+  exponential backtracking problem if there are lots of backslashes in
+  the (invalid) input string.  A good way to write this would be
+  ``r'"([^\\]|\\.)*?"'``, where the inner group can only match in one
+  way.  Better yet is to use a dedicated state, which not only
+  sidesteps the issue without headaches, but allows you to highlight
+  string escapes.
+
+  ```python
+  'root': [
+      ...,
+      (r'"', String, 'string'),
+      ...
+  ],
+  'string': [
+      (r'\\.', String.Escape),
+      (r'"', String, '#pop'),
+      (r'[^\\"]+', String),
+  ]
+  ```
+
+* When writing rules for patterns such as comments or strings, match as many
+  characters as possible in each token.  This is an example of what not to
+  do:
+
+  ```python
+  'comment': [
+      (r'\*/', Comment.Multiline, '#pop'),
+      (r'.', Comment.Multiline),
+  ]
+  ```
+
+  This generates one token per character in the comment, which slows
+  down the lexing process, and also makes the raw token output (and in
+  particular the test output) hard to read.  Do this instead:
+
+  ```python
+  'comment': [
+      (r'\*/', Comment.Multiline, '#pop'),
+      (r'[^*]+', Comment.Multiline),
+      (r'\*', Comment.Multiline),
+  ]
+  ```
+
+* Don't add imports of your lexer anywhere in the codebase. (In case you're
+  curious about ``compiled.py`` -- this file exists for backwards compatibility
+  reasons.)
+
+* Use the standard importing convention: ``from token import Punctuation``
+
+* For test cases that assert on the tokens produced by a lexer, use tools:
+
+  * You can use the ``testcase`` formatter to produce a piece of code that
+    can be pasted into a unittest file:
+    ``python -m pygments -l lua -f testcase <<< "local a = 5"``
+
+  * Most snippets should instead be put as a sample file under
+    ``tests/snippets//*.txt``. These files are automatically
+    picked up as individual tests, asserting that the input produces the
+    expected tokens.
+
+    To add a new test, create a file with just your code snippet under a
+    subdirectory based on your lexer's main alias. Then run
+    ``pytest --update-goldens `` to auto-populate the currently
+    expected tokens. Check that they look good and check in the file.
+
+    Also run the same command whenever you need to update the test if the
+    actual produced tokens change (assuming the change is expected).
+
+  * Large test files should go in ``tests/examplefiles``.  This works
+    similar to ``snippets``, but the token output is stored in a separate
+    file.  Output can also be regenerated with ``--update-goldens``.
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..446a1a8
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,25 @@
+Copyright (c) 2006-2022 by the respective authors (see AUTHORS file).
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 0000000..c6a8567
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1,5 @@
+include Makefile CHANGES LICENSE AUTHORS
+include external/*
+recursive-include tests *
+recursive-include doc *
+recursive-include scripts *
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..e674732
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,68 @@
+#
+# Makefile for Pygments
+# ~~~~~~~~~~~~~~~~~~~~~
+#
+# Combines scripts for common tasks.
+#
+# :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS.
+# :license: BSD, see LICENSE for details.
+#
+
+PYTHON ?= python3
+
+export PYTHONPATH = $(shell echo "$$PYTHONPATH"):$(shell python -c 'import os; print ":".join(os.path.abspath(line.strip()) for line in file("PYTHONPATH"))' 2>/dev/null)
+
+.PHONY: all check clean clean-pyc docs mapfiles \
+	pylint reindent test test-coverage \
+	tox-test tox-test-coverage regexlint
+
+all: clean-pyc check test
+
+check:
+	@$(PYTHON) scripts/check_crlf.py pygments build external
+	@$(PYTHON) scripts/detect_missing_analyse_text.py --skip-no-aliases
+	@pyflakes pygments | grep -v 'but unused' || true
+	@$(PYTHON) scripts/check_sources.py -i build -i dist -i pygments/lexers/_mapping.py \
+		   -i docs/build -i pygments/formatters/_mapping.py -i pygments/unistring.py \
+		   -i tests/support/empty.py
+	@$(PYTHON) scripts/count_token_references.py --minfiles=1 --maxfiles=1 \
+		   --minlines=1 --maxlines=3 --subtoken
+
+clean: clean-pyc
+	-rm -rf doc/_build build Pygments.egg-info
+	-rm -f codetags.html
+
+clean-pyc:
+	find . -name '__pycache__' -exec rm -rf {} +
+
+docs:
+	make -C doc html
+
+mapfiles:
+	$(PYTHON) scripts/gen_mapfiles.py
+
+pylint:
+	@pylint --rcfile scripts/pylintrc pygments
+
+reindent:
+	@$(PYTHON) scripts/reindent.py -r -B .
+
+TEST = tests
+
+test:
+	@$(PYTHON) -m pytest $(TEST)
+
+test-coverage:
+	@$(PYTHON) -m pytest --cov --cov-report=html --cov-report=term $(TEST)
+
+tox-test:
+	@tox -- $(TEST)
+
+tox-test-coverage:
+	@tox -- --with-coverage --cover-package=pygments --cover-erase $(TEST)
+
+RLMODULES = pygments.lexers
+
+regexlint:
+	@if [ -z "$(REGEXLINT)" ]; then echo "Please set REGEXLINT=checkout path"; exit 1; fi
+	PYTHONPATH=`pwd`:$(REGEXLINT) $(PYTHON) $(REGEXLINT)/regexlint/cmdline.py $(RLMODULES)
diff --git a/README.rst b/README.rst
new file mode 100644
index 0000000..204e46b
--- /dev/null
+++ b/README.rst
@@ -0,0 +1,93 @@
+Welcome to Pygments
+===================
+
+This is the source of Pygments.  It is a **generic syntax highlighter** written
+in Python that supports over 500 languages and text formats, for use in code
+hosting, forums, wikis or other applications that need to prettify source code.
+
+Installing
+----------
+
+... works as usual, use ``pip install Pygments`` to get published versions,
+or ``python setup.py install`` to install from a checkout.
+
+Documentation
+-------------
+
+... can be found online at https://pygments.org/ or created with Sphinx by ::
+
+   make docs
+
+By default, the documentation does not include the demo page, as it requires
+having Docker installed for building Pyodide. To build the documentation with
+the demo page, use ::
+
+   WEBSITE_BUILD=1 make docs
+
+The initial build might take some time, but subsequent ones should be instant
+because of Docker caching.
+
+To view the generated documentation, serve it using Python's ``http.server``
+module (this step is required for the demo to work) ::
+
+   python3 -m http.server --directory doc/_build/html
+
+
+Development
+-----------
+
+... takes place on `GitHub `_, where the
+Git repository, tickets and pull requests can be viewed.
+
+Continuous testing runs on GitHub workflows:
+
+.. image:: https://github.com/pygments/pygments/workflows/Pygments/badge.svg
+   :target: https://github.com/pygments/pygments/actions?query=workflow%3APygments
+
+Contribution guidelines are found in Contributing.md_.
+
+.. _Contributing.md: https://github.com/pygments/pygments/blob/master/Contributing.md
+
+Security considerations
+-----------------------
+
+Pygments provides no guarantees on execution time, which needs to be taken
+into consideration when using Pygments to process arbitrary user inputs. For
+example, if you have a web service which uses Pygments for highlighting, there
+may be inputs which will cause the Pygments process to run "forever" and/or use
+significant amounts of memory. This can subsequently be used to perform a
+remote denial-of-service attack on the server if the processes are not
+terminated quickly.
+
+Unfortunately, it's practically impossible to harden Pygments itself against 
+those issues: Some regular expressions can result in "catastrophic 
+backtracking", but other bugs like incorrect matchers can also
+cause similar problems, and there is no way to find them in an automated fashion
+(short of solving the halting problem.) Pygments has extensive unit tests, 
+automated randomized testing, and is also tested by `OSS-Fuzz `_, 
+but we will never be able to eliminate all bugs in this area.
+
+Our recommendations are:
+
+* Ensure that the Pygments process is *terminated* after a reasonably short
+  timeout. In general Pygments should take seconds at most for reasonably-sized
+  input.
+* *Limit* the number of concurrent Pygments processes to avoid oversubscription
+  of resources.
+
+The Pygments authors will treat any bug resulting in long processing times with
+high priority -- it's one of those things that will be fixed in a patch release.
+When reporting a bug where you suspect super-linear execution times, please make
+sure to attach an input to reproduce it.
+
+The authors
+-----------
+
+Pygments is maintained by **Georg Brandl**, e-mail address *georg*\ *@*\ *python.org*, **Matthäus Chajdas** and **Jean Abou-Samra**.
+
+Many lexers and fixes have been contributed by **Armin Ronacher**, the rest of
+the `Pocoo `_ team and **Tim Hatch**.
+
+The code is distributed under the BSD 2-clause license.  Contributors making pull
+requests must agree that they are able and willing to put their contributions
+under that license.
diff --git a/description.rst b/description.rst
new file mode 100644
index 0000000..de34cf1
--- /dev/null
+++ b/description.rst
@@ -0,0 +1,18 @@
+Pygments
+~~~~~~~~
+
+Pygments is a syntax highlighting package written in Python.
+
+It is a generic syntax highlighter suitable for use in code hosting, forums,
+wikis or other applications that need to prettify source code.  Highlights
+are:
+
+* a wide range of over 500 languages and other text formats is supported
+* special attention is paid to details, increasing quality by a fair amount
+* support for new languages and formats are added easily
+* a number of output formats, presently HTML, LaTeX, RTF, SVG, all image
+  formats that PIL supports and ANSI sequences
+* it is usable as a command-line tool and as a library
+
+Copyright 2006-2022 by the Pygments team, see ``AUTHORS``.
+Licensed under the BSD, see ``LICENSE`` for details.
\ No newline at end of file
diff --git a/doc/Makefile b/doc/Makefile
new file mode 100644
index 0000000..a0dcaaa
--- /dev/null
+++ b/doc/Makefile
@@ -0,0 +1,163 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = PYTHONPATH=.. sphinx-build
+PAPER         =
+BUILDDIR      = _build
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help clean pyodide html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
+
+help:
+	@echo "Please use \`make ' where  is one of"
+	@echo "  pyodide    to make Pyodide with currently checked out Pygments"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  singlehtml to make a single large HTML file"
+	@echo "  pickle     to make pickle files"
+	@echo "  json       to make JSON files"
+	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  qthelp     to make HTML files and a qthelp project"
+	@echo "  devhelp    to make HTML files and a Devhelp project"
+	@echo "  epub       to make an epub"
+	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
+	@echo "  text       to make text files"
+	@echo "  man        to make manual pages"
+	@echo "  texinfo    to make Texinfo files"
+	@echo "  info       to make Texinfo files and run them through makeinfo"
+	@echo "  gettext    to make PO message catalogs"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+
+clean:
+	-rm -rf $(BUILDDIR)/*
+
+pyodide:
+	$(if $(test ! -f docker), $(error "Could not find Docker. Please install that before continuing."))
+	# Enable the BuildKit backend to use the --output option.
+	DOCKER_BUILDKIT=1 docker build --file pyodide/Dockerfile --output $(BUILDDIR)/pyodide/pyodide ..
+	@echo
+	@echo "Pyodide build finished. The Pyodide artifacts are in $(BUILDDIR)/pyodide."
+
+html:
+	$(if $(WEBSITE_BUILD), $(MAKE) pyodide)
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+dirhtml:
+	$(if $(WEBSITE_BUILD), $(MAKE) pyodide)
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	@echo
+	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+	@echo
+	@echo "Build finished; now you can process the pickle files."
+
+json:
+	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+	@echo
+	@echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+	@echo
+	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
+	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Pygments.qhcp"
+	@echo "To view the help file:"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Pygments.qhc"
+
+devhelp:
+	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+	@echo
+	@echo "Build finished."
+	@echo "To view the help file:"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/Pygments"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Pygments"
+	@echo "# devhelp"
+
+epub:
+	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+	@echo
+	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+latex:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Run \`make' in that directory to run these through (pdf)latex" \
+	      "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through pdflatex..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+	@echo
+	@echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+man:
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+	@echo
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo
+	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Run \`make' in that directory to run these through makeinfo" \
+	      "(use \`make info' here to do that automatically)."
+
+info:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo "Running Texinfo files through makeinfo..."
+	make -C $(BUILDDIR)/texinfo info
+	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+gettext:
+	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+	@echo
+	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."
diff --git a/doc/_static/demo-worker.js b/doc/_static/demo-worker.js
new file mode 100644
index 0000000..22b8b3d
--- /dev/null
+++ b/doc/_static/demo-worker.js
@@ -0,0 +1,74 @@
+importScripts('/_static/pyodide/pyodide.js');
+
+async function loadPyodideAndPygments() {
+    self.pyodide = await loadPyodide();
+    await self.pyodide.loadPackage(["Pygments"]);
+    const styles = self.pyodide.runPython(`
+        from pygments.formatters.html import HtmlFormatter
+        from pygments.styles import STYLE_MAP
+        {s: HtmlFormatter(style=s).get_style_defs('.demo-highlight') for s in STYLE_MAP}
+    `).toJs();
+    self.postMessage({loaded: {styles}})
+}
+let pyodideReadyPromise = loadPyodideAndPygments();
+
+self.onmessage = async (event) => {
+    // Make sure loading is done.
+    await pyodideReadyPromise;
+    if (event.data.highlight) {
+        self.pyodide.globals.set('code', event.data.highlight.code);
+        self.pyodide.globals.set('lexer_name', event.data.highlight.lexer);
+
+        self.pyodide.runPython(`
+            import pygments.lexers
+
+            lexer = pygments.lexers.get_lexer_by_name(lexer_name)
+            if type(code) == memoryview:
+                code = bytes(code)
+            tokens = lexer.get_tokens(code)
+        `);
+
+        const formatter = event.data.highlight.formatter;
+        if (formatter == 'html') {
+
+            const html = self.pyodide.runPython(`
+                import io
+                from pygments.formatters.html import HtmlFormatter
+
+                fmter = HtmlFormatter(cssclass='demo-highlight')
+                buf = io.StringIO()
+                fmter.format(tokens, buf)
+                buf.getvalue()
+            `);
+            self.postMessage({html});
+        } else if (formatter == 'tokens') {
+            const tokens = self.pyodide.runPython('list(tokens)').toJs();
+            self.postMessage({tokens});
+        } else {
+            console.warn('unknown formatter:', formatter);
+        }
+    } else if (event.data.guess_lexer) {
+        self.pyodide.globals.set('code', event.data.guess_lexer.code);
+        self.pyodide.globals.set('filename', event.data.guess_lexer.filename);
+        const lexer = self.pyodide.runPython(`
+            import sys
+            sys.setrecursionlimit(1000)
+            # TODO: remove after upgrading to Pyodide 0.19
+
+            import pygments.lexers
+            import pygments.util
+
+            if type(code) == memoryview:
+                code = bytes(code)
+
+            if filename:
+                lexer = pygments.lexers.guess_lexer_for_filename(filename, code)
+            else:
+                lexer = pygments.lexers.guess_lexer(code)
+            lexer.aliases[0]
+        `);
+        self.postMessage({lexer});
+    } else {
+        console.warn('unknown command: expected highlight or guess_lexer but received ', event.data);
+    }
+}
diff --git a/doc/_static/demo.css b/doc/_static/demo.css
new file mode 100644
index 0000000..eaa4410
--- /dev/null
+++ b/doc/_static/demo.css
@@ -0,0 +1,89 @@
+#try {
+    background-color: #f6f6f6;
+    border-radius: 0;
+    border: 1px solid #ccc;
+    margin-top: 15px;
+    margin-bottom: 10px;
+    padding: 10px 15px 5px 10px;
+    position: relative;
+}
+
+#try h2 {
+    margin-top: 0;
+}
+
+#try textarea {
+    border: 1px solid #999;
+    padding: 2px;
+    width: 100%;
+    min-height: 150px;
+    resize: vertical;
+}
+
+#hlcode {
+    margin: 10px 0;
+    max-height: 500px;
+    overflow: auto;
+    border: 1px solid #ccc;
+}
+
+#hlcode:empty {
+    display: none;
+}
+
+#hlcode pre {
+    background-color: transparent;
+    border: 0;
+    margin: 0;
+}
+#hlcode table {
+    /* unset negative margin from pygments14.css */
+    margin: unset;
+}
+
+#code-header:not([hidden]) {
+    display: flex;
+    gap: 1em;
+    padding: 0 15px;
+}
+.flex-grow-1 {
+    flex-grow: 1;
+}
+#lexer {
+    margin-right: 0.5em;
+}
+#guessed-lexer:not(:empty):before {
+    content: '(guessed ';
+}
+#guessed-lexer:not(:empty):after {
+    content: ')';
+}
+
+#loading[hidden] {
+    visibility: hidden;
+    display: flex;
+}
+
+#loading {
+    display: flex;
+    align-items: center;
+    gap: 1em;
+}
+
+#format-settings {
+    display: flex;
+    gap: 1em;
+    border-top: 1px solid #ccc;
+    padding-top: 0.5em;
+    margin-top: 0.5em;
+}
+
+.tokens code {
+    /* make whitespace visible */
+    white-space: pre;
+    background: #d9d9d9;
+}
+
+#contrast-warning {
+    color: darkred;
+}
diff --git a/doc/_static/demo.js b/doc/_static/demo.js
new file mode 100644
index 0000000..b193d10
--- /dev/null
+++ b/doc/_static/demo.js
@@ -0,0 +1,200 @@
+const loadingDiv = document.getElementById("loading");
+const langSelect = document.getElementById("lang");
+const styleSelect = document.getElementById("style");
+const formatterSelect = document.getElementById("formatter");
+const outputDiv = document.getElementById("hlcode");
+const codeHeader = document.getElementById("code-header");
+const copyLink = document.getElementById("copylink");
+const style = document.getElementById("css-style");
+const textarea = document.getElementById("code");
+const uriTooLongMsg = document.getElementById('uri-too-long');
+const contrastWarning = document.getElementById('contrast-warning');
+const fileInput = document.getElementById("file");
+const fileInputResetButton = document.getElementById('reset-file');
+
+const qvars = Object.fromEntries(new URLSearchParams(window.location.search));
+if (qvars.lexer) {
+    langSelect.value = qvars.lexer;
+}
+if (qvars.code !== undefined) {
+    textarea.value = qvars.code;
+    loadingDiv.hidden = false;
+}
+if (qvars.style !== undefined) {
+    styleSelect.value = qvars.style;
+    updateContrastWarning();
+}
+if (qvars.formatter !== undefined) {
+    formatterSelect.value = qvars.formatter;
+}
+
+styleSelect.addEventListener('change', () => {
+    if (!styles)
+        // Worker has not loaded yet.
+        return;
+    style.textContent = styles.get(styleSelect.value);
+    updateCopyLink();
+    updateContrastWarning();
+});
+
+function updateContrastWarning() {
+    contrastWarning.hidden = styleSelect.selectedOptions[0].dataset.wcag == 'aa';
+}
+
+function debounce(func, timeout) {
+    let timer;
+    return (...args) => {
+        clearTimeout(timer);
+        timer = setTimeout(() => func.apply(this, args), timeout);
+    };
+}
+
+const highlightShortDebounce = debounce(highlight, 50);
+const highlightLongDebounce = debounce(highlight, 500);
+
+function debouncedUpdate() {
+    if (fileInput.files.length > 0)
+        return;
+
+    if (textarea.value.length < 1000) {
+        highlightShortDebounce();
+    } else {
+        highlightLongDebounce();
+    }
+}
+
+langSelect.addEventListener('change', debouncedUpdate);
+textarea.addEventListener('input', debouncedUpdate);
+formatterSelect.addEventListener('change', debouncedUpdate);
+fileInput.addEventListener('change', () => {
+    fileInputResetButton.hidden = false;
+    highlight();
+});
+fileInputResetButton.hidden = fileInput.files.length == 0;
+fileInputResetButton.addEventListener('click', () => {
+    fileInputResetButton.hidden = true;
+    fileInput.value = '';
+    highlight();
+});
+
+let styles;
+
+const highlightWorker = new Worker("/_static/demo-worker.js");
+highlightWorker.onmessage = (msg) => {
+    if (msg.data.loaded) {
+        styles = msg.data.loaded.styles;
+
+        if (qvars.code !== undefined || textarea.value) {
+            loadingDiv.hidden = true;
+            highlight();
+        }
+    } else if (msg.data.html) {
+        outputDiv.innerHTML = msg.data.html;
+        codeHeader.hidden = false;
+        loadingDiv.hidden = true;
+        style.textContent = styles.get(styleSelect.value);
+    } else if (msg.data.tokens) {
+        const table = document.createElement('table');
+        table.className = 'tokens';
+        for (const [tokenType, value] of msg.data.tokens) {
+            const tr = document.createElement('tr');
+            const td1 = document.createElement('td');
+            td1.textContent = tokenType.join('.');
+            const td2 = document.createElement('td');
+            const inlineCode = document.createElement('code');
+            inlineCode.textContent = value;
+            td2.appendChild(inlineCode);
+            tr.appendChild(td1);
+            tr.appendChild(td2);
+            table.appendChild(tr);
+        }
+        outputDiv.innerHTML = '';
+        outputDiv.appendChild(table);
+
+        codeHeader.hidden = false;
+        loadingDiv.hidden = true;
+    } else if (msg.data.lexer) {
+        highlight(msg.data.lexer);
+    } else {
+        console.warn('unexpected message from highlight worker', msg);
+    }
+};
+
+function updateCopyLink() {
+    var url = document.location.origin + document.location.pathname +
+        "?" + new URLSearchParams({
+            lexer: langSelect.value,
+            style: styleSelect.value,
+            formatter: formatterSelect.value,
+            code: textarea.value,
+        }).toString()
+    if (url.length > 8201) {
+        // pygments.org is hosted on GitHub pages which does not support URIs longer than 8201
+        copyLink.hidden = true;
+        uriTooLongMsg.hidden = false;
+    } else {
+        copyLink.href = url;
+        copyLink.textContent = 'Copy link';
+        copyLink.hidden = false;
+        uriTooLongMsg.hidden = true;
+    }
+}
+
+async function highlight(guessedLexer) {
+    var lexer = langSelect.value || guessedLexer;
+    var file = fileInput.files[0];
+
+    let code;
+    if (file) {
+        code = await file.arrayBuffer();
+    } else {
+        code = textarea.value;
+    }
+
+    loadingDiv.hidden = false;
+
+    if (!lexer) {
+        const guess_lexer = {code};
+        if (file)
+            guess_lexer.filename = file.name;
+        highlightWorker.postMessage({guess_lexer});
+        document.getElementById('loading-text').textContent = 'guessing lexer...';
+        return;
+    }
+
+    document.getElementById('loading-text').textContent = 'highlighting code...';
+
+    document.getElementById('guessed-lexer').textContent = guessedLexer;
+
+    highlightWorker.postMessage({highlight: {code, lexer, formatter: formatterSelect.value}});
+
+    if (code instanceof ArrayBuffer) {
+        copyLink.hidden = true;
+        uriTooLongMsg.hidden = true;
+    } else {
+        updateCopyLink();
+    }
+}
+
+copyLink.addEventListener('click', async (e) => {
+    e.preventDefault();
+    await navigator.clipboard.writeText(e.target.href);
+});
+
+function download_code() {
+    var filename = "highlighted.html";
+    var hlcode = document.getElementById("hlcode").innerHTML + style.outerHTML;
+    var blob = new Blob([hlcode], {type: 'text/html'});
+    if (window.navigator.msSaveOrOpenBlob) {
+        window.navigator.msSaveBlob(blob, filename);
+    }
+    else{
+        var elem = window.document.createElement('a');
+        elem.href = window.URL.createObjectURL(blob);
+        elem.download = filename;
+        document.body.appendChild(elem);
+        elem.click();
+        document.body.removeChild(elem);
+        window.URL.revokeObjectURL(elem.href);
+    }
+}
diff --git a/doc/_static/favicon.ico b/doc/_static/favicon.ico
new file mode 100644
index 0000000..777f617
Binary files /dev/null and b/doc/_static/favicon.ico differ
diff --git a/doc/_static/github.png b/doc/_static/github.png
new file mode 100644
index 0000000..5d146ad
Binary files /dev/null and b/doc/_static/github.png differ
diff --git a/doc/_static/logo_new.png b/doc/_static/logo_new.png
new file mode 100644
index 0000000..0ae4b20
Binary files /dev/null and b/doc/_static/logo_new.png differ
diff --git a/doc/_static/logo_only.png b/doc/_static/logo_only.png
new file mode 100644
index 0000000..fdebcc4
Binary files /dev/null and b/doc/_static/logo_only.png differ
diff --git a/doc/_static/spinner.gif b/doc/_static/spinner.gif
new file mode 100644
index 0000000..2212db9
Binary files /dev/null and b/doc/_static/spinner.gif differ
diff --git a/doc/_templates/demo.html b/doc/_templates/demo.html
new file mode 100644
index 0000000..8e2a7c6
--- /dev/null
+++ b/doc/_templates/demo.html
@@ -0,0 +1,97 @@
+{% extends "layout.html" %}
+{% set sidebars = sidebars + ["demo_sidebar.html"] %}
+
+{% block extrahead %}
+{{ super() }}
+
+{% endblock %}
+
+{% block htmltitle %}Demo{{ titlesuffix }}{% endblock %}
+
+{% block body %}
+{{ body }}
+
+
Try out Pygments!

+
+
+
+

+    

+        
+        
+    

+    

+        
+    

+    

+        
+        
+    

+	

+        
+        
+        
+    

+    
+

+
+
+
+
+
+

+
+
+
+
The highlighting here is performed in-browser using
+    a WebAssembly translation of the latest Pygments master branch, courtesy of
+    Pyodide.

+
Your content is neither sent over the web nor stored anywhere.

+
+
+{% endblock %}
diff --git a/doc/_templates/demo_sidebar.html b/doc/_templates/demo_sidebar.html
new file mode 100644
index 0000000..3f2a86c
--- /dev/null
+++ b/doc/_templates/demo_sidebar.html
@@ -0,0 +1 @@
+
Back to top

diff --git a/doc/_templates/docssidebar.html b/doc/_templates/docssidebar.html
new file mode 100644
index 0000000..913acaa
--- /dev/null
+++ b/doc/_templates/docssidebar.html
@@ -0,0 +1,3 @@
+{% if pagename != 'docs/index' %}
+« Back to docs index
+{% endif %}
diff --git a/doc/_templates/index_with_try.html b/doc/_templates/index_with_try.html
new file mode 100644
index 0000000..e69de29
diff --git a/doc/_templates/indexsidebar.html b/doc/_templates/indexsidebar.html
new file mode 100644
index 0000000..5aa5019
--- /dev/null
+++ b/doc/_templates/indexsidebar.html
@@ -0,0 +1,18 @@
+

+
Download

+
Current version: {{ version }}
Changelog

+
Get Pygments from the Python Package
+    Index, or install it with:

+
pip install Pygments

+

+

+
Questions? Suggestions?

+
+

+    Clone at GitHub.

+
You can also open an issue at the
+  tracker.

+

+
+
diff --git a/doc/_templates/styles.html b/doc/_templates/styles.html
new file mode 100644
index 0000000..137fa24
--- /dev/null
+++ b/doc/_templates/styles.html
@@ -0,0 +1,55 @@
+{% extends "layout.html" %}
+
+{% block htmltitle %}Styles{{ titlesuffix }}{% endblock %}
+
+{% block body %}
+
+{{ body }}
+
+
Styles

+
+
Pygments comes with the following builtin styles.
+For more information about styles refer to the documentation.
+

+
+

+{% for style in styles_aa %}
+    

+    
{{style.name}}

+    {{style.html|safe}}
+    

+{% endfor %}
+

+
+
Styles with a lower contrast

+

+The following styles do not meet the WCAG 2.1 AA contrast minimum,
+so they might be difficult to read for people with suboptimal vision.
+If you want your highlighted code to be well readable for other people, you
+should use one of the earlier styles instead.
+

+

+    {% for style in styles_sub_aa %}
+    

+    
{{style.name}}

+    {{style.html|safe}}
+    

+    {% endfor %}
+

+
+{% endblock %}
diff --git a/doc/_themes/pygments14/layout.html b/doc/_themes/pygments14/layout.html
new file mode 100644
index 0000000..34e86ef
--- /dev/null
+++ b/doc/_themes/pygments14/layout.html
@@ -0,0 +1,101 @@
+{#
+    sphinxdoc/layout.html
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Sphinx layout template for the sphinxdoc theme.
+
+    :copyright: Copyright 2007-2013 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+#}
+{%- extends "basic/layout.html" %}
+
+{# put the sidebar before the body #}
+{% block sidebar1 %}{{ sidebar() }}{% endblock %}
+{% block sidebar2 %}{% endblock %}
+
+{% block relbar1 %}{% endblock %}
+{% block relbar2 %}{% endblock %}
+
+{% block extrahead %}
+    
+{{ super() }}
+{%- if not embedded %}
+    
+    
+{%- endif %}
+{% endblock %}
+
+{% block header %}
+

+

+  
+  

+    
+      Pygments logo
+    
+  

+

+

+{% endblock %}
+
+{% block footer %}
+    
 {# closes "flexwrapper" div #}
+    
+  
 {# closes "outerwrapper" div #}
+{% endblock %}
+
+{% block sidebarrel %}
+{% endblock %}
+
+{% block sidebarsourcelink %}
+{% endblock %}
diff --git a/doc/_themes/pygments14/localtoc.html b/doc/_themes/pygments14/localtoc.html
new file mode 100644
index 0000000..c0e2de0
--- /dev/null
+++ b/doc/_themes/pygments14/localtoc.html
@@ -0,0 +1,17 @@
+{#
+    basic/localtoc.html
+    ~~~~~~~~~~~~~~~~~~~
+
+    Sphinx sidebar template: local table of contents.
+
+    This file can be removed once https://github.com/sphinx-doc/sphinx/pull/9815 has landed.
+
+    :copyright: Copyright 2007-2021 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+#}
+{%- if display_toc %}
+  
+{%- endif %}
diff --git a/doc/_themes/pygments14/relations.html b/doc/_themes/pygments14/relations.html
new file mode 100644
index 0000000..372894d
--- /dev/null
+++ b/doc/_themes/pygments14/relations.html
@@ -0,0 +1,25 @@
+{#
+    basic/relations.html
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Sphinx sidebar template: relation links.
+
+    This file can be removed once https://github.com/sphinx-doc/sphinx/pull/9815 has landed.
+
+    :copyright: Copyright 2007-2021 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+#}
+{%- if prev %}
+

+  
{{ _('Previous topic') }}

+  
{{ prev.title }}

+

+{%- endif %}
+{%- if next %}
+

+  
{{ _('Next topic') }}

+  
{{ next.title }}

+

+{%- endif %}
diff --git a/doc/_themes/pygments14/static/bodybg.png b/doc/_themes/pygments14/static/bodybg.png
new file mode 100644
index 0000000..46892b8
Binary files /dev/null and b/doc/_themes/pygments14/static/bodybg.png differ
diff --git a/doc/_themes/pygments14/static/docbg.png b/doc/_themes/pygments14/static/docbg.png
new file mode 100644
index 0000000..13e61f3
Binary files /dev/null and b/doc/_themes/pygments14/static/docbg.png differ
diff --git a/doc/_themes/pygments14/static/listitem.png b/doc/_themes/pygments14/static/listitem.png
new file mode 100644
index 0000000..e45715f
Binary files /dev/null and b/doc/_themes/pygments14/static/listitem.png differ
diff --git a/doc/_themes/pygments14/static/logo.png b/doc/_themes/pygments14/static/logo.png
new file mode 100644
index 0000000..2c1a24d
Binary files /dev/null and b/doc/_themes/pygments14/static/logo.png differ
diff --git a/doc/_themes/pygments14/static/pocoo.png b/doc/_themes/pygments14/static/pocoo.png
new file mode 100644
index 0000000..4174149
Binary files /dev/null and b/doc/_themes/pygments14/static/pocoo.png differ
diff --git a/doc/_themes/pygments14/static/pygments14.css_t b/doc/_themes/pygments14/static/pygments14.css_t
new file mode 100644
index 0000000..4355074
--- /dev/null
+++ b/doc/_themes/pygments14/static/pygments14.css_t
@@ -0,0 +1,422 @@
+/*
+ * pygments14.css
+ * ~~~~~~~~~~~~~~
+ *
+ * Sphinx stylesheet -- pygments14 theme.  Heavily copied from sphinx13.
+ *
+ * :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS.
+ * :license: BSD, see LICENSE for details.
+ *
+ */
+
+@import url("basic.css");
+
+/* -- page layout ----------------------------------------------------------- */
+
+body {
+    font-family: {{ theme_font }}, 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva',
+                 'Verdana', sans-serif;
+    font-size: 14px;
+    text-align: center;
+    background-image: url(bodybg.png);
+    background-color: {{ theme_background }};
+    color: black;
+    padding: 0;
+    /*
+    border-right: 1px solid {{ theme_border }};
+    border-left: 1px solid {{ theme_border }};
+    */
+
+    margin: 0 auto;
+    max-width: 1080px;
+}
+
+.outerwrapper {
+    background-image: url(docbg.png);
+    background-attachment: fixed;
+}
+
+.pageheader {
+    text-align: left;
+    padding: 10px 15px;
+}
+
+.pageheader ul {
+    float: right;
+    color: white;
+    list-style-type: none;
+    padding-left: 0;
+    margin-top: 40px;
+    margin-right: 10px;
+}
+
+.pageheader li {
+    float: left;
+    margin: 0 0 0 10px;
+}
+
+.pageheader li a {
+    border-radius: 3px;
+    padding: 8px 12px;
+    color: {{ theme_darkgray }};
+    text-shadow: 0 0 5px rgba(0, 0, 0, 0.2);
+}
+
+.pageheader li a:hover {
+    background-color: {{ theme_yellow }};
+    color: black;
+    text-shadow: none;
+}
+
+div.document {
+    width: 700px;
+    flex-grow: 100;
+    text-align: left;
+    /*border-left: 1em solid {{ theme_lightyellow }};*/
+    min-width: 500px;
+}
+
+@media screen and (max-width: 550px) {
+    div.document {
+        min-width: inherit;
+    }
+}
+
+div.bodywrapper {
+    background-color: white;
+/*    border-right: 1px solid {{ theme_border }}; */
+}
+
+.flexwrapper {
+    display: flex;
+    gap: 15px;
+    flex-wrap: wrap;
+    padding-right: 12px;
+}
+
+div.body {
+    margin: 0;
+    padding: 0.5em 20px 20px 20px;
+    width: 100%;
+    box-sizing: border-box;
+}
+
+div.related {
+    font-size: 1em;
+    color: {{ theme_darkgray }};
+}
+
+div.related ul {
+    background-image: url(relbg.png);
+    background-repeat: repeat-y;
+    background-color: {{ theme_yellow }};
+    height: 1.9em;
+    /*
+    border-top: 1px solid {{ theme_border }};
+    border-bottom: 1px solid {{ theme_border }};
+    */
+}
+
+div.related ul li {
+    margin: 0 5px 0 0;
+    padding: 0;
+    float: left;
+}
+
+div.related ul li.right {
+    float: right;
+    margin-right: 5px;
+}
+
+div.related ul li a {
+    margin: 0;
+    padding: 0 5px 0 5px;
+    line-height: 1.75em;
+    color: {{ theme_darkgray }};
+    /*text-shadow: 0px 0px 1px rgba(0, 0, 0, 0.5);*/
+}
+
+div.related ul li a:hover {
+    text-decoration: underline;
+    text-shadow: 0px 0px 1px rgba(255, 255, 255, 0.5);
+}
+
+div.sphinxsidebar {
+    margin: 0;
+    padding: 0 0px 15px 15px;
+    width: 210px;
+    float: none;
+    font-size: 1em;
+    text-align: left;
+    flex-grow: 1;
+}
+
+.sphinxsidebarwrapper > * {
+    flex: 1 1 0px;
+    min-width: 200px;
+}
+
+div.sphinxsidebar .logo {
+    font-size: 1.8em;
+    color: #666;
+    font-weight: 300;
+    text-align: center;
+}
+
+div.sphinxsidebar .logo img {
+    vertical-align: middle;
+}
+
+div.sphinxsidebar input {
+    border: 1px solid #aaa;
+    font-family: {{ theme_font }}, 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva',
+                 'Verdana', sans-serif;
+    font-size: 1em;
+}
+
+div.sphinxsidebar h3 {
+    font-size: 1.5em;
+    /* border-top: 1px solid {{ theme_border }}; */
+    margin-top: 0;
+    margin-bottom: 0.5em;
+    padding-top: 0.5em;
+}
+
+div.sphinxsidebar h4 {
+    font-size: 1.2em;
+    margin-bottom: 0;
+}
+
+div.sphinxsidebar h3, div.sphinxsidebar h4 {
+    margin-left: -15px;
+    padding-right: 14px;
+    padding-left: 14px;
+    color: #333;
+    font-weight: 300;
+    /*text-shadow: 0px 0px 0.5px rgba(0, 0, 0, 0.4);*/
+}
+
+div.sphinxsidebarwrapper {
+    padding: 0;
+    display: flex;
+    flex-wrap: wrap;
+    gap: 15px;
+}
+
+div.sphinxsidebarwrapper > h3:first-child {
+    margin-top: 0.5em;
+    border: none;
+}
+
+div.sphinxsidebar h3 a {
+    color: #333;
+}
+
+div.sphinxsidebar ul {
+    color: #444;
+    margin-top: 7px;
+    padding: 0;
+    line-height: 130%;
+}
+
+div.sphinxsidebar ul ul {
+    margin-left: 20px;
+    list-style-image: url(listitem.png);
+}
+
+div.footer {
+    color: {{ theme_darkgray }};
+    text-shadow: 0 0 .2px rgba(255, 255, 255, 0.8);
+    padding: 2em;
+    text-align: center;
+    clear: both;
+    font-size: 0.8em;
+}
+
+/* -- body styles ----------------------------------------------------------- */
+
+p {    
+    margin: 0.8em 0 0.5em 0;
+}
+
+a {
+    color: {{ theme_darkgreen }};
+    text-decoration: none;
+}
+
+a:hover {
+    color: {{ theme_darkyellow }};
+}
+
+div.body a {
+    text-decoration: underline;
+}
+
+h1 {
+    margin: 10px 0 0 0;
+    font-size: 2.4em;
+    color: {{ theme_darkgray }};
+    font-weight: 300;
+}
+
+h2 {
+    margin: 1.em 0 0.2em 0;
+    font-size: 1.5em;
+    font-weight: 300;
+    padding: 0;
+    color: {{ theme_darkgreen }};
+}
+
+h3 {
+    margin: 1em 0 -0.3em 0;
+    font-size: 1.3em;
+    font-weight: 300;
+}
+
+div.body h1 a, div.body h2 a, div.body h3 a, div.body h4 a, div.body h5 a, div.body h6 a {
+    text-decoration: none;
+}
+
+div.body h1 a tt, div.body h2 a tt, div.body h3 a tt, div.body h4 a tt, div.body h5 a tt, div.body h6 a tt {
+    color: {{ theme_darkgreen }} !important;
+    font-size: inherit !important;
+}
+
+a.headerlink {
+    color: {{ theme_green }} !important;
+    font-size: 12px;
+    margin-left: 6px;
+    padding: 0 4px 0 4px;
+    text-decoration: none !important;
+    float: right;
+}
+
+a.headerlink:hover {
+    background-color: #ccc;
+    color: white!important;
+}
+
+cite, code, tt {
+    font-family: 'Consolas', 'DejaVu Sans Mono',
+                 'Bitstream Vera Sans Mono', monospace;
+    font-size: 14px;
+    letter-spacing: -0.02em;
+}
+
+tt {
+    background-color: #f2f2f2;
+    border: 1px solid #ddd;
+    border-radius: 2px;
+    color: #333;
+    padding: 1px;
+}
+
+tt.descname, tt.descclassname, tt.xref {
+    border: 0;
+}
+
+hr {
+    border: 1px solid #abc;
+    margin: 2em;
+}
+
+a tt {
+    border: 0;
+    color: {{ theme_darkgreen }};
+}
+
+a tt:hover {
+    color: {{ theme_darkyellow }};
+}
+
+pre {
+    font-family: 'Consolas', 'DejaVu Sans Mono',
+                 'Bitstream Vera Sans Mono', monospace;
+    font-size: 13px;
+    letter-spacing: 0.015em;
+    line-height: 120%;
+    padding: 0.5em;
+    border: 1px solid #ccc;
+    border-radius: 2px;
+    background-color: #f8f8f8;
+}
+
+pre a {
+    color: inherit;
+    text-decoration: underline;
+}
+
+td.linenos pre {
+    padding: 0.5em 0;
+}
+
+div.quotebar {
+    background-color: #f8f8f8;
+    max-width: 250px;
+    float: right;
+    padding: 0px 7px;
+    border: 1px solid #ccc;
+    margin-left: 1em;
+}
+
+div.topic {
+    background-color: #f8f8f8;
+}
+
+table {
+    border-collapse: collapse;
+    margin: 0 -0.5em 0 -0.5em;
+}
+
+table td, table th {
+    padding: 0.2em 0.5em 0.2em 0.5em;
+}
+
+div.admonition, div.warning {
+    font-size: 0.9em;
+    margin: 1em 0 1em 0;
+    border: 1px solid #86989B;
+    border-radius: 2px;
+    background-color: #f7f7f7;
+    padding: 0;
+    padding-bottom: 0.5rem;
+}
+
+div.admonition p, div.warning p {
+    margin: 0.5em 1em 0.5em 1em;
+    padding: 0;
+}
+
+div.admonition pre, div.warning pre {
+    margin: 0.4em 1em 0.4em 1em;
+}
+
+div.admonition p.admonition-title,
+div.warning p.admonition-title {
+    font-weight: bold;
+}
+
+div.warning {
+    border: 1px solid #940000;
+/*    background-color: #FFCCCF;*/
+}
+
+div.warning p.admonition-title {
+}
+
+div.admonition ul, div.admonition ol,
+div.warning ul, div.warning ol {
+    margin: 0.1em 0.5em 0.5em 3em;
+    padding: 0;
+}
+
+.viewcode-back {
+    font-family: {{ theme_font }}, 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva',
+                 'Verdana', sans-serif;
+}
+
+div.viewcode-block:target {
+    background-color: #f4debf;
+    border-top: 1px solid #ac9;
+    border-bottom: 1px solid #ac9;
+}
diff --git a/doc/_themes/pygments14/theme.conf b/doc/_themes/pygments14/theme.conf
new file mode 100644
index 0000000..8d2988f
--- /dev/null
+++ b/doc/_themes/pygments14/theme.conf
@@ -0,0 +1,17 @@
+[theme]
+inherit = basic
+stylesheet = pygments14.css
+pygments_style = friendly
+
+[options]
+body_min_width = inherit
+body_max_width = inherit
+green = #66b55e
+darkgreen = #36852e
+darkgray = #666666
+border = #66b55e
+yellow = #f4cd00
+darkyellow = #d4ad00
+lightyellow = #fffbe3
+background = #f9f9f9
+font = PT Sans
diff --git a/doc/conf.py b/doc/conf.py
new file mode 100644
index 0000000..f42c355
--- /dev/null
+++ b/doc/conf.py
@@ -0,0 +1,291 @@
+#
+# Pygments documentation build configuration file
+#
+
+import re, sys, os, itertools
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+sys.path.insert(0, os.path.abspath('..'))
+
+import pygments
+import pygments.formatters
+import pygments.lexers
+import pygments.styles
+import tests.contrast.test_contrasts as test_contrasts
+
+# -- General configuration -----------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'pygments.sphinxext']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = 'Pygments'
+copyright = '2006-2022, Georg Brandl and Pygments contributors'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = pygments.__version__
+# The full version, including alpha/beta/rc tags.
+release = version
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ['_build']
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+#pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'pygments14'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+html_theme_path = ['_themes']
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# " v documentation".
+html_title = 'Pygments'
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+html_favicon = '_static/favicon.ico'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+html_sidebars = {'index': ['indexsidebar.html', 'searchbox.html']}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+html_additional_pages = {
+    'styles': 'styles.html',
+    }
+
+if os.environ.get('WEBSITE_BUILD'):
+    html_additional_pages['demo'] = 'demo.html'
+    html_static_path.append('_build/pyodide')
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a  tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'Pygments'
+
+
+# -- Options for LaTeX output --------------------------------------------------
+
+latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',
+
+# The font size ('10pt', '11pt' or '12pt').
+#'pointsize': '10pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, documentclass [howto/manual]).
+latex_documents = [
+    ('docs/index', 'Pygments.tex', 'Pygments Documentation',
+     'Pygments authors', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output --------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    ('docs/index', 'pygments', 'Pygments Documentation',
+     ['Pygments authors'], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# Example configuration for intersphinx: refer to the Python standard library.
+#intersphinx_mapping = {'http://docs.python.org/': None}
+
+rst_prolog = '.. |language_count| replace:: {}'.format(len(list(pygments.lexers.get_all_lexers())))
+
+def pg_context(app, pagename, templatename, ctx, event_arg):
+    ctx['demo_active'] = bool(os.environ.get('WEBSITE_BUILD'))
+
+    if pagename == 'demo':
+        ctx['lexers'] = sorted(pygments.lexers.get_all_lexers(plugins=False), key=lambda x: x[0].lower())
+
+    if pagename in ('styles', 'demo'):
+        with open('examples/example.py') as f:
+            html = f.read()
+        lexer = pygments.lexers.get_lexer_for_filename('example.py')
+        min_contrasts = test_contrasts.min_contrasts()
+        ctx['styles_aa'] = []
+        ctx['styles_sub_aa'] = []
+        # Use STYLE_MAP directly so we don't get plugins as with get_all_styles().
+        for style in pygments.styles.STYLE_MAP:
+            if not pygments.styles.get_style_by_name(style).web_style_gallery_exclude:
+                aa = min_contrasts[style] >= test_contrasts.WCAG_AA_CONTRAST
+                bg_r, bg_g, bg_b = test_contrasts.hex2rgb(pygments.styles.get_style_by_name(style).background_color)
+                ctx['styles_aa' if aa else 'styles_sub_aa'].append(
+                    dict(
+                        name=style,
+                        html=pygments.highlight(
+                            html,
+                            lexer,
+                            pygments.formatters.HtmlFormatter(noclasses=True, style=style),
+                        ),
+                        # from https://en.wikipedia.org/wiki/Relative_luminance
+                        bg_luminance=(0.2126*bg_r + 0.7152*bg_g + 0.0722*bg_b)
+                    )
+                )
+
+        # sort styles according to their background luminance (light styles first)
+        # if styles have the same background luminance sort them by their name
+        sortkey = lambda s: (-s['bg_luminance'], s['name'])
+        # the default style is always displayed first
+        default_style = ctx['styles_aa'].pop(0)
+        ctx['styles_aa'].sort(key=sortkey)
+        ctx['styles_aa'].insert(0, default_style)
+        ctx['styles_sub_aa'].sort(key=sortkey)
+
+
+def source_read(app, docname, source):
+    # linkify issue / PR numbers in changelog
+    if docname == 'docs/changelog':
+        with open('../CHANGES') as f:
+            changelog = f.read()
+
+        idx = changelog.find('\nVersion 2.4.2\n')
+
+        def linkify(match):
+            url = 'https://github.com/pygments/pygments/issues/' + match[1]
+            return '`{} <{}>`_'.format(match[0], url)
+
+        linkified = re.sub(r'(?:PR)?#([0-9]+)\b', linkify, changelog[:idx])
+        source[0] = linkified + changelog[idx:]
+
+
+def setup(app):
+    app.connect('html-page-context', pg_context)
+    app.connect('source-read', source_read)
diff --git a/doc/docs/api.rst b/doc/docs/api.rst
new file mode 100644
index 0000000..4d330bf
--- /dev/null
+++ b/doc/docs/api.rst
@@ -0,0 +1,360 @@
+.. -*- mode: rst -*-
+
+=====================
+The full Pygments API
+=====================
+
+This page describes the Pygments API.
+
+High-level API
+==============
+
+.. module:: pygments
+
+Functions from the :mod:`pygments` module:
+
+.. function:: lex(code, lexer)
+
+    Lex `code` with the `lexer` (must be a `Lexer` instance)
+    and return an iterable of tokens. Currently, this only calls
+    `lexer.get_tokens()`.
+
+.. function:: format(tokens, formatter, outfile=None)
+
+    Format a token stream (iterable of tokens) `tokens` with the
+    `formatter` (must be a `Formatter` instance). The result is
+    written to `outfile`, or if that is ``None``, returned as a
+    string.
+
+.. function:: highlight(code, lexer, formatter, outfile=None)
+
+    This is the most high-level highlighting function.
+    It combines `lex` and `format` in one function.
+
+
+.. module:: pygments.lexers
+
+Functions from :mod:`pygments.lexers`:
+
+.. function:: get_lexer_by_name(alias, **options)
+
+    Return an instance of a `Lexer` subclass that has `alias` in its
+    aliases list. The lexer is given the `options` at its
+    instantiation.
+
+    Will raise :exc:`pygments.util.ClassNotFound` if no lexer with that alias is
+    found.
+
+.. function:: get_lexer_for_filename(fn, **options)
+
+    Return a `Lexer` subclass instance that has a filename pattern
+    matching `fn`. The lexer is given the `options` at its
+    instantiation.
+
+    Will raise :exc:`pygments.util.ClassNotFound` if no lexer for that filename
+    is found.
+
+.. function:: get_lexer_for_mimetype(mime, **options)
+
+    Return a `Lexer` subclass instance that has `mime` in its mimetype
+    list. The lexer is given the `options` at its instantiation.
+
+    Will raise :exc:`pygments.util.ClassNotFound` if not lexer for that mimetype
+    is found.
+
+.. function:: load_lexer_from_file(filename, lexername="CustomLexer", **options)
+
+    Return a `Lexer` subclass instance loaded from the provided file, relative
+    to the current directory. The file is expected to contain a Lexer class
+    named `lexername` (by default, CustomLexer). Users should be very careful with
+    the input, because this method is equivalent to running eval on the input file.
+    The lexer is given the `options` at its instantiation.
+
+    :exc:`ClassNotFound` is raised if there are any errors loading the Lexer
+
+    .. versionadded:: 2.2
+
+.. function:: guess_lexer(text, **options)
+
+    Return a `Lexer` subclass instance that's guessed from the text in
+    `text`. For that, the :meth:`.analyse_text()` method of every known lexer
+    class is called with the text as argument, and the lexer which returned the
+    highest value will be instantiated and returned.
+
+    :exc:`pygments.util.ClassNotFound` is raised if no lexer thinks it can
+    handle the content.
+
+.. function:: guess_lexer_for_filename(filename, text, **options)
+
+    As :func:`guess_lexer()`, but only lexers which have a pattern in `filenames`
+    or `alias_filenames` that matches `filename` are taken into consideration.
+
+    :exc:`pygments.util.ClassNotFound` is raised if no lexer thinks it can
+    handle the content.
+
+.. function:: get_all_lexers()
+
+    Return an iterable over all registered lexers, yielding tuples in the
+    format::
+
+    	(longname, tuple of aliases, tuple of filename patterns, tuple of mimetypes)
+
+    .. versionadded:: 0.6
+
+.. function:: find_lexer_class_by_name(alias)
+
+    Return the `Lexer` subclass that has `alias` in its aliases list, without
+    instantiating it.
+
+    Will raise :exc:`pygments.util.ClassNotFound` if no lexer with that alias is
+    found.
+
+    .. versionadded:: 2.2
+
+.. function:: find_lexer_class(name)
+
+    Return the `Lexer` subclass that with the *name* attribute as given by
+    the *name* argument.
+
+
+.. module:: pygments.formatters
+
+Functions from :mod:`pygments.formatters`:
+
+.. function:: get_formatter_by_name(alias, **options)
+
+    Return an instance of a :class:`.Formatter` subclass that has `alias` in its
+    aliases list. The formatter is given the `options` at its instantiation.
+
+    Will raise :exc:`pygments.util.ClassNotFound` if no formatter with that
+    alias is found.
+
+.. function:: get_formatter_for_filename(fn, **options)
+
+    Return a :class:`.Formatter` subclass instance that has a filename pattern
+    matching `fn`. The formatter is given the `options` at its instantiation.
+
+    Will raise :exc:`pygments.util.ClassNotFound` if no formatter for that filename
+    is found.
+
+.. function:: load_formatter_from_file(filename, formattername="CustomFormatter", **options)
+
+    Return a `Formatter` subclass instance loaded from the provided file, relative
+    to the current directory. The file is expected to contain a Formatter class
+    named ``formattername`` (by default, CustomFormatter). Users should be very
+    careful with the input, because this method is equivalent to running eval
+    on the input file. The formatter is given the `options` at its instantiation.
+
+    :exc:`ClassNotFound` is raised if there are any errors loading the Formatter
+
+    .. versionadded:: 2.2
+
+.. module:: pygments.styles
+
+Functions from :mod:`pygments.styles`:
+
+.. function:: get_style_by_name(name)
+
+    Return a style class by its short name. The names of the builtin styles
+    are listed in :data:`pygments.styles.STYLE_MAP`.
+
+    Will raise :exc:`pygments.util.ClassNotFound` if no style of that name is
+    found.
+
+.. function:: get_all_styles()
+
+    Return an iterable over all registered styles, yielding their names.
+
+    .. versionadded:: 0.6
+
+
+.. module:: pygments.lexer
+
+Lexers
+======
+
+The base lexer class from which all lexers are derived is:
+
+.. class:: Lexer(**options)
+
+    The constructor takes a \*\*keywords dictionary of options.
+    Every subclass must first process its own options and then call
+    the `Lexer` constructor, since it processes the `stripnl`,
+    `stripall` and `tabsize` options.
+
+    An example looks like this:
+
+    .. sourcecode:: python
+
+        def __init__(self, **options):
+            self.compress = options.get('compress', '')
+            Lexer.__init__(self, **options)
+
+    As these options must all be specifiable as strings (due to the
+    command line usage), there are various utility functions
+    available to help with that, see `Option processing`_.
+
+    .. method:: get_tokens(text)
+
+        This method is the basic interface of a lexer. It is called by
+        the `highlight()` function. It must process the text and return an
+        iterable of ``(tokentype, value)`` pairs from `text`.
+
+        Normally, you don't need to override this method. The default
+        implementation processes the `stripnl`, `stripall` and `tabsize`
+        options and then yields all tokens from `get_tokens_unprocessed()`,
+        with the ``index`` dropped.
+
+    .. method:: get_tokens_unprocessed(text)
+
+        This method should process the text and return an iterable of
+        ``(index, tokentype, value)`` tuples where ``index`` is the starting
+        position of the token within the input text.
+
+        This method must be overridden by subclasses.
+
+    .. staticmethod:: analyse_text(text)
+
+        A static method which is called for lexer guessing. It should analyse
+        the text and return a float in the range from ``0.0`` to ``1.0``.
+        If it returns ``0.0``, the lexer will not be selected as the most
+        probable one, if it returns ``1.0``, it will be selected immediately.
+
+        .. note:: You don't have to add ``@staticmethod`` to the definition of
+                  this method, this will be taken care of by the Lexer's metaclass.
+
+    For a list of known tokens have a look at the :doc:`tokens` page.
+
+    A lexer also can have the following attributes (in fact, they are mandatory
+    except `alias_filenames`) that are used by the builtin lookup mechanism.
+
+    .. attribute:: name
+
+        Full name for the lexer, in human-readable form.
+
+    .. attribute:: aliases
+
+        A list of short, unique identifiers that can be used to lookup
+        the lexer from a list, e.g. using `get_lexer_by_name()`.
+
+    .. attribute:: filenames
+
+        A list of `fnmatch` patterns that match filenames which contain
+        content for this lexer. The patterns in this list should be unique among
+        all lexers.
+
+    .. attribute:: alias_filenames
+
+        A list of `fnmatch` patterns that match filenames which may or may not
+        contain content for this lexer. This list is used by the
+        :func:`.guess_lexer_for_filename()` function, to determine which lexers
+        are then included in guessing the correct one. That means that
+        e.g. every lexer for HTML and a template language should include
+        ``\*.html`` in this list.
+
+    .. attribute:: mimetypes
+
+        A list of MIME types for content that can be lexed with this
+        lexer.
+
+
+.. module:: pygments.formatter
+
+Formatters
+==========
+
+A formatter is derived from this class:
+
+
+.. class:: Formatter(**options)
+
+    As with lexers, this constructor processes options and then must call the
+    base class :meth:`__init__`.
+
+    The :class:`Formatter` class recognizes the options `style`, `full` and
+    `title`.  It is up to the formatter class whether it uses them.
+
+    .. method:: get_style_defs(arg='')
+
+        This method must return statements or declarations suitable to define
+        the current style for subsequent highlighted text (e.g. CSS classes
+        in the `HTMLFormatter`).
+
+        The optional argument `arg` can be used to modify the generation and
+        is formatter dependent (it is standardized because it can be given on
+        the command line).
+
+        This method is called by the ``-S`` :doc:`command-line option `,
+        the `arg` is then given by the ``-a`` option.
+
+    .. method:: format(tokensource, outfile)
+
+        This method must format the tokens from the `tokensource` iterable and
+        write the formatted version to the file object `outfile`.
+
+        Formatter options can control how exactly the tokens are converted.
+
+    .. versionadded:: 0.7
+       A formatter must have the following attributes that are used by the
+       builtin lookup mechanism.
+
+    .. attribute:: name
+
+        Full name for the formatter, in human-readable form.
+
+    .. attribute:: aliases
+
+        A list of short, unique identifiers that can be used to lookup
+        the formatter from a list, e.g. using :func:`.get_formatter_by_name()`.
+
+    .. attribute:: filenames
+
+        A list of :mod:`fnmatch` patterns that match filenames for which this
+        formatter can produce output. The patterns in this list should be unique
+        among all formatters.
+
+
+.. module:: pygments.util
+
+Option processing
+=================
+
+The :mod:`pygments.util` module has some utility functions usable for processing
+command line options. All of the following functions get values from a
+dictionary of options. If the value is already in the type expected by the
+option, it is returned as-is. Otherwise, if the value is a string, it is first
+converted to the expected type if possible.
+
+.. exception:: OptionError
+
+    This exception will be raised by all option processing functions if
+    the type or value of the argument is not correct.
+
+.. function:: get_bool_opt(options, optname, default=None)
+
+    Intuitively, this is `options.get(optname, default)`, but restricted to
+    Boolean value. The Booleans can be represented as string, in order to accept
+    Boolean value from the command line arguments. If the key `optname` is
+    present in the dictionary `options` and is not associated with a Boolean,
+    raise an `OptionError`. If it is absent, `default` is returned instead.
+
+    The valid string values for ``True`` are ``1``, ``yes``, ``true`` and
+    ``on``, the ones for ``False`` are ``0``, ``no``, ``false`` and ``off``
+    (matched case-insensitively).
+
+.. function:: get_int_opt(options, optname, default=None)
+
+    As :func:`get_bool_opt`, but interpret the value as an integer.
+
+.. function:: get_list_opt(options, optname, default=None)
+
+    If the key `optname` from the dictionary `options` is a string,
+    split it at whitespace and return it. If it is already a list
+    or a tuple, it is returned as a list.
+
+.. function:: get_choice_opt(options, optname, allowed, default=None)
+
+    If the key `optname` from the dictionary is not in the sequence
+    `allowed`, raise an error, otherwise return it.
+
+    .. versionadded:: 0.8
diff --git a/doc/docs/authors.rst b/doc/docs/authors.rst
new file mode 100644
index 0000000..f8373f0
--- /dev/null
+++ b/doc/docs/authors.rst
@@ -0,0 +1,4 @@
+Full contributor list
+=====================
+
+.. include:: ../../AUTHORS
diff --git a/doc/docs/changelog.rst b/doc/docs/changelog.rst
new file mode 100644
index 0000000..f264cab
--- /dev/null
+++ b/doc/docs/changelog.rst
@@ -0,0 +1 @@
+.. include:: ../../CHANGES
diff --git a/doc/docs/cmdline.rst b/doc/docs/cmdline.rst
new file mode 100644
index 0000000..b07b3e4
--- /dev/null
+++ b/doc/docs/cmdline.rst
@@ -0,0 +1,218 @@
+.. -*- mode: rst -*-
+
+======================
+Command Line Interface
+======================
+
+You can use Pygments from the shell, provided you installed the
+:program:`pygmentize` script::
+
+    $ pygmentize test.py
+    print "Hello World"
+
+will print the file test.py to standard output, using the Python lexer
+(inferred from the file name extension) and the terminal formatter (because
+you didn't give an explicit formatter name).
+:program:`pygmentize` attempts to
+detect the maximum number of colors that the terminal supports. The difference
+between color formatters for 16 and 256 colors is immense, but there is a less
+noticeable difference between color formatters for 256 and 16 million colors.
+
+Here's the process of how it detects the maxiumum number of colors
+supported by your terminal. If the ``COLORTERM`` environment variable is set to
+either ``truecolor`` or ``24bit``, it will use a 16 million color representation
+(like ``terminal16m``). Next, it will try to find ``256`` is anywhere in the
+environment variable ``TERM``, which it will use a 256-color representaion
+(such as ``terminal256``).  When neither of those are found, it falls back to a
+the 16 color representation (like ``terminal``).
+
+If you want HTML output::
+
+    $ pygmentize -f html -l python -o test.html test.py
+
+As you can see, the -l option explicitly selects a lexer. As seen above, if you
+give an input file name and it has an extension that Pygments recognizes, you can
+omit this option.
+
+The ``-o`` option gives an output file name. If it is not given, output is
+written to stdout.
+
+The ``-f`` option selects a formatter (as with ``-l``, it can also be omitted
+if an output file name is given and has a supported extension).
+If no output file name is given and ``-f`` is omitted, the
+:class:`.TerminalFormatter` is used.
+
+The above command could therefore also be given as::
+
+    $ pygmentize -o test.html test.py
+
+To create a full HTML document, including line numbers and stylesheet (using the
+"emacs" style), highlighting the Python file ``test.py`` to ``test.html``::
+
+    $ pygmentize -O full,style=emacs,linenos=1 -o test.html test.py
+
+
+Options and filters
+-------------------
+
+Lexer and formatter options can be given using the ``-O`` option::
+
+    $ pygmentize -f html -O style=colorful,linenos=1 -l python test.py
+
+Be sure to enclose the option string in quotes if it contains any special shell
+characters, such as spaces or expansion wildcards like ``*``. If an option
+expects a list value, separate the list entries with spaces (you'll have to
+quote the option value in this case too, so that the shell doesn't split it).
+
+Since the ``-O`` option argument is split at commas and expects the split values
+to be of the form ``name=value``, you can't give an option value that contains
+commas or equals signs.  Therefore, an option ``-P`` is provided (as of Pygments
+0.9) that works like ``-O`` but can only pass one option per ``-P``. Its value
+can then contain all characters::
+
+    $ pygmentize -P "heading=Pygments, the Python highlighter" ...
+
+Filters are added to the token stream using the ``-F`` option::
+
+    $ pygmentize -f html -l pascal -F keywordcase:case=upper main.pas
+
+As you see, options for the filter are given after a colon. As for ``-O``, the
+filter name and options must be one shell word, so there may not be any spaces
+around the colon.
+
+
+Generating styles
+-----------------
+
+Formatters normally don't output full style information.  For example, the HTML
+formatter by default only outputs ```` tags with ``class`` attributes.
+Therefore, there's a special ``-S`` option for generating style definitions.
+Usage is as follows::
+
+    $ pygmentize -f html -S colorful -a .syntax
+
+generates a CSS style sheet (because you selected the HTML formatter) for
+the "colorful" style prepending a ".syntax" selector to all style rules.
+
+For an explanation what ``-a`` means for :doc:`a particular formatter
+`, look for the `arg` argument for the formatter's
+:meth:`.get_style_defs()` method.
+
+
+Getting lexer names
+-------------------
+
+.. versionadded:: 1.0
+
+The ``-N`` option guesses a lexer name for a given filename, so that ::
+
+    $ pygmentize -N setup.py
+
+will print out ``python``.  It won't highlight anything yet.  If no specific
+lexer is known for that filename, ``text`` is printed.
+
+Additionally, there is the ``-C`` option, which is just like like ``-N``, except
+that it prints out a lexer name based solely on a given content from standard
+input.
+
+
+Guessing the lexer from the file contents
+-----------------------------------------
+
+The ``-g`` option will try to guess the correct lexer from the file contents,
+or pass through as plain text if nothing can be guessed. This option also looks
+for Vim modelines in the text, and for *some* languages, shebangs. Usage is as
+follows::
+
+    $ pygmentize -g setup.py
+
+Note though, that this option is not very relaiable, and probably should be
+used only if Pygments is not able to guess the correct lexer from the file's
+extension.
+
+
+Highlighting stdin until EOF
+----------------------------
+
+The ``-s`` option processes lines one at a time until EOF, rather than waiting
+to process the entire file. This only works for stdin, only for lexers with no 
+line-spanning constructs, and is intended for streaming input such as you get
+from `tail -f`. Usage is as follows::
+
+    $ tail -f sql.log | pygmentize -s -l sql
+
+
+Custom Lexers and Formatters
+----------------------------
+
+.. versionadded:: 2.2
+
+The ``-x`` flag enables custom lexers and formatters to be loaded
+from files relative to the current directory. Create a file with a class named
+CustomLexer or CustomFormatter, then specify it on the command line::
+
+    $ pygmentize -l your_lexer.py -f your_formatter.py -x
+
+You can also specify the name of your class with a colon::
+
+    $ pygmentize -l your_lexer.py:SomeLexer -x
+
+For more information, see :doc:`the Pygments documentation on Lexer development
+`.
+
+
+Getting help
+------------
+
+The ``-L`` option lists lexers, formatters, along with their short
+names and supported file name extensions, styles and filters. If you want to see
+only one category, give it as an argument::
+
+    $ pygmentize -L filters
+
+will list only all installed filters.
+
+.. versionadded:: 2.11
+
+The ``--json`` option can be used in conjunction with the ``-L`` option to
+output it's contents as JSON. Thus, to print all the installed styles and their
+description in JSON, use the command::
+
+    $ pygmentize -L styles --json
+
+The ``-H`` option will give you detailed information (the same that can be found
+in this documentation) about a lexer, formatter or filter. Usage is as follows::
+
+    $ pygmentize -H formatter html
+
+will print the help for the HTML formatter, while ::
+
+    $ pygmentize -H lexer python
+
+will print the help for the Python lexer, etc.
+
+
+A note on encodings
+-------------------
+
+.. versionadded:: 0.9
+
+Pygments tries to be smart regarding encodings in the formatting process:
+
+* If you give an ``encoding`` option, it will be used as the input and
+  output encoding.
+
+* If you give an ``outencoding`` option, it will override ``encoding``
+  as the output encoding.
+
+* If you give an ``inencoding`` option, it will override ``encoding``
+  as the input encoding.
+
+* If you don't give an encoding and have given an output file, the default
+  encoding for lexer and formatter is the terminal encoding or the default
+  locale encoding of the system.  As a last resort, ``latin1`` is used (which
+  will pass through all non-ASCII characters).
+
+* If you don't give an encoding and haven't given an output file (that means
+  output is written to the console), the default encoding for lexer and
+  formatter is the terminal encoding (``sys.stdout.encoding``).
diff --git a/doc/docs/filterdevelopment.rst b/doc/docs/filterdevelopment.rst
new file mode 100644
index 0000000..004919e
--- /dev/null
+++ b/doc/docs/filterdevelopment.rst
@@ -0,0 +1,75 @@
+.. -*- mode: rst -*-
+
+=====================
+Write your own filter
+=====================
+
+.. versionadded:: 0.7
+
+Writing own filters is very easy. All you have to do is to subclass
+the `Filter` class and override the `filter` method. Additionally a
+filter is instantiated with some keyword arguments you can use to
+adjust the behavior of your filter.
+
+
+Subclassing Filters
+===================
+
+As an example, we write a filter that converts all `Name.Function` tokens
+to normal `Name` tokens to make the output less colorful.
+
+.. sourcecode:: python
+
+    from pygments.util import get_bool_opt
+    from pygments.token import Name
+    from pygments.filter import Filter
+
+    class UncolorFilter(Filter):
+
+        def __init__(self, **options):
+            Filter.__init__(self, **options)
+            self.class_too = get_bool_opt(options, 'classtoo')
+
+        def filter(self, lexer, stream):
+            for ttype, value in stream:
+                if ttype is Name.Function or (self.class_too and
+                                              ttype is Name.Class):
+                    ttype = Name
+                yield ttype, value
+
+Some notes on the `lexer` argument: that can be quite confusing since it doesn't
+need to be a lexer instance. If a filter was added by using the `add_filter()`
+function of lexers, that lexer is registered for the filter. In that case
+`lexer` will refer to the lexer that has registered the filter. It *can* be used
+to access options passed to a lexer. Because it could be `None` you always have
+to check for that case if you access it.
+
+
+Using a decorator
+=================
+
+You can also use the `simplefilter` decorator from the `pygments.filter` module:
+
+.. sourcecode:: python
+
+    from pygments.util import get_bool_opt
+    from pygments.token import Name
+    from pygments.filter import simplefilter
+
+
+    @simplefilter
+    def uncolor(self, lexer, stream, options):
+        class_too = get_bool_opt(options, 'classtoo')
+        for ttype, value in stream:
+            if ttype is Name.Function or (class_too and
+                                          ttype is Name.Class):
+                ttype = Name
+            yield ttype, value
+
+
+You can instantiate this filter by calling `uncolor(classtoo=True)`, the same
+way that you would have instantiated the previous filter by calling
+`UncolorFilter(classtoo=True)`. Indeed, The decorator automatically ensures that
+`uncolor` is a class which subclasses an internal filter class. The class
+`uncolo` uses the decorated function as a method for filtering.  (That's why
+there is a `self` argument that you probably won't end up using in the method.)
diff --git a/doc/docs/filters.rst b/doc/docs/filters.rst
new file mode 100644
index 0000000..5cdcb4c
--- /dev/null
+++ b/doc/docs/filters.rst
@@ -0,0 +1,48 @@
+.. -*- mode: rst -*-
+
+=======
+Filters
+=======
+
+.. versionadded:: 0.7
+
+Transforming a stream of tokens into another stream is called "filtering" and is
+done by filters. The most common example of filters transform each token by
+applying a simple rules such as highlighting the token if it is a TODO or
+another special word, or converting keywords to uppercase to enforce a style
+guide. More complex filters can transform the stream of tokens, such as removing
+the line indentation or merging tokens together. It should be noted that pygments
+filters are entirely unrelated to Python's `filter
+`_.
+
+An arbitrary number of filters can be applied to token streams coming from
+lexers to improve or annotate the output. To apply a filter, you can use the
+`add_filter()` method of a lexer:
+
+.. sourcecode:: pycon
+
+    >>> from pygments.lexers import PythonLexer
+    >>> l = PythonLexer()
+    >>> # add a filter given by a string and options
+    >>> l.add_filter('codetagify', case='lower')
+    >>> l.filters
+    []
+    >>> from pygments.filters import KeywordCaseFilter
+    >>> # or give an instance
+    >>> l.add_filter(KeywordCaseFilter(case='lower'))
+
+The `add_filter()` method takes keyword arguments which are forwarded to
+the constructor of the filter.
+
+To get a list of all registered filters by name, you can use the
+`get_all_filters()` function from the `pygments.filters` module that returns an
+iterable for all known filters.
+
+If you want to write your own filter, have a look at :doc:`Write your own filter
+`.
+
+
+Builtin Filters
+===============
+
+.. pygmentsdoc:: filters
diff --git a/doc/docs/formatterdevelopment.rst b/doc/docs/formatterdevelopment.rst
new file mode 100644
index 0000000..2bfac05
--- /dev/null
+++ b/doc/docs/formatterdevelopment.rst
@@ -0,0 +1,169 @@
+.. -*- mode: rst -*-
+
+========================
+Write your own formatter
+========================
+
+As well as creating :doc:`your own lexer `, writing a new
+formatter for Pygments is easy and straightforward.
+
+A formatter is a class that is initialized with some keyword arguments (the
+formatter options) and that must provides a `format()` method.
+Additionally a formatter should provide a `get_style_defs()` method that
+returns the style definitions from the style in a form usable for the
+formatter's output format.
+
+
+Quickstart
+==========
+
+The most basic formatter shipped with Pygments is the `NullFormatter`. It just
+sends the value of a token to the output stream:
+
+.. sourcecode:: python
+
+    from pygments.formatter import Formatter
+
+    class NullFormatter(Formatter):
+        def format(self, tokensource, outfile):
+            for ttype, value in tokensource:
+                outfile.write(value)
+
+As you can see, the `format()` method is passed two parameters: `tokensource`
+and `outfile`. The first is an iterable of ``(token_type, value)`` tuples,
+the latter a file like object with a `write()` method.
+
+Because the formatter is that basic it doesn't overwrite the `get_style_defs()`
+method.
+
+
+Styles
+======
+
+Styles aren't instantiated but their metaclass provides some class functions
+so that you can access the style definitions easily.
+
+Styles are iterable and yield tuples in the form ``(ttype, d)`` where `ttype`
+is a token and `d` is a dict with the following keys:
+
+``'color'``
+    Hexadecimal color value (eg: ``'ff0000'`` for red) or `None` if not
+    defined.
+
+``'bold'``
+    `True` if the value should be bold
+
+``'italic'``
+    `True` if the value should be italic
+
+``'underline'``
+    `True` if the value should be underlined
+
+``'bgcolor'``
+    Hexadecimal color value for the background (eg: ``'eeeeeee'`` for light
+    gray) or `None` if not defined.
+
+``'border'``
+    Hexadecimal color value for the border (eg: ``'0000aa'`` for a dark
+    blue) or `None` for no border.
+
+Additional keys might appear in the future, formatters should ignore all keys
+they don't support.
+
+
+HTML 3.2 Formatter
+==================
+
+For an more complex example, let's implement a HTML 3.2 Formatter. We don't
+use CSS but inline markup (````, ````, etc). Because this isn't good
+style this formatter isn't in the standard library ;-)
+
+.. sourcecode:: python
+
+    from pygments.formatter import Formatter
+
+    class OldHtmlFormatter(Formatter):
+
+        def __init__(self, **options):
+            Formatter.__init__(self, **options)
+
+            # create a dict of (start, end) tuples that wrap the
+            # value of a token so that we can use it in the format
+            # method later
+            self.styles = {}
+
+            # we iterate over the `_styles` attribute of a style item
+            # that contains the parsed style values.
+            for token, style in self.style:
+                start = end = ''
+                # a style item is a tuple in the following form:
+                # colors are readily specified in hex: 'RRGGBB'
+                if style['color']:
+                    start += '' % style['color']
+                    end = '' + end
+                if style['bold']:
+                    start += ''
+                    end = '' + end
+                if style['italic']:
+                    start += ''
+                    end = '' + end
+                if style['underline']:
+                    start += ''
+                    end = '' + end
+                self.styles[token] = (start, end)
+
+        def format(self, tokensource, outfile):
+            # lastval is a string we use for caching
+            # because it's possible that an lexer yields a number
+            # of consecutive tokens with the same token type.
+            # to minimize the size of the generated html markup we
+            # try to join the values of same-type tokens here
+            lastval = ''
+            lasttype = None
+
+            # wrap the whole output with 
+            outfile.write('
')
+
+            for ttype, value in tokensource:
+                # if the token type doesn't exist in the stylemap
+                # we try it with the parent of the token type
+                # eg: parent of Token.Literal.String.Double is
+                # Token.Literal.String
+                while ttype not in self.styles:
+                    ttype = ttype.parent
+                if ttype == lasttype:
+                    # the current token type is the same of the last
+                    # iteration. cache it
+                    lastval += value
+                else:
+                    # not the same token as last iteration, but we
+                    # have some data in the buffer. wrap it with the
+                    # defined style and write it to the output file
+                    if lastval:
+                        stylebegin, styleend = self.styles[lasttype]
+                        outfile.write(stylebegin + lastval + styleend)
+                    # set lastval/lasttype to current values
+                    lastval = value
+                    lasttype = ttype
+
+            # if something is left in the buffer, write it to the
+            # output file, then close the opened 
 tag
+            if lastval:
+                stylebegin, styleend = self.styles[lasttype]
+                outfile.write(stylebegin + lastval + styleend)
+            outfile.write('
\n')
+
+The comments should explain it. Again, this formatter doesn't override the
+`get_style_defs()` method. If we would have used CSS classes instead of
+inline HTML markup, we would need to generate the CSS first. For that
+purpose the `get_style_defs()` method exists:
+
+
+Generating Style Definitions
+============================
+
+Some formatters like the `LatexFormatter` and the `HtmlFormatter` don't
+output inline markup but reference either macros or css classes. Because
+the definitions of those are not part of the output, the `get_style_defs()`
+method exists. It is passed one parameter (if it's used and how it's used
+is up to the formatter) and has to return a string or ``None``.
diff --git a/doc/docs/formatters.rst b/doc/docs/formatters.rst
new file mode 100644
index 0000000..9e7074e
--- /dev/null
+++ b/doc/docs/formatters.rst
@@ -0,0 +1,48 @@
+.. -*- mode: rst -*-
+
+====================
+Available formatters
+====================
+
+This page lists all builtin formatters.
+
+Common options
+==============
+
+All formatters support these options:
+
+`encoding`
+    If given, must be an encoding name (such as ``"utf-8"``). This will
+    be used to convert the token strings (which are Unicode strings)
+    to byte strings in the output (default: ``None``).
+    It will also be written in an encoding declaration suitable for the
+    document format if the `full` option is given (e.g. a ``meta
+    content-type`` directive in HTML or an invocation of the `inputenc`
+    package in LaTeX).
+
+    If this is ``""`` or ``None``, Unicode strings will be written
+    to the output file, which most file-like objects do not support.
+    For example, `pygments.highlight()` will return a Unicode string if
+    called with no `outfile` argument and a formatter that has `encoding`
+    set to ``None`` because it uses a `StringIO.StringIO` object that
+    supports Unicode arguments to `write()`. Using a regular file object
+    wouldn't work.
+
+    .. versionadded:: 0.6
+
+`outencoding`
+    When using Pygments from the command line, any `encoding` option given is
+    passed to the lexer and the formatter. This is sometimes not desirable,
+    for example if you want to set the input encoding to ``"guess"``.
+    Therefore, `outencoding` has been introduced which overrides `encoding`
+    for the formatter if given.
+
+    .. versionadded:: 0.7
+
+
+Formatter classes
+=================
+
+All these classes are importable from :mod:`pygments.formatters`.
+
+.. pygmentsdoc:: formatters
diff --git a/doc/docs/index.rst b/doc/docs/index.rst
new file mode 100644
index 0000000..d35fe6f
--- /dev/null
+++ b/doc/docs/index.rst
@@ -0,0 +1,64 @@
+Pygments documentation
+======================
+
+**Starting with Pygments**
+
+.. toctree::
+   :maxdepth: 1
+
+   ../download
+   quickstart
+   cmdline
+
+**Builtin components**
+
+.. toctree::
+   :maxdepth: 1
+
+   lexers
+   filters
+   formatters
+   styles
+
+**Reference**
+
+.. toctree::
+   :maxdepth: 1
+
+   unicode
+   tokens
+   api
+   terminal-sessions
+
+**Hacking for Pygments**
+
+.. toctree::
+   :maxdepth: 1
+
+   lexerdevelopment
+   formatterdevelopment
+   filterdevelopment
+   styledevelopment
+   plugins
+
+**Hints and tricks**
+
+.. toctree::
+   :maxdepth: 1
+
+   rstdirective
+   moinmoin
+   java
+   integrate
+
+**About Pygments**
+
+.. toctree::
+   :maxdepth: 1
+
+   changelog
+   authors
+   security
+
+If you find bugs or have suggestions for the documentation, please submit them
+on `GitHub `_.
diff --git a/doc/docs/integrate.rst b/doc/docs/integrate.rst
new file mode 100644
index 0000000..2a030b7
--- /dev/null
+++ b/doc/docs/integrate.rst
@@ -0,0 +1,40 @@
+.. -*- mode: rst -*-
+
+===================================
+Using Pygments in various scenarios
+===================================
+
+Markdown
+--------
+
+Since Pygments 0.9, the distribution ships Markdown_ preprocessor sample code
+that uses Pygments to render source code in
+:file:`external/markdown-processor.py`.  You can copy and adapt it to your
+liking.
+
+.. _Markdown: https://pypi.org/project/Markdown/
+
+TextMate
+--------
+
+Antonio Cangiano has created a Pygments bundle for TextMate that allows to
+colorize code via a simple menu option.  It can be found here_.
+
+.. _here: https://programmingzen.com/pygments-textmate-bundle/
+
+Bash completion
+---------------
+
+The source distribution contains a file ``external/pygments.bashcomp`` that
+sets up completion for the ``pygmentize`` command in bash.
+
+Wrappers for other languages
+----------------------------
+
+These libraries provide Pygments highlighting for users of other languages
+than Python:
+
+* `pygments.rb `_, a pygments wrapper for Ruby
+* `Clygments `_, a pygments wrapper for
+  Clojure
+* `PHPygments `_, a pygments wrapper for PHP
diff --git a/doc/docs/java.rst b/doc/docs/java.rst
new file mode 100644
index 0000000..a8a5beb
--- /dev/null
+++ b/doc/docs/java.rst
@@ -0,0 +1,70 @@
+=====================
+Use Pygments in Java
+=====================
+
+Thanks to `Jython `_ it is possible to use Pygments in
+Java.
+
+This page is a simple tutorial to get an idea of how this works. You can
+then look at the `Jython documentation `_ for more
+advanced uses.
+
+Since version 1.5, Pygments is deployed on `Maven Central
+`_ as a JAR, as is Jython
+which makes it a lot easier to create a Java project.
+
+Here is an example of a `Maven `_ ``pom.xml`` file for a
+project running Pygments:
+
+.. sourcecode:: xml
+
+    
+
+    
+      4.0.0
+      example
+      example
+      1.0-SNAPSHOT
+      
+        
+          org.python
+          jython-standalone
+          2.5.3
+        
+        
+          org.pygments
+          pygments
+          1.5
+          runtime
+        
+      
+    
+
+The following Java example:
+
+.. sourcecode:: java
+
+    PythonInterpreter interpreter = new PythonInterpreter();
+
+    // Set a variable with the content you want to work with
+    interpreter.set("code", code);
+
+    // Simple use Pygments as you would in Python
+    interpreter.exec("from pygments import highlight\n"
+        + "from pygments.lexers import PythonLexer\n"
+        + "from pygments.formatters import HtmlFormatter\n"
+        + "\nresult = highlight(code, PythonLexer(), HtmlFormatter())");
+
+    // Get the result that has been set in a variable
+    System.out.println(interpreter.get("result", String.class));
+
+will print something like:
+
+.. sourcecode:: html
+
+    

+    
print "Hello World"

+    

diff --git a/doc/docs/lexerdevelopment.rst b/doc/docs/lexerdevelopment.rst
new file mode 100644
index 0000000..354b1d4
--- /dev/null
+++ b/doc/docs/lexerdevelopment.rst
@@ -0,0 +1,748 @@
+.. -*- mode: rst -*-
+
+.. highlight:: python
+
+====================
+Write your own lexer
+====================
+
+If a lexer for your favorite language is missing in the Pygments package, you
+can easily write your own and extend Pygments.
+
+All you need can be found inside the :mod:`pygments.lexer` module.  As you can
+read in the :doc:`API documentation `, a lexer is a class that is
+initialized with some keyword arguments (the lexer options) and that provides a
+:meth:`.get_tokens_unprocessed()` method which is given a string or unicode
+object with the data to lex.
+
+The :meth:`.get_tokens_unprocessed()` method must return an iterator or iterable
+containing tuples in the form ``(index, token, value)``.  Normally you don't
+need to do this since there are base lexers that do most of the work and that
+you can subclass.
+
+RegexLexer
+==========
+
+The lexer base class used by almost all of Pygments' lexers is the
+:class:`RegexLexer`.  This class allows you to define lexing rules in terms of
+*regular expressions* for different *states*.
+
+States are groups of regular expressions that are matched against the input
+string at the *current position*.  If one of these expressions matches, a
+corresponding action is performed (such as yielding a token with a specific
+type, or changing state), the current position is set to where the last match
+ended and the matching process continues with the first regex of the current
+state.
+
+Lexer states are kept on a stack: each time a new state is entered, the new
+state is pushed onto the stack.  The most basic lexers (like the `DiffLexer`)
+just need one state.
+
+Each state is defined as a list of tuples in the form (`regex`, `action`,
+`new_state`) where the last item is optional.  In the most basic form, `action`
+is a token type (like `Name.Builtin`).  That means: When `regex` matches, emit a
+token with the match text and type `tokentype` and push `new_state` on the state
+stack.  If the new state is ``'#pop'``, the topmost state is popped from the
+stack instead.  To pop more than one state, use ``'#pop:2'`` and so on.
+``'#push'`` is a synonym for pushing a second time the current state on top of
+the stack.
+
+The following example shows the `DiffLexer` from the builtin lexers.  Note that
+it contains some additional attributes `name`, `aliases` and `filenames` which
+aren't required for a lexer.  They are used by the builtin lexer lookup
+functions. ::
+
+    from pygments.lexer import RegexLexer
+    from pygments.token import *
+
+    class DiffLexer(RegexLexer):
+        name = 'Diff'
+        aliases = ['diff']
+        filenames = ['*.diff']
+
+        tokens = {
+            'root': [
+                (r' .*\n', Text),
+                (r'\+.*\n', Generic.Inserted),
+                (r'-.*\n', Generic.Deleted),
+                (r'@.*\n', Generic.Subheading),
+                (r'Index.*\n', Generic.Heading),
+                (r'=.*\n', Generic.Heading),
+                (r'.*\n', Text),
+            ]
+        }
+
+As you can see this lexer only uses one state.  When the lexer starts scanning
+the text, it first checks if the current character is a space.  If this is true
+it scans everything until newline and returns the data as a `Text` token (which
+is the "no special highlighting" token).
+
+If this rule doesn't match, it checks if the current char is a plus sign.  And
+so on.
+
+If no rule matches at the current position, the current char is emitted as an
+`Error` token that indicates a lexing error, and the position is increased by
+one.
+
+
+Adding and testing a new lexer
+==============================
+
+The easiest way to use a new lexer is to use Pygments' support for loading
+the lexer from a file relative to your current directory.
+
+First, change the name of your lexer class to CustomLexer:
+
+.. code-block:: python
+
+    from pygments.lexer import RegexLexer
+    from pygments.token import *
+
+    class CustomLexer(RegexLexer):
+        """All your lexer code goes here!"""
+
+Then you can load and test the lexer from the command line with the additional
+flag ``-x``:
+
+.. code-block:: console
+
+    $ python -m pygments -x -l your_lexer_file.py 
+
+To specify a class name other than CustomLexer, append it with a colon:
+
+.. code-block:: console
+
+    $ python -m pygments -x -l your_lexer.py:SomeLexer 
+
+Or, using the Python API:
+
+.. code-block:: python
+
+    # For a lexer named CustomLexer
+    your_lexer = load_lexer_from_file(filename, **options)
+
+    # For a lexer named MyNewLexer
+    your_named_lexer = load_lexer_from_file(filename, "MyNewLexer", **options)
+
+When loading custom lexers and formatters, be extremely careful to use only
+trusted files; Pygments will perform the equivalent of ``eval`` on them.
+
+If you only want to use your lexer with the Pygments API, you can import and
+instantiate the lexer yourself, then pass it to :func:`pygments.highlight`.
+
+Use the ``-f`` flag to select a different output format than terminal
+escape sequences. The :class:`pygments.formatters.html.HtmlFormatter` helps
+you with debugging your lexer. You can use the ``debug_token_types`` option
+to display the token types assigned to each part of your input file:
+
+.. code-block:: console
+
+    $ python -m pygments -x -f html -Ofull,debug_token_types -l your_lexer.py:SomeLexer 
+
+Hover over each token to see the token type displayed as a tooltip.
+
+To prepare your new lexer for inclusion in the Pygments distribution, so that it
+will be found when passing filenames or lexer aliases from the command line, you
+have to perform the following steps.
+
+First, change to the current directory containing the Pygments source code.  You
+will need to have either an unpacked source tarball, or (preferably) a copy
+cloned from GitHub.
+
+.. code-block:: console
+
+    $ cd pygments
+
+Select a matching module under ``pygments/lexers``, or create a new module for
+your lexer class.
+
+.. note::
+
+  We encourage you to put your lexer class into its own module, unless it's a
+  very small derivative of an already existing lexer.
+
+Next, make sure the lexer is known from outside of the module.  All modules in
+the ``pygments.lexers`` package specify ``__all__``. For example,
+``esoteric.py`` sets::
+
+    __all__ = ['BrainfuckLexer', 'BefungeLexer', ...]
+
+Add the name of your lexer class to this list (or create the list if your lexer
+is the only class in the module).
+
+Finally the lexer can be made publicly known by rebuilding the lexer mapping.
+In the root directory of the source (where the ``Makefile`` is located), run:
+
+.. code-block:: console
+
+    $ make mapfiles
+
+To test the new lexer, store an example file in
+``tests/examplefiles/``.  For example, to test your
+``DiffLexer``, add a ``tests/examplefiles/diff/example.diff`` containing a
+sample diff output.  To (re)generate the lexer output which the file is checked
+against, use the command ``pytest tests/examplefiles/diff --update-goldens``.
+
+Now you can use ``python -m pygments`` from the current root of the checkout to
+render your example to HTML:
+
+.. code-block:: console
+
+    $ python -m pygments -O full -f html -o /tmp/example.html tests/examplefiles/diff/example.diff
+
+Note that this explicitly calls the ``pygments`` module in the current
+directory. This ensures your modifications are used. Otherwise a possibly
+already installed, unmodified version without your new lexer would have been
+called from the system search path (``$PATH``).
+
+To view the result, open ``/tmp/example.html`` in your browser.
+
+Once the example renders as expected, you should run the complete test suite:
+
+.. code-block:: console
+
+    $ make test
+
+It also tests that your lexer fulfills the lexer API and certain invariants,
+such as that the concatenation of all token text is the same as the input text.
+
+
+Regex Flags
+===========
+
+You can either define regex flags locally in the regex (``r'(?x)foo bar'``) or
+globally by adding a `flags` attribute to your lexer class.  If no attribute is
+defined, it defaults to `re.MULTILINE`.  For more information about regular
+expression flags see the page about `regular expressions`_ in the Python
+documentation.
+
+.. _regular expressions: https://docs.python.org/library/re.html#regular-expression-syntax
+
+
+Scanning multiple tokens at once
+================================
+
+So far, the `action` element in the rule tuple of regex, action and state has
+been a single token type.  Now we look at the first of several other possible
+values.
+
+Here is a more complex lexer that highlights INI files.  INI files consist of
+sections, comments and ``key = value`` pairs::
+
+    from pygments.lexer import RegexLexer, bygroups
+    from pygments.token import *
+
+    class IniLexer(RegexLexer):
+        name = 'INI'
+        aliases = ['ini', 'cfg']
+        filenames = ['*.ini', '*.cfg']
+
+        tokens = {
+            'root': [
+                (r'\s+', Text),
+                (r';.*?$', Comment),
+                (r'\[.*?\]$', Keyword),
+                (r'(.*?)(\s*)(=)(\s*)(.*?)$',
+                 bygroups(Name.Attribute, Text, Operator, Text, String))
+            ]
+        }
+
+The lexer first looks for whitespace, comments and section names.  Later it
+looks for a line that looks like a key, value pair, separated by an ``'='``
+sign, and optional whitespace.
+
+The `bygroups` helper yields each capturing group in the regex with a different
+token type.  First the `Name.Attribute` token, then a `Text` token for the
+optional whitespace, after that a `Operator` token for the equals sign. Then a
+`Text` token for the whitespace again.  The rest of the line is returned as
+`String`.
+
+Note that for this to work, every part of the match must be inside a capturing
+group (a ``(...)``), and there must not be any nested capturing groups.  If you
+nevertheless need a group, use a non-capturing group defined using this syntax:
+``(?:some|words|here)`` (note the ``?:`` after the beginning parenthesis).
+
+If you find yourself needing a capturing group inside the regex which shouldn't
+be part of the output but is used in the regular expressions for backreferencing
+(eg: ``r'(<(foo|bar)>)(.*?)()'``), you can pass `None` to the bygroups
+function and that group will be skipped in the output.
+
+
+Changing states
+===============
+
+Many lexers need multiple states to work as expected.  For example, some
+languages allow multiline comments to be nested.  Since this is a recursive
+pattern it's impossible to lex just using regular expressions.
+
+Here is a lexer that recognizes C++ style comments (multi-line with ``/* */``
+and single-line with ``//`` until end of line)::
+
+    from pygments.lexer import RegexLexer
+    from pygments.token import *
+
+    class CppCommentLexer(RegexLexer):
+        name = 'Example Lexer with states'
+
+        tokens = {
+            'root': [
+                (r'[^/]+', Text),
+                (r'/\*', Comment.Multiline, 'comment'),
+                (r'//.*?$', Comment.Singleline),
+                (r'/', Text)
+            ],
+            'comment': [
+                (r'[^*/]+', Comment.Multiline),
+                (r'/\*', Comment.Multiline, '#push'),
+                (r'\*/', Comment.Multiline, '#pop'),
+                (r'[*/]', Comment.Multiline)
+            ]
+        }
+
+This lexer starts lexing in the ``'root'`` state. It tries to match as much as
+possible until it finds a slash (``'/'``).  If the next character after the slash
+is an asterisk (``'*'``) the `RegexLexer` sends those two characters to the
+output stream marked as `Comment.Multiline` and continues lexing with the rules
+defined in the ``'comment'`` state.
+
+If there wasn't an asterisk after the slash, the `RegexLexer` checks if it's a
+Singleline comment (i.e. followed by a second slash).  If this also wasn't the
+case it must be a single slash, which is not a comment starter (the separate
+regex for a single slash must also be given, else the slash would be marked as
+an error token).
+
+Inside the ``'comment'`` state, we do the same thing again.  Scan until the
+lexer finds a star or slash.  If it's the opening of a multiline comment, push
+the ``'comment'`` state on the stack and continue scanning, again in the
+``'comment'`` state.  Else, check if it's the end of the multiline comment.  If
+yes, pop one state from the stack.
+
+Note: If you pop from an empty stack you'll get an `IndexError`.  (There is an
+easy way to prevent this from happening: don't ``'#pop'`` in the root state).
+
+If the `RegexLexer` encounters a newline that is flagged as an error token, the
+stack is emptied and the lexer continues scanning in the ``'root'`` state.  This
+can help producing error-tolerant highlighting for erroneous input, e.g. when a
+single-line string is not closed.
+
+
+Advanced state tricks
+=====================
+
+There are a few more things you can do with states:
+
+- You can push multiple states onto the stack if you give a tuple instead of a
+  simple string as the third item in a rule tuple.  For example, if you want to
+  match a comment containing a directive, something like:
+
+  .. code-block:: text
+
+      /*     rest of comment */
+
+  you can use this rule::
+
+      tokens = {
+          'root': [
+              (r'/\* <', Comment, ('comment', 'directive')),
+              ...
+          ],
+          'directive': [
+              (r'[^>]+', Comment.Directive),
+              (r'>', Comment, '#pop'),
+          ],
+          'comment': [
+              (r'[^*]+', Comment),
+              (r'\*/', Comment, '#pop'),
+              (r'\*', Comment),
+          ]
+      }
+
+  When this encounters the above sample, first ``'comment'`` and ``'directive'``
+  are pushed onto the stack, then the lexer continues in the directive state
+  until it finds the closing ``>``, then it continues in the comment state until
+  the closing ``*/``.  Then, both states are popped from the stack again and
+  lexing continues in the root state.
+
+  .. versionadded:: 0.9
+     The tuple can contain the special ``'#push'`` and ``'#pop'`` (but not
+     ``'#pop:n'``) directives.
+
+
+- You can include the rules of a state in the definition of another.  This is
+  done by using `include` from `pygments.lexer`::
+
+      from pygments.lexer import RegexLexer, bygroups, include
+      from pygments.token import *
+
+      class ExampleLexer(RegexLexer):
+          tokens = {
+              'comments': [
+                  (r'(?s)/\*.*?\*/', Comment),
+                  (r'//.*?\n', Comment),
+              ],
+              'root': [
+                  include('comments'),
+                  (r'(function)( )(\w+)( )({)',
+                   bygroups(Keyword, Whitespace, Name, Whitespace, Punctuation), 'function'),
+                  (r'.*\n', Text),
+              ],
+              'function': [
+                  (r'[^}/]+', Text),
+                  include('comments'),
+                  (r'/', Text),
+                  (r'\}', Punctuation, '#pop'),
+              ]
+          }
+
+  This is a hypothetical lexer for a language that consist of functions and
+  comments.  Because comments can occur at toplevel and in functions, we need
+  rules for comments in both states.  As you can see, the `include` helper saves
+  repeating rules that occur more than once (in this example, the state
+  ``'comment'`` will never be entered by the lexer, as it's only there to be
+  included in ``'root'`` and ``'function'``).
+
+- Sometimes, you may want to "combine" a state from existing ones.  This is
+  possible with the `combined` helper from `pygments.lexer`.
+
+  If you, instead of a new state, write ``combined('state1', 'state2')`` as the
+  third item of a rule tuple, a new anonymous state will be formed from state1
+  and state2 and if the rule matches, the lexer will enter this state.
+
+  This is not used very often, but can be helpful in some cases, such as the
+  `PythonLexer`'s string literal processing.
+
+- If you want your lexer to start lexing in a different state you can modify the
+  stack by overriding the `get_tokens_unprocessed()` method::
+
+      from pygments.lexer import RegexLexer
+
+      class ExampleLexer(RegexLexer):
+          tokens = {...}
+
+          def get_tokens_unprocessed(self, text, stack=('root', 'otherstate')):
+              for item in RegexLexer.get_tokens_unprocessed(self, text, stack):
+                  yield item
+
+  Some lexers like the `PhpLexer` use this to make the leading ``', Name.Tag),
+            ],
+            'script-content': [
+                (r'(.+?)(<\s*/\s*script\s*>)',
+                 bygroups(using(JavascriptLexer), Name.Tag),
+                 '#pop'),
+            ]
+        }
+
+Here the content of a ```` end tag is processed by the `JavascriptLexer`,
+while the end tag is yielded as a normal token with the `Name.Tag` type.
+
+Also note the ``(r'<\s*script\s*', Name.Tag, ('script-content', 'tag'))`` rule.
+Here, two states are pushed onto the state stack, ``'script-content'`` and
+``'tag'``.  That means that first ``'tag'`` is processed, which will lex
+attributes and the closing ``>``, then the ``'tag'`` state is popped and the
+next state on top of the stack will be ``'script-content'``.
+
+Since you cannot refer to the class currently being defined, use `this`
+(imported from `pygments.lexer`) to refer to the current lexer class, i.e.
+``using(this)``.  This construct may seem unnecessary, but this is often the
+most obvious way of lexing arbitrary syntax between fixed delimiters without
+introducing deeply nested states.
+
+The `using()` helper has a special keyword argument, `state`, which works as
+follows: if given, the lexer to use initially is not in the ``"root"`` state,
+but in the state given by this argument.  This does not work with advanced
+`RegexLexer` subclasses such as `ExtendedRegexLexer` (see below).
+
+Any other keywords arguments passed to `using()` are added to the keyword
+arguments used to create the lexer.
+
+
+Delegating Lexer
+================
+
+Another approach for nested lexers is the `DelegatingLexer` which is for example
+used for the template engine lexers.  It takes two lexers as arguments on
+initialisation: a `root_lexer` and a `language_lexer`.
+
+The input is processed as follows: First, the whole text is lexed with the
+`language_lexer`.  All tokens yielded with the special type of ``Other`` are
+then concatenated and given to the `root_lexer`.  The language tokens of the
+`language_lexer` are then inserted into the `root_lexer`'s token stream at the
+appropriate positions. ::
+
+    from pygments.lexer import DelegatingLexer
+    from pygments.lexers.web import HtmlLexer, PhpLexer
+
+    class HtmlPhpLexer(DelegatingLexer):
+        def __init__(self, **options):
+            super().__init__(HtmlLexer, PhpLexer, **options)
+
+This procedure ensures that e.g. HTML with template tags in it is highlighted
+correctly even if the template tags are put into HTML tags or attributes.
+
+If you want to change the needle token ``Other`` to something else, you can give
+the lexer another token type as the third parameter::
+
+    DelegatingLexer.__init__(MyLexer, OtherLexer, Text, **options)
+
+
+Callbacks
+=========
+
+Sometimes the grammar of a language is so complex that a lexer would be unable
+to process it just by using regular expressions and stacks.
+
+For this, the `RegexLexer` allows callbacks to be given in rule tuples, instead
+of token types (`bygroups` and `using` are nothing else but preimplemented
+callbacks).  The callback must be a function taking two arguments:
+
+* the lexer itself
+* the match object for the last matched rule
+
+The callback must then return an iterable of (or simply yield) ``(index,
+tokentype, value)`` tuples, which are then just passed through by
+`get_tokens_unprocessed()`.  The ``index`` here is the position of the token in
+the input string, ``tokentype`` is the normal token type (like `Name.Builtin`),
+and ``value`` the associated part of the input string.
+
+You can see an example here::
+
+    from pygments.lexer import RegexLexer
+    from pygments.token import Generic
+
+    class HypotheticLexer(RegexLexer):
+
+        def headline_callback(lexer, match):
+            equal_signs = match.group(1)
+            text = match.group(2)
+            yield match.start(), Generic.Headline, equal_signs + text + equal_signs
+
+        tokens = {
+            'root': [
+                (r'(=+)(.*?)(\1)', headline_callback)
+            ]
+        }
+
+If the regex for the `headline_callback` matches, the function is called with
+the match object.  Note that after the callback is done, processing continues
+normally, that is, after the end of the previous match.  The callback has no
+possibility to influence the position.
+
+There are not really any simple examples for lexer callbacks, but you can see
+them in action e.g. in the `SMLLexer` class in `ml.py`_.
+
+.. _ml.py: https://github.com/pygments/pygments/blob/master/pygments/lexers/ml.py
+
+
+The ExtendedRegexLexer class
+============================
+
+The `RegexLexer`, even with callbacks, unfortunately isn't powerful enough for
+the funky syntax rules of languages such as Ruby.
+
+But fear not; even then you don't have to abandon the regular expression
+approach: Pygments has a subclass of `RegexLexer`, the `ExtendedRegexLexer`.
+All features known from RegexLexers are available here too, and the tokens are
+specified in exactly the same way, *except* for one detail:
+
+The `get_tokens_unprocessed()` method holds its internal state data not as local
+variables, but in an instance of the `pygments.lexer.LexerContext` class, and
+that instance is passed to callbacks as a third argument. This means that you
+can modify the lexer state in callbacks.
+
+The `LexerContext` class has the following members:
+
+* `text` -- the input text
+* `pos` -- the current starting position that is used for matching regexes
+* `stack` -- a list containing the state stack
+* `end` -- the maximum position to which regexes are matched, this defaults to
+  the length of `text`
+
+Additionally, the `get_tokens_unprocessed()` method can be given a
+`LexerContext` instead of a string and will then process this context instead of
+creating a new one for the string argument.
+
+Note that because you can set the current position to anything in the callback,
+it won't be automatically be set by the caller after the callback is finished.
+For example, this is how the hypothetical lexer above would be written with the
+`ExtendedRegexLexer`::
+
+    from pygments.lexer import ExtendedRegexLexer
+    from pygments.token import Generic
+
+    class ExHypotheticLexer(ExtendedRegexLexer):
+
+        def headline_callback(lexer, match, ctx):
+            equal_signs = match.group(1)
+            text = match.group(2)
+            yield match.start(), Generic.Headline, equal_signs + text + equal_signs
+            ctx.pos = match.end()
+
+        tokens = {
+            'root': [
+                (r'(=+)(.*?)(\1)', headline_callback)
+            ]
+        }
+
+This might sound confusing (and it can really be). But it is needed, and for an
+example look at the Ruby lexer in `ruby.py`_.
+
+.. _ruby.py: https://github.com/pygments/pygments/blob/master/pygments/lexers/ruby.py
+
+
+Handling Lists of Keywords
+==========================
+
+For a relatively short list (hundreds) you can construct an optimized regular
+expression directly using ``words()`` (longer lists, see next section).  This
+function handles a few things for you automatically, including escaping
+metacharacters and Python's first-match rather than longest-match in
+alternations.  Feel free to put the lists themselves in
+``pygments/lexers/_$lang_builtins.py`` (see examples there), and generated by
+code if possible.
+
+An example of using ``words()`` is something like::
+
+    from pygments.lexer import RegexLexer, words, Name
+
+    class MyLexer(RegexLexer):
+
+        tokens = {
+            'root': [
+                (words(('else', 'elseif'), suffix=r'\b'), Name.Builtin),
+                (r'\w+', Name),
+            ],
+        }
+
+As you can see, you can add ``prefix`` and ``suffix`` parts to the constructed
+regex.
+
+
+Modifying Token Streams
+=======================
+
+Some languages ship a lot of builtin functions (for example PHP).  The total
+amount of those functions differs from system to system because not everybody
+has every extension installed.  In the case of PHP there are over 3000 builtin
+functions.  That's an incredibly huge amount of functions, much more than you
+want to put into a regular expression.
+
+But because only `Name` tokens can be function names this is solvable by
+overriding the ``get_tokens_unprocessed()`` method.  The following lexer
+subclasses the `PythonLexer` so that it highlights some additional names as
+pseudo keywords::
+
+    from pygments.lexers.python import PythonLexer
+    from pygments.token import Name, Keyword
+
+    class MyPythonLexer(PythonLexer):
+        EXTRA_KEYWORDS = set(('foo', 'bar', 'foobar', 'barfoo', 'spam', 'eggs'))
+
+        def get_tokens_unprocessed(self, text):
+            for index, token, value in PythonLexer.get_tokens_unprocessed(self, text):
+                if token is Name and value in self.EXTRA_KEYWORDS:
+                    yield index, Keyword.Pseudo, value
+                else:
+                    yield index, token, value
+
+The `PhpLexer` and `LuaLexer` use this method to resolve builtin functions.
diff --git a/doc/docs/lexers.rst b/doc/docs/lexers.rst
new file mode 100644
index 0000000..446c5a9
--- /dev/null
+++ b/doc/docs/lexers.rst
@@ -0,0 +1,69 @@
+.. -*- mode: rst -*-
+
+================
+Available lexers
+================
+
+This page lists all available builtin lexers and the options they take.
+
+Currently, **all lexers** support these options:
+
+`stripnl`
+    Strip leading and trailing newlines from the input (default: ``True``)
+
+`stripall`
+    Strip all leading and trailing whitespace from the input (default:
+    ``False``).
+
+`ensurenl`
+    Make sure that the input ends with a newline (default: ``True``).  This
+    is required for some lexers that consume input linewise.
+
+    .. versionadded:: 1.3
+
+`tabsize`
+    If given and greater than 0, expand tabs in the input (default: ``0``).
+
+`encoding`
+    If given, must be an encoding name (such as ``"utf-8"``). This encoding
+    will be used to convert the input string to Unicode (if it is not already
+    a Unicode string). The default is ``"guess"``.
+
+    If this option is set to ``"guess"``, a simple UTF-8 vs. Latin-1
+    detection is used, if it is set to ``"chardet"``, the
+    `chardet library `_ is used to
+    guess the encoding of the input.
+
+    .. versionadded:: 0.6
+
+
+The "Short Names" field lists the identifiers that can be used with the
+`get_lexer_by_name()` function.
+
+These lexers are builtin and can be imported from `pygments.lexers`:
+
+.. pygmentsdoc:: lexers
+
+
+Iterating over all lexers
+-------------------------
+
+.. versionadded:: 0.6
+
+To get all lexers (both the builtin and the plugin ones), you can
+use the `get_all_lexers()` function from the `pygments.lexers`
+module:
+
+.. sourcecode:: pycon
+
+    >>> from pygments.lexers import get_all_lexers
+    >>> i = get_all_lexers()
+    >>> i.next()
+    ('Diff', ('diff',), ('*.diff', '*.patch'), ('text/x-diff', 'text/x-patch'))
+    >>> i.next()
+    ('Delphi', ('delphi', 'objectpascal', 'pas', 'pascal'), ('*.pas',), ('text/x-pascal',))
+    >>> i.next()
+    ('XML+Ruby', ('xml+ruby', 'xml+erb'), (), ())
+
+As you can see, the return value is an iterator which yields tuples
+in the form ``(name, aliases, filetypes, mimetypes)``.
diff --git a/doc/docs/moinmoin.rst b/doc/docs/moinmoin.rst
new file mode 100644
index 0000000..80ed25c
--- /dev/null
+++ b/doc/docs/moinmoin.rst
@@ -0,0 +1,39 @@
+.. -*- mode: rst -*-
+
+============================
+Using Pygments with MoinMoin
+============================
+
+From Pygments 0.7, the source distribution ships a `Moin`_ parser plugin that
+can be used to get Pygments highlighting in Moin wiki pages.
+
+To use it, copy the file `external/moin-parser.py` from the Pygments
+distribution to the `data/plugin/parser` subdirectory of your Moin instance.
+Edit the options at the top of the file (currently ``ATTACHMENTS`` and
+``INLINESTYLES``) and rename the file to the name that the parser directive
+should have. For example, if you name the file ``code.py``, you can get a
+highlighted Python code sample with this Wiki markup::
+
+    {{{
+    #!code python
+    [...]
+    }}}
+
+where ``python`` is the Pygments name of the lexer to use.
+
+Additionally, if you set the ``ATTACHMENTS`` option to True, Pygments will also
+be called for all attachments for whose filenames there is no other parser
+registered.
+
+You are responsible for including CSS rules that will map the Pygments CSS
+classes to colors. You can output a stylesheet file with `pygmentize`, put it
+into the `htdocs` directory of your Moin instance and then include it in the
+`stylesheets` configuration option in the Moin config, e.g.::
+
+    stylesheets = [('screen', '/htdocs/pygments.css')]
+
+If you do not want to do that and are willing to accept larger HTML output, you
+can set the ``INLINESTYLES`` option to True.
+
+
+.. _Moin: https://moinmo.in/
diff --git a/doc/docs/plugins.rst b/doc/docs/plugins.rst
new file mode 100644
index 0000000..6738860
--- /dev/null
+++ b/doc/docs/plugins.rst
@@ -0,0 +1,122 @@
+=======
+Plugins
+=======
+
+If you want to extend Pygments without hacking the sources, but want to
+use the lexer/formatter/style/filter lookup functions (`lexers.get_lexer_by_name`
+et al.), you can use `setuptools`_ entrypoints to add new lexers, formatters
+or styles as if they were in the Pygments core.
+
+.. _setuptools: https://pypi.org/project/setuptools/
+
+That means you can use your highlighter modules with the `pygmentize` script,
+which relies on the mentioned functions.
+
+
+Plugin discovery
+================
+
+At runtime, discovering plugins is preferentially done using Python's
+standard library module `importlib.metadata`_, available in Python 3.8
+and higher. In earlier Python versions, Pygments attempts to use the
+`importlib_metadata`_ backport, if available. If not available, a
+fallback is attempted on the older `pkg_resources`_ module. Finally, if
+``pkg_resources`` is not available, no plugins will be loaded at
+all. Note that ``pkg_resources`` is distributed with `setuptools`_, and
+thus available on most Python environments. However, ``pkg_resources``
+is considerably slower than ``importlib.metadata`` or its
+``importlib_metadata`` backport. For this reason, if you run Pygments
+under Python older than 3.8, it is recommended to install
+``importlib-metadata``. Pygments defines a ``plugins`` packaging extra,
+so you can ensure it is installed with best plugin support (i.e., that
+``importlib-metadata`` is also installed in case you are running Python
+earlier than 3.8) by specifying ``pygments[plugins]`` as the
+requirement, for example, with ``pip``:
+
+.. sourcecode:: shell
+
+   $ python -m pip install --user pygments[plugins]
+
+.. _importlib.metadata: https://docs.python.org/3.10/library/importlib.metadata.html
+.. _importlib_metadata: https://pypi.org/project/importlib-metadata
+.. _pkg_resources: https://setuptools.pypa.io/en/latest/pkg_resources.html
+
+
+Defining plugins through entrypoints
+====================================
+
+Here is a list of setuptools entrypoints that Pygments understands:
+
+`pygments.lexers`
+
+    This entrypoint is used for adding new lexers to the Pygments core.
+    The name of the entrypoint values doesn't really matter, Pygments extracts
+    required metadata from the class definition:
+
+    .. sourcecode:: ini
+
+        [pygments.lexers]
+        yourlexer = yourmodule:YourLexer
+
+    Note that you have to define ``name``, ``aliases`` and ``filename``
+    attributes so that you can use the highlighter from the command line:
+
+    .. sourcecode:: python
+
+        class YourLexer(...):
+            name = 'Name Of Your Lexer'
+            aliases = ['alias']
+            filenames = ['*.ext']
+
+
+`pygments.formatters`
+
+    You can use this entrypoint to add new formatters to Pygments. The
+    name of an entrypoint item is the name of the formatter. If you
+    prefix the name with a slash it's used as a filename pattern:
+
+    .. sourcecode:: ini
+
+        [pygments.formatters]
+        yourformatter = yourmodule:YourFormatter
+        /.ext = yourmodule:YourFormatter
+
+
+`pygments.styles`
+
+    To add a new style you can use this entrypoint. The name of the entrypoint
+    is the name of the style:
+
+    .. sourcecode:: ini
+
+        [pygments.styles]
+        yourstyle = yourmodule:YourStyle
+
+
+`pygments.filters`
+
+    Use this entrypoint to register a new filter. The name of the
+    entrypoint is the name of the filter:
+
+    .. sourcecode:: ini
+
+        [pygments.filters]
+        yourfilter = yourmodule:YourFilter
+
+
+How To Use Entrypoints
+======================
+
+This documentation doesn't explain how to use those entrypoints because this is
+covered in the `setuptools documentation`_. That page should cover everything
+you need to write a plugin.
+
+.. _setuptools documentation: https://setuptools.readthedocs.io/en/latest/
+
+
+Extending The Core
+==================
+
+If you have written a Pygments plugin that is open source, please inform us
+about that. There is a high chance that we'll add it to the Pygments
+distribution.
diff --git a/doc/docs/quickstart.rst b/doc/docs/quickstart.rst
new file mode 100644
index 0000000..b2a9c29
--- /dev/null
+++ b/doc/docs/quickstart.rst
@@ -0,0 +1,205 @@
+.. -*- mode: rst -*-
+
+===========================
+Introduction and Quickstart
+===========================
+
+
+Welcome to Pygments! This document explains the basic concepts and terms and
+gives a few examples of how to use the library.
+
+
+Architecture
+============
+
+There are four types of components that work together highlighting a piece of
+code:
+
+* A **lexer** splits the source into tokens, fragments of the source that
+  have a token type that determines what the text represents semantically
+  (e.g., keyword, string, or comment). There is a lexer for every language
+  or markup format that Pygments supports.
+* The token stream can be piped through **filters**, which usually modify
+  the token types or text fragments, e.g. uppercasing all keywords.
+* A **formatter** then takes the token stream and writes it to an output
+  file, in a format such as HTML, LaTeX or RTF.
+* While writing the output, a **style** determines how to highlight all the
+  different token types. It maps them to attributes like "red and bold".
+
+
+Example
+=======
+
+Here is a small example for highlighting Python code:
+
+.. sourcecode:: python
+
+    from pygments import highlight
+    from pygments.lexers import PythonLexer
+    from pygments.formatters import HtmlFormatter
+
+    code = 'print "Hello World"'
+    print(highlight(code, PythonLexer(), HtmlFormatter()))
+
+which prints something like this:
+
+.. sourcecode:: html
+
+    

+    
print "Hello World"

+    

+
+As you can see, Pygments uses CSS classes (by default, but you can change that)
+instead of inline styles in order to avoid outputting redundant style information over
+and over. A CSS stylesheet that contains all CSS classes possibly used in the output
+can be produced by:
+
+.. sourcecode:: python
+
+    print(HtmlFormatter().get_style_defs('.highlight'))
+
+The argument to :func:`get_style_defs` is used as an additional CSS selector:
+the output may look like this:
+
+.. sourcecode:: css
+
+    .highlight .k { color: #AA22FF; font-weight: bold }
+    .highlight .s { color: #BB4444 }
+    ...
+
+
+Options
+=======
+
+The :func:`highlight()` function supports a fourth argument called *outfile*, it
+must be a file object if given. The formatted output will then be written to
+this file instead of being returned as a string.
+
+Lexers and formatters both support options. They are given to them as keyword
+arguments either to the class or to the lookup method:
+
+.. sourcecode:: python
+
+    from pygments import highlight
+    from pygments.lexers import get_lexer_by_name
+    from pygments.formatters import HtmlFormatter
+
+    lexer = get_lexer_by_name("python", stripall=True)
+    formatter = HtmlFormatter(linenos=True, cssclass="source")
+    result = highlight(code, lexer, formatter)
+
+This makes the lexer strip all leading and trailing whitespace from the input
+(`stripall` option), lets the formatter output line numbers (`linenos` option),
+and sets the wrapping ``
``'s class to ``source`` (instead of
+``highlight``).
+
+Important options include:
+
+`encoding` : for lexers and formatters
+   Since Pygments uses Unicode strings internally, this determines which
+   encoding will be used to convert to or from byte strings.
+`style` : for formatters
+   The name of the style to use when writing the output.
+
+
+For an overview of builtin lexers and formatters and their options, visit the
+:doc:`lexer ` and :doc:`formatters ` lists.
+
+For a documentation on filters, see :doc:`this page `.
+
+
+Lexer and formatter lookup
+==========================
+
+If you want to lookup a built-in lexer by its alias or a filename, you can use
+one of the following methods:
+
+.. sourcecode:: pycon
+
+    >>> from pygments.lexers import (get_lexer_by_name,
+    ...     get_lexer_for_filename, get_lexer_for_mimetype)
+
+    >>> get_lexer_by_name('python')
+    
+
+    >>> get_lexer_for_filename('spam.rb')
+    
+
+    >>> get_lexer_for_mimetype('text/x-perl')
+    
+
+All these functions accept keyword arguments; they will be passed to the lexer
+as options.
+
+A similar API is available for formatters: use :func:`.get_formatter_by_name()`
+and :func:`.get_formatter_for_filename()` from the :mod:`pygments.formatters`
+module for this purpose.
+
+
+Guessing lexers
+===============
+
+If you don't know the content of the file, or you want to highlight a file
+whose extension is ambiguous, such as ``.html`` (which could contain plain HTML
+or some template tags), use these functions:
+
+.. sourcecode:: pycon
+
+    >>> from pygments.lexers import guess_lexer, guess_lexer_for_filename
+
+    >>> guess_lexer('#!/usr/bin/python\nprint "Hello World!"')
+    
+
+    >>> guess_lexer_for_filename('test.py', 'print "Hello World!"')
+    
+
+:func:`.guess_lexer()` passes the given content to the lexer classes'
+:meth:`analyse_text()` method and returns the one for which it returns the
+highest number.
+
+All lexers have two different filename pattern lists: the primary and the
+secondary one. The :func:`.get_lexer_for_filename()` function only uses the
+primary list, whose entries are supposed to be unique among all lexers.
+:func:`.guess_lexer_for_filename()`, however, will first loop through all lexers
+and look at the primary and secondary filename patterns if the filename matches.
+If only one lexer matches, it is returned, else the guessing mechanism of
+:func:`.guess_lexer()` is used with the matching lexers.
+
+As usual, keyword arguments to these functions are given to the created lexer
+as options.    
+
+
+Command line usage
+==================
+
+You can use Pygments from the command line, using the :program:`pygmentize`
+script::
+
+    $ pygmentize test.py
+
+will highlight the Python file test.py using ANSI escape sequences
+(a.k.a. terminal colors) and print the result to standard output.
+
+To output HTML, use the ``-f`` option::
+
+    $ pygmentize -f html -o test.html test.py
+
+to write an HTML-highlighted version of test.py to the file test.html.
+Note that it will only be a snippet of HTML, if you want a full HTML document,
+use the "full" option::
+
+    $ pygmentize -f html -O full -o test.html test.py
+
+This will produce a full HTML document with included stylesheet.
+
+A style can be selected with ``-O style=``.
+
+If you need a stylesheet for an existing HTML file using Pygments CSS classes,
+it can be created with::
+
+    $ pygmentize -S default -f html > style.css
+
+where ``default`` is the style name.
+
+More options and tricks can be found in the :doc:`command line reference
+`.
diff --git a/doc/docs/rstdirective.rst b/doc/docs/rstdirective.rst
new file mode 100644
index 0000000..edc117d
--- /dev/null
+++ b/doc/docs/rstdirective.rst
@@ -0,0 +1,22 @@
+.. -*- mode: rst -*-
+
+================================
+Using Pygments in ReST documents
+================================
+
+Many Python people use `ReST`_ for documentation their sourcecode, programs,
+scripts et cetera. This also means that documentation often includes sourcecode
+samples or snippets.
+
+You can easily enable Pygments support for your ReST texts using a custom
+directive -- this is also how this documentation displays source code.
+
+From Pygments 0.9, the directive is shipped in the distribution as
+`external/rst-directive.py`.  You can copy and adapt this code to your liking.
+
+.. removed -- too confusing
+   *Loosely related note:* The ReST lexer now recognizes ``.. sourcecode::`` and
+   ``.. code::`` directives and highlights the contents in the specified language
+   if the `handlecodeblocks` option is true.
+
+.. _ReST: https://docutils.sourceforge.io/rst.html
diff --git a/doc/docs/security.rst b/doc/docs/security.rst
new file mode 100644
index 0000000..72f2d05
--- /dev/null
+++ b/doc/docs/security.rst
@@ -0,0 +1,31 @@
+Security considerations
+-----------------------
+
+Pygments provides no guarantees on execution time, which needs to be taken
+into consideration when using Pygments to process arbitrary user inputs. For
+example, if you have a web service which uses Pygments for highlighting, there
+may be inputs which will cause the Pygments process to run "forever" and/or use
+significant amounts of memory. This can subsequently be used to perform a
+remote denial-of-service attack on the server if the processes are not
+terminated quickly.
+
+Unfortunately, it's practically impossible to harden Pygments itself against 
+those issues: Some regular expressions can result in "catastrophic 
+backtracking", but other bugs like incorrect matchers can also
+cause similar problems, and there is no way to find them in an automated fashion
+(short of solving the halting problem.) Pygments has extensive unit tests, 
+automated randomized testing, and is also tested by `OSS-Fuzz `_, 
+but we will never be able to eliminate all bugs in this area.
+
+Our recommendations are:
+
+* Ensure that the Pygments process is *terminated* after a reasonably short
+  timeout. In general Pygments should take seconds at most for reasonably-sized
+  input.
+* *Limit* the number of concurrent Pygments processes to avoid oversubscription
+  of resources.
+
+The Pygments authors will treat any bug resulting in long processing times with
+high priority -- it's one of those things that will be fixed in a patch release.
+When reporting a bug where you suspect super-linear execution times, please make
+sure to attach an input to reproduce it.
\ No newline at end of file
diff --git a/doc/docs/styledevelopment.rst b/doc/docs/styledevelopment.rst
new file mode 100644
index 0000000..8c4ec2d
--- /dev/null
+++ b/doc/docs/styledevelopment.rst
@@ -0,0 +1,96 @@
+.. -*- mode: rst -*-
+
+.. _creating-own-styles:
+
+Creating Own Styles
+===================
+
+So, how to create a style? All you have to do is to subclass `Style` and
+define some styles:
+
+.. sourcecode:: python
+
+    from pygments.style import Style
+    from pygments.token import Token, Comment, Keyword, Name, String, \
+         Error, Generic, Number, Operator
+
+
+    class YourStyle(Style):
+
+        styles = {
+            Token:                  '',
+            Comment:                'italic #888',
+            Keyword:                'bold #005',
+            Name:                   '#f00',
+            Name.Class:             'bold #0f0',
+            Name.Function:          '#0f0',
+            String:                 'bg:#eee #111'
+        }
+
+That's it, save it as ``your.py``. There are just a few rules. When you define a style for `Name`
+the style automatically also affects `Name.Function` and so on. If you
+defined ``'bold'`` and you don't want boldface for a subtoken use ``'nobold'``.
+
+(Philosophy: the styles aren't written in CSS syntax since this way
+they can be used for a variety of formatters.)
+
+``Token`` is the default style inherited by all token types.
+
+To make the style usable for Pygments, you must
+
+* either register it as a plugin (see :doc:`the plugin docs `)
+* or update the ``pygments.styles`` subpackage directory. For example:
+
+  * add ``your.py`` file
+  * register the new style by adding a line to the ``__init__.py`` file:
+  
+  .. sourcecode:: python
+  
+      STYLE_MAP = {
+          ...
+          'your':  'your::YourStyle',
+
+.. note::
+
+    You should *only* add it to the ``pygments.styles`` subdirectory if you are
+    working on a contribution to Pygments. You should not use that
+    method to extend an already existing copy of Pygments, use the plugins
+    mechanism for that.
+
+
+Style Rules
+===========
+
+Here a small overview of all allowed styles:
+
+``bold``
+    render text as bold
+``nobold``
+    don't render text as bold (to prevent subtokens being highlighted bold)
+``italic``
+    render text italic
+``noitalic``
+    don't render text as italic
+``underline``
+    render text underlined
+``nounderline``
+    don't render text underlined
+``bg:``
+    transparent background
+``bg:#000000``
+    background color (black)
+``border:``
+    no border
+``border:#ffffff``
+    border color (white)
+``#ff0000``
+    text color (red)
+``noinherit``
+    don't inherit styles from supertoken
+
+Note that there may not be a space between ``bg:`` and the color value
+since the style definition string is split at whitespace.
+Also, using named colors is not allowed since the supported color names
+vary for different formatters.
+
+Furthermore, not all lexers might support every style.
diff --git a/doc/docs/styles.rst b/doc/docs/styles.rst
new file mode 100644
index 0000000..91689d3
--- /dev/null
+++ b/doc/docs/styles.rst
@@ -0,0 +1,157 @@
+.. -*- mode: rst -*-
+
+======
+Styles
+======
+
+Pygments comes with :doc:`some builtin styles ` that work for both the
+HTML and LaTeX formatter.
+
+The builtin styles can be looked up with the `get_style_by_name` function:
+
+.. sourcecode:: pycon
+
+    >>> from pygments.styles import get_style_by_name
+    >>> get_style_by_name('colorful')
+    
+
+You can pass a instance of a `Style` class to a formatter as the `style`
+option in form of a string:
+
+.. sourcecode:: pycon
+
+    >>> from pygments.styles import get_style_by_name
+    >>> from pygments.formatters import HtmlFormatter
+    >>> HtmlFormatter(style='colorful').style
+    
+
+Or you can also import your own style (which must be a subclass of
+`pygments.style.Style`) and pass it to the formatter:
+
+.. sourcecode:: pycon
+
+    >>> from yourapp.yourmodule import YourStyle
+    >>> from pygments.formatters import HtmlFormatter
+    >>> HtmlFormatter(style=YourStyle).style
+    
+
+
+Creating Own Styles
+===================
+
+See :ref:`creating-own-styles`.
+
+
+Builtin Styles
+==============
+
+Pygments ships some builtin styles which are maintained by the Pygments team.
+
+To get a list of known styles you can use this snippet:
+
+.. sourcecode:: pycon
+
+    >>> from pygments.styles import STYLE_MAP
+    >>> STYLE_MAP.keys()
+    ['default', 'emacs', 'friendly', 'colorful']
+
+
+Getting a list of available styles
+==================================
+
+.. versionadded:: 0.6
+
+Because it could be that a plugin registered a style, there is
+a way to iterate over all styles:
+
+.. sourcecode:: pycon
+
+    >>> from pygments.styles import get_all_styles
+    >>> styles = list(get_all_styles())
+
+
+.. _AnsiTerminalStyle:
+
+Terminal Styles
+===============
+
+.. versionadded:: 2.2
+
+Custom styles used with the 256-color terminal formatter can also map colors to
+use the 8 default ANSI colors.  To do so, use ``ansigreen``, ``ansibrightred`` or
+any other colors defined in :attr:`pygments.style.ansicolors`.  Foreground ANSI
+colors will be mapped to the corresponding `escape codes 30 to 37
+`_ thus respecting any
+custom color mapping and themes provided by many terminal emulators.  Light
+variants are treated as foreground color with and an added bold flag.
+``bg:ansi`` will also be respected, except the light variant will be the
+same shade as their dark variant.
+
+See the following example where the color of the string ``"hello world"`` is
+governed by the escape sequence ``\x1b[34;01m`` (Ansi bright blue, Bold, 41 being red
+background) instead of an extended foreground & background color.
+
+.. sourcecode:: pycon
+
+    >>> from pygments import highlight
+    >>> from pygments.style import Style
+    >>> from pygments.token import Token
+    >>> from pygments.lexers import Python3Lexer
+    >>> from pygments.formatters import Terminal256Formatter
+
+    >>> class MyStyle(Style):
+            styles = {
+                Token.String:     'ansibrightblue bg:ansibrightred',
+            }
+
+    >>> code = 'print("Hello World")'
+    >>> result = highlight(code, Python3Lexer(), Terminal256Formatter(style=MyStyle))
+    >>> print(result.encode())
+    b'\x1b[34;41;01m"\x1b[39;49;00m\x1b[34;41;01mHello World\x1b[39;49;00m\x1b[34;41;01m"\x1b[39;49;00m'
+
+Colors specified using ``ansi*`` are converted to a default set of RGB colors
+when used with formatters other than the terminal-256 formatter.
+
+By definition of ANSI, the following colors are considered "light" colors, and
+will be rendered by most terminals as bold:
+
+- "brightblack" (darkgrey), "brightred", "brightgreen", "brightyellow", "brightblue",
+  "brightmagenta", "brightcyan", "white"
+
+The following are considered "dark" colors and will be rendered as non-bold:
+
+- "black", "red", "green", "yellow", "blue", "magenta", "cyan",
+  "gray"
+
+Exact behavior might depends on the terminal emulator you are using, and its
+settings.
+
+.. _new-ansi-color-names:
+
+.. versionchanged:: 2.4
+
+The definition of the ANSI color names has changed.
+New names are easier to understand and align to the colors used in other projects.
+
+===================== ====================
+New names             Pygments up to 2.3
+===================== ====================
+``ansiblack``         ``#ansiblack``
+``ansired``           ``#ansidarkred``
+``ansigreen``         ``#ansidarkgreen``
+``ansiyellow``        ``#ansibrown``
+``ansiblue``          ``#ansidarkblue``
+``ansimagenta``       ``#ansipurple``
+``ansicyan``          ``#ansiteal``
+``ansigray``          ``#ansilightgray``
+``ansibrightblack``   ``#ansidarkgray``
+``ansibrightred``     ``#ansired``
+``ansibrightgreen``   ``#ansigreen``
+``ansibrightyellow``  ``#ansiyellow``
+``ansibrightblue``    ``#ansiblue``
+``ansibrightmagenta`` ``#ansifuchsia``
+``ansibrightcyan``    ``#ansiturquoise``
+``ansiwhite``         ``#ansiwhite``
+===================== ====================
+
+Old ANSI color names are deprecated but will still work.
diff --git a/doc/docs/terminal-sessions.rst b/doc/docs/terminal-sessions.rst
new file mode 100644
index 0000000..45af0eb
--- /dev/null
+++ b/doc/docs/terminal-sessions.rst
@@ -0,0 +1,46 @@
+Interactive terminal/shell sessions
+-----------------------------------
+
+To highlight an interactive terminal or shell session, prefix your code snippet
+with a specially formatted prompt.
+
+Supported shells with examples are shown below. In each example, prompt parts in
+brackets ``[any]`` represent optional parts of the prompt, and prompt parts
+without brackets or in parenthesis ``(any)`` represent required parts of the
+prompt.
+
+* **Bash Session** (console, shell-session):
+
+  .. code-block:: console
+
+     [any@any]$ ls -lh
+     [any@any]# ls -lh
+     [any@any]% ls -lh
+     $ ls -lh
+     # ls -lh
+     % ls -lh
+     > ls -lh
+
+* **MSDOS Session** (doscon):
+
+  .. code-block:: doscon
+
+     [any]> dir
+     > dir
+     More? dir
+
+* **Tcsh Session** (tcshcon):
+
+  .. code-block:: tcshcon
+
+     (any)> ls -lh
+     ? ls -lh
+
+* **PowerShell Session** (ps1con):
+
+  .. code-block:: ps1con
+
+     PS[any]> Get-ChildItem
+     PS> Get-ChildItem
+     >> Get-ChildItem
+
diff --git a/doc/docs/tokens.rst b/doc/docs/tokens.rst
new file mode 100644
index 0000000..0bc7586
--- /dev/null
+++ b/doc/docs/tokens.rst
@@ -0,0 +1,376 @@
+.. -*- mode: rst -*-
+
+==============
+Builtin Tokens
+==============
+
+.. module:: pygments.token
+
+In the :mod:`pygments.token` module, there is a special object called `Token`
+that is used to create token types.
+
+You can create a new token type by accessing an attribute of `Token` whose
+name starts with an uppercase letter:
+
+.. sourcecode:: pycon
+
+    >>> from pygments.token import Token
+    >>> Token.String
+    Token.String
+    >>> Token.String is Token.String
+    True
+
+Note that tokens are singletons so you can use the ``is`` operator for comparing
+token types.
+
+You can also use the ``in`` operator to perform set tests:
+
+.. sourcecode:: pycon
+
+    >>> from pygments.token import Comment
+    >>> Comment.Single in Comment
+    True
+    >>> Comment in Comment.Multi
+    False
+
+This can be useful in :doc:`filters ` and if you write lexers on your
+own without using the base lexers.
+
+You can also split a token type into a hierarchy, and get the parent of it:
+
+.. sourcecode:: pycon
+
+    >>> String.split()
+    [Token, Token.Literal, Token.Literal.String]
+    >>> String.parent
+    Token.Literal
+
+In principle, you can create an unlimited number of token types but nobody can
+guarantee that a style would define style rules for a token type. Because of
+that, Pygments proposes some global token types defined in the
+`pygments.token.STANDARD_TYPES` dict.
+
+For some tokens aliases are already defined:
+
+.. sourcecode:: pycon
+
+    >>> from pygments.token import String
+    >>> String
+    Token.Literal.String
+
+Inside the :mod:`pygments.token` module the following aliases are defined:
+
+============= ============================ ====================================
+`Text`        `Token.Text`                 for any type of text data
+`Whitespace`  `Token.Text.Whitespace`      for whitespace
+`Error`       `Token.Error`                represents lexer errors
+`Other`       `Token.Other`                special token for data not
+                                           matched by a parser (e.g. HTML
+                                           markup in PHP code)
+`Keyword`     `Token.Keyword`              any kind of keywords
+`Name`        `Token.Name`                 variable/function names
+`Literal`     `Token.Literal`              Any literals
+`String`      `Token.Literal.String`       string literals
+`Number`      `Token.Literal.Number`       number literals
+`Operator`    `Token.Operator`             operators (``+``, ``not``...)
+`Punctuation` `Token.Punctuation`          punctuation (``[``, ``(``...)
+`Comment`     `Token.Comment`              any kind of comments
+`Generic`     `Token.Generic`              generic tokens (have a look at
+                                           the explanation below)
+============= ============================ ====================================
+
+Normally you just create token types using the already defined aliases. For each
+of those token aliases, a number of subtypes exists (excluding the special tokens
+`Token.Text`, `Token.Error` and `Token.Other`)
+
+It's also possible to convert strings to token types (for example
+if you want to supply a token from the command line):
+
+.. sourcecode:: pycon
+
+    >>> from pygments.token import String, string_to_tokentype
+    >>> string_to_tokentype("String")
+    Token.Literal.String
+    >>> string_to_tokentype("Token.Literal.String")
+    Token.Literal.String
+    >>> string_to_tokentype(String)
+    Token.Literal.String
+
+
+Keyword Tokens
+==============
+
+`Keyword`
+    For any kind of keyword (especially if it doesn't match any of the
+    subtypes of course).
+
+`Keyword.Constant`
+    For keywords that are constants (e.g. ``None`` in future Python versions).
+
+`Keyword.Declaration`
+    For keywords used for variable declaration (e.g. ``var`` in some programming
+    languages like JavaScript).
+
+`Keyword.Namespace`
+    For keywords used for namespace declarations (e.g. ``import`` in Python and
+    Java and ``package`` in Java).
+
+`Keyword.Pseudo`
+    For keywords that aren't really keywords (e.g. ``None`` in old Python
+    versions).
+
+`Keyword.Reserved`
+    For reserved keywords.
+
+`Keyword.Type`
+    For builtin types that can't be used as identifiers (e.g. ``int``,
+    ``char`` etc. in C).
+
+
+Name Tokens
+===========
+
+`Name`
+    For any name (variable names, function names, classes).
+
+`Name.Attribute`
+    For all attributes (e.g. in HTML tags).
+
+`Name.Builtin`
+    Builtin names; names that are available in the global namespace.
+
+`Name.Builtin.Pseudo`
+    Builtin names that are implicit (e.g. ``self`` in Ruby, ``this`` in Java).
+
+`Name.Class`
+    Class names. Because no lexer can know if a name is a class or a function
+    or something else this token is meant for class declarations.
+
+`Name.Constant`
+    Token type for constants. In some languages you can recognise a token by the
+    way it's defined (the value after a ``const`` keyword for example). In
+    other languages constants are uppercase by definition (Ruby).
+
+`Name.Decorator`
+    Token type for decorators. Decorators are syntactic elements in the Python
+    language. Similar syntax elements exist in C# and Java.
+
+`Name.Entity`
+    Token type for special entities. (e.g. `` `` in HTML).
+
+`Name.Exception`
+    Token type for exception names (e.g. ``RuntimeError`` in Python). Some languages
+    define exceptions in the function signature (Java). You can highlight
+    the name of that exception using this token then.
+
+`Name.Function`
+    Token type for function names.
+
+`Name.Function.Magic`
+    same as `Name.Function` but for special function names that have an implicit use
+    in a language (e.g. ``__init__`` method in Python).
+
+`Name.Label`
+    Token type for label names (e.g. in languages that support ``goto``).
+
+`Name.Namespace`
+    Token type for namespaces. (e.g. import paths in Java/Python), names following
+    the ``module``/``namespace`` keyword in other languages.
+
+`Name.Other`
+    Other names. Normally unused.
+
+`Name.Property`
+    Additional token type occasionally used for class attributes.
+
+`Name.Tag`
+    Tag names (in HTML/XML markup or configuration files).
+
+`Name.Variable`
+    Token type for variables. Some languages have prefixes for variable names
+    (PHP, Ruby, Perl). You can highlight them using this token.
+
+`Name.Variable.Class`
+    same as `Name.Variable` but for class variables (also static variables).
+
+`Name.Variable.Global`
+    same as `Name.Variable` but for global variables (used in Ruby, for
+    example).
+
+`Name.Variable.Instance`
+    same as `Name.Variable` but for instance variables.
+
+`Name.Variable.Magic`
+    same as `Name.Variable` but for special variable names that have an implicit use
+    in a language (e.g. ``__doc__`` in Python).
+
+
+Literals
+========
+
+`Literal`
+    For any literal (if not further defined).
+
+`Literal.Date`
+    for date literals (e.g. ``42d`` in Boo).
+
+
+`String`
+    For any string literal.
+
+`String.Affix`
+    Token type for affixes that further specify the type of the string they're
+    attached to (e.g. the prefixes ``r`` and ``u8`` in ``r"foo"`` and ``u8"foo"``).
+
+`String.Backtick`
+    Token type for strings enclosed in backticks.
+
+`String.Char`
+    Token type for single characters (e.g. Java, C).
+
+`String.Delimiter`
+    Token type for delimiting identifiers in "heredoc", raw and other similar
+    strings (e.g. the word ``END`` in Perl code ``print <<'END';``).
+
+`String.Doc`
+    Token type for documentation strings (for example Python).
+
+`String.Double`
+    Double quoted strings.
+
+`String.Escape`
+    Token type for escape sequences in strings.
+
+`String.Heredoc`
+    Token type for "heredoc" strings (e.g. in Ruby or Perl).
+
+`String.Interpol`
+    Token type for interpolated parts in strings (e.g. ``#{foo}`` in Ruby).
+
+`String.Other`
+    Token type for any other strings (for example ``%q{foo}`` string constructs
+    in Ruby).
+
+`String.Regex`
+    Token type for regular expression literals (e.g. ``/foo/`` in JavaScript).
+
+`String.Single`
+    Token type for single quoted strings.
+
+`String.Symbol`
+    Token type for symbols (e.g. ``:foo`` in LISP or Ruby).
+
+
+`Number`
+    Token type for any number literal.
+
+`Number.Bin`
+    Token type for binary literals (e.g. ``0b101010``).
+
+`Number.Float`
+    Token type for float literals (e.g. ``42.0``).
+
+`Number.Hex`
+    Token type for hexadecimal number literals (e.g. ``0xdeadbeef``).
+
+`Number.Integer`
+    Token type for integer literals (e.g. ``42``).
+
+`Number.Integer.Long`
+    Token type for long integer literals (e.g. ``42L`` in Python).
+
+`Number.Oct`
+    Token type for octal literals.
+
+
+Operators
+=========
+
+`Operator`
+    For any punctuation operator (e.g. ``+``, ``-``).
+
+`Operator.Word`
+    For any operator that is a word (e.g. ``not``).
+
+
+Punctuation
+===========
+
+.. versionadded:: 0.7
+
+`Punctuation`
+    For any punctuation which is not an operator (e.g. ``[``, ``(``...)
+
+`Punctuation.Marker`
+    For markers that point to a location (e.g., carets in Python
+    tracebacks for syntax errors).
+
+    .. versionadded:: 2.10
+
+
+Comments
+========
+
+`Comment`
+    Token type for any comment.
+
+`Comment.Hashbang`
+    Token type for hashbang comments (i.e. first lines of files that start with
+     ``#!``).
+
+`Comment.Multiline`
+    Token type for multiline comments.
+
+`Comment.Preproc`
+    Token type for preprocessor comments (also ```.
+
+.. versionadded:: 0.7
+   The formatters now also accept an `outencoding` option which will override
+   the `encoding` option if given. This makes it possible to use a single
+   options dict with lexers and formatters, and still have different input and
+   output encodings.
+
+.. _chardet: https://chardet.github.io/
diff --git a/doc/download.rst b/doc/download.rst
new file mode 100644
index 0000000..7ac0868
--- /dev/null
+++ b/doc/download.rst
@@ -0,0 +1,39 @@
+Download and installation
+=========================
+
+The current release is version |version|.
+
+Packaged versions
+-----------------
+
+You can download it `from the Python Package Index
+`_.  For installation of packages from
+PyPI, we recommend `Pip `_, which works on all
+major platforms.
+
+Under Linux, most distributions include a package for Pygments, usually called
+``pygments`` or ``python-pygments``.  You can install it with the package
+manager as usual.
+
+Development sources
+-------------------
+
+We're using the Git version control system.  You can get the development source
+using this command::
+
+    git clone https://github.com/pygments/pygments
+
+Development takes place at `GitHub `_.
+
+The latest changes in the development source code are listed in the `changelog
+`_.
+
+.. Documentation
+   -------------
+
+.. XXX todo
+
+   You can download the documentation either as
+      a bunch of rst files from the Git repository, see above, or
+      as a tar.gz containing rendered HTML files:

+      
pygmentsdocs.tar.gz

diff --git a/doc/examples/example.py b/doc/examples/example.py
new file mode 100644
index 0000000..6c9e2f1
--- /dev/null
+++ b/doc/examples/example.py
@@ -0,0 +1,14 @@
+from typing import Iterator
+
+# This is an example
+class Math:
+    @staticmethod
+    def fib(n: int) -> Iterator[int]:
+        """Fibonacci series up to n."""
+        a, b = 0, 1
+        while a < n:
+            yield a
+            a, b = b, a + b
+
+result = sum(Math.fib(42))
+print("The answer is {}".format(result))
diff --git a/doc/faq.rst b/doc/faq.rst
new file mode 100644
index 0000000..4e078dc
--- /dev/null
+++ b/doc/faq.rst
@@ -0,0 +1,142 @@
+:orphan:
+
+Pygments FAQ
+=============
+
+What is Pygments?
+-----------------
+
+Pygments is a syntax highlighting engine written in Python. That means, it will
+take source code (or other markup) in a supported language and output a
+processed version (in different formats) containing syntax highlighting markup.
+
+Its features include:
+
+* a wide range of common :doc:`languages and markup formats ` is supported
+* new languages and formats are added easily
+* a number of output formats is available, including:
+
+  - HTML
+  - ANSI sequences (console output)
+  - LaTeX
+  - RTF
+
+* it is usable as a command-line tool and as a library
+* parsing and formatting is fast
+
+Pygments is licensed under the BSD license.
+
+Where does the name Pygments come from?
+---------------------------------------
+
+*Py* of course stands for Python, while *pigments* are used for coloring paint,
+and in this case, source code!
+
+What are the system requirements?
+---------------------------------
+
+Pygments only needs a standard Python install, version 3.6 or higher. No
+additional libraries are needed.
+
+How can I use Pygments?
+-----------------------
+
+Pygments is usable as a command-line tool as well as a library.
+
+From the command-line, usage looks like this (assuming the pygmentize script is
+properly installed)::
+
+    pygmentize -f html /path/to/file.py
+
+This will print a HTML-highlighted version of /path/to/file.py to standard output.
+
+For a complete help, please run ``pygmentize -h``.
+
+Usage as a library is thoroughly demonstrated in the Documentation section.
+
+How do I make a new style?
+--------------------------
+
+Please see the :doc:`documentation on styles `.
+
+How can I report a bug or suggest a feature?
+--------------------------------------------
+
+Please report bugs and feature wishes in the tracker at GitHub.
+
+You can also e-mail the authors, see the contact details.
+
+I want this support for this language!
+--------------------------------------
+
+Instead of waiting for others to include language support, why not write it
+yourself? All you have to know is :doc:`outlined in the docs
+`.
+
+Can I use Pygments for programming language processing?
+-------------------------------------------------------
+
+The Pygments lexing machinery is quite powerful can be used to build lexers for
+basically all languages. However, parsing them is not possible, though some
+lexers go some steps in this direction in order to e.g. highlight function names
+differently.
+
+Also, error reporting is not the scope of Pygments. It focuses on correctly
+highlighting syntactically valid documents, not finding and compensating errors.
+
+Who uses Pygments?
+------------------
+
+This is an (incomplete) list of projects and sites known to use the Pygments highlighter.
+
+* `Wikipedia `_
+* `BitBucket `_, a Mercurial and Git hosting site
+* `The Sphinx documentation builder `_, for embedded source examples
+* `rst2pdf `_, a reStructuredText to PDF converter
+* `Codecov `_, a code coverage CI service
+* `Trac `_, the universal project management tool
+* `AsciiDoc `_, a text-based documentation generator
+* `ActiveState Code `_, the Python Cookbook successor
+* `ViewVC `_, a web-based version control repository browser
+* `BzrFruit `_, a Bazaar branch viewer
+* `QBzr `_, a cross-platform Qt-based GUI front end for Bazaar
+* `Review Board `_, a collaborative code reviewing tool
+* `Diamanda `_, a Django powered wiki system with support for Pygments
+* `Progopedia `_ (`English `_),
+  an encyclopedia of programming languages
+* `Bruce `_, a reStructuredText presentation tool
+* `PIDA `_, a universal IDE written in Python
+* `BPython `_, a curses-based intelligent Python shell
+* `PuDB `_, a console Python debugger
+* `XWiki `_, a wiki-based development framework in Java, using Jython
+* `roux `_, a script for running R scripts
+  and creating beautiful output including graphs
+* `hurl `_, a web service for making HTTP requests
+* `wxHTMLPygmentizer `_ is
+  a GUI utility, used to make code-colorization easier
+* `Postmarkup `_, a BBCode to XHTML generator
+* `WpPygments `_, and `WPygments
+  `_, highlighter plugins for WordPress
+* `Siafoo `_, a tool for sharing and storing useful code and programming experience
+* `D source `_, a community for the D programming language
+* `dpaste.com `_, another Django pastebin
+* `Django snippets `_, a pastebin for Django code
+* `Fayaa `_, a Chinese pastebin
+* `Incollo.com `_, a free collaborative debugging tool
+* `PasteBox `_, a pastebin focused on privacy
+* `hilite.me `_, a site to highlight code snippets
+* `patx.me `_, a pastebin
+* `Fluidic `_, an experiment in
+  integrating shells with a GUI
+* `pygments.rb `_, a pygments wrapper for Ruby
+* `Clygments `_, a pygments wrapper for
+  Clojure
+* `PHPygments `_, a pygments wrapper for PHP
+* `Spyder `_, the Scientific Python Development
+  Environment, uses pygments for the multi-language syntax highlighting in its
+  `editor `_.
+* `snippet.host `_, minimal text and code snippet hosting
+* `sourcehut `_, the hacker's forge
+
+If you have a project or web site using Pygments, `open an issue or PR
+`_ and we'll add a line here.
diff --git a/doc/index.rst b/doc/index.rst
new file mode 100644
index 0000000..dbd1596
--- /dev/null
+++ b/doc/index.rst
@@ -0,0 +1,47 @@
+Welcome!
+========
+
+This is the home of Pygments.  It is a generic syntax highlighter suitable for
+use in code hosting, forums, wikis or other applications that need to prettify
+source code.  Highlights are:
+
+* a wide range of |language_count| languages and other text formats is supported
+* special attention is paid to details that increase highlighting quality
+* support for new languages and formats are added easily; most languages use a
+  simple regex-based lexing mechanism
+* a number of output formats is available, among them HTML, RTF, LaTeX and ANSI
+  sequences
+* it is usable as a command-line tool and as a library
+
+Read more in the :doc:`FAQ list ` or the :doc:`documentation `,
+or `download the latest release `_.
+
+.. _contribute:
+
+Contribute
+----------
+
+Like every open-source project, we are always looking for volunteers to help us
+with programming. Python knowledge is required, but don't fear: Python is a very
+clear and easy to learn language.
+
+Development takes place on `GitHub `_.
+
+If you found a bug, just open a ticket in the GitHub tracker. Be sure to log
+in to be notified when the issue is fixed -- development is not fast-paced as
+the library is quite stable.  You can also send an e-mail to the developers, see
+below.
+
+The authors
+-----------
+
+Pygments is maintained by **Georg Brandl**, e-mail address *georg*\ *@*\ *python.org*, **Matthäus Chajdas** and **Jean Abou-Samra**.
+
+Many lexers and fixes have been contributed by **Armin Ronacher**, the rest of
+the `Pocoo `_ team and **Tim Hatch**.
+
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+
+   docs/index
diff --git a/doc/languages.rst b/doc/languages.rst
new file mode 100644
index 0000000..8136442
--- /dev/null
+++ b/doc/languages.rst
@@ -0,0 +1,18 @@
+:orphan:
+
+Languages
+=========
+
+.. pygmentsdoc:: lexers_overview
+
+... that's all?
+---------------
+
+Well, why not write your own? Contributing to Pygments is easy and fun. Take a
+look at the :doc:`docs on lexer development `. Pull
+requests are welcome on `GitHub `_.
+
+.. note:: 
+    
+    The languages listed here are supported in the development version. The
+    latest release may lack a few of them.
diff --git a/doc/make.bat b/doc/make.bat
new file mode 100644
index 0000000..8803c98
--- /dev/null
+++ b/doc/make.bat
@@ -0,0 +1,190 @@
+@ECHO OFF
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set BUILDDIR=_build
+set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
+set I18NSPHINXOPTS=%SPHINXOPTS% .
+if NOT "%PAPER%" == "" (
+	set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
+	set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS%
+)
+
+if "%1" == "" goto help
+
+if "%1" == "help" (
+	:help
+	echo.Please use `make ^` where ^ is one of
+	echo.  html       to make standalone HTML files
+	echo.  dirhtml    to make HTML files named index.html in directories
+	echo.  singlehtml to make a single large HTML file
+	echo.  pickle     to make pickle files
+	echo.  json       to make JSON files
+	echo.  htmlhelp   to make HTML files and a HTML help project
+	echo.  qthelp     to make HTML files and a qthelp project
+	echo.  devhelp    to make HTML files and a Devhelp project
+	echo.  epub       to make an epub
+	echo.  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter
+	echo.  text       to make text files
+	echo.  man        to make manual pages
+	echo.  texinfo    to make Texinfo files
+	echo.  gettext    to make PO message catalogs
+	echo.  changes    to make an overview over all changed/added/deprecated items
+	echo.  linkcheck  to check all external links for integrity
+	echo.  doctest    to run all doctests embedded in the documentation if enabled
+	goto end
+)
+
+if "%1" == "clean" (
+	for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
+	del /q /s %BUILDDIR%\*
+	goto end
+)
+
+if "%1" == "html" (
+	%SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The HTML pages are in %BUILDDIR%/html.
+	goto end
+)
+
+if "%1" == "dirhtml" (
+	%SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
+	goto end
+)
+
+if "%1" == "singlehtml" (
+	%SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
+	goto end
+)
+
+if "%1" == "pickle" (
+	%SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; now you can process the pickle files.
+	goto end
+)
+
+if "%1" == "json" (
+	%SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; now you can process the JSON files.
+	goto end
+)
+
+if "%1" == "htmlhelp" (
+	%SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; now you can run HTML Help Workshop with the ^
+.hhp project file in %BUILDDIR%/htmlhelp.
+	goto end
+)
+
+if "%1" == "qthelp" (
+	%SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; now you can run "qcollectiongenerator" with the ^
+.qhcp project file in %BUILDDIR%/qthelp, like this:
+	echo.^> qcollectiongenerator %BUILDDIR%\qthelp\Pygments.qhcp
+	echo.To view the help file:
+	echo.^> assistant -collectionFile %BUILDDIR%\qthelp\Pygments.ghc
+	goto end
+)
+
+if "%1" == "devhelp" (
+	%SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished.
+	goto end
+)
+
+if "%1" == "epub" (
+	%SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The epub file is in %BUILDDIR%/epub.
+	goto end
+)
+
+if "%1" == "latex" (
+	%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
+	goto end
+)
+
+if "%1" == "text" (
+	%SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The text files are in %BUILDDIR%/text.
+	goto end
+)
+
+if "%1" == "man" (
+	%SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The manual pages are in %BUILDDIR%/man.
+	goto end
+)
+
+if "%1" == "texinfo" (
+	%SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo.
+	goto end
+)
+
+if "%1" == "gettext" (
+	%SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The message catalogs are in %BUILDDIR%/locale.
+	goto end
+)
+
+if "%1" == "changes" (
+	%SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.The overview file is in %BUILDDIR%/changes.
+	goto end
+)
+
+if "%1" == "linkcheck" (
+	%SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Link check complete; look for any errors in the above output ^
+or in %BUILDDIR%/linkcheck/output.txt.
+	goto end
+)
+
+if "%1" == "doctest" (
+	%SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Testing of doctests in the sources finished, look at the ^
+results in %BUILDDIR%/doctest/output.txt.
+	goto end
+)
+
+:end
diff --git a/doc/pygmentize.1 b/doc/pygmentize.1
new file mode 100644
index 0000000..5ac8fe6
--- /dev/null
+++ b/doc/pygmentize.1
@@ -0,0 +1,112 @@
+.TH PYGMENTIZE 1 "January 20, 2021"
+
+.SH NAME
+pygmentize \- highlights the input file
+
+.SH SYNOPSIS
+.B \fBpygmentize\fP
+.RI  [-l\ \fI\fP\ |\ -g]\ [-F\ \fI\fP[:\fI\fP]]\ [-f\ \fI\fP]
+.RI  [-O\ \fI\fP]\ [-P\ \fI\fP]\ [-o\ \fI\fP]\ [\fI\fP]
+.br
+.B \fBpygmentize\fP
+.RI -S\ \fI
+
+
+
%(title)s

+
+'''
+
+DOC_HEADER_EXTERNALCSS = '''\
+
+
+
+
+  %(title)s
+  
+  
+
+
+
%(title)s

+
+'''
+
+DOC_FOOTER = '''\
+
+
+'''
+
+
+class HtmlFormatter(Formatter):
+    r"""
+    Format tokens as HTML 4 ```` tags within a ``
`` tag, wrapped
+    in a ``
`` tag. The ``
``'s CSS class can be set by the `cssclass`
+    option.
+
+    If the `linenos` option is set to ``"table"``, the ``
`` is
+    additionally wrapped inside a ``
`` which has one row and two + cells: one containing the line numbers and one containing the code. + Example: + + .. sourcecode:: html + +
+
+ + +
+ 
1
+            2
+ 		+ 
def foo(bar):
+              pass
+
+
+ + (whitespace added to improve clarity). + + Wrapping can be disabled using the `nowrap` option. + + A list of lines can be specified using the `hl_lines` option to make these + lines highlighted (as of Pygments 0.11). + + With the `full` option, a complete HTML 4 document is output, including + the style definitions inside a ``$)', _handle_cssblock), + + include('keywords'), + include('inline'), + ], + 'keywords': [ + (words(( + '\\define', '\\end', 'caption', 'created', 'modified', 'tags', + 'title', 'type'), prefix=r'^', suffix=r'\b'), + Keyword), + ], + 'inline': [ + # escape + (r'\\.', Text), + # created or modified date + (r'\d{17}', Number.Integer), + # italics + (r'(\s)(//[^/]+//)((?=\W|\n))', + bygroups(Text, Generic.Emph, Text)), + # superscript + (r'(\s)(\^\^[^\^]+\^\^)', bygroups(Text, Generic.Emph)), + # subscript + (r'(\s)(,,[^,]+,,)', bygroups(Text, Generic.Emph)), + # underscore + (r'(\s)(__[^_]+__)', bygroups(Text, Generic.Strong)), + # bold + (r"(\s)(''[^']+'')((?=\W|\n))", + bygroups(Text, Generic.Strong, Text)), + # strikethrough + (r'(\s)(~~[^~]+~~)((?=\W|\n))', + bygroups(Text, Generic.Deleted, Text)), + # TiddlyWiki variables + (r'<<[^>]+>>', Name.Tag), + (r'\$\$[^$]+\$\$', Name.Tag), + (r'\$\([^)]+\)\$', Name.Tag), + # TiddlyWiki style or class + (r'^@@.*$', Name.Tag), + # HTML tags + (r']+>', Name.Tag), + # inline code + (r'`[^`]+`', String.Backtick), + # HTML escaped symbols + (r'&\S*?;', String.Regex), + # Wiki links + (r'(\[{2})([^]\|]+)(\]{2})', bygroups(Text, Name.Tag, Text)), + # External links + (r'(\[{2})([^]\|]+)(\|)([^]\|]+)(\]{2})', + bygroups(Text, Name.Tag, Text, Name.Attribute, Text)), + # Transclusion + (r'(\{{2})([^}]+)(\}{2})', bygroups(Text, Name.Tag, Text)), + # URLs + (r'(\b.?.?tps?://[^\s"]+)', bygroups(Name.Attribute)), + + # general text, must come last! + (r'[\w]+', Text), + (r'.', Text) + ], + } + + def __init__(self, **options): + self.handlecodeblocks = get_bool_opt(options, 'handlecodeblocks', True) + RegexLexer.__init__(self, **options) diff --git a/pygments/lexers/math.py b/pygments/lexers/math.py new file mode 100644 index 0000000..22144fa --- /dev/null +++ b/pygments/lexers/math.py @@ -0,0 +1,20 @@ +""" + pygments.lexers.math + ~~~~~~~~~~~~~~~~~~~~ + + Just export lexers that were contained in this module. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexers.python import NumPyLexer +from pygments.lexers.matlab import MatlabLexer, MatlabSessionLexer, \ + OctaveLexer, ScilabLexer +from pygments.lexers.julia import JuliaLexer, JuliaConsoleLexer +from pygments.lexers.r import RConsoleLexer, SLexer, RdLexer +from pygments.lexers.modeling import BugsLexer, JagsLexer, StanLexer +from pygments.lexers.idl import IDLLexer +from pygments.lexers.algebra import MuPADLexer + +__all__ = [] diff --git a/pygments/lexers/matlab.py b/pygments/lexers/matlab.py new file mode 100644 index 0000000..058661e --- /dev/null +++ b/pygments/lexers/matlab.py @@ -0,0 +1,3308 @@ +""" + pygments.lexers.matlab + ~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for Matlab and related languages. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import Lexer, RegexLexer, bygroups, default, words, \ + do_insertions, include +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Generic, Whitespace + +from pygments.lexers import _scilab_builtins + +__all__ = ['MatlabLexer', 'MatlabSessionLexer', 'OctaveLexer', 'ScilabLexer'] + + +class MatlabLexer(RegexLexer): + """ + For Matlab source code. + + .. versionadded:: 0.10 + """ + name = 'Matlab' + aliases = ['matlab'] + filenames = ['*.m'] + mimetypes = ['text/matlab'] + + _operators = r'-|==|~=|<=|>=|<|>|&&|&|~|\|\|?|\.\*|\*|\+|\.\^|\.\\|\./|/|\\' + + tokens = { + 'expressions': [ + # operators: + (_operators, Operator), + + # numbers (must come before punctuation to handle `.5`; cannot use + # `\b` due to e.g. `5. + .5`). The negative lookahead on operators + # avoids including the dot in `1./x` (the dot is part of `./`). + (r'(? and then + # (equal | open-parenthesis | | ). + (r'(?:^|(?<=;))(\s*)(\w+)(\s+)(?!=|\(|%s\s|\s)' % _operators, + bygroups(Whitespace, Name, Whitespace), 'commandargs'), + + include('expressions') + ], + 'blockcomment': [ + (r'^\s*%\}', Comment.Multiline, '#pop'), + (r'^.*\n', Comment.Multiline), + (r'.', Comment.Multiline), + ], + 'deffunc': [ + (r'(\s*)(?:(\S+)(\s*)(=)(\s*))?(.+)(\()(.*)(\))(\s*)', + bygroups(Whitespace, Text, Whitespace, Punctuation, + Whitespace, Name.Function, Punctuation, Text, + Punctuation, Whitespace), '#pop'), + # function with no args + (r'(\s*)([a-zA-Z_]\w*)', + bygroups(Whitespace, Name.Function), '#pop'), + ], + 'propattrs': [ + (r'(\w+)(\s*)(=)(\s*)(\d+)', + bygroups(Name.Builtin, Whitespace, Punctuation, Whitespace, + Number)), + (r'(\w+)(\s*)(=)(\s*)([a-zA-Z]\w*)', + bygroups(Name.Builtin, Whitespace, Punctuation, Whitespace, + Keyword)), + (r',', Punctuation), + (r'\)', Punctuation, '#pop'), + (r'\s+', Whitespace), + (r'.', Text), + ], + 'defprops': [ + (r'%\{\s*\n', Comment.Multiline, 'blockcomment'), + (r'%.*$', Comment), + (r'(?. + + .. versionadded:: 0.10 + """ + name = 'Matlab session' + aliases = ['matlabsession'] + + def get_tokens_unprocessed(self, text): + mlexer = MatlabLexer(**self.options) + + curcode = '' + insertions = [] + continuation = False + + for match in line_re.finditer(text): + line = match.group() + + if line.startswith('>> '): + insertions.append((len(curcode), + [(0, Generic.Prompt, line[:3])])) + curcode += line[3:] + + elif line.startswith('>>'): + insertions.append((len(curcode), + [(0, Generic.Prompt, line[:2])])) + curcode += line[2:] + + elif line.startswith('???'): + + idx = len(curcode) + + # without is showing error on same line as before...? + # line = "\n" + line + token = (0, Generic.Traceback, line) + insertions.append((idx, [token])) + elif continuation and insertions: + # line_start is the length of the most recent prompt symbol + line_start = len(insertions[-1][-1][-1]) + # Set leading spaces with the length of the prompt to be a generic prompt + # This keeps code aligned when prompts are removed, say with some Javascript + if line.startswith(' '*line_start): + insertions.append( + (len(curcode), [(0, Generic.Prompt, line[:line_start])])) + curcode += line[line_start:] + else: + curcode += line + else: + if curcode: + yield from do_insertions( + insertions, mlexer.get_tokens_unprocessed(curcode)) + curcode = '' + insertions = [] + + yield match.start(), Generic.Output, line + + # Does not allow continuation if a comment is included after the ellipses. + # Continues any line that ends with ..., even comments (lines that start with %) + if line.strip().endswith('...'): + continuation = True + else: + continuation = False + + if curcode: # or item: + yield from do_insertions( + insertions, mlexer.get_tokens_unprocessed(curcode)) + + +class OctaveLexer(RegexLexer): + """ + For GNU Octave source code. + + .. versionadded:: 1.5 + """ + name = 'Octave' + url = 'https://www.gnu.org/software/octave/index' + aliases = ['octave'] + filenames = ['*.m'] + mimetypes = ['text/octave'] + + # These lists are generated automatically. + # Run the following in bash shell: + # + # First dump all of the Octave manual into a plain text file: + # + # $ info octave --subnodes -o octave-manual + # + # Now grep through it: + + # for i in \ + # "Built-in Function" "Command" "Function File" \ + # "Loadable Function" "Mapping Function"; + # do + # perl -e '@name = qw('"$i"'); + # print lc($name[0]),"_kw = [\n"'; + # + # perl -n -e 'print "\"$1\",\n" if /-- '"$i"': .* (\w*) \(/;' \ + # octave-manual | sort | uniq ; + # echo "]" ; + # echo; + # done + + # taken from Octave Mercurial changeset 8cc154f45e37 (30-jan-2011) + + builtin_kw = ( + "addlistener", "addpath", "addproperty", "all", + "and", "any", "argnames", "argv", "assignin", + "atexit", "autoload", + "available_graphics_toolkits", "beep_on_error", + "bitand", "bitmax", "bitor", "bitshift", "bitxor", + "cat", "cell", "cellstr", "char", "class", "clc", + "columns", "command_line_path", + "completion_append_char", "completion_matches", + "complex", "confirm_recursive_rmdir", "cputime", + "crash_dumps_octave_core", "ctranspose", "cumprod", + "cumsum", "debug_on_error", "debug_on_interrupt", + "debug_on_warning", "default_save_options", + "dellistener", "diag", "diff", "disp", + "doc_cache_file", "do_string_escapes", "double", + "drawnow", "e", "echo_executing_commands", "eps", + "eq", "errno", "errno_list", "error", "eval", + "evalin", "exec", "exist", "exit", "eye", "false", + "fclear", "fclose", "fcntl", "fdisp", "feof", + "ferror", "feval", "fflush", "fgetl", "fgets", + "fieldnames", "file_in_loadpath", "file_in_path", + "filemarker", "filesep", "find_dir_in_path", + "fixed_point_format", "fnmatch", "fopen", "fork", + "formula", "fprintf", "fputs", "fread", "freport", + "frewind", "fscanf", "fseek", "fskipl", "ftell", + "functions", "fwrite", "ge", "genpath", "get", + "getegid", "getenv", "geteuid", "getgid", + "getpgrp", "getpid", "getppid", "getuid", "glob", + "gt", "gui_mode", "history_control", + "history_file", "history_size", + "history_timestamp_format_string", "home", + "horzcat", "hypot", "ifelse", + "ignore_function_time_stamp", "inferiorto", + "info_file", "info_program", "inline", "input", + "intmax", "intmin", "ipermute", + "is_absolute_filename", "isargout", "isbool", + "iscell", "iscellstr", "ischar", "iscomplex", + "isempty", "isfield", "isfloat", "isglobal", + "ishandle", "isieee", "isindex", "isinteger", + "islogical", "ismatrix", "ismethod", "isnull", + "isnumeric", "isobject", "isreal", + "is_rooted_relative_filename", "issorted", + "isstruct", "isvarname", "kbhit", "keyboard", + "kill", "lasterr", "lasterror", "lastwarn", + "ldivide", "le", "length", "link", "linspace", + "logical", "lstat", "lt", "make_absolute_filename", + "makeinfo_program", "max_recursion_depth", "merge", + "methods", "mfilename", "minus", "mislocked", + "mkdir", "mkfifo", "mkstemp", "mldivide", "mlock", + "mouse_wheel_zoom", "mpower", "mrdivide", "mtimes", + "munlock", "nargin", "nargout", + "native_float_format", "ndims", "ne", "nfields", + "nnz", "norm", "not", "numel", "nzmax", + "octave_config_info", "octave_core_file_limit", + "octave_core_file_name", + "octave_core_file_options", "ones", "or", + "output_max_field_width", "output_precision", + "page_output_immediately", "page_screen_output", + "path", "pathsep", "pause", "pclose", "permute", + "pi", "pipe", "plus", "popen", "power", + "print_empty_dimensions", "printf", + "print_struct_array_contents", "prod", + "program_invocation_name", "program_name", + "putenv", "puts", "pwd", "quit", "rats", "rdivide", + "readdir", "readlink", "read_readline_init_file", + "realmax", "realmin", "rehash", "rename", + "repelems", "re_read_readline_init_file", "reset", + "reshape", "resize", "restoredefaultpath", + "rethrow", "rmdir", "rmfield", "rmpath", "rows", + "save_header_format_string", "save_precision", + "saving_history", "scanf", "set", "setenv", + "shell_cmd", "sighup_dumps_octave_core", + "sigterm_dumps_octave_core", "silent_functions", + "single", "size", "size_equal", "sizemax", + "sizeof", "sleep", "source", "sparse_auto_mutate", + "split_long_rows", "sprintf", "squeeze", "sscanf", + "stat", "stderr", "stdin", "stdout", "strcmp", + "strcmpi", "string_fill_char", "strncmp", + "strncmpi", "struct", "struct_levels_to_print", + "strvcat", "subsasgn", "subsref", "sum", "sumsq", + "superiorto", "suppress_verbose_help_message", + "symlink", "system", "tic", "tilde_expand", + "times", "tmpfile", "tmpnam", "toc", "toupper", + "transpose", "true", "typeinfo", "umask", "uminus", + "uname", "undo_string_escapes", "unlink", "uplus", + "upper", "usage", "usleep", "vec", "vectorize", + "vertcat", "waitpid", "warning", "warranty", + "whos_line_format", "yes_or_no", "zeros", + "inf", "Inf", "nan", "NaN") + + command_kw = ("close", "load", "who", "whos") + + function_kw = ( + "accumarray", "accumdim", "acosd", "acotd", + "acscd", "addtodate", "allchild", "ancestor", + "anova", "arch_fit", "arch_rnd", "arch_test", + "area", "arma_rnd", "arrayfun", "ascii", "asctime", + "asecd", "asind", "assert", "atand", + "autoreg_matrix", "autumn", "axes", "axis", "bar", + "barh", "bartlett", "bartlett_test", "beep", + "betacdf", "betainv", "betapdf", "betarnd", + "bicgstab", "bicubic", "binary", "binocdf", + "binoinv", "binopdf", "binornd", "bitcmp", + "bitget", "bitset", "blackman", "blanks", + "blkdiag", "bone", "box", "brighten", "calendar", + "cast", "cauchy_cdf", "cauchy_inv", "cauchy_pdf", + "cauchy_rnd", "caxis", "celldisp", "center", "cgs", + "chisquare_test_homogeneity", + "chisquare_test_independence", "circshift", "cla", + "clabel", "clf", "clock", "cloglog", "closereq", + "colon", "colorbar", "colormap", "colperm", + "comet", "common_size", "commutation_matrix", + "compan", "compare_versions", "compass", + "computer", "cond", "condest", "contour", + "contourc", "contourf", "contrast", "conv", + "convhull", "cool", "copper", "copyfile", "cor", + "corrcoef", "cor_test", "cosd", "cotd", "cov", + "cplxpair", "cross", "cscd", "cstrcat", "csvread", + "csvwrite", "ctime", "cumtrapz", "curl", "cut", + "cylinder", "date", "datenum", "datestr", + "datetick", "datevec", "dblquad", "deal", + "deblank", "deconv", "delaunay", "delaunayn", + "delete", "demo", "detrend", "diffpara", "diffuse", + "dir", "discrete_cdf", "discrete_inv", + "discrete_pdf", "discrete_rnd", "display", + "divergence", "dlmwrite", "dos", "dsearch", + "dsearchn", "duplication_matrix", "durbinlevinson", + "ellipsoid", "empirical_cdf", "empirical_inv", + "empirical_pdf", "empirical_rnd", "eomday", + "errorbar", "etime", "etreeplot", "example", + "expcdf", "expinv", "expm", "exppdf", "exprnd", + "ezcontour", "ezcontourf", "ezmesh", "ezmeshc", + "ezplot", "ezpolar", "ezsurf", "ezsurfc", "factor", + "factorial", "fail", "fcdf", "feather", "fftconv", + "fftfilt", "fftshift", "figure", "fileattrib", + "fileparts", "fill", "findall", "findobj", + "findstr", "finv", "flag", "flipdim", "fliplr", + "flipud", "fpdf", "fplot", "fractdiff", "freqz", + "freqz_plot", "frnd", "fsolve", + "f_test_regression", "ftp", "fullfile", "fzero", + "gamcdf", "gaminv", "gampdf", "gamrnd", "gca", + "gcbf", "gcbo", "gcf", "genvarname", "geocdf", + "geoinv", "geopdf", "geornd", "getfield", "ginput", + "glpk", "gls", "gplot", "gradient", + "graphics_toolkit", "gray", "grid", "griddata", + "griddatan", "gtext", "gunzip", "gzip", "hadamard", + "hamming", "hankel", "hanning", "hggroup", + "hidden", "hilb", "hist", "histc", "hold", "hot", + "hotelling_test", "housh", "hsv", "hurst", + "hygecdf", "hygeinv", "hygepdf", "hygernd", + "idivide", "ifftshift", "image", "imagesc", + "imfinfo", "imread", "imshow", "imwrite", "index", + "info", "inpolygon", "inputname", "interpft", + "interpn", "intersect", "invhilb", "iqr", "isa", + "isdefinite", "isdir", "is_duplicate_entry", + "isequal", "isequalwithequalnans", "isfigure", + "ishermitian", "ishghandle", "is_leap_year", + "isletter", "ismac", "ismember", "ispc", "isprime", + "isprop", "isscalar", "issquare", "isstrprop", + "issymmetric", "isunix", "is_valid_file_id", + "isvector", "jet", "kendall", + "kolmogorov_smirnov_cdf", + "kolmogorov_smirnov_test", "kruskal_wallis_test", + "krylov", "kurtosis", "laplace_cdf", "laplace_inv", + "laplace_pdf", "laplace_rnd", "legend", "legendre", + "license", "line", "linkprop", "list_primes", + "loadaudio", "loadobj", "logistic_cdf", + "logistic_inv", "logistic_pdf", "logistic_rnd", + "logit", "loglog", "loglogerr", "logm", "logncdf", + "logninv", "lognpdf", "lognrnd", "logspace", + "lookfor", "ls_command", "lsqnonneg", "magic", + "mahalanobis", "manova", "matlabroot", + "mcnemar_test", "mean", "meansq", "median", "menu", + "mesh", "meshc", "meshgrid", "meshz", "mexext", + "mget", "mkpp", "mode", "moment", "movefile", + "mpoles", "mput", "namelengthmax", "nargchk", + "nargoutchk", "nbincdf", "nbininv", "nbinpdf", + "nbinrnd", "nchoosek", "ndgrid", "newplot", "news", + "nonzeros", "normcdf", "normest", "norminv", + "normpdf", "normrnd", "now", "nthroot", "null", + "ocean", "ols", "onenormest", "optimget", + "optimset", "orderfields", "orient", "orth", + "pack", "pareto", "parseparams", "pascal", "patch", + "pathdef", "pcg", "pchip", "pcolor", "pcr", + "peaks", "periodogram", "perl", "perms", "pie", + "pink", "planerot", "playaudio", "plot", + "plotmatrix", "plotyy", "poisscdf", "poissinv", + "poisspdf", "poissrnd", "polar", "poly", + "polyaffine", "polyarea", "polyderiv", "polyfit", + "polygcd", "polyint", "polyout", "polyreduce", + "polyval", "polyvalm", "postpad", "powerset", + "ppder", "ppint", "ppjumps", "ppplot", "ppval", + "pqpnonneg", "prepad", "primes", "print", + "print_usage", "prism", "probit", "qp", "qqplot", + "quadcc", "quadgk", "quadl", "quadv", "quiver", + "qzhess", "rainbow", "randi", "range", "rank", + "ranks", "rat", "reallog", "realpow", "realsqrt", + "record", "rectangle_lw", "rectangle_sw", + "rectint", "refresh", "refreshdata", + "regexptranslate", "repmat", "residue", "ribbon", + "rindex", "roots", "rose", "rosser", "rotdim", + "rref", "run", "run_count", "rundemos", "run_test", + "runtests", "saveas", "saveaudio", "saveobj", + "savepath", "scatter", "secd", "semilogx", + "semilogxerr", "semilogy", "semilogyerr", + "setaudio", "setdiff", "setfield", "setxor", + "shading", "shift", "shiftdim", "sign_test", + "sinc", "sind", "sinetone", "sinewave", "skewness", + "slice", "sombrero", "sortrows", "spaugment", + "spconvert", "spdiags", "spearman", "spectral_adf", + "spectral_xdf", "specular", "speed", "spencer", + "speye", "spfun", "sphere", "spinmap", "spline", + "spones", "sprand", "sprandn", "sprandsym", + "spring", "spstats", "spy", "sqp", "stairs", + "statistics", "std", "stdnormal_cdf", + "stdnormal_inv", "stdnormal_pdf", "stdnormal_rnd", + "stem", "stft", "strcat", "strchr", "strjust", + "strmatch", "strread", "strsplit", "strtok", + "strtrim", "strtrunc", "structfun", "studentize", + "subplot", "subsindex", "subspace", "substr", + "substruct", "summer", "surf", "surface", "surfc", + "surfl", "surfnorm", "svds", "swapbytes", + "sylvester_matrix", "symvar", "synthesis", "table", + "tand", "tar", "tcdf", "tempdir", "tempname", + "test", "text", "textread", "textscan", "tinv", + "title", "toeplitz", "tpdf", "trace", "trapz", + "treelayout", "treeplot", "triangle_lw", + "triangle_sw", "tril", "trimesh", "triplequad", + "triplot", "trisurf", "triu", "trnd", "tsearchn", + "t_test", "t_test_regression", "type", "unidcdf", + "unidinv", "unidpdf", "unidrnd", "unifcdf", + "unifinv", "unifpdf", "unifrnd", "union", "unique", + "unix", "unmkpp", "unpack", "untabify", "untar", + "unwrap", "unzip", "u_test", "validatestring", + "vander", "var", "var_test", "vech", "ver", + "version", "view", "voronoi", "voronoin", + "waitforbuttonpress", "wavread", "wavwrite", + "wblcdf", "wblinv", "wblpdf", "wblrnd", "weekday", + "welch_test", "what", "white", "whitebg", + "wienrnd", "wilcoxon_test", "wilkinson", "winter", + "xlabel", "xlim", "ylabel", "yulewalker", "zip", + "zlabel", "z_test") + + loadable_kw = ( + "airy", "amd", "balance", "besselh", "besseli", + "besselj", "besselk", "bessely", "bitpack", + "bsxfun", "builtin", "ccolamd", "cellfun", + "cellslices", "chol", "choldelete", "cholinsert", + "cholinv", "cholshift", "cholupdate", "colamd", + "colloc", "convhulln", "convn", "csymamd", + "cummax", "cummin", "daspk", "daspk_options", + "dasrt", "dasrt_options", "dassl", "dassl_options", + "dbclear", "dbdown", "dbstack", "dbstatus", + "dbstop", "dbtype", "dbup", "dbwhere", "det", + "dlmread", "dmperm", "dot", "eig", "eigs", + "endgrent", "endpwent", "etree", "fft", "fftn", + "fftw", "filter", "find", "full", "gcd", + "getgrent", "getgrgid", "getgrnam", "getpwent", + "getpwnam", "getpwuid", "getrusage", "givens", + "gmtime", "gnuplot_binary", "hess", "ifft", + "ifftn", "inv", "isdebugmode", "issparse", "kron", + "localtime", "lookup", "lsode", "lsode_options", + "lu", "luinc", "luupdate", "matrix_type", "max", + "min", "mktime", "pinv", "qr", "qrdelete", + "qrinsert", "qrshift", "qrupdate", "quad", + "quad_options", "qz", "rand", "rande", "randg", + "randn", "randp", "randperm", "rcond", "regexp", + "regexpi", "regexprep", "schur", "setgrent", + "setpwent", "sort", "spalloc", "sparse", "spparms", + "sprank", "sqrtm", "strfind", "strftime", + "strptime", "strrep", "svd", "svd_driver", "syl", + "symamd", "symbfact", "symrcm", "time", "tsearch", + "typecast", "urlread", "urlwrite") + + mapping_kw = ( + "abs", "acos", "acosh", "acot", "acoth", "acsc", + "acsch", "angle", "arg", "asec", "asech", "asin", + "asinh", "atan", "atanh", "beta", "betainc", + "betaln", "bincoeff", "cbrt", "ceil", "conj", "cos", + "cosh", "cot", "coth", "csc", "csch", "erf", "erfc", + "erfcx", "erfinv", "exp", "finite", "fix", "floor", + "fmod", "gamma", "gammainc", "gammaln", "imag", + "isalnum", "isalpha", "isascii", "iscntrl", + "isdigit", "isfinite", "isgraph", "isinf", + "islower", "isna", "isnan", "isprint", "ispunct", + "isspace", "isupper", "isxdigit", "lcm", "lgamma", + "log", "lower", "mod", "real", "rem", "round", + "roundb", "sec", "sech", "sign", "sin", "sinh", + "sqrt", "tan", "tanh", "toascii", "tolower", "xor") + + builtin_consts = ( + "EDITOR", "EXEC_PATH", "I", "IMAGE_PATH", "NA", + "OCTAVE_HOME", "OCTAVE_VERSION", "PAGER", + "PAGER_FLAGS", "SEEK_CUR", "SEEK_END", "SEEK_SET", + "SIG", "S_ISBLK", "S_ISCHR", "S_ISDIR", "S_ISFIFO", + "S_ISLNK", "S_ISREG", "S_ISSOCK", "WCONTINUE", + "WCOREDUMP", "WEXITSTATUS", "WIFCONTINUED", + "WIFEXITED", "WIFSIGNALED", "WIFSTOPPED", "WNOHANG", + "WSTOPSIG", "WTERMSIG", "WUNTRACED") + + tokens = { + 'root': [ + (r'%\{\s*\n', Comment.Multiline, 'percentblockcomment'), + (r'#\{\s*\n', Comment.Multiline, 'hashblockcomment'), + (r'[%#].*$', Comment), + (r'^\s*function\b', Keyword, 'deffunc'), + + # from 'iskeyword' on hg changeset 8cc154f45e37 + (words(( + '__FILE__', '__LINE__', 'break', 'case', 'catch', 'classdef', + 'continue', 'do', 'else', 'elseif', 'end', 'end_try_catch', + 'end_unwind_protect', 'endclassdef', 'endevents', 'endfor', + 'endfunction', 'endif', 'endmethods', 'endproperties', 'endswitch', + 'endwhile', 'events', 'for', 'function', 'get', 'global', 'if', + 'methods', 'otherwise', 'persistent', 'properties', 'return', + 'set', 'static', 'switch', 'try', 'until', 'unwind_protect', + 'unwind_protect_cleanup', 'while'), suffix=r'\b'), + Keyword), + + (words(builtin_kw + command_kw + function_kw + loadable_kw + mapping_kw, + suffix=r'\b'), Name.Builtin), + + (words(builtin_consts, suffix=r'\b'), Name.Constant), + + # operators in Octave but not Matlab: + (r'-=|!=|!|/=|--', Operator), + # operators: + (r'-|==|~=|<|>|<=|>=|&&|&|~|\|\|?', Operator), + # operators in Octave but not Matlab requiring escape for re: + (r'\*=|\+=|\^=|\/=|\\=|\*\*|\+\+|\.\*\*', Operator), + # operators requiring escape for re: + (r'\.\*|\*|\+|\.\^|\.\\|\.\/|\/|\\', Operator), + + + # punctuation: + (r'[\[\](){}:@.,]', Punctuation), + (r'=|:|;', Punctuation), + + (r'"[^"]*"', String), + + (r'(\d+\.\d*|\d*\.\d+)([eEf][+-]?[0-9]+)?', Number.Float), + (r'\d+[eEf][+-]?[0-9]+', Number.Float), + (r'\d+', Number.Integer), + + # quote can be transpose, instead of string: + # (not great, but handles common cases...) + (r'(?<=[\w)\].])\'+', Operator), + (r'(?|<=|>=|&&|&|~|\|\|?', Operator), + # operators requiring escape for re: + (r'\.\*|\*|\+|\.\^|\.\\|\.\/|\/|\\', Operator), + + # punctuation: + (r'[\[\](){}@.,=:;]+', Punctuation), + + (r'"[^"]*"', String), + + # quote can be transpose, instead of string: + # (not great, but handles common cases...) + (r'(?<=[\w)\].])\'+', Operator), + (r'(?', r'<', r'|', r'!', r"'") + + operator_words = ('and', 'or', 'not') + + tokens = { + 'root': [ + (r'/\*', Comment.Multiline, 'comment'), + (r'"(?:[^"\\]|\\.)*"', String), + (r'\(|\)|\[|\]|\{|\}', Punctuation), + (r'[,;$]', Punctuation), + (words (constants), Name.Constant), + (words (keywords), Keyword), + (words (operators), Operator), + (words (operator_words), Operator.Word), + (r'''(?x) + ((?:[a-zA-Z_#][\w#]*|`[^`]*`) + (?:::[a-zA-Z_#][\w#]*|`[^`]*`)*)(\s*)([(])''', + bygroups(Name.Function, Text.Whitespace, Punctuation)), + (r'''(?x) + (?:[a-zA-Z_#%][\w#%]*|`[^`]*`) + (?:::[a-zA-Z_#%][\w#%]*|`[^`]*`)*''', Name.Variable), + (r'[-+]?(\d*\.\d+([bdefls][-+]?\d+)?|\d+(\.\d*)?[bdefls][-+]?\d+)', Number.Float), + (r'[-+]?\d+', Number.Integer), + (r'\s+', Text.Whitespace), + (r'.', Text) + ], + 'comment': [ + (r'[^*/]+', Comment.Multiline), + (r'/\*', Comment.Multiline, '#push'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[*/]', Comment.Multiline) + ] + } + + def analyse_text (text): + strength = 0.0 + # Input expression terminator. + if re.search (r'\$\s*$', text, re.MULTILINE): + strength += 0.05 + # Function definition operator. + if ':=' in text: + strength += 0.02 + return strength diff --git a/pygments/lexers/meson.py b/pygments/lexers/meson.py new file mode 100644 index 0000000..e8d1f08 --- /dev/null +++ b/pygments/lexers/meson.py @@ -0,0 +1,140 @@ +""" + pygments.lexers.meson + ~~~~~~~~~~~~~~~~~~~~~ + + Pygments lexer for the Meson build system + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, words, include +from pygments.token import Comment, Name, Number, Punctuation, Operator, \ + Keyword, String, Whitespace + +__all__ = ['MesonLexer'] + + +class MesonLexer(RegexLexer): + """Meson language lexer. + + The grammar definition use to transcribe the syntax was retrieved from + https://mesonbuild.com/Syntax.html#grammar for version 0.58. + Some of those definitions are improperly transcribed, so the Meson++ + implementation was also checked: https://github.com/dcbaker/meson-plus-plus. + + .. versionadded:: 2.10 + """ + + # TODO String interpolation @VARNAME@ inner matches + # TODO keyword_arg: value inner matches + + name = 'Meson' + url = 'https://mesonbuild.com/' + aliases = ['meson', 'meson.build'] + filenames = ['meson.build', 'meson_options.txt'] + mimetypes = ['text/x-meson'] + + tokens = { + 'root': [ + (r'#.*?$', Comment), + (r"'''.*'''", String.Single), + (r'[1-9][0-9]*', Number.Integer), + (r'0o[0-7]+', Number.Oct), + (r'0x[a-fA-F0-9]+', Number.Hex), + include('string'), + include('keywords'), + include('expr'), + (r'[a-zA-Z_][a-zA-Z_0-9]*', Name), + (r'\s+', Whitespace), + ], + 'string': [ + (r"[']{3}([']{0,2}([^\\']|\\(.|\n)))*[']{3}", String), + (r"'.*?(?`_. + + .. versionadded:: 2.12.0 + """ + + name = "MCFunction" + url = "https://minecraft.fandom.com/wiki/Commands" + aliases = ["mcfunction", "mcf"] + filenames = ["*.mcfunction"] + mimetypes = ["text/mcfunction"] + + # Used to denotate the start of a block comment, borrowed from Github's mcfunction + _block_comment_prefix = "[>!]" + + tokens = { + "root": [ + include("names"), + include("comments"), + include("literals"), + include("whitespace"), + include("property"), + include("operators"), + include("selectors"), + ], + + "names": [ + # The start of a command (either beginning of line OR after the run keyword) + # We don't encode a list of keywords since mods, plugins, or even pre-processors + # may add new commands, so we have a 'close-enough' regex which catches them. + (r"^(\s*)([a-z_]+)", bygroups(Whitespace, Name.Builtin)), + (r"(?<=run)\s+[a-z_]+", Name.Builtin), + + # UUID + (r"\b[0-9a-fA-F]+(?:-[0-9a-fA-F]+){4}\b", Name.Variable), + include("resource-name"), + # normal command names and scoreboards + # there's no way to know the differences unfortuntely + (r"[A-Za-z_][\w.#%$]+", Keyword.Constant), + (r"[#%$][\w.#%$]+", Name.Variable.Magic), + ], + + "resource-name": [ + # resource names have to be lowercase + (r"#?[a-z_][a-z_.-]*:[a-z0-9_./-]+", Name.Function), + # similar to above except optional `:`` + # a `/` must be present "somewhere" + (r"#?[a-z0-9_\.\-]+\/[a-z0-9_\.\-\/]+", Name.Function), + ], + + "whitespace": [ + (r"\s+", Whitespace), + ], + + "comments": [ + (rf"^\s*(#{_block_comment_prefix})", Comment.Multiline, + ("comments.block", "comments.block.emphasized")), + (r"#.*$", Comment.Single), + ], + "comments.block": [ + (rf"^\s*#{_block_comment_prefix}", Comment.Multiline, + "comments.block.emphasized"), + (r"^\s*#", Comment.Multiline, "comments.block.normal"), + default("#pop"), + ], + "comments.block.normal": [ + include("comments.block.special"), + (r"\S+", Comment.Multiline), + (r"\n", Text, "#pop"), + include("whitespace"), + ], + "comments.block.emphasized": [ + include("comments.block.special"), + (r"\S+", String.Doc), + (r"\n", Text, "#pop"), + include("whitespace"), + ], + "comments.block.special": [ + # Params + (r"@\S+", Name.Decorator), + + include("resource-name"), + + # Scoreboard player names + (r"[#%$][\w.#%$]+", Name.Variable.Magic), + ], + + "operators": [ + (r"[\-~%^?!+*<>\\/|&=.]", Operator), + ], + + "literals": [ + (r"\.\.", Literal), + (r"(true|false)", Keyword.Pseudo), + + # these are like unquoted strings and appear in many places + (r"[A-Za-z_]+", Name.Variable.Class), + + (r"[0-7]b", Number.Byte), + (r"[+-]?\d*\.?\d+([eE]?[+-]?\d+)?[df]?\b", Number.Float), + (r"[+-]?\d+\b", Number.Integer), + (r'"', String.Double, "literals.string-double"), + (r"'", String.Single, "literals.string-single"), + ], + "literals.string-double": [ + (r"\\.", String.Escape), + (r'[^\\"\n]+', String.Double), + (r'"', String.Double, "#pop"), + ], + "literals.string-single": [ + (r"\\.", String.Escape), + (r"[^\\'\n]+", String.Single), + (r"'", String.Single, "#pop"), + ], + + "selectors": [ + (r"@[a-z]", Name.Variable), + ], + + + ## Generic Property Container + # There are several, differing instances where the language accepts + # specific contained keys or contained key, value pairings. + # + # Property Maps: + # - Starts with either `[` or `{` + # - Key separated by `:` or `=` + # - Deliminated by `,` + # + # Property Lists: + # - Starts with `[` + # - Deliminated by `,` + # + # For simplicity, these patterns match a generic, nestable structure + # which follow a key, value pattern. For normal lists, there's only keys. + # This allow some "illegal" structures, but we'll accept those for + # sake of simplicity + # + # Examples: + # - `[facing=up, powered=true]` (blockstate) + # - `[name="hello world", nbt={key: 1b}]` (selector + nbt) + # - `[{"text": "value"}, "literal"]` (json) + ## + "property": [ + # This state gets included in root and also several substates + # We do this to shortcut the starting of new properties + # within other properties. Lists can have sublists and compounds + # and values can start a new property (see the `difficult_1.txt` + # snippet). + (r"\{", Punctuation, ("property.curly", "property.key")), + (r"\[", Punctuation, ("property.square", "property.key")), + ], + "property.curly": [ + include("whitespace"), + include("property"), + (r"\}", Punctuation, "#pop"), + ], + "property.square": [ + include("whitespace"), + include("property"), + (r"\]", Punctuation, "#pop"), + + # lists can have sequences of items + (r",", Punctuation), + ], + "property.key": [ + include("whitespace"), + + # resource names (for advancements) + # can omit `:` to default `minecraft:` + # must check if there is a future equals sign if `:` is in the name + (r"#?[a-z_][a-z_\.\-]*\:[a-z0-9_\.\-/]+(?=\s*\=)", Name.Attribute, "property.delimiter"), + (r"#?[a-z_][a-z0-9_\.\-/]+", Name.Attribute, "property.delimiter"), + + # unquoted NBT key + (r"[A-Za-z_\-\+]+", Name.Attribute, "property.delimiter"), + + # quoted JSON or NBT key + (r'"', Name.Attribute, "property.delimiter", "literals.string-double"), + (r"'", Name.Attribute, "property.delimiter", "literals.string-single"), + + # index for a list + (r"-?\d+", Number.Integer, "property.delimiter"), + + default("#pop"), + ], + "property.key.string-double": [ + (r"\\.", String.Escape), + (r'[^\\"\n]+', Name.Attribute), + (r'"', Name.Attribute, "#pop"), + ], + "property.key.string-single": [ + (r"\\.", String.Escape), + (r"[^\\'\n]+", Name.Attribute), + (r"'", Name.Attribute, "#pop"), + ], + "property.delimiter": [ + include("whitespace"), + + (r"[:=]!?", Punctuation, "property.value"), + (r",", Punctuation), + + default("#pop"), + ], + "property.value": [ + include("whitespace"), + + # unquoted resource names are valid literals here + (r"#?[a-z_][a-z_\.\-]*\:[a-z0-9_\.\-/]+", Name.Tag), + (r"#?[a-z_][a-z0-9_\.\-/]+", Name.Tag), + + include("literals"), + include("property"), + + default("#pop"), + ], + } + + +class MCSchemaLexer(RegexLexer): + """Lexer for Minecraft Add-ons data Schemas, an interface structure standard used in Minecraft + + .. versionadded:: 2.14.0 + """ + + name = 'MCSchema' + url = 'https://learn.microsoft.com/en-us/minecraft/creator/reference/content/schemasreference/' + aliases = ['mcschema'] + filenames = ['*.mcschema'] + mimetypes = ['text/mcschema'] + + tokens = { + 'commentsandwhitespace': [ + (r'\s+', Whitespace), + (r'//.*?$', Comment.Single), + (r'/\*.*?\*/', Comment.Multiline) + ], + 'slashstartsregex': [ + include('commentsandwhitespace'), + (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/' + r'([gimuysd]+\b|\B)', String.Regex, '#pop'), + (r'(?=/)', Text, ('#pop', 'badregex')), + default('#pop') + ], + 'badregex': [ + (r'\n', Whitespace, '#pop') + ], + 'singlestring': [ + (r'\\.', String.Escape), + (r"'", String.Single, '#pop'), + (r"[^\\']+", String.Single), + ], + 'doublestring': [ + (r'\\.', String.Escape), + (r'"', String.Double, '#pop'), + (r'[^\\"]+', String.Double), + ], + 'root': [ + (r'^(?=\s|/|', Comment, '#pop'), + (r'[^\-]+|-', Comment), + ], + } + + +class ReasonLexer(RegexLexer): + """ + For the ReasonML language. + + .. versionadded:: 2.6 + """ + + name = 'ReasonML' + url = 'https://reasonml.github.io/' + aliases = ['reasonml', 'reason'] + filenames = ['*.re', '*.rei'] + mimetypes = ['text/x-reasonml'] + + keywords = ( + 'as', 'assert', 'begin', 'class', 'constraint', 'do', 'done', 'downto', + 'else', 'end', 'exception', 'external', 'false', 'for', 'fun', 'esfun', + 'function', 'functor', 'if', 'in', 'include', 'inherit', 'initializer', 'lazy', + 'let', 'switch', 'module', 'pub', 'mutable', 'new', 'nonrec', 'object', 'of', + 'open', 'pri', 'rec', 'sig', 'struct', 'then', 'to', 'true', 'try', + 'type', 'val', 'virtual', 'when', 'while', 'with', + ) + keyopts = ( + '!=', '#', '&', '&&', r'\(', r'\)', r'\*', r'\+', ',', '-', + r'-\.', '=>', r'\.', r'\.\.', r'\.\.\.', ':', '::', ':=', ':>', ';', ';;', '<', + '<-', '=', '>', '>]', r'>\}', r'\?', r'\?\?', r'\[', r'\[<', r'\[>', + r'\[\|', ']', '_', '`', r'\{', r'\{<', r'\|', r'\|\|', r'\|]', r'\}', '~' + ) + + operators = r'[!$%&*+\./:<=>?@^|~-]' + word_operators = ('and', 'asr', 'land', 'lor', 'lsl', 'lsr', 'lxor', 'mod', 'or') + prefix_syms = r'[!?~]' + infix_syms = r'[=<>@^|&+\*/$%-]' + primitives = ('unit', 'int', 'float', 'bool', 'string', 'char', 'list', 'array') + + tokens = { + 'escape-sequence': [ + (r'\\[\\"\'ntbr]', String.Escape), + (r'\\[0-9]{3}', String.Escape), + (r'\\x[0-9a-fA-F]{2}', String.Escape), + ], + 'root': [ + (r'\s+', Text), + (r'false|true|\(\)|\[\]', Name.Builtin.Pseudo), + (r'\b([A-Z][\w\']*)(?=\s*\.)', Name.Namespace, 'dotted'), + (r'\b([A-Z][\w\']*)', Name.Class), + (r'//.*?\n', Comment.Single), + (r'\/\*(?!/)', Comment.Multiline, 'comment'), + (r'\b(%s)\b' % '|'.join(keywords), Keyword), + (r'(%s)' % '|'.join(keyopts[::-1]), Operator.Word), + (r'(%s|%s)?%s' % (infix_syms, prefix_syms, operators), Operator), + (r'\b(%s)\b' % '|'.join(word_operators), Operator.Word), + (r'\b(%s)\b' % '|'.join(primitives), Keyword.Type), + + (r"[^\W\d][\w']*", Name), + + (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)', Number.Float), + (r'0[xX][\da-fA-F][\da-fA-F_]*', Number.Hex), + (r'0[oO][0-7][0-7_]*', Number.Oct), + (r'0[bB][01][01_]*', Number.Bin), + (r'\d[\d_]*', Number.Integer), + + (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'", + String.Char), + (r"'.'", String.Char), + (r"'", Keyword), + + (r'"', String.Double, 'string'), + + (r'[~?][a-z][\w\']*:', Name.Variable), + ], + 'comment': [ + (r'[^/*]+', Comment.Multiline), + (r'\/\*', Comment.Multiline, '#push'), + (r'\*\/', Comment.Multiline, '#pop'), + (r'\*', Comment.Multiline), + ], + 'string': [ + (r'[^\\"]+', String.Double), + include('escape-sequence'), + (r'\\\n', String.Double), + (r'"', String.Double, '#pop'), + ], + 'dotted': [ + (r'\s+', Text), + (r'\.', Punctuation), + (r'[A-Z][\w\']*(?=\s*\.)', Name.Namespace), + (r'[A-Z][\w\']*', Name.Class, '#pop'), + (r'[a-z_][\w\']*', Name, '#pop'), + default('#pop'), + ], + } + + +class FStarLexer(RegexLexer): + """ + For the F* language. + .. versionadded:: 2.7 + """ + + name = 'FStar' + url = 'https://www.fstar-lang.org/' + aliases = ['fstar'] + filenames = ['*.fst', '*.fsti'] + mimetypes = ['text/x-fstar'] + + keywords = ( + 'abstract', 'attributes', 'noeq', 'unopteq', 'and' + 'begin', 'by', 'default', 'effect', 'else', 'end', 'ensures', + 'exception', 'exists', 'false', 'forall', 'fun', 'function', 'if', + 'in', 'include', 'inline', 'inline_for_extraction', 'irreducible', + 'logic', 'match', 'module', 'mutable', 'new', 'new_effect', 'noextract', + 'of', 'open', 'opaque', 'private', 'range_of', 'reifiable', + 'reify', 'reflectable', 'requires', 'set_range_of', 'sub_effect', + 'synth', 'then', 'total', 'true', 'try', 'type', 'unfold', 'unfoldable', + 'val', 'when', 'with', 'not' + ) + decl_keywords = ('let', 'rec') + assume_keywords = ('assume', 'admit', 'assert', 'calc') + keyopts = ( + r'~', r'-', r'/\\', r'\\/', r'<:', r'<@', r'\(\|', r'\|\)', r'#', r'u#', + r'&', r'\(', r'\)', r'\(\)', r',', r'~>', r'->', r'<-', r'<--', r'<==>', + r'==>', r'\.', r'\?', r'\?\.', r'\.\[', r'\.\(', r'\.\(\|', r'\.\[\|', + r'\{:pattern', r':', r'::', r':=', r';', r';;', r'=', r'%\[', r'!\{', + r'\[', r'\[@', r'\[\|', r'\|>', r'\]', r'\|\]', r'\{', r'\|', r'\}', r'\$' + ) + + operators = r'[!$%&*+\./:<=>?@^|~-]' + prefix_syms = r'[!?~]' + infix_syms = r'[=<>@^|&+\*/$%-]' + primitives = ('unit', 'int', 'float', 'bool', 'string', 'char', 'list', 'array') + + tokens = { + 'escape-sequence': [ + (r'\\[\\"\'ntbr]', String.Escape), + (r'\\[0-9]{3}', String.Escape), + (r'\\x[0-9a-fA-F]{2}', String.Escape), + ], + 'root': [ + (r'\s+', Text), + (r'false|true|False|True|\(\)|\[\]', Name.Builtin.Pseudo), + (r'\b([A-Z][\w\']*)(?=\s*\.)', Name.Namespace, 'dotted'), + (r'\b([A-Z][\w\']*)', Name.Class), + (r'\(\*(?![)])', Comment, 'comment'), + (r'\/\/.+$', Comment), + (r'\b(%s)\b' % '|'.join(keywords), Keyword), + (r'\b(%s)\b' % '|'.join(assume_keywords), Name.Exception), + (r'\b(%s)\b' % '|'.join(decl_keywords), Keyword.Declaration), + (r'(%s)' % '|'.join(keyopts[::-1]), Operator), + (r'(%s|%s)?%s' % (infix_syms, prefix_syms, operators), Operator), + (r'\b(%s)\b' % '|'.join(primitives), Keyword.Type), + + (r"[^\W\d][\w']*", Name), + + (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)', Number.Float), + (r'0[xX][\da-fA-F][\da-fA-F_]*', Number.Hex), + (r'0[oO][0-7][0-7_]*', Number.Oct), + (r'0[bB][01][01_]*', Number.Bin), + (r'\d[\d_]*', Number.Integer), + + (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'", + String.Char), + (r"'.'", String.Char), + (r"'", Keyword), # a stray quote is another syntax element + (r"\`([\w\'.]+)\`", Operator.Word), # for infix applications + (r"\`", Keyword), # for quoting + (r'"', String.Double, 'string'), + + (r'[~?][a-z][\w\']*:', Name.Variable), + ], + 'comment': [ + (r'[^(*)]+', Comment), + (r'\(\*', Comment, '#push'), + (r'\*\)', Comment, '#pop'), + (r'[(*)]', Comment), + ], + 'string': [ + (r'[^\\"]+', String.Double), + include('escape-sequence'), + (r'\\\n', String.Double), + (r'"', String.Double, '#pop'), + ], + 'dotted': [ + (r'\s+', Text), + (r'\.', Punctuation), + (r'[A-Z][\w\']*(?=\s*\.)', Name.Namespace), + (r'[A-Z][\w\']*', Name.Class, '#pop'), + (r'[a-z_][\w\']*', Name, '#pop'), + default('#pop'), + ], + } diff --git a/pygments/lexers/modeling.py b/pygments/lexers/modeling.py new file mode 100644 index 0000000..56448f5 --- /dev/null +++ b/pygments/lexers/modeling.py @@ -0,0 +1,369 @@ +""" + pygments.lexers.modeling + ~~~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for modeling languages. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, include, bygroups, using, default +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Whitespace + +from pygments.lexers.html import HtmlLexer +from pygments.lexers import _stan_builtins + +__all__ = ['ModelicaLexer', 'BugsLexer', 'JagsLexer', 'StanLexer'] + + +class ModelicaLexer(RegexLexer): + """ + For Modelica source code. + + .. versionadded:: 1.1 + """ + name = 'Modelica' + url = 'http://www.modelica.org/' + aliases = ['modelica'] + filenames = ['*.mo'] + mimetypes = ['text/x-modelica'] + + flags = re.DOTALL | re.MULTILINE + + _name = r"(?:'(?:[^\\']|\\.)+'|[a-zA-Z_]\w*)" + + tokens = { + 'whitespace': [ + (r'[\s\ufeff]+', Text), + (r'//[^\n]*\n?', Comment.Single), + (r'/\*.*?\*/', Comment.Multiline) + ], + 'root': [ + include('whitespace'), + (r'"', String.Double, 'string'), + (r'[()\[\]{},;]+', Punctuation), + (r'\.?[*^/+-]|\.|<>|[<>:=]=?', Operator), + (r'\d+(\.?\d*[eE][-+]?\d+|\.\d*)', Number.Float), + (r'\d+', Number.Integer), + (r'(abs|acos|actualStream|array|asin|assert|AssertionLevel|atan|' + r'atan2|backSample|Boolean|cardinality|cat|ceil|change|Clock|' + r'Connections|cos|cosh|cross|delay|diagonal|div|edge|exp|' + r'ExternalObject|fill|floor|getInstanceName|hold|homotopy|' + r'identity|inStream|integer|Integer|interval|inverse|isPresent|' + r'linspace|log|log10|matrix|max|min|mod|ndims|noClock|noEvent|' + r'ones|outerProduct|pre|previous|product|Real|reinit|rem|rooted|' + r'sample|scalar|semiLinear|shiftSample|sign|sin|sinh|size|skew|' + r'smooth|spatialDistribution|sqrt|StateSelect|String|subSample|' + r'sum|superSample|symmetric|tan|tanh|terminal|terminate|time|' + r'transpose|vector|zeros)\b', Name.Builtin), + (r'(algorithm|annotation|break|connect|constant|constrainedby|der|' + r'discrete|each|else|elseif|elsewhen|encapsulated|enumeration|' + r'equation|exit|expandable|extends|external|firstTick|final|flow|for|if|' + r'import|impure|in|initial|inner|input|interval|loop|nondiscrete|outer|' + r'output|parameter|partial|protected|public|pure|redeclare|' + r'replaceable|return|stream|then|when|while)\b', + Keyword.Reserved), + (r'(and|not|or)\b', Operator.Word), + (r'(block|class|connector|end|function|model|operator|package|' + r'record|type)\b', Keyword.Reserved, 'class'), + (r'(false|true)\b', Keyword.Constant), + (r'within\b', Keyword.Reserved, 'package-prefix'), + (_name, Name) + ], + 'class': [ + include('whitespace'), + (r'(function|record)\b', Keyword.Reserved), + (r'(if|for|when|while)\b', Keyword.Reserved, '#pop'), + (_name, Name.Class, '#pop'), + default('#pop') + ], + 'package-prefix': [ + include('whitespace'), + (_name, Name.Namespace, '#pop'), + default('#pop') + ], + 'string': [ + (r'"', String.Double, '#pop'), + (r'\\[\'"?\\abfnrtv]', String.Escape), + (r'(?i)<\s*html\s*>([^\\"]|\\.)+?(<\s*/\s*html\s*>|(?="))', + using(HtmlLexer)), + (r'<|\\?[^"\\<]+', String.Double) + ] + } + + +class BugsLexer(RegexLexer): + """ + Pygments Lexer for OpenBugs and WinBugs + models. + + .. versionadded:: 1.6 + """ + + name = 'BUGS' + aliases = ['bugs', 'winbugs', 'openbugs'] + filenames = ['*.bug'] + + _FUNCTIONS = ( + # Scalar functions + 'abs', 'arccos', 'arccosh', 'arcsin', 'arcsinh', 'arctan', 'arctanh', + 'cloglog', 'cos', 'cosh', 'cumulative', 'cut', 'density', 'deviance', + 'equals', 'expr', 'gammap', 'ilogit', 'icloglog', 'integral', 'log', + 'logfact', 'loggam', 'logit', 'max', 'min', 'phi', 'post.p.value', + 'pow', 'prior.p.value', 'probit', 'replicate.post', 'replicate.prior', + 'round', 'sin', 'sinh', 'solution', 'sqrt', 'step', 'tan', 'tanh', + 'trunc', + # Vector functions + 'inprod', 'interp.lin', 'inverse', 'logdet', 'mean', 'eigen.vals', + 'ode', 'prod', 'p.valueM', 'rank', 'ranked', 'replicate.postM', + 'sd', 'sort', 'sum', + # Special + 'D', 'I', 'F', 'T', 'C') + """ OpenBUGS built-in functions + + From http://www.openbugs.info/Manuals/ModelSpecification.html#ContentsAII + + This also includes + + - T, C, I : Truncation and censoring. + ``T`` and ``C`` are in OpenBUGS. ``I`` in WinBUGS. + - D : ODE + - F : Functional http://www.openbugs.info/Examples/Functionals.html + + """ + + _DISTRIBUTIONS = ('dbern', 'dbin', 'dcat', 'dnegbin', 'dpois', + 'dhyper', 'dbeta', 'dchisqr', 'ddexp', 'dexp', + 'dflat', 'dgamma', 'dgev', 'df', 'dggamma', 'dgpar', + 'dloglik', 'dlnorm', 'dlogis', 'dnorm', 'dpar', + 'dt', 'dunif', 'dweib', 'dmulti', 'ddirch', 'dmnorm', + 'dmt', 'dwish') + """ OpenBUGS built-in distributions + + Functions from + http://www.openbugs.info/Manuals/ModelSpecification.html#ContentsAI + """ + + tokens = { + 'whitespace': [ + (r"\s+", Text), + ], + 'comments': [ + # Comments + (r'#.*$', Comment.Single), + ], + 'root': [ + # Comments + include('comments'), + include('whitespace'), + # Block start + (r'(model)(\s+)(\{)', + bygroups(Keyword.Namespace, Text, Punctuation)), + # Reserved Words + (r'(for|in)(?![\w.])', Keyword.Reserved), + # Built-in Functions + (r'(%s)(?=\s*\()' + % r'|'.join(_FUNCTIONS + _DISTRIBUTIONS), + Name.Builtin), + # Regular variable names + (r'[A-Za-z][\w.]*', Name), + # Number Literals + (r'[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?', Number), + # Punctuation + (r'\[|\]|\(|\)|:|,|;', Punctuation), + # Assignment operators + # SLexer makes these tokens Operators. + (r'<-|~', Operator), + # Infix and prefix operators + (r'\+|-|\*|/', Operator), + # Block + (r'[{}]', Punctuation), + ] + } + + def analyse_text(text): + if re.search(r"^\s*model\s*{", text, re.M): + return 0.7 + else: + return 0.0 + + +class JagsLexer(RegexLexer): + """ + Pygments Lexer for JAGS. + + .. versionadded:: 1.6 + """ + + name = 'JAGS' + aliases = ['jags'] + filenames = ['*.jag', '*.bug'] + + # JAGS + _FUNCTIONS = ( + 'abs', 'arccos', 'arccosh', 'arcsin', 'arcsinh', 'arctan', 'arctanh', + 'cos', 'cosh', 'cloglog', + 'equals', 'exp', 'icloglog', 'ifelse', 'ilogit', 'log', 'logfact', + 'loggam', 'logit', 'phi', 'pow', 'probit', 'round', 'sin', 'sinh', + 'sqrt', 'step', 'tan', 'tanh', 'trunc', 'inprod', 'interp.lin', + 'logdet', 'max', 'mean', 'min', 'prod', 'sum', 'sd', 'inverse', + 'rank', 'sort', 't', 'acos', 'acosh', 'asin', 'asinh', 'atan', + # Truncation/Censoring (should I include) + 'T', 'I') + # Distributions with density, probability and quartile functions + _DISTRIBUTIONS = tuple('[dpq]%s' % x for x in + ('bern', 'beta', 'dchiqsqr', 'ddexp', 'dexp', + 'df', 'gamma', 'gen.gamma', 'logis', 'lnorm', + 'negbin', 'nchisqr', 'norm', 'par', 'pois', 'weib')) + # Other distributions without density and probability + _OTHER_DISTRIBUTIONS = ( + 'dt', 'dunif', 'dbetabin', 'dbern', 'dbin', 'dcat', 'dhyper', + 'ddirch', 'dmnorm', 'dwish', 'dmt', 'dmulti', 'dbinom', 'dchisq', + 'dnbinom', 'dweibull', 'ddirich') + + tokens = { + 'whitespace': [ + (r"\s+", Text), + ], + 'names': [ + # Regular variable names + (r'[a-zA-Z][\w.]*\b', Name), + ], + 'comments': [ + # do not use stateful comments + (r'(?s)/\*.*?\*/', Comment.Multiline), + # Comments + (r'#.*$', Comment.Single), + ], + 'root': [ + # Comments + include('comments'), + include('whitespace'), + # Block start + (r'(model|data)(\s+)(\{)', + bygroups(Keyword.Namespace, Text, Punctuation)), + (r'var(?![\w.])', Keyword.Declaration), + # Reserved Words + (r'(for|in)(?![\w.])', Keyword.Reserved), + # Builtins + # Need to use lookahead because . is a valid char + (r'(%s)(?=\s*\()' % r'|'.join(_FUNCTIONS + + _DISTRIBUTIONS + + _OTHER_DISTRIBUTIONS), + Name.Builtin), + # Names + include('names'), + # Number Literals + (r'[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?', Number), + (r'\[|\]|\(|\)|:|,|;', Punctuation), + # Assignment operators + (r'<-|~', Operator), + # # JAGS includes many more than OpenBUGS + (r'\+|-|\*|\/|\|\|[&]{2}|[<>=]=?|\^|%.*?%', Operator), + (r'[{}]', Punctuation), + ] + } + + def analyse_text(text): + if re.search(r'^\s*model\s*\{', text, re.M): + if re.search(r'^\s*data\s*\{', text, re.M): + return 0.9 + elif re.search(r'^\s*var', text, re.M): + return 0.9 + else: + return 0.3 + else: + return 0 + + +class StanLexer(RegexLexer): + """Pygments Lexer for Stan models. + + The Stan modeling language is specified in the *Stan Modeling Language + User's Guide and Reference Manual, v2.17.0*, + `pdf `__. + + .. versionadded:: 1.6 + """ + + name = 'Stan' + aliases = ['stan'] + filenames = ['*.stan'] + + tokens = { + 'whitespace': [ + (r"\s+", Text), + ], + 'comments': [ + (r'(?s)/\*.*?\*/', Comment.Multiline), + # Comments + (r'(//|#).*$', Comment.Single), + ], + 'root': [ + (r'"[^"]*"', String), + # Comments + include('comments'), + # block start + include('whitespace'), + # Block start + (r'(%s)(\s*)(\{)' % + r'|'.join(('functions', 'data', r'transformed\s+?data', + 'parameters', r'transformed\s+parameters', + 'model', r'generated\s+quantities')), + bygroups(Keyword.Namespace, Text, Punctuation)), + # target keyword + (r'target\s*\+=', Keyword), + # Reserved Words + (r'(%s)\b' % r'|'.join(_stan_builtins.KEYWORDS), Keyword), + # Truncation + (r'T(?=\s*\[)', Keyword), + # Data types + (r'(%s)\b' % r'|'.join(_stan_builtins.TYPES), Keyword.Type), + # < should be punctuation, but elsewhere I can't tell if it is in + # a range constraint + (r'(<)(\s*)(upper|lower|offset|multiplier)(\s*)(=)', + bygroups(Operator, Whitespace, Keyword, Whitespace, Punctuation)), + (r'(,)(\s*)(upper)(\s*)(=)', + bygroups(Punctuation, Whitespace, Keyword, Whitespace, Punctuation)), + # Punctuation + (r"[;,\[\]()]", Punctuation), + # Builtin + (r'(%s)(?=\s*\()' % '|'.join(_stan_builtins.FUNCTIONS), Name.Builtin), + (r'(~)(\s*)(%s)(?=\s*\()' % '|'.join(_stan_builtins.DISTRIBUTIONS), + bygroups(Operator, Whitespace, Name.Builtin)), + # Special names ending in __, like lp__ + (r'[A-Za-z]\w*__\b', Name.Builtin.Pseudo), + (r'(%s)\b' % r'|'.join(_stan_builtins.RESERVED), Keyword.Reserved), + # user-defined functions + (r'[A-Za-z]\w*(?=\s*\()]', Name.Function), + # Imaginary Literals + (r'[0-9]+(\.[0-9]*)?([eE][+-]?[0-9]+)?i', Number.Float), + (r'\.[0-9]+([eE][+-]?[0-9]+)?i', Number.Float), + (r'[0-9]+i', Number.Float), + # Real Literals + (r'[0-9]+(\.[0-9]*)?([eE][+-]?[0-9]+)?', Number.Float), + (r'\.[0-9]+([eE][+-]?[0-9]+)?', Number.Float), + # Integer Literals + (r'[0-9]+', Number.Integer), + # Regular variable names + (r'[A-Za-z]\w*\b', Name), + # Assignment operators + (r'<-|(?:\+|-|\.?/|\.?\*|=)?=|~', Operator), + # Infix, prefix and postfix operators (and = ) + (r"\+|-|\.?\*|\.?/|\\|'|\.?\^|!=?|<=?|>=?|\|\||&&|%|\?|:|%/%|!", Operator), + # Block delimiters + (r'[{}]', Punctuation), + # Distribution | + (r'\|', Punctuation) + ] + } + + def analyse_text(text): + if re.search(r'^\s*parameters\s*\{', text, re.M): + return 1.0 + else: + return 0.0 diff --git a/pygments/lexers/modula2.py b/pygments/lexers/modula2.py new file mode 100644 index 0000000..a94d486 --- /dev/null +++ b/pygments/lexers/modula2.py @@ -0,0 +1,1580 @@ +""" + pygments.lexers.modula2 + ~~~~~~~~~~~~~~~~~~~~~~~ + + Multi-Dialect Lexer for Modula-2. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, include +from pygments.util import get_bool_opt, get_list_opt +from pygments.token import Text, Comment, Operator, Keyword, Name, \ + String, Number, Punctuation, Error + +__all__ = ['Modula2Lexer'] + + +# Multi-Dialect Modula-2 Lexer +class Modula2Lexer(RegexLexer): + """ + For Modula-2 source code. + + The Modula-2 lexer supports several dialects. By default, it operates in + fallback mode, recognising the *combined* literals, punctuation symbols + and operators of all supported dialects, and the *combined* reserved words + and builtins of PIM Modula-2, ISO Modula-2 and Modula-2 R10, while not + differentiating between library defined identifiers. + + To select a specific dialect, a dialect option may be passed + or a dialect tag may be embedded into a source file. + + Dialect Options: + + `m2pim` + Select PIM Modula-2 dialect. + `m2iso` + Select ISO Modula-2 dialect. + `m2r10` + Select Modula-2 R10 dialect. + `objm2` + Select Objective Modula-2 dialect. + + The PIM and ISO dialect options may be qualified with a language extension. + + Language Extensions: + + `+aglet` + Select Aglet Modula-2 extensions, available with m2iso. + `+gm2` + Select GNU Modula-2 extensions, available with m2pim. + `+p1` + Select p1 Modula-2 extensions, available with m2iso. + `+xds` + Select XDS Modula-2 extensions, available with m2iso. + + + Passing a Dialect Option via Unix Commandline Interface + + Dialect options may be passed to the lexer using the `dialect` key. + Only one such option should be passed. If multiple dialect options are + passed, the first valid option is used, any subsequent options are ignored. + + Examples: + + `$ pygmentize -O full,dialect=m2iso -f html -o /path/to/output /path/to/input` + Use ISO dialect to render input to HTML output + `$ pygmentize -O full,dialect=m2iso+p1 -f rtf -o /path/to/output /path/to/input` + Use ISO dialect with p1 extensions to render input to RTF output + + + Embedding a Dialect Option within a source file + + A dialect option may be embedded in a source file in form of a dialect + tag, a specially formatted comment that specifies a dialect option. + + Dialect Tag EBNF:: + + dialectTag : + OpeningCommentDelim Prefix dialectOption ClosingCommentDelim ; + + dialectOption : + 'm2pim' | 'm2iso' | 'm2r10' | 'objm2' | + 'm2iso+aglet' | 'm2pim+gm2' | 'm2iso+p1' | 'm2iso+xds' ; + + Prefix : '!' ; + + OpeningCommentDelim : '(*' ; + + ClosingCommentDelim : '*)' ; + + No whitespace is permitted between the tokens of a dialect tag. + + In the event that a source file contains multiple dialect tags, the first + tag that contains a valid dialect option will be used and any subsequent + dialect tags will be ignored. Ideally, a dialect tag should be placed + at the beginning of a source file. + + An embedded dialect tag overrides a dialect option set via command line. + + Examples: + + ``(*!m2r10*) DEFINITION MODULE Foobar; ...`` + Use Modula2 R10 dialect to render this source file. + ``(*!m2pim+gm2*) DEFINITION MODULE Bazbam; ...`` + Use PIM dialect with GNU extensions to render this source file. + + + Algol Publication Mode: + + In Algol publication mode, source text is rendered for publication of + algorithms in scientific papers and academic texts, following the format + of the Revised Algol-60 Language Report. It is activated by passing + one of two corresponding styles as an option: + + `algol` + render reserved words lowercase underline boldface + and builtins lowercase boldface italic + `algol_nu` + render reserved words lowercase boldface (no underlining) + and builtins lowercase boldface italic + + The lexer automatically performs the required lowercase conversion when + this mode is activated. + + Example: + + ``$ pygmentize -O full,style=algol -f latex -o /path/to/output /path/to/input`` + Render input file in Algol publication mode to LaTeX output. + + + Rendering Mode of First Class ADT Identifiers: + + The rendering of standard library first class ADT identifiers is controlled + by option flag "treat_stdlib_adts_as_builtins". + + When this option is turned on, standard library ADT identifiers are rendered + as builtins. When it is turned off, they are rendered as ordinary library + identifiers. + + `treat_stdlib_adts_as_builtins` (default: On) + + The option is useful for dialects that support ADTs as first class objects + and provide ADTs in the standard library that would otherwise be built-in. + + At present, only Modula-2 R10 supports library ADTs as first class objects + and therefore, no ADT identifiers are defined for any other dialects. + + Example: + + ``$ pygmentize -O full,dialect=m2r10,treat_stdlib_adts_as_builtins=Off ...`` + Render standard library ADTs as ordinary library types. + + .. versionadded:: 1.3 + + .. versionchanged:: 2.1 + Added multi-dialect support. + """ + name = 'Modula-2' + url = 'http://www.modula2.org/' + aliases = ['modula2', 'm2'] + filenames = ['*.def', '*.mod'] + mimetypes = ['text/x-modula2'] + + flags = re.MULTILINE | re.DOTALL + + tokens = { + 'whitespace': [ + (r'\n+', Text), # blank lines + (r'\s+', Text), # whitespace + ], + 'dialecttags': [ + # PIM Dialect Tag + (r'\(\*!m2pim\*\)', Comment.Special), + # ISO Dialect Tag + (r'\(\*!m2iso\*\)', Comment.Special), + # M2R10 Dialect Tag + (r'\(\*!m2r10\*\)', Comment.Special), + # ObjM2 Dialect Tag + (r'\(\*!objm2\*\)', Comment.Special), + # Aglet Extensions Dialect Tag + (r'\(\*!m2iso\+aglet\*\)', Comment.Special), + # GNU Extensions Dialect Tag + (r'\(\*!m2pim\+gm2\*\)', Comment.Special), + # p1 Extensions Dialect Tag + (r'\(\*!m2iso\+p1\*\)', Comment.Special), + # XDS Extensions Dialect Tag + (r'\(\*!m2iso\+xds\*\)', Comment.Special), + ], + 'identifiers': [ + (r'([a-zA-Z_$][\w$]*)', Name), + ], + 'prefixed_number_literals': [ + # + # Base-2, whole number + (r'0b[01]+(\'[01]+)*', Number.Bin), + # + # Base-16, whole number + (r'0[ux][0-9A-F]+(\'[0-9A-F]+)*', Number.Hex), + ], + 'plain_number_literals': [ + # + # Base-10, real number with exponent + (r'[0-9]+(\'[0-9]+)*' # integral part + r'\.[0-9]+(\'[0-9]+)*' # fractional part + r'[eE][+-]?[0-9]+(\'[0-9]+)*', # exponent + Number.Float), + # + # Base-10, real number without exponent + (r'[0-9]+(\'[0-9]+)*' # integral part + r'\.[0-9]+(\'[0-9]+)*', # fractional part + Number.Float), + # + # Base-10, whole number + (r'[0-9]+(\'[0-9]+)*', Number.Integer), + ], + 'suffixed_number_literals': [ + # + # Base-8, whole number + (r'[0-7]+B', Number.Oct), + # + # Base-8, character code + (r'[0-7]+C', Number.Oct), + # + # Base-16, number + (r'[0-9A-F]+H', Number.Hex), + ], + 'string_literals': [ + (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double), + (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single), + ], + 'digraph_operators': [ + # Dot Product Operator + (r'\*\.', Operator), + # Array Concatenation Operator + (r'\+>', Operator), # M2R10 + ObjM2 + # Inequality Operator + (r'<>', Operator), # ISO + PIM + # Less-Or-Equal, Subset + (r'<=', Operator), + # Greater-Or-Equal, Superset + (r'>=', Operator), + # Identity Operator + (r'==', Operator), # M2R10 + ObjM2 + # Type Conversion Operator + (r'::', Operator), # M2R10 + ObjM2 + # Assignment Symbol + (r':=', Operator), + # Postfix Increment Mutator + (r'\+\+', Operator), # M2R10 + ObjM2 + # Postfix Decrement Mutator + (r'--', Operator), # M2R10 + ObjM2 + ], + 'unigraph_operators': [ + # Arithmetic Operators + (r'[+-]', Operator), + (r'[*/]', Operator), + # ISO 80000-2 compliant Set Difference Operator + (r'\\', Operator), # M2R10 + ObjM2 + # Relational Operators + (r'[=#<>]', Operator), + # Dereferencing Operator + (r'\^', Operator), + # Dereferencing Operator Synonym + (r'@', Operator), # ISO + # Logical AND Operator Synonym + (r'&', Operator), # PIM + ISO + # Logical NOT Operator Synonym + (r'~', Operator), # PIM + ISO + # Smalltalk Message Prefix + (r'`', Operator), # ObjM2 + ], + 'digraph_punctuation': [ + # Range Constructor + (r'\.\.', Punctuation), + # Opening Chevron Bracket + (r'<<', Punctuation), # M2R10 + ISO + # Closing Chevron Bracket + (r'>>', Punctuation), # M2R10 + ISO + # Blueprint Punctuation + (r'->', Punctuation), # M2R10 + ISO + # Distinguish |# and # in M2 R10 + (r'\|#', Punctuation), + # Distinguish ## and # in M2 R10 + (r'##', Punctuation), + # Distinguish |* and * in M2 R10 + (r'\|\*', Punctuation), + ], + 'unigraph_punctuation': [ + # Common Punctuation + (r'[()\[\]{},.:;|]', Punctuation), + # Case Label Separator Synonym + (r'!', Punctuation), # ISO + # Blueprint Punctuation + (r'\?', Punctuation), # M2R10 + ObjM2 + ], + 'comments': [ + # Single Line Comment + (r'^//.*?\n', Comment.Single), # M2R10 + ObjM2 + # Block Comment + (r'\(\*([^$].*?)\*\)', Comment.Multiline), + # Template Block Comment + (r'/\*(.*?)\*/', Comment.Multiline), # M2R10 + ObjM2 + ], + 'pragmas': [ + # ISO Style Pragmas + (r'<\*.*?\*>', Comment.Preproc), # ISO, M2R10 + ObjM2 + # Pascal Style Pragmas + (r'\(\*\$.*?\*\)', Comment.Preproc), # PIM + ], + 'root': [ + include('whitespace'), + include('dialecttags'), + include('pragmas'), + include('comments'), + include('identifiers'), + include('suffixed_number_literals'), # PIM + ISO + include('prefixed_number_literals'), # M2R10 + ObjM2 + include('plain_number_literals'), + include('string_literals'), + include('digraph_punctuation'), + include('digraph_operators'), + include('unigraph_punctuation'), + include('unigraph_operators'), + ] + } + +# C o m m o n D a t a s e t s + + # Common Reserved Words Dataset + common_reserved_words = ( + # 37 common reserved words + 'AND', 'ARRAY', 'BEGIN', 'BY', 'CASE', 'CONST', 'DEFINITION', 'DIV', + 'DO', 'ELSE', 'ELSIF', 'END', 'EXIT', 'FOR', 'FROM', 'IF', + 'IMPLEMENTATION', 'IMPORT', 'IN', 'LOOP', 'MOD', 'MODULE', 'NOT', + 'OF', 'OR', 'POINTER', 'PROCEDURE', 'RECORD', 'REPEAT', 'RETURN', + 'SET', 'THEN', 'TO', 'TYPE', 'UNTIL', 'VAR', 'WHILE', + ) + + # Common Builtins Dataset + common_builtins = ( + # 16 common builtins + 'ABS', 'BOOLEAN', 'CARDINAL', 'CHAR', 'CHR', 'FALSE', 'INTEGER', + 'LONGINT', 'LONGREAL', 'MAX', 'MIN', 'NIL', 'ODD', 'ORD', 'REAL', + 'TRUE', + ) + + # Common Pseudo-Module Builtins Dataset + common_pseudo_builtins = ( + # 4 common pseudo builtins + 'ADDRESS', 'BYTE', 'WORD', 'ADR' + ) + +# P I M M o d u l a - 2 D a t a s e t s + + # Lexemes to Mark as Error Tokens for PIM Modula-2 + pim_lexemes_to_reject = ( + '!', '`', '@', '$', '%', '?', '\\', '==', '++', '--', '::', '*.', + '+>', '->', '<<', '>>', '|#', '##', + ) + + # PIM Modula-2 Additional Reserved Words Dataset + pim_additional_reserved_words = ( + # 3 additional reserved words + 'EXPORT', 'QUALIFIED', 'WITH', + ) + + # PIM Modula-2 Additional Builtins Dataset + pim_additional_builtins = ( + # 16 additional builtins + 'BITSET', 'CAP', 'DEC', 'DISPOSE', 'EXCL', 'FLOAT', 'HALT', 'HIGH', + 'INC', 'INCL', 'NEW', 'NIL', 'PROC', 'SIZE', 'TRUNC', 'VAL', + ) + + # PIM Modula-2 Additional Pseudo-Module Builtins Dataset + pim_additional_pseudo_builtins = ( + # 5 additional pseudo builtins + 'SYSTEM', 'PROCESS', 'TSIZE', 'NEWPROCESS', 'TRANSFER', + ) + +# I S O M o d u l a - 2 D a t a s e t s + + # Lexemes to Mark as Error Tokens for ISO Modula-2 + iso_lexemes_to_reject = ( + '`', '$', '%', '?', '\\', '==', '++', '--', '::', '*.', '+>', '->', + '<<', '>>', '|#', '##', + ) + + # ISO Modula-2 Additional Reserved Words Dataset + iso_additional_reserved_words = ( + # 9 additional reserved words (ISO 10514-1) + 'EXCEPT', 'EXPORT', 'FINALLY', 'FORWARD', 'PACKEDSET', 'QUALIFIED', + 'REM', 'RETRY', 'WITH', + # 10 additional reserved words (ISO 10514-2 & ISO 10514-3) + 'ABSTRACT', 'AS', 'CLASS', 'GUARD', 'INHERIT', 'OVERRIDE', 'READONLY', + 'REVEAL', 'TRACED', 'UNSAFEGUARDED', + ) + + # ISO Modula-2 Additional Builtins Dataset + iso_additional_builtins = ( + # 26 additional builtins (ISO 10514-1) + 'BITSET', 'CAP', 'CMPLX', 'COMPLEX', 'DEC', 'DISPOSE', 'EXCL', 'FLOAT', + 'HALT', 'HIGH', 'IM', 'INC', 'INCL', 'INT', 'INTERRUPTIBLE', 'LENGTH', + 'LFLOAT', 'LONGCOMPLEX', 'NEW', 'PROC', 'PROTECTION', 'RE', 'SIZE', + 'TRUNC', 'UNINTERRUBTIBLE', 'VAL', + # 5 additional builtins (ISO 10514-2 & ISO 10514-3) + 'CREATE', 'DESTROY', 'EMPTY', 'ISMEMBER', 'SELF', + ) + + # ISO Modula-2 Additional Pseudo-Module Builtins Dataset + iso_additional_pseudo_builtins = ( + # 14 additional builtins (SYSTEM) + 'SYSTEM', 'BITSPERLOC', 'LOCSPERBYTE', 'LOCSPERWORD', 'LOC', + 'ADDADR', 'SUBADR', 'DIFADR', 'MAKEADR', 'ADR', + 'ROTATE', 'SHIFT', 'CAST', 'TSIZE', + # 13 additional builtins (COROUTINES) + 'COROUTINES', 'ATTACH', 'COROUTINE', 'CURRENT', 'DETACH', 'HANDLER', + 'INTERRUPTSOURCE', 'IOTRANSFER', 'IsATTACHED', 'LISTEN', + 'NEWCOROUTINE', 'PROT', 'TRANSFER', + # 9 additional builtins (EXCEPTIONS) + 'EXCEPTIONS', 'AllocateSource', 'CurrentNumber', 'ExceptionNumber', + 'ExceptionSource', 'GetMessage', 'IsCurrentSource', + 'IsExceptionalExecution', 'RAISE', + # 3 additional builtins (TERMINATION) + 'TERMINATION', 'IsTerminating', 'HasHalted', + # 4 additional builtins (M2EXCEPTION) + 'M2EXCEPTION', 'M2Exceptions', 'M2Exception', 'IsM2Exception', + 'indexException', 'rangeException', 'caseSelectException', + 'invalidLocation', 'functionException', 'wholeValueException', + 'wholeDivException', 'realValueException', 'realDivException', + 'complexValueException', 'complexDivException', 'protException', + 'sysException', 'coException', 'exException', + ) + +# M o d u l a - 2 R 1 0 D a t a s e t s + + # Lexemes to Mark as Error Tokens for Modula-2 R10 + m2r10_lexemes_to_reject = ( + '!', '`', '@', '$', '%', '&', '<>', + ) + + # Modula-2 R10 reserved words in addition to the common set + m2r10_additional_reserved_words = ( + # 12 additional reserved words + 'ALIAS', 'ARGLIST', 'BLUEPRINT', 'COPY', 'GENLIB', 'INDETERMINATE', + 'NEW', 'NONE', 'OPAQUE', 'REFERENTIAL', 'RELEASE', 'RETAIN', + # 2 additional reserved words with symbolic assembly option + 'ASM', 'REG', + ) + + # Modula-2 R10 builtins in addition to the common set + m2r10_additional_builtins = ( + # 26 additional builtins + 'CARDINAL', 'COUNT', 'EMPTY', 'EXISTS', 'INSERT', 'LENGTH', 'LONGCARD', + 'OCTET', 'PTR', 'PRED', 'READ', 'READNEW', 'REMOVE', 'RETRIEVE', 'SORT', + 'STORE', 'SUBSET', 'SUCC', 'TLIMIT', 'TMAX', 'TMIN', 'TRUE', 'TSIZE', + 'UNICHAR', 'WRITE', 'WRITEF', + ) + + # Modula-2 R10 Additional Pseudo-Module Builtins Dataset + m2r10_additional_pseudo_builtins = ( + # 13 additional builtins (TPROPERTIES) + 'TPROPERTIES', 'PROPERTY', 'LITERAL', 'TPROPERTY', 'TLITERAL', + 'TBUILTIN', 'TDYN', 'TREFC', 'TNIL', 'TBASE', 'TPRECISION', + 'TMAXEXP', 'TMINEXP', + # 4 additional builtins (CONVERSION) + 'CONVERSION', 'TSXFSIZE', 'SXF', 'VAL', + # 35 additional builtins (UNSAFE) + 'UNSAFE', 'CAST', 'INTRINSIC', 'AVAIL', 'ADD', 'SUB', 'ADDC', 'SUBC', + 'FETCHADD', 'FETCHSUB', 'SHL', 'SHR', 'ASHR', 'ROTL', 'ROTR', 'ROTLC', + 'ROTRC', 'BWNOT', 'BWAND', 'BWOR', 'BWXOR', 'BWNAND', 'BWNOR', + 'SETBIT', 'TESTBIT', 'LSBIT', 'MSBIT', 'CSBITS', 'BAIL', 'HALT', + 'TODO', 'FFI', 'ADDR', 'VARGLIST', 'VARGC', + # 11 additional builtins (ATOMIC) + 'ATOMIC', 'INTRINSIC', 'AVAIL', 'SWAP', 'CAS', 'INC', 'DEC', 'BWAND', + 'BWNAND', 'BWOR', 'BWXOR', + # 7 additional builtins (COMPILER) + 'COMPILER', 'DEBUG', 'MODNAME', 'PROCNAME', 'LINENUM', 'DEFAULT', + 'HASH', + # 5 additional builtins (ASSEMBLER) + 'ASSEMBLER', 'REGISTER', 'SETREG', 'GETREG', 'CODE', + ) + +# O b j e c t i v e M o d u l a - 2 D a t a s e t s + + # Lexemes to Mark as Error Tokens for Objective Modula-2 + objm2_lexemes_to_reject = ( + '!', '$', '%', '&', '<>', + ) + + # Objective Modula-2 Extensions + # reserved words in addition to Modula-2 R10 + objm2_additional_reserved_words = ( + # 16 additional reserved words + 'BYCOPY', 'BYREF', 'CLASS', 'CONTINUE', 'CRITICAL', 'INOUT', 'METHOD', + 'ON', 'OPTIONAL', 'OUT', 'PRIVATE', 'PROTECTED', 'PROTOCOL', 'PUBLIC', + 'SUPER', 'TRY', + ) + + # Objective Modula-2 Extensions + # builtins in addition to Modula-2 R10 + objm2_additional_builtins = ( + # 3 additional builtins + 'OBJECT', 'NO', 'YES', + ) + + # Objective Modula-2 Extensions + # pseudo-module builtins in addition to Modula-2 R10 + objm2_additional_pseudo_builtins = ( + # None + ) + +# A g l e t M o d u l a - 2 D a t a s e t s + + # Aglet Extensions + # reserved words in addition to ISO Modula-2 + aglet_additional_reserved_words = ( + # None + ) + + # Aglet Extensions + # builtins in addition to ISO Modula-2 + aglet_additional_builtins = ( + # 9 additional builtins + 'BITSET8', 'BITSET16', 'BITSET32', 'CARDINAL8', 'CARDINAL16', + 'CARDINAL32', 'INTEGER8', 'INTEGER16', 'INTEGER32', + ) + + # Aglet Modula-2 Extensions + # pseudo-module builtins in addition to ISO Modula-2 + aglet_additional_pseudo_builtins = ( + # None + ) + +# G N U M o d u l a - 2 D a t a s e t s + + # GNU Extensions + # reserved words in addition to PIM Modula-2 + gm2_additional_reserved_words = ( + # 10 additional reserved words + 'ASM', '__ATTRIBUTE__', '__BUILTIN__', '__COLUMN__', '__DATE__', + '__FILE__', '__FUNCTION__', '__LINE__', '__MODULE__', 'VOLATILE', + ) + + # GNU Extensions + # builtins in addition to PIM Modula-2 + gm2_additional_builtins = ( + # 21 additional builtins + 'BITSET8', 'BITSET16', 'BITSET32', 'CARDINAL8', 'CARDINAL16', + 'CARDINAL32', 'CARDINAL64', 'COMPLEX32', 'COMPLEX64', 'COMPLEX96', + 'COMPLEX128', 'INTEGER8', 'INTEGER16', 'INTEGER32', 'INTEGER64', + 'REAL8', 'REAL16', 'REAL32', 'REAL96', 'REAL128', 'THROW', + ) + + # GNU Extensions + # pseudo-module builtins in addition to PIM Modula-2 + gm2_additional_pseudo_builtins = ( + # None + ) + +# p 1 M o d u l a - 2 D a t a s e t s + + # p1 Extensions + # reserved words in addition to ISO Modula-2 + p1_additional_reserved_words = ( + # None + ) + + # p1 Extensions + # builtins in addition to ISO Modula-2 + p1_additional_builtins = ( + # None + ) + + # p1 Modula-2 Extensions + # pseudo-module builtins in addition to ISO Modula-2 + p1_additional_pseudo_builtins = ( + # 1 additional builtin + 'BCD', + ) + +# X D S M o d u l a - 2 D a t a s e t s + + # XDS Extensions + # reserved words in addition to ISO Modula-2 + xds_additional_reserved_words = ( + # 1 additional reserved word + 'SEQ', + ) + + # XDS Extensions + # builtins in addition to ISO Modula-2 + xds_additional_builtins = ( + # 9 additional builtins + 'ASH', 'ASSERT', 'DIFFADR_TYPE', 'ENTIER', 'INDEX', 'LEN', + 'LONGCARD', 'SHORTCARD', 'SHORTINT', + ) + + # XDS Modula-2 Extensions + # pseudo-module builtins in addition to ISO Modula-2 + xds_additional_pseudo_builtins = ( + # 22 additional builtins (SYSTEM) + 'PROCESS', 'NEWPROCESS', 'BOOL8', 'BOOL16', 'BOOL32', 'CARD8', + 'CARD16', 'CARD32', 'INT8', 'INT16', 'INT32', 'REF', 'MOVE', + 'FILL', 'GET', 'PUT', 'CC', 'int', 'unsigned', 'size_t', 'void' + # 3 additional builtins (COMPILER) + 'COMPILER', 'OPTION', 'EQUATION' + ) + +# P I M S t a n d a r d L i b r a r y D a t a s e t s + + # PIM Modula-2 Standard Library Modules Dataset + pim_stdlib_module_identifiers = ( + 'Terminal', 'FileSystem', 'InOut', 'RealInOut', 'MathLib0', 'Storage', + ) + + # PIM Modula-2 Standard Library Types Dataset + pim_stdlib_type_identifiers = ( + 'Flag', 'FlagSet', 'Response', 'Command', 'Lock', 'Permission', + 'MediumType', 'File', 'FileProc', 'DirectoryProc', 'FileCommand', + 'DirectoryCommand', + ) + + # PIM Modula-2 Standard Library Procedures Dataset + pim_stdlib_proc_identifiers = ( + 'Read', 'BusyRead', 'ReadAgain', 'Write', 'WriteString', 'WriteLn', + 'Create', 'Lookup', 'Close', 'Delete', 'Rename', 'SetRead', 'SetWrite', + 'SetModify', 'SetOpen', 'Doio', 'SetPos', 'GetPos', 'Length', 'Reset', + 'Again', 'ReadWord', 'WriteWord', 'ReadChar', 'WriteChar', + 'CreateMedium', 'DeleteMedium', 'AssignName', 'DeassignName', + 'ReadMedium', 'LookupMedium', 'OpenInput', 'OpenOutput', 'CloseInput', + 'CloseOutput', 'ReadString', 'ReadInt', 'ReadCard', 'ReadWrd', + 'WriteInt', 'WriteCard', 'WriteOct', 'WriteHex', 'WriteWrd', + 'ReadReal', 'WriteReal', 'WriteFixPt', 'WriteRealOct', 'sqrt', 'exp', + 'ln', 'sin', 'cos', 'arctan', 'entier', 'ALLOCATE', 'DEALLOCATE', + ) + + # PIM Modula-2 Standard Library Variables Dataset + pim_stdlib_var_identifiers = ( + 'Done', 'termCH', 'in', 'out' + ) + + # PIM Modula-2 Standard Library Constants Dataset + pim_stdlib_const_identifiers = ( + 'EOL', + ) + +# I S O S t a n d a r d L i b r a r y D a t a s e t s + + # ISO Modula-2 Standard Library Modules Dataset + iso_stdlib_module_identifiers = ( + # TO DO + ) + + # ISO Modula-2 Standard Library Types Dataset + iso_stdlib_type_identifiers = ( + # TO DO + ) + + # ISO Modula-2 Standard Library Procedures Dataset + iso_stdlib_proc_identifiers = ( + # TO DO + ) + + # ISO Modula-2 Standard Library Variables Dataset + iso_stdlib_var_identifiers = ( + # TO DO + ) + + # ISO Modula-2 Standard Library Constants Dataset + iso_stdlib_const_identifiers = ( + # TO DO + ) + +# M 2 R 1 0 S t a n d a r d L i b r a r y D a t a s e t s + + # Modula-2 R10 Standard Library ADTs Dataset + m2r10_stdlib_adt_identifiers = ( + 'BCD', 'LONGBCD', 'BITSET', 'SHORTBITSET', 'LONGBITSET', + 'LONGLONGBITSET', 'COMPLEX', 'LONGCOMPLEX', 'SHORTCARD', 'LONGLONGCARD', + 'SHORTINT', 'LONGLONGINT', 'POSINT', 'SHORTPOSINT', 'LONGPOSINT', + 'LONGLONGPOSINT', 'BITSET8', 'BITSET16', 'BITSET32', 'BITSET64', + 'BITSET128', 'BS8', 'BS16', 'BS32', 'BS64', 'BS128', 'CARDINAL8', + 'CARDINAL16', 'CARDINAL32', 'CARDINAL64', 'CARDINAL128', 'CARD8', + 'CARD16', 'CARD32', 'CARD64', 'CARD128', 'INTEGER8', 'INTEGER16', + 'INTEGER32', 'INTEGER64', 'INTEGER128', 'INT8', 'INT16', 'INT32', + 'INT64', 'INT128', 'STRING', 'UNISTRING', + ) + + # Modula-2 R10 Standard Library Blueprints Dataset + m2r10_stdlib_blueprint_identifiers = ( + 'ProtoRoot', 'ProtoComputational', 'ProtoNumeric', 'ProtoScalar', + 'ProtoNonScalar', 'ProtoCardinal', 'ProtoInteger', 'ProtoReal', + 'ProtoComplex', 'ProtoVector', 'ProtoTuple', 'ProtoCompArray', + 'ProtoCollection', 'ProtoStaticArray', 'ProtoStaticSet', + 'ProtoStaticString', 'ProtoArray', 'ProtoString', 'ProtoSet', + 'ProtoMultiSet', 'ProtoDictionary', 'ProtoMultiDict', 'ProtoExtension', + 'ProtoIO', 'ProtoCardMath', 'ProtoIntMath', 'ProtoRealMath', + ) + + # Modula-2 R10 Standard Library Modules Dataset + m2r10_stdlib_module_identifiers = ( + 'ASCII', 'BooleanIO', 'CharIO', 'UnicharIO', 'OctetIO', + 'CardinalIO', 'LongCardIO', 'IntegerIO', 'LongIntIO', 'RealIO', + 'LongRealIO', 'BCDIO', 'LongBCDIO', 'CardMath', 'LongCardMath', + 'IntMath', 'LongIntMath', 'RealMath', 'LongRealMath', 'BCDMath', + 'LongBCDMath', 'FileIO', 'FileSystem', 'Storage', 'IOSupport', + ) + + # Modula-2 R10 Standard Library Types Dataset + m2r10_stdlib_type_identifiers = ( + 'File', 'Status', + # TO BE COMPLETED + ) + + # Modula-2 R10 Standard Library Procedures Dataset + m2r10_stdlib_proc_identifiers = ( + 'ALLOCATE', 'DEALLOCATE', 'SIZE', + # TO BE COMPLETED + ) + + # Modula-2 R10 Standard Library Variables Dataset + m2r10_stdlib_var_identifiers = ( + 'stdIn', 'stdOut', 'stdErr', + ) + + # Modula-2 R10 Standard Library Constants Dataset + m2r10_stdlib_const_identifiers = ( + 'pi', 'tau', + ) + +# D i a l e c t s + + # Dialect modes + dialects = ( + 'unknown', + 'm2pim', 'm2iso', 'm2r10', 'objm2', + 'm2iso+aglet', 'm2pim+gm2', 'm2iso+p1', 'm2iso+xds', + ) + +# D a t a b a s e s + + # Lexemes to Mark as Errors Database + lexemes_to_reject_db = { + # Lexemes to reject for unknown dialect + 'unknown': ( + # LEAVE THIS EMPTY + ), + # Lexemes to reject for PIM Modula-2 + 'm2pim': ( + pim_lexemes_to_reject, + ), + # Lexemes to reject for ISO Modula-2 + 'm2iso': ( + iso_lexemes_to_reject, + ), + # Lexemes to reject for Modula-2 R10 + 'm2r10': ( + m2r10_lexemes_to_reject, + ), + # Lexemes to reject for Objective Modula-2 + 'objm2': ( + objm2_lexemes_to_reject, + ), + # Lexemes to reject for Aglet Modula-2 + 'm2iso+aglet': ( + iso_lexemes_to_reject, + ), + # Lexemes to reject for GNU Modula-2 + 'm2pim+gm2': ( + pim_lexemes_to_reject, + ), + # Lexemes to reject for p1 Modula-2 + 'm2iso+p1': ( + iso_lexemes_to_reject, + ), + # Lexemes to reject for XDS Modula-2 + 'm2iso+xds': ( + iso_lexemes_to_reject, + ), + } + + # Reserved Words Database + reserved_words_db = { + # Reserved words for unknown dialect + 'unknown': ( + common_reserved_words, + pim_additional_reserved_words, + iso_additional_reserved_words, + m2r10_additional_reserved_words, + ), + + # Reserved words for PIM Modula-2 + 'm2pim': ( + common_reserved_words, + pim_additional_reserved_words, + ), + + # Reserved words for Modula-2 R10 + 'm2iso': ( + common_reserved_words, + iso_additional_reserved_words, + ), + + # Reserved words for ISO Modula-2 + 'm2r10': ( + common_reserved_words, + m2r10_additional_reserved_words, + ), + + # Reserved words for Objective Modula-2 + 'objm2': ( + common_reserved_words, + m2r10_additional_reserved_words, + objm2_additional_reserved_words, + ), + + # Reserved words for Aglet Modula-2 Extensions + 'm2iso+aglet': ( + common_reserved_words, + iso_additional_reserved_words, + aglet_additional_reserved_words, + ), + + # Reserved words for GNU Modula-2 Extensions + 'm2pim+gm2': ( + common_reserved_words, + pim_additional_reserved_words, + gm2_additional_reserved_words, + ), + + # Reserved words for p1 Modula-2 Extensions + 'm2iso+p1': ( + common_reserved_words, + iso_additional_reserved_words, + p1_additional_reserved_words, + ), + + # Reserved words for XDS Modula-2 Extensions + 'm2iso+xds': ( + common_reserved_words, + iso_additional_reserved_words, + xds_additional_reserved_words, + ), + } + + # Builtins Database + builtins_db = { + # Builtins for unknown dialect + 'unknown': ( + common_builtins, + pim_additional_builtins, + iso_additional_builtins, + m2r10_additional_builtins, + ), + + # Builtins for PIM Modula-2 + 'm2pim': ( + common_builtins, + pim_additional_builtins, + ), + + # Builtins for ISO Modula-2 + 'm2iso': ( + common_builtins, + iso_additional_builtins, + ), + + # Builtins for ISO Modula-2 + 'm2r10': ( + common_builtins, + m2r10_additional_builtins, + ), + + # Builtins for Objective Modula-2 + 'objm2': ( + common_builtins, + m2r10_additional_builtins, + objm2_additional_builtins, + ), + + # Builtins for Aglet Modula-2 Extensions + 'm2iso+aglet': ( + common_builtins, + iso_additional_builtins, + aglet_additional_builtins, + ), + + # Builtins for GNU Modula-2 Extensions + 'm2pim+gm2': ( + common_builtins, + pim_additional_builtins, + gm2_additional_builtins, + ), + + # Builtins for p1 Modula-2 Extensions + 'm2iso+p1': ( + common_builtins, + iso_additional_builtins, + p1_additional_builtins, + ), + + # Builtins for XDS Modula-2 Extensions + 'm2iso+xds': ( + common_builtins, + iso_additional_builtins, + xds_additional_builtins, + ), + } + + # Pseudo-Module Builtins Database + pseudo_builtins_db = { + # Builtins for unknown dialect + 'unknown': ( + common_pseudo_builtins, + pim_additional_pseudo_builtins, + iso_additional_pseudo_builtins, + m2r10_additional_pseudo_builtins, + ), + + # Builtins for PIM Modula-2 + 'm2pim': ( + common_pseudo_builtins, + pim_additional_pseudo_builtins, + ), + + # Builtins for ISO Modula-2 + 'm2iso': ( + common_pseudo_builtins, + iso_additional_pseudo_builtins, + ), + + # Builtins for ISO Modula-2 + 'm2r10': ( + common_pseudo_builtins, + m2r10_additional_pseudo_builtins, + ), + + # Builtins for Objective Modula-2 + 'objm2': ( + common_pseudo_builtins, + m2r10_additional_pseudo_builtins, + objm2_additional_pseudo_builtins, + ), + + # Builtins for Aglet Modula-2 Extensions + 'm2iso+aglet': ( + common_pseudo_builtins, + iso_additional_pseudo_builtins, + aglet_additional_pseudo_builtins, + ), + + # Builtins for GNU Modula-2 Extensions + 'm2pim+gm2': ( + common_pseudo_builtins, + pim_additional_pseudo_builtins, + gm2_additional_pseudo_builtins, + ), + + # Builtins for p1 Modula-2 Extensions + 'm2iso+p1': ( + common_pseudo_builtins, + iso_additional_pseudo_builtins, + p1_additional_pseudo_builtins, + ), + + # Builtins for XDS Modula-2 Extensions + 'm2iso+xds': ( + common_pseudo_builtins, + iso_additional_pseudo_builtins, + xds_additional_pseudo_builtins, + ), + } + + # Standard Library ADTs Database + stdlib_adts_db = { + # Empty entry for unknown dialect + 'unknown': ( + # LEAVE THIS EMPTY + ), + # Standard Library ADTs for PIM Modula-2 + 'm2pim': ( + # No first class library types + ), + + # Standard Library ADTs for ISO Modula-2 + 'm2iso': ( + # No first class library types + ), + + # Standard Library ADTs for Modula-2 R10 + 'm2r10': ( + m2r10_stdlib_adt_identifiers, + ), + + # Standard Library ADTs for Objective Modula-2 + 'objm2': ( + m2r10_stdlib_adt_identifiers, + ), + + # Standard Library ADTs for Aglet Modula-2 + 'm2iso+aglet': ( + # No first class library types + ), + + # Standard Library ADTs for GNU Modula-2 + 'm2pim+gm2': ( + # No first class library types + ), + + # Standard Library ADTs for p1 Modula-2 + 'm2iso+p1': ( + # No first class library types + ), + + # Standard Library ADTs for XDS Modula-2 + 'm2iso+xds': ( + # No first class library types + ), + } + + # Standard Library Modules Database + stdlib_modules_db = { + # Empty entry for unknown dialect + 'unknown': ( + # LEAVE THIS EMPTY + ), + # Standard Library Modules for PIM Modula-2 + 'm2pim': ( + pim_stdlib_module_identifiers, + ), + + # Standard Library Modules for ISO Modula-2 + 'm2iso': ( + iso_stdlib_module_identifiers, + ), + + # Standard Library Modules for Modula-2 R10 + 'm2r10': ( + m2r10_stdlib_blueprint_identifiers, + m2r10_stdlib_module_identifiers, + m2r10_stdlib_adt_identifiers, + ), + + # Standard Library Modules for Objective Modula-2 + 'objm2': ( + m2r10_stdlib_blueprint_identifiers, + m2r10_stdlib_module_identifiers, + ), + + # Standard Library Modules for Aglet Modula-2 + 'm2iso+aglet': ( + iso_stdlib_module_identifiers, + ), + + # Standard Library Modules for GNU Modula-2 + 'm2pim+gm2': ( + pim_stdlib_module_identifiers, + ), + + # Standard Library Modules for p1 Modula-2 + 'm2iso+p1': ( + iso_stdlib_module_identifiers, + ), + + # Standard Library Modules for XDS Modula-2 + 'm2iso+xds': ( + iso_stdlib_module_identifiers, + ), + } + + # Standard Library Types Database + stdlib_types_db = { + # Empty entry for unknown dialect + 'unknown': ( + # LEAVE THIS EMPTY + ), + # Standard Library Types for PIM Modula-2 + 'm2pim': ( + pim_stdlib_type_identifiers, + ), + + # Standard Library Types for ISO Modula-2 + 'm2iso': ( + iso_stdlib_type_identifiers, + ), + + # Standard Library Types for Modula-2 R10 + 'm2r10': ( + m2r10_stdlib_type_identifiers, + ), + + # Standard Library Types for Objective Modula-2 + 'objm2': ( + m2r10_stdlib_type_identifiers, + ), + + # Standard Library Types for Aglet Modula-2 + 'm2iso+aglet': ( + iso_stdlib_type_identifiers, + ), + + # Standard Library Types for GNU Modula-2 + 'm2pim+gm2': ( + pim_stdlib_type_identifiers, + ), + + # Standard Library Types for p1 Modula-2 + 'm2iso+p1': ( + iso_stdlib_type_identifiers, + ), + + # Standard Library Types for XDS Modula-2 + 'm2iso+xds': ( + iso_stdlib_type_identifiers, + ), + } + + # Standard Library Procedures Database + stdlib_procedures_db = { + # Empty entry for unknown dialect + 'unknown': ( + # LEAVE THIS EMPTY + ), + # Standard Library Procedures for PIM Modula-2 + 'm2pim': ( + pim_stdlib_proc_identifiers, + ), + + # Standard Library Procedures for ISO Modula-2 + 'm2iso': ( + iso_stdlib_proc_identifiers, + ), + + # Standard Library Procedures for Modula-2 R10 + 'm2r10': ( + m2r10_stdlib_proc_identifiers, + ), + + # Standard Library Procedures for Objective Modula-2 + 'objm2': ( + m2r10_stdlib_proc_identifiers, + ), + + # Standard Library Procedures for Aglet Modula-2 + 'm2iso+aglet': ( + iso_stdlib_proc_identifiers, + ), + + # Standard Library Procedures for GNU Modula-2 + 'm2pim+gm2': ( + pim_stdlib_proc_identifiers, + ), + + # Standard Library Procedures for p1 Modula-2 + 'm2iso+p1': ( + iso_stdlib_proc_identifiers, + ), + + # Standard Library Procedures for XDS Modula-2 + 'm2iso+xds': ( + iso_stdlib_proc_identifiers, + ), + } + + # Standard Library Variables Database + stdlib_variables_db = { + # Empty entry for unknown dialect + 'unknown': ( + # LEAVE THIS EMPTY + ), + # Standard Library Variables for PIM Modula-2 + 'm2pim': ( + pim_stdlib_var_identifiers, + ), + + # Standard Library Variables for ISO Modula-2 + 'm2iso': ( + iso_stdlib_var_identifiers, + ), + + # Standard Library Variables for Modula-2 R10 + 'm2r10': ( + m2r10_stdlib_var_identifiers, + ), + + # Standard Library Variables for Objective Modula-2 + 'objm2': ( + m2r10_stdlib_var_identifiers, + ), + + # Standard Library Variables for Aglet Modula-2 + 'm2iso+aglet': ( + iso_stdlib_var_identifiers, + ), + + # Standard Library Variables for GNU Modula-2 + 'm2pim+gm2': ( + pim_stdlib_var_identifiers, + ), + + # Standard Library Variables for p1 Modula-2 + 'm2iso+p1': ( + iso_stdlib_var_identifiers, + ), + + # Standard Library Variables for XDS Modula-2 + 'm2iso+xds': ( + iso_stdlib_var_identifiers, + ), + } + + # Standard Library Constants Database + stdlib_constants_db = { + # Empty entry for unknown dialect + 'unknown': ( + # LEAVE THIS EMPTY + ), + # Standard Library Constants for PIM Modula-2 + 'm2pim': ( + pim_stdlib_const_identifiers, + ), + + # Standard Library Constants for ISO Modula-2 + 'm2iso': ( + iso_stdlib_const_identifiers, + ), + + # Standard Library Constants for Modula-2 R10 + 'm2r10': ( + m2r10_stdlib_const_identifiers, + ), + + # Standard Library Constants for Objective Modula-2 + 'objm2': ( + m2r10_stdlib_const_identifiers, + ), + + # Standard Library Constants for Aglet Modula-2 + 'm2iso+aglet': ( + iso_stdlib_const_identifiers, + ), + + # Standard Library Constants for GNU Modula-2 + 'm2pim+gm2': ( + pim_stdlib_const_identifiers, + ), + + # Standard Library Constants for p1 Modula-2 + 'm2iso+p1': ( + iso_stdlib_const_identifiers, + ), + + # Standard Library Constants for XDS Modula-2 + 'm2iso+xds': ( + iso_stdlib_const_identifiers, + ), + } + +# M e t h o d s + + # initialise a lexer instance + def __init__(self, **options): + # + # check dialect options + # + dialects = get_list_opt(options, 'dialect', []) + # + for dialect_option in dialects: + if dialect_option in self.dialects[1:-1]: + # valid dialect option found + self.set_dialect(dialect_option) + break + # + # Fallback Mode (DEFAULT) + else: + # no valid dialect option + self.set_dialect('unknown') + # + self.dialect_set_by_tag = False + # + # check style options + # + styles = get_list_opt(options, 'style', []) + # + # use lowercase mode for Algol style + if 'algol' in styles or 'algol_nu' in styles: + self.algol_publication_mode = True + else: + self.algol_publication_mode = False + # + # Check option flags + # + self.treat_stdlib_adts_as_builtins = get_bool_opt( + options, 'treat_stdlib_adts_as_builtins', True) + # + # call superclass initialiser + RegexLexer.__init__(self, **options) + + # Set lexer to a specified dialect + def set_dialect(self, dialect_id): + # + # if __debug__: + # print 'entered set_dialect with arg: ', dialect_id + # + # check dialect name against known dialects + if dialect_id not in self.dialects: + dialect = 'unknown' # default + else: + dialect = dialect_id + # + # compose lexemes to reject set + lexemes_to_reject_set = set() + # add each list of reject lexemes for this dialect + for list in self.lexemes_to_reject_db[dialect]: + lexemes_to_reject_set.update(set(list)) + # + # compose reserved words set + reswords_set = set() + # add each list of reserved words for this dialect + for list in self.reserved_words_db[dialect]: + reswords_set.update(set(list)) + # + # compose builtins set + builtins_set = set() + # add each list of builtins for this dialect excluding reserved words + for list in self.builtins_db[dialect]: + builtins_set.update(set(list).difference(reswords_set)) + # + # compose pseudo-builtins set + pseudo_builtins_set = set() + # add each list of builtins for this dialect excluding reserved words + for list in self.pseudo_builtins_db[dialect]: + pseudo_builtins_set.update(set(list).difference(reswords_set)) + # + # compose ADTs set + adts_set = set() + # add each list of ADTs for this dialect excluding reserved words + for list in self.stdlib_adts_db[dialect]: + adts_set.update(set(list).difference(reswords_set)) + # + # compose modules set + modules_set = set() + # add each list of builtins for this dialect excluding builtins + for list in self.stdlib_modules_db[dialect]: + modules_set.update(set(list).difference(builtins_set)) + # + # compose types set + types_set = set() + # add each list of types for this dialect excluding builtins + for list in self.stdlib_types_db[dialect]: + types_set.update(set(list).difference(builtins_set)) + # + # compose procedures set + procedures_set = set() + # add each list of procedures for this dialect excluding builtins + for list in self.stdlib_procedures_db[dialect]: + procedures_set.update(set(list).difference(builtins_set)) + # + # compose variables set + variables_set = set() + # add each list of variables for this dialect excluding builtins + for list in self.stdlib_variables_db[dialect]: + variables_set.update(set(list).difference(builtins_set)) + # + # compose constants set + constants_set = set() + # add each list of constants for this dialect excluding builtins + for list in self.stdlib_constants_db[dialect]: + constants_set.update(set(list).difference(builtins_set)) + # + # update lexer state + self.dialect = dialect + self.lexemes_to_reject = lexemes_to_reject_set + self.reserved_words = reswords_set + self.builtins = builtins_set + self.pseudo_builtins = pseudo_builtins_set + self.adts = adts_set + self.modules = modules_set + self.types = types_set + self.procedures = procedures_set + self.variables = variables_set + self.constants = constants_set + # + # if __debug__: + # print 'exiting set_dialect' + # print ' self.dialect: ', self.dialect + # print ' self.lexemes_to_reject: ', self.lexemes_to_reject + # print ' self.reserved_words: ', self.reserved_words + # print ' self.builtins: ', self.builtins + # print ' self.pseudo_builtins: ', self.pseudo_builtins + # print ' self.adts: ', self.adts + # print ' self.modules: ', self.modules + # print ' self.types: ', self.types + # print ' self.procedures: ', self.procedures + # print ' self.variables: ', self.variables + # print ' self.types: ', self.types + # print ' self.constants: ', self.constants + + # Extracts a dialect name from a dialect tag comment string and checks + # the extracted name against known dialects. If a match is found, the + # matching name is returned, otherwise dialect id 'unknown' is returned + def get_dialect_from_dialect_tag(self, dialect_tag): + # + # if __debug__: + # print 'entered get_dialect_from_dialect_tag with arg: ', dialect_tag + # + # constants + left_tag_delim = '(*!' + right_tag_delim = '*)' + left_tag_delim_len = len(left_tag_delim) + right_tag_delim_len = len(right_tag_delim) + indicator_start = left_tag_delim_len + indicator_end = -(right_tag_delim_len) + # + # check comment string for dialect indicator + if len(dialect_tag) > (left_tag_delim_len + right_tag_delim_len) \ + and dialect_tag.startswith(left_tag_delim) \ + and dialect_tag.endswith(right_tag_delim): + # + # if __debug__: + # print 'dialect tag found' + # + # extract dialect indicator + indicator = dialect_tag[indicator_start:indicator_end] + # + # if __debug__: + # print 'extracted: ', indicator + # + # check against known dialects + for index in range(1, len(self.dialects)): + # + # if __debug__: + # print 'dialects[', index, ']: ', self.dialects[index] + # + if indicator == self.dialects[index]: + # + # if __debug__: + # print 'matching dialect found' + # + # indicator matches known dialect + return indicator + else: + # indicator does not match any dialect + return 'unknown' # default + else: + # invalid indicator string + return 'unknown' # default + + # intercept the token stream, modify token attributes and return them + def get_tokens_unprocessed(self, text): + for index, token, value in RegexLexer.get_tokens_unprocessed(self, text): + # + # check for dialect tag if dialect has not been set by tag + if not self.dialect_set_by_tag and token == Comment.Special: + indicated_dialect = self.get_dialect_from_dialect_tag(value) + if indicated_dialect != 'unknown': + # token is a dialect indicator + # reset reserved words and builtins + self.set_dialect(indicated_dialect) + self.dialect_set_by_tag = True + # + # check for reserved words, predefined and stdlib identifiers + if token is Name: + if value in self.reserved_words: + token = Keyword.Reserved + if self.algol_publication_mode: + value = value.lower() + # + elif value in self.builtins: + token = Name.Builtin + if self.algol_publication_mode: + value = value.lower() + # + elif value in self.pseudo_builtins: + token = Name.Builtin.Pseudo + if self.algol_publication_mode: + value = value.lower() + # + elif value in self.adts: + if not self.treat_stdlib_adts_as_builtins: + token = Name.Namespace + else: + token = Name.Builtin.Pseudo + if self.algol_publication_mode: + value = value.lower() + # + elif value in self.modules: + token = Name.Namespace + # + elif value in self.types: + token = Name.Class + # + elif value in self.procedures: + token = Name.Function + # + elif value in self.variables: + token = Name.Variable + # + elif value in self.constants: + token = Name.Constant + # + elif token in Number: + # + # mark prefix number literals as error for PIM and ISO dialects + if self.dialect not in ('unknown', 'm2r10', 'objm2'): + if "'" in value or value[0:2] in ('0b', '0x', '0u'): + token = Error + # + elif self.dialect in ('m2r10', 'objm2'): + # mark base-8 number literals as errors for M2 R10 and ObjM2 + if token is Number.Oct: + token = Error + # mark suffix base-16 literals as errors for M2 R10 and ObjM2 + elif token is Number.Hex and 'H' in value: + token = Error + # mark real numbers with E as errors for M2 R10 and ObjM2 + elif token is Number.Float and 'E' in value: + token = Error + # + elif token in Comment: + # + # mark single line comment as error for PIM and ISO dialects + if token is Comment.Single: + if self.dialect not in ('unknown', 'm2r10', 'objm2'): + token = Error + # + if token is Comment.Preproc: + # mark ISO pragma as error for PIM dialects + if value.startswith('<*') and \ + self.dialect.startswith('m2pim'): + token = Error + # mark PIM pragma as comment for other dialects + elif value.startswith('(*$') and \ + self.dialect != 'unknown' and \ + not self.dialect.startswith('m2pim'): + token = Comment.Multiline + # + else: # token is neither Name nor Comment + # + # mark lexemes matching the dialect's error token set as errors + if value in self.lexemes_to_reject: + token = Error + # + # substitute lexemes when in Algol mode + if self.algol_publication_mode: + if value == '#': + value = '≠' + elif value == '<=': + value = '≤' + elif value == '>=': + value = '≥' + elif value == '==': + value = '≡' + elif value == '*.': + value = '•' + + # return result + yield index, token, value + + def analyse_text(text): + """It's Pascal-like, but does not use FUNCTION -- uses PROCEDURE + instead.""" + + # Check if this looks like Pascal, if not, bail out early + if not ('(*' in text and '*)' in text and ':=' in text): + return + + result = 0 + # Procedure is in Modula2 + if re.search(r'\bPROCEDURE\b', text): + result += 0.6 + + # FUNCTION is only valid in Pascal, but not in Modula2 + if re.search(r'\bFUNCTION\b', text): + result = 0.0 + + return result diff --git a/pygments/lexers/monte.py b/pygments/lexers/monte.py new file mode 100644 index 0000000..e22fb93 --- /dev/null +++ b/pygments/lexers/monte.py @@ -0,0 +1,204 @@ +""" + pygments.lexers.monte + ~~~~~~~~~~~~~~~~~~~~~ + + Lexer for the Monte programming language. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.token import Comment, Error, Keyword, Name, Number, Operator, \ + Punctuation, String, Whitespace +from pygments.lexer import RegexLexer, include, words + +__all__ = ['MonteLexer'] + + +# `var` handled separately +# `interface` handled separately +_declarations = ['bind', 'def', 'fn', 'object'] +_methods = ['method', 'to'] +_keywords = [ + 'as', 'break', 'catch', 'continue', 'else', 'escape', 'exit', 'exports', + 'extends', 'finally', 'for', 'guards', 'if', 'implements', 'import', + 'in', 'match', 'meta', 'pass', 'return', 'switch', 'try', 'via', 'when', + 'while', +] +_operators = [ + # Unary + '~', '!', + # Binary + '+', '-', '*', '/', '%', '**', '&', '|', '^', '<<', '>>', + # Binary augmented + '+=', '-=', '*=', '/=', '%=', '**=', '&=', '|=', '^=', '<<=', '>>=', + # Comparison + '==', '!=', '<', '<=', '>', '>=', '<=>', + # Patterns and assignment + ':=', '?', '=~', '!~', '=>', + # Calls and sends + '.', '<-', '->', +] +_escape_pattern = ( + r'(?:\\x[0-9a-fA-F]{2}|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|' + r'\\["\'\\bftnr])') +# _char = _escape_chars + [('.', String.Char)] +_identifier = r'[_a-zA-Z]\w*' + +_constants = [ + # Void constants + 'null', + # Bool constants + 'false', 'true', + # Double constants + 'Infinity', 'NaN', + # Special objects + 'M', 'Ref', 'throw', 'traceln', +] + +_guards = [ + 'Any', 'Binding', 'Bool', 'Bytes', 'Char', 'DeepFrozen', 'Double', + 'Empty', 'Int', 'List', 'Map', 'Near', 'NullOk', 'Same', 'Selfless', + 'Set', 'Str', 'SubrangeGuard', 'Transparent', 'Void', +] + +_safeScope = [ + '_accumulateList', '_accumulateMap', '_auditedBy', '_bind', + '_booleanFlow', '_comparer', '_equalizer', '_iterForever', '_loop', + '_makeBytes', '_makeDouble', '_makeFinalSlot', '_makeInt', '_makeList', + '_makeMap', '_makeMessageDesc', '_makeOrderedSpace', '_makeParamDesc', + '_makeProtocolDesc', '_makeSourceSpan', '_makeString', '_makeVarSlot', + '_makeVerbFacet', '_mapExtract', '_matchSame', '_quasiMatcher', + '_slotToBinding', '_splitList', '_suchThat', '_switchFailed', + '_validateFor', 'b__quasiParser', 'eval', 'import', 'm__quasiParser', + 'makeBrandPair', 'makeLazySlot', 'safeScope', 'simple__quasiParser', +] + + +class MonteLexer(RegexLexer): + """ + Lexer for the Monte programming language. + + .. versionadded:: 2.2 + """ + name = 'Monte' + url = 'https://monte.readthedocs.io/' + aliases = ['monte'] + filenames = ['*.mt'] + + tokens = { + 'root': [ + # Comments + (r'#[^\n]*\n', Comment), + + # Docstrings + # Apologies for the non-greedy matcher here. + (r'/\*\*.*?\*/', String.Doc), + + # `var` declarations + (r'\bvar\b', Keyword.Declaration, 'var'), + + # `interface` declarations + (r'\binterface\b', Keyword.Declaration, 'interface'), + + # method declarations + (words(_methods, prefix='\\b', suffix='\\b'), + Keyword, 'method'), + + # All other declarations + (words(_declarations, prefix='\\b', suffix='\\b'), + Keyword.Declaration), + + # Keywords + (words(_keywords, prefix='\\b', suffix='\\b'), Keyword), + + # Literals + ('[+-]?0x[_0-9a-fA-F]+', Number.Hex), + (r'[+-]?[_0-9]+\.[_0-9]*([eE][+-]?[_0-9]+)?', Number.Float), + ('[+-]?[_0-9]+', Number.Integer), + ("'", String.Double, 'char'), + ('"', String.Double, 'string'), + + # Quasiliterals + ('`', String.Backtick, 'ql'), + + # Operators + (words(_operators), Operator), + + # Verb operators + (_identifier + '=', Operator.Word), + + # Safe scope constants + (words(_constants, prefix='\\b', suffix='\\b'), + Keyword.Pseudo), + + # Safe scope guards + (words(_guards, prefix='\\b', suffix='\\b'), Keyword.Type), + + # All other safe scope names + (words(_safeScope, prefix='\\b', suffix='\\b'), + Name.Builtin), + + # Identifiers + (_identifier, Name), + + # Punctuation + (r'\(|\)|\{|\}|\[|\]|:|,', Punctuation), + + # Whitespace + (' +', Whitespace), + + # Definite lexer errors + ('=', Error), + ], + 'char': [ + # It is definitely an error to have a char of width == 0. + ("'", Error, 'root'), + (_escape_pattern, String.Escape, 'charEnd'), + ('.', String.Char, 'charEnd'), + ], + 'charEnd': [ + ("'", String.Char, '#pop:2'), + # It is definitely an error to have a char of width > 1. + ('.', Error), + ], + # The state of things coming into an interface. + 'interface': [ + (' +', Whitespace), + (_identifier, Name.Class, '#pop'), + include('root'), + ], + # The state of things coming into a method. + 'method': [ + (' +', Whitespace), + (_identifier, Name.Function, '#pop'), + include('root'), + ], + 'string': [ + ('"', String.Double, 'root'), + (_escape_pattern, String.Escape), + (r'\n', String.Double), + ('.', String.Double), + ], + 'ql': [ + ('`', String.Backtick, 'root'), + (r'\$' + _escape_pattern, String.Escape), + (r'\$\$', String.Escape), + (r'@@', String.Escape), + (r'\$\{', String.Interpol, 'qlNest'), + (r'@\{', String.Interpol, 'qlNest'), + (r'\$' + _identifier, Name), + ('@' + _identifier, Name), + ('.', String.Backtick), + ], + 'qlNest': [ + (r'\}', String.Interpol, '#pop'), + include('root'), + ], + # The state of things immediately following `var`. + 'var': [ + (' +', Whitespace), + (_identifier, Name.Variable, '#pop'), + include('root'), + ], + } diff --git a/pygments/lexers/mosel.py b/pygments/lexers/mosel.py new file mode 100644 index 0000000..1214ba4 --- /dev/null +++ b/pygments/lexers/mosel.py @@ -0,0 +1,447 @@ +""" + pygments.lexers.mosel + ~~~~~~~~~~~~~~~~~~~~~ + + Lexers for the mosel language. + http://www.fico.com/en/products/fico-xpress-optimization + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, words +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation + +__all__ = ['MoselLexer'] + +FUNCTIONS = ( + # core functions + '_', + 'abs', + 'arctan', + 'asproc', + 'assert', + 'bitflip', + 'bitneg', + 'bitset', + 'bitshift', + 'bittest', + 'bitval', + 'ceil', + 'cos', + 'create', + 'currentdate', + 'currenttime', + 'cutelt', + 'cutfirst', + 'cuthead', + 'cutlast', + 'cuttail', + 'datablock', + 'delcell', + 'exists', + 'exit', + 'exp', + 'exportprob', + 'fclose', + 'fflush', + 'finalize', + 'findfirst', + 'findlast', + 'floor', + 'fopen', + 'fselect', + 'fskipline', + 'fwrite', + 'fwrite_', + 'fwriteln', + 'fwriteln_', + 'getact', + 'getcoeff', + 'getcoeffs', + 'getdual', + 'getelt', + 'getfid', + 'getfirst', + 'getfname', + 'gethead', + 'getlast', + 'getobjval', + 'getparam', + 'getrcost', + 'getreadcnt', + 'getreverse', + 'getsize', + 'getslack', + 'getsol', + 'gettail', + 'gettype', + 'getvars', + 'isdynamic', + 'iseof', + 'isfinite', + 'ishidden', + 'isinf', + 'isnan', + 'isodd', + 'ln', + 'localsetparam', + 'log', + 'makesos1', + 'makesos2', + 'maxlist', + 'memoryuse', + 'minlist', + 'newmuid', + 'publish', + 'random', + 'read', + 'readln', + 'reset', + 'restoreparam', + 'reverse', + 'round', + 'setcoeff', + 'sethidden', + 'setioerr', + 'setmatherr', + 'setname', + 'setparam', + 'setrandseed', + 'setrange', + 'settype', + 'sin', + 'splithead', + 'splittail', + 'sqrt', + 'strfmt', + 'substr', + 'timestamp', + 'unpublish', + 'versionnum', + 'versionstr', + 'write', + 'write_', + 'writeln', + 'writeln_', + + # mosel exam mmxprs | sed -n -e "s/ [pf][a-z]* \([a-zA-Z0-9_]*\).*/'\1',/p" | sort -u + 'addcut', + 'addcuts', + 'addmipsol', + 'basisstability', + 'calcsolinfo', + 'clearmipdir', + 'clearmodcut', + 'command', + 'copysoltoinit', + 'crossoverlpsol', + 'defdelayedrows', + 'defsecurevecs', + 'delcuts', + 'dropcuts', + 'estimatemarginals', + 'fixglobal', + 'flushmsgq', + 'getbstat', + 'getcnlist', + 'getcplist', + 'getdualray', + 'getiis', + 'getiissense', + 'getiistype', + 'getinfcause', + 'getinfeas', + 'getlb', + 'getlct', + 'getleft', + 'getloadedlinctrs', + 'getloadedmpvars', + 'getname', + 'getprimalray', + 'getprobstat', + 'getrange', + 'getright', + 'getsensrng', + 'getsize', + 'getsol', + 'gettype', + 'getub', + 'getvars', + 'gety', + 'hasfeature', + 'implies', + 'indicator', + 'initglobal', + 'ishidden', + 'isiisvalid', + 'isintegral', + 'loadbasis', + 'loadcuts', + 'loadlpsol', + 'loadmipsol', + 'loadprob', + 'maximise', + 'maximize', + 'minimise', + 'minimize', + 'postsolve', + 'readbasis', + 'readdirs', + 'readsol', + 'refinemipsol', + 'rejectintsol', + 'repairinfeas', + 'repairinfeas_deprec', + 'resetbasis', + 'resetiis', + 'resetsol', + 'savebasis', + 'savemipsol', + 'savesol', + 'savestate', + 'selectsol', + 'setarchconsistency', + 'setbstat', + 'setcallback', + 'setcbcutoff', + 'setgndata', + 'sethidden', + 'setlb', + 'setmipdir', + 'setmodcut', + 'setsol', + 'setub', + 'setucbdata', + 'stopoptimise', + 'stopoptimize', + 'storecut', + 'storecuts', + 'unloadprob', + 'uselastbarsol', + 'writebasis', + 'writedirs', + 'writeprob', + 'writesol', + 'xor', + 'xprs_addctr', + 'xprs_addindic', + + # mosel exam mmsystem | sed -n -e "s/ [pf][a-z]* \([a-zA-Z0-9_]*\).*/'\1',/p" | sort -u + 'addmonths', + 'copytext', + 'cuttext', + 'deltext', + 'endswith', + 'erase', + 'expandpath', + 'fcopy', + 'fdelete', + 'findfiles', + 'findtext', + 'fmove', + 'formattext', + 'getasnumber', + 'getchar', + 'getcwd', + 'getdate', + 'getday', + 'getdaynum', + 'getdays', + 'getdirsep', + 'getdsoparam', + 'getendparse', + 'getenv', + 'getfsize', + 'getfstat', + 'getftime', + 'gethour', + 'getminute', + 'getmonth', + 'getmsec', + 'getoserrmsg', + 'getoserror', + 'getpathsep', + 'getqtype', + 'getsecond', + 'getsepchar', + 'getsize', + 'getstart', + 'getsucc', + 'getsysinfo', + 'getsysstat', + 'gettime', + 'gettmpdir', + 'gettrim', + 'getweekday', + 'getyear', + 'inserttext', + 'isvalid', + 'jointext', + 'makedir', + 'makepath', + 'newtar', + 'newzip', + 'nextfield', + 'openpipe', + 'parseextn', + 'parseint', + 'parsereal', + 'parsetext', + 'pastetext', + 'pathmatch', + 'pathsplit', + 'qsort', + 'quote', + 'readtextline', + 'regmatch', + 'regreplace', + 'removedir', + 'removefiles', + 'setchar', + 'setdate', + 'setday', + 'setdsoparam', + 'setendparse', + 'setenv', + 'sethour', + 'setminute', + 'setmonth', + 'setmsec', + 'setoserror', + 'setqtype', + 'setsecond', + 'setsepchar', + 'setstart', + 'setsucc', + 'settime', + 'settrim', + 'setyear', + 'sleep', + 'splittext', + 'startswith', + 'system', + 'tarlist', + 'textfmt', + 'tolower', + 'toupper', + 'trim', + 'untar', + 'unzip', + 'ziplist', + + # mosel exam mmjobs | sed -n -e "s/ [pf][a-z]* \([a-zA-Z0-9_]*\).*/'\1',/p" | sort -u + 'canceltimer', + 'clearaliases', + 'compile', + 'connect', + 'detach', + 'disconnect', + 'dropnextevent', + 'findxsrvs', + 'getaliases', + 'getannidents', + 'getannotations', + 'getbanner', + 'getclass', + 'getdsoprop', + 'getdsopropnum', + 'getexitcode', + 'getfromgid', + 'getfromid', + 'getfromuid', + 'getgid', + 'gethostalias', + 'getid', + 'getmodprop', + 'getmodpropnum', + 'getnextevent', + 'getnode', + 'getrmtid', + 'getstatus', + 'getsysinfo', + 'gettimer', + 'getuid', + 'getvalue', + 'isqueueempty', + 'load', + 'nullevent', + 'peeknextevent', + 'resetmodpar', + 'run', + 'send', + 'setcontrol', + 'setdefstream', + 'setgid', + 'sethostalias', + 'setmodpar', + 'settimer', + 'setuid', + 'setworkdir', + 'stop', + 'unload', + 'wait', + 'waitexpired', + 'waitfor', + 'waitforend', +) + + +class MoselLexer(RegexLexer): + """ + For the Mosel optimization language. + + .. versionadded:: 2.6 + """ + name = 'Mosel' + aliases = ['mosel'] + filenames = ['*.mos'] + + tokens = { + 'root': [ + (r'\n', Text), + (r'\s+', Text.Whitespace), + (r'!.*?\n', Comment.Single), + (r'\(!(.|\n)*?!\)', Comment.Multiline), + (words(( + 'and', 'as', 'break', 'case', 'count', 'declarations', 'do', + 'dynamic', 'elif', 'else', 'end-', 'end', 'evaluation', 'false', + 'forall', 'forward', 'from', 'function', 'hashmap', 'if', + 'imports', 'include', 'initialisations', 'initializations', 'inter', + 'max', 'min', 'model', 'namespace', 'next', 'not', 'nsgroup', + 'nssearch', 'of', 'options', 'or', 'package', 'parameters', + 'procedure', 'public', 'prod', 'record', 'repeat', 'requirements', + 'return', 'sum', 'then', 'to', 'true', 'union', 'until', 'uses', + 'version', 'while', 'with'), prefix=r'\b', suffix=r'\b'), + Keyword.Builtin), + (words(( + 'range', 'array', 'set', 'list', 'mpvar', 'mpproblem', 'linctr', + 'nlctr', 'integer', 'string', 'real', 'boolean', 'text', 'time', + 'date', 'datetime', 'returned', 'Model', 'Mosel', 'counter', + 'xmldoc', 'is_sos1', 'is_sos2', 'is_integer', 'is_binary', + 'is_continuous', 'is_free', 'is_semcont', 'is_semint', + 'is_partint'), prefix=r'\b', suffix=r'\b'), + Keyword.Type), + (r'(\+|\-|\*|/|=|<=|>=|\||\^|<|>|<>|\.\.|\.|:=|::|:|in|mod|div)', + Operator), + (r'[()\[\]{},;]+', Punctuation), + (words(FUNCTIONS, prefix=r'\b', suffix=r'\b'), Name.Function), + (r'(\d+\.(?!\.)\d*|\.(?!.)\d+)([eE][+-]?\d+)?', Number.Float), + (r'\d+([eE][+-]?\d+)?', Number.Integer), + (r'[+-]?Infinity', Number.Integer), + (r'0[xX][0-9a-fA-F]+', Number), + (r'"', String.Double, 'double_quote'), + (r'\'', String.Single, 'single_quote'), + (r'(\w+|(\.(?!\.)))', Text), + ], + 'single_quote': [ + (r'\'', String.Single, '#pop'), + (r'[^\']+', String.Single), + ], + 'double_quote': [ + (r'(\\"|\\[0-7]{1,3}\D|\\[abfnrtv]|\\\\)', String.Escape), + (r'\"', String.Double, '#pop'), + (r'[^"\\]+', String.Double), + ], + } diff --git a/pygments/lexers/ncl.py b/pygments/lexers/ncl.py new file mode 100644 index 0000000..30cc510 --- /dev/null +++ b/pygments/lexers/ncl.py @@ -0,0 +1,893 @@ +""" + pygments.lexers.ncl + ~~~~~~~~~~~~~~~~~~~ + + Lexers for NCAR Command Language. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, include, words +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation + +__all__ = ['NCLLexer'] + + +class NCLLexer(RegexLexer): + """ + Lexer for NCL code. + + .. versionadded:: 2.2 + """ + name = 'NCL' + aliases = ['ncl'] + filenames = ['*.ncl'] + mimetypes = ['text/ncl'] + flags = re.MULTILINE + + tokens = { + 'root': [ + (r';.*\n', Comment), + include('strings'), + include('core'), + (r'[a-zA-Z_]\w*', Name), + include('nums'), + (r'[\s]+', Text), + ], + 'core': [ + # Statements + (words(( + 'begin', 'break', 'continue', 'create', 'defaultapp', 'do', + 'else', 'end', 'external', 'exit', 'True', 'False', 'file', 'function', + 'getvalues', 'graphic', 'group', 'if', 'list', 'load', 'local', + 'new', '_Missing', 'Missing', 'noparent', 'procedure', + 'quit', 'QUIT', 'Quit', 'record', 'return', 'setvalues', 'stop', + 'then', 'while'), prefix=r'\b', suffix=r'\s*\b'), + Keyword), + + # Data Types + (words(( + 'ubyte', 'uint', 'uint64', 'ulong', 'string', 'byte', + 'character', 'double', 'float', 'integer', 'int64', 'logical', + 'long', 'short', 'ushort', 'enumeric', 'numeric', 'snumeric'), + prefix=r'\b', suffix=r'\s*\b'), + Keyword.Type), + + # Operators + (r'[\%^*+\-/<>]', Operator), + + # punctuation: + (r'[\[\]():@$!&|.,\\{}]', Punctuation), + (r'[=:]', Punctuation), + + # Intrinsics + (words(( + 'abs', 'acos', 'addfile', 'addfiles', 'all', 'angmom_atm', 'any', + 'area_conserve_remap', 'area_hi2lores', 'area_poly_sphere', + 'asciiread', 'asciiwrite', 'asin', 'atan', 'atan2', 'attsetvalues', + 'avg', 'betainc', 'bin_avg', 'bin_sum', 'bw_bandpass_filter', + 'cancor', 'cbinread', 'cbinwrite', 'cd_calendar', 'cd_inv_calendar', + 'cdfbin_p', 'cdfbin_pr', 'cdfbin_s', 'cdfbin_xn', 'cdfchi_p', + 'cdfchi_x', 'cdfgam_p', 'cdfgam_x', 'cdfnor_p', 'cdfnor_x', + 'cdft_p', 'cdft_t', 'ceil', 'center_finite_diff', + 'center_finite_diff_n', 'cfftb', 'cfftf', 'cfftf_frq_reorder', + 'charactertodouble', 'charactertofloat', 'charactertointeger', + 'charactertolong', 'charactertoshort', 'charactertostring', + 'chartodouble', 'chartofloat', 'chartoint', 'chartointeger', + 'chartolong', 'chartoshort', 'chartostring', 'chiinv', 'clear', + 'color_index_to_rgba', 'conform', 'conform_dims', 'cos', 'cosh', + 'count_unique_values', 'covcorm', 'covcorm_xy', 'craybinnumrec', + 'craybinrecread', 'create_graphic', 'csa1', 'csa1d', 'csa1s', + 'csa1x', 'csa1xd', 'csa1xs', 'csa2', 'csa2d', 'csa2l', 'csa2ld', + 'csa2ls', 'csa2lx', 'csa2lxd', 'csa2lxs', 'csa2s', 'csa2x', + 'csa2xd', 'csa2xs', 'csa3', 'csa3d', 'csa3l', 'csa3ld', 'csa3ls', + 'csa3lx', 'csa3lxd', 'csa3lxs', 'csa3s', 'csa3x', 'csa3xd', + 'csa3xs', 'csc2s', 'csgetp', 'css2c', 'cssetp', 'cssgrid', 'csstri', + 'csvoro', 'cumsum', 'cz2ccm', 'datatondc', 'day_of_week', + 'day_of_year', 'days_in_month', 'default_fillvalue', 'delete', + 'depth_to_pres', 'destroy', 'determinant', 'dewtemp_trh', + 'dgeevx_lapack', 'dim_acumrun_n', 'dim_avg', 'dim_avg_n', + 'dim_avg_wgt', 'dim_avg_wgt_n', 'dim_cumsum', 'dim_cumsum_n', + 'dim_gamfit_n', 'dim_gbits', 'dim_max', 'dim_max_n', 'dim_median', + 'dim_median_n', 'dim_min', 'dim_min_n', 'dim_num', 'dim_num_n', + 'dim_numrun_n', 'dim_pqsort', 'dim_pqsort_n', 'dim_product', + 'dim_product_n', 'dim_rmsd', 'dim_rmsd_n', 'dim_rmvmean', + 'dim_rmvmean_n', 'dim_rmvmed', 'dim_rmvmed_n', 'dim_spi_n', + 'dim_standardize', 'dim_standardize_n', 'dim_stat4', 'dim_stat4_n', + 'dim_stddev', 'dim_stddev_n', 'dim_sum', 'dim_sum_n', 'dim_sum_wgt', + 'dim_sum_wgt_n', 'dim_variance', 'dim_variance_n', 'dimsizes', + 'doubletobyte', 'doubletochar', 'doubletocharacter', + 'doubletofloat', 'doubletoint', 'doubletointeger', 'doubletolong', + 'doubletoshort', 'dpres_hybrid_ccm', 'dpres_plevel', 'draw', + 'draw_color_palette', 'dsgetp', 'dsgrid2', 'dsgrid2d', 'dsgrid2s', + 'dsgrid3', 'dsgrid3d', 'dsgrid3s', 'dspnt2', 'dspnt2d', 'dspnt2s', + 'dspnt3', 'dspnt3d', 'dspnt3s', 'dssetp', 'dtrend', 'dtrend_msg', + 'dtrend_msg_n', 'dtrend_n', 'dtrend_quadratic', + 'dtrend_quadratic_msg_n', 'dv2uvf', 'dv2uvg', 'dz_height', + 'echo_off', 'echo_on', 'eof2data', 'eof_varimax', 'eofcor', + 'eofcor_pcmsg', 'eofcor_ts', 'eofcov', 'eofcov_pcmsg', 'eofcov_ts', + 'eofunc', 'eofunc_ts', 'eofunc_varimax', 'equiv_sample_size', 'erf', + 'erfc', 'esacr', 'esacv', 'esccr', 'esccv', 'escorc', 'escorc_n', + 'escovc', 'exit', 'exp', 'exp_tapersh', 'exp_tapersh_wgts', + 'exp_tapershC', 'ezfftb', 'ezfftb_n', 'ezfftf', 'ezfftf_n', + 'f2fosh', 'f2foshv', 'f2fsh', 'f2fshv', 'f2gsh', 'f2gshv', 'fabs', + 'fbindirread', 'fbindirwrite', 'fbinnumrec', 'fbinread', + 'fbinrecread', 'fbinrecwrite', 'fbinwrite', 'fft2db', 'fft2df', + 'fftshift', 'fileattdef', 'filechunkdimdef', 'filedimdef', + 'fileexists', 'filegrpdef', 'filevarattdef', 'filevarchunkdef', + 'filevarcompressleveldef', 'filevardef', 'filevardimsizes', + 'filwgts_lancos', 'filwgts_lanczos', 'filwgts_normal', + 'floattobyte', 'floattochar', 'floattocharacter', 'floattoint', + 'floattointeger', 'floattolong', 'floattoshort', 'floor', + 'fluxEddy', 'fo2fsh', 'fo2fshv', 'fourier_info', 'frame', 'fspan', + 'ftcurv', 'ftcurvd', 'ftcurvi', 'ftcurvp', 'ftcurvpi', 'ftcurvps', + 'ftcurvs', 'ftest', 'ftgetp', 'ftkurv', 'ftkurvd', 'ftkurvp', + 'ftkurvpd', 'ftsetp', 'ftsurf', 'g2fsh', 'g2fshv', 'g2gsh', + 'g2gshv', 'gamma', 'gammainc', 'gaus', 'gaus_lobat', + 'gaus_lobat_wgt', 'gc_aangle', 'gc_clkwise', 'gc_dangle', + 'gc_inout', 'gc_latlon', 'gc_onarc', 'gc_pnt2gc', 'gc_qarea', + 'gc_tarea', 'generate_2d_array', 'get_color_index', + 'get_color_rgba', 'get_cpu_time', 'get_isolines', 'get_ncl_version', + 'get_script_name', 'get_script_prefix_name', 'get_sphere_radius', + 'get_unique_values', 'getbitsone', 'getenv', 'getfiledimsizes', + 'getfilegrpnames', 'getfilepath', 'getfilevaratts', + 'getfilevarchunkdimsizes', 'getfilevardims', 'getfilevardimsizes', + 'getfilevarnames', 'getfilevartypes', 'getvaratts', 'getvardims', + 'gradsf', 'gradsg', 'greg2jul', 'grid2triple', 'hlsrgb', 'hsvrgb', + 'hydro', 'hyi2hyo', 'idsfft', 'igradsf', 'igradsg', 'ilapsf', + 'ilapsg', 'ilapvf', 'ilapvg', 'ind', 'ind_resolve', 'int2p', + 'int2p_n', 'integertobyte', 'integertochar', 'integertocharacter', + 'integertoshort', 'inttobyte', 'inttochar', 'inttoshort', + 'inverse_matrix', 'isatt', 'isbigendian', 'isbyte', 'ischar', + 'iscoord', 'isdefined', 'isdim', 'isdimnamed', 'isdouble', + 'isenumeric', 'isfile', 'isfilepresent', 'isfilevar', + 'isfilevaratt', 'isfilevarcoord', 'isfilevardim', 'isfloat', + 'isfunc', 'isgraphic', 'isint', 'isint64', 'isinteger', + 'isleapyear', 'islogical', 'islong', 'ismissing', 'isnan_ieee', + 'isnumeric', 'ispan', 'isproc', 'isshort', 'issnumeric', 'isstring', + 'isubyte', 'isuint', 'isuint64', 'isulong', 'isunlimited', + 'isunsigned', 'isushort', 'isvar', 'jul2greg', 'kmeans_as136', + 'kolsm2_n', 'kron_product', 'lapsf', 'lapsg', 'lapvf', 'lapvg', + 'latlon2utm', 'lclvl', 'lderuvf', 'lderuvg', 'linint1', 'linint1_n', + 'linint2', 'linint2_points', 'linmsg', 'linmsg_n', 'linrood_latwgt', + 'linrood_wgt', 'list_files', 'list_filevars', 'list_hlus', + 'list_procfuncs', 'list_vars', 'ListAppend', 'ListCount', + 'ListGetType', 'ListIndex', 'ListIndexFromName', 'ListPop', + 'ListPush', 'ListSetType', 'loadscript', 'local_max', 'local_min', + 'log', 'log10', 'longtobyte', 'longtochar', 'longtocharacter', + 'longtoint', 'longtointeger', 'longtoshort', 'lspoly', 'lspoly_n', + 'mask', 'max', 'maxind', 'min', 'minind', 'mixed_layer_depth', + 'mixhum_ptd', 'mixhum_ptrh', 'mjo_cross_coh2pha', + 'mjo_cross_segment', 'moc_globe_atl', 'monthday', 'natgrid', + 'natgridd', 'natgrids', 'ncargpath', 'ncargversion', 'ndctodata', + 'ndtooned', 'new', 'NewList', 'ngezlogo', 'nggcog', 'nggetp', + 'nglogo', 'ngsetp', 'NhlAddAnnotation', 'NhlAddData', + 'NhlAddOverlay', 'NhlAddPrimitive', 'NhlAppGetDefaultParentId', + 'NhlChangeWorkstation', 'NhlClassName', 'NhlClearWorkstation', + 'NhlDataPolygon', 'NhlDataPolyline', 'NhlDataPolymarker', + 'NhlDataToNDC', 'NhlDestroy', 'NhlDraw', 'NhlFrame', 'NhlFreeColor', + 'NhlGetBB', 'NhlGetClassResources', 'NhlGetErrorObjectId', + 'NhlGetNamedColorIndex', 'NhlGetParentId', + 'NhlGetParentWorkstation', 'NhlGetWorkspaceObjectId', + 'NhlIsAllocatedColor', 'NhlIsApp', 'NhlIsDataComm', 'NhlIsDataItem', + 'NhlIsDataSpec', 'NhlIsTransform', 'NhlIsView', 'NhlIsWorkstation', + 'NhlName', 'NhlNDCPolygon', 'NhlNDCPolyline', 'NhlNDCPolymarker', + 'NhlNDCToData', 'NhlNewColor', 'NhlNewDashPattern', 'NhlNewMarker', + 'NhlPalGetDefined', 'NhlRemoveAnnotation', 'NhlRemoveData', + 'NhlRemoveOverlay', 'NhlRemovePrimitive', 'NhlSetColor', + 'NhlSetDashPattern', 'NhlSetMarker', 'NhlUpdateData', + 'NhlUpdateWorkstation', 'nice_mnmxintvl', 'nngetaspectd', + 'nngetaspects', 'nngetp', 'nngetsloped', 'nngetslopes', 'nngetwts', + 'nngetwtsd', 'nnpnt', 'nnpntd', 'nnpntend', 'nnpntendd', + 'nnpntinit', 'nnpntinitd', 'nnpntinits', 'nnpnts', 'nnsetp', 'num', + 'obj_anal_ic', 'omega_ccm', 'onedtond', 'overlay', 'paleo_outline', + 'pdfxy_bin', 'poisson_grid_fill', 'pop_remap', 'potmp_insitu_ocn', + 'prcwater_dp', 'pres2hybrid', 'pres_hybrid_ccm', 'pres_sigma', + 'print', 'print_table', 'printFileVarSummary', 'printVarSummary', + 'product', 'pslec', 'pslhor', 'pslhyp', 'qsort', 'rand', + 'random_chi', 'random_gamma', 'random_normal', 'random_setallseed', + 'random_uniform', 'rcm2points', 'rcm2rgrid', 'rdsstoi', + 'read_colormap_file', 'reg_multlin', 'regcoef', 'regCoef_n', + 'regline', 'relhum', 'replace_ieeenan', 'reshape', 'reshape_ind', + 'rgba_to_color_index', 'rgbhls', 'rgbhsv', 'rgbyiq', 'rgrid2rcm', + 'rhomb_trunc', 'rip_cape_2d', 'rip_cape_3d', 'round', 'rtest', + 'runave', 'runave_n', 'set_default_fillvalue', 'set_sphere_radius', + 'setfileoption', 'sfvp2uvf', 'sfvp2uvg', 'shaec', 'shagc', + 'shgetnp', 'shgetp', 'shgrid', 'shorttobyte', 'shorttochar', + 'shorttocharacter', 'show_ascii', 'shsec', 'shsetp', 'shsgc', + 'shsgc_R42', 'sigma2hybrid', 'simpeq', 'simpne', 'sin', + 'sindex_yrmo', 'sinh', 'sizeof', 'sleep', 'smth9', 'snindex_yrmo', + 'solve_linsys', 'span_color_indexes', 'span_color_rgba', + 'sparse_matrix_mult', 'spcorr', 'spcorr_n', 'specx_anal', + 'specxy_anal', 'spei', 'sprintf', 'sprinti', 'sqrt', 'sqsort', + 'srand', 'stat2', 'stat4', 'stat_medrng', 'stat_trim', + 'status_exit', 'stdatmus_p2tdz', 'stdatmus_z2tdp', 'stddev', + 'str_capital', 'str_concat', 'str_fields_count', 'str_get_cols', + 'str_get_dq', 'str_get_field', 'str_get_nl', 'str_get_sq', + 'str_get_tab', 'str_index_of_substr', 'str_insert', 'str_is_blank', + 'str_join', 'str_left_strip', 'str_lower', 'str_match', + 'str_match_ic', 'str_match_ic_regex', 'str_match_ind', + 'str_match_ind_ic', 'str_match_ind_ic_regex', 'str_match_ind_regex', + 'str_match_regex', 'str_right_strip', 'str_split', + 'str_split_by_length', 'str_split_csv', 'str_squeeze', 'str_strip', + 'str_sub_str', 'str_switch', 'str_upper', 'stringtochar', + 'stringtocharacter', 'stringtodouble', 'stringtofloat', + 'stringtoint', 'stringtointeger', 'stringtolong', 'stringtoshort', + 'strlen', 'student_t', 'sum', 'svd_lapack', 'svdcov', 'svdcov_sv', + 'svdstd', 'svdstd_sv', 'system', 'systemfunc', 'tan', 'tanh', + 'taper', 'taper_n', 'tdclrs', 'tdctri', 'tdcudp', 'tdcurv', + 'tddtri', 'tdez2d', 'tdez3d', 'tdgetp', 'tdgrds', 'tdgrid', + 'tdgtrs', 'tdinit', 'tditri', 'tdlbla', 'tdlblp', 'tdlbls', + 'tdline', 'tdlndp', 'tdlnpa', 'tdlpdp', 'tdmtri', 'tdotri', + 'tdpara', 'tdplch', 'tdprpa', 'tdprpi', 'tdprpt', 'tdsetp', + 'tdsort', 'tdstri', 'tdstrs', 'tdttri', 'thornthwaite', 'tobyte', + 'tochar', 'todouble', 'tofloat', 'toint', 'toint64', 'tointeger', + 'tolong', 'toshort', 'tosigned', 'tostring', 'tostring_with_format', + 'totype', 'toubyte', 'touint', 'touint64', 'toulong', 'tounsigned', + 'toushort', 'trend_manken', 'tri_trunc', 'triple2grid', + 'triple2grid2d', 'trop_wmo', 'ttest', 'typeof', 'undef', + 'unique_string', 'update', 'ushorttoint', 'ut_calendar', + 'ut_inv_calendar', 'utm2latlon', 'uv2dv_cfd', 'uv2dvf', 'uv2dvg', + 'uv2sfvpf', 'uv2sfvpg', 'uv2vr_cfd', 'uv2vrdvf', 'uv2vrdvg', + 'uv2vrf', 'uv2vrg', 'v5d_close', 'v5d_create', 'v5d_setLowLev', + 'v5d_setUnits', 'v5d_write', 'v5d_write_var', 'variance', 'vhaec', + 'vhagc', 'vhsec', 'vhsgc', 'vibeta', 'vinth2p', 'vinth2p_ecmwf', + 'vinth2p_ecmwf_nodes', 'vinth2p_nodes', 'vintp2p_ecmwf', 'vr2uvf', + 'vr2uvg', 'vrdv2uvf', 'vrdv2uvg', 'wavelet', 'wavelet_default', + 'weibull', 'wgt_area_smooth', 'wgt_areaave', 'wgt_areaave2', + 'wgt_arearmse', 'wgt_arearmse2', 'wgt_areasum2', 'wgt_runave', + 'wgt_runave_n', 'wgt_vert_avg_beta', 'wgt_volave', 'wgt_volave_ccm', + 'wgt_volrmse', 'wgt_volrmse_ccm', 'where', 'wk_smooth121', 'wmbarb', + 'wmbarbmap', 'wmdrft', 'wmgetp', 'wmlabs', 'wmsetp', 'wmstnm', + 'wmvect', 'wmvectmap', 'wmvlbl', 'wrf_avo', 'wrf_cape_2d', + 'wrf_cape_3d', 'wrf_dbz', 'wrf_eth', 'wrf_helicity', 'wrf_ij_to_ll', + 'wrf_interp_1d', 'wrf_interp_2d_xy', 'wrf_interp_3d_z', + 'wrf_latlon_to_ij', 'wrf_ll_to_ij', 'wrf_omega', 'wrf_pvo', + 'wrf_rh', 'wrf_slp', 'wrf_smooth_2d', 'wrf_td', 'wrf_tk', + 'wrf_updraft_helicity', 'wrf_uvmet', 'wrf_virtual_temp', + 'wrf_wetbulb', 'wrf_wps_close_int', 'wrf_wps_open_int', + 'wrf_wps_rddata_int', 'wrf_wps_rdhead_int', 'wrf_wps_read_int', + 'wrf_wps_write_int', 'write_matrix', 'write_table', 'yiqrgb', + 'z2geouv', 'zonal_mpsi', 'addfiles_GetVar', 'advect_variable', + 'area_conserve_remap_Wrap', 'area_hi2lores_Wrap', + 'array_append_record', 'assignFillValue', 'byte2flt', + 'byte2flt_hdf', 'calcDayAnomTLL', 'calcMonAnomLLLT', + 'calcMonAnomLLT', 'calcMonAnomTLL', 'calcMonAnomTLLL', + 'calculate_monthly_values', 'cd_convert', 'changeCase', + 'changeCaseChar', 'clmDayTLL', 'clmDayTLLL', 'clmMon2clmDay', + 'clmMonLLLT', 'clmMonLLT', 'clmMonTLL', 'clmMonTLLL', 'closest_val', + 'copy_VarAtts', 'copy_VarCoords', 'copy_VarCoords_1', + 'copy_VarCoords_2', 'copy_VarMeta', 'copyatt', 'crossp3', + 'cshstringtolist', 'cssgrid_Wrap', 'dble2flt', 'decimalPlaces', + 'delete_VarAtts', 'dim_avg_n_Wrap', 'dim_avg_wgt_n_Wrap', + 'dim_avg_wgt_Wrap', 'dim_avg_Wrap', 'dim_cumsum_n_Wrap', + 'dim_cumsum_Wrap', 'dim_max_n_Wrap', 'dim_min_n_Wrap', + 'dim_rmsd_n_Wrap', 'dim_rmsd_Wrap', 'dim_rmvmean_n_Wrap', + 'dim_rmvmean_Wrap', 'dim_rmvmed_n_Wrap', 'dim_rmvmed_Wrap', + 'dim_standardize_n_Wrap', 'dim_standardize_Wrap', + 'dim_stddev_n_Wrap', 'dim_stddev_Wrap', 'dim_sum_n_Wrap', + 'dim_sum_wgt_n_Wrap', 'dim_sum_wgt_Wrap', 'dim_sum_Wrap', + 'dim_variance_n_Wrap', 'dim_variance_Wrap', 'dpres_plevel_Wrap', + 'dtrend_leftdim', 'dv2uvF_Wrap', 'dv2uvG_Wrap', 'eof_north', + 'eofcor_Wrap', 'eofcov_Wrap', 'eofunc_north', 'eofunc_ts_Wrap', + 'eofunc_varimax_reorder', 'eofunc_varimax_Wrap', 'eofunc_Wrap', + 'epsZero', 'f2fosh_Wrap', 'f2foshv_Wrap', 'f2fsh_Wrap', + 'f2fshv_Wrap', 'f2gsh_Wrap', 'f2gshv_Wrap', 'fbindirSwap', + 'fbinseqSwap1', 'fbinseqSwap2', 'flt2dble', 'flt2string', + 'fo2fsh_Wrap', 'fo2fshv_Wrap', 'g2fsh_Wrap', 'g2fshv_Wrap', + 'g2gsh_Wrap', 'g2gshv_Wrap', 'generate_resample_indices', + 'generate_sample_indices', 'generate_unique_indices', + 'genNormalDist', 'get1Dindex', 'get1Dindex_Collapse', + 'get1Dindex_Exclude', 'get_file_suffix', 'GetFillColor', + 'GetFillColorIndex', 'getFillValue', 'getind_latlon2d', + 'getVarDimNames', 'getVarFillValue', 'grib_stime2itime', + 'hyi2hyo_Wrap', 'ilapsF_Wrap', 'ilapsG_Wrap', 'ind_nearest_coord', + 'indStrSubset', 'int2dble', 'int2flt', 'int2p_n_Wrap', 'int2p_Wrap', + 'isMonotonic', 'isStrSubset', 'latGau', 'latGauWgt', 'latGlobeF', + 'latGlobeFo', 'latRegWgt', 'linint1_n_Wrap', 'linint1_Wrap', + 'linint2_points_Wrap', 'linint2_Wrap', 'local_max_1d', + 'local_min_1d', 'lonFlip', 'lonGlobeF', 'lonGlobeFo', 'lonPivot', + 'merge_levels_sfc', 'mod', 'month_to_annual', + 'month_to_annual_weighted', 'month_to_season', 'month_to_season12', + 'month_to_seasonN', 'monthly_total_to_daily_mean', 'nameDim', + 'natgrid_Wrap', 'NewCosWeight', 'niceLatLon2D', 'NormCosWgtGlobe', + 'numAsciiCol', 'numAsciiRow', 'numeric2int', + 'obj_anal_ic_deprecated', 'obj_anal_ic_Wrap', 'omega_ccm_driver', + 'omega_to_w', 'oneDtostring', 'pack_values', 'pattern_cor', 'pdfx', + 'pdfxy', 'pdfxy_conform', 'pot_temp', 'pot_vort_hybrid', + 'pot_vort_isobaric', 'pres2hybrid_Wrap', 'print_clock', + 'printMinMax', 'quadroots', 'rcm2points_Wrap', 'rcm2rgrid_Wrap', + 'readAsciiHead', 'readAsciiTable', 'reg_multlin_stats', + 'region_ind', 'regline_stats', 'relhum_ttd', 'replaceSingleChar', + 'RGBtoCmap', 'rgrid2rcm_Wrap', 'rho_mwjf', 'rm_single_dims', + 'rmAnnCycle1D', 'rmInsufData', 'rmMonAnnCycLLLT', 'rmMonAnnCycLLT', + 'rmMonAnnCycTLL', 'runave_n_Wrap', 'runave_Wrap', 'short2flt', + 'short2flt_hdf', 'shsgc_R42_Wrap', 'sign_f90', 'sign_matlab', + 'smth9_Wrap', 'smthClmDayTLL', 'smthClmDayTLLL', 'SqrtCosWeight', + 'stat_dispersion', 'static_stability', 'stdMonLLLT', 'stdMonLLT', + 'stdMonTLL', 'stdMonTLLL', 'symMinMaxPlt', 'table_attach_columns', + 'table_attach_rows', 'time_to_newtime', 'transpose', + 'triple2grid_Wrap', 'ut_convert', 'uv2dvF_Wrap', 'uv2dvG_Wrap', + 'uv2vrF_Wrap', 'uv2vrG_Wrap', 'vr2uvF_Wrap', 'vr2uvG_Wrap', + 'w_to_omega', 'wallClockElapseTime', 'wave_number_spc', + 'wgt_areaave_Wrap', 'wgt_runave_leftdim', 'wgt_runave_n_Wrap', + 'wgt_runave_Wrap', 'wgt_vertical_n', 'wind_component', + 'wind_direction', 'yyyyddd_to_yyyymmdd', 'yyyymm_time', + 'yyyymm_to_yyyyfrac', 'yyyymmdd_time', 'yyyymmdd_to_yyyyddd', + 'yyyymmdd_to_yyyyfrac', 'yyyymmddhh_time', 'yyyymmddhh_to_yyyyfrac', + 'zonal_mpsi_Wrap', 'zonalAve', 'calendar_decode2', 'cd_string', + 'kf_filter', 'run_cor', 'time_axis_labels', 'ut_string', + 'wrf_contour', 'wrf_map', 'wrf_map_overlay', 'wrf_map_overlays', + 'wrf_map_resources', 'wrf_map_zoom', 'wrf_overlay', 'wrf_overlays', + 'wrf_user_getvar', 'wrf_user_ij_to_ll', 'wrf_user_intrp2d', + 'wrf_user_intrp3d', 'wrf_user_latlon_to_ij', 'wrf_user_list_times', + 'wrf_user_ll_to_ij', 'wrf_user_unstagger', 'wrf_user_vert_interp', + 'wrf_vector', 'gsn_add_annotation', 'gsn_add_polygon', + 'gsn_add_polyline', 'gsn_add_polymarker', + 'gsn_add_shapefile_polygons', 'gsn_add_shapefile_polylines', + 'gsn_add_shapefile_polymarkers', 'gsn_add_text', 'gsn_attach_plots', + 'gsn_blank_plot', 'gsn_contour', 'gsn_contour_map', + 'gsn_contour_shade', 'gsn_coordinates', 'gsn_create_labelbar', + 'gsn_create_legend', 'gsn_create_text', + 'gsn_csm_attach_zonal_means', 'gsn_csm_blank_plot', + 'gsn_csm_contour', 'gsn_csm_contour_map', 'gsn_csm_contour_map_ce', + 'gsn_csm_contour_map_overlay', 'gsn_csm_contour_map_polar', + 'gsn_csm_hov', 'gsn_csm_lat_time', 'gsn_csm_map', 'gsn_csm_map_ce', + 'gsn_csm_map_polar', 'gsn_csm_pres_hgt', + 'gsn_csm_pres_hgt_streamline', 'gsn_csm_pres_hgt_vector', + 'gsn_csm_streamline', 'gsn_csm_streamline_contour_map', + 'gsn_csm_streamline_contour_map_ce', + 'gsn_csm_streamline_contour_map_polar', 'gsn_csm_streamline_map', + 'gsn_csm_streamline_map_ce', 'gsn_csm_streamline_map_polar', + 'gsn_csm_streamline_scalar', 'gsn_csm_streamline_scalar_map', + 'gsn_csm_streamline_scalar_map_ce', + 'gsn_csm_streamline_scalar_map_polar', 'gsn_csm_time_lat', + 'gsn_csm_vector', 'gsn_csm_vector_map', 'gsn_csm_vector_map_ce', + 'gsn_csm_vector_map_polar', 'gsn_csm_vector_scalar', + 'gsn_csm_vector_scalar_map', 'gsn_csm_vector_scalar_map_ce', + 'gsn_csm_vector_scalar_map_polar', 'gsn_csm_x2y', 'gsn_csm_x2y2', + 'gsn_csm_xy', 'gsn_csm_xy2', 'gsn_csm_xy3', 'gsn_csm_y', + 'gsn_define_colormap', 'gsn_draw_colormap', 'gsn_draw_named_colors', + 'gsn_histogram', 'gsn_labelbar_ndc', 'gsn_legend_ndc', 'gsn_map', + 'gsn_merge_colormaps', 'gsn_open_wks', 'gsn_panel', 'gsn_polygon', + 'gsn_polygon_ndc', 'gsn_polyline', 'gsn_polyline_ndc', + 'gsn_polymarker', 'gsn_polymarker_ndc', 'gsn_retrieve_colormap', + 'gsn_reverse_colormap', 'gsn_streamline', 'gsn_streamline_map', + 'gsn_streamline_scalar', 'gsn_streamline_scalar_map', 'gsn_table', + 'gsn_text', 'gsn_text_ndc', 'gsn_vector', 'gsn_vector_map', + 'gsn_vector_scalar', 'gsn_vector_scalar_map', 'gsn_xy', 'gsn_y', + 'hsv2rgb', 'maximize_output', 'namedcolor2rgb', 'namedcolor2rgba', + 'reset_device_coordinates', 'span_named_colors'), prefix=r'\b'), + Name.Builtin), + + # Resources + (words(( + 'amDataXF', 'amDataYF', 'amJust', 'amOn', 'amOrthogonalPosF', + 'amParallelPosF', 'amResizeNotify', 'amSide', 'amTrackData', + 'amViewId', 'amZone', 'appDefaultParent', 'appFileSuffix', + 'appResources', 'appSysDir', 'appUsrDir', 'caCopyArrays', + 'caXArray', 'caXCast', 'caXMaxV', 'caXMinV', 'caXMissingV', + 'caYArray', 'caYCast', 'caYMaxV', 'caYMinV', 'caYMissingV', + 'cnCellFillEdgeColor', 'cnCellFillMissingValEdgeColor', + 'cnConpackParams', 'cnConstFEnableFill', 'cnConstFLabelAngleF', + 'cnConstFLabelBackgroundColor', 'cnConstFLabelConstantSpacingF', + 'cnConstFLabelFont', 'cnConstFLabelFontAspectF', + 'cnConstFLabelFontColor', 'cnConstFLabelFontHeightF', + 'cnConstFLabelFontQuality', 'cnConstFLabelFontThicknessF', + 'cnConstFLabelFormat', 'cnConstFLabelFuncCode', 'cnConstFLabelJust', + 'cnConstFLabelOn', 'cnConstFLabelOrthogonalPosF', + 'cnConstFLabelParallelPosF', 'cnConstFLabelPerimColor', + 'cnConstFLabelPerimOn', 'cnConstFLabelPerimSpaceF', + 'cnConstFLabelPerimThicknessF', 'cnConstFLabelSide', + 'cnConstFLabelString', 'cnConstFLabelTextDirection', + 'cnConstFLabelZone', 'cnConstFUseInfoLabelRes', + 'cnExplicitLabelBarLabelsOn', 'cnExplicitLegendLabelsOn', + 'cnExplicitLineLabelsOn', 'cnFillBackgroundColor', 'cnFillColor', + 'cnFillColors', 'cnFillDotSizeF', 'cnFillDrawOrder', 'cnFillMode', + 'cnFillOn', 'cnFillOpacityF', 'cnFillPalette', 'cnFillPattern', + 'cnFillPatterns', 'cnFillScaleF', 'cnFillScales', 'cnFixFillBleed', + 'cnGridBoundFillColor', 'cnGridBoundFillPattern', + 'cnGridBoundFillScaleF', 'cnGridBoundPerimColor', + 'cnGridBoundPerimDashPattern', 'cnGridBoundPerimOn', + 'cnGridBoundPerimThicknessF', 'cnHighLabelAngleF', + 'cnHighLabelBackgroundColor', 'cnHighLabelConstantSpacingF', + 'cnHighLabelCount', 'cnHighLabelFont', 'cnHighLabelFontAspectF', + 'cnHighLabelFontColor', 'cnHighLabelFontHeightF', + 'cnHighLabelFontQuality', 'cnHighLabelFontThicknessF', + 'cnHighLabelFormat', 'cnHighLabelFuncCode', 'cnHighLabelPerimColor', + 'cnHighLabelPerimOn', 'cnHighLabelPerimSpaceF', + 'cnHighLabelPerimThicknessF', 'cnHighLabelString', 'cnHighLabelsOn', + 'cnHighLowLabelOverlapMode', 'cnHighUseLineLabelRes', + 'cnInfoLabelAngleF', 'cnInfoLabelBackgroundColor', + 'cnInfoLabelConstantSpacingF', 'cnInfoLabelFont', + 'cnInfoLabelFontAspectF', 'cnInfoLabelFontColor', + 'cnInfoLabelFontHeightF', 'cnInfoLabelFontQuality', + 'cnInfoLabelFontThicknessF', 'cnInfoLabelFormat', + 'cnInfoLabelFuncCode', 'cnInfoLabelJust', 'cnInfoLabelOn', + 'cnInfoLabelOrthogonalPosF', 'cnInfoLabelParallelPosF', + 'cnInfoLabelPerimColor', 'cnInfoLabelPerimOn', + 'cnInfoLabelPerimSpaceF', 'cnInfoLabelPerimThicknessF', + 'cnInfoLabelSide', 'cnInfoLabelString', 'cnInfoLabelTextDirection', + 'cnInfoLabelZone', 'cnLabelBarEndLabelsOn', 'cnLabelBarEndStyle', + 'cnLabelDrawOrder', 'cnLabelMasking', 'cnLabelScaleFactorF', + 'cnLabelScaleValueF', 'cnLabelScalingMode', 'cnLegendLevelFlags', + 'cnLevelCount', 'cnLevelFlag', 'cnLevelFlags', 'cnLevelSelectionMode', + 'cnLevelSpacingF', 'cnLevels', 'cnLineColor', 'cnLineColors', + 'cnLineDashPattern', 'cnLineDashPatterns', 'cnLineDashSegLenF', + 'cnLineDrawOrder', 'cnLineLabelAngleF', 'cnLineLabelBackgroundColor', + 'cnLineLabelConstantSpacingF', 'cnLineLabelCount', + 'cnLineLabelDensityF', 'cnLineLabelFont', 'cnLineLabelFontAspectF', + 'cnLineLabelFontColor', 'cnLineLabelFontColors', + 'cnLineLabelFontHeightF', 'cnLineLabelFontQuality', + 'cnLineLabelFontThicknessF', 'cnLineLabelFormat', + 'cnLineLabelFuncCode', 'cnLineLabelInterval', 'cnLineLabelPerimColor', + 'cnLineLabelPerimOn', 'cnLineLabelPerimSpaceF', + 'cnLineLabelPerimThicknessF', 'cnLineLabelPlacementMode', + 'cnLineLabelStrings', 'cnLineLabelsOn', 'cnLinePalette', + 'cnLineThicknessF', 'cnLineThicknesses', 'cnLinesOn', + 'cnLowLabelAngleF', 'cnLowLabelBackgroundColor', + 'cnLowLabelConstantSpacingF', 'cnLowLabelCount', 'cnLowLabelFont', + 'cnLowLabelFontAspectF', 'cnLowLabelFontColor', + 'cnLowLabelFontHeightF', 'cnLowLabelFontQuality', + 'cnLowLabelFontThicknessF', 'cnLowLabelFormat', 'cnLowLabelFuncCode', + 'cnLowLabelPerimColor', 'cnLowLabelPerimOn', 'cnLowLabelPerimSpaceF', + 'cnLowLabelPerimThicknessF', 'cnLowLabelString', 'cnLowLabelsOn', + 'cnLowUseHighLabelRes', 'cnMaxDataValueFormat', 'cnMaxLevelCount', + 'cnMaxLevelValF', 'cnMaxPointDistanceF', 'cnMinLevelValF', + 'cnMissingValFillColor', 'cnMissingValFillPattern', + 'cnMissingValFillScaleF', 'cnMissingValPerimColor', + 'cnMissingValPerimDashPattern', 'cnMissingValPerimGridBoundOn', + 'cnMissingValPerimOn', 'cnMissingValPerimThicknessF', + 'cnMonoFillColor', 'cnMonoFillPattern', 'cnMonoFillScale', + 'cnMonoLevelFlag', 'cnMonoLineColor', 'cnMonoLineDashPattern', + 'cnMonoLineLabelFontColor', 'cnMonoLineThickness', 'cnNoDataLabelOn', + 'cnNoDataLabelString', 'cnOutOfRangeFillColor', + 'cnOutOfRangeFillPattern', 'cnOutOfRangeFillScaleF', + 'cnOutOfRangePerimColor', 'cnOutOfRangePerimDashPattern', + 'cnOutOfRangePerimOn', 'cnOutOfRangePerimThicknessF', + 'cnRasterCellSizeF', 'cnRasterMinCellSizeF', 'cnRasterModeOn', + 'cnRasterSampleFactorF', 'cnRasterSmoothingOn', 'cnScalarFieldData', + 'cnSmoothingDistanceF', 'cnSmoothingOn', 'cnSmoothingTensionF', + 'cnSpanFillPalette', 'cnSpanLinePalette', 'ctCopyTables', + 'ctXElementSize', 'ctXMaxV', 'ctXMinV', 'ctXMissingV', 'ctXTable', + 'ctXTableLengths', 'ctXTableType', 'ctYElementSize', 'ctYMaxV', + 'ctYMinV', 'ctYMissingV', 'ctYTable', 'ctYTableLengths', + 'ctYTableType', 'dcDelayCompute', 'errBuffer', + 'errFileName', 'errFilePtr', 'errLevel', 'errPrint', 'errUnitNumber', + 'gsClipOn', 'gsColors', 'gsEdgeColor', 'gsEdgeDashPattern', + 'gsEdgeDashSegLenF', 'gsEdgeThicknessF', 'gsEdgesOn', + 'gsFillBackgroundColor', 'gsFillColor', 'gsFillDotSizeF', + 'gsFillIndex', 'gsFillLineThicknessF', 'gsFillOpacityF', + 'gsFillScaleF', 'gsFont', 'gsFontAspectF', 'gsFontColor', + 'gsFontHeightF', 'gsFontOpacityF', 'gsFontQuality', + 'gsFontThicknessF', 'gsLineColor', 'gsLineDashPattern', + 'gsLineDashSegLenF', 'gsLineLabelConstantSpacingF', 'gsLineLabelFont', + 'gsLineLabelFontAspectF', 'gsLineLabelFontColor', + 'gsLineLabelFontHeightF', 'gsLineLabelFontQuality', + 'gsLineLabelFontThicknessF', 'gsLineLabelFuncCode', + 'gsLineLabelString', 'gsLineOpacityF', 'gsLineThicknessF', + 'gsMarkerColor', 'gsMarkerIndex', 'gsMarkerOpacityF', 'gsMarkerSizeF', + 'gsMarkerThicknessF', 'gsSegments', 'gsTextAngleF', + 'gsTextConstantSpacingF', 'gsTextDirection', 'gsTextFuncCode', + 'gsTextJustification', 'gsnAboveYRefLineBarColors', + 'gsnAboveYRefLineBarFillScales', 'gsnAboveYRefLineBarPatterns', + 'gsnAboveYRefLineColor', 'gsnAddCyclic', 'gsnAttachBorderOn', + 'gsnAttachPlotsXAxis', 'gsnBelowYRefLineBarColors', + 'gsnBelowYRefLineBarFillScales', 'gsnBelowYRefLineBarPatterns', + 'gsnBelowYRefLineColor', 'gsnBoxMargin', 'gsnCenterString', + 'gsnCenterStringFontColor', 'gsnCenterStringFontHeightF', + 'gsnCenterStringFuncCode', 'gsnCenterStringOrthogonalPosF', + 'gsnCenterStringParallelPosF', 'gsnContourLineThicknessesScale', + 'gsnContourNegLineDashPattern', 'gsnContourPosLineDashPattern', + 'gsnContourZeroLineThicknessF', 'gsnDebugWriteFileName', 'gsnDraw', + 'gsnFrame', 'gsnHistogramBarWidthPercent', 'gsnHistogramBinIntervals', + 'gsnHistogramBinMissing', 'gsnHistogramBinWidth', + 'gsnHistogramClassIntervals', 'gsnHistogramCompare', + 'gsnHistogramComputePercentages', + 'gsnHistogramComputePercentagesNoMissing', + 'gsnHistogramDiscreteBinValues', 'gsnHistogramDiscreteClassValues', + 'gsnHistogramHorizontal', 'gsnHistogramMinMaxBinsOn', + 'gsnHistogramNumberOfBins', 'gsnHistogramPercentSign', + 'gsnHistogramSelectNiceIntervals', 'gsnLeftString', + 'gsnLeftStringFontColor', 'gsnLeftStringFontHeightF', + 'gsnLeftStringFuncCode', 'gsnLeftStringOrthogonalPosF', + 'gsnLeftStringParallelPosF', 'gsnMajorLatSpacing', + 'gsnMajorLonSpacing', 'gsnMaskLambertConformal', + 'gsnMaskLambertConformalOutlineOn', 'gsnMaximize', + 'gsnMinorLatSpacing', 'gsnMinorLonSpacing', 'gsnPanelBottom', + 'gsnPanelCenter', 'gsnPanelDebug', 'gsnPanelFigureStrings', + 'gsnPanelFigureStringsBackgroundFillColor', + 'gsnPanelFigureStringsFontHeightF', 'gsnPanelFigureStringsJust', + 'gsnPanelFigureStringsPerimOn', 'gsnPanelLabelBar', 'gsnPanelLeft', + 'gsnPanelMainFont', 'gsnPanelMainFontColor', + 'gsnPanelMainFontHeightF', 'gsnPanelMainString', 'gsnPanelRight', + 'gsnPanelRowSpec', 'gsnPanelScalePlotIndex', 'gsnPanelTop', + 'gsnPanelXF', 'gsnPanelXWhiteSpacePercent', 'gsnPanelYF', + 'gsnPanelYWhiteSpacePercent', 'gsnPaperHeight', 'gsnPaperMargin', + 'gsnPaperOrientation', 'gsnPaperWidth', 'gsnPolar', + 'gsnPolarLabelDistance', 'gsnPolarLabelFont', + 'gsnPolarLabelFontHeightF', 'gsnPolarLabelSpacing', 'gsnPolarTime', + 'gsnPolarUT', 'gsnRightString', 'gsnRightStringFontColor', + 'gsnRightStringFontHeightF', 'gsnRightStringFuncCode', + 'gsnRightStringOrthogonalPosF', 'gsnRightStringParallelPosF', + 'gsnScalarContour', 'gsnScale', 'gsnShape', 'gsnSpreadColorEnd', + 'gsnSpreadColorStart', 'gsnSpreadColors', 'gsnStringFont', + 'gsnStringFontColor', 'gsnStringFontHeightF', 'gsnStringFuncCode', + 'gsnTickMarksOn', 'gsnXAxisIrregular2Linear', 'gsnXAxisIrregular2Log', + 'gsnXRefLine', 'gsnXRefLineColor', 'gsnXRefLineDashPattern', + 'gsnXRefLineThicknessF', 'gsnXYAboveFillColors', 'gsnXYBarChart', + 'gsnXYBarChartBarWidth', 'gsnXYBarChartColors', + 'gsnXYBarChartColors2', 'gsnXYBarChartFillDotSizeF', + 'gsnXYBarChartFillLineThicknessF', 'gsnXYBarChartFillOpacityF', + 'gsnXYBarChartFillScaleF', 'gsnXYBarChartOutlineOnly', + 'gsnXYBarChartOutlineThicknessF', 'gsnXYBarChartPatterns', + 'gsnXYBarChartPatterns2', 'gsnXYBelowFillColors', 'gsnXYFillColors', + 'gsnXYFillOpacities', 'gsnXYLeftFillColors', 'gsnXYRightFillColors', + 'gsnYAxisIrregular2Linear', 'gsnYAxisIrregular2Log', 'gsnYRefLine', + 'gsnYRefLineColor', 'gsnYRefLineColors', 'gsnYRefLineDashPattern', + 'gsnYRefLineDashPatterns', 'gsnYRefLineThicknessF', + 'gsnYRefLineThicknesses', 'gsnZonalMean', 'gsnZonalMeanXMaxF', + 'gsnZonalMeanXMinF', 'gsnZonalMeanYRefLine', 'lbAutoManage', + 'lbBottomMarginF', 'lbBoxCount', 'lbBoxEndCapStyle', 'lbBoxFractions', + 'lbBoxLineColor', 'lbBoxLineDashPattern', 'lbBoxLineDashSegLenF', + 'lbBoxLineThicknessF', 'lbBoxLinesOn', 'lbBoxMajorExtentF', + 'lbBoxMinorExtentF', 'lbBoxSeparatorLinesOn', 'lbBoxSizing', + 'lbFillBackground', 'lbFillColor', 'lbFillColors', 'lbFillDotSizeF', + 'lbFillLineThicknessF', 'lbFillPattern', 'lbFillPatterns', + 'lbFillScaleF', 'lbFillScales', 'lbJustification', 'lbLabelAlignment', + 'lbLabelAngleF', 'lbLabelAutoStride', 'lbLabelBarOn', + 'lbLabelConstantSpacingF', 'lbLabelDirection', 'lbLabelFont', + 'lbLabelFontAspectF', 'lbLabelFontColor', 'lbLabelFontHeightF', + 'lbLabelFontQuality', 'lbLabelFontThicknessF', 'lbLabelFuncCode', + 'lbLabelJust', 'lbLabelOffsetF', 'lbLabelPosition', 'lbLabelStride', + 'lbLabelStrings', 'lbLabelsOn', 'lbLeftMarginF', 'lbMaxLabelLenF', + 'lbMinLabelSpacingF', 'lbMonoFillColor', 'lbMonoFillPattern', + 'lbMonoFillScale', 'lbOrientation', 'lbPerimColor', + 'lbPerimDashPattern', 'lbPerimDashSegLenF', 'lbPerimFill', + 'lbPerimFillColor', 'lbPerimOn', 'lbPerimThicknessF', + 'lbRasterFillOn', 'lbRightMarginF', 'lbTitleAngleF', + 'lbTitleConstantSpacingF', 'lbTitleDirection', 'lbTitleExtentF', + 'lbTitleFont', 'lbTitleFontAspectF', 'lbTitleFontColor', + 'lbTitleFontHeightF', 'lbTitleFontQuality', 'lbTitleFontThicknessF', + 'lbTitleFuncCode', 'lbTitleJust', 'lbTitleOffsetF', 'lbTitleOn', + 'lbTitlePosition', 'lbTitleString', 'lbTopMarginF', 'lgAutoManage', + 'lgBottomMarginF', 'lgBoxBackground', 'lgBoxLineColor', + 'lgBoxLineDashPattern', 'lgBoxLineDashSegLenF', 'lgBoxLineThicknessF', + 'lgBoxLinesOn', 'lgBoxMajorExtentF', 'lgBoxMinorExtentF', + 'lgDashIndex', 'lgDashIndexes', 'lgItemCount', 'lgItemOrder', + 'lgItemPlacement', 'lgItemPositions', 'lgItemType', 'lgItemTypes', + 'lgJustification', 'lgLabelAlignment', 'lgLabelAngleF', + 'lgLabelAutoStride', 'lgLabelConstantSpacingF', 'lgLabelDirection', + 'lgLabelFont', 'lgLabelFontAspectF', 'lgLabelFontColor', + 'lgLabelFontHeightF', 'lgLabelFontQuality', 'lgLabelFontThicknessF', + 'lgLabelFuncCode', 'lgLabelJust', 'lgLabelOffsetF', 'lgLabelPosition', + 'lgLabelStride', 'lgLabelStrings', 'lgLabelsOn', 'lgLeftMarginF', + 'lgLegendOn', 'lgLineColor', 'lgLineColors', 'lgLineDashSegLenF', + 'lgLineDashSegLens', 'lgLineLabelConstantSpacingF', 'lgLineLabelFont', + 'lgLineLabelFontAspectF', 'lgLineLabelFontColor', + 'lgLineLabelFontColors', 'lgLineLabelFontHeightF', + 'lgLineLabelFontHeights', 'lgLineLabelFontQuality', + 'lgLineLabelFontThicknessF', 'lgLineLabelFuncCode', + 'lgLineLabelStrings', 'lgLineLabelsOn', 'lgLineThicknessF', + 'lgLineThicknesses', 'lgMarkerColor', 'lgMarkerColors', + 'lgMarkerIndex', 'lgMarkerIndexes', 'lgMarkerSizeF', 'lgMarkerSizes', + 'lgMarkerThicknessF', 'lgMarkerThicknesses', 'lgMonoDashIndex', + 'lgMonoItemType', 'lgMonoLineColor', 'lgMonoLineDashSegLen', + 'lgMonoLineLabelFontColor', 'lgMonoLineLabelFontHeight', + 'lgMonoLineThickness', 'lgMonoMarkerColor', 'lgMonoMarkerIndex', + 'lgMonoMarkerSize', 'lgMonoMarkerThickness', 'lgOrientation', + 'lgPerimColor', 'lgPerimDashPattern', 'lgPerimDashSegLenF', + 'lgPerimFill', 'lgPerimFillColor', 'lgPerimOn', 'lgPerimThicknessF', + 'lgRightMarginF', 'lgTitleAngleF', 'lgTitleConstantSpacingF', + 'lgTitleDirection', 'lgTitleExtentF', 'lgTitleFont', + 'lgTitleFontAspectF', 'lgTitleFontColor', 'lgTitleFontHeightF', + 'lgTitleFontQuality', 'lgTitleFontThicknessF', 'lgTitleFuncCode', + 'lgTitleJust', 'lgTitleOffsetF', 'lgTitleOn', 'lgTitlePosition', + 'lgTitleString', 'lgTopMarginF', 'mpAreaGroupCount', + 'mpAreaMaskingOn', 'mpAreaNames', 'mpAreaTypes', 'mpBottomAngleF', + 'mpBottomMapPosF', 'mpBottomNDCF', 'mpBottomNPCF', + 'mpBottomPointLatF', 'mpBottomPointLonF', 'mpBottomWindowF', + 'mpCenterLatF', 'mpCenterLonF', 'mpCenterRotF', 'mpCountyLineColor', + 'mpCountyLineDashPattern', 'mpCountyLineDashSegLenF', + 'mpCountyLineThicknessF', 'mpDataBaseVersion', 'mpDataResolution', + 'mpDataSetName', 'mpDefaultFillColor', 'mpDefaultFillPattern', + 'mpDefaultFillScaleF', 'mpDynamicAreaGroups', 'mpEllipticalBoundary', + 'mpFillAreaSpecifiers', 'mpFillBoundarySets', 'mpFillColor', + 'mpFillColors', 'mpFillColors-default', 'mpFillDotSizeF', + 'mpFillDrawOrder', 'mpFillOn', 'mpFillPatternBackground', + 'mpFillPattern', 'mpFillPatterns', 'mpFillPatterns-default', + 'mpFillScaleF', 'mpFillScales', 'mpFillScales-default', + 'mpFixedAreaGroups', 'mpGeophysicalLineColor', + 'mpGeophysicalLineDashPattern', 'mpGeophysicalLineDashSegLenF', + 'mpGeophysicalLineThicknessF', 'mpGreatCircleLinesOn', + 'mpGridAndLimbDrawOrder', 'mpGridAndLimbOn', 'mpGridLatSpacingF', + 'mpGridLineColor', 'mpGridLineDashPattern', 'mpGridLineDashSegLenF', + 'mpGridLineThicknessF', 'mpGridLonSpacingF', 'mpGridMaskMode', + 'mpGridMaxLatF', 'mpGridPolarLonSpacingF', 'mpGridSpacingF', + 'mpInlandWaterFillColor', 'mpInlandWaterFillPattern', + 'mpInlandWaterFillScaleF', 'mpLabelDrawOrder', 'mpLabelFontColor', + 'mpLabelFontHeightF', 'mpLabelsOn', 'mpLambertMeridianF', + 'mpLambertParallel1F', 'mpLambertParallel2F', 'mpLandFillColor', + 'mpLandFillPattern', 'mpLandFillScaleF', 'mpLeftAngleF', + 'mpLeftCornerLatF', 'mpLeftCornerLonF', 'mpLeftMapPosF', + 'mpLeftNDCF', 'mpLeftNPCF', 'mpLeftPointLatF', + 'mpLeftPointLonF', 'mpLeftWindowF', 'mpLimbLineColor', + 'mpLimbLineDashPattern', 'mpLimbLineDashSegLenF', + 'mpLimbLineThicknessF', 'mpLimitMode', 'mpMaskAreaSpecifiers', + 'mpMaskOutlineSpecifiers', 'mpMaxLatF', 'mpMaxLonF', + 'mpMinLatF', 'mpMinLonF', 'mpMonoFillColor', 'mpMonoFillPattern', + 'mpMonoFillScale', 'mpNationalLineColor', 'mpNationalLineDashPattern', + 'mpNationalLineThicknessF', 'mpOceanFillColor', 'mpOceanFillPattern', + 'mpOceanFillScaleF', 'mpOutlineBoundarySets', 'mpOutlineDrawOrder', + 'mpOutlineMaskingOn', 'mpOutlineOn', 'mpOutlineSpecifiers', + 'mpPerimDrawOrder', 'mpPerimLineColor', 'mpPerimLineDashPattern', + 'mpPerimLineDashSegLenF', 'mpPerimLineThicknessF', 'mpPerimOn', + 'mpPolyMode', 'mpProjection', 'mpProvincialLineColor', + 'mpProvincialLineDashPattern', 'mpProvincialLineDashSegLenF', + 'mpProvincialLineThicknessF', 'mpRelativeCenterLat', + 'mpRelativeCenterLon', 'mpRightAngleF', 'mpRightCornerLatF', + 'mpRightCornerLonF', 'mpRightMapPosF', 'mpRightNDCF', + 'mpRightNPCF', 'mpRightPointLatF', 'mpRightPointLonF', + 'mpRightWindowF', 'mpSatelliteAngle1F', 'mpSatelliteAngle2F', + 'mpSatelliteDistF', 'mpShapeMode', 'mpSpecifiedFillColors', + 'mpSpecifiedFillDirectIndexing', 'mpSpecifiedFillPatterns', + 'mpSpecifiedFillPriority', 'mpSpecifiedFillScales', + 'mpTopAngleF', 'mpTopMapPosF', 'mpTopNDCF', 'mpTopNPCF', + 'mpTopPointLatF', 'mpTopPointLonF', 'mpTopWindowF', + 'mpUSStateLineColor', 'mpUSStateLineDashPattern', + 'mpUSStateLineDashSegLenF', 'mpUSStateLineThicknessF', + 'pmAnnoManagers', 'pmAnnoViews', 'pmLabelBarDisplayMode', + 'pmLabelBarHeightF', 'pmLabelBarKeepAspect', 'pmLabelBarOrthogonalPosF', + 'pmLabelBarParallelPosF', 'pmLabelBarSide', 'pmLabelBarWidthF', + 'pmLabelBarZone', 'pmLegendDisplayMode', 'pmLegendHeightF', + 'pmLegendKeepAspect', 'pmLegendOrthogonalPosF', + 'pmLegendParallelPosF', 'pmLegendSide', 'pmLegendWidthF', + 'pmLegendZone', 'pmOverlaySequenceIds', 'pmTickMarkDisplayMode', + 'pmTickMarkZone', 'pmTitleDisplayMode', 'pmTitleZone', + 'prGraphicStyle', 'prPolyType', 'prXArray', 'prYArray', + 'sfCopyData', 'sfDataArray', 'sfDataMaxV', 'sfDataMinV', + 'sfElementNodes', 'sfExchangeDimensions', 'sfFirstNodeIndex', + 'sfMissingValueV', 'sfXArray', 'sfXCActualEndF', 'sfXCActualStartF', + 'sfXCEndIndex', 'sfXCEndSubsetV', 'sfXCEndV', 'sfXCStartIndex', + 'sfXCStartSubsetV', 'sfXCStartV', 'sfXCStride', 'sfXCellBounds', + 'sfYArray', 'sfYCActualEndF', 'sfYCActualStartF', 'sfYCEndIndex', + 'sfYCEndSubsetV', 'sfYCEndV', 'sfYCStartIndex', 'sfYCStartSubsetV', + 'sfYCStartV', 'sfYCStride', 'sfYCellBounds', 'stArrowLengthF', + 'stArrowStride', 'stCrossoverCheckCount', + 'stExplicitLabelBarLabelsOn', 'stLabelBarEndLabelsOn', + 'stLabelFormat', 'stLengthCheckCount', 'stLevelColors', + 'stLevelCount', 'stLevelPalette', 'stLevelSelectionMode', + 'stLevelSpacingF', 'stLevels', 'stLineColor', 'stLineOpacityF', + 'stLineStartStride', 'stLineThicknessF', 'stMapDirection', + 'stMaxLevelCount', 'stMaxLevelValF', 'stMinArrowSpacingF', + 'stMinDistanceF', 'stMinLevelValF', 'stMinLineSpacingF', + 'stMinStepFactorF', 'stMonoLineColor', 'stNoDataLabelOn', + 'stNoDataLabelString', 'stScalarFieldData', 'stScalarMissingValColor', + 'stSpanLevelPalette', 'stStepSizeF', 'stStreamlineDrawOrder', + 'stUseScalarArray', 'stVectorFieldData', 'stZeroFLabelAngleF', + 'stZeroFLabelBackgroundColor', 'stZeroFLabelConstantSpacingF', + 'stZeroFLabelFont', 'stZeroFLabelFontAspectF', + 'stZeroFLabelFontColor', 'stZeroFLabelFontHeightF', + 'stZeroFLabelFontQuality', 'stZeroFLabelFontThicknessF', + 'stZeroFLabelFuncCode', 'stZeroFLabelJust', 'stZeroFLabelOn', + 'stZeroFLabelOrthogonalPosF', 'stZeroFLabelParallelPosF', + 'stZeroFLabelPerimColor', 'stZeroFLabelPerimOn', + 'stZeroFLabelPerimSpaceF', 'stZeroFLabelPerimThicknessF', + 'stZeroFLabelSide', 'stZeroFLabelString', 'stZeroFLabelTextDirection', + 'stZeroFLabelZone', 'tfDoNDCOverlay', 'tfPlotManagerOn', + 'tfPolyDrawList', 'tfPolyDrawOrder', 'tiDeltaF', 'tiMainAngleF', + 'tiMainConstantSpacingF', 'tiMainDirection', 'tiMainFont', + 'tiMainFontAspectF', 'tiMainFontColor', 'tiMainFontHeightF', + 'tiMainFontQuality', 'tiMainFontThicknessF', 'tiMainFuncCode', + 'tiMainJust', 'tiMainOffsetXF', 'tiMainOffsetYF', 'tiMainOn', + 'tiMainPosition', 'tiMainSide', 'tiMainString', 'tiUseMainAttributes', + 'tiXAxisAngleF', 'tiXAxisConstantSpacingF', 'tiXAxisDirection', + 'tiXAxisFont', 'tiXAxisFontAspectF', 'tiXAxisFontColor', + 'tiXAxisFontHeightF', 'tiXAxisFontQuality', 'tiXAxisFontThicknessF', + 'tiXAxisFuncCode', 'tiXAxisJust', 'tiXAxisOffsetXF', + 'tiXAxisOffsetYF', 'tiXAxisOn', 'tiXAxisPosition', 'tiXAxisSide', + 'tiXAxisString', 'tiYAxisAngleF', 'tiYAxisConstantSpacingF', + 'tiYAxisDirection', 'tiYAxisFont', 'tiYAxisFontAspectF', + 'tiYAxisFontColor', 'tiYAxisFontHeightF', 'tiYAxisFontQuality', + 'tiYAxisFontThicknessF', 'tiYAxisFuncCode', 'tiYAxisJust', + 'tiYAxisOffsetXF', 'tiYAxisOffsetYF', 'tiYAxisOn', 'tiYAxisPosition', + 'tiYAxisSide', 'tiYAxisString', 'tmBorderLineColor', + 'tmBorderThicknessF', 'tmEqualizeXYSizes', 'tmLabelAutoStride', + 'tmSciNoteCutoff', 'tmXBAutoPrecision', 'tmXBBorderOn', + 'tmXBDataLeftF', 'tmXBDataRightF', 'tmXBFormat', 'tmXBIrrTensionF', + 'tmXBIrregularPoints', 'tmXBLabelAngleF', 'tmXBLabelConstantSpacingF', + 'tmXBLabelDeltaF', 'tmXBLabelDirection', 'tmXBLabelFont', + 'tmXBLabelFontAspectF', 'tmXBLabelFontColor', 'tmXBLabelFontHeightF', + 'tmXBLabelFontQuality', 'tmXBLabelFontThicknessF', + 'tmXBLabelFuncCode', 'tmXBLabelJust', 'tmXBLabelStride', 'tmXBLabels', + 'tmXBLabelsOn', 'tmXBMajorLengthF', 'tmXBMajorLineColor', + 'tmXBMajorOutwardLengthF', 'tmXBMajorThicknessF', 'tmXBMaxLabelLenF', + 'tmXBMaxTicks', 'tmXBMinLabelSpacingF', 'tmXBMinorLengthF', + 'tmXBMinorLineColor', 'tmXBMinorOn', 'tmXBMinorOutwardLengthF', + 'tmXBMinorPerMajor', 'tmXBMinorThicknessF', 'tmXBMinorValues', + 'tmXBMode', 'tmXBOn', 'tmXBPrecision', 'tmXBStyle', 'tmXBTickEndF', + 'tmXBTickSpacingF', 'tmXBTickStartF', 'tmXBValues', 'tmXMajorGrid', + 'tmXMajorGridLineColor', 'tmXMajorGridLineDashPattern', + 'tmXMajorGridThicknessF', 'tmXMinorGrid', 'tmXMinorGridLineColor', + 'tmXMinorGridLineDashPattern', 'tmXMinorGridThicknessF', + 'tmXTAutoPrecision', 'tmXTBorderOn', 'tmXTDataLeftF', + 'tmXTDataRightF', 'tmXTFormat', 'tmXTIrrTensionF', + 'tmXTIrregularPoints', 'tmXTLabelAngleF', 'tmXTLabelConstantSpacingF', + 'tmXTLabelDeltaF', 'tmXTLabelDirection', 'tmXTLabelFont', + 'tmXTLabelFontAspectF', 'tmXTLabelFontColor', 'tmXTLabelFontHeightF', + 'tmXTLabelFontQuality', 'tmXTLabelFontThicknessF', + 'tmXTLabelFuncCode', 'tmXTLabelJust', 'tmXTLabelStride', 'tmXTLabels', + 'tmXTLabelsOn', 'tmXTMajorLengthF', 'tmXTMajorLineColor', + 'tmXTMajorOutwardLengthF', 'tmXTMajorThicknessF', 'tmXTMaxLabelLenF', + 'tmXTMaxTicks', 'tmXTMinLabelSpacingF', 'tmXTMinorLengthF', + 'tmXTMinorLineColor', 'tmXTMinorOn', 'tmXTMinorOutwardLengthF', + 'tmXTMinorPerMajor', 'tmXTMinorThicknessF', 'tmXTMinorValues', + 'tmXTMode', 'tmXTOn', 'tmXTPrecision', 'tmXTStyle', 'tmXTTickEndF', + 'tmXTTickSpacingF', 'tmXTTickStartF', 'tmXTValues', 'tmXUseBottom', + 'tmYLAutoPrecision', 'tmYLBorderOn', 'tmYLDataBottomF', + 'tmYLDataTopF', 'tmYLFormat', 'tmYLIrrTensionF', + 'tmYLIrregularPoints', 'tmYLLabelAngleF', 'tmYLLabelConstantSpacingF', + 'tmYLLabelDeltaF', 'tmYLLabelDirection', 'tmYLLabelFont', + 'tmYLLabelFontAspectF', 'tmYLLabelFontColor', 'tmYLLabelFontHeightF', + 'tmYLLabelFontQuality', 'tmYLLabelFontThicknessF', + 'tmYLLabelFuncCode', 'tmYLLabelJust', 'tmYLLabelStride', 'tmYLLabels', + 'tmYLLabelsOn', 'tmYLMajorLengthF', 'tmYLMajorLineColor', + 'tmYLMajorOutwardLengthF', 'tmYLMajorThicknessF', 'tmYLMaxLabelLenF', + 'tmYLMaxTicks', 'tmYLMinLabelSpacingF', 'tmYLMinorLengthF', + 'tmYLMinorLineColor', 'tmYLMinorOn', 'tmYLMinorOutwardLengthF', + 'tmYLMinorPerMajor', 'tmYLMinorThicknessF', 'tmYLMinorValues', + 'tmYLMode', 'tmYLOn', 'tmYLPrecision', 'tmYLStyle', 'tmYLTickEndF', + 'tmYLTickSpacingF', 'tmYLTickStartF', 'tmYLValues', 'tmYMajorGrid', + 'tmYMajorGridLineColor', 'tmYMajorGridLineDashPattern', + 'tmYMajorGridThicknessF', 'tmYMinorGrid', 'tmYMinorGridLineColor', + 'tmYMinorGridLineDashPattern', 'tmYMinorGridThicknessF', + 'tmYRAutoPrecision', 'tmYRBorderOn', 'tmYRDataBottomF', + 'tmYRDataTopF', 'tmYRFormat', 'tmYRIrrTensionF', + 'tmYRIrregularPoints', 'tmYRLabelAngleF', 'tmYRLabelConstantSpacingF', + 'tmYRLabelDeltaF', 'tmYRLabelDirection', 'tmYRLabelFont', + 'tmYRLabelFontAspectF', 'tmYRLabelFontColor', 'tmYRLabelFontHeightF', + 'tmYRLabelFontQuality', 'tmYRLabelFontThicknessF', + 'tmYRLabelFuncCode', 'tmYRLabelJust', 'tmYRLabelStride', 'tmYRLabels', + 'tmYRLabelsOn', 'tmYRMajorLengthF', 'tmYRMajorLineColor', + 'tmYRMajorOutwardLengthF', 'tmYRMajorThicknessF', 'tmYRMaxLabelLenF', + 'tmYRMaxTicks', 'tmYRMinLabelSpacingF', 'tmYRMinorLengthF', + 'tmYRMinorLineColor', 'tmYRMinorOn', 'tmYRMinorOutwardLengthF', + 'tmYRMinorPerMajor', 'tmYRMinorThicknessF', 'tmYRMinorValues', + 'tmYRMode', 'tmYROn', 'tmYRPrecision', 'tmYRStyle', 'tmYRTickEndF', + 'tmYRTickSpacingF', 'tmYRTickStartF', 'tmYRValues', 'tmYUseLeft', + 'trGridType', 'trLineInterpolationOn', + 'trXAxisType', 'trXCoordPoints', 'trXInterPoints', 'trXLog', + 'trXMaxF', 'trXMinF', 'trXReverse', 'trXSamples', 'trXTensionF', + 'trYAxisType', 'trYCoordPoints', 'trYInterPoints', 'trYLog', + 'trYMaxF', 'trYMinF', 'trYReverse', 'trYSamples', 'trYTensionF', + 'txAngleF', 'txBackgroundFillColor', 'txConstantSpacingF', 'txDirection', + 'txFont', 'HLU-Fonts', 'txFontAspectF', 'txFontColor', + 'txFontHeightF', 'txFontOpacityF', 'txFontQuality', + 'txFontThicknessF', 'txFuncCode', 'txJust', 'txPerimColor', + 'txPerimDashLengthF', 'txPerimDashPattern', 'txPerimOn', + 'txPerimSpaceF', 'txPerimThicknessF', 'txPosXF', 'txPosYF', + 'txString', 'vcExplicitLabelBarLabelsOn', 'vcFillArrowEdgeColor', + 'vcFillArrowEdgeThicknessF', 'vcFillArrowFillColor', + 'vcFillArrowHeadInteriorXF', 'vcFillArrowHeadMinFracXF', + 'vcFillArrowHeadMinFracYF', 'vcFillArrowHeadXF', 'vcFillArrowHeadYF', + 'vcFillArrowMinFracWidthF', 'vcFillArrowWidthF', 'vcFillArrowsOn', + 'vcFillOverEdge', 'vcGlyphOpacityF', 'vcGlyphStyle', + 'vcLabelBarEndLabelsOn', 'vcLabelFontColor', 'vcLabelFontHeightF', + 'vcLabelsOn', 'vcLabelsUseVectorColor', 'vcLevelColors', + 'vcLevelCount', 'vcLevelPalette', 'vcLevelSelectionMode', + 'vcLevelSpacingF', 'vcLevels', 'vcLineArrowColor', + 'vcLineArrowHeadMaxSizeF', 'vcLineArrowHeadMinSizeF', + 'vcLineArrowThicknessF', 'vcMagnitudeFormat', + 'vcMagnitudeScaleFactorF', 'vcMagnitudeScaleValueF', + 'vcMagnitudeScalingMode', 'vcMapDirection', 'vcMaxLevelCount', + 'vcMaxLevelValF', 'vcMaxMagnitudeF', 'vcMinAnnoAngleF', + 'vcMinAnnoArrowAngleF', 'vcMinAnnoArrowEdgeColor', + 'vcMinAnnoArrowFillColor', 'vcMinAnnoArrowLineColor', + 'vcMinAnnoArrowMinOffsetF', 'vcMinAnnoArrowSpaceF', + 'vcMinAnnoArrowUseVecColor', 'vcMinAnnoBackgroundColor', + 'vcMinAnnoConstantSpacingF', 'vcMinAnnoExplicitMagnitudeF', + 'vcMinAnnoFont', 'vcMinAnnoFontAspectF', 'vcMinAnnoFontColor', + 'vcMinAnnoFontHeightF', 'vcMinAnnoFontQuality', + 'vcMinAnnoFontThicknessF', 'vcMinAnnoFuncCode', 'vcMinAnnoJust', + 'vcMinAnnoOn', 'vcMinAnnoOrientation', 'vcMinAnnoOrthogonalPosF', + 'vcMinAnnoParallelPosF', 'vcMinAnnoPerimColor', 'vcMinAnnoPerimOn', + 'vcMinAnnoPerimSpaceF', 'vcMinAnnoPerimThicknessF', 'vcMinAnnoSide', + 'vcMinAnnoString1', 'vcMinAnnoString1On', 'vcMinAnnoString2', + 'vcMinAnnoString2On', 'vcMinAnnoTextDirection', 'vcMinAnnoZone', + 'vcMinDistanceF', 'vcMinFracLengthF', 'vcMinLevelValF', + 'vcMinMagnitudeF', 'vcMonoFillArrowEdgeColor', + 'vcMonoFillArrowFillColor', 'vcMonoLineArrowColor', + 'vcMonoWindBarbColor', 'vcNoDataLabelOn', 'vcNoDataLabelString', + 'vcPositionMode', 'vcRefAnnoAngleF', 'vcRefAnnoArrowAngleF', + 'vcRefAnnoArrowEdgeColor', 'vcRefAnnoArrowFillColor', + 'vcRefAnnoArrowLineColor', 'vcRefAnnoArrowMinOffsetF', + 'vcRefAnnoArrowSpaceF', 'vcRefAnnoArrowUseVecColor', + 'vcRefAnnoBackgroundColor', 'vcRefAnnoConstantSpacingF', + 'vcRefAnnoExplicitMagnitudeF', 'vcRefAnnoFont', + 'vcRefAnnoFontAspectF', 'vcRefAnnoFontColor', 'vcRefAnnoFontHeightF', + 'vcRefAnnoFontQuality', 'vcRefAnnoFontThicknessF', + 'vcRefAnnoFuncCode', 'vcRefAnnoJust', 'vcRefAnnoOn', + 'vcRefAnnoOrientation', 'vcRefAnnoOrthogonalPosF', + 'vcRefAnnoParallelPosF', 'vcRefAnnoPerimColor', 'vcRefAnnoPerimOn', + 'vcRefAnnoPerimSpaceF', 'vcRefAnnoPerimThicknessF', 'vcRefAnnoSide', + 'vcRefAnnoString1', 'vcRefAnnoString1On', 'vcRefAnnoString2', + 'vcRefAnnoString2On', 'vcRefAnnoTextDirection', 'vcRefAnnoZone', + 'vcRefLengthF', 'vcRefMagnitudeF', 'vcScalarFieldData', + 'vcScalarMissingValColor', 'vcScalarValueFormat', + 'vcScalarValueScaleFactorF', 'vcScalarValueScaleValueF', + 'vcScalarValueScalingMode', 'vcSpanLevelPalette', 'vcUseRefAnnoRes', + 'vcUseScalarArray', 'vcVectorDrawOrder', 'vcVectorFieldData', + 'vcWindBarbCalmCircleSizeF', 'vcWindBarbColor', + 'vcWindBarbLineThicknessF', 'vcWindBarbScaleFactorF', + 'vcWindBarbTickAngleF', 'vcWindBarbTickLengthF', + 'vcWindBarbTickSpacingF', 'vcZeroFLabelAngleF', + 'vcZeroFLabelBackgroundColor', 'vcZeroFLabelConstantSpacingF', + 'vcZeroFLabelFont', 'vcZeroFLabelFontAspectF', + 'vcZeroFLabelFontColor', 'vcZeroFLabelFontHeightF', + 'vcZeroFLabelFontQuality', 'vcZeroFLabelFontThicknessF', + 'vcZeroFLabelFuncCode', 'vcZeroFLabelJust', 'vcZeroFLabelOn', + 'vcZeroFLabelOrthogonalPosF', 'vcZeroFLabelParallelPosF', + 'vcZeroFLabelPerimColor', 'vcZeroFLabelPerimOn', + 'vcZeroFLabelPerimSpaceF', 'vcZeroFLabelPerimThicknessF', + 'vcZeroFLabelSide', 'vcZeroFLabelString', 'vcZeroFLabelTextDirection', + 'vcZeroFLabelZone', 'vfCopyData', 'vfDataArray', + 'vfExchangeDimensions', 'vfExchangeUVData', 'vfMagMaxV', 'vfMagMinV', + 'vfMissingUValueV', 'vfMissingVValueV', 'vfPolarData', + 'vfSingleMissingValue', 'vfUDataArray', 'vfUMaxV', 'vfUMinV', + 'vfVDataArray', 'vfVMaxV', 'vfVMinV', 'vfXArray', 'vfXCActualEndF', + 'vfXCActualStartF', 'vfXCEndIndex', 'vfXCEndSubsetV', 'vfXCEndV', + 'vfXCStartIndex', 'vfXCStartSubsetV', 'vfXCStartV', 'vfXCStride', + 'vfYArray', 'vfYCActualEndF', 'vfYCActualStartF', 'vfYCEndIndex', + 'vfYCEndSubsetV', 'vfYCEndV', 'vfYCStartIndex', 'vfYCStartSubsetV', + 'vfYCStartV', 'vfYCStride', 'vpAnnoManagerId', 'vpClipOn', + 'vpHeightF', 'vpKeepAspect', 'vpOn', 'vpUseSegments', 'vpWidthF', + 'vpXF', 'vpYF', 'wkAntiAlias', 'wkBackgroundColor', 'wkBackgroundOpacityF', + 'wkColorMapLen', 'wkColorMap', 'wkColorModel', 'wkDashTableLength', + 'wkDefGraphicStyleId', 'wkDeviceLowerX', 'wkDeviceLowerY', + 'wkDeviceUpperX', 'wkDeviceUpperY', 'wkFileName', 'wkFillTableLength', + 'wkForegroundColor', 'wkFormat', 'wkFullBackground', 'wkGksWorkId', + 'wkHeight', 'wkMarkerTableLength', 'wkMetaName', 'wkOrientation', + 'wkPDFFileName', 'wkPDFFormat', 'wkPDFResolution', 'wkPSFileName', + 'wkPSFormat', 'wkPSResolution', 'wkPaperHeightF', 'wkPaperSize', + 'wkPaperWidthF', 'wkPause', 'wkTopLevelViews', 'wkViews', + 'wkVisualType', 'wkWidth', 'wkWindowId', 'wkXColorMode', 'wsCurrentSize', + 'wsMaximumSize', 'wsThresholdSize', 'xyComputeXMax', + 'xyComputeXMin', 'xyComputeYMax', 'xyComputeYMin', 'xyCoordData', + 'xyCoordDataSpec', 'xyCurveDrawOrder', 'xyDashPattern', + 'xyDashPatterns', 'xyExplicitLabels', 'xyExplicitLegendLabels', + 'xyLabelMode', 'xyLineColor', 'xyLineColors', 'xyLineDashSegLenF', + 'xyLineLabelConstantSpacingF', 'xyLineLabelFont', + 'xyLineLabelFontAspectF', 'xyLineLabelFontColor', + 'xyLineLabelFontColors', 'xyLineLabelFontHeightF', + 'xyLineLabelFontQuality', 'xyLineLabelFontThicknessF', + 'xyLineLabelFuncCode', 'xyLineThicknessF', 'xyLineThicknesses', + 'xyMarkLineMode', 'xyMarkLineModes', 'xyMarker', 'xyMarkerColor', + 'xyMarkerColors', 'xyMarkerSizeF', 'xyMarkerSizes', + 'xyMarkerThicknessF', 'xyMarkerThicknesses', 'xyMarkers', + 'xyMonoDashPattern', 'xyMonoLineColor', 'xyMonoLineLabelFontColor', + 'xyMonoLineThickness', 'xyMonoMarkLineMode', 'xyMonoMarker', + 'xyMonoMarkerColor', 'xyMonoMarkerSize', 'xyMonoMarkerThickness', + 'xyXIrrTensionF', 'xyXIrregularPoints', 'xyXStyle', 'xyYIrrTensionF', + 'xyYIrregularPoints', 'xyYStyle'), prefix=r'\b'), + Name.Builtin), + + # Booleans + (r'\.(True|False)\.', Name.Builtin), + # Comparing Operators + (r'\.(eq|ne|lt|le|gt|ge|not|and|or|xor)\.', Operator.Word), + ], + + 'strings': [ + (r'(?s)"(\\\\|\\[0-7]+|\\.|[^"\\])*"', String.Double), + ], + + 'nums': [ + (r'\d+(?![.e])(_[a-z]\w+)?', Number.Integer), + (r'[+-]?\d*\.\d+(e[-+]?\d+)?(_[a-z]\w+)?', Number.Float), + (r'[+-]?\d+\.\d*(e[-+]?\d+)?(_[a-z]\w+)?', Number.Float), + ], + } diff --git a/pygments/lexers/nimrod.py b/pygments/lexers/nimrod.py new file mode 100644 index 0000000..bf5c948 --- /dev/null +++ b/pygments/lexers/nimrod.py @@ -0,0 +1,200 @@ +""" + pygments.lexers.nimrod + ~~~~~~~~~~~~~~~~~~~~~~ + + Lexer for the Nim language (formerly known as Nimrod). + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, include, default, bygroups +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Error + +__all__ = ['NimrodLexer'] + + +class NimrodLexer(RegexLexer): + """ + For Nim source code. + + .. versionadded:: 1.5 + """ + + name = 'Nimrod' + url = 'http://nim-lang.org/' + aliases = ['nimrod', 'nim'] + filenames = ['*.nim', '*.nimrod'] + mimetypes = ['text/x-nim'] + + flags = re.MULTILINE | re.IGNORECASE + + def underscorize(words): + newWords = [] + new = [] + for word in words: + for ch in word: + new.append(ch) + new.append("_?") + newWords.append(''.join(new)) + new = [] + return "|".join(newWords) + + keywords = [ + 'addr', 'and', 'as', 'asm', 'bind', 'block', 'break', 'case', + 'cast', 'concept', 'const', 'continue', 'converter', 'defer', 'discard', + 'distinct', 'div', 'do', 'elif', 'else', 'end', 'enum', 'except', + 'export', 'finally', 'for', 'if', 'in', 'yield', 'interface', + 'is', 'isnot', 'iterator', 'let', 'mixin', 'mod', + 'not', 'notin', 'object', 'of', 'or', 'out', 'ptr', 'raise', + 'ref', 'return', 'shl', 'shr', 'static', 'try', + 'tuple', 'type', 'using', 'when', 'while', 'xor' + ] + + keywordsPseudo = [ + 'nil', 'true', 'false' + ] + + opWords = [ + 'and', 'or', 'not', 'xor', 'shl', 'shr', 'div', 'mod', 'in', + 'notin', 'is', 'isnot' + ] + + types = [ + 'int', 'int8', 'int16', 'int32', 'int64', 'float', 'float32', 'float64', + 'bool', 'char', 'range', 'array', 'seq', 'set', 'string' + ] + + tokens = { + 'root': [ + # Comments + (r'##\[', String.Doc, 'doccomment'), + (r'##.*$', String.Doc), + (r'#\[', Comment.Multiline, 'comment'), + (r'#.*$', Comment), + + # Pragmas + (r'\{\.', String.Other, 'pragma'), + + # Operators + (r'[*=><+\-/@$~&%!?|\\\[\]]', Operator), + (r'\.\.|\.|,|\[\.|\.\]|\{\.|\.\}|\(\.|\.\)|\{|\}|\(|\)|:|\^|`|;', + Punctuation), + + # Case statement branch + (r'(\n\s*)(of)(\s)', bygroups(Text.Whitespace, Keyword, + Text.Whitespace), 'casebranch'), + + # Strings + (r'(?:[\w]+)"', String, 'rdqs'), + (r'"""', String.Double, 'tdqs'), + ('"', String, 'dqs'), + + # Char + ("'", String.Char, 'chars'), + + # Keywords + (r'(%s)\b' % underscorize(opWords), Operator.Word), + (r'(proc|func|method|macro|template)(\s)(?![(\[\]])', + bygroups(Keyword, Text.Whitespace), 'funcname'), + (r'(%s)\b' % underscorize(keywords), Keyword), + (r'(%s)\b' % underscorize(['from', 'import', 'include', 'export']), + Keyword.Namespace), + (r'(v_?a_?r)\b', Keyword.Declaration), + (r'(%s)\b' % underscorize(types), Name.Builtin), + (r'(%s)\b' % underscorize(keywordsPseudo), Keyword.Pseudo), + + # Identifiers + (r'\b((?![_\d])\w)(((?!_)\w)|(_(?!_)\w))*', Name), + + # Numbers + (r'[0-9][0-9_]*(?=([e.]|\'f(32|64)))', + Number.Float, ('float-suffix', 'float-number')), + (r'0x[a-f0-9][a-f0-9_]*', Number.Hex, 'int-suffix'), + (r'0b[01][01_]*', Number.Bin, 'int-suffix'), + (r'0o[0-7][0-7_]*', Number.Oct, 'int-suffix'), + (r'[0-9][0-9_]*', Number.Integer, 'int-suffix'), + + # Whitespace + (r'\s+', Text.Whitespace), + (r'.+$', Error), + ], + 'chars': [ + (r'\\([\\abcefnrtvl"\']|x[a-f0-9]{2}|[0-9]{1,3})', String.Escape), + (r"'", String.Char, '#pop'), + (r".", String.Char) + ], + 'strings': [ + (r'(?|>=|>>|>|<=|<<|<|\+|-|=|/|\*|%|\+=|-=|!|@', Operator), + (r'\(|\)|\[|\]|,|\.\.\.|\.\.|\.|::|:', Punctuation), + (r'`\{[^`]*`\}', Text), # Extern blocks won't be Lexed by Nit + (r'[\r\n\t ]+', Text), + ], + } diff --git a/pygments/lexers/nix.py b/pygments/lexers/nix.py new file mode 100644 index 0000000..55e2617 --- /dev/null +++ b/pygments/lexers/nix.py @@ -0,0 +1,135 @@ +""" + pygments.lexers.nix + ~~~~~~~~~~~~~~~~~~~ + + Lexers for the NixOS Nix language. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, include +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Literal + +__all__ = ['NixLexer'] + + +class NixLexer(RegexLexer): + """ + For the Nix language. + + .. versionadded:: 2.0 + """ + + name = 'Nix' + url = 'http://nixos.org/nix/' + aliases = ['nixos', 'nix'] + filenames = ['*.nix'] + mimetypes = ['text/x-nix'] + + keywords = ['rec', 'with', 'let', 'in', 'inherit', 'assert', 'if', + 'else', 'then', '...'] + builtins = ['import', 'abort', 'baseNameOf', 'dirOf', 'isNull', 'builtins', + 'map', 'removeAttrs', 'throw', 'toString', 'derivation'] + operators = ['++', '+', '?', '.', '!', '//', '==', + '!=', '&&', '||', '->', '='] + + punctuations = ["(", ")", "[", "]", ";", "{", "}", ":", ",", "@"] + + tokens = { + 'root': [ + # comments starting with # + (r'#.*$', Comment.Single), + + # multiline comments + (r'/\*', Comment.Multiline, 'comment'), + + # whitespace + (r'\s+', Text), + + # keywords + ('(%s)' % '|'.join(re.escape(entry) + '\\b' for entry in keywords), Keyword), + + # highlight the builtins + ('(%s)' % '|'.join(re.escape(entry) + '\\b' for entry in builtins), + Name.Builtin), + + (r'\b(true|false|null)\b', Name.Constant), + + # operators + ('(%s)' % '|'.join(re.escape(entry) for entry in operators), + Operator), + + # word operators + (r'\b(or|and)\b', Operator.Word), + + # punctuations + ('(%s)' % '|'.join(re.escape(entry) for entry in punctuations), Punctuation), + + # integers + (r'[0-9]+', Number.Integer), + + # strings + (r'"', String.Double, 'doublequote'), + (r"''", String.Single, 'singlequote'), + + # paths + (r'[\w.+-]*(\/[\w.+-]+)+', Literal), + (r'\<[\w.+-]+(\/[\w.+-]+)*\>', Literal), + + # urls + (r'[a-zA-Z][a-zA-Z0-9\+\-\.]*\:[\w%/?:@&=+$,\\.!~*\'-]+', Literal), + + # names of variables + (r'[\w-]+\s*=', String.Symbol), + (r'[a-zA-Z_][\w\'-]*', Text), + + ], + 'comment': [ + (r'[^/*]+', Comment.Multiline), + (r'/\*', Comment.Multiline, '#push'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[*/]', Comment.Multiline), + ], + 'singlequote': [ + (r"'''", String.Escape), + (r"''\$\{", String.Escape), + (r"''\n", String.Escape), + (r"''\r", String.Escape), + (r"''\t", String.Escape), + (r"''", String.Single, '#pop'), + (r'\$\{', String.Interpol, 'antiquote'), + (r"['$]", String.Single), + (r"[^'$]+", String.Single), + ], + 'doublequote': [ + (r'\\', String.Escape), + (r'\\"', String.Escape), + (r'\\$\{', String.Escape), + (r'"', String.Double, '#pop'), + (r'\$\{', String.Interpol, 'antiquote'), + (r'[^"]', String.Double), + ], + 'antiquote': [ + (r"\}", String.Interpol, '#pop'), + # TODO: we should probably escape also here ''${ \${ + (r"\$\{", String.Interpol, '#push'), + include('root'), + ], + } + + def analyse_text(text): + rv = 0.0 + # TODO: let/in + if re.search(r'import.+?<[^>]+>', text): + rv += 0.4 + if re.search(r'mkDerivation\s+(\(|\{|rec)', text): + rv += 0.4 + if re.search(r'=\s+mkIf\s+', text): + rv += 0.4 + if re.search(r'\{[a-zA-Z,\s]+\}:', text): + rv += 0.1 + return rv diff --git a/pygments/lexers/oberon.py b/pygments/lexers/oberon.py new file mode 100644 index 0000000..da7a248 --- /dev/null +++ b/pygments/lexers/oberon.py @@ -0,0 +1,120 @@ +""" + pygments.lexers.oberon + ~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for Oberon family languages. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, include, words +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation + +__all__ = ['ComponentPascalLexer'] + + +class ComponentPascalLexer(RegexLexer): + """ + For Component Pascal source code. + + .. versionadded:: 2.1 + """ + name = 'Component Pascal' + aliases = ['componentpascal', 'cp'] + filenames = ['*.cp', '*.cps'] + mimetypes = ['text/x-component-pascal'] + + flags = re.MULTILINE | re.DOTALL + + tokens = { + 'root': [ + include('whitespace'), + include('comments'), + include('punctuation'), + include('numliterals'), + include('strings'), + include('operators'), + include('builtins'), + include('identifiers'), + ], + 'whitespace': [ + (r'\n+', Text), # blank lines + (r'\s+', Text), # whitespace + ], + 'comments': [ + (r'\(\*([^$].*?)\*\)', Comment.Multiline), + # TODO: nested comments (* (* ... *) ... (* ... *) *) not supported! + ], + 'punctuation': [ + (r'[()\[\]{},.:;|]', Punctuation), + ], + 'numliterals': [ + (r'[0-9A-F]+X\b', Number.Hex), # char code + (r'[0-9A-F]+[HL]\b', Number.Hex), # hexadecimal number + (r'[0-9]+\.[0-9]+E[+-][0-9]+', Number.Float), # real number + (r'[0-9]+\.[0-9]+', Number.Float), # real number + (r'[0-9]+', Number.Integer), # decimal whole number + ], + 'strings': [ + (r"'[^\n']*'", String), # single quoted string + (r'"[^\n"]*"', String), # double quoted string + ], + 'operators': [ + # Arithmetic Operators + (r'[+-]', Operator), + (r'[*/]', Operator), + # Relational Operators + (r'[=#<>]', Operator), + # Dereferencing Operator + (r'\^', Operator), + # Logical AND Operator + (r'&', Operator), + # Logical NOT Operator + (r'~', Operator), + # Assignment Symbol + (r':=', Operator), + # Range Constructor + (r'\.\.', Operator), + (r'\$', Operator), + ], + 'identifiers': [ + (r'([a-zA-Z_$][\w$]*)', Name), + ], + 'builtins': [ + (words(( + 'ANYPTR', 'ANYREC', 'BOOLEAN', 'BYTE', 'CHAR', 'INTEGER', 'LONGINT', + 'REAL', 'SET', 'SHORTCHAR', 'SHORTINT', 'SHORTREAL' + ), suffix=r'\b'), Keyword.Type), + (words(( + 'ABS', 'ABSTRACT', 'ARRAY', 'ASH', 'ASSERT', 'BEGIN', 'BITS', 'BY', + 'CAP', 'CASE', 'CHR', 'CLOSE', 'CONST', 'DEC', 'DIV', 'DO', 'ELSE', + 'ELSIF', 'EMPTY', 'END', 'ENTIER', 'EXCL', 'EXIT', 'EXTENSIBLE', 'FOR', + 'HALT', 'IF', 'IMPORT', 'IN', 'INC', 'INCL', 'IS', 'LEN', 'LIMITED', + 'LONG', 'LOOP', 'MAX', 'MIN', 'MOD', 'MODULE', 'NEW', 'ODD', 'OF', + 'OR', 'ORD', 'OUT', 'POINTER', 'PROCEDURE', 'RECORD', 'REPEAT', 'RETURN', + 'SHORT', 'SHORTCHAR', 'SHORTINT', 'SIZE', 'THEN', 'TYPE', 'TO', 'UNTIL', + 'VAR', 'WHILE', 'WITH' + ), suffix=r'\b'), Keyword.Reserved), + (r'(TRUE|FALSE|NIL|INF)\b', Keyword.Constant), + ] + } + + def analyse_text(text): + """The only other lexer using .cp is the C++ one, so we check if for + a few common Pascal keywords here. Those are unfortunately quite + common across various business languages as well.""" + result = 0 + if 'BEGIN' in text: + result += 0.01 + if 'END' in text: + result += 0.01 + if 'PROCEDURE' in text: + result += 0.01 + if 'END' in text: + result += 0.01 + + return result diff --git a/pygments/lexers/objective.py b/pygments/lexers/objective.py new file mode 100644 index 0000000..c9c4278 --- /dev/null +++ b/pygments/lexers/objective.py @@ -0,0 +1,505 @@ +""" + pygments.lexers.objective + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for Objective-C family languages. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, include, bygroups, using, this, words, \ + inherit, default +from pygments.token import Text, Keyword, Name, String, Operator, \ + Number, Punctuation, Literal, Comment + +from pygments.lexers.c_cpp import CLexer, CppLexer + +__all__ = ['ObjectiveCLexer', 'ObjectiveCppLexer', 'LogosLexer', 'SwiftLexer'] + + +def objective(baselexer): + """ + Generate a subclass of baselexer that accepts the Objective-C syntax + extensions. + """ + + # Have to be careful not to accidentally match JavaDoc/Doxygen syntax here, + # since that's quite common in ordinary C/C++ files. It's OK to match + # JavaDoc/Doxygen keywords that only apply to Objective-C, mind. + # + # The upshot of this is that we CANNOT match @class or @interface + _oc_keywords = re.compile(r'@(?:end|implementation|protocol)') + + # Matches [ ? identifier ( identifier ? ] | identifier? : ) + # (note the identifier is *optional* when there is a ':'!) + _oc_message = re.compile(r'\[\s*[a-zA-Z_]\w*\s+' + r'(?:[a-zA-Z_]\w*\s*\]|' + r'(?:[a-zA-Z_]\w*)?:)') + + class GeneratedObjectiveCVariant(baselexer): + """ + Implements Objective-C syntax on top of an existing C family lexer. + """ + + tokens = { + 'statements': [ + (r'@"', String, 'string'), + (r'@(YES|NO)', Number), + (r"@'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char), + (r'@(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[lL]?', Number.Float), + (r'@(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float), + (r'@0x[0-9a-fA-F]+[Ll]?', Number.Hex), + (r'@0[0-7]+[Ll]?', Number.Oct), + (r'@\d+[Ll]?', Number.Integer), + (r'@\(', Literal, 'literal_number'), + (r'@\[', Literal, 'literal_array'), + (r'@\{', Literal, 'literal_dictionary'), + (words(( + '@selector', '@private', '@protected', '@public', '@encode', + '@synchronized', '@try', '@throw', '@catch', '@finally', + '@end', '@property', '@synthesize', '__bridge', '__bridge_transfer', + '__autoreleasing', '__block', '__weak', '__strong', 'weak', 'strong', + 'copy', 'retain', 'assign', 'unsafe_unretained', 'atomic', 'nonatomic', + 'readonly', 'readwrite', 'setter', 'getter', 'typeof', 'in', + 'out', 'inout', 'release', 'class', '@dynamic', '@optional', + '@required', '@autoreleasepool', '@import'), suffix=r'\b'), + Keyword), + (words(('id', 'instancetype', 'Class', 'IMP', 'SEL', 'BOOL', + 'IBOutlet', 'IBAction', 'unichar'), suffix=r'\b'), + Keyword.Type), + (r'@(true|false|YES|NO)\n', Name.Builtin), + (r'(YES|NO|nil|self|super)\b', Name.Builtin), + # Carbon types + (r'(Boolean|UInt8|SInt8|UInt16|SInt16|UInt32|SInt32)\b', Keyword.Type), + # Carbon built-ins + (r'(TRUE|FALSE)\b', Name.Builtin), + (r'(@interface|@implementation)(\s+)', bygroups(Keyword, Text), + ('#pop', 'oc_classname')), + (r'(@class|@protocol)(\s+)', bygroups(Keyword, Text), + ('#pop', 'oc_forward_classname')), + # @ can also prefix other expressions like @{...} or @(...) + (r'@', Punctuation), + inherit, + ], + 'oc_classname': [ + # interface definition that inherits + (r'([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?(\s*)(\{)', + bygroups(Name.Class, Text, Name.Class, Text, Punctuation), + ('#pop', 'oc_ivars')), + (r'([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?', + bygroups(Name.Class, Text, Name.Class), '#pop'), + # interface definition for a category + (r'([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))(\s*)(\{)', + bygroups(Name.Class, Text, Name.Label, Text, Punctuation), + ('#pop', 'oc_ivars')), + (r'([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))', + bygroups(Name.Class, Text, Name.Label), '#pop'), + # simple interface / implementation + (r'([a-zA-Z$_][\w$]*)(\s*)(\{)', + bygroups(Name.Class, Text, Punctuation), ('#pop', 'oc_ivars')), + (r'([a-zA-Z$_][\w$]*)', Name.Class, '#pop') + ], + 'oc_forward_classname': [ + (r'([a-zA-Z$_][\w$]*)(\s*,\s*)', + bygroups(Name.Class, Text), 'oc_forward_classname'), + (r'([a-zA-Z$_][\w$]*)(\s*;?)', + bygroups(Name.Class, Text), '#pop') + ], + 'oc_ivars': [ + include('whitespace'), + include('statements'), + (';', Punctuation), + (r'\{', Punctuation, '#push'), + (r'\}', Punctuation, '#pop'), + ], + 'root': [ + # methods + (r'^([-+])(\s*)' # method marker + r'(\(.*?\))?(\s*)' # return type + r'([a-zA-Z$_][\w$]*:?)', # begin of method name + bygroups(Punctuation, Text, using(this), + Text, Name.Function), + 'method'), + inherit, + ], + 'method': [ + include('whitespace'), + # TODO unsure if ellipses are allowed elsewhere, see + # discussion in Issue 789 + (r',', Punctuation), + (r'\.\.\.', Punctuation), + (r'(\(.*?\))(\s*)([a-zA-Z$_][\w$]*)', + bygroups(using(this), Text, Name.Variable)), + (r'[a-zA-Z$_][\w$]*:', Name.Function), + (';', Punctuation, '#pop'), + (r'\{', Punctuation, 'function'), + default('#pop'), + ], + 'literal_number': [ + (r'\(', Punctuation, 'literal_number_inner'), + (r'\)', Literal, '#pop'), + include('statement'), + ], + 'literal_number_inner': [ + (r'\(', Punctuation, '#push'), + (r'\)', Punctuation, '#pop'), + include('statement'), + ], + 'literal_array': [ + (r'\[', Punctuation, 'literal_array_inner'), + (r'\]', Literal, '#pop'), + include('statement'), + ], + 'literal_array_inner': [ + (r'\[', Punctuation, '#push'), + (r'\]', Punctuation, '#pop'), + include('statement'), + ], + 'literal_dictionary': [ + (r'\}', Literal, '#pop'), + include('statement'), + ], + } + + def analyse_text(text): + if _oc_keywords.search(text): + return 1.0 + elif '@"' in text: # strings + return 0.8 + elif re.search('@[0-9]+', text): + return 0.7 + elif _oc_message.search(text): + return 0.8 + return 0 + + def get_tokens_unprocessed(self, text, stack=('root',)): + from pygments.lexers._cocoa_builtins import COCOA_INTERFACES, \ + COCOA_PROTOCOLS, COCOA_PRIMITIVES + + for index, token, value in \ + baselexer.get_tokens_unprocessed(self, text, stack): + if token is Name or token is Name.Class: + if value in COCOA_INTERFACES or value in COCOA_PROTOCOLS \ + or value in COCOA_PRIMITIVES: + token = Name.Builtin.Pseudo + + yield index, token, value + + return GeneratedObjectiveCVariant + + +class ObjectiveCLexer(objective(CLexer)): + """ + For Objective-C source code with preprocessor directives. + """ + + name = 'Objective-C' + url = 'https://developer.apple.com/library/archive/documentation/Cocoa/Conceptual/ProgrammingWithObjectiveC/Introduction/Introduction.html' + aliases = ['objective-c', 'objectivec', 'obj-c', 'objc'] + filenames = ['*.m', '*.h'] + mimetypes = ['text/x-objective-c'] + priority = 0.05 # Lower than C + + +class ObjectiveCppLexer(objective(CppLexer)): + """ + For Objective-C++ source code with preprocessor directives. + """ + + name = 'Objective-C++' + aliases = ['objective-c++', 'objectivec++', 'obj-c++', 'objc++'] + filenames = ['*.mm', '*.hh'] + mimetypes = ['text/x-objective-c++'] + priority = 0.05 # Lower than C++ + + +class LogosLexer(ObjectiveCppLexer): + """ + For Logos + Objective-C source code with preprocessor directives. + + .. versionadded:: 1.6 + """ + + name = 'Logos' + aliases = ['logos'] + filenames = ['*.x', '*.xi', '*.xm', '*.xmi'] + mimetypes = ['text/x-logos'] + priority = 0.25 + + tokens = { + 'statements': [ + (r'(%orig|%log)\b', Keyword), + (r'(%c)\b(\()(\s*)([a-zA-Z$_][\w$]*)(\s*)(\))', + bygroups(Keyword, Punctuation, Text, Name.Class, Text, Punctuation)), + (r'(%init)\b(\()', + bygroups(Keyword, Punctuation), 'logos_init_directive'), + (r'(%init)(?=\s*;)', bygroups(Keyword)), + (r'(%hook|%group)(\s+)([a-zA-Z$_][\w$]+)', + bygroups(Keyword, Text, Name.Class), '#pop'), + (r'(%subclass)(\s+)', bygroups(Keyword, Text), + ('#pop', 'logos_classname')), + inherit, + ], + 'logos_init_directive': [ + (r'\s+', Text), + (',', Punctuation, ('logos_init_directive', '#pop')), + (r'([a-zA-Z$_][\w$]*)(\s*)(=)(\s*)([^);]*)', + bygroups(Name.Class, Text, Punctuation, Text, Text)), + (r'([a-zA-Z$_][\w$]*)', Name.Class), + (r'\)', Punctuation, '#pop'), + ], + 'logos_classname': [ + (r'([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?', + bygroups(Name.Class, Text, Name.Class), '#pop'), + (r'([a-zA-Z$_][\w$]*)', Name.Class, '#pop') + ], + 'root': [ + (r'(%subclass)(\s+)', bygroups(Keyword, Text), + 'logos_classname'), + (r'(%hook|%group)(\s+)([a-zA-Z$_][\w$]+)', + bygroups(Keyword, Text, Name.Class)), + (r'(%config)(\s*\(\s*)(\w+)(\s*=)(.*?)(\)\s*)', + bygroups(Keyword, Text, Name.Variable, Text, String, Text)), + (r'(%ctor)(\s*)(\{)', bygroups(Keyword, Text, Punctuation), + 'function'), + (r'(%new)(\s*)(\()(.*?)(\))', + bygroups(Keyword, Text, Keyword, String, Keyword)), + (r'(\s*)(%end)(\s*)', bygroups(Text, Keyword, Text)), + inherit, + ], + } + + _logos_keywords = re.compile(r'%(?:hook|ctor|init|c\()') + + def analyse_text(text): + if LogosLexer._logos_keywords.search(text): + return 1.0 + return 0 + + +class SwiftLexer(RegexLexer): + """ + For Swift source. + + .. versionadded:: 2.0 + """ + name = 'Swift' + url = 'https://www.swift.org/' + filenames = ['*.swift'] + aliases = ['swift'] + mimetypes = ['text/x-swift'] + + tokens = { + 'root': [ + # Whitespace and Comments + (r'\n', Text), + (r'\s+', Text), + (r'//', Comment.Single, 'comment-single'), + (r'/\*', Comment.Multiline, 'comment-multi'), + (r'#(if|elseif|else|endif|available)\b', Comment.Preproc, 'preproc'), + + # Keywords + include('keywords'), + + # Global Types + (words(( + 'Array', 'AutoreleasingUnsafeMutablePointer', 'BidirectionalReverseView', + 'Bit', 'Bool', 'CFunctionPointer', 'COpaquePointer', 'CVaListPointer', + 'Character', 'ClosedInterval', 'CollectionOfOne', 'ContiguousArray', + 'Dictionary', 'DictionaryGenerator', 'DictionaryIndex', 'Double', + 'EmptyCollection', 'EmptyGenerator', 'EnumerateGenerator', + 'EnumerateSequence', 'FilterCollectionView', + 'FilterCollectionViewIndex', 'FilterGenerator', 'FilterSequenceView', + 'Float', 'Float80', 'FloatingPointClassification', 'GeneratorOf', + 'GeneratorOfOne', 'GeneratorSequence', 'HalfOpenInterval', 'HeapBuffer', + 'HeapBufferStorage', 'ImplicitlyUnwrappedOptional', 'IndexingGenerator', + 'Int', 'Int16', 'Int32', 'Int64', 'Int8', 'LazyBidirectionalCollection', + 'LazyForwardCollection', 'LazyRandomAccessCollection', + 'LazySequence', 'MapCollectionView', 'MapSequenceGenerator', + 'MapSequenceView', 'MirrorDisposition', 'ObjectIdentifier', 'OnHeap', + 'Optional', 'PermutationGenerator', 'QuickLookObject', + 'RandomAccessReverseView', 'Range', 'RangeGenerator', 'RawByte', 'Repeat', + 'ReverseBidirectionalIndex', 'ReverseRandomAccessIndex', 'SequenceOf', + 'SinkOf', 'Slice', 'StaticString', 'StrideThrough', 'StrideThroughGenerator', + 'StrideTo', 'StrideToGenerator', 'String', 'UInt', 'UInt16', 'UInt32', + 'UInt64', 'UInt8', 'UTF16', 'UTF32', 'UTF8', 'UnicodeDecodingResult', + 'UnicodeScalar', 'Unmanaged', 'UnsafeBufferPointer', + 'UnsafeBufferPointerGenerator', 'UnsafeMutableBufferPointer', + 'UnsafeMutablePointer', 'UnsafePointer', 'Zip2', 'ZipGenerator2', + # Protocols + 'AbsoluteValuable', 'AnyObject', 'ArrayLiteralConvertible', + 'BidirectionalIndexType', 'BitwiseOperationsType', + 'BooleanLiteralConvertible', 'BooleanType', 'CVarArgType', + 'CollectionType', 'Comparable', 'DebugPrintable', + 'DictionaryLiteralConvertible', 'Equatable', + 'ExtendedGraphemeClusterLiteralConvertible', + 'ExtensibleCollectionType', 'FloatLiteralConvertible', + 'FloatingPointType', 'ForwardIndexType', 'GeneratorType', 'Hashable', + 'IntegerArithmeticType', 'IntegerLiteralConvertible', 'IntegerType', + 'IntervalType', 'MirrorType', 'MutableCollectionType', 'MutableSliceable', + 'NilLiteralConvertible', 'OutputStreamType', 'Printable', + 'RandomAccessIndexType', 'RangeReplaceableCollectionType', + 'RawOptionSetType', 'RawRepresentable', 'Reflectable', 'SequenceType', + 'SignedIntegerType', 'SignedNumberType', 'SinkType', 'Sliceable', + 'Streamable', 'Strideable', 'StringInterpolationConvertible', + 'StringLiteralConvertible', 'UnicodeCodecType', + 'UnicodeScalarLiteralConvertible', 'UnsignedIntegerType', + '_ArrayBufferType', '_BidirectionalIndexType', '_CocoaStringType', + '_CollectionType', '_Comparable', '_ExtensibleCollectionType', + '_ForwardIndexType', '_Incrementable', '_IntegerArithmeticType', + '_IntegerType', '_ObjectiveCBridgeable', '_RandomAccessIndexType', + '_RawOptionSetType', '_SequenceType', '_Sequence_Type', + '_SignedIntegerType', '_SignedNumberType', '_Sliceable', '_Strideable', + '_SwiftNSArrayRequiredOverridesType', '_SwiftNSArrayType', + '_SwiftNSCopyingType', '_SwiftNSDictionaryRequiredOverridesType', + '_SwiftNSDictionaryType', '_SwiftNSEnumeratorType', + '_SwiftNSFastEnumerationType', '_SwiftNSStringRequiredOverridesType', + '_SwiftNSStringType', '_UnsignedIntegerType', + # Variables + 'C_ARGC', 'C_ARGV', 'Process', + # Typealiases + 'Any', 'AnyClass', 'BooleanLiteralType', 'CBool', 'CChar', 'CChar16', + 'CChar32', 'CDouble', 'CFloat', 'CInt', 'CLong', 'CLongLong', 'CShort', + 'CSignedChar', 'CUnsignedInt', 'CUnsignedLong', 'CUnsignedShort', + 'CWideChar', 'ExtendedGraphemeClusterType', 'Float32', 'Float64', + 'FloatLiteralType', 'IntMax', 'IntegerLiteralType', 'StringLiteralType', + 'UIntMax', 'UWord', 'UnicodeScalarType', 'Void', 'Word', + # Foundation/Cocoa + 'NSErrorPointer', 'NSObjectProtocol', 'Selector'), suffix=r'\b'), + Name.Builtin), + # Functions + (words(( + 'abs', 'advance', 'alignof', 'alignofValue', 'assert', 'assertionFailure', + 'contains', 'count', 'countElements', 'debugPrint', 'debugPrintln', + 'distance', 'dropFirst', 'dropLast', 'dump', 'enumerate', 'equal', + 'extend', 'fatalError', 'filter', 'find', 'first', 'getVaList', 'indices', + 'insert', 'isEmpty', 'join', 'last', 'lazy', 'lexicographicalCompare', + 'map', 'max', 'maxElement', 'min', 'minElement', 'numericCast', 'overlaps', + 'partition', 'precondition', 'preconditionFailure', 'prefix', 'print', + 'println', 'reduce', 'reflect', 'removeAll', 'removeAtIndex', 'removeLast', + 'removeRange', 'reverse', 'sizeof', 'sizeofValue', 'sort', 'sorted', + 'splice', 'split', 'startsWith', 'stride', 'strideof', 'strideofValue', + 'suffix', 'swap', 'toDebugString', 'toString', 'transcode', + 'underestimateCount', 'unsafeAddressOf', 'unsafeBitCast', 'unsafeDowncast', + 'withExtendedLifetime', 'withUnsafeMutablePointer', + 'withUnsafeMutablePointers', 'withUnsafePointer', 'withUnsafePointers', + 'withVaList'), suffix=r'\b'), + Name.Builtin.Pseudo), + + # Implicit Block Variables + (r'\$\d+', Name.Variable), + + # Binary Literal + (r'0b[01_]+', Number.Bin), + # Octal Literal + (r'0o[0-7_]+', Number.Oct), + # Hexadecimal Literal + (r'0x[0-9a-fA-F_]+', Number.Hex), + # Decimal Literal + (r'[0-9][0-9_]*(\.[0-9_]+[eE][+\-]?[0-9_]+|' + r'\.[0-9_]*|[eE][+\-]?[0-9_]+)', Number.Float), + (r'[0-9][0-9_]*', Number.Integer), + # String Literal + (r'"', String, 'string'), + + # Operators and Punctuation + (r'[(){}\[\].,:;=@#`?]|->|[<&?](?=\w)|(?<=\w)[>!?]', Punctuation), + (r'[/=\-+!*%<>&|^?~]+', Operator), + + # Identifier + (r'[a-zA-Z_]\w*', Name) + ], + 'keywords': [ + (words(( + 'as', 'async', 'await', 'break', 'case', 'catch', 'continue', 'default', 'defer', + 'do', 'else', 'fallthrough', 'for', 'guard', 'if', 'in', 'is', + 'repeat', 'return', '#selector', 'switch', 'throw', 'try', + 'where', 'while'), suffix=r'\b'), + Keyword), + (r'@availability\([^)]+\)', Keyword.Reserved), + (words(( + 'associativity', 'convenience', 'dynamic', 'didSet', 'final', + 'get', 'indirect', 'infix', 'inout', 'lazy', 'left', 'mutating', + 'none', 'nonmutating', 'optional', 'override', 'postfix', + 'precedence', 'prefix', 'Protocol', 'required', 'rethrows', + 'right', 'set', 'throws', 'Type', 'unowned', 'weak', 'willSet', + '@availability', '@autoclosure', '@noreturn', + '@NSApplicationMain', '@NSCopying', '@NSManaged', '@objc', + '@UIApplicationMain', '@IBAction', '@IBDesignable', + '@IBInspectable', '@IBOutlet'), suffix=r'\b'), + Keyword.Reserved), + (r'(as|dynamicType|false|is|nil|self|Self|super|true|__COLUMN__' + r'|__FILE__|__FUNCTION__|__LINE__|_' + r'|#(?:file|line|column|function))\b', Keyword.Constant), + (r'import\b', Keyword.Declaration, 'module'), + (r'(class|enum|extension|struct|protocol)(\s+)([a-zA-Z_]\w*)', + bygroups(Keyword.Declaration, Text, Name.Class)), + (r'(func)(\s+)([a-zA-Z_]\w*)', + bygroups(Keyword.Declaration, Text, Name.Function)), + (r'(var|let)(\s+)([a-zA-Z_]\w*)', bygroups(Keyword.Declaration, + Text, Name.Variable)), + (words(( + 'actor', 'associatedtype', 'class', 'deinit', 'enum', 'extension', 'func', 'import', + 'init', 'internal', 'let', 'operator', 'private', 'protocol', 'public', + 'static', 'struct', 'subscript', 'typealias', 'var'), suffix=r'\b'), + Keyword.Declaration) + ], + 'comment': [ + (r':param: [a-zA-Z_]\w*|:returns?:|(FIXME|MARK|TODO):', + Comment.Special) + ], + + # Nested + 'comment-single': [ + (r'\n', Text, '#pop'), + include('comment'), + (r'[^\n]', Comment.Single) + ], + 'comment-multi': [ + include('comment'), + (r'[^*/]', Comment.Multiline), + (r'/\*', Comment.Multiline, '#push'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[*/]', Comment.Multiline) + ], + 'module': [ + (r'\n', Text, '#pop'), + (r'[a-zA-Z_]\w*', Name.Class), + include('root') + ], + 'preproc': [ + (r'\n', Text, '#pop'), + include('keywords'), + (r'[A-Za-z]\w*', Comment.Preproc), + include('root') + ], + 'string': [ + (r'\\\(', String.Interpol, 'string-intp'), + (r'"', String, '#pop'), + (r"""\\['"\\nrt]|\\x[0-9a-fA-F]{2}|\\[0-7]{1,3}""" + r"""|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}""", String.Escape), + (r'[^\\"]+', String), + (r'\\', String) + ], + 'string-intp': [ + (r'\(', String.Interpol, '#push'), + (r'\)', String.Interpol, '#pop'), + include('root') + ] + } + + def get_tokens_unprocessed(self, text): + from pygments.lexers._cocoa_builtins import COCOA_INTERFACES, \ + COCOA_PROTOCOLS, COCOA_PRIMITIVES + + for index, token, value in \ + RegexLexer.get_tokens_unprocessed(self, text): + if token is Name or token is Name.Class: + if value in COCOA_INTERFACES or value in COCOA_PROTOCOLS \ + or value in COCOA_PRIMITIVES: + token = Name.Builtin.Pseudo + + yield index, token, value diff --git a/pygments/lexers/ooc.py b/pygments/lexers/ooc.py new file mode 100644 index 0000000..5b1df2c --- /dev/null +++ b/pygments/lexers/ooc.py @@ -0,0 +1,85 @@ +""" + pygments.lexers.ooc + ~~~~~~~~~~~~~~~~~~~ + + Lexers for the Ooc language. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups, words +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation + +__all__ = ['OocLexer'] + + +class OocLexer(RegexLexer): + """ + For Ooc source code + + .. versionadded:: 1.2 + """ + name = 'Ooc' + url = 'http://ooc-lang.org/' + aliases = ['ooc'] + filenames = ['*.ooc'] + mimetypes = ['text/x-ooc'] + + tokens = { + 'root': [ + (words(( + 'class', 'interface', 'implement', 'abstract', 'extends', 'from', + 'this', 'super', 'new', 'const', 'final', 'static', 'import', + 'use', 'extern', 'inline', 'proto', 'break', 'continue', + 'fallthrough', 'operator', 'if', 'else', 'for', 'while', 'do', + 'switch', 'case', 'as', 'in', 'version', 'return', 'true', + 'false', 'null'), prefix=r'\b', suffix=r'\b'), + Keyword), + (r'include\b', Keyword, 'include'), + (r'(cover)([ \t]+)(from)([ \t]+)(\w+[*@]?)', + bygroups(Keyword, Text, Keyword, Text, Name.Class)), + (r'(func)((?:[ \t]|\\\n)+)(~[a-z_]\w*)', + bygroups(Keyword, Text, Name.Function)), + (r'\bfunc\b', Keyword), + # Note: %= and ^= not listed on http://ooc-lang.org/syntax + (r'//.*', Comment), + (r'(?s)/\*.*?\*/', Comment.Multiline), + (r'(==?|\+=?|-[=>]?|\*=?|/=?|:=|!=?|%=?|\?|>{1,3}=?|<{1,3}=?|\.\.|' + r'&&?|\|\|?|\^=?)', Operator), + (r'(\.)([ \t]*)([a-z]\w*)', bygroups(Operator, Text, + Name.Function)), + (r'[A-Z][A-Z0-9_]+', Name.Constant), + (r'[A-Z]\w*([@*]|\[[ \t]*\])?', Name.Class), + + (r'([a-z]\w*(?:~[a-z]\w*)?)((?:[ \t]|\\\n)*)(?=\()', + bygroups(Name.Function, Text)), + (r'[a-z]\w*', Name.Variable), + + # : introduces types + (r'[:(){}\[\];,]', Punctuation), + + (r'0x[0-9a-fA-F]+', Number.Hex), + (r'0c[0-9]+', Number.Oct), + (r'0b[01]+', Number.Bin), + (r'[0-9_]\.[0-9_]*(?!\.)', Number.Float), + (r'[0-9_]+', Number.Decimal), + + (r'"(?:\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\"])*"', + String.Double), + (r"'(?:\\.|\\[0-9]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", + String.Char), + (r'@', Punctuation), # pointer dereference + (r'\.', Punctuation), # imports or chain operator + + (r'\\[ \t\n]', Text), + (r'[ \t]+', Text), + ], + 'include': [ + (r'[\w/]+', Name), + (r',', Punctuation), + (r'[ \t]', Text), + (r'[;\n]', Text, '#pop'), + ], + } diff --git a/pygments/lexers/other.py b/pygments/lexers/other.py new file mode 100644 index 0000000..6c73c01 --- /dev/null +++ b/pygments/lexers/other.py @@ -0,0 +1,40 @@ +""" + pygments.lexers.other + ~~~~~~~~~~~~~~~~~~~~~ + + Just export lexer classes previously contained in this module. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexers.sql import SqlLexer, MySqlLexer, SqliteConsoleLexer +from pygments.lexers.shell import BashLexer, BashSessionLexer, BatchLexer, \ + TcshLexer +from pygments.lexers.robotframework import RobotFrameworkLexer +from pygments.lexers.testing import GherkinLexer +from pygments.lexers.esoteric import BrainfuckLexer, BefungeLexer, RedcodeLexer +from pygments.lexers.prolog import LogtalkLexer +from pygments.lexers.snobol import SnobolLexer +from pygments.lexers.rebol import RebolLexer +from pygments.lexers.configs import KconfigLexer, Cfengine3Lexer +from pygments.lexers.modeling import ModelicaLexer +from pygments.lexers.scripting import AppleScriptLexer, MOOCodeLexer, \ + HybrisLexer +from pygments.lexers.graphics import PostScriptLexer, GnuplotLexer, \ + AsymptoteLexer, PovrayLexer +from pygments.lexers.business import ABAPLexer, OpenEdgeLexer, \ + GoodDataCLLexer, MaqlLexer +from pygments.lexers.automation import AutoItLexer, AutohotkeyLexer +from pygments.lexers.dsls import ProtoBufLexer, BroLexer, PuppetLexer, \ + MscgenLexer, VGLLexer +from pygments.lexers.basic import CbmBasicV2Lexer +from pygments.lexers.pawn import SourcePawnLexer, PawnLexer +from pygments.lexers.ecl import ECLLexer +from pygments.lexers.urbi import UrbiscriptLexer +from pygments.lexers.smalltalk import SmalltalkLexer, NewspeakLexer +from pygments.lexers.installers import NSISLexer, RPMSpecLexer +from pygments.lexers.textedit import AwkLexer +from pygments.lexers.smv import NuSMVLexer + +__all__ = [] diff --git a/pygments/lexers/parasail.py b/pygments/lexers/parasail.py new file mode 100644 index 0000000..de50c65 --- /dev/null +++ b/pygments/lexers/parasail.py @@ -0,0 +1,79 @@ +""" + pygments.lexers.parasail + ~~~~~~~~~~~~~~~~~~~~~~~~ + + Lexer for ParaSail. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, include +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Literal + +__all__ = ['ParaSailLexer'] + + +class ParaSailLexer(RegexLexer): + """ + For ParaSail source code. + + .. versionadded:: 2.1 + """ + + name = 'ParaSail' + url = 'http://www.parasail-lang.org' + aliases = ['parasail'] + filenames = ['*.psi', '*.psl'] + mimetypes = ['text/x-parasail'] + + flags = re.MULTILINE + + tokens = { + 'root': [ + (r'[^\S\n]+', Text), + (r'//.*?\n', Comment.Single), + (r'\b(and|or|xor)=', Operator.Word), + (r'\b(and(\s+then)?|or(\s+else)?|xor|rem|mod|' + r'(is|not)\s+null)\b', + Operator.Word), + # Keywords + (r'\b(abs|abstract|all|block|class|concurrent|const|continue|' + r'each|end|exit|extends|exports|forward|func|global|implements|' + r'import|in|interface|is|lambda|locked|new|not|null|of|op|' + r'optional|private|queued|ref|return|reverse|separate|some|' + r'type|until|var|with|' + # Control flow + r'if|then|else|elsif|case|for|while|loop)\b', + Keyword.Reserved), + (r'(abstract\s+)?(interface|class|op|func|type)', + Keyword.Declaration), + # Literals + (r'"[^"]*"', String), + (r'\\[\'ntrf"0]', String.Escape), + (r'#[a-zA-Z]\w*', Literal), # Enumeration + include('numbers'), + (r"'[^']'", String.Char), + (r'[a-zA-Z]\w*', Name), + # Operators and Punctuation + (r'(<==|==>|<=>|\*\*=|<\|=|<<=|>>=|==|!=|=\?|<=|>=|' + r'\*\*|<<|>>|=>|:=|\+=|-=|\*=|\|=|\||/=|\+|-|\*|/|' + r'\.\.|<\.\.|\.\.<|<\.\.<)', + Operator), + (r'(<|>|\[|\]|\(|\)|\||:|;|,|.|\{|\}|->)', + Punctuation), + (r'\n+', Text), + ], + 'numbers': [ + (r'\d[0-9_]*#[0-9a-fA-F][0-9a-fA-F_]*#', Number.Hex), # any base + (r'0[xX][0-9a-fA-F][0-9a-fA-F_]*', Number.Hex), # C-like hex + (r'0[bB][01][01_]*', Number.Bin), # C-like bin + (r'\d[0-9_]*\.\d[0-9_]*[eE][+-]\d[0-9_]*', # float exp + Number.Float), + (r'\d[0-9_]*\.\d[0-9_]*', Number.Float), # float + (r'\d[0-9_]*', Number.Integer), # integer + ], + } diff --git a/pygments/lexers/parsers.py b/pygments/lexers/parsers.py new file mode 100644 index 0000000..20a0cc3 --- /dev/null +++ b/pygments/lexers/parsers.py @@ -0,0 +1,801 @@ +""" + pygments.lexers.parsers + ~~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for parser generators. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, DelegatingLexer, \ + include, bygroups, using +from pygments.token import Punctuation, Other, Text, Comment, Operator, \ + Keyword, Name, String, Number, Whitespace +from pygments.lexers.jvm import JavaLexer +from pygments.lexers.c_cpp import CLexer, CppLexer +from pygments.lexers.objective import ObjectiveCLexer +from pygments.lexers.d import DLexer +from pygments.lexers.dotnet import CSharpLexer +from pygments.lexers.ruby import RubyLexer +from pygments.lexers.python import PythonLexer +from pygments.lexers.perl import PerlLexer + +__all__ = ['RagelLexer', 'RagelEmbeddedLexer', 'RagelCLexer', 'RagelDLexer', + 'RagelCppLexer', 'RagelObjectiveCLexer', 'RagelRubyLexer', + 'RagelJavaLexer', 'AntlrLexer', 'AntlrPythonLexer', + 'AntlrPerlLexer', 'AntlrRubyLexer', 'AntlrCppLexer', + 'AntlrCSharpLexer', 'AntlrObjectiveCLexer', + 'AntlrJavaLexer', 'AntlrActionScriptLexer', + 'TreetopLexer', 'EbnfLexer'] + + +class RagelLexer(RegexLexer): + """A pure `Ragel `_ lexer. Use this + for fragments of Ragel. For ``.rl`` files, use + :class:`RagelEmbeddedLexer` instead (or one of the + language-specific subclasses). + + .. versionadded:: 1.1 + + """ + + name = 'Ragel' + url = 'http://www.colm.net/open-source/ragel/' + aliases = ['ragel'] + filenames = [] + + tokens = { + 'whitespace': [ + (r'\s+', Whitespace) + ], + 'comments': [ + (r'\#.*$', Comment), + ], + 'keywords': [ + (r'(access|action|alphtype)\b', Keyword), + (r'(getkey|write|machine|include)\b', Keyword), + (r'(any|ascii|extend|alpha|digit|alnum|lower|upper)\b', Keyword), + (r'(xdigit|cntrl|graph|print|punct|space|zlen|empty)\b', Keyword) + ], + 'numbers': [ + (r'0x[0-9A-Fa-f]+', Number.Hex), + (r'[+-]?[0-9]+', Number.Integer), + ], + 'literals': [ + (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double), + (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single), + (r'\[(\\\\|\\[^\\]|[^\\\]])*\]', String), # square bracket literals + (r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/', String.Regex), # regular expressions + ], + 'identifiers': [ + (r'[a-zA-Z_]\w*', Name.Variable), + ], + 'operators': [ + (r',', Operator), # Join + (r'\||&|--?', Operator), # Union, Intersection and Subtraction + (r'\.|<:|:>>?', Operator), # Concatention + (r':', Operator), # Label + (r'->', Operator), # Epsilon Transition + (r'(>|\$|%|<|@|<>)(/|eof\b)', Operator), # EOF Actions + (r'(>|\$|%|<|@|<>)(!|err\b)', Operator), # Global Error Actions + (r'(>|\$|%|<|@|<>)(\^|lerr\b)', Operator), # Local Error Actions + (r'(>|\$|%|<|@|<>)(~|to\b)', Operator), # To-State Actions + (r'(>|\$|%|<|@|<>)(\*|from\b)', Operator), # From-State Actions + (r'>|@|\$|%', Operator), # Transition Actions and Priorities + (r'\*|\?|\+|\{[0-9]*,[0-9]*\}', Operator), # Repetition + (r'!|\^', Operator), # Negation + (r'\(|\)', Operator), # Grouping + ], + 'root': [ + include('literals'), + include('whitespace'), + include('comments'), + include('keywords'), + include('numbers'), + include('identifiers'), + include('operators'), + (r'\{', Punctuation, 'host'), + (r'=', Operator), + (r';', Punctuation), + ], + 'host': [ + (r'(' + r'|'.join(( # keep host code in largest possible chunks + r'[^{}\'"/#]+', # exclude unsafe characters + r'[^\\]\\[{}]', # allow escaped { or } + + # strings and comments may safely contain unsafe characters + r'"(\\\\|\\[^\\]|[^"\\])*"', + r"'(\\\\|\\[^\\]|[^'\\])*'", + r'//.*$\n?', # single line comment + r'/\*(.|\n)*?\*/', # multi-line javadoc-style comment + r'\#.*$\n?', # ruby comment + + # regular expression: There's no reason for it to start + # with a * and this stops confusion with comments. + r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/', + + # / is safe now that we've handled regex and javadoc comments + r'/', + )) + r')+', Other), + + (r'\{', Punctuation, '#push'), + (r'\}', Punctuation, '#pop'), + ], + } + + +class RagelEmbeddedLexer(RegexLexer): + """ + A lexer for Ragel embedded in a host language file. + + This will only highlight Ragel statements. If you want host language + highlighting then call the language-specific Ragel lexer. + + .. versionadded:: 1.1 + """ + + name = 'Embedded Ragel' + aliases = ['ragel-em'] + filenames = ['*.rl'] + + tokens = { + 'root': [ + (r'(' + r'|'.join(( # keep host code in largest possible chunks + r'[^%\'"/#]+', # exclude unsafe characters + r'%(?=[^%]|$)', # a single % sign is okay, just not 2 of them + + # strings and comments may safely contain unsafe characters + r'"(\\\\|\\[^\\]|[^"\\])*"', + r"'(\\\\|\\[^\\]|[^'\\])*'", + r'/\*(.|\n)*?\*/', # multi-line javadoc-style comment + r'//.*$\n?', # single line comment + r'\#.*$\n?', # ruby/ragel comment + r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/', # regular expression + + # / is safe now that we've handled regex and javadoc comments + r'/', + )) + r')+', Other), + + # Single Line FSM. + # Please don't put a quoted newline in a single line FSM. + # That's just mean. It will break this. + (r'(%%)(?![{%])(.*)($|;)(\n?)', bygroups(Punctuation, + using(RagelLexer), + Punctuation, Text)), + + # Multi Line FSM. + (r'(%%%%|%%)\{', Punctuation, 'multi-line-fsm'), + ], + 'multi-line-fsm': [ + (r'(' + r'|'.join(( # keep ragel code in largest possible chunks. + r'(' + r'|'.join(( + r'[^}\'"\[/#]', # exclude unsafe characters + r'\}(?=[^%]|$)', # } is okay as long as it's not followed by % + r'\}%(?=[^%]|$)', # ...well, one %'s okay, just not two... + r'[^\\]\\[{}]', # ...and } is okay if it's escaped + + # allow / if it's preceded with one of these symbols + # (ragel EOF actions) + r'(>|\$|%|<|@|<>)/', + + # specifically allow regex followed immediately by * + # so it doesn't get mistaken for a comment + r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/\*', + + # allow / as long as it's not followed by another / or by a * + r'/(?=[^/*]|$)', + + # We want to match as many of these as we can in one block. + # Not sure if we need the + sign here, + # does it help performance? + )) + r')+', + + # strings and comments may safely contain unsafe characters + r'"(\\\\|\\[^\\]|[^"\\])*"', + r"'(\\\\|\\[^\\]|[^'\\])*'", + r"\[(\\\\|\\[^\\]|[^\]\\])*\]", # square bracket literal + r'/\*(.|\n)*?\*/', # multi-line javadoc-style comment + r'//.*$\n?', # single line comment + r'\#.*$\n?', # ruby/ragel comment + )) + r')+', using(RagelLexer)), + + (r'\}%%', Punctuation, '#pop'), + ] + } + + def analyse_text(text): + return '@LANG: indep' in text + + +class RagelRubyLexer(DelegatingLexer): + """ + A lexer for Ragel in a Ruby host file. + + .. versionadded:: 1.1 + """ + + name = 'Ragel in Ruby Host' + aliases = ['ragel-ruby', 'ragel-rb'] + filenames = ['*.rl'] + + def __init__(self, **options): + super().__init__(RubyLexer, RagelEmbeddedLexer, **options) + + def analyse_text(text): + return '@LANG: ruby' in text + + +class RagelCLexer(DelegatingLexer): + """ + A lexer for Ragel in a C host file. + + .. versionadded:: 1.1 + """ + + name = 'Ragel in C Host' + aliases = ['ragel-c'] + filenames = ['*.rl'] + + def __init__(self, **options): + super().__init__(CLexer, RagelEmbeddedLexer, **options) + + def analyse_text(text): + return '@LANG: c' in text + + +class RagelDLexer(DelegatingLexer): + """ + A lexer for Ragel in a D host file. + + .. versionadded:: 1.1 + """ + + name = 'Ragel in D Host' + aliases = ['ragel-d'] + filenames = ['*.rl'] + + def __init__(self, **options): + super().__init__(DLexer, RagelEmbeddedLexer, **options) + + def analyse_text(text): + return '@LANG: d' in text + + +class RagelCppLexer(DelegatingLexer): + """ + A lexer for Ragel in a C++ host file. + + .. versionadded:: 1.1 + """ + + name = 'Ragel in CPP Host' + aliases = ['ragel-cpp'] + filenames = ['*.rl'] + + def __init__(self, **options): + super().__init__(CppLexer, RagelEmbeddedLexer, **options) + + def analyse_text(text): + return '@LANG: c++' in text + + +class RagelObjectiveCLexer(DelegatingLexer): + """ + A lexer for Ragel in an Objective C host file. + + .. versionadded:: 1.1 + """ + + name = 'Ragel in Objective C Host' + aliases = ['ragel-objc'] + filenames = ['*.rl'] + + def __init__(self, **options): + super().__init__(ObjectiveCLexer, RagelEmbeddedLexer, **options) + + def analyse_text(text): + return '@LANG: objc' in text + + +class RagelJavaLexer(DelegatingLexer): + """ + A lexer for Ragel in a Java host file. + + .. versionadded:: 1.1 + """ + + name = 'Ragel in Java Host' + aliases = ['ragel-java'] + filenames = ['*.rl'] + + def __init__(self, **options): + super().__init__(JavaLexer, RagelEmbeddedLexer, **options) + + def analyse_text(text): + return '@LANG: java' in text + + +class AntlrLexer(RegexLexer): + """ + Generic `ANTLR`_ Lexer. + Should not be called directly, instead + use DelegatingLexer for your target language. + + .. versionadded:: 1.1 + + .. _ANTLR: http://www.antlr.org/ + """ + + name = 'ANTLR' + aliases = ['antlr'] + filenames = [] + + _id = r'[A-Za-z]\w*' + _TOKEN_REF = r'[A-Z]\w*' + _RULE_REF = r'[a-z]\w*' + _STRING_LITERAL = r'\'(?:\\\\|\\\'|[^\']*)\'' + _INT = r'[0-9]+' + + tokens = { + 'whitespace': [ + (r'\s+', Whitespace), + ], + 'comments': [ + (r'//.*$', Comment), + (r'/\*(.|\n)*?\*/', Comment), + ], + 'root': [ + include('whitespace'), + include('comments'), + + (r'(lexer|parser|tree)?(\s*)(grammar\b)(\s*)(' + _id + ')(;)', + bygroups(Keyword, Whitespace, Keyword, Whitespace, Name.Class, + Punctuation)), + # optionsSpec + (r'options\b', Keyword, 'options'), + # tokensSpec + (r'tokens\b', Keyword, 'tokens'), + # attrScope + (r'(scope)(\s*)(' + _id + r')(\s*)(\{)', + bygroups(Keyword, Whitespace, Name.Variable, Whitespace, + Punctuation), 'action'), + # exception + (r'(catch|finally)\b', Keyword, 'exception'), + # action + (r'(@' + _id + r')(\s*)(::)?(\s*)(' + _id + r')(\s*)(\{)', + bygroups(Name.Label, Whitespace, Punctuation, Whitespace, + Name.Label, Whitespace, Punctuation), 'action'), + # rule + (r'((?:protected|private|public|fragment)\b)?(\s*)(' + _id + ')(!)?', + bygroups(Keyword, Whitespace, Name.Label, Punctuation), + ('rule-alts', 'rule-prelims')), + ], + 'exception': [ + (r'\n', Whitespace, '#pop'), + (r'\s', Whitespace), + include('comments'), + + (r'\[', Punctuation, 'nested-arg-action'), + (r'\{', Punctuation, 'action'), + ], + 'rule-prelims': [ + include('whitespace'), + include('comments'), + + (r'returns\b', Keyword), + (r'\[', Punctuation, 'nested-arg-action'), + (r'\{', Punctuation, 'action'), + # throwsSpec + (r'(throws)(\s+)(' + _id + ')', + bygroups(Keyword, Whitespace, Name.Label)), + (r'(,)(\s*)(' + _id + ')', + bygroups(Punctuation, Whitespace, Name.Label)), # Additional throws + # optionsSpec + (r'options\b', Keyword, 'options'), + # ruleScopeSpec - scope followed by target language code or name of action + # TODO finish implementing other possibilities for scope + # L173 ANTLRv3.g from ANTLR book + (r'(scope)(\s+)(\{)', bygroups(Keyword, Whitespace, Punctuation), + 'action'), + (r'(scope)(\s+)(' + _id + r')(\s*)(;)', + bygroups(Keyword, Whitespace, Name.Label, Whitespace, Punctuation)), + # ruleAction + (r'(@' + _id + r')(\s*)(\{)', + bygroups(Name.Label, Whitespace, Punctuation), 'action'), + # finished prelims, go to rule alts! + (r':', Punctuation, '#pop') + ], + 'rule-alts': [ + include('whitespace'), + include('comments'), + + # These might need to go in a separate 'block' state triggered by ( + (r'options\b', Keyword, 'options'), + (r':', Punctuation), + + # literals + (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double), + (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single), + (r'<<([^>]|>[^>])>>', String), + # identifiers + # Tokens start with capital letter. + (r'\$?[A-Z_]\w*', Name.Constant), + # Rules start with small letter. + (r'\$?[a-z_]\w*', Name.Variable), + # operators + (r'(\+|\||->|=>|=|\(|\)|\.\.|\.|\?|\*|\^|!|\#|~)', Operator), + (r',', Punctuation), + (r'\[', Punctuation, 'nested-arg-action'), + (r'\{', Punctuation, 'action'), + (r';', Punctuation, '#pop') + ], + 'tokens': [ + include('whitespace'), + include('comments'), + (r'\{', Punctuation), + (r'(' + _TOKEN_REF + r')(\s*)(=)?(\s*)(' + _STRING_LITERAL + + r')?(\s*)(;)', + bygroups(Name.Label, Whitespace, Punctuation, Whitespace, + String, Whitespace, Punctuation)), + (r'\}', Punctuation, '#pop'), + ], + 'options': [ + include('whitespace'), + include('comments'), + (r'\{', Punctuation), + (r'(' + _id + r')(\s*)(=)(\s*)(' + + '|'.join((_id, _STRING_LITERAL, _INT, r'\*')) + r')(\s*)(;)', + bygroups(Name.Variable, Whitespace, Punctuation, Whitespace, + Text, Whitespace, Punctuation)), + (r'\}', Punctuation, '#pop'), + ], + 'action': [ + (r'(' + r'|'.join(( # keep host code in largest possible chunks + r'[^${}\'"/\\]+', # exclude unsafe characters + + # strings and comments may safely contain unsafe characters + r'"(\\\\|\\[^\\]|[^"\\])*"', + r"'(\\\\|\\[^\\]|[^'\\])*'", + r'//.*$\n?', # single line comment + r'/\*(.|\n)*?\*/', # multi-line javadoc-style comment + + # regular expression: There's no reason for it to start + # with a * and this stops confusion with comments. + r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/', + + # backslashes are okay, as long as we are not backslashing a % + r'\\(?!%)', + + # Now that we've handled regex and javadoc comments + # it's safe to let / through. + r'/', + )) + r')+', Other), + (r'(\\)(%)', bygroups(Punctuation, Other)), + (r'(\$[a-zA-Z]+)(\.?)(text|value)?', + bygroups(Name.Variable, Punctuation, Name.Property)), + (r'\{', Punctuation, '#push'), + (r'\}', Punctuation, '#pop'), + ], + 'nested-arg-action': [ + (r'(' + r'|'.join(( # keep host code in largest possible chunks. + r'[^$\[\]\'"/]+', # exclude unsafe characters + + # strings and comments may safely contain unsafe characters + r'"(\\\\|\\[^\\]|[^"\\])*"', + r"'(\\\\|\\[^\\]|[^'\\])*'", + r'//.*$\n?', # single line comment + r'/\*(.|\n)*?\*/', # multi-line javadoc-style comment + + # regular expression: There's no reason for it to start + # with a * and this stops confusion with comments. + r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/', + + # Now that we've handled regex and javadoc comments + # it's safe to let / through. + r'/', + )) + r')+', Other), + + + (r'\[', Punctuation, '#push'), + (r'\]', Punctuation, '#pop'), + (r'(\$[a-zA-Z]+)(\.?)(text|value)?', + bygroups(Name.Variable, Punctuation, Name.Property)), + (r'(\\\\|\\\]|\\\[|[^\[\]])+', Other), + ] + } + + def analyse_text(text): + return re.search(r'^\s*grammar\s+[a-zA-Z0-9]+\s*;', text, re.M) + + +# http://www.antlr.org/wiki/display/ANTLR3/Code+Generation+Targets + +class AntlrCppLexer(DelegatingLexer): + """ + ANTLR with C++ Target + + .. versionadded:: 1.1 + """ + + name = 'ANTLR With CPP Target' + aliases = ['antlr-cpp'] + filenames = ['*.G', '*.g'] + + def __init__(self, **options): + super().__init__(CppLexer, AntlrLexer, **options) + + def analyse_text(text): + return AntlrLexer.analyse_text(text) and \ + re.search(r'^\s*language\s*=\s*C\s*;', text, re.M) + + +class AntlrObjectiveCLexer(DelegatingLexer): + """ + ANTLR with Objective-C Target + + .. versionadded:: 1.1 + """ + + name = 'ANTLR With ObjectiveC Target' + aliases = ['antlr-objc'] + filenames = ['*.G', '*.g'] + + def __init__(self, **options): + super().__init__(ObjectiveCLexer, AntlrLexer, **options) + + def analyse_text(text): + return AntlrLexer.analyse_text(text) and \ + re.search(r'^\s*language\s*=\s*ObjC\s*;', text) + + +class AntlrCSharpLexer(DelegatingLexer): + """ + ANTLR with C# Target + + .. versionadded:: 1.1 + """ + + name = 'ANTLR With C# Target' + aliases = ['antlr-csharp', 'antlr-c#'] + filenames = ['*.G', '*.g'] + + def __init__(self, **options): + super().__init__(CSharpLexer, AntlrLexer, **options) + + def analyse_text(text): + return AntlrLexer.analyse_text(text) and \ + re.search(r'^\s*language\s*=\s*CSharp2\s*;', text, re.M) + + +class AntlrPythonLexer(DelegatingLexer): + """ + ANTLR with Python Target + + .. versionadded:: 1.1 + """ + + name = 'ANTLR With Python Target' + aliases = ['antlr-python'] + filenames = ['*.G', '*.g'] + + def __init__(self, **options): + super().__init__(PythonLexer, AntlrLexer, **options) + + def analyse_text(text): + return AntlrLexer.analyse_text(text) and \ + re.search(r'^\s*language\s*=\s*Python\s*;', text, re.M) + + +class AntlrJavaLexer(DelegatingLexer): + """ + ANTLR with Java Target + + .. versionadded:: 1. + """ + + name = 'ANTLR With Java Target' + aliases = ['antlr-java'] + filenames = ['*.G', '*.g'] + + def __init__(self, **options): + super().__init__(JavaLexer, AntlrLexer, **options) + + def analyse_text(text): + # Antlr language is Java by default + return AntlrLexer.analyse_text(text) and 0.9 + + +class AntlrRubyLexer(DelegatingLexer): + """ + ANTLR with Ruby Target + + .. versionadded:: 1.1 + """ + + name = 'ANTLR With Ruby Target' + aliases = ['antlr-ruby', 'antlr-rb'] + filenames = ['*.G', '*.g'] + + def __init__(self, **options): + super().__init__(RubyLexer, AntlrLexer, **options) + + def analyse_text(text): + return AntlrLexer.analyse_text(text) and \ + re.search(r'^\s*language\s*=\s*Ruby\s*;', text, re.M) + + +class AntlrPerlLexer(DelegatingLexer): + """ + ANTLR with Perl Target + + .. versionadded:: 1.1 + """ + + name = 'ANTLR With Perl Target' + aliases = ['antlr-perl'] + filenames = ['*.G', '*.g'] + + def __init__(self, **options): + super().__init__(PerlLexer, AntlrLexer, **options) + + def analyse_text(text): + return AntlrLexer.analyse_text(text) and \ + re.search(r'^\s*language\s*=\s*Perl5\s*;', text, re.M) + + +class AntlrActionScriptLexer(DelegatingLexer): + """ + ANTLR with ActionScript Target + + .. versionadded:: 1.1 + """ + + name = 'ANTLR With ActionScript Target' + aliases = ['antlr-actionscript', 'antlr-as'] + filenames = ['*.G', '*.g'] + + def __init__(self, **options): + from pygments.lexers.actionscript import ActionScriptLexer + super().__init__(ActionScriptLexer, AntlrLexer, **options) + + def analyse_text(text): + return AntlrLexer.analyse_text(text) and \ + re.search(r'^\s*language\s*=\s*ActionScript\s*;', text, re.M) + + +class TreetopBaseLexer(RegexLexer): + """ + A base lexer for `Treetop `_ grammars. + Not for direct use; use :class:`TreetopLexer` instead. + + .. versionadded:: 1.6 + """ + + tokens = { + 'root': [ + include('space'), + (r'require[ \t]+[^\n\r]+[\n\r]', Other), + (r'module\b', Keyword.Namespace, 'module'), + (r'grammar\b', Keyword, 'grammar'), + ], + 'module': [ + include('space'), + include('end'), + (r'module\b', Keyword, '#push'), + (r'grammar\b', Keyword, 'grammar'), + (r'[A-Z]\w*(?:::[A-Z]\w*)*', Name.Namespace), + ], + 'grammar': [ + include('space'), + include('end'), + (r'rule\b', Keyword, 'rule'), + (r'include\b', Keyword, 'include'), + (r'[A-Z]\w*', Name), + ], + 'include': [ + include('space'), + (r'[A-Z]\w*(?:::[A-Z]\w*)*', Name.Class, '#pop'), + ], + 'rule': [ + include('space'), + include('end'), + (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double), + (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single), + (r'([A-Za-z_]\w*)(:)', bygroups(Name.Label, Punctuation)), + (r'[A-Za-z_]\w*', Name), + (r'[()]', Punctuation), + (r'[?+*/&!~]', Operator), + (r'\[(?:\\.|\[:\^?[a-z]+:\]|[^\\\]])+\]', String.Regex), + (r'([0-9]*)(\.\.)([0-9]*)', + bygroups(Number.Integer, Operator, Number.Integer)), + (r'(<)([^>]+)(>)', bygroups(Punctuation, Name.Class, Punctuation)), + (r'\{', Punctuation, 'inline_module'), + (r'\.', String.Regex), + ], + 'inline_module': [ + (r'\{', Other, 'ruby'), + (r'\}', Punctuation, '#pop'), + (r'[^{}]+', Other), + ], + 'ruby': [ + (r'\{', Other, '#push'), + (r'\}', Other, '#pop'), + (r'[^{}]+', Other), + ], + 'space': [ + (r'[ \t\n\r]+', Whitespace), + (r'#[^\n]*', Comment.Single), + ], + 'end': [ + (r'end\b', Keyword, '#pop'), + ], + } + + +class TreetopLexer(DelegatingLexer): + """ + A lexer for `Treetop `_ grammars. + + .. versionadded:: 1.6 + """ + + name = 'Treetop' + aliases = ['treetop'] + filenames = ['*.treetop', '*.tt'] + + def __init__(self, **options): + super().__init__(RubyLexer, TreetopBaseLexer, **options) + + +class EbnfLexer(RegexLexer): + """ + Lexer for `ISO/IEC 14977 EBNF + `_ + grammars. + + .. versionadded:: 2.0 + """ + + name = 'EBNF' + aliases = ['ebnf'] + filenames = ['*.ebnf'] + mimetypes = ['text/x-ebnf'] + + tokens = { + 'root': [ + include('whitespace'), + include('comment_start'), + include('identifier'), + (r'=', Operator, 'production'), + ], + 'production': [ + include('whitespace'), + include('comment_start'), + include('identifier'), + (r'"[^"]*"', String.Double), + (r"'[^']*'", String.Single), + (r'(\?[^?]*\?)', Name.Entity), + (r'[\[\]{}(),|]', Punctuation), + (r'-', Operator), + (r';', Punctuation, '#pop'), + (r'\.', Punctuation, '#pop'), + ], + 'whitespace': [ + (r'\s+', Text), + ], + 'comment_start': [ + (r'\(\*', Comment.Multiline, 'comment'), + ], + 'comment': [ + (r'[^*)]', Comment.Multiline), + include('comment_start'), + (r'\*\)', Comment.Multiline, '#pop'), + (r'[*)]', Comment.Multiline), + ], + 'identifier': [ + (r'([a-zA-Z][\w \-]*)', Keyword), + ], + } diff --git a/pygments/lexers/pascal.py b/pygments/lexers/pascal.py new file mode 100644 index 0000000..cab170d --- /dev/null +++ b/pygments/lexers/pascal.py @@ -0,0 +1,641 @@ +""" + pygments.lexers.pascal + ~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for Pascal family languages. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import Lexer +from pygments.util import get_bool_opt, get_list_opt +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Error, Whitespace +from pygments.scanner import Scanner + +# compatibility import +from pygments.lexers.modula2 import Modula2Lexer + +__all__ = ['DelphiLexer', 'PortugolLexer'] + + +class PortugolLexer(Lexer): + """For Portugol, a Pascal dialect with keywords in Portuguese.""" + name = 'Portugol' + aliases = ['portugol'] + filenames = ['*.alg', '*.portugol'] + mimetypes = [] + url = "https://www.apoioinformatica.inf.br/produtos/visualg/linguagem" + + def __init__(self, **options): + Lexer.__init__(self, **options) + self.lexer = DelphiLexer(**options, portugol=True) + + def get_tokens_unprocessed(self, text): + return self.lexer.get_tokens_unprocessed(text) + + +class DelphiLexer(Lexer): + """ + For Delphi (Borland Object Pascal), + Turbo Pascal and Free Pascal source code. + + Additional options accepted: + + `turbopascal` + Highlight Turbo Pascal specific keywords (default: ``True``). + `delphi` + Highlight Borland Delphi specific keywords (default: ``True``). + `freepascal` + Highlight Free Pascal specific keywords (default: ``True``). + `units` + A list of units that should be considered builtin, supported are + ``System``, ``SysUtils``, ``Classes`` and ``Math``. + Default is to consider all of them builtin. + """ + name = 'Delphi' + aliases = ['delphi', 'pas', 'pascal', 'objectpascal'] + filenames = ['*.pas', '*.dpr'] + mimetypes = ['text/x-pascal'] + + TURBO_PASCAL_KEYWORDS = ( + 'absolute', 'and', 'array', 'asm', 'begin', 'break', 'case', + 'const', 'constructor', 'continue', 'destructor', 'div', 'do', + 'downto', 'else', 'end', 'file', 'for', 'function', 'goto', + 'if', 'implementation', 'in', 'inherited', 'inline', 'interface', + 'label', 'mod', 'nil', 'not', 'object', 'of', 'on', 'operator', + 'or', 'packed', 'procedure', 'program', 'record', 'reintroduce', + 'repeat', 'self', 'set', 'shl', 'shr', 'string', 'then', 'to', + 'type', 'unit', 'until', 'uses', 'var', 'while', 'with', 'xor' + ) + + DELPHI_KEYWORDS = ( + 'as', 'class', 'except', 'exports', 'finalization', 'finally', + 'initialization', 'is', 'library', 'on', 'property', 'raise', + 'threadvar', 'try' + ) + + FREE_PASCAL_KEYWORDS = ( + 'dispose', 'exit', 'false', 'new', 'true' + ) + + BLOCK_KEYWORDS = { + 'begin', 'class', 'const', 'constructor', 'destructor', 'end', + 'finalization', 'function', 'implementation', 'initialization', + 'label', 'library', 'operator', 'procedure', 'program', 'property', + 'record', 'threadvar', 'type', 'unit', 'uses', 'var' + } + + FUNCTION_MODIFIERS = { + 'alias', 'cdecl', 'export', 'inline', 'interrupt', 'nostackframe', + 'pascal', 'register', 'safecall', 'softfloat', 'stdcall', + 'varargs', 'name', 'dynamic', 'near', 'virtual', 'external', + 'override', 'assembler' + } + + # XXX: those aren't global. but currently we know no way for defining + # them just for the type context. + DIRECTIVES = { + 'absolute', 'abstract', 'assembler', 'cppdecl', 'default', 'far', + 'far16', 'forward', 'index', 'oldfpccall', 'private', 'protected', + 'published', 'public' + } + + BUILTIN_TYPES = { + 'ansichar', 'ansistring', 'bool', 'boolean', 'byte', 'bytebool', + 'cardinal', 'char', 'comp', 'currency', 'double', 'dword', + 'extended', 'int64', 'integer', 'iunknown', 'longbool', 'longint', + 'longword', 'pansichar', 'pansistring', 'pbool', 'pboolean', + 'pbyte', 'pbytearray', 'pcardinal', 'pchar', 'pcomp', 'pcurrency', + 'pdate', 'pdatetime', 'pdouble', 'pdword', 'pextended', 'phandle', + 'pint64', 'pinteger', 'plongint', 'plongword', 'pointer', + 'ppointer', 'pshortint', 'pshortstring', 'psingle', 'psmallint', + 'pstring', 'pvariant', 'pwidechar', 'pwidestring', 'pword', + 'pwordarray', 'pwordbool', 'real', 'real48', 'shortint', + 'shortstring', 'single', 'smallint', 'string', 'tclass', 'tdate', + 'tdatetime', 'textfile', 'thandle', 'tobject', 'ttime', 'variant', + 'widechar', 'widestring', 'word', 'wordbool' + } + + BUILTIN_UNITS = { + 'System': ( + 'abs', 'acquireexceptionobject', 'addr', 'ansitoutf8', + 'append', 'arctan', 'assert', 'assigned', 'assignfile', + 'beginthread', 'blockread', 'blockwrite', 'break', 'chdir', + 'chr', 'close', 'closefile', 'comptocurrency', 'comptodouble', + 'concat', 'continue', 'copy', 'cos', 'dec', 'delete', + 'dispose', 'doubletocomp', 'endthread', 'enummodules', + 'enumresourcemodules', 'eof', 'eoln', 'erase', 'exceptaddr', + 'exceptobject', 'exclude', 'exit', 'exp', 'filepos', 'filesize', + 'fillchar', 'finalize', 'findclasshinstance', 'findhinstance', + 'findresourcehinstance', 'flush', 'frac', 'freemem', + 'get8087cw', 'getdir', 'getlasterror', 'getmem', + 'getmemorymanager', 'getmodulefilename', 'getvariantmanager', + 'halt', 'hi', 'high', 'inc', 'include', 'initialize', 'insert', + 'int', 'ioresult', 'ismemorymanagerset', 'isvariantmanagerset', + 'length', 'ln', 'lo', 'low', 'mkdir', 'move', 'new', 'odd', + 'olestrtostring', 'olestrtostrvar', 'ord', 'paramcount', + 'paramstr', 'pi', 'pos', 'pred', 'ptr', 'pucs4chars', 'random', + 'randomize', 'read', 'readln', 'reallocmem', + 'releaseexceptionobject', 'rename', 'reset', 'rewrite', 'rmdir', + 'round', 'runerror', 'seek', 'seekeof', 'seekeoln', + 'set8087cw', 'setlength', 'setlinebreakstyle', + 'setmemorymanager', 'setstring', 'settextbuf', + 'setvariantmanager', 'sin', 'sizeof', 'slice', 'sqr', 'sqrt', + 'str', 'stringofchar', 'stringtoolestr', 'stringtowidechar', + 'succ', 'swap', 'trunc', 'truncate', 'typeinfo', + 'ucs4stringtowidestring', 'unicodetoutf8', 'uniquestring', + 'upcase', 'utf8decode', 'utf8encode', 'utf8toansi', + 'utf8tounicode', 'val', 'vararrayredim', 'varclear', + 'widecharlentostring', 'widecharlentostrvar', + 'widechartostring', 'widechartostrvar', + 'widestringtoucs4string', 'write', 'writeln' + ), + 'SysUtils': ( + 'abort', 'addexitproc', 'addterminateproc', 'adjustlinebreaks', + 'allocmem', 'ansicomparefilename', 'ansicomparestr', + 'ansicomparetext', 'ansidequotedstr', 'ansiextractquotedstr', + 'ansilastchar', 'ansilowercase', 'ansilowercasefilename', + 'ansipos', 'ansiquotedstr', 'ansisamestr', 'ansisametext', + 'ansistrcomp', 'ansistricomp', 'ansistrlastchar', 'ansistrlcomp', + 'ansistrlicomp', 'ansistrlower', 'ansistrpos', 'ansistrrscan', + 'ansistrscan', 'ansistrupper', 'ansiuppercase', + 'ansiuppercasefilename', 'appendstr', 'assignstr', 'beep', + 'booltostr', 'bytetocharindex', 'bytetocharlen', 'bytetype', + 'callterminateprocs', 'changefileext', 'charlength', + 'chartobyteindex', 'chartobytelen', 'comparemem', 'comparestr', + 'comparetext', 'createdir', 'createguid', 'currentyear', + 'currtostr', 'currtostrf', 'date', 'datetimetofiledate', + 'datetimetostr', 'datetimetostring', 'datetimetosystemtime', + 'datetimetotimestamp', 'datetostr', 'dayofweek', 'decodedate', + 'decodedatefully', 'decodetime', 'deletefile', 'directoryexists', + 'diskfree', 'disksize', 'disposestr', 'encodedate', 'encodetime', + 'exceptionerrormessage', 'excludetrailingbackslash', + 'excludetrailingpathdelimiter', 'expandfilename', + 'expandfilenamecase', 'expanduncfilename', 'extractfiledir', + 'extractfiledrive', 'extractfileext', 'extractfilename', + 'extractfilepath', 'extractrelativepath', 'extractshortpathname', + 'fileage', 'fileclose', 'filecreate', 'filedatetodatetime', + 'fileexists', 'filegetattr', 'filegetdate', 'fileisreadonly', + 'fileopen', 'fileread', 'filesearch', 'fileseek', 'filesetattr', + 'filesetdate', 'filesetreadonly', 'filewrite', 'finalizepackage', + 'findclose', 'findcmdlineswitch', 'findfirst', 'findnext', + 'floattocurr', 'floattodatetime', 'floattodecimal', 'floattostr', + 'floattostrf', 'floattotext', 'floattotextfmt', 'fmtloadstr', + 'fmtstr', 'forcedirectories', 'format', 'formatbuf', 'formatcurr', + 'formatdatetime', 'formatfloat', 'freeandnil', 'getcurrentdir', + 'getenvironmentvariable', 'getfileversion', 'getformatsettings', + 'getlocaleformatsettings', 'getmodulename', 'getpackagedescription', + 'getpackageinfo', 'gettime', 'guidtostring', 'incamonth', + 'includetrailingbackslash', 'includetrailingpathdelimiter', + 'incmonth', 'initializepackage', 'interlockeddecrement', + 'interlockedexchange', 'interlockedexchangeadd', + 'interlockedincrement', 'inttohex', 'inttostr', 'isdelimiter', + 'isequalguid', 'isleapyear', 'ispathdelimiter', 'isvalidident', + 'languages', 'lastdelimiter', 'loadpackage', 'loadstr', + 'lowercase', 'msecstotimestamp', 'newstr', 'nextcharindex', 'now', + 'outofmemoryerror', 'quotedstr', 'raiselastoserror', + 'raiselastwin32error', 'removedir', 'renamefile', 'replacedate', + 'replacetime', 'safeloadlibrary', 'samefilename', 'sametext', + 'setcurrentdir', 'showexception', 'sleep', 'stralloc', 'strbufsize', + 'strbytetype', 'strcat', 'strcharlength', 'strcomp', 'strcopy', + 'strdispose', 'strecopy', 'strend', 'strfmt', 'stricomp', + 'stringreplace', 'stringtoguid', 'strlcat', 'strlcomp', 'strlcopy', + 'strlen', 'strlfmt', 'strlicomp', 'strlower', 'strmove', 'strnew', + 'strnextchar', 'strpas', 'strpcopy', 'strplcopy', 'strpos', + 'strrscan', 'strscan', 'strtobool', 'strtobooldef', 'strtocurr', + 'strtocurrdef', 'strtodate', 'strtodatedef', 'strtodatetime', + 'strtodatetimedef', 'strtofloat', 'strtofloatdef', 'strtoint', + 'strtoint64', 'strtoint64def', 'strtointdef', 'strtotime', + 'strtotimedef', 'strupper', 'supports', 'syserrormessage', + 'systemtimetodatetime', 'texttofloat', 'time', 'timestamptodatetime', + 'timestamptomsecs', 'timetostr', 'trim', 'trimleft', 'trimright', + 'tryencodedate', 'tryencodetime', 'tryfloattocurr', 'tryfloattodatetime', + 'trystrtobool', 'trystrtocurr', 'trystrtodate', 'trystrtodatetime', + 'trystrtofloat', 'trystrtoint', 'trystrtoint64', 'trystrtotime', + 'unloadpackage', 'uppercase', 'widecomparestr', 'widecomparetext', + 'widefmtstr', 'wideformat', 'wideformatbuf', 'widelowercase', + 'widesamestr', 'widesametext', 'wideuppercase', 'win32check', + 'wraptext' + ), + 'Classes': ( + 'activateclassgroup', 'allocatehwnd', 'bintohex', 'checksynchronize', + 'collectionsequal', 'countgenerations', 'deallocatehwnd', 'equalrect', + 'extractstrings', 'findclass', 'findglobalcomponent', 'getclass', + 'groupdescendantswith', 'hextobin', 'identtoint', + 'initinheritedcomponent', 'inttoident', 'invalidpoint', + 'isuniqueglobalcomponentname', 'linestart', 'objectbinarytotext', + 'objectresourcetotext', 'objecttexttobinary', 'objecttexttoresource', + 'pointsequal', 'readcomponentres', 'readcomponentresex', + 'readcomponentresfile', 'rect', 'registerclass', 'registerclassalias', + 'registerclasses', 'registercomponents', 'registerintegerconsts', + 'registernoicon', 'registernonactivex', 'smallpoint', 'startclassgroup', + 'teststreamformat', 'unregisterclass', 'unregisterclasses', + 'unregisterintegerconsts', 'unregistermoduleclasses', + 'writecomponentresfile' + ), + 'Math': ( + 'arccos', 'arccosh', 'arccot', 'arccoth', 'arccsc', 'arccsch', 'arcsec', + 'arcsech', 'arcsin', 'arcsinh', 'arctan2', 'arctanh', 'ceil', + 'comparevalue', 'cosecant', 'cosh', 'cot', 'cotan', 'coth', 'csc', + 'csch', 'cycletodeg', 'cycletograd', 'cycletorad', 'degtocycle', + 'degtograd', 'degtorad', 'divmod', 'doubledecliningbalance', + 'ensurerange', 'floor', 'frexp', 'futurevalue', 'getexceptionmask', + 'getprecisionmode', 'getroundmode', 'gradtocycle', 'gradtodeg', + 'gradtorad', 'hypot', 'inrange', 'interestpayment', 'interestrate', + 'internalrateofreturn', 'intpower', 'isinfinite', 'isnan', 'iszero', + 'ldexp', 'lnxp1', 'log10', 'log2', 'logn', 'max', 'maxintvalue', + 'maxvalue', 'mean', 'meanandstddev', 'min', 'minintvalue', 'minvalue', + 'momentskewkurtosis', 'netpresentvalue', 'norm', 'numberofperiods', + 'payment', 'periodpayment', 'poly', 'popnstddev', 'popnvariance', + 'power', 'presentvalue', 'radtocycle', 'radtodeg', 'radtograd', + 'randg', 'randomrange', 'roundto', 'samevalue', 'sec', 'secant', + 'sech', 'setexceptionmask', 'setprecisionmode', 'setroundmode', + 'sign', 'simpleroundto', 'sincos', 'sinh', 'slndepreciation', 'stddev', + 'sum', 'sumint', 'sumofsquares', 'sumsandsquares', 'syddepreciation', + 'tan', 'tanh', 'totalvariance', 'variance' + ) + } + + ASM_REGISTERS = { + 'ah', 'al', 'ax', 'bh', 'bl', 'bp', 'bx', 'ch', 'cl', 'cr0', + 'cr1', 'cr2', 'cr3', 'cr4', 'cs', 'cx', 'dh', 'di', 'dl', 'dr0', + 'dr1', 'dr2', 'dr3', 'dr4', 'dr5', 'dr6', 'dr7', 'ds', 'dx', + 'eax', 'ebp', 'ebx', 'ecx', 'edi', 'edx', 'es', 'esi', 'esp', + 'fs', 'gs', 'mm0', 'mm1', 'mm2', 'mm3', 'mm4', 'mm5', 'mm6', + 'mm7', 'si', 'sp', 'ss', 'st0', 'st1', 'st2', 'st3', 'st4', 'st5', + 'st6', 'st7', 'xmm0', 'xmm1', 'xmm2', 'xmm3', 'xmm4', 'xmm5', + 'xmm6', 'xmm7' + } + + ASM_INSTRUCTIONS = { + 'aaa', 'aad', 'aam', 'aas', 'adc', 'add', 'and', 'arpl', 'bound', + 'bsf', 'bsr', 'bswap', 'bt', 'btc', 'btr', 'bts', 'call', 'cbw', + 'cdq', 'clc', 'cld', 'cli', 'clts', 'cmc', 'cmova', 'cmovae', + 'cmovb', 'cmovbe', 'cmovc', 'cmovcxz', 'cmove', 'cmovg', + 'cmovge', 'cmovl', 'cmovle', 'cmovna', 'cmovnae', 'cmovnb', + 'cmovnbe', 'cmovnc', 'cmovne', 'cmovng', 'cmovnge', 'cmovnl', + 'cmovnle', 'cmovno', 'cmovnp', 'cmovns', 'cmovnz', 'cmovo', + 'cmovp', 'cmovpe', 'cmovpo', 'cmovs', 'cmovz', 'cmp', 'cmpsb', + 'cmpsd', 'cmpsw', 'cmpxchg', 'cmpxchg486', 'cmpxchg8b', 'cpuid', + 'cwd', 'cwde', 'daa', 'das', 'dec', 'div', 'emms', 'enter', 'hlt', + 'ibts', 'icebp', 'idiv', 'imul', 'in', 'inc', 'insb', 'insd', + 'insw', 'int', 'int01', 'int03', 'int1', 'int3', 'into', 'invd', + 'invlpg', 'iret', 'iretd', 'iretw', 'ja', 'jae', 'jb', 'jbe', + 'jc', 'jcxz', 'jcxz', 'je', 'jecxz', 'jg', 'jge', 'jl', 'jle', + 'jmp', 'jna', 'jnae', 'jnb', 'jnbe', 'jnc', 'jne', 'jng', 'jnge', + 'jnl', 'jnle', 'jno', 'jnp', 'jns', 'jnz', 'jo', 'jp', 'jpe', + 'jpo', 'js', 'jz', 'lahf', 'lar', 'lcall', 'lds', 'lea', 'leave', + 'les', 'lfs', 'lgdt', 'lgs', 'lidt', 'ljmp', 'lldt', 'lmsw', + 'loadall', 'loadall286', 'lock', 'lodsb', 'lodsd', 'lodsw', + 'loop', 'loope', 'loopne', 'loopnz', 'loopz', 'lsl', 'lss', 'ltr', + 'mov', 'movd', 'movq', 'movsb', 'movsd', 'movsw', 'movsx', + 'movzx', 'mul', 'neg', 'nop', 'not', 'or', 'out', 'outsb', 'outsd', + 'outsw', 'pop', 'popa', 'popad', 'popaw', 'popf', 'popfd', 'popfw', + 'push', 'pusha', 'pushad', 'pushaw', 'pushf', 'pushfd', 'pushfw', + 'rcl', 'rcr', 'rdmsr', 'rdpmc', 'rdshr', 'rdtsc', 'rep', 'repe', + 'repne', 'repnz', 'repz', 'ret', 'retf', 'retn', 'rol', 'ror', + 'rsdc', 'rsldt', 'rsm', 'sahf', 'sal', 'salc', 'sar', 'sbb', + 'scasb', 'scasd', 'scasw', 'seta', 'setae', 'setb', 'setbe', + 'setc', 'setcxz', 'sete', 'setg', 'setge', 'setl', 'setle', + 'setna', 'setnae', 'setnb', 'setnbe', 'setnc', 'setne', 'setng', + 'setnge', 'setnl', 'setnle', 'setno', 'setnp', 'setns', 'setnz', + 'seto', 'setp', 'setpe', 'setpo', 'sets', 'setz', 'sgdt', 'shl', + 'shld', 'shr', 'shrd', 'sidt', 'sldt', 'smi', 'smint', 'smintold', + 'smsw', 'stc', 'std', 'sti', 'stosb', 'stosd', 'stosw', 'str', + 'sub', 'svdc', 'svldt', 'svts', 'syscall', 'sysenter', 'sysexit', + 'sysret', 'test', 'ud1', 'ud2', 'umov', 'verr', 'verw', 'wait', + 'wbinvd', 'wrmsr', 'wrshr', 'xadd', 'xbts', 'xchg', 'xlat', + 'xlatb', 'xor' + } + + PORTUGOL_KEYWORDS = ( + 'aleatorio', + 'algoritmo', + 'arquivo', + 'ate', + 'caso', + 'cronometro', + 'debug', + 'e', + 'eco', + 'enquanto', + 'entao', + 'escolha', + 'escreva', + 'escreval', + 'faca', + 'falso', + 'fimalgoritmo', + 'fimenquanto', + 'fimescolha', + 'fimfuncao', + 'fimpara', + 'fimprocedimento', + 'fimrepita', + 'fimse', + 'funcao', + 'inicio', + 'int', + 'interrompa', + 'leia', + 'limpatela', + 'mod', + 'nao', + 'ou', + 'outrocaso', + 'para', + 'passo', + 'pausa', + 'procedimento', + 'repita', + 'retorne', + 'se', + 'senao', + 'timer', + 'var', + 'vetor', + 'verdadeiro', + 'xou', + 'div', + 'mod', + 'abs', + 'arccos', + 'arcsen', + 'arctan', + 'cos', + 'cotan', + 'Exp', + 'grauprad', + 'int', + 'log', + 'logn', + 'pi', + 'quad', + 'radpgrau', + 'raizq', + 'rand', + 'randi', + 'sen', + 'Tan', + 'asc', + 'carac', + 'caracpnum', + 'compr', + 'copia', + 'maiusc', + 'minusc', + 'numpcarac', + 'pos', + ) + + PORTUGOL_BUILTIN_TYPES = { + 'inteiro', 'real', 'caractere', 'logico' + } + + def __init__(self, **options): + Lexer.__init__(self, **options) + self.keywords = set() + self.builtins = set() + if get_bool_opt(options, 'portugol', False): + self.keywords.update(self.PORTUGOL_KEYWORDS) + self.builtins.update(self.PORTUGOL_BUILTIN_TYPES) + self.is_portugol = True + else: + self.is_portugol = False + + if get_bool_opt(options, 'turbopascal', True): + self.keywords.update(self.TURBO_PASCAL_KEYWORDS) + if get_bool_opt(options, 'delphi', True): + self.keywords.update(self.DELPHI_KEYWORDS) + if get_bool_opt(options, 'freepascal', True): + self.keywords.update(self.FREE_PASCAL_KEYWORDS) + for unit in get_list_opt(options, 'units', list(self.BUILTIN_UNITS)): + self.builtins.update(self.BUILTIN_UNITS[unit]) + + def get_tokens_unprocessed(self, text): + scanner = Scanner(text, re.DOTALL | re.MULTILINE | re.IGNORECASE) + stack = ['initial'] + in_function_block = False + in_property_block = False + was_dot = False + next_token_is_function = False + next_token_is_property = False + collect_labels = False + block_labels = set() + brace_balance = [0, 0] + + while not scanner.eos: + token = Error + + if stack[-1] == 'initial': + if scanner.scan(r'\s+'): + token = Whitespace + elif not self.is_portugol and scanner.scan(r'\{.*?\}|\(\*.*?\*\)'): + if scanner.match.startswith('$'): + token = Comment.Preproc + else: + token = Comment.Multiline + elif scanner.scan(r'//.*?$'): + token = Comment.Single + elif self.is_portugol and scanner.scan(r'(<\-)|(>=)|(<=)|%|<|>|-|\+|\*|\=|(<>)|\/|\.|:|,'): + token = Operator + elif not self.is_portugol and scanner.scan(r'[-+*\/=<>:;,.@\^]'): + token = Operator + # stop label highlighting on next ";" + if collect_labels and scanner.match == ';': + collect_labels = False + elif scanner.scan(r'[\(\)\[\]]+'): + token = Punctuation + # abort function naming ``foo = Function(...)`` + next_token_is_function = False + # if we are in a function block we count the open + # braces because ootherwise it's impossible to + # determine the end of the modifier context + if in_function_block or in_property_block: + if scanner.match == '(': + brace_balance[0] += 1 + elif scanner.match == ')': + brace_balance[0] -= 1 + elif scanner.match == '[': + brace_balance[1] += 1 + elif scanner.match == ']': + brace_balance[1] -= 1 + elif scanner.scan(r'[A-Za-z_][A-Za-z_0-9]*'): + lowercase_name = scanner.match.lower() + if lowercase_name == 'result': + token = Name.Builtin.Pseudo + elif lowercase_name in self.keywords: + token = Keyword + # if we are in a special block and a + # block ending keyword occurs (and the parenthesis + # is balanced) we end the current block context + if self.is_portugol: + if lowercase_name in ('funcao', 'procedimento'): + in_function_block = True + next_token_is_function = True + else: + if (in_function_block or in_property_block) and \ + lowercase_name in self.BLOCK_KEYWORDS and \ + brace_balance[0] <= 0 and \ + brace_balance[1] <= 0: + in_function_block = False + in_property_block = False + brace_balance = [0, 0] + block_labels = set() + if lowercase_name in ('label', 'goto'): + collect_labels = True + elif lowercase_name == 'asm': + stack.append('asm') + elif lowercase_name == 'property': + in_property_block = True + next_token_is_property = True + elif lowercase_name in ('procedure', 'operator', + 'function', 'constructor', + 'destructor'): + in_function_block = True + next_token_is_function = True + # we are in a function block and the current name + # is in the set of registered modifiers. highlight + # it as pseudo keyword + elif not self.is_portugol and in_function_block and \ + lowercase_name in self.FUNCTION_MODIFIERS: + token = Keyword.Pseudo + # if we are in a property highlight some more + # modifiers + elif not self.is_portugol and in_property_block and \ + lowercase_name in ('read', 'write'): + token = Keyword.Pseudo + next_token_is_function = True + # if the last iteration set next_token_is_function + # to true we now want this name highlighted as + # function. so do that and reset the state + elif next_token_is_function: + # Look if the next token is a dot. If yes it's + # not a function, but a class name and the + # part after the dot a function name + if not self.is_portugol and scanner.test(r'\s*\.\s*'): + token = Name.Class + # it's not a dot, our job is done + else: + token = Name.Function + next_token_is_function = False + + if self.is_portugol: + block_labels.add(scanner.match.lower()) + + # same for properties + elif not self.is_portugol and next_token_is_property: + token = Name.Property + next_token_is_property = False + # Highlight this token as label and add it + # to the list of known labels + elif not self.is_portugol and collect_labels: + token = Name.Label + block_labels.add(scanner.match.lower()) + # name is in list of known labels + elif lowercase_name in block_labels: + token = Name.Label + elif self.is_portugol and lowercase_name in self.PORTUGOL_BUILTIN_TYPES: + token = Keyword.Type + elif not self.is_portugol and lowercase_name in self.BUILTIN_TYPES: + token = Keyword.Type + elif not self.is_portugol and lowercase_name in self.DIRECTIVES: + token = Keyword.Pseudo + # builtins are just builtins if the token + # before isn't a dot + elif not self.is_portugol and not was_dot and lowercase_name in self.builtins: + token = Name.Builtin + else: + token = Name + elif self.is_portugol and scanner.scan(r"\""): + token = String + stack.append('string') + elif not self.is_portugol and scanner.scan(r"'"): + token = String + stack.append('string') + elif not self.is_portugol and scanner.scan(r'\#(\d+|\$[0-9A-Fa-f]+)'): + token = String.Char + elif not self.is_portugol and scanner.scan(r'\$[0-9A-Fa-f]+'): + token = Number.Hex + elif scanner.scan(r'\d+(?![eE]|\.[^.])'): + token = Number.Integer + elif scanner.scan(r'\d+(\.\d+([eE][+-]?\d+)?|[eE][+-]?\d+)'): + token = Number.Float + else: + # if the stack depth is deeper than once, pop + if len(stack) > 1: + stack.pop() + scanner.get_char() + + elif stack[-1] == 'string': + if self.is_portugol: + if scanner.scan(r"''"): + token = String.Escape + elif scanner.scan(r"\""): + token = String + stack.pop() + elif scanner.scan(r"[^\"]*"): + token = String + else: + scanner.get_char() + stack.pop() + else: + if scanner.scan(r"''"): + token = String.Escape + elif scanner.scan(r"'"): + token = String + stack.pop() + elif scanner.scan(r"[^']*"): + token = String + else: + scanner.get_char() + stack.pop() + elif not self.is_portugol and stack[-1] == 'asm': + if scanner.scan(r'\s+'): + token = Whitespace + elif scanner.scan(r'end'): + token = Keyword + stack.pop() + elif scanner.scan(r'\{.*?\}|\(\*.*?\*\)'): + if scanner.match.startswith('$'): + token = Comment.Preproc + else: + token = Comment.Multiline + elif scanner.scan(r'//.*?$'): + token = Comment.Single + elif scanner.scan(r"'"): + token = String + stack.append('string') + elif scanner.scan(r'@@[A-Za-z_][A-Za-z_0-9]*'): + token = Name.Label + elif scanner.scan(r'[A-Za-z_][A-Za-z_0-9]*'): + lowercase_name = scanner.match.lower() + if lowercase_name in self.ASM_INSTRUCTIONS: + token = Keyword + elif lowercase_name in self.ASM_REGISTERS: + token = Name.Builtin + else: + token = Name + elif scanner.scan(r'[-+*\/=<>:;,.@\^]+'): + token = Operator + elif scanner.scan(r'[\(\)\[\]]+'): + token = Punctuation + elif scanner.scan(r'\$[0-9A-Fa-f]+'): + token = Number.Hex + elif scanner.scan(r'\d+(?![eE]|\.[^.])'): + token = Number.Integer + elif scanner.scan(r'\d+(\.\d+([eE][+-]?\d+)?|[eE][+-]?\d+)'): + token = Number.Float + else: + scanner.get_char() + stack.pop() + + # save the dot!!!11 + if not self.is_portugol and scanner.match.strip(): + was_dot = scanner.match == '.' + + yield scanner.start_pos, token, scanner.match or '' diff --git a/pygments/lexers/pawn.py b/pygments/lexers/pawn.py new file mode 100644 index 0000000..41b19da --- /dev/null +++ b/pygments/lexers/pawn.py @@ -0,0 +1,202 @@ +""" + pygments.lexers.pawn + ~~~~~~~~~~~~~~~~~~~~ + + Lexers for the Pawn languages. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation +from pygments.util import get_bool_opt + +__all__ = ['SourcePawnLexer', 'PawnLexer'] + + +class SourcePawnLexer(RegexLexer): + """ + For SourcePawn source code with preprocessor directives. + + .. versionadded:: 1.6 + """ + name = 'SourcePawn' + aliases = ['sp'] + filenames = ['*.sp'] + mimetypes = ['text/x-sourcepawn'] + + #: optional Comment or Whitespace + _ws = r'(?:\s|//.*?\n|/\*.*?\*/)+' + #: only one /* */ style comment + _ws1 = r'\s*(?:/[*].*?[*]/\s*)*' + + tokens = { + 'root': [ + # preprocessor directives: without whitespace + (r'^#if\s+0', Comment.Preproc, 'if0'), + ('^#', Comment.Preproc, 'macro'), + # or with whitespace + ('^' + _ws1 + r'#if\s+0', Comment.Preproc, 'if0'), + ('^' + _ws1 + '#', Comment.Preproc, 'macro'), + (r'\n', Text), + (r'\s+', Text), + (r'\\\n', Text), # line continuation + (r'/(\\\n)?/(\n|(.|\n)*?[^\\]\n)', Comment.Single), + (r'/(\\\n)?\*(.|\n)*?\*(\\\n)?/', Comment.Multiline), + (r'[{}]', Punctuation), + (r'L?"', String, 'string'), + (r"L?'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char), + (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*', Number.Float), + (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float), + (r'0x[0-9a-fA-F]+[LlUu]*', Number.Hex), + (r'0[0-7]+[LlUu]*', Number.Oct), + (r'\d+[LlUu]*', Number.Integer), + (r'[~!%^&*+=|?:<>/-]', Operator), + (r'[()\[\],.;]', Punctuation), + (r'(case|const|continue|native|' + r'default|else|enum|for|if|new|operator|' + r'public|return|sizeof|static|decl|struct|switch)\b', Keyword), + (r'(bool|Float)\b', Keyword.Type), + (r'(true|false)\b', Keyword.Constant), + (r'[a-zA-Z_]\w*', Name), + ], + 'string': [ + (r'"', String, '#pop'), + (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape), + (r'[^\\"\n]+', String), # all other characters + (r'\\\n', String), # line continuation + (r'\\', String), # stray backslash + ], + 'macro': [ + (r'[^/\n]+', Comment.Preproc), + (r'/\*(.|\n)*?\*/', Comment.Multiline), + (r'//.*?\n', Comment.Single, '#pop'), + (r'/', Comment.Preproc), + (r'(?<=\\)\n', Comment.Preproc), + (r'\n', Comment.Preproc, '#pop'), + ], + 'if0': [ + (r'^\s*#if.*?(?/-]', Operator), + (r'[()\[\],.;]', Punctuation), + (r'(switch|case|default|const|new|static|char|continue|break|' + r'if|else|for|while|do|operator|enum|' + r'public|return|sizeof|tagof|state|goto)\b', Keyword), + (r'(bool|Float)\b', Keyword.Type), + (r'(true|false)\b', Keyword.Constant), + (r'[a-zA-Z_]\w*', Name), + ], + 'string': [ + (r'"', String, '#pop'), + (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape), + (r'[^\\"\n]+', String), # all other characters + (r'\\\n', String), # line continuation + (r'\\', String), # stray backslash + ], + 'macro': [ + (r'[^/\n]+', Comment.Preproc), + (r'/\*(.|\n)*?\*/', Comment.Multiline), + (r'//.*?\n', Comment.Single, '#pop'), + (r'/', Comment.Preproc), + (r'(?<=\\)\n', Comment.Preproc), + (r'\n', Comment.Preproc, '#pop'), + ], + 'if0': [ + (r'^\s*#if.*?(?])*>[egimosx]*', String.Regex, '#pop'), + (r'\[(\\\\|\\[^\\]|[^\\\]])*\][egimosx]*', String.Regex, '#pop'), + (r'\((\\\\|\\[^\\]|[^\\)])*\)[egimosx]*', String.Regex, '#pop'), + (r'@(\\\\|\\[^\\]|[^\\@])*@[egimosx]*', String.Regex, '#pop'), + (r'%(\\\\|\\[^\\]|[^\\%])*%[egimosx]*', String.Regex, '#pop'), + (r'\$(\\\\|\\[^\\]|[^\\$])*\$[egimosx]*', String.Regex, '#pop'), + ], + 'root': [ + (r'\A\#!.+?$', Comment.Hashbang), + (r'\#.*?$', Comment.Single), + (r'^=[a-zA-Z0-9]+\s+.*?\n=cut', Comment.Multiline), + (words(( + 'case', 'continue', 'do', 'else', 'elsif', 'for', 'foreach', + 'if', 'last', 'my', 'next', 'our', 'redo', 'reset', 'then', + 'unless', 'until', 'while', 'print', 'new', 'BEGIN', + 'CHECK', 'INIT', 'END', 'return'), suffix=r'\b'), + Keyword), + (r'(format)(\s+)(\w+)(\s*)(=)(\s*\n)', + bygroups(Keyword, Whitespace, Name, Whitespace, Punctuation, Whitespace), 'format'), + (r'(eq|lt|gt|le|ge|ne|not|and|or|cmp)\b', Operator.Word), + # common delimiters + (r's/(\\\\|\\[^\\]|[^\\/])*/(\\\\|\\[^\\]|[^\\/])*/[egimosx]*', + String.Regex), + (r's!(\\\\|\\!|[^!])*!(\\\\|\\!|[^!])*![egimosx]*', String.Regex), + (r's\\(\\\\|[^\\])*\\(\\\\|[^\\])*\\[egimosx]*', String.Regex), + (r's@(\\\\|\\[^\\]|[^\\@])*@(\\\\|\\[^\\]|[^\\@])*@[egimosx]*', + String.Regex), + (r's%(\\\\|\\[^\\]|[^\\%])*%(\\\\|\\[^\\]|[^\\%])*%[egimosx]*', + String.Regex), + # balanced delimiters + (r's\{(\\\\|\\[^\\]|[^\\}])*\}\s*', String.Regex, 'balanced-regex'), + (r's<(\\\\|\\[^\\]|[^\\>])*>\s*', String.Regex, 'balanced-regex'), + (r's\[(\\\\|\\[^\\]|[^\\\]])*\]\s*', String.Regex, + 'balanced-regex'), + (r's\((\\\\|\\[^\\]|[^\\)])*\)\s*', String.Regex, + 'balanced-regex'), + + (r'm?/(\\\\|\\[^\\]|[^\\/\n])*/[gcimosx]*', String.Regex), + (r'm(?=[/!\\{<\[(@%$])', String.Regex, 'balanced-regex'), + (r'((?<==~)|(?<=\())\s*/(\\\\|\\[^\\]|[^\\/])*/[gcimosx]*', + String.Regex), + (r'\s+', Whitespace), + (words(( + 'abs', 'accept', 'alarm', 'atan2', 'bind', 'binmode', 'bless', 'caller', 'chdir', + 'chmod', 'chomp', 'chop', 'chown', 'chr', 'chroot', 'close', 'closedir', 'connect', + 'continue', 'cos', 'crypt', 'dbmclose', 'dbmopen', 'defined', 'delete', 'die', + 'dump', 'each', 'endgrent', 'endhostent', 'endnetent', 'endprotoent', + 'endpwent', 'endservent', 'eof', 'eval', 'exec', 'exists', 'exit', 'exp', 'fcntl', + 'fileno', 'flock', 'fork', 'format', 'formline', 'getc', 'getgrent', 'getgrgid', + 'getgrnam', 'gethostbyaddr', 'gethostbyname', 'gethostent', 'getlogin', + 'getnetbyaddr', 'getnetbyname', 'getnetent', 'getpeername', 'getpgrp', + 'getppid', 'getpriority', 'getprotobyname', 'getprotobynumber', + 'getprotoent', 'getpwent', 'getpwnam', 'getpwuid', 'getservbyname', + 'getservbyport', 'getservent', 'getsockname', 'getsockopt', 'glob', 'gmtime', + 'goto', 'grep', 'hex', 'import', 'index', 'int', 'ioctl', 'join', 'keys', 'kill', 'last', + 'lc', 'lcfirst', 'length', 'link', 'listen', 'local', 'localtime', 'log', 'lstat', + 'map', 'mkdir', 'msgctl', 'msgget', 'msgrcv', 'msgsnd', 'my', 'next', 'oct', 'open', + 'opendir', 'ord', 'our', 'pack', 'pipe', 'pop', 'pos', 'printf', + 'prototype', 'push', 'quotemeta', 'rand', 'read', 'readdir', + 'readline', 'readlink', 'readpipe', 'recv', 'redo', 'ref', 'rename', + 'reverse', 'rewinddir', 'rindex', 'rmdir', 'scalar', 'seek', 'seekdir', + 'select', 'semctl', 'semget', 'semop', 'send', 'setgrent', 'sethostent', 'setnetent', + 'setpgrp', 'setpriority', 'setprotoent', 'setpwent', 'setservent', + 'setsockopt', 'shift', 'shmctl', 'shmget', 'shmread', 'shmwrite', 'shutdown', + 'sin', 'sleep', 'socket', 'socketpair', 'sort', 'splice', 'split', 'sprintf', 'sqrt', + 'srand', 'stat', 'study', 'substr', 'symlink', 'syscall', 'sysopen', 'sysread', + 'sysseek', 'system', 'syswrite', 'tell', 'telldir', 'tie', 'tied', 'time', 'times', 'tr', + 'truncate', 'uc', 'ucfirst', 'umask', 'undef', 'unlink', 'unpack', 'unshift', 'untie', + 'utime', 'values', 'vec', 'wait', 'waitpid', 'wantarray', 'warn', 'write'), suffix=r'\b'), + Name.Builtin), + (r'((__(DATA|DIE|WARN)__)|(STD(IN|OUT|ERR)))\b', Name.Builtin.Pseudo), + (r'(<<)([\'"]?)([a-zA-Z_]\w*)(\2;?\n.*?\n)(\3)(\n)', + bygroups(String, String, String.Delimiter, String, String.Delimiter, Whitespace)), + (r'__END__', Comment.Preproc, 'end-part'), + (r'\$\^[ADEFHILMOPSTWX]', Name.Variable.Global), + (r"\$[\\\"\[\]'&`+*.,;=%~?@$!<>(^|/-](?!\w)", Name.Variable.Global), + (r'[$@%#]+', Name.Variable, 'varname'), + (r'0_?[0-7]+(_[0-7]+)*', Number.Oct), + (r'0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*', Number.Hex), + (r'0b[01]+(_[01]+)*', Number.Bin), + (r'(?i)(\d*(_\d*)*\.\d+(_\d*)*|\d+(_\d*)*\.\d+(_\d*)*)(e[+-]?\d+)?', + Number.Float), + (r'(?i)\d+(_\d*)*e[+-]?\d+(_\d*)*', Number.Float), + (r'\d+(_\d+)*', Number.Integer), + (r"'(\\\\|\\[^\\]|[^'\\])*'", String), + (r'"(\\\\|\\[^\\]|[^"\\])*"', String), + (r'`(\\\\|\\[^\\]|[^`\\])*`', String.Backtick), + (r'<([^\s>]+)>', String.Regex), + (r'(q|qq|qw|qr|qx)\{', String.Other, 'cb-string'), + (r'(q|qq|qw|qr|qx)\(', String.Other, 'rb-string'), + (r'(q|qq|qw|qr|qx)\[', String.Other, 'sb-string'), + (r'(q|qq|qw|qr|qx)\<', String.Other, 'lt-string'), + (r'(q|qq|qw|qr|qx)([\W_])(.|\n)*?\2', String.Other), + (r'(package)(\s+)([a-zA-Z_]\w*(?:::[a-zA-Z_]\w*)*)', + bygroups(Keyword, Whitespace, Name.Namespace)), + (r'(use|require|no)(\s+)([a-zA-Z_]\w*(?:::[a-zA-Z_]\w*)*)', + bygroups(Keyword, Whitespace, Name.Namespace)), + (r'(sub)(\s+)', bygroups(Keyword, Whitespace), 'funcname'), + (words(( + 'no', 'package', 'require', 'use'), suffix=r'\b'), + Keyword), + (r'(\[\]|\*\*|::|<<|>>|>=|<=>|<=|={3}|!=|=~|' + r'!~|&&?|\|\||\.{1,3})', Operator), + (r'[-+/*%=<>&^|!\\~]=?', Operator), + (r'[()\[\]:;,<>/?{}]', Punctuation), # yes, there's no shortage + # of punctuation in Perl! + (r'(?=\w)', Name, 'name'), + ], + 'format': [ + (r'\.\n', String.Interpol, '#pop'), + (r'[^\n]*\n', String.Interpol), + ], + 'varname': [ + (r'\s+', Whitespace), + (r'\{', Punctuation, '#pop'), # hash syntax? + (r'\)|,', Punctuation, '#pop'), # argument specifier + (r'\w+::', Name.Namespace), + (r'[\w:]+', Name.Variable, '#pop'), + ], + 'name': [ + (r'[a-zA-Z_]\w*(::[a-zA-Z_]\w*)*(::)?(?=\s*->)', Name.Namespace, '#pop'), + (r'[a-zA-Z_]\w*(::[a-zA-Z_]\w*)*::', Name.Namespace, '#pop'), + (r'[\w:]+', Name, '#pop'), + (r'[A-Z_]+(?=\W)', Name.Constant, '#pop'), + (r'(?=\W)', Text, '#pop'), + ], + 'funcname': [ + (r'[a-zA-Z_]\w*[!?]?', Name.Function), + (r'\s+', Whitespace), + # argument declaration + (r'(\([$@%]*\))(\s*)', bygroups(Punctuation, Whitespace)), + (r';', Punctuation, '#pop'), + (r'.*?\{', Punctuation, '#pop'), + ], + 'cb-string': [ + (r'\\[{}\\]', String.Other), + (r'\\', String.Other), + (r'\{', String.Other, 'cb-string'), + (r'\}', String.Other, '#pop'), + (r'[^{}\\]+', String.Other) + ], + 'rb-string': [ + (r'\\[()\\]', String.Other), + (r'\\', String.Other), + (r'\(', String.Other, 'rb-string'), + (r'\)', String.Other, '#pop'), + (r'[^()]+', String.Other) + ], + 'sb-string': [ + (r'\\[\[\]\\]', String.Other), + (r'\\', String.Other), + (r'\[', String.Other, 'sb-string'), + (r'\]', String.Other, '#pop'), + (r'[^\[\]]+', String.Other) + ], + 'lt-string': [ + (r'\\[<>\\]', String.Other), + (r'\\', String.Other), + (r'\<', String.Other, 'lt-string'), + (r'\>', String.Other, '#pop'), + (r'[^<>]+', String.Other) + ], + 'end-part': [ + (r'.+', Comment.Preproc, '#pop') + ] + } + + def analyse_text(text): + if shebang_matches(text, r'perl'): + return True + + result = 0 + + if re.search(r'(?:my|our)\s+[$@%(]', text): + result += 0.9 + + if ':=' in text: + # := is not valid Perl, but it appears in unicon, so we should + # become less confident if we think we found Perl with := + result /= 2 + + return result + + +class Perl6Lexer(ExtendedRegexLexer): + """ + For Raku (a.k.a. Perl 6) source code. + + .. versionadded:: 2.0 + """ + + name = 'Perl6' + url = 'https://www.raku.org' + aliases = ['perl6', 'pl6', 'raku'] + filenames = ['*.pl', '*.pm', '*.nqp', '*.p6', '*.6pl', '*.p6l', '*.pl6', + '*.6pm', '*.p6m', '*.pm6', '*.t', '*.raku', '*.rakumod', + '*.rakutest', '*.rakudoc'] + mimetypes = ['text/x-perl6', 'application/x-perl6'] + flags = re.MULTILINE | re.DOTALL + + PERL6_IDENTIFIER_RANGE = r"['\w:-]" + + PERL6_KEYWORDS = ( + #Phasers + 'BEGIN','CATCH','CHECK','CLOSE','CONTROL','DOC','END','ENTER','FIRST', + 'INIT','KEEP','LAST','LEAVE','NEXT','POST','PRE','QUIT','UNDO', + #Keywords + 'anon','augment','but','class','constant','default','does','else', + 'elsif','enum','for','gather','given','grammar','has','if','import', + 'is','let','loop','made','make','method','module','multi','my','need', + 'orwith','our','proceed','proto','repeat','require','return', + 'return-rw','returns','role','rule','state','sub','submethod','subset', + 'succeed','supersede','token','try','unit','unless','until','use', + 'when','while','with','without', + #Traits + 'export','native','repr','required','rw','symbol', + ) + + PERL6_BUILTINS = ( + 'ACCEPTS','abs','abs2rel','absolute','accept','accessed','acos', + 'acosec','acosech','acosh','acotan','acotanh','acquire','act','action', + 'actions','add','add_attribute','add_enum_value','add_fallback', + 'add_method','add_parent','add_private_method','add_role','add_trustee', + 'adverb','after','all','allocate','allof','allowed','alternative-names', + 'annotations','antipair','antipairs','any','anyof','app_lifetime', + 'append','arch','archname','args','arity','Array','asec','asech','asin', + 'asinh','ASSIGN-KEY','ASSIGN-POS','assuming','ast','at','atan','atan2', + 'atanh','AT-KEY','atomic-assign','atomic-dec-fetch','atomic-fetch', + 'atomic-fetch-add','atomic-fetch-dec','atomic-fetch-inc', + 'atomic-fetch-sub','atomic-inc-fetch','AT-POS','attributes','auth', + 'await','backtrace','Bag','BagHash','bail-out','base','basename', + 'base-repeating','batch','BIND-KEY','BIND-POS','bind-stderr', + 'bind-stdin','bind-stdout','bind-udp','bits','bless','block','Bool', + 'bool-only','bounds','break','Bridge','broken','BUILD','build-date', + 'bytes','cache','callframe','calling-package','CALL-ME','callsame', + 'callwith','can','cancel','candidates','cando','can-ok','canonpath', + 'caps','caption','Capture','cas','catdir','categorize','categorize-list', + 'catfile','catpath','cause','ceiling','cglobal','changed','Channel', + 'chars','chdir','child','child-name','child-typename','chmod','chomp', + 'chop','chr','chrs','chunks','cis','classify','classify-list','cleanup', + 'clone','close','closed','close-stdin','cmp-ok','code','codes','collate', + 'column','comb','combinations','command','comment','compiler','Complex', + 'compose','compose_type','composer','condition','config', + 'configure_destroy','configure_type_checking','conj','connect', + 'constraints','construct','contains','contents','copy','cos','cosec', + 'cosech','cosh','cotan','cotanh','count','count-only','cpu-cores', + 'cpu-usage','CREATE','create_type','cross','cue','curdir','curupdir','d', + 'Date','DateTime','day','daycount','day-of-month','day-of-week', + 'day-of-year','days-in-month','declaration','decode','decoder','deepmap', + 'default','defined','DEFINITE','delayed','DELETE-KEY','DELETE-POS', + 'denominator','desc','DESTROY','destroyers','devnull','diag', + 'did-you-mean','die','dies-ok','dir','dirname','dir-sep','DISTROnames', + 'do','does','does-ok','done','done-testing','duckmap','dynamic','e', + 'eager','earlier','elems','emit','enclosing','encode','encoder', + 'encoding','end','ends-with','enum_from_value','enum_value_list', + 'enum_values','enums','eof','EVAL','eval-dies-ok','EVALFILE', + 'eval-lives-ok','exception','excludes-max','excludes-min','EXISTS-KEY', + 'EXISTS-POS','exit','exitcode','exp','expected','explicitly-manage', + 'expmod','extension','f','fail','fails-like','fc','feature','file', + 'filename','find_method','find_method_qualified','finish','first','flat', + 'flatmap','flip','floor','flunk','flush','fmt','format','formatter', + 'freeze','from','from-list','from-loop','from-posix','full', + 'full-barrier','get','get_value','getc','gist','got','grab','grabpairs', + 'grep','handle','handled','handles','hardware','has_accessor','Hash', + 'head','headers','hh-mm-ss','hidden','hides','hour','how','hyper','id', + 'illegal','im','in','indent','index','indices','indir','infinite', + 'infix','infix:<+>','infix:<->','install_method_cache','Instant', + 'instead','Int','int-bounds','interval','in-timezone','invalid-str', + 'invert','invocant','IO','IO::Notification.watch-path','is_trusted', + 'is_type','isa','is-absolute','isa-ok','is-approx','is-deeply', + 'is-hidden','is-initial-thread','is-int','is-lazy','is-leap-year', + 'isNaN','isnt','is-prime','is-relative','is-routine','is-setting', + 'is-win','item','iterator','join','keep','kept','KERNELnames','key', + 'keyof','keys','kill','kv','kxxv','l','lang','last','lastcall','later', + 'lazy','lc','leading','level','like','line','lines','link','List', + 'listen','live','lives-ok','local','lock','log','log10','lookup','lsb', + 'made','MAIN','make','Map','match','max','maxpairs','merge','message', + 'method','method_table','methods','migrate','min','minmax','minpairs', + 'minute','misplaced','Mix','MixHash','mkdir','mode','modified','month', + 'move','mro','msb','multi','multiness','my','name','named','named_names', + 'narrow','nativecast','native-descriptor','nativesizeof','new','new_type', + 'new-from-daycount','new-from-pairs','next','nextcallee','next-handle', + 'nextsame','nextwith','NFC','NFD','NFKC','NFKD','nl-in','nl-out', + 'nodemap','nok','none','norm','not','note','now','nude','Num', + 'numerator','Numeric','of','offset','offset-in-hours','offset-in-minutes', + 'ok','old','on-close','one','on-switch','open','opened','operation', + 'optional','ord','ords','orig','os-error','osname','out-buffer','pack', + 'package','package-kind','package-name','packages','pair','pairs', + 'pairup','parameter','params','parent','parent-name','parents','parse', + 'parse-base','parsefile','parse-names','parts','pass','path','path-sep', + 'payload','peer-host','peer-port','periods','perl','permutations','phaser', + 'pick','pickpairs','pid','placeholder','plan','plus','polar','poll', + 'polymod','pop','pos','positional','posix','postfix','postmatch', + 'precomp-ext','precomp-target','pred','prefix','prematch','prepend', + 'print','printf','print-nl','print-to','private','private_method_table', + 'proc','produce','Promise','prompt','protect','pull-one','push', + 'push-all','push-at-least','push-exactly','push-until-lazy','put', + 'qualifier-type','quit','r','race','radix','rand','range','Rat','raw', + 're','read','readchars','readonly','ready','Real','reallocate','reals', + 'reason','rebless','receive','recv','redispatcher','redo','reduce', + 'rel2abs','relative','release','rename','repeated','replacement', + 'report','reserved','resolve','restore','result','resume','rethrow', + 'reverse','right','rindex','rmdir','role','roles_to_compose','rolish', + 'roll','rootdir','roots','rotate','rotor','round','roundrobin', + 'routine-type','run','rwx','s','samecase','samemark','samewith','say', + 'schedule-on','scheduler','scope','sec','sech','second','seek','self', + 'send','Set','set_hidden','set_name','set_package','set_rw','set_value', + 'SetHash','set-instruments','setup_finalization','shape','share','shell', + 'shift','sibling','sigil','sign','signal','signals','signature','sin', + 'sinh','sink','sink-all','skip','skip-at-least','skip-at-least-pull-one', + 'skip-one','skip-rest','sleep','sleep-timer','sleep-until','Slip','slurp', + 'slurp-rest','slurpy','snap','snapper','so','socket-host','socket-port', + 'sort','source','source-package','spawn','SPEC','splice','split', + 'splitdir','splitpath','sprintf','spurt','sqrt','squish','srand','stable', + 'start','started','starts-with','status','stderr','stdout','Str', + 'sub_signature','subbuf','subbuf-rw','subname','subparse','subst', + 'subst-mutate','substr','substr-eq','substr-rw','subtest','succ','sum', + 'Supply','symlink','t','tail','take','take-rw','tan','tanh','tap', + 'target','target-name','tc','tclc','tell','then','throttle','throw', + 'throws-like','timezone','tmpdir','to','today','todo','toggle','to-posix', + 'total','trailing','trans','tree','trim','trim-leading','trim-trailing', + 'truncate','truncated-to','trusts','try_acquire','trying','twigil','type', + 'type_captures','typename','uc','udp','uncaught_handler','unimatch', + 'uniname','uninames','uniparse','uniprop','uniprops','unique','unival', + 'univals','unlike','unlink','unlock','unpack','unpolar','unshift', + 'unwrap','updir','USAGE','use-ok','utc','val','value','values','VAR', + 'variable','verbose-config','version','VMnames','volume','vow','w','wait', + 'warn','watch','watch-path','week','weekday-of-month','week-number', + 'week-year','WHAT','when','WHERE','WHEREFORE','WHICH','WHO', + 'whole-second','WHY','wordcase','words','workaround','wrap','write', + 'write-to','x','yada','year','yield','yyyy-mm-dd','z','zip','zip-latest', + + ) + + PERL6_BUILTIN_CLASSES = ( + #Booleans + 'False','True', + #Classes + 'Any','Array','Associative','AST','atomicint','Attribute','Backtrace', + 'Backtrace::Frame','Bag','Baggy','BagHash','Blob','Block','Bool','Buf', + 'Callable','CallFrame','Cancellation','Capture','CArray','Channel','Code', + 'compiler','Complex','ComplexStr','Cool','CurrentThreadScheduler', + 'Cursor','Date','Dateish','DateTime','Distro','Duration','Encoding', + 'Exception','Failure','FatRat','Grammar','Hash','HyperWhatever','Instant', + 'Int','int16','int32','int64','int8','IntStr','IO','IO::ArgFiles', + 'IO::CatHandle','IO::Handle','IO::Notification','IO::Path', + 'IO::Path::Cygwin','IO::Path::QNX','IO::Path::Unix','IO::Path::Win32', + 'IO::Pipe','IO::Socket','IO::Socket::Async','IO::Socket::INET','IO::Spec', + 'IO::Spec::Cygwin','IO::Spec::QNX','IO::Spec::Unix','IO::Spec::Win32', + 'IO::Special','Iterable','Iterator','Junction','Kernel','Label','List', + 'Lock','Lock::Async','long','longlong','Macro','Map','Match', + 'Metamodel::AttributeContainer','Metamodel::C3MRO','Metamodel::ClassHOW', + 'Metamodel::EnumHOW','Metamodel::Finalization','Metamodel::MethodContainer', + 'Metamodel::MROBasedMethodDispatch','Metamodel::MultipleInheritance', + 'Metamodel::Naming','Metamodel::Primitives','Metamodel::PrivateMethodContainer', + 'Metamodel::RoleContainer','Metamodel::Trusting','Method','Mix','MixHash', + 'Mixy','Mu','NFC','NFD','NFKC','NFKD','Nil','Num','num32','num64', + 'Numeric','NumStr','ObjAt','Order','Pair','Parameter','Perl','Pod::Block', + 'Pod::Block::Code','Pod::Block::Comment','Pod::Block::Declarator', + 'Pod::Block::Named','Pod::Block::Para','Pod::Block::Table','Pod::Heading', + 'Pod::Item','Pointer','Positional','PositionalBindFailover','Proc', + 'Proc::Async','Promise','Proxy','PseudoStash','QuantHash','Range','Rat', + 'Rational','RatStr','Real','Regex','Routine','Scalar','Scheduler', + 'Semaphore','Seq','Set','SetHash','Setty','Signature','size_t','Slip', + 'Stash','Str','StrDistance','Stringy','Sub','Submethod','Supplier', + 'Supplier::Preserving','Supply','Systemic','Tap','Telemetry', + 'Telemetry::Instrument::Thread','Telemetry::Instrument::Usage', + 'Telemetry::Period','Telemetry::Sampler','Thread','ThreadPoolScheduler', + 'UInt','uint16','uint32','uint64','uint8','Uni','utf8','Variable', + 'Version','VM','Whatever','WhateverCode','WrapHandle' + ) + + PERL6_OPERATORS = ( + 'X', 'Z', 'after', 'also', 'and', 'andthen', 'before', 'cmp', 'div', + 'eq', 'eqv', 'extra', 'ff', 'fff', 'ge', 'gt', 'le', 'leg', 'lt', 'm', + 'mm', 'mod', 'ne', 'or', 'orelse', 'rx', 's', 'tr', 'x', 'xor', 'xx', + '++', '--', '**', '!', '+', '-', '~', '?', '|', '||', '+^', '~^', '?^', + '^', '*', '/', '%', '%%', '+&', '+<', '+>', '~&', '~<', '~>', '?&', + 'gcd', 'lcm', '+', '-', '+|', '+^', '~|', '~^', '?|', '?^', + '~', '&', '^', 'but', 'does', '<=>', '..', '..^', '^..', '^..^', + '!=', '==', '<', '<=', '>', '>=', '~~', '===', '!eqv', + '&&', '||', '^^', '//', 'min', 'max', '??', '!!', 'ff', 'fff', 'so', + 'not', '<==', '==>', '<<==', '==>>','unicmp', + ) + + # Perl 6 has a *lot* of possible bracketing characters + # this list was lifted from STD.pm6 (https://github.com/perl6/std) + PERL6_BRACKETS = { + '\u0028': '\u0029', '\u003c': '\u003e', '\u005b': '\u005d', + '\u007b': '\u007d', '\u00ab': '\u00bb', '\u0f3a': '\u0f3b', + '\u0f3c': '\u0f3d', '\u169b': '\u169c', '\u2018': '\u2019', + '\u201a': '\u2019', '\u201b': '\u2019', '\u201c': '\u201d', + '\u201e': '\u201d', '\u201f': '\u201d', '\u2039': '\u203a', + '\u2045': '\u2046', '\u207d': '\u207e', '\u208d': '\u208e', + '\u2208': '\u220b', '\u2209': '\u220c', '\u220a': '\u220d', + '\u2215': '\u29f5', '\u223c': '\u223d', '\u2243': '\u22cd', + '\u2252': '\u2253', '\u2254': '\u2255', '\u2264': '\u2265', + '\u2266': '\u2267', '\u2268': '\u2269', '\u226a': '\u226b', + '\u226e': '\u226f', '\u2270': '\u2271', '\u2272': '\u2273', + '\u2274': '\u2275', '\u2276': '\u2277', '\u2278': '\u2279', + '\u227a': '\u227b', '\u227c': '\u227d', '\u227e': '\u227f', + '\u2280': '\u2281', '\u2282': '\u2283', '\u2284': '\u2285', + '\u2286': '\u2287', '\u2288': '\u2289', '\u228a': '\u228b', + '\u228f': '\u2290', '\u2291': '\u2292', '\u2298': '\u29b8', + '\u22a2': '\u22a3', '\u22a6': '\u2ade', '\u22a8': '\u2ae4', + '\u22a9': '\u2ae3', '\u22ab': '\u2ae5', '\u22b0': '\u22b1', + '\u22b2': '\u22b3', '\u22b4': '\u22b5', '\u22b6': '\u22b7', + '\u22c9': '\u22ca', '\u22cb': '\u22cc', '\u22d0': '\u22d1', + '\u22d6': '\u22d7', '\u22d8': '\u22d9', '\u22da': '\u22db', + '\u22dc': '\u22dd', '\u22de': '\u22df', '\u22e0': '\u22e1', + '\u22e2': '\u22e3', '\u22e4': '\u22e5', '\u22e6': '\u22e7', + '\u22e8': '\u22e9', '\u22ea': '\u22eb', '\u22ec': '\u22ed', + '\u22f0': '\u22f1', '\u22f2': '\u22fa', '\u22f3': '\u22fb', + '\u22f4': '\u22fc', '\u22f6': '\u22fd', '\u22f7': '\u22fe', + '\u2308': '\u2309', '\u230a': '\u230b', '\u2329': '\u232a', + '\u23b4': '\u23b5', '\u2768': '\u2769', '\u276a': '\u276b', + '\u276c': '\u276d', '\u276e': '\u276f', '\u2770': '\u2771', + '\u2772': '\u2773', '\u2774': '\u2775', '\u27c3': '\u27c4', + '\u27c5': '\u27c6', '\u27d5': '\u27d6', '\u27dd': '\u27de', + '\u27e2': '\u27e3', '\u27e4': '\u27e5', '\u27e6': '\u27e7', + '\u27e8': '\u27e9', '\u27ea': '\u27eb', '\u2983': '\u2984', + '\u2985': '\u2986', '\u2987': '\u2988', '\u2989': '\u298a', + '\u298b': '\u298c', '\u298d': '\u298e', '\u298f': '\u2990', + '\u2991': '\u2992', '\u2993': '\u2994', '\u2995': '\u2996', + '\u2997': '\u2998', '\u29c0': '\u29c1', '\u29c4': '\u29c5', + '\u29cf': '\u29d0', '\u29d1': '\u29d2', '\u29d4': '\u29d5', + '\u29d8': '\u29d9', '\u29da': '\u29db', '\u29f8': '\u29f9', + '\u29fc': '\u29fd', '\u2a2b': '\u2a2c', '\u2a2d': '\u2a2e', + '\u2a34': '\u2a35', '\u2a3c': '\u2a3d', '\u2a64': '\u2a65', + '\u2a79': '\u2a7a', '\u2a7d': '\u2a7e', '\u2a7f': '\u2a80', + '\u2a81': '\u2a82', '\u2a83': '\u2a84', '\u2a8b': '\u2a8c', + '\u2a91': '\u2a92', '\u2a93': '\u2a94', '\u2a95': '\u2a96', + '\u2a97': '\u2a98', '\u2a99': '\u2a9a', '\u2a9b': '\u2a9c', + '\u2aa1': '\u2aa2', '\u2aa6': '\u2aa7', '\u2aa8': '\u2aa9', + '\u2aaa': '\u2aab', '\u2aac': '\u2aad', '\u2aaf': '\u2ab0', + '\u2ab3': '\u2ab4', '\u2abb': '\u2abc', '\u2abd': '\u2abe', + '\u2abf': '\u2ac0', '\u2ac1': '\u2ac2', '\u2ac3': '\u2ac4', + '\u2ac5': '\u2ac6', '\u2acd': '\u2ace', '\u2acf': '\u2ad0', + '\u2ad1': '\u2ad2', '\u2ad3': '\u2ad4', '\u2ad5': '\u2ad6', + '\u2aec': '\u2aed', '\u2af7': '\u2af8', '\u2af9': '\u2afa', + '\u2e02': '\u2e03', '\u2e04': '\u2e05', '\u2e09': '\u2e0a', + '\u2e0c': '\u2e0d', '\u2e1c': '\u2e1d', '\u2e20': '\u2e21', + '\u3008': '\u3009', '\u300a': '\u300b', '\u300c': '\u300d', + '\u300e': '\u300f', '\u3010': '\u3011', '\u3014': '\u3015', + '\u3016': '\u3017', '\u3018': '\u3019', '\u301a': '\u301b', + '\u301d': '\u301e', '\ufd3e': '\ufd3f', '\ufe17': '\ufe18', + '\ufe35': '\ufe36', '\ufe37': '\ufe38', '\ufe39': '\ufe3a', + '\ufe3b': '\ufe3c', '\ufe3d': '\ufe3e', '\ufe3f': '\ufe40', + '\ufe41': '\ufe42', '\ufe43': '\ufe44', '\ufe47': '\ufe48', + '\ufe59': '\ufe5a', '\ufe5b': '\ufe5c', '\ufe5d': '\ufe5e', + '\uff08': '\uff09', '\uff1c': '\uff1e', '\uff3b': '\uff3d', + '\uff5b': '\uff5d', '\uff5f': '\uff60', '\uff62': '\uff63', + } + + def _build_word_match(words, boundary_regex_fragment=None, prefix='', suffix=''): + if boundary_regex_fragment is None: + return r'\b(' + prefix + r'|'.join(re.escape(x) for x in words) + \ + suffix + r')\b' + else: + return r'(? 0: + next_open_pos = text.find(opening_chars, search_pos + n_chars) + next_close_pos = text.find(closing_chars, search_pos + n_chars) + + if next_close_pos == -1: + next_close_pos = len(text) + nesting_level = 0 + elif next_open_pos != -1 and next_open_pos < next_close_pos: + nesting_level += 1 + search_pos = next_open_pos + else: # next_close_pos < next_open_pos + nesting_level -= 1 + search_pos = next_close_pos + + end_pos = next_close_pos + + if end_pos < 0: # if we didn't find a closer, just highlight the + # rest of the text in this class + end_pos = len(text) + + if adverbs is not None and re.search(r':to\b', adverbs): + heredoc_terminator = text[match.start('delimiter') + n_chars:end_pos] + end_heredoc = re.search(r'^\s*' + re.escape(heredoc_terminator) + + r'\s*$', text[end_pos:], re.MULTILINE) + + if end_heredoc: + end_pos += end_heredoc.end() + else: + end_pos = len(text) + + yield match.start(), token_class, text[match.start():end_pos + n_chars] + context.pos = end_pos + n_chars + + return callback + + def opening_brace_callback(lexer, match, context): + stack = context.stack + + yield match.start(), Text, context.text[match.start():match.end()] + context.pos = match.end() + + # if we encounter an opening brace and we're one level + # below a token state, it means we need to increment + # the nesting level for braces so we know later when + # we should return to the token rules. + if len(stack) > 2 and stack[-2] == 'token': + context.perl6_token_nesting_level += 1 + + def closing_brace_callback(lexer, match, context): + stack = context.stack + + yield match.start(), Text, context.text[match.start():match.end()] + context.pos = match.end() + + # if we encounter a free closing brace and we're one level + # below a token state, it means we need to check the nesting + # level to see if we need to return to the token state. + if len(stack) > 2 and stack[-2] == 'token': + context.perl6_token_nesting_level -= 1 + if context.perl6_token_nesting_level == 0: + stack.pop() + + def embedded_perl6_callback(lexer, match, context): + context.perl6_token_nesting_level = 1 + yield match.start(), Text, context.text[match.start():match.end()] + context.pos = match.end() + context.stack.append('root') + + # If you're modifying these rules, be careful if you need to process '{' or '}' + # characters. We have special logic for processing these characters (due to the fact + # that you can nest Perl 6 code in regex blocks), so if you need to process one of + # them, make sure you also process the corresponding one! + tokens = { + 'common': [ + (r'#[`|=](?P(?P[' + ''.join(PERL6_BRACKETS) + r'])(?P=first_char)*)', + brackets_callback(Comment.Multiline)), + (r'#[^\n]*$', Comment.Single), + (r'^(\s*)=begin\s+(\w+)\b.*?^\1=end\s+\2', Comment.Multiline), + (r'^(\s*)=for.*?\n\s*?\n', Comment.Multiline), + (r'^=.*?\n\s*?\n', Comment.Multiline), + (r'(regex|token|rule)(\s*' + PERL6_IDENTIFIER_RANGE + '+:sym)', + bygroups(Keyword, Name), 'token-sym-brackets'), + (r'(regex|token|rule)(?!' + PERL6_IDENTIFIER_RANGE + r')(\s*' + PERL6_IDENTIFIER_RANGE + '+)?', + bygroups(Keyword, Name), 'pre-token'), + # deal with a special case in the Perl 6 grammar (role q { ... }) + (r'(role)(\s+)(q)(\s*)', bygroups(Keyword, Whitespace, Name, Whitespace)), + (_build_word_match(PERL6_KEYWORDS, PERL6_IDENTIFIER_RANGE), Keyword), + (_build_word_match(PERL6_BUILTIN_CLASSES, PERL6_IDENTIFIER_RANGE, suffix='(?::[UD])?'), + Name.Builtin), + (_build_word_match(PERL6_BUILTINS, PERL6_IDENTIFIER_RANGE), Name.Builtin), + # copied from PerlLexer + (r'[$@%&][.^:?=!~]?' + PERL6_IDENTIFIER_RANGE + '+(?:<<.*?>>|<.*?>|«.*?»)*', + Name.Variable), + (r'\$[!/](?:<<.*?>>|<.*?>|«.*?»)*', Name.Variable.Global), + (r'::\?\w+', Name.Variable.Global), + (r'[$@%&]\*' + PERL6_IDENTIFIER_RANGE + '+(?:<<.*?>>|<.*?>|«.*?»)*', + Name.Variable.Global), + (r'\$(?:<.*?>)+', Name.Variable), + (r'(?:q|qq|Q)[a-zA-Z]?\s*(?P:[\w\s:]+)?\s*(?P(?P[^0-9a-zA-Z:\s])' + r'(?P=first_char)*)', brackets_callback(String)), + # copied from PerlLexer + (r'0_?[0-7]+(_[0-7]+)*', Number.Oct), + (r'0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*', Number.Hex), + (r'0b[01]+(_[01]+)*', Number.Bin), + (r'(?i)(\d*(_\d*)*\.\d+(_\d*)*|\d+(_\d*)*\.\d+(_\d*)*)(e[+-]?\d+)?', + Number.Float), + (r'(?i)\d+(_\d*)*e[+-]?\d+(_\d*)*', Number.Float), + (r'\d+(_\d+)*', Number.Integer), + (r'(?<=~~)\s*/(?:\\\\|\\/|.)*?/', String.Regex), + (r'(?<=[=(,])\s*/(?:\\\\|\\/|.)*?/', String.Regex), + (r'm\w+(?=\()', Name), + (r'(?:m|ms|rx)\s*(?P:[\w\s:]+)?\s*(?P(?P[^\w:\s])' + r'(?P=first_char)*)', brackets_callback(String.Regex)), + (r'(?:s|ss|tr)\s*(?::[\w\s:]+)?\s*/(?:\\\\|\\/|.)*?/(?:\\\\|\\/|.)*?/', + String.Regex), + (r'<[^\s=].*?\S>', String), + (_build_word_match(PERL6_OPERATORS), Operator), + (r'\w' + PERL6_IDENTIFIER_RANGE + '*', Name), + (r"'(\\\\|\\[^\\]|[^'\\])*'", String), + (r'"(\\\\|\\[^\\]|[^"\\])*"', String), + ], + 'root': [ + include('common'), + (r'\{', opening_brace_callback), + (r'\}', closing_brace_callback), + (r'.+?', Text), + ], + 'pre-token': [ + include('common'), + (r'\{', Text, ('#pop', 'token')), + (r'.+?', Text), + ], + 'token-sym-brackets': [ + (r'(?P(?P[' + ''.join(PERL6_BRACKETS) + '])(?P=first_char)*)', + brackets_callback(Name), ('#pop', 'pre-token')), + default(('#pop', 'pre-token')), + ], + 'token': [ + (r'\}', Text, '#pop'), + (r'(?<=:)(?:my|our|state|constant|temp|let).*?;', using(this)), + # make sure that quotes in character classes aren't treated as strings + (r'<(?:[-!?+.]\s*)?\[.*?\]>', String.Regex), + # make sure that '#' characters in quotes aren't treated as comments + (r"(?my|our)\s+)?(?:module|class|role|enum|grammar)', line) + if class_decl: + if saw_perl_decl or class_decl.group('scope') is not None: + return True + rating = 0.05 + continue + break + + if ':=' in text: + # Same logic as above for PerlLexer + rating /= 2 + + return rating + + def __init__(self, **options): + super().__init__(**options) + self.encoding = options.get('encoding', 'utf-8') diff --git a/pygments/lexers/phix.py b/pygments/lexers/phix.py new file mode 100644 index 0000000..b292aa5 --- /dev/null +++ b/pygments/lexers/phix.py @@ -0,0 +1,364 @@ +""" + pygments.lexers.phix + ~~~~~~~~~~~~~~~~~~~~ + + Lexers for Phix. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, words +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Whitespace + +__all__ = ['PhixLexer'] + + +class PhixLexer(RegexLexer): + """ + Pygments Lexer for Phix files (.exw). + See http://phix.x10.mx + + .. versionadded:: 2.14.0 + """ + + name = 'Phix' + url = 'http://phix.x10.mx' + aliases = ['phix'] + filenames = ['*.exw'] + mimetypes = ['text/x-phix'] + + flags = re.MULTILINE # nb: **NOT** re.DOTALL! (totally spanners comment handling) + + preproc = ( + 'ifdef', 'elsifdef', 'elsedef' + ) + # Note these lists are auto-generated by pwa/p2js.exw, when pwa\src\p2js_keywords.e (etc) + # change, though of course subsequent copy/commit/pull requests are all manual steps. + types = ( + 'string', 'nullable_string', 'atom_string', 'atom', 'bool', 'boolean', + 'cdCanvan', 'cdCanvas', 'complex', 'CURLcode', 'dictionary', 'int', + 'integer', 'Ihandle', 'Ihandles', 'Ihandln', 'mpfr', 'mpq', 'mpz', + 'mpz_or_string', 'number', 'rid_string', 'seq', 'sequence', 'timedate', + 'object' + ) + keywords = ( + 'abstract', 'class', 'continue', 'export', 'extends', 'nullable', + 'private', 'public', 'static', 'struct', 'trace', + 'and', 'break', 'by', 'case', 'catch', 'const', 'constant', 'debug', + 'default', 'do', 'else', 'elsif', 'end', 'enum', 'exit', 'fallthru', + 'fallthrough', 'for', 'forward', 'function', 'global', 'if', 'in', + 'include', 'js', 'javascript', 'javascript_semantics', 'let', 'not', + 'or', 'procedure', 'profile', 'profile_time', 'return', 'safe_mode', + 'switch', 'then', 'to', 'try', 'type', 'type_check', 'until', 'warning', + 'while', 'with', 'without', 'xor' + ) + routines = ( + 'abort', 'abs', 'adjust_timedate', 'and_bits', 'and_bitsu', 'apply', + 'append', 'arccos', 'arcsin', 'arctan', 'assert', 'atan2', + 'atom_to_float32', 'atom_to_float64', 'bankers_rounding', 'beep', + 'begins', 'binary_search', 'bits_to_int', 'bk_color', 'bytes_to_int', + 'call_func', 'call_proc', 'cdCanvasActivate', 'cdCanvasArc', + 'cdCanvasBegin', 'cdCanvasBox', 'cdCanvasChord', 'cdCanvasCircle', + 'cdCanvasClear', 'cdCanvasEnd', 'cdCanvasFlush', 'cdCanvasFont', + 'cdCanvasGetImageRGB', 'cdCanvasGetSize', 'cdCanvasGetTextAlignment', + 'cdCanvasGetTextSize', 'cdCanvasLine', 'cdCanvasMark', + 'cdCanvasMarkSize', 'cdCanvasMultiLineVectorText', 'cdCanvasPixel', + 'cdCanvasRect', 'cdCanvasRoundedBox', 'cdCanvasRoundedRect', + 'cdCanvasSector', 'cdCanvasSetAttribute', 'cdCanvasSetBackground', + 'cdCanvasSetFillMode', 'cdCanvasSetForeground', + 'cdCanvasSetInteriorStyle', 'cdCanvasSetLineStyle', + 'cdCanvasSetLineWidth', 'cdCanvasSetTextAlignment', 'cdCanvasText', + 'cdCanvasSetTextOrientation', 'cdCanvasGetTextOrientation', + 'cdCanvasVectorText', 'cdCanvasVectorTextDirection', + 'cdCanvasVectorTextSize', 'cdCanvasVertex', 'cdCreateCanvas', + 'cdDecodeAlpha', 'cdDecodeColor', 'cdDecodeColorAlpha', 'cdEncodeAlpha', + 'cdEncodeColor', 'cdEncodeColorAlpha', 'cdKillCanvas', 'cdVersion', + 'cdVersionDate', 'ceil', 'change_timezone', 'choose', 'clear_screen', + 'columnize', 'command_line', 'compare', 'complex_abs', 'complex_add', + 'complex_arg', 'complex_conjugate', 'complex_cos', 'complex_cosh', + 'complex_div', 'complex_exp', 'complex_imag', 'complex_inv', + 'complex_log', 'complex_mul', 'complex_neg', 'complex_new', + 'complex_norm', 'complex_power', 'complex_rho', 'complex_real', + 'complex_round', 'complex_sin', 'complex_sinh', 'complex_sprint', + 'complex_sqrt', 'complex_sub', 'complex_theta', 'concat', 'cos', + 'crash', 'custom_sort', 'date', 'day_of_week', 'day_of_year', + 'days_in_month', 'decode_base64', 'decode_flags', 'deep_copy', 'deld', + 'deserialize', 'destroy_dict', 'destroy_queue', 'destroy_stack', + 'dict_name', 'dict_size', 'elapsed', 'elapsed_short', 'encode_base64', + 'equal', 'even', 'exp', 'extract', 'factorial', 'factors', + 'file_size_k', 'find', 'find_all', 'find_any', 'find_replace', 'filter', + 'flatten', 'float32_to_atom', 'float64_to_atom', 'floor', + 'format_timedate', 'free_console', 'from_polar', 'gcd', 'get_file_base', + 'get_file_extension', 'get_file_name', 'get_file_name_and_path', + 'get_file_path', 'get_file_path_and_name', 'get_maxprime', 'get_prime', + 'get_primes', 'get_primes_le', 'get_proper_dir', 'get_proper_path', + 'get_rand', 'get_routine_info', 'get_test_abort', 'get_test_logfile', + 'get_test_pause', 'get_test_verbosity', 'get_tzid', 'getd', 'getdd', + 'getd_all_keys', 'getd_by_index', 'getd_index', 'getd_partial_key', + 'glAttachShader', 'glBindBuffer', 'glBindTexture', 'glBufferData', + 'glCanvasSpecialText', 'glClear', 'glClearColor', 'glColor', + 'glCompileShader', 'glCreateBuffer', 'glCreateProgram', + 'glCreateShader', 'glCreateTexture', 'glDeleteProgram', + 'glDeleteShader', 'glDrawArrays', 'glEnable', + 'glEnableVertexAttribArray', 'glFloat32Array', 'glInt32Array', + 'glFlush', 'glGetAttribLocation', 'glGetError', 'glGetProgramInfoLog', + 'glGetProgramParameter', 'glGetShaderInfoLog', 'glGetShaderParameter', + 'glGetUniformLocation', 'glLinkProgram', 'glLoadIdentity', + 'glMatrixMode', 'glOrtho', 'glRotatef', 'glShadeModel', + 'glShaderSource', 'glSimpleA7texcoords', 'glTexImage2Dc', + 'glTexParameteri', 'glTranslate', 'glUniform1f', 'glUniform1i', + 'glUniformMatrix4fv', 'glUseProgram', 'glVertex', + 'glVertexAttribPointer', 'glViewport', 'head', 'hsv_to_rgb', 'iff', + 'iif', 'include_file', 'incl0de_file', 'insert', 'instance', + 'int_to_bits', 'int_to_bytes', 'is_dict', 'is_integer', 's_leap_year', + 'is_prime', 'is_prime2', 'islower', 'isupper', 'Icallback', + 'iup_isdouble', 'iup_isprint', 'iup_XkeyBase', 'IupAppend', 'IupAlarm', + 'IupBackgroundBox', 'IupButton', 'IupCalendar', 'IupCanvas', + 'IupClipboard', 'IupClose', 'IupCloseOnEscape', 'IupControlsOpen', + 'IupDatePick', 'IupDestroy', 'IupDialog', 'IupDrawArc', 'IupDrawBegin', + 'IupDrawEnd', 'IupDrawGetSize', 'IupDrawGetTextSize', 'IupDrawLine', + 'IupDrawRectangle', 'IupDrawText', 'IupExpander', 'IupFill', + 'IupFlatLabel', 'IupFlatList', 'IupFlatTree', 'IupFlush', 'IupFrame', + 'IupGetAttribute', 'IupGetAttributeId', 'IupGetAttributePtr', + 'IupGetBrother', 'IupGetChild', 'IupGetChildCount', 'IupGetClassName', + 'IupGetDialog', 'IupGetDialogChild', 'IupGetDouble', 'IupGetFocus', + 'IupGetGlobal', 'IupGetGlobalInt', 'IupGetGlobalIntInt', 'IupGetInt', + 'IupGetInt2', 'IupGetIntId', 'IupGetIntInt', 'IupGetParent', + 'IupGLCanvas', 'IupGLCanvasOpen', 'IupGLMakeCurrent', 'IupGraph', + 'IupHbox', 'IupHide', 'IupImage', 'IupImageRGBA', 'IupItem', + 'iupKeyCodeToName', 'IupLabel', 'IupLink', 'IupList', 'IupMap', + 'IupMenu', 'IupMenuItem', 'IupMessage', 'IupMessageDlg', 'IupMultiBox', + 'IupMultiLine', 'IupNextField', 'IupNormaliser', 'IupOpen', + 'IupPlayInput', 'IupPopup', 'IupPreviousField', 'IupProgressBar', + 'IupRadio', 'IupRecordInput', 'IupRedraw', 'IupRefresh', + 'IupRefreshChildren', 'IupSeparator', 'IupSetAttribute', + 'IupSetAttributes', 'IupSetAttributeHandle', 'IupSetAttributeId', + 'IupSetAttributePtr', 'IupSetCallback', 'IupSetCallbacks', + 'IupSetDouble', 'IupSetFocus', 'IupSetGlobal', 'IupSetGlobalInt', + 'IupSetGlobalFunction', 'IupSetHandle', 'IupSetInt', + 'IupSetStrAttribute', 'IupSetStrGlobal', 'IupShow', 'IupShowXY', + 'IupSplit', 'IupStoreAttribute', 'IupSubmenu', 'IupTable', + 'IupTableClearSelected', 'IupTableClick_cb', 'IupTableGetSelected', + 'IupTableResize_cb', 'IupTableSetData', 'IupTabs', 'IupText', + 'IupTimer', 'IupToggle', 'IupTreeAddNodes', 'IupTreeView', 'IupUpdate', + 'IupValuator', 'IupVbox', 'join', 'join_by', 'join_path', 'k_perm', + 'largest', 'lcm', 'length', 'log', 'log10', 'log2', 'lower', + 'm4_crossProduct', 'm4_inverse', 'm4_lookAt', 'm4_multiply', + 'm4_normalize', 'm4_perspective', 'm4_subtractVectors', 'm4_xRotate', + 'm4_yRotate', 'machine_bits', 'machine_word', 'match', 'match_all', + 'match_replace', 'max', 'maxsq', 'min', 'minsq', 'mod', 'mpfr_add', + 'mpfr_ceil', 'mpfr_cmp', 'mpfr_cmp_si', 'mpfr_const_pi', 'mpfr_div', + 'mpfr_div_si', 'mpfr_div_z', 'mpfr_floor', 'mpfr_free', 'mpfr_get_d', + 'mpfr_get_default_precision', 'mpfr_get_default_rounding_mode', + 'mpfr_get_fixed', 'mpfr_get_precision', 'mpfr_get_si', 'mpfr_init', + 'mpfr_inits', 'mpfr_init_set', 'mpfr_init_set_q', 'mpfr_init_set_z', + 'mpfr_mul', 'mpfr_mul_si', 'mpfr_pow_si', 'mpfr_set', 'mpfr_set_d', + 'mpfr_set_default_precision', 'mpfr_set_default_rounding_mode', + 'mpfr_set_precision', 'mpfr_set_q', 'mpfr_set_si', 'mpfr_set_str', + 'mpfr_set_z', 'mpfr_si_div', 'mpfr_si_sub', 'mpfr_sqrt', 'mpfr_sub', + 'mpfr_sub_si', 'mpq_abs', 'mpq_add', 'mpq_add_si', 'mpq_canonicalize', + 'mpq_cmp', 'mpq_cmp_si', 'mpq_div', 'mpq_div_2exp', 'mpq_free', + 'mpq_get_den', 'mpq_get_num', 'mpq_get_str', 'mpq_init', 'mpq_init_set', + 'mpq_init_set_si', 'mpq_init_set_str', 'mpq_init_set_z', 'mpq_inits', + 'mpq_inv', 'mpq_mul', 'mpq_neg', 'mpq_set', 'mpq_set_si', 'mpq_set_str', + 'mpq_set_z', 'mpq_sub', 'mpz_abs', 'mpz_add', 'mpz_addmul', + 'mpz_addmul_ui', 'mpz_addmul_si', 'mpz_add_si', 'mpz_add_ui', 'mpz_and', + 'mpz_bin_uiui', 'mpz_cdiv_q', 'mpz_cmp', 'mpz_cmp_si', 'mpz_divexact', + 'mpz_divexact_ui', 'mpz_divisible_p', 'mpz_divisible_ui_p', 'mpz_even', + 'mpz_fac_ui', 'mpz_factorstring', 'mpz_fdiv_q', 'mpz_fdiv_q_2exp', + 'mpz_fdiv_q_ui', 'mpz_fdiv_qr', 'mpz_fdiv_r', 'mpz_fdiv_ui', + 'mpz_fib_ui', 'mpz_fib2_ui', 'mpz_fits_atom', 'mpz_fits_integer', + 'mpz_free', 'mpz_gcd', 'mpz_gcd_ui', 'mpz_get_atom', 'mpz_get_integer', + 'mpz_get_short_str', 'mpz_get_str', 'mpz_init', 'mpz_init_set', + 'mpz_inits', 'mpz_invert', 'mpz_lcm', 'mpz_lcm_ui', 'mpz_max', + 'mpz_min', 'mpz_mod', 'mpz_mod_ui', 'mpz_mul', 'mpz_mul_2exp', + 'mpz_mul_d', 'mpz_mul_si', 'mpz_neg', 'mpz_nthroot', 'mpz_odd', + 'mpz_pollard_rho', 'mpz_pow_ui', 'mpz_powm', 'mpz_powm_ui', 'mpz_prime', + 'mpz_prime_factors', 'mpz_prime_mr', 'mpz_rand', 'mpz_rand_ui', + 'mpz_re_compose', 'mpz_remove', 'mpz_scan0', 'mpz_scan1', 'mpz_set', + 'mpz_set_d', 'mpz_set_si', 'mpz_set_str', 'mpz_set_v', 'mpz_sign', + 'mpz_sizeinbase', 'mpz_sqrt', 'mpz_sub', 'mpz_sub_si', 'mpz_sub_ui', + 'mpz_si_sub', 'mpz_tdiv_q_2exp', 'mpz_tdiv_r_2exp', 'mpz_tstbit', + 'mpz_ui_pow_ui', 'mpz_xor', 'named_dict', 'new_dict', 'new_queue', + 'new_stack', 'not_bits', 'not_bitsu', 'odd', 'or_all', 'or_allu', + 'or_bits', 'or_bitsu', 'ord', 'ordinal', 'ordinant', + 'override_timezone', 'pad', 'pad_head', 'pad_tail', 'parse_date_string', + 'papply', 'peep', 'peepn', 'peep_dict', 'permute', 'permutes', + 'platform', 'pop', 'popn', 'pop_dict', 'power', 'pp', 'ppEx', 'ppExf', + 'ppf', 'ppOpt', 'pq_add', 'pq_destroy', 'pq_empty', 'pq_new', 'pq_peek', + 'pq_pop', 'pq_pop_data', 'pq_size', 'prepend', 'prime_factors', + 'printf', 'product', 'proper', 'push', 'pushn', 'putd', 'puts', + 'queue_empty', 'queue_size', 'rand', 'rand_range', 'reinstate', + 'remainder', 'remove', 'remove_all', 'repeat', 'repeatch', 'replace', + 'requires', 'reverse', 'rfind', 'rgb', 'rmatch', 'rmdr', 'rnd', 'round', + 'routine_id', 'scanf', 'serialize', 'series', 'set_rand', + 'set_test_abort', 'set_test_logfile', 'set_test_module', + 'set_test_pause', 'set_test_verbosity', 'set_timedate_formats', + 'set_timezone', 'setd', 'setd_default', 'shorten', 'sha256', + 'shift_bits', 'shuffle', 'sign', 'sin', 'smallest', 'sort', + 'sort_columns', 'speak', 'splice', 'split', 'split_any', 'split_by', + 'sprint', 'sprintf', 'sq_abs', 'sq_add', 'sq_and', 'sq_and_bits', + 'sq_arccos', 'sq_arcsin', 'sq_arctan', 'sq_atom', 'sq_ceil', 'sq_cmp', + 'sq_cos', 'sq_div', 'sq_even', 'sq_eq', 'sq_floor', 'sq_floor_div', + 'sq_ge', 'sq_gt', 'sq_int', 'sq_le', 'sq_log', 'sq_log10', 'sq_log2', + 'sq_lt', 'sq_max', 'sq_min', 'sq_mod', 'sq_mul', 'sq_ne', 'sq_not', + 'sq_not_bits', 'sq_odd', 'sq_or', 'sq_or_bits', 'sq_power', 'sq_rand', + 'sq_remainder', 'sq_rmdr', 'sq_rnd', 'sq_round', 'sq_seq', 'sq_sign', + 'sq_sin', 'sq_sqrt', 'sq_str', 'sq_sub', 'sq_tan', 'sq_trunc', + 'sq_uminus', 'sq_xor', 'sq_xor_bits', 'sqrt', 'square_free', + 'stack_empty', 'stack_size', 'substitute', 'substitute_all', 'sum', + 'tail', 'tan', 'test_equal', 'test_fail', 'test_false', + 'test_not_equal', 'test_pass', 'test_summary', 'test_true', + 'text_color', 'throw', 'time', 'timedate_diff', 'timedelta', + 'to_integer', 'to_number', 'to_rgb', 'to_string', 'traverse_dict', + 'traverse_dict_partial_key', 'trim', 'trim_head', 'trim_tail', 'trunc', + 'tagset', 'tagstart', 'typeof', 'unique', 'unix_dict', 'upper', + 'utf8_to_utf32', 'utf32_to_utf8', 'version', 'vlookup', 'vslice', + 'wglGetProcAddress', 'wildcard_file', 'wildcard_match', 'with_rho', + 'with_theta', 'xml_new_doc', 'xml_new_element', 'xml_set_attribute', + 'xml_sprint', 'xor_bits', 'xor_bitsu', + 'accept', 'allocate', 'allocate_string', 'allow_break', 'ARM', + 'atom_to_float80', 'c_func', 'c_proc', 'call_back', 'chdir', + 'check_break', 'clearDib', 'close', 'closesocket', 'console', + 'copy_file', 'create', 'create_directory', 'create_thread', + 'curl_easy_cleanup', 'curl_easy_get_file', 'curl_easy_init', + 'curl_easy_perform', 'curl_easy_perform_ex', 'curl_easy_setopt', + 'curl_easy_strerror', 'curl_global_cleanup', 'curl_global_init', + 'curl_slist_append', 'curl_slist_free_all', 'current_dir', 'cursor', + 'define_c_func', 'define_c_proc', 'delete', 'delete_cs', 'delete_file', + 'dir', 'DLL', 'drawDib', 'drawShadedPolygonToDib', 'ELF32', 'ELF64', + 'enter_cs', 'eval', 'exit_thread', 'free', 'file_exists', 'final', + 'float80_to_atom', 'format', 'get_bytes', 'get_file_date', + 'get_file_size', 'get_file_type', 'get_interpreter', 'get_key', + 'get_socket_error', 'get_text', 'get_thread_exitcode', 'get_thread_id', + 'getc', 'getenv', 'gets', 'getsockaddr', 'glBegin', 'glCallList', + 'glFrustum', 'glGenLists', 'glGetString', 'glLight', 'glMaterial', + 'glNewList', 'glNormal', 'glPopMatrix', 'glPushMatrix', 'glRotate', + 'glEnd', 'glEndList', 'glTexImage2D', 'goto', 'GUI', 'icons', 'ilASM', + 'include_files', 'include_paths', 'init_cs', 'ip_to_string', + 'IupConfig', 'IupConfigDialogClosed', 'IupConfigDialogShow', + 'IupConfigGetVariableInt', 'IupConfigLoad', 'IupConfigSave', + 'IupConfigSetVariableInt', 'IupExitLoop', 'IupFileDlg', 'IupFileList', + 'IupGLSwapBuffers', 'IupHelp', 'IupLoopStep', 'IupMainLoop', + 'IupNormalizer', 'IupPlot', 'IupPlotAdd', 'IupPlotBegin', 'IupPlotEnd', + 'IupPlotInsert', 'IupSaveImage', 'IupTreeGetUserId', 'IupUser', + 'IupVersion', 'IupVersionDate', 'IupVersionNumber', 'IupVersionShow', + 'killDib', 'leave_cs', 'listen', 'manifest', 'mem_copy', 'mem_set', + 'mpfr_gamma', 'mpfr_printf', 'mpfr_sprintf', 'mpz_export', 'mpz_import', + 'namespace', 'new', 'newDib', 'open', 'open_dll', 'PE32', 'PE64', + 'peek', 'peek_string', 'peek1s', 'peek1u', 'peek2s', 'peek2u', 'peek4s', + 'peek4u', 'peek8s', 'peek8u', 'peekNS', 'peekns', 'peeknu', 'poke', + 'poke2', 'poke4', 'poke8', 'pokeN', 'poke_string', 'poke_wstring', + 'position', 'progress', 'prompt_number', 'prompt_string', 'read_file', + 'read_lines', 'recv', 'resume_thread', 'seek', 'select', 'send', + 'setHandler', 'shutdown', 'sleep', 'SO', 'sockaddr_in', 'socket', + 'split_path', 'suspend_thread', 'system', 'system_exec', 'system_open', + 'system_wait', 'task_clock_start', 'task_clock_stop', 'task_create', + 'task_delay', 'task_list', 'task_schedule', 'task_self', 'task_status', + 'task_suspend', 'task_yield', 'thread_safe_string', 'try_cs', + 'utf8_to_utf16', 'utf16_to_utf8', 'utf16_to_utf32', 'utf32_to_utf16', + 'video_config', 'WSACleanup', 'wait_thread', 'walk_dir', 'where', + 'write_lines', 'wait_key' + ) + constants = ( + 'ANY_QUEUE', 'ASCENDING', 'BLACK', 'BLOCK_CURSOR', 'BLUE', + 'BRIGHT_CYAN', 'BRIGHT_BLUE', 'BRIGHT_GREEN', 'BRIGHT_MAGENTA', + 'BRIGHT_RED', 'BRIGHT_WHITE', 'BROWN', 'C_DWORD', 'C_INT', 'C_POINTER', + 'C_USHORT', 'C_WORD', 'CD_AMBER', 'CD_BLACK', 'CD_BLUE', 'CD_BOLD', + 'CD_BOLD_ITALIC', 'CD_BOX', 'CD_CENTER', 'CD_CIRCLE', 'CD_CLOSED_LINES', + 'CD_CONTINUOUS', 'CD_CUSTOM', 'CD_CYAN', 'CD_DARK_BLUE', 'CD_DARK_CYAN', + 'CD_DARK_GRAY', 'CD_DARK_GREY', 'CD_DARK_GREEN', 'CD_DARK_MAGENTA', + 'CD_DARK_RED', 'CD_DARK_YELLOW', 'CD_DASH_DOT', 'CD_DASH_DOT_DOT', + 'CD_DASHED', 'CD_DBUFFER', 'CD_DEG2RAD', 'CD_DIAMOND', 'CD_DOTTED', + 'CD_EAST', 'CD_EVENODD', 'CD_FILL', 'CD_GL', 'CD_GRAY', 'CD_GREY', + 'CD_GREEN', 'CD_HATCH', 'CD_HOLLOW', 'CD_HOLLOW_BOX', + 'CD_HOLLOW_CIRCLE', 'CD_HOLLOW_DIAMOND', 'CD_INDIGO', 'CD_ITALIC', + 'CD_IUP', 'CD_IUPDBUFFER', 'CD_LIGHT_BLUE', 'CD_LIGHT_GRAY', + 'CD_LIGHT_GREY', 'CD_LIGHT_GREEN', 'CD_LIGHT_PARCHMENT', 'CD_MAGENTA', + 'CD_NAVY', 'CD_NORTH', 'CD_NORTH_EAST', 'CD_NORTH_WEST', 'CD_OLIVE', + 'CD_OPEN_LINES', 'CD_ORANGE', 'CD_PARCHMENT', 'CD_PATTERN', + 'CD_PRINTER', 'CD_PURPLE', 'CD_PLAIN', 'CD_PLUS', 'CD_QUERY', + 'CD_RAD2DEG', 'CD_RED', 'CD_SILVER', 'CD_SOLID', 'CD_SOUTH_EAST', + 'CD_SOUTH_WEST', 'CD_STAR', 'CD_STIPPLE', 'CD_STRIKEOUT', + 'CD_UNDERLINE', 'CD_WEST', 'CD_WHITE', 'CD_WINDING', 'CD_VIOLET', + 'CD_X', 'CD_YELLOW', 'CURLE_OK', 'CURLOPT_MAIL_FROM', + 'CURLOPT_MAIL_RCPT', 'CURLOPT_PASSWORD', 'CURLOPT_READDATA', + 'CURLOPT_READFUNCTION', 'CURLOPT_SSL_VERIFYPEER', + 'CURLOPT_SSL_VERIFYHOST', 'CURLOPT_UPLOAD', 'CURLOPT_URL', + 'CURLOPT_USE_SSL', 'CURLOPT_USERNAME', 'CURLOPT_VERBOSE', + 'CURLOPT_WRITEFUNCTION', 'CURLUSESSL_ALL', 'CYAN', 'D_NAME', + 'D_ATTRIBUTES', 'D_SIZE', 'D_YEAR', 'D_MONTH', 'D_DAY', 'D_HOUR', + 'D_MINUTE', 'D_SECOND', 'D_CREATION', 'D_LASTACCESS', 'D_MODIFICATION', + 'DT_YEAR', 'DT_MONTH', 'DT_DAY', 'DT_HOUR', 'DT_MINUTE', 'DT_SECOND', + 'DT_DOW', 'DT_MSEC', 'DT_DOY', 'DT_GMT', 'EULER', 'E_CODE', 'E_ADDR', + 'E_LINE', 'E_RTN', 'E_NAME', 'E_FILE', 'E_PATH', 'E_USER', 'false', + 'False', 'FALSE', 'FIFO_QUEUE', 'FILETYPE_DIRECTORY', 'FILETYPE_FILE', + 'GET_EOF', 'GET_FAIL', 'GET_IGNORE', 'GET_SUCCESS', + 'GL_AMBIENT_AND_DIFFUSE', 'GL_ARRAY_BUFFER', 'GL_CLAMP', + 'GL_CLAMP_TO_BORDER', 'GL_CLAMP_TO_EDGE', 'GL_COLOR_BUFFER_BIT', + 'GL_COMPILE', 'GL_COMPILE_STATUS', 'GL_CULL_FACE', + 'GL_DEPTH_BUFFER_BIT', 'GL_DEPTH_TEST', 'GL_EXTENSIONS', 'GL_FLAT', + 'GL_FLOAT', 'GL_FRAGMENT_SHADER', 'GL_FRONT', 'GL_LIGHT0', + 'GL_LIGHTING', 'GL_LINEAR', 'GL_LINK_STATUS', 'GL_MODELVIEW', + 'GL_NEAREST', 'GL_NO_ERROR', 'GL_NORMALIZE', 'GL_POSITION', + 'GL_PROJECTION', 'GL_QUAD_STRIP', 'GL_QUADS', 'GL_RENDERER', + 'GL_REPEAT', 'GL_RGB', 'GL_RGBA', 'GL_SMOOTH', 'GL_STATIC_DRAW', + 'GL_TEXTURE_2D', 'GL_TEXTURE_MAG_FILTER', 'GL_TEXTURE_MIN_FILTER', + 'GL_TEXTURE_WRAP_S', 'GL_TEXTURE_WRAP_T', 'GL_TRIANGLES', + 'GL_UNSIGNED_BYTE', 'GL_VENDOR', 'GL_VERSION', 'GL_VERTEX_SHADER', + 'GRAY', 'GREEN', 'GT_LF_STRIPPED', 'GT_WHOLE_FILE', 'INVLN10', + 'IUP_CLOSE', 'IUP_CONTINUE', 'IUP_DEFAULT', 'IUP_BLACK', 'IUP_BLUE', + 'IUP_BUTTON1', 'IUP_BUTTON3', 'IUP_CENTER', 'IUP_CYAN', 'IUP_DARK_BLUE', + 'IUP_DARK_CYAN', 'IUP_DARK_GRAY', 'IUP_DARK_GREY', 'IUP_DARK_GREEN', + 'IUP_DARK_MAGENTA', 'IUP_DARK_RED', 'IUP_GRAY', 'IUP_GREY', 'IUP_GREEN', + 'IUP_IGNORE', 'IUP_INDIGO', 'IUP_MAGENTA', 'IUP_MASK_INT', + 'IUP_MASK_UINT', 'IUP_MOUSEPOS', 'IUP_NAVY', 'IUP_OLIVE', 'IUP_RECTEXT', + 'IUP_RED', 'IUP_LIGHT_BLUE', 'IUP_LIGHT_GRAY', 'IUP_LIGHT_GREY', + 'IUP_LIGHT_GREEN', 'IUP_ORANGE', 'IUP_PARCHMENT', 'IUP_PURPLE', + 'IUP_SILVER', 'IUP_TEAL', 'IUP_VIOLET', 'IUP_WHITE', 'IUP_YELLOW', + 'K_BS', 'K_cA', 'K_cC', 'K_cD', 'K_cF5', 'K_cK', 'K_cM', 'K_cN', 'K_cO', + 'K_cP', 'K_cR', 'K_cS', 'K_cT', 'K_cW', 'K_CR', 'K_DEL', 'K_DOWN', + 'K_END', 'K_ESC', 'K_F1', 'K_F2', 'K_F3', 'K_F4', 'K_F5', 'K_F6', + 'K_F7', 'K_F8', 'K_F9', 'K_F10', 'K_F11', 'K_F12', 'K_HOME', 'K_INS', + 'K_LEFT', 'K_MIDDLE', 'K_PGDN', 'K_PGUP', 'K_RIGHT', 'K_SP', 'K_TAB', + 'K_UP', 'K_h', 'K_i', 'K_j', 'K_p', 'K_r', 'K_s', 'JS', 'LIFO_QUEUE', + 'LINUX', 'MAX_HEAP', 'MAGENTA', 'MIN_HEAP', 'Nan', 'NO_CURSOR', 'null', + 'NULL', 'PI', 'pp_Ascii', 'pp_Brkt', 'pp_Date', 'pp_File', 'pp_FltFmt', + 'pp_Indent', 'pp_IntCh', 'pp_IntFmt', 'pp_Maxlen', 'pp_Nest', + 'pp_Pause', 'pp_Q22', 'pp_StrFmt', 'RED', 'SEEK_OK', 'SLASH', + 'TEST_ABORT', 'TEST_CRASH', 'TEST_PAUSE', 'TEST_PAUSE_FAIL', + 'TEST_QUIET', 'TEST_SHOW_ALL', 'TEST_SHOW_FAILED', 'TEST_SUMMARY', + 'true', 'True', 'TRUE', 'VC_SCRNLINES', 'WHITE', 'WINDOWS', 'YELLOW' + ) + + tokens = { + 'root': [ + (r"\s+", Whitespace), + (r'/\*|--/\*|#\[', Comment.Multiline, 'comment'), + (r'(?://|--|#!).*$', Comment.Single), +#Alt: +# (r'//.*$|--.*$|#!.*$', Comment.Single), + (r'"([^"\\]|\\.)*"', String.Other), + (r'\'[^\']*\'', String.Other), + (r'`[^`]*`', String.Other), + + (words(types, prefix=r'\b', suffix=r'\b'), Name.Function), + (words(routines, prefix=r'\b', suffix=r'\b'), Name.Function), + (words(preproc, prefix=r'\b', suffix=r'\b'), Keyword.Declaration), + (words(keywords, prefix=r'\b', suffix=r'\b'), Keyword.Declaration), + (words(constants, prefix=r'\b', suffix=r'\b'), Name.Constant), + # Aside: Phix only supports/uses the ascii/non-unicode tilde + (r'!=|==|<<|>>|:=|[-~+/*%=<>&^|\.(){},?:\[\]$\\;#]', Operator), + (r'[\w-]+', Text) + ], + 'comment': [ + (r'[^*/#]+', Comment.Multiline), + (r'/\*|#\[', Comment.Multiline, '#push'), + (r'\*/|#\]', Comment.Multiline, '#pop'), + (r'[*/#]', Comment.Multiline) + ] + } diff --git a/pygments/lexers/php.py b/pygments/lexers/php.py new file mode 100644 index 0000000..61d552a --- /dev/null +++ b/pygments/lexers/php.py @@ -0,0 +1,319 @@ +""" + pygments.lexers.php + ~~~~~~~~~~~~~~~~~~~ + + Lexers for PHP and related languages. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import Lexer, RegexLexer, include, bygroups, default, \ + using, this, words, do_insertions, line_re +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Other, Generic +from pygments.util import get_bool_opt, get_list_opt, shebang_matches + +__all__ = ['ZephirLexer', 'PsyshConsoleLexer', 'PhpLexer'] + + +class ZephirLexer(RegexLexer): + """ + For Zephir language source code. + + Zephir is a compiled high level language aimed + to the creation of C-extensions for PHP. + + .. versionadded:: 2.0 + """ + + name = 'Zephir' + url = 'http://zephir-lang.com/' + aliases = ['zephir'] + filenames = ['*.zep'] + + zephir_keywords = ['fetch', 'echo', 'isset', 'empty'] + zephir_type = ['bit', 'bits', 'string'] + + flags = re.DOTALL | re.MULTILINE + + tokens = { + 'commentsandwhitespace': [ + (r'\s+', Text), + (r'//.*?\n', Comment.Single), + (r'/\*.*?\*/', Comment.Multiline) + ], + 'slashstartsregex': [ + include('commentsandwhitespace'), + (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/' + r'([gim]+\b|\B)', String.Regex, '#pop'), + (r'/', Operator, '#pop'), + default('#pop') + ], + 'badregex': [ + (r'\n', Text, '#pop') + ], + 'root': [ + (r'^(?=\s|/)', Text, 'slashstartsregex'), + include('commentsandwhitespace'), + (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|' + r'(<<|>>>?|==?|!=?|->|[-<>+*%&|^/])=?', Operator, 'slashstartsregex'), + (r'[{(\[;,]', Punctuation, 'slashstartsregex'), + (r'[})\].]', Punctuation), + (r'(for|in|while|do|break|return|continue|switch|case|default|if|else|loop|' + r'require|inline|throw|try|catch|finally|new|delete|typeof|instanceof|void|' + r'namespace|use|extends|this|fetch|isset|unset|echo|fetch|likely|unlikely|' + r'empty)\b', Keyword, 'slashstartsregex'), + (r'(var|let|with|function)\b', Keyword.Declaration, 'slashstartsregex'), + (r'(abstract|boolean|bool|char|class|const|double|enum|export|extends|final|' + r'native|goto|implements|import|int|string|interface|long|ulong|char|uchar|' + r'float|unsigned|private|protected|public|short|static|self|throws|reverse|' + r'transient|volatile|readonly)\b', Keyword.Reserved), + (r'(true|false|null|undefined)\b', Keyword.Constant), + (r'(Array|Boolean|Date|_REQUEST|_COOKIE|_SESSION|' + r'_GET|_POST|_SERVER|this|stdClass|range|count|iterator|' + r'window)\b', Name.Builtin), + (r'[$a-zA-Z_][\w\\]*', Name.Other), + (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), + (r'0x[0-9a-fA-F]+', Number.Hex), + (r'[0-9]+', Number.Integer), + (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double), + (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single), + ] + } + + +class PsyshConsoleLexer(Lexer): + """ + For PsySH console output, such as: + + .. sourcecode:: psysh + + >>> $greeting = function($name): string { + ... return "Hello, {$name}"; + ... }; + => Closure($name): string {#2371 …3} + >>> $greeting('World') + => "Hello, World" + + .. versionadded:: 2.7 + """ + name = 'PsySH console session for PHP' + url = 'https://psysh.org/' + aliases = ['psysh'] + + def __init__(self, **options): + options['startinline'] = True + Lexer.__init__(self, **options) + + def get_tokens_unprocessed(self, text): + phplexer = PhpLexer(**self.options) + curcode = '' + insertions = [] + for match in line_re.finditer(text): + line = match.group() + if line.startswith('>>> ') or line.startswith('... '): + insertions.append((len(curcode), + [(0, Generic.Prompt, line[:4])])) + curcode += line[4:] + elif line.rstrip() == '...': + insertions.append((len(curcode), + [(0, Generic.Prompt, '...')])) + curcode += line[3:] + else: + if curcode: + yield from do_insertions( + insertions, phplexer.get_tokens_unprocessed(curcode)) + curcode = '' + insertions = [] + yield match.start(), Generic.Output, line + if curcode: + yield from do_insertions(insertions, + phplexer.get_tokens_unprocessed(curcode)) + + +class PhpLexer(RegexLexer): + """ + For PHP source code. + For PHP embedded in HTML, use the `HtmlPhpLexer`. + + Additional options accepted: + + `startinline` + If given and ``True`` the lexer starts highlighting with + php code (i.e.: no starting ``>> from pygments.lexers._php_builtins import MODULES + >>> MODULES.keys() + ['PHP Options/Info', 'Zip', 'dba', ...] + + In fact the names of those modules match the module names from + the php documentation. + """ + + name = 'PHP' + url = 'https://www.php.net/' + aliases = ['php', 'php3', 'php4', 'php5'] + filenames = ['*.php', '*.php[345]', '*.inc'] + mimetypes = ['text/x-php'] + + # Note that a backslash is included, PHP uses a backslash as a namespace + # separator. + _ident_inner = r'(?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*' + # But not inside strings. + _ident_nons = r'(?:[_a-z]|[^\x00-\x7f])(?:\w|[^\x00-\x7f])*' + + flags = re.IGNORECASE | re.DOTALL | re.MULTILINE + tokens = { + 'root': [ + (r'<\?(php)?', Comment.Preproc, 'php'), + (r'[^<]+', Other), + (r'<', Other) + ], + 'php': [ + (r'\?>', Comment.Preproc, '#pop'), + (r'(<<<)([\'"]?)(' + _ident_nons + r')(\2\n.*?\n\s*)(\3)(;?)(\n)', + bygroups(String, String, String.Delimiter, String, String.Delimiter, + Punctuation, Text)), + (r'\s+', Text), + (r'#.*?\n', Comment.Single), + (r'//.*?\n', Comment.Single), + # put the empty comment here, it is otherwise seen as + # the start of a docstring + (r'/\*\*/', Comment.Multiline), + (r'/\*\*.*?\*/', String.Doc), + (r'/\*.*?\*/', Comment.Multiline), + (r'(->|::)(\s*)(' + _ident_nons + ')', + bygroups(Operator, Text, Name.Attribute)), + (r'[~!%^&*+=|:.<>/@-]+', Operator), + (r'\?', Operator), # don't add to the charclass above! + (r'[\[\]{}();,]+', Punctuation), + (r'(class)(\s+)', bygroups(Keyword, Text), 'classname'), + (r'(function)(\s*)(?=\()', bygroups(Keyword, Text)), + (r'(function)(\s+)(&?)(\s*)', + bygroups(Keyword, Text, Operator, Text), 'functionname'), + (r'(const)(\s+)(' + _ident_inner + ')', + bygroups(Keyword, Text, Name.Constant)), + (r'(and|E_PARSE|old_function|E_ERROR|or|as|E_WARNING|parent|' + r'eval|PHP_OS|break|exit|case|extends|PHP_VERSION|cfunction|' + r'FALSE|print|for|require|continue|foreach|require_once|' + r'declare|return|default|static|do|switch|die|stdClass|' + r'echo|else|TRUE|elseif|var|empty|if|xor|enddeclare|include|' + r'virtual|endfor|include_once|while|endforeach|global|' + r'endif|list|endswitch|new|endwhile|not|' + r'array|E_ALL|NULL|final|php_user_filter|interface|' + r'implements|public|private|protected|abstract|clone|try|' + r'catch|throw|this|use|namespace|trait|yield|' + r'finally|match)\b', Keyword), + (r'(true|false|null)\b', Keyword.Constant), + include('magicconstants'), + (r'\$\{\$+' + _ident_inner + r'\}', Name.Variable), + (r'\$+' + _ident_inner, Name.Variable), + (_ident_inner, Name.Other), + (r'(\d+\.\d*|\d*\.\d+)(e[+-]?[0-9]+)?', Number.Float), + (r'\d+e[+-]?[0-9]+', Number.Float), + (r'0[0-7]+', Number.Oct), + (r'0x[a-f0-9]+', Number.Hex), + (r'\d+', Number.Integer), + (r'0b[01]+', Number.Bin), + (r"'([^'\\]*(?:\\.[^'\\]*)*)'", String.Single), + (r'`([^`\\]*(?:\\.[^`\\]*)*)`', String.Backtick), + (r'"', String.Double, 'string'), + ], + 'magicfuncs': [ + # source: http://php.net/manual/en/language.oop5.magic.php + (words(( + '__construct', '__destruct', '__call', '__callStatic', '__get', '__set', + '__isset', '__unset', '__sleep', '__wakeup', '__toString', '__invoke', + '__set_state', '__clone', '__debugInfo',), suffix=r'\b'), + Name.Function.Magic), + ], + 'magicconstants': [ + # source: http://php.net/manual/en/language.constants.predefined.php + (words(( + '__LINE__', '__FILE__', '__DIR__', '__FUNCTION__', '__CLASS__', + '__TRAIT__', '__METHOD__', '__NAMESPACE__',), + suffix=r'\b'), + Name.Constant), + ], + 'classname': [ + (_ident_inner, Name.Class, '#pop') + ], + 'functionname': [ + include('magicfuncs'), + (_ident_inner, Name.Function, '#pop'), + default('#pop') + ], + 'string': [ + (r'"', String.Double, '#pop'), + (r'[^{$"\\]+', String.Double), + (r'\\([nrt"$\\]|[0-7]{1,3}|x[0-9a-f]{1,2})', String.Escape), + (r'\$' + _ident_nons + r'(\[\S+?\]|->' + _ident_nons + ')?', + String.Interpol), + (r'(\{\$\{)(.*?)(\}\})', + bygroups(String.Interpol, using(this, _startinline=True), + String.Interpol)), + (r'(\{)(\$.*?)(\})', + bygroups(String.Interpol, using(this, _startinline=True), + String.Interpol)), + (r'(\$\{)(\S+)(\})', + bygroups(String.Interpol, Name.Variable, String.Interpol)), + (r'[${\\]', String.Double) + ], + } + + def __init__(self, **options): + self.funcnamehighlighting = get_bool_opt( + options, 'funcnamehighlighting', True) + self.disabledmodules = get_list_opt( + options, 'disabledmodules', ['unknown']) + self.startinline = get_bool_opt(options, 'startinline', False) + + # private option argument for the lexer itself + if '_startinline' in options: + self.startinline = options.pop('_startinline') + + # collect activated functions in a set + self._functions = set() + if self.funcnamehighlighting: + from pygments.lexers._php_builtins import MODULES + for key, value in MODULES.items(): + if key not in self.disabledmodules: + self._functions.update(value) + RegexLexer.__init__(self, **options) + + def get_tokens_unprocessed(self, text): + stack = ['root'] + if self.startinline: + stack.append('php') + for index, token, value in \ + RegexLexer.get_tokens_unprocessed(self, text, stack): + if token is Name.Other: + if value in self._functions: + yield index, Name.Builtin, value + continue + yield index, token, value + + def analyse_text(text): + if shebang_matches(text, r'php'): + return True + rv = 0.0 + if re.search(r'<\?(?!xml)', text): + rv += 0.3 + return rv diff --git a/pygments/lexers/pointless.py b/pygments/lexers/pointless.py new file mode 100644 index 0000000..9fea7ea --- /dev/null +++ b/pygments/lexers/pointless.py @@ -0,0 +1,71 @@ +""" + pygments.lexers.pointless + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for Pointless. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, words +from pygments.token import Comment, Error, Keyword, Name, Number, Operator, \ + Punctuation, String, Text + +__all__ = ['PointlessLexer'] + + +class PointlessLexer(RegexLexer): + """ + For Pointless source code. + + .. versionadded:: 2.7 + """ + + name = 'Pointless' + url = 'https://ptls.dev' + aliases = ['pointless'] + filenames = ['*.ptls'] + + ops = words([ + "+", "-", "*", "/", "**", "%", "+=", "-=", "*=", + "/=", "**=", "%=", "|>", "=", "==", "!=", "<", ">", + "<=", ">=", "=>", "$", "++", + ]) + + keywords = words([ + "if", "then", "else", "where", "with", "cond", + "case", "and", "or", "not", "in", "as", "for", + "requires", "throw", "try", "catch", "when", + "yield", "upval", + ], suffix=r'\b') + + tokens = { + 'root': [ + (r'[ \n\r]+', Text), + (r'--.*$', Comment.Single), + (r'"""', String, 'multiString'), + (r'"', String, 'string'), + (r'[\[\](){}:;,.]', Punctuation), + (ops, Operator), + (keywords, Keyword), + (r'\d+|\d*\.\d+', Number), + (r'(true|false)\b', Name.Builtin), + (r'[A-Z][a-zA-Z0-9]*\b', String.Symbol), + (r'output\b', Name.Variable.Magic), + (r'(export|import)\b', Keyword.Namespace), + (r'[a-z][a-zA-Z0-9]*\b', Name.Variable) + ], + 'multiString': [ + (r'\\.', String.Escape), + (r'"""', String, '#pop'), + (r'"', String), + (r'[^\\"]+', String), + ], + 'string': [ + (r'\\.', String.Escape), + (r'"', String, '#pop'), + (r'\n', Error), + (r'[^\\"]+', String), + ], + } diff --git a/pygments/lexers/pony.py b/pygments/lexers/pony.py new file mode 100644 index 0000000..014f8f8 --- /dev/null +++ b/pygments/lexers/pony.py @@ -0,0 +1,93 @@ +""" + pygments.lexers.pony + ~~~~~~~~~~~~~~~~~~~~ + + Lexers for Pony and related languages. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups, words +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation + +__all__ = ['PonyLexer'] + + +class PonyLexer(RegexLexer): + """ + For Pony source code. + + .. versionadded:: 2.4 + """ + + name = 'Pony' + aliases = ['pony'] + filenames = ['*.pony'] + + _caps = r'(iso|trn|ref|val|box|tag)' + + tokens = { + 'root': [ + (r'\n', Text), + (r'[^\S\n]+', Text), + (r'//.*\n', Comment.Single), + (r'/\*', Comment.Multiline, 'nested_comment'), + (r'"""(?:.|\n)*?"""', String.Doc), + (r'"', String, 'string'), + (r'\'.*\'', String.Char), + (r'=>|[]{}:().~;,|&!^?[]', Punctuation), + (words(( + 'addressof', 'and', 'as', 'consume', 'digestof', 'is', 'isnt', + 'not', 'or'), + suffix=r'\b'), + Operator.Word), + (r'!=|==|<<|>>|[-+/*%=<>]', Operator), + (words(( + 'box', 'break', 'compile_error', 'compile_intrinsic', + 'continue', 'do', 'else', 'elseif', 'embed', 'end', 'error', + 'for', 'if', 'ifdef', 'in', 'iso', 'lambda', 'let', 'match', + 'object', 'recover', 'ref', 'repeat', 'return', 'tag', 'then', + 'this', 'trn', 'try', 'until', 'use', 'var', 'val', 'where', + 'while', 'with', '#any', '#read', '#send', '#share'), + suffix=r'\b'), + Keyword), + (r'(actor|class|struct|primitive|interface|trait|type)((?:\s)+)', + bygroups(Keyword, Text), 'typename'), + (r'(new|fun|be)((?:\s)+)', bygroups(Keyword, Text), 'methodname'), + (words(( + 'I8', 'U8', 'I16', 'U16', 'I32', 'U32', 'I64', 'U64', 'I128', + 'U128', 'ILong', 'ULong', 'ISize', 'USize', 'F32', 'F64', + 'Bool', 'Pointer', 'None', 'Any', 'Array', 'String', + 'Iterator'), + suffix=r'\b'), + Name.Builtin.Type), + (r'_?[A-Z]\w*', Name.Type), + (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+', Number.Float), + (r'0x[0-9a-fA-F]+', Number.Hex), + (r'\d+', Number.Integer), + (r'(true|false)\b', Name.Builtin), + (r'_\d*', Name), + (r'_?[a-z][\w\']*', Name) + ], + 'typename': [ + (_caps + r'?((?:\s)*)(_?[A-Z]\w*)', + bygroups(Keyword, Text, Name.Class), '#pop') + ], + 'methodname': [ + (_caps + r'?((?:\s)*)(_?[a-z]\w*)', + bygroups(Keyword, Text, Name.Function), '#pop') + ], + 'nested_comment': [ + (r'[^*/]+', Comment.Multiline), + (r'/\*', Comment.Multiline, '#push'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[*/]', Comment.Multiline) + ], + 'string': [ + (r'"', String, '#pop'), + (r'\\"', String), + (r'[^\\"]+', String) + ] + } diff --git a/pygments/lexers/praat.py b/pygments/lexers/praat.py new file mode 100644 index 0000000..b39979d --- /dev/null +++ b/pygments/lexers/praat.py @@ -0,0 +1,304 @@ +""" + pygments.lexers.praat + ~~~~~~~~~~~~~~~~~~~~~ + + Lexer for Praat + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, words, bygroups, include +from pygments.token import Name, Text, Comment, Keyword, String, Punctuation, \ + Number, Operator, Whitespace + +__all__ = ['PraatLexer'] + + +class PraatLexer(RegexLexer): + """ + For Praat scripts. + + .. versionadded:: 2.1 + """ + + name = 'Praat' + url = 'http://www.praat.org' + aliases = ['praat'] + filenames = ['*.praat', '*.proc', '*.psc'] + + keywords = ( + 'if', 'then', 'else', 'elsif', 'elif', 'endif', 'fi', 'for', 'from', 'to', + 'endfor', 'endproc', 'while', 'endwhile', 'repeat', 'until', 'select', 'plus', + 'minus', 'demo', 'assert', 'stopwatch', 'nocheck', 'nowarn', 'noprogress', + 'editor', 'endeditor', 'clearinfo', + ) + + functions_string = ( + 'backslashTrigraphsToUnicode', 'chooseDirectory', 'chooseReadFile', + 'chooseWriteFile', 'date', 'demoKey', 'do', 'environment', 'extractLine', + 'extractWord', 'fixed', 'info', 'left', 'mid', 'percent', 'readFile', 'replace', + 'replace_regex', 'right', 'selected', 'string', 'unicodeToBackslashTrigraphs', + ) + + functions_numeric = ( + 'abs', 'appendFile', 'appendFileLine', 'appendInfo', 'appendInfoLine', 'arccos', + 'arccosh', 'arcsin', 'arcsinh', 'arctan', 'arctan2', 'arctanh', 'barkToHertz', + 'beginPause', 'beginSendPraat', 'besselI', 'besselK', 'beta', 'beta2', + 'binomialP', 'binomialQ', 'boolean', 'ceiling', 'chiSquareP', 'chiSquareQ', + 'choice', 'comment', 'cos', 'cosh', 'createDirectory', 'deleteFile', + 'demoClicked', 'demoClickedIn', 'demoCommandKeyPressed', + 'demoExtraControlKeyPressed', 'demoInput', 'demoKeyPressed', + 'demoOptionKeyPressed', 'demoShiftKeyPressed', 'demoShow', 'demoWaitForInput', + 'demoWindowTitle', 'demoX', 'demoY', 'differenceLimensToPhon', 'do', 'editor', + 'endPause', 'endSendPraat', 'endsWith', 'erb', 'erbToHertz', 'erf', 'erfc', + 'exitScript', 'exp', 'extractNumber', 'fileReadable', 'fisherP', 'fisherQ', + 'floor', 'gaussP', 'gaussQ', 'hertzToBark', 'hertzToErb', 'hertzToMel', + 'hertzToSemitones', 'imax', 'imin', 'incompleteBeta', 'incompleteGammaP', 'index', + 'index_regex', 'integer', 'invBinomialP', 'invBinomialQ', 'invChiSquareQ', 'invFisherQ', + 'invGaussQ', 'invSigmoid', 'invStudentQ', 'length', 'ln', 'lnBeta', 'lnGamma', + 'log10', 'log2', 'max', 'melToHertz', 'min', 'minusObject', 'natural', 'number', + 'numberOfColumns', 'numberOfRows', 'numberOfSelected', 'objectsAreIdentical', + 'option', 'optionMenu', 'pauseScript', 'phonToDifferenceLimens', 'plusObject', + 'positive', 'randomBinomial', 'randomGauss', 'randomInteger', 'randomPoisson', + 'randomUniform', 'real', 'readFile', 'removeObject', 'rindex', 'rindex_regex', + 'round', 'runScript', 'runSystem', 'runSystem_nocheck', 'selectObject', + 'selected', 'semitonesToHertz', 'sentence', 'sentencetext', 'sigmoid', 'sin', 'sinc', + 'sincpi', 'sinh', 'soundPressureToPhon', 'sqrt', 'startsWith', 'studentP', + 'studentQ', 'tan', 'tanh', 'text', 'variableExists', 'word', 'writeFile', 'writeFileLine', + 'writeInfo', 'writeInfoLine', + ) + + functions_array = ( + 'linear', 'randomGauss', 'randomInteger', 'randomUniform', 'zero', + ) + + objects = ( + 'Activation', 'AffineTransform', 'AmplitudeTier', 'Art', 'Artword', + 'Autosegment', 'BarkFilter', 'BarkSpectrogram', 'CCA', 'Categories', + 'Cepstrogram', 'Cepstrum', 'Cepstrumc', 'ChebyshevSeries', 'ClassificationTable', + 'Cochleagram', 'Collection', 'ComplexSpectrogram', 'Configuration', 'Confusion', + 'ContingencyTable', 'Corpus', 'Correlation', 'Covariance', + 'CrossCorrelationTable', 'CrossCorrelationTables', 'DTW', 'DataModeler', + 'Diagonalizer', 'Discriminant', 'Dissimilarity', 'Distance', 'Distributions', + 'DurationTier', 'EEG', 'ERP', 'ERPTier', 'EditCostsTable', 'EditDistanceTable', + 'Eigen', 'Excitation', 'Excitations', 'ExperimentMFC', 'FFNet', 'FeatureWeights', + 'FileInMemory', 'FilesInMemory', 'Formant', 'FormantFilter', 'FormantGrid', + 'FormantModeler', 'FormantPoint', 'FormantTier', 'GaussianMixture', 'HMM', + 'HMM_Observation', 'HMM_ObservationSequence', 'HMM_State', 'HMM_StateSequence', + 'Harmonicity', 'ISpline', 'Index', 'Intensity', 'IntensityTier', 'IntervalTier', + 'KNN', 'KlattGrid', 'KlattTable', 'LFCC', 'LPC', 'Label', 'LegendreSeries', + 'LinearRegression', 'LogisticRegression', 'LongSound', 'Ltas', 'MFCC', 'MSpline', + 'ManPages', 'Manipulation', 'Matrix', 'MelFilter', 'MelSpectrogram', + 'MixingMatrix', 'Movie', 'Network', 'Object', 'OTGrammar', 'OTHistory', 'OTMulti', + 'PCA', 'PairDistribution', 'ParamCurve', 'Pattern', 'Permutation', 'Photo', + 'Pitch', 'PitchModeler', 'PitchTier', 'PointProcess', 'Polygon', 'Polynomial', + 'PowerCepstrogram', 'PowerCepstrum', 'Procrustes', 'RealPoint', 'RealTier', + 'ResultsMFC', 'Roots', 'SPINET', 'SSCP', 'SVD', 'Salience', 'ScalarProduct', + 'Similarity', 'SimpleString', 'SortedSetOfString', 'Sound', 'Speaker', + 'Spectrogram', 'Spectrum', 'SpectrumTier', 'SpeechSynthesizer', 'SpellingChecker', + 'Strings', 'StringsIndex', 'Table', 'TableOfReal', 'TextGrid', 'TextInterval', + 'TextPoint', 'TextTier', 'Tier', 'Transition', 'VocalTract', 'VocalTractTier', + 'Weight', 'WordList', + ) + + variables_numeric = ( + 'macintosh', 'windows', 'unix', 'praatVersion', 'pi', 'e', 'undefined', + ) + + variables_string = ( + 'praatVersion', 'tab', 'shellDirectory', 'homeDirectory', + 'preferencesDirectory', 'newline', 'temporaryDirectory', + 'defaultDirectory', + ) + + object_attributes = ( + 'ncol', 'nrow', 'xmin', 'ymin', 'xmax', 'ymax', 'nx', 'ny', 'dx', 'dy', + ) + + tokens = { + 'root': [ + (r'(\s+)(#.*?$)', bygroups(Whitespace, Comment.Single)), + (r'^#.*?$', Comment.Single), + (r';[^\n]*', Comment.Single), + (r'\s+', Whitespace), + + (r'\bprocedure\b', Keyword, 'procedure_definition'), + (r'\bcall\b', Keyword, 'procedure_call'), + (r'@', Name.Function, 'procedure_call'), + + include('function_call'), + + (words(keywords, suffix=r'\b'), Keyword), + + (r'(\bform\b)(\s+)([^\n]+)', + bygroups(Keyword, Whitespace, String), 'old_form'), + + (r'(print(?:line|tab)?|echo|exit|asserterror|pause|send(?:praat|socket)|' + r'include|execute|system(?:_nocheck)?)(\s+)', + bygroups(Keyword, Whitespace), 'string_unquoted'), + + (r'(goto|label)(\s+)(\w+)', bygroups(Keyword, Whitespace, Name.Label)), + + include('variable_name'), + include('number'), + + (r'"', String, 'string'), + + (words((objects), suffix=r'(?=\s+\S+\n)'), Name.Class, 'string_unquoted'), + + (r'\b[A-Z]', Keyword, 'command'), + (r'(\.{3}|[)(,])', Punctuation), + ], + 'command': [ + (r'( ?[\w()-]+ ?)', Keyword), + + include('string_interpolated'), + + (r'\.{3}', Keyword, ('#pop', 'old_arguments')), + (r':', Keyword, ('#pop', 'comma_list')), + (r'\s', Whitespace, '#pop'), + ], + 'procedure_call': [ + (r'\s+', Whitespace), + (r'([\w.]+)(?:(:)|(?:(\s*)(\()))', + bygroups(Name.Function, Punctuation, + Text.Whitespace, Punctuation), '#pop'), + (r'([\w.]+)', Name.Function, ('#pop', 'old_arguments')), + ], + 'procedure_definition': [ + (r'\s', Whitespace), + (r'([\w.]+)(\s*?[(:])', + bygroups(Name.Function, Whitespace), '#pop'), + (r'([\w.]+)([^\n]*)', + bygroups(Name.Function, Text), '#pop'), + ], + 'function_call': [ + (words(functions_string, suffix=r'\$(?=\s*[:(])'), Name.Function, 'function'), + (words(functions_array, suffix=r'#(?=\s*[:(])'), Name.Function, 'function'), + (words(functions_numeric, suffix=r'(?=\s*[:(])'), Name.Function, 'function'), + ], + 'function': [ + (r'\s+', Whitespace), + (r':', Punctuation, ('#pop', 'comma_list')), + (r'\s*\(', Punctuation, ('#pop', 'comma_list')), + ], + 'comma_list': [ + (r'(\s*\n\s*)(\.{3})', bygroups(Whitespace, Punctuation)), + + (r'(\s*)(?:([)\]])|(\n))', bygroups( + Whitespace, Punctuation, Whitespace), '#pop'), + + (r'\s+', Whitespace), + (r'"', String, 'string'), + (r'\b(if|then|else|fi|endif)\b', Keyword), + + include('function_call'), + include('variable_name'), + include('operator'), + include('number'), + + (r'[()]', Text), + (r',', Punctuation), + ], + 'old_arguments': [ + (r'\n', Whitespace, '#pop'), + + include('variable_name'), + include('operator'), + include('number'), + + (r'"', String, 'string'), + (r'[^\n]', Text), + ], + 'number': [ + (r'\n', Whitespace, '#pop'), + (r'\b\d+(\.\d*)?([eE][-+]?\d+)?%?', Number), + ], + 'object_reference': [ + include('string_interpolated'), + (r'([a-z][a-zA-Z0-9_]*|\d+)', Name.Builtin), + + (words(object_attributes, prefix=r'\.'), Name.Builtin, '#pop'), + + (r'\$', Name.Builtin), + (r'\[', Text, '#pop'), + ], + 'variable_name': [ + include('operator'), + include('number'), + + (words(variables_string, suffix=r'\$'), Name.Variable.Global), + (words(variables_numeric, + suffix=r'(?=[^a-zA-Z0-9_."\'$#\[:(]|\s|^|$)'), + Name.Variable.Global), + + (words(objects, prefix=r'\b', suffix=r"(_)"), + bygroups(Name.Builtin, Name.Builtin), + 'object_reference'), + + (r'\.?_?[a-z][\w.]*(\$|#)?', Text), + (r'[\[\]]', Punctuation, 'comma_list'), + + include('string_interpolated'), + ], + 'operator': [ + (r'([+\/*<>=!-]=?|[&*|][&*|]?|\^|<>)', Operator), + (r'(?', Punctuation), + (r'"(?:\\x[0-9a-fA-F]+\\|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|' + r'\\[0-7]+\\|\\["\\abcefnrstv]|[^\\"])*"', String.Double), + (r"'(?:''|[^'])*'", String.Atom), # quoted atom + # Needs to not be followed by an atom. + # (r'=(?=\s|[a-zA-Z\[])', Operator), + (r'is\b', Operator), + (r'(<|>|=<|>=|==|=:=|=|/|//|\*|\+|-)(?=\s|[a-zA-Z0-9\[])', + Operator), + (r'(mod|div|not)\b', Operator), + (r'_', Keyword), # The don't-care variable + (r'([a-z]+)(:)', bygroups(Name.Namespace, Punctuation)), + (r'([a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]' + r'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)' + r'(\s*)(:-|-->)', + bygroups(Name.Function, Text, Operator)), # function defn + (r'([a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]' + r'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)' + r'(\s*)(\()', + bygroups(Name.Function, Text, Punctuation)), + (r'[a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]' + r'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*', + String.Atom), # atom, characters + # This one includes ! + (r'[#&*+\-./:<=>?@\\^~\u00a1-\u00bf\u2010-\u303f]+', + String.Atom), # atom, graphics + (r'[A-Z_]\w*', Name.Variable), + (r'\s+|[\u2000-\u200f\ufff0-\ufffe\uffef]', Text), + ], + 'nested-comment': [ + (r'\*/', Comment.Multiline, '#pop'), + (r'/\*', Comment.Multiline, '#push'), + (r'[^*/]+', Comment.Multiline), + (r'[*/]', Comment.Multiline), + ], + } + + def analyse_text(text): + return ':-' in text + + +class LogtalkLexer(RegexLexer): + """ + For Logtalk source code. + + .. versionadded:: 0.10 + """ + + name = 'Logtalk' + url = 'http://logtalk.org/' + aliases = ['logtalk'] + filenames = ['*.lgt', '*.logtalk'] + mimetypes = ['text/x-logtalk'] + + tokens = { + 'root': [ + # Directives + (r'^\s*:-\s', Punctuation, 'directive'), + # Comments + (r'%.*?\n', Comment), + (r'/\*(.|\n)*?\*/', Comment), + # Whitespace + (r'\n', Text), + (r'\s+', Text), + # Numbers + (r"0'[\\]?.", Number), + (r'0b[01]+', Number.Bin), + (r'0o[0-7]+', Number.Oct), + (r'0x[0-9a-fA-F]+', Number.Hex), + (r'\d+\.?\d*((e|E)(\+|-)?\d+)?', Number), + # Variables + (r'([A-Z_][a-zA-Z0-9_]*)', Name.Variable), + # Event handlers + (r'(after|before)(?=[(])', Keyword), + # Message forwarding handler + (r'forward(?=[(])', Keyword), + # Execution-context methods + (r'(context|parameter|this|se(lf|nder))(?=[(])', Keyword), + # Reflection + (r'(current_predicate|predicate_property)(?=[(])', Keyword), + # DCGs and term expansion + (r'(expand_(goal|term)|(goal|term)_expansion|phrase)(?=[(])', Keyword), + # Entity + (r'(abolish|c(reate|urrent))_(object|protocol|category)(?=[(])', Keyword), + (r'(object|protocol|category)_property(?=[(])', Keyword), + # Entity relations + (r'co(mplements_object|nforms_to_protocol)(?=[(])', Keyword), + (r'extends_(object|protocol|category)(?=[(])', Keyword), + (r'imp(lements_protocol|orts_category)(?=[(])', Keyword), + (r'(instantiat|specializ)es_class(?=[(])', Keyword), + # Events + (r'(current_event|(abolish|define)_events)(?=[(])', Keyword), + # Flags + (r'(create|current|set)_logtalk_flag(?=[(])', Keyword), + # Compiling, loading, and library paths + (r'logtalk_(compile|l(ibrary_path|oad|oad_context)|make(_target_action)?)(?=[(])', Keyword), + (r'\blogtalk_make\b', Keyword), + # Database + (r'(clause|retract(all)?)(?=[(])', Keyword), + (r'a(bolish|ssert(a|z))(?=[(])', Keyword), + # Control constructs + (r'(ca(ll|tch)|throw)(?=[(])', Keyword), + (r'(fa(il|lse)|true|(instantiation|system)_error)\b', Keyword), + (r'(type|domain|existence|permission|representation|evaluation|resource|syntax)_error(?=[(])', Keyword), + # All solutions + (r'((bag|set)of|f(ind|or)all)(?=[(])', Keyword), + # Multi-threading predicates + (r'threaded(_(ca(ll|ncel)|once|ignore|exit|peek|wait|notify))?(?=[(])', Keyword), + # Engine predicates + (r'threaded_engine(_(create|destroy|self|next|next_reified|yield|post|fetch))?(?=[(])', Keyword), + # Term unification + (r'(subsumes_term|unify_with_occurs_check)(?=[(])', Keyword), + # Term creation and decomposition + (r'(functor|arg|copy_term|numbervars|term_variables)(?=[(])', Keyword), + # Evaluable functors + (r'(div|rem|m(ax|in|od)|abs|sign)(?=[(])', Keyword), + (r'float(_(integer|fractional)_part)?(?=[(])', Keyword), + (r'(floor|t(an|runcate)|round|ceiling)(?=[(])', Keyword), + # Other arithmetic functors + (r'(cos|a(cos|sin|tan|tan2)|exp|log|s(in|qrt)|xor)(?=[(])', Keyword), + # Term testing + (r'(var|atom(ic)?|integer|float|c(allable|ompound)|n(onvar|umber)|ground|acyclic_term)(?=[(])', Keyword), + # Term comparison + (r'compare(?=[(])', Keyword), + # Stream selection and control + (r'(curren|se)t_(in|out)put(?=[(])', Keyword), + (r'(open|close)(?=[(])', Keyword), + (r'flush_output(?=[(])', Keyword), + (r'(at_end_of_stream|flush_output)\b', Keyword), + (r'(stream_property|at_end_of_stream|set_stream_position)(?=[(])', Keyword), + # Character and byte input/output + (r'(nl|(get|peek|put)_(byte|c(har|ode)))(?=[(])', Keyword), + (r'\bnl\b', Keyword), + # Term input/output + (r'read(_term)?(?=[(])', Keyword), + (r'write(q|_(canonical|term))?(?=[(])', Keyword), + (r'(current_)?op(?=[(])', Keyword), + (r'(current_)?char_conversion(?=[(])', Keyword), + # Atomic term processing + (r'atom_(length|c(hars|o(ncat|des)))(?=[(])', Keyword), + (r'(char_code|sub_atom)(?=[(])', Keyword), + (r'number_c(har|ode)s(?=[(])', Keyword), + # Implementation defined hooks functions + (r'(se|curren)t_prolog_flag(?=[(])', Keyword), + (r'\bhalt\b', Keyword), + (r'halt(?=[(])', Keyword), + # Message sending operators + (r'(::|:|\^\^)', Operator), + # External call + (r'[{}]', Keyword), + # Logic and control + (r'(ignore|once)(?=[(])', Keyword), + (r'\brepeat\b', Keyword), + # Sorting + (r'(key)?sort(?=[(])', Keyword), + # Bitwise functors + (r'(>>|<<|/\\|\\\\|\\)', Operator), + # Predicate aliases + (r'\bas\b', Operator), + # Arithmetic evaluation + (r'\bis\b', Keyword), + # Arithmetic comparison + (r'(=:=|=\\=|<|=<|>=|>)', Operator), + # Term creation and decomposition + (r'=\.\.', Operator), + # Term unification + (r'(=|\\=)', Operator), + # Term comparison + (r'(==|\\==|@=<|@<|@>=|@>)', Operator), + # Evaluable functors + (r'(//|[-+*/])', Operator), + (r'\b(e|pi|div|mod|rem)\b', Operator), + # Other arithmetic functors + (r'\b\*\*\b', Operator), + # DCG rules + (r'-->', Operator), + # Control constructs + (r'([!;]|->)', Operator), + # Logic and control + (r'\\+', Operator), + # Mode operators + (r'[?@]', Operator), + # Existential quantifier + (r'\^', Operator), + # Strings + (r'"(\\\\|\\[^\\]|[^"\\])*"', String), + # Punctuation + (r'[()\[\],.|]', Text), + # Atoms + (r"[a-z][a-zA-Z0-9_]*", Text), + (r"'", String, 'quoted_atom'), + ], + + 'quoted_atom': [ + (r"''", String), + (r"'", String, '#pop'), + (r'\\([\\abfnrtv"\']|(x[a-fA-F0-9]+|[0-7]+)\\)', String.Escape), + (r"[^\\'\n]+", String), + (r'\\', String), + ], + + 'directive': [ + # Conditional compilation directives + (r'(el)?if(?=[(])', Keyword, 'root'), + (r'(e(lse|ndif))(?=[.])', Keyword, 'root'), + # Entity directives + (r'(category|object|protocol)(?=[(])', Keyword, 'entityrelations'), + (r'(end_(category|object|protocol))(?=[.])', Keyword, 'root'), + # Predicate scope directives + (r'(public|protected|private)(?=[(])', Keyword, 'root'), + # Other directives + (r'e(n(coding|sure_loaded)|xport)(?=[(])', Keyword, 'root'), + (r'in(clude|itialization|fo)(?=[(])', Keyword, 'root'), + (r'(built_in|dynamic|synchronized|threaded)(?=[.])', Keyword, 'root'), + (r'(alias|d(ynamic|iscontiguous)|m(eta_(non_terminal|predicate)|ode|ultifile)|s(et_(logtalk|prolog)_flag|ynchronized))(?=[(])', Keyword, 'root'), + (r'op(?=[(])', Keyword, 'root'), + (r'(c(alls|oinductive)|module|reexport|use(s|_module))(?=[(])', Keyword, 'root'), + (r'[a-z][a-zA-Z0-9_]*(?=[(])', Text, 'root'), + (r'[a-z][a-zA-Z0-9_]*(?=[.])', Text, 'root'), + ], + + 'entityrelations': [ + (r'(complements|extends|i(nstantiates|mp(lements|orts))|specializes)(?=[(])', Keyword), + # Numbers + (r"0'[\\]?.", Number), + (r'0b[01]+', Number.Bin), + (r'0o[0-7]+', Number.Oct), + (r'0x[0-9a-fA-F]+', Number.Hex), + (r'\d+\.?\d*((e|E)(\+|-)?\d+)?', Number), + # Variables + (r'([A-Z_][a-zA-Z0-9_]*)', Name.Variable), + # Atoms + (r"[a-z][a-zA-Z0-9_]*", Text), + (r"'", String, 'quoted_atom'), + # Strings + (r'"(\\\\|\\[^\\]|[^"\\])*"', String), + # End of entity-opening directive + (r'([)]\.)', Text, 'root'), + # Scope operator + (r'(::)', Operator), + # Punctuation + (r'[()\[\],.|]', Text), + # Comments + (r'%.*?\n', Comment), + (r'/\*(.|\n)*?\*/', Comment), + # Whitespace + (r'\n', Text), + (r'\s+', Text), + ] + } + + def analyse_text(text): + if ':- object(' in text: + return 1.0 + elif ':- protocol(' in text: + return 1.0 + elif ':- category(' in text: + return 1.0 + elif re.search(r'^:-\s[a-z]', text, re.M): + return 0.9 + else: + return 0.0 diff --git a/pygments/lexers/promql.py b/pygments/lexers/promql.py new file mode 100644 index 0000000..dcb8b33 --- /dev/null +++ b/pygments/lexers/promql.py @@ -0,0 +1,175 @@ +""" + pygments.lexers.promql + ~~~~~~~~~~~~~~~~~~~~~~ + + Lexer for Prometheus Query Language. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups, default, words +from pygments.token import Comment, Keyword, Name, Number, Operator, \ + Punctuation, String, Whitespace + +__all__ = ["PromQLLexer"] + + +class PromQLLexer(RegexLexer): + """ + For PromQL queries. + + For details about the grammar see: + https://github.com/prometheus/prometheus/tree/master/promql/parser + + .. versionadded: 2.7 + """ + + name = "PromQL" + url = 'https://prometheus.io/docs/prometheus/latest/querying/basics/' + aliases = ["promql"] + filenames = ["*.promql"] + + base_keywords = ( + words( + ( + "bool", + "by", + "group_left", + "group_right", + "ignoring", + "offset", + "on", + "without", + ), + suffix=r"\b", + ), + Keyword, + ) + + aggregator_keywords = ( + words( + ( + "sum", + "min", + "max", + "avg", + "group", + "stddev", + "stdvar", + "count", + "count_values", + "bottomk", + "topk", + "quantile", + ), + suffix=r"\b", + ), + Keyword, + ) + + function_keywords = ( + words( + ( + "abs", + "absent", + "absent_over_time", + "avg_over_time", + "ceil", + "changes", + "clamp_max", + "clamp_min", + "count_over_time", + "day_of_month", + "day_of_week", + "days_in_month", + "delta", + "deriv", + "exp", + "floor", + "histogram_quantile", + "holt_winters", + "hour", + "idelta", + "increase", + "irate", + "label_join", + "label_replace", + "ln", + "log10", + "log2", + "max_over_time", + "min_over_time", + "minute", + "month", + "predict_linear", + "quantile_over_time", + "rate", + "resets", + "round", + "scalar", + "sort", + "sort_desc", + "sqrt", + "stddev_over_time", + "stdvar_over_time", + "sum_over_time", + "time", + "timestamp", + "vector", + "year", + ), + suffix=r"\b", + ), + Keyword.Reserved, + ) + + tokens = { + "root": [ + (r"\n", Whitespace), + (r"\s+", Whitespace), + (r",", Punctuation), + # Keywords + base_keywords, + aggregator_keywords, + function_keywords, + # Offsets + (r"[1-9][0-9]*[smhdwy]", String), + # Numbers + (r"-?[0-9]+\.[0-9]+", Number.Float), + (r"-?[0-9]+", Number.Integer), + # Comments + (r"#.*?$", Comment.Single), + # Operators + (r"(\+|\-|\*|\/|\%|\^)", Operator), + (r"==|!=|>=|<=|<|>", Operator), + (r"and|or|unless", Operator.Word), + # Metrics + (r"[_a-zA-Z][a-zA-Z0-9_]+", Name.Variable), + # Params + (r'(["\'])(.*?)(["\'])', bygroups(Punctuation, String, Punctuation)), + # Other states + (r"\(", Operator, "function"), + (r"\)", Operator), + (r"\{", Punctuation, "labels"), + (r"\[", Punctuation, "range"), + ], + "labels": [ + (r"\}", Punctuation, "#pop"), + (r"\n", Whitespace), + (r"\s+", Whitespace), + (r",", Punctuation), + (r'([_a-zA-Z][a-zA-Z0-9_]*?)(\s*?)(=~|!=|=|!~)(\s*?)("|\')(.*?)("|\')', + bygroups(Name.Label, Whitespace, Operator, Whitespace, + Punctuation, String, Punctuation)), + ], + "range": [ + (r"\]", Punctuation, "#pop"), + (r"[1-9][0-9]*[smhdwy]", String), + ], + "function": [ + (r"\)", Operator, "#pop"), + (r"\(", Operator, "#push"), + default("#pop"), + ], + } diff --git a/pygments/lexers/python.py b/pygments/lexers/python.py new file mode 100644 index 0000000..0a318a9 --- /dev/null +++ b/pygments/lexers/python.py @@ -0,0 +1,1204 @@ +""" + pygments.lexers.python + ~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for Python and related languages. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re +import keyword + +from pygments.lexer import Lexer, RegexLexer, include, bygroups, using, \ + default, words, combined, do_insertions, this, line_re +from pygments.util import get_bool_opt, shebang_matches +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Generic, Other, Error, Whitespace +from pygments import unistring as uni + +__all__ = ['PythonLexer', 'PythonConsoleLexer', 'PythonTracebackLexer', + 'Python2Lexer', 'Python2TracebackLexer', + 'CythonLexer', 'DgLexer', 'NumPyLexer'] + + +class PythonLexer(RegexLexer): + """ + For Python source code (version 3.x). + + .. versionadded:: 0.10 + + .. versionchanged:: 2.5 + This is now the default ``PythonLexer``. It is still available as the + alias ``Python3Lexer``. + """ + + name = 'Python' + url = 'http://www.python.org' + aliases = ['python', 'py', 'sage', 'python3', 'py3'] + filenames = [ + '*.py', + '*.pyw', + # Type stubs + '*.pyi', + # Jython + '*.jy', + # Sage + '*.sage', + # SCons + '*.sc', + 'SConstruct', + 'SConscript', + # Skylark/Starlark (used by Bazel, Buck, and Pants) + '*.bzl', + 'BUCK', + 'BUILD', + 'BUILD.bazel', + 'WORKSPACE', + # Twisted Application infrastructure + '*.tac', + ] + mimetypes = ['text/x-python', 'application/x-python', + 'text/x-python3', 'application/x-python3'] + + uni_name = "[%s][%s]*" % (uni.xid_start, uni.xid_continue) + + def innerstring_rules(ttype): + return [ + # the old style '%s' % (...) string formatting (still valid in Py3) + (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' + '[hlL]?[E-GXc-giorsaux%]', String.Interpol), + # the new style '{}'.format(...) string formatting + (r'\{' + r'((\w+)((\.\w+)|(\[[^\]]+\]))*)?' # field name + r'(\![sra])?' # conversion + r'(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?' + r'\}', String.Interpol), + + # backslashes, quotes and formatting signs must be parsed one at a time + (r'[^\\\'"%{\n]+', ttype), + (r'[\'"\\]', ttype), + # unhandled string formatting sign + (r'%|(\{{1,2})', ttype) + # newlines are an error (use "nl" state) + ] + + def fstring_rules(ttype): + return [ + # Assuming that a '}' is the closing brace after format specifier. + # Sadly, this means that we won't detect syntax error. But it's + # more important to parse correct syntax correctly, than to + # highlight invalid syntax. + (r'\}', String.Interpol), + (r'\{', String.Interpol, 'expr-inside-fstring'), + # backslashes, quotes and formatting signs must be parsed one at a time + (r'[^\\\'"{}\n]+', ttype), + (r'[\'"\\]', ttype), + # newlines are an error (use "nl" state) + ] + + tokens = { + 'root': [ + (r'\n', Whitespace), + (r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")', + bygroups(Whitespace, String.Affix, String.Doc)), + (r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')", + bygroups(Whitespace, String.Affix, String.Doc)), + (r'\A#!.+$', Comment.Hashbang), + (r'#.*$', Comment.Single), + (r'\\\n', Text), + (r'\\', Text), + include('keywords'), + include('soft-keywords'), + (r'(def)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'funcname'), + (r'(class)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'classname'), + (r'(from)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text), + 'fromimport'), + (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text), + 'import'), + include('expr'), + ], + 'expr': [ + # raw f-strings + ('(?i)(rf|fr)(""")', + bygroups(String.Affix, String.Double), + combined('rfstringescape', 'tdqf')), + ("(?i)(rf|fr)(''')", + bygroups(String.Affix, String.Single), + combined('rfstringescape', 'tsqf')), + ('(?i)(rf|fr)(")', + bygroups(String.Affix, String.Double), + combined('rfstringescape', 'dqf')), + ("(?i)(rf|fr)(')", + bygroups(String.Affix, String.Single), + combined('rfstringescape', 'sqf')), + # non-raw f-strings + ('([fF])(""")', bygroups(String.Affix, String.Double), + combined('fstringescape', 'tdqf')), + ("([fF])(''')", bygroups(String.Affix, String.Single), + combined('fstringescape', 'tsqf')), + ('([fF])(")', bygroups(String.Affix, String.Double), + combined('fstringescape', 'dqf')), + ("([fF])(')", bygroups(String.Affix, String.Single), + combined('fstringescape', 'sqf')), + # raw bytes and strings + ('(?i)(rb|br|r)(""")', + bygroups(String.Affix, String.Double), 'tdqs'), + ("(?i)(rb|br|r)(''')", + bygroups(String.Affix, String.Single), 'tsqs'), + ('(?i)(rb|br|r)(")', + bygroups(String.Affix, String.Double), 'dqs'), + ("(?i)(rb|br|r)(')", + bygroups(String.Affix, String.Single), 'sqs'), + # non-raw strings + ('([uU]?)(""")', bygroups(String.Affix, String.Double), + combined('stringescape', 'tdqs')), + ("([uU]?)(''')", bygroups(String.Affix, String.Single), + combined('stringescape', 'tsqs')), + ('([uU]?)(")', bygroups(String.Affix, String.Double), + combined('stringescape', 'dqs')), + ("([uU]?)(')", bygroups(String.Affix, String.Single), + combined('stringescape', 'sqs')), + # non-raw bytes + ('([bB])(""")', bygroups(String.Affix, String.Double), + combined('bytesescape', 'tdqs')), + ("([bB])(''')", bygroups(String.Affix, String.Single), + combined('bytesescape', 'tsqs')), + ('([bB])(")', bygroups(String.Affix, String.Double), + combined('bytesescape', 'dqs')), + ("([bB])(')", bygroups(String.Affix, String.Single), + combined('bytesescape', 'sqs')), + + (r'[^\S\n]+', Text), + include('numbers'), + (r'!=|==|<<|>>|:=|[-~+/*%=<>&^|.]', Operator), + (r'[]{}:(),;[]', Punctuation), + (r'(in|is|and|or|not)\b', Operator.Word), + include('expr-keywords'), + include('builtins'), + include('magicfuncs'), + include('magicvars'), + include('name'), + ], + 'expr-inside-fstring': [ + (r'[{([]', Punctuation, 'expr-inside-fstring-inner'), + # without format specifier + (r'(=\s*)?' # debug (https://bugs.python.org/issue36817) + r'(\![sraf])?' # conversion + r'\}', String.Interpol, '#pop'), + # with format specifier + # we'll catch the remaining '}' in the outer scope + (r'(=\s*)?' # debug (https://bugs.python.org/issue36817) + r'(\![sraf])?' # conversion + r':', String.Interpol, '#pop'), + (r'\s+', Whitespace), # allow new lines + include('expr'), + ], + 'expr-inside-fstring-inner': [ + (r'[{([]', Punctuation, 'expr-inside-fstring-inner'), + (r'[])}]', Punctuation, '#pop'), + (r'\s+', Whitespace), # allow new lines + include('expr'), + ], + 'expr-keywords': [ + # Based on https://docs.python.org/3/reference/expressions.html + (words(( + 'async for', 'await', 'else', 'for', 'if', 'lambda', + 'yield', 'yield from'), suffix=r'\b'), + Keyword), + (words(('True', 'False', 'None'), suffix=r'\b'), Keyword.Constant), + ], + 'keywords': [ + (words(( + 'assert', 'async', 'await', 'break', 'continue', 'del', 'elif', + 'else', 'except', 'finally', 'for', 'global', 'if', 'lambda', + 'pass', 'raise', 'nonlocal', 'return', 'try', 'while', 'yield', + 'yield from', 'as', 'with'), suffix=r'\b'), + Keyword), + (words(('True', 'False', 'None'), suffix=r'\b'), Keyword.Constant), + ], + 'soft-keywords': [ + # `match`, `case` and `_` soft keywords + (r'(^[ \t]*)' # at beginning of line + possible indentation + r'(match|case)\b' # a possible keyword + r'(?![ \t]*(?:' # not followed by... + r'[:,;=^&|@~)\]}]|(?:' + # characters and keywords that mean this isn't + r'|'.join(keyword.kwlist) + r')\b))', # pattern matching + bygroups(Text, Keyword), 'soft-keywords-inner'), + ], + 'soft-keywords-inner': [ + # optional `_` keyword + (r'(\s+)([^\n_]*)(_\b)', bygroups(Whitespace, using(this), Keyword)), + default('#pop') + ], + 'builtins': [ + (words(( + '__import__', 'abs', 'all', 'any', 'bin', 'bool', 'bytearray', + 'breakpoint', 'bytes', 'chr', 'classmethod', 'compile', 'complex', + 'delattr', 'dict', 'dir', 'divmod', 'enumerate', 'eval', 'filter', + 'float', 'format', 'frozenset', 'getattr', 'globals', 'hasattr', + 'hash', 'hex', 'id', 'input', 'int', 'isinstance', 'issubclass', + 'iter', 'len', 'list', 'locals', 'map', 'max', 'memoryview', + 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'print', + 'property', 'range', 'repr', 'reversed', 'round', 'set', 'setattr', + 'slice', 'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', + 'type', 'vars', 'zip'), prefix=r'(?>|[-~+/*%=<>&^|.]', Operator), + include('keywords'), + (r'(def)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'funcname'), + (r'(class)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'classname'), + (r'(from)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text), + 'fromimport'), + (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text), + 'import'), + include('builtins'), + include('magicfuncs'), + include('magicvars'), + include('backtick'), + ('([rR]|[uUbB][rR]|[rR][uUbB])(""")', + bygroups(String.Affix, String.Double), 'tdqs'), + ("([rR]|[uUbB][rR]|[rR][uUbB])(''')", + bygroups(String.Affix, String.Single), 'tsqs'), + ('([rR]|[uUbB][rR]|[rR][uUbB])(")', + bygroups(String.Affix, String.Double), 'dqs'), + ("([rR]|[uUbB][rR]|[rR][uUbB])(')", + bygroups(String.Affix, String.Single), 'sqs'), + ('([uUbB]?)(""")', bygroups(String.Affix, String.Double), + combined('stringescape', 'tdqs')), + ("([uUbB]?)(''')", bygroups(String.Affix, String.Single), + combined('stringescape', 'tsqs')), + ('([uUbB]?)(")', bygroups(String.Affix, String.Double), + combined('stringescape', 'dqs')), + ("([uUbB]?)(')", bygroups(String.Affix, String.Single), + combined('stringescape', 'sqs')), + include('name'), + include('numbers'), + ], + 'keywords': [ + (words(( + 'assert', 'break', 'continue', 'del', 'elif', 'else', 'except', + 'exec', 'finally', 'for', 'global', 'if', 'lambda', 'pass', + 'print', 'raise', 'return', 'try', 'while', 'yield', + 'yield from', 'as', 'with'), suffix=r'\b'), + Keyword), + ], + 'builtins': [ + (words(( + '__import__', 'abs', 'all', 'any', 'apply', 'basestring', 'bin', + 'bool', 'buffer', 'bytearray', 'bytes', 'callable', 'chr', 'classmethod', + 'cmp', 'coerce', 'compile', 'complex', 'delattr', 'dict', 'dir', 'divmod', + 'enumerate', 'eval', 'execfile', 'exit', 'file', 'filter', 'float', + 'frozenset', 'getattr', 'globals', 'hasattr', 'hash', 'hex', 'id', + 'input', 'int', 'intern', 'isinstance', 'issubclass', 'iter', 'len', + 'list', 'locals', 'long', 'map', 'max', 'min', 'next', 'object', + 'oct', 'open', 'ord', 'pow', 'property', 'range', 'raw_input', 'reduce', + 'reload', 'repr', 'reversed', 'round', 'set', 'setattr', 'slice', + 'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', 'type', + 'unichr', 'unicode', 'vars', 'xrange', 'zip'), + prefix=r'(?>> a = 'foo' + >>> print a + foo + >>> 1 / 0 + Traceback (most recent call last): + File "", line 1, in + ZeroDivisionError: integer division or modulo by zero + + Additional options: + + `python3` + Use Python 3 lexer for code. Default is ``True``. + + .. versionadded:: 1.0 + .. versionchanged:: 2.5 + Now defaults to ``True``. + """ + name = 'Python console session' + aliases = ['pycon'] + mimetypes = ['text/x-python-doctest'] + + def __init__(self, **options): + self.python3 = get_bool_opt(options, 'python3', True) + Lexer.__init__(self, **options) + + def get_tokens_unprocessed(self, text): + if self.python3: + pylexer = PythonLexer(**self.options) + tblexer = PythonTracebackLexer(**self.options) + else: + pylexer = Python2Lexer(**self.options) + tblexer = Python2TracebackLexer(**self.options) + + curcode = '' + insertions = [] + curtb = '' + tbindex = 0 + tb = 0 + for match in line_re.finditer(text): + line = match.group() + if line.startswith('>>> ') or line.startswith('... '): + tb = 0 + insertions.append((len(curcode), + [(0, Generic.Prompt, line[:4])])) + curcode += line[4:] + elif line.rstrip() == '...' and not tb: + # only a new >>> prompt can end an exception block + # otherwise an ellipsis in place of the traceback frames + # will be mishandled + insertions.append((len(curcode), + [(0, Generic.Prompt, '...')])) + curcode += line[3:] + else: + if curcode: + yield from do_insertions( + insertions, pylexer.get_tokens_unprocessed(curcode)) + curcode = '' + insertions = [] + if (line.startswith('Traceback (most recent call last):') or + re.match(' File "[^"]+", line \\d+\\n$', line)): + tb = 1 + curtb = line + tbindex = match.start() + elif line == 'KeyboardInterrupt\n': + yield match.start(), Name.Class, line + elif tb: + curtb += line + if not (line.startswith(' ') or line.strip() == '...'): + tb = 0 + for i, t, v in tblexer.get_tokens_unprocessed(curtb): + yield tbindex+i, t, v + curtb = '' + else: + yield match.start(), Generic.Output, line + if curcode: + yield from do_insertions(insertions, + pylexer.get_tokens_unprocessed(curcode)) + if curtb: + for i, t, v in tblexer.get_tokens_unprocessed(curtb): + yield tbindex+i, t, v + + +class PythonTracebackLexer(RegexLexer): + """ + For Python 3.x tracebacks, with support for chained exceptions. + + .. versionadded:: 1.0 + + .. versionchanged:: 2.5 + This is now the default ``PythonTracebackLexer``. It is still available + as the alias ``Python3TracebackLexer``. + """ + + name = 'Python Traceback' + aliases = ['pytb', 'py3tb'] + filenames = ['*.pytb', '*.py3tb'] + mimetypes = ['text/x-python-traceback', 'text/x-python3-traceback'] + + tokens = { + 'root': [ + (r'\n', Whitespace), + (r'^Traceback \(most recent call last\):\n', Generic.Traceback, 'intb'), + (r'^During handling of the above exception, another ' + r'exception occurred:\n\n', Generic.Traceback), + (r'^The above exception was the direct cause of the ' + r'following exception:\n\n', Generic.Traceback), + (r'^(?= File "[^"]+", line \d+)', Generic.Traceback, 'intb'), + (r'^.*\n', Other), + ], + 'intb': [ + (r'^( File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)', + bygroups(Text, Name.Builtin, Text, Number, Text, Name, Whitespace)), + (r'^( File )("[^"]+")(, line )(\d+)(\n)', + bygroups(Text, Name.Builtin, Text, Number, Whitespace)), + (r'^( )(.+)(\n)', + bygroups(Whitespace, using(PythonLexer), Whitespace), 'markers'), + (r'^([ \t]*)(\.\.\.)(\n)', + bygroups(Whitespace, Comment, Whitespace)), # for doctests... + (r'^([^:]+)(: )(.+)(\n)', + bygroups(Generic.Error, Text, Name, Whitespace), '#pop'), + (r'^([a-zA-Z_][\w.]*)(:?\n)', + bygroups(Generic.Error, Whitespace), '#pop') + ], + 'markers': [ + # Either `PEP 657 ` + # error locations in Python 3.11+, or single-caret markers + # for syntax errors before that. + (r'^( {4,})([~^]+)(\n)', + bygroups(Whitespace, Punctuation.Marker, Whitespace), + '#pop'), + default('#pop'), + ], + } + + +Python3TracebackLexer = PythonTracebackLexer + + +class Python2TracebackLexer(RegexLexer): + """ + For Python tracebacks. + + .. versionadded:: 0.7 + + .. versionchanged:: 2.5 + This class has been renamed from ``PythonTracebackLexer``. + ``PythonTracebackLexer`` now refers to the Python 3 variant. + """ + + name = 'Python 2.x Traceback' + aliases = ['py2tb'] + filenames = ['*.py2tb'] + mimetypes = ['text/x-python2-traceback'] + + tokens = { + 'root': [ + # Cover both (most recent call last) and (innermost last) + # The optional ^C allows us to catch keyboard interrupt signals. + (r'^(\^C)?(Traceback.*\n)', + bygroups(Text, Generic.Traceback), 'intb'), + # SyntaxError starts with this. + (r'^(?= File "[^"]+", line \d+)', Generic.Traceback, 'intb'), + (r'^.*\n', Other), + ], + 'intb': [ + (r'^( File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)', + bygroups(Text, Name.Builtin, Text, Number, Text, Name, Whitespace)), + (r'^( File )("[^"]+")(, line )(\d+)(\n)', + bygroups(Text, Name.Builtin, Text, Number, Whitespace)), + (r'^( )(.+)(\n)', + bygroups(Text, using(Python2Lexer), Whitespace), 'marker'), + (r'^([ \t]*)(\.\.\.)(\n)', + bygroups(Text, Comment, Whitespace)), # for doctests... + (r'^([^:]+)(: )(.+)(\n)', + bygroups(Generic.Error, Text, Name, Whitespace), '#pop'), + (r'^([a-zA-Z_]\w*)(:?\n)', + bygroups(Generic.Error, Whitespace), '#pop') + ], + 'marker': [ + # For syntax errors. + (r'( {4,})(\^)', bygroups(Text, Punctuation.Marker), '#pop'), + default('#pop'), + ], + } + + +class CythonLexer(RegexLexer): + """ + For Pyrex and Cython source code. + + .. versionadded:: 1.1 + """ + + name = 'Cython' + url = 'http://cython.org' + aliases = ['cython', 'pyx', 'pyrex'] + filenames = ['*.pyx', '*.pxd', '*.pxi'] + mimetypes = ['text/x-cython', 'application/x-cython'] + + tokens = { + 'root': [ + (r'\n', Whitespace), + (r'^(\s*)("""(?:.|\n)*?""")', bygroups(Whitespace, String.Doc)), + (r"^(\s*)('''(?:.|\n)*?''')", bygroups(Whitespace, String.Doc)), + (r'[^\S\n]+', Text), + (r'#.*$', Comment), + (r'[]{}:(),;[]', Punctuation), + (r'\\\n', Whitespace), + (r'\\', Text), + (r'(in|is|and|or|not)\b', Operator.Word), + (r'(<)([a-zA-Z0-9.?]+)(>)', + bygroups(Punctuation, Keyword.Type, Punctuation)), + (r'!=|==|<<|>>|[-~+/*%=<>&^|.?]', Operator), + (r'(from)(\d+)(<=)(\s+)(<)(\d+)(:)', + bygroups(Keyword, Number.Integer, Operator, Name, Operator, + Name, Punctuation)), + include('keywords'), + (r'(def|property)(\s+)', bygroups(Keyword, Text), 'funcname'), + (r'(cp?def)(\s+)', bygroups(Keyword, Text), 'cdef'), + # (should actually start a block with only cdefs) + (r'(cdef)(:)', bygroups(Keyword, Punctuation)), + (r'(class|struct)(\s+)', bygroups(Keyword, Text), 'classname'), + (r'(from)(\s+)', bygroups(Keyword, Text), 'fromimport'), + (r'(c?import)(\s+)', bygroups(Keyword, Text), 'import'), + include('builtins'), + include('backtick'), + ('(?:[rR]|[uU][rR]|[rR][uU])"""', String, 'tdqs'), + ("(?:[rR]|[uU][rR]|[rR][uU])'''", String, 'tsqs'), + ('(?:[rR]|[uU][rR]|[rR][uU])"', String, 'dqs'), + ("(?:[rR]|[uU][rR]|[rR][uU])'", String, 'sqs'), + ('[uU]?"""', String, combined('stringescape', 'tdqs')), + ("[uU]?'''", String, combined('stringescape', 'tsqs')), + ('[uU]?"', String, combined('stringescape', 'dqs')), + ("[uU]?'", String, combined('stringescape', 'sqs')), + include('name'), + include('numbers'), + ], + 'keywords': [ + (words(( + 'assert', 'async', 'await', 'break', 'by', 'continue', 'ctypedef', 'del', 'elif', + 'else', 'except', 'except?', 'exec', 'finally', 'for', 'fused', 'gil', + 'global', 'if', 'include', 'lambda', 'nogil', 'pass', 'print', + 'raise', 'return', 'try', 'while', 'yield', 'as', 'with'), suffix=r'\b'), + Keyword), + (r'(DEF|IF|ELIF|ELSE)\b', Comment.Preproc), + ], + 'builtins': [ + (words(( + '__import__', 'abs', 'all', 'any', 'apply', 'basestring', 'bin', 'bint', + 'bool', 'buffer', 'bytearray', 'bytes', 'callable', 'chr', + 'classmethod', 'cmp', 'coerce', 'compile', 'complex', 'delattr', + 'dict', 'dir', 'divmod', 'enumerate', 'eval', 'execfile', 'exit', + 'file', 'filter', 'float', 'frozenset', 'getattr', 'globals', + 'hasattr', 'hash', 'hex', 'id', 'input', 'int', 'intern', 'isinstance', + 'issubclass', 'iter', 'len', 'list', 'locals', 'long', 'map', 'max', + 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'property', 'Py_ssize_t', + 'range', 'raw_input', 'reduce', 'reload', 'repr', 'reversed', + 'round', 'set', 'setattr', 'slice', 'sorted', 'staticmethod', + 'str', 'sum', 'super', 'tuple', 'type', 'unichr', 'unicode', 'unsigned', + 'vars', 'xrange', 'zip'), prefix=r'(?`_ source code. + + .. versionadded:: 2.12 + """ + + name = "K" + aliases = ["k"] + filenames = ["*.k"] + + tokens = { + "whitespace": [ + # hashbang script + (r"^#!.*", Comment.Hashbang), + # Comments + (r"^/\s*\n", Comment.Multiline, "comments"), + (r"(??/\\:']?:)(\s*)(\{)", + bygroups(Name.Function, Whitespace, Operator, Whitespace, Punctuation), + "functions"), + # Variable Names + (r"([.]?[a-zA-Z][\w.]*)(\s*)([-.~=!@#$%^&*_+|,<>?/\\:']?:)", + bygroups(Name.Variable, Whitespace, Operator)), + # Functions + (r"\{", Punctuation, "functions"), + # Parentheses + (r"\(", Punctuation, "parentheses"), + # Brackets + (r"\[", Punctuation, "brackets"), + # Errors + (r"'`([a-zA-Z][\w.]*)?", Name.Exception), + # File Symbols + (r"`:([a-zA-Z/][\w./]*)?", String.Symbol), + # Symbols + (r"`([a-zA-Z][\w.]*)?", String.Symbol), + # Numbers + include("numbers"), + # Variable Names + (r"[a-zA-Z][\w.]*", Name), + # Operators + (r"[-=+*#$%@!~^&:.,<>'\\|/?_]", Operator), + # Punctuation + (r";", Punctuation), + ], + "functions": [ + include("root"), + (r"\}", Punctuation, "#pop"), + ], + "parentheses": [ + include("root"), + (r"\)", Punctuation, "#pop"), + ], + "brackets": [ + include("root"), + (r"\]", Punctuation, "#pop"), + ], + "numbers": [ + # Binary Values + (r"[01]+b", Number.Bin), + # Nulls/Infinities + (r"0[nNwW][cefghijmndzuvtp]?", Number), + # Timestamps + ((r"(?:[0-9]{4}[.][0-9]{2}[.][0-9]{2}|[0-9]+)" + "D(?:[0-9](?:[0-9](?::[0-9]{2}" + "(?::[0-9]{2}(?:[.][0-9]*)?)?)?)?)?"), Literal.Date), + # Datetimes + ((r"[0-9]{4}[.][0-9]{2}" + "(?:m|[.][0-9]{2}(?:T(?:[0-9]{2}:[0-9]{2}" + "(?::[0-9]{2}(?:[.][0-9]*)?)?)?)?)"), Literal.Date), + # Times + (r"[0-9]{2}:[0-9]{2}(?::[0-9]{2}(?:[.][0-9]{1,3})?)?", + Literal.Date), + # GUIDs + (r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", + Number.Hex), + # Byte Vectors + (r"0x[0-9a-fA-F]+", Number.Hex), + # Floats + (r"([0-9]*[.]?[0-9]+|[0-9]+[.]?[0-9]*)[eE][+-]?[0-9]+[ef]?", + Number.Float), + (r"([0-9]*[.][0-9]+|[0-9]+[.][0-9]*)[ef]?", Number.Float), + (r"[0-9]+[ef]", Number.Float), + # Characters + (r"[0-9]+c", Number), + # Integers + (r"[0-9]+[ihtuv]", Number.Integer), + # Long Integers + (r"[0-9]+[jnp]?", Number.Integer.Long), + ], + "comments": [ + (r"[^\\]+", Comment.Multiline), + (r"^\\", Comment.Multiline, "#pop"), + (r"\\", Comment.Multiline), + ], + "strings": [ + (r'[^"\\]+', String.Double), + (r"\\.", String.Escape), + (r'"', String.Double, "#pop"), + ], + } + + +class QLexer(KLexer): + """ + For `Q `_ source code. + + .. versionadded:: 2.12 + """ + + name = "Q" + aliases = ["q"] + filenames = ["*.q"] + + tokens = { + "root": [ + (words(("aj", "aj0", "ajf", "ajf0", "all", "and", "any", "asc", + "asof", "attr", "avgs", "ceiling", "cols", "count", "cross", + "csv", "cut", "deltas", "desc", "differ", "distinct", "dsave", + "each", "ej", "ema", "eval", "except", "fby", "fills", "first", + "fkeys", "flip", "floor", "get", "group", "gtime", "hclose", + "hcount", "hdel", "hsym", "iasc", "idesc", "ij", "ijf", + "inter", "inv", "key", "keys", "lj", "ljf", "load", "lower", + "lsq", "ltime", "ltrim", "mavg", "maxs", "mcount", "md5", + "mdev", "med", "meta", "mins", "mmax", "mmin", "mmu", "mod", + "msum", "neg", "next", "not", "null", "or", "over", "parse", + "peach", "pj", "prds", "prior", "prev", "rand", "rank", "ratios", + "raze", "read0", "read1", "reciprocal", "reval", "reverse", + "rload", "rotate", "rsave", "rtrim", "save", "scan", "scov", + "sdev", "set", "show", "signum", "ssr", "string", "sublist", + "sums", "sv", "svar", "system", "tables", "til", "trim", "txf", + "type", "uj", "ujf", "ungroup", "union", "upper", "upsert", + "value", "view", "views", "vs", "where", "wj", "wj1", "ww", + "xasc", "xbar", "xcol", "xcols", "xdesc", "xgroup", "xkey", + "xlog", "xprev", "xrank"), + suffix=r"\b"), Name.Builtin, + ), + inherit, + ], + } diff --git a/pygments/lexers/qlik.py b/pygments/lexers/qlik.py new file mode 100644 index 0000000..bb4defd --- /dev/null +++ b/pygments/lexers/qlik.py @@ -0,0 +1,117 @@ +""" + pygments.lexers.qlik + ~~~~~~~~~~~~~~~~~~~~ + + Lexer for the qlik scripting language + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, include, bygroups, words +from pygments.token import Comment, Keyword, Name, Number, Operator, \ + Punctuation, String, Text +from pygments.lexers._qlik_builtins import OPERATORS_LIST, STATEMENT_LIST, \ + SCRIPT_FUNCTIONS, CONSTANT_LIST + +__all__ = ["QlikLexer"] + + +class QlikLexer(RegexLexer): + """ + Lexer for qlik code, including .qvs files + + .. versionadded:: 2.12 + """ + + name = "Qlik" + aliases = ["qlik", "qlikview", "qliksense", "qlikscript"] + filenames = ["*.qvs", "*.qvw"] + + flags = re.IGNORECASE + + tokens = { + # Handle multi-line comments + "comment": [ + (r"\*/", Comment.Multiline, "#pop"), + (r"[^*]+", Comment.Multiline), + ], + # Handle numbers + "numerics": [ + (r"\b\d+\.\d+(e\d+)?[fd]?\b", Number.Float), + (r"\b\d+\b", Number.Integer), + ], + # Handle variable names in things + "interp": [ + ( + r"(\$\()(\w+)(\))", + bygroups(String.Interpol, Name.Variable, String.Interpol), + ), + ], + # Handle strings + "string": [ + (r"'", String, "#pop"), + include("interp"), + (r"[^'$]+", String), + (r"\$", String), + ], + # + "assignment": [ + (r";", Punctuation, "#pop"), + include("root"), + ], + "field_name_quote": [ + (r'"', String.Symbol, "#pop"), + include("interp"), + (r"[^\"$]+", String.Symbol), + (r"\$", String.Symbol), + ], + "field_name_bracket": [ + (r"\]", String.Symbol, "#pop"), + include("interp"), + (r"[^\]$]+", String.Symbol), + (r"\$", String.Symbol), + ], + "function": [(r"\)", Punctuation, "#pop"), include("root")], + "root": [ + # Whitespace and comments + (r"\s+", Text.Whitespace), + (r"/\*", Comment.Multiline, "comment"), + (r"//.*\n", Comment.Single), + # variable assignment + (r"(let|set)(\s+)", bygroups(Keyword.Declaration, Text.Whitespace), + "assignment"), + # Word operators + (words(OPERATORS_LIST["words"], prefix=r"\b", suffix=r"\b"), + Operator.Word), + # Statements + (words(STATEMENT_LIST, suffix=r"\b"), Keyword), + # Table names + (r"[a-z]\w*:", Keyword.Declaration), + # Constants + (words(CONSTANT_LIST, suffix=r"\b"), Keyword.Constant), + # Functions + (words(SCRIPT_FUNCTIONS, suffix=r"(?=\s*\()"), Name.Builtin, + "function"), + # interpolation - e.g. $(variableName) + include("interp"), + # Quotes denote a field/file name + (r'"', String.Symbol, "field_name_quote"), + # Square brackets denote a field/file name + (r"\[", String.Symbol, "field_name_bracket"), + # Strings + (r"'", String, "string"), + # Numbers + include("numerics"), + # Operator symbols + (words(OPERATORS_LIST["symbols"]), Operator), + # Strings denoted by single quotes + (r"'.+?'", String), + # Words as text + (r"\b\w+\b", Text), + # Basic punctuation + (r"[,;.()\\/]", Punctuation), + ], + } diff --git a/pygments/lexers/qvt.py b/pygments/lexers/qvt.py new file mode 100644 index 0000000..07d1818 --- /dev/null +++ b/pygments/lexers/qvt.py @@ -0,0 +1,151 @@ +""" + pygments.lexers.qvt + ~~~~~~~~~~~~~~~~~~~ + + Lexer for QVT Operational language. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups, include, combined, default, \ + words +from pygments.token import Text, Comment, Operator, Keyword, Punctuation, \ + Name, String, Number + +__all__ = ['QVToLexer'] + + +class QVToLexer(RegexLexer): + """ + For the `QVT Operational Mapping language `_. + + Reference for implementing this: «Meta Object Facility (MOF) 2.0 + Query/View/Transformation Specification», Version 1.1 - January 2011 + (http://www.omg.org/spec/QVT/1.1/), see §8.4, «Concrete Syntax» in + particular. + + Notable tokens assignments: + + - Name.Class is assigned to the identifier following any of the following + keywords: metamodel, class, exception, primitive, enum, transformation + or library + + - Name.Function is assigned to the names of mappings and queries + + - Name.Builtin.Pseudo is assigned to the pre-defined variables 'this', + 'self' and 'result'. + """ + # With obvious borrowings & inspiration from the Java, Python and C lexers + + name = 'QVTO' + aliases = ['qvto', 'qvt'] + filenames = ['*.qvto'] + + tokens = { + 'root': [ + (r'\n', Text), + (r'[^\S\n]+', Text), + (r'(--|//)(\s*)(directive:)?(.*)$', + bygroups(Comment, Comment, Comment.Preproc, Comment)), + # Uncomment the following if you want to distinguish between + # '/*' and '/**', à la javadoc + # (r'/[*]{2}(.|\n)*?[*]/', Comment.Multiline), + (r'/[*](.|\n)*?[*]/', Comment.Multiline), + (r'\\\n', Text), + (r'(and|not|or|xor|##?)\b', Operator.Word), + (r'(:{1,2}=|[-+]=)\b', Operator.Word), + (r'(@|<<|>>)\b', Keyword), # stereotypes + (r'!=|<>|==|=|!->|->|>=|<=|[.]{3}|[+/*%=<>&|.~]', Operator), + (r'[]{}:(),;[]', Punctuation), + (r'(true|false|unlimited|null)\b', Keyword.Constant), + (r'(this|self|result)\b', Name.Builtin.Pseudo), + (r'(var)\b', Keyword.Declaration), + (r'(from|import)\b', Keyword.Namespace, 'fromimport'), + (r'(metamodel|class|exception|primitive|enum|transformation|' + r'library)(\s+)(\w+)', + bygroups(Keyword.Word, Text, Name.Class)), + (r'(exception)(\s+)(\w+)', + bygroups(Keyword.Word, Text, Name.Exception)), + (r'(main)\b', Name.Function), + (r'(mapping|helper|query)(\s+)', + bygroups(Keyword.Declaration, Text), 'operation'), + (r'(assert)(\s+)\b', bygroups(Keyword, Text), 'assert'), + (r'(Bag|Collection|Dict|OrderedSet|Sequence|Set|Tuple|List)\b', + Keyword.Type), + include('keywords'), + ('"', String, combined('stringescape', 'dqs')), + ("'", String, combined('stringescape', 'sqs')), + include('name'), + include('numbers'), + # (r'([a-zA-Z_]\w*)(::)([a-zA-Z_]\w*)', + # bygroups(Text, Text, Text)), + ], + + 'fromimport': [ + (r'(?:[ \t]|\\\n)+', Text), + (r'[a-zA-Z_][\w.]*', Name.Namespace), + default('#pop'), + ], + + 'operation': [ + (r'::', Text), + (r'(.*::)([a-zA-Z_]\w*)([ \t]*)(\()', + bygroups(Text, Name.Function, Text, Punctuation), '#pop') + ], + + 'assert': [ + (r'(warning|error|fatal)\b', Keyword, '#pop'), + default('#pop'), # all else: go back + ], + + 'keywords': [ + (words(( + 'abstract', 'access', 'any', 'assert', 'blackbox', 'break', + 'case', 'collect', 'collectNested', 'collectOne', 'collectselect', + 'collectselectOne', 'composes', 'compute', 'configuration', + 'constructor', 'continue', 'datatype', 'default', 'derived', + 'disjuncts', 'do', 'elif', 'else', 'end', 'endif', 'except', + 'exists', 'extends', 'forAll', 'forEach', 'forOne', 'from', 'if', + 'implies', 'in', 'inherits', 'init', 'inout', 'intermediate', + 'invresolve', 'invresolveIn', 'invresolveone', 'invresolveoneIn', + 'isUnique', 'iterate', 'late', 'let', 'literal', 'log', 'map', + 'merges', 'modeltype', 'new', 'object', 'one', 'ordered', 'out', + 'package', 'population', 'property', 'raise', 'readonly', + 'references', 'refines', 'reject', 'resolve', 'resolveIn', + 'resolveone', 'resolveoneIn', 'return', 'select', 'selectOne', + 'sortedBy', 'static', 'switch', 'tag', 'then', 'try', 'typedef', + 'unlimited', 'uses', 'when', 'where', 'while', 'with', 'xcollect', + 'xmap', 'xselect'), suffix=r'\b'), Keyword), + ], + + # There is no need to distinguish between String.Single and + # String.Double: 'strings' is factorised for 'dqs' and 'sqs' + 'strings': [ + (r'[^\\\'"\n]+', String), + # quotes, percents and backslashes must be parsed one at a time + (r'[\'"\\]', String), + ], + 'stringescape': [ + (r'\\([\\btnfr"\']|u[0-3][0-7]{2}|u[0-7]{1,2})', String.Escape) + ], + 'dqs': [ # double-quoted string + (r'"', String, '#pop'), + (r'\\\\|\\"', String.Escape), + include('strings') + ], + 'sqs': [ # single-quoted string + (r"'", String, '#pop'), + (r"\\\\|\\'", String.Escape), + include('strings') + ], + 'name': [ + (r'[a-zA-Z_]\w*', Name), + ], + # numbers: excerpt taken from the python lexer + 'numbers': [ + (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float), + (r'\d+[eE][+-]?[0-9]+', Number.Float), + (r'\d+', Number.Integer) + ], + } diff --git a/pygments/lexers/r.py b/pygments/lexers/r.py new file mode 100644 index 0000000..02c61d2 --- /dev/null +++ b/pygments/lexers/r.py @@ -0,0 +1,190 @@ +""" + pygments.lexers.r + ~~~~~~~~~~~~~~~~~ + + Lexers for the R/S languages. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import Lexer, RegexLexer, include, do_insertions +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Generic, Whitespace + +__all__ = ['RConsoleLexer', 'SLexer', 'RdLexer'] + + +line_re = re.compile('.*?\n') + + +class RConsoleLexer(Lexer): + """ + For R console transcripts or R CMD BATCH output files. + """ + + name = 'RConsole' + aliases = ['rconsole', 'rout'] + filenames = ['*.Rout'] + + def get_tokens_unprocessed(self, text): + slexer = SLexer(**self.options) + + current_code_block = '' + insertions = [] + + for match in line_re.finditer(text): + line = match.group() + if line.startswith('>') or line.startswith('+'): + # Colorize the prompt as such, + # then put rest of line into current_code_block + insertions.append((len(current_code_block), + [(0, Generic.Prompt, line[:2])])) + current_code_block += line[2:] + else: + # We have reached a non-prompt line! + # If we have stored prompt lines, need to process them first. + if current_code_block: + # Weave together the prompts and highlight code. + yield from do_insertions( + insertions, slexer.get_tokens_unprocessed(current_code_block)) + # Reset vars for next code block. + current_code_block = '' + insertions = [] + # Now process the actual line itself, this is output from R. + yield match.start(), Generic.Output, line + + # If we happen to end on a code block with nothing after it, need to + # process the last code block. This is neither elegant nor DRY so + # should be changed. + if current_code_block: + yield from do_insertions( + insertions, slexer.get_tokens_unprocessed(current_code_block)) + + +class SLexer(RegexLexer): + """ + For S, S-plus, and R source code. + + .. versionadded:: 0.10 + """ + + name = 'S' + aliases = ['splus', 's', 'r'] + filenames = ['*.S', '*.R', '.Rhistory', '.Rprofile', '.Renviron'] + mimetypes = ['text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', + 'text/x-R', 'text/x-r-history', 'text/x-r-profile'] + + valid_name = r'`[^`\\]*(?:\\.[^`\\]*)*`|(?:[a-zA-Z]|\.[A-Za-z_.])[\w.]*|\.' + tokens = { + 'comments': [ + (r'#.*$', Comment.Single), + ], + 'valid_name': [ + (valid_name, Name), + ], + 'punctuation': [ + (r'\[{1,2}|\]{1,2}|\(|\)|;|,', Punctuation), + ], + 'keywords': [ + (r'(if|else|for|while|repeat|in|next|break|return|switch|function)' + r'(?![\w.])', + Keyword.Reserved), + ], + 'operators': [ + (r'<>?|-|==|<=|>=|<|>|&&?|!=|\|\|?|\?', Operator), + (r'\*|\+|\^|/|!|%[^%]*%|=|~|\$|@|:{1,3}', Operator), + ], + 'builtin_symbols': [ + (r'(NULL|NA(_(integer|real|complex|character)_)?|' + r'letters|LETTERS|Inf|TRUE|FALSE|NaN|pi|\.\.(\.|[0-9]+))' + r'(?![\w.])', + Keyword.Constant), + (r'(T|F)\b', Name.Builtin.Pseudo), + ], + 'numbers': [ + # hex number + (r'0[xX][a-fA-F0-9]+([pP][0-9]+)?[Li]?', Number.Hex), + # decimal number + (r'[+-]?([0-9]+(\.[0-9]+)?|\.[0-9]+|\.)([eE][+-]?[0-9]+)?[Li]?', + Number), + ], + 'statements': [ + include('comments'), + # whitespaces + (r'\s+', Whitespace), + (r'\'', String, 'string_squote'), + (r'\"', String, 'string_dquote'), + include('builtin_symbols'), + include('valid_name'), + include('numbers'), + include('keywords'), + include('punctuation'), + include('operators'), + ], + 'root': [ + # calls: + (r'(%s)\s*(?=\()' % valid_name, Name.Function), + include('statements'), + # blocks: + (r'\{|\}', Punctuation), + # (r'\{', Punctuation, 'block'), + (r'.', Text), + ], + # 'block': [ + # include('statements'), + # ('\{', Punctuation, '#push'), + # ('\}', Punctuation, '#pop') + # ], + 'string_squote': [ + (r'([^\'\\]|\\.)*\'', String, '#pop'), + ], + 'string_dquote': [ + (r'([^"\\]|\\.)*"', String, '#pop'), + ], + } + + def analyse_text(text): + if re.search(r'[a-z0-9_\])\s]<-(?!-)', text): + return 0.11 + + +class RdLexer(RegexLexer): + """ + Pygments Lexer for R documentation (Rd) files + + This is a very minimal implementation, highlighting little more + than the macros. A description of Rd syntax is found in `Writing R + Extensions `_ + and `Parsing Rd files `_. + + .. versionadded:: 1.6 + """ + name = 'Rd' + aliases = ['rd'] + filenames = ['*.Rd'] + mimetypes = ['text/x-r-doc'] + + # To account for verbatim / LaTeX-like / and R-like areas + # would require parsing. + tokens = { + 'root': [ + # catch escaped brackets and percent sign + (r'\\[\\{}%]', String.Escape), + # comments + (r'%.*$', Comment), + # special macros with no arguments + (r'\\(?:cr|l?dots|R|tab)\b', Keyword.Constant), + # macros + (r'\\[a-zA-Z]+\b', Keyword), + # special preprocessor macros + (r'^\s*#(?:ifn?def|endif).*\b', Comment.Preproc), + # non-escaped brackets + (r'[{}]', Name.Builtin), + # everything else + (r'[^\\%\n{}]+', Text), + (r'.', Text), + ] + } diff --git a/pygments/lexers/rdf.py b/pygments/lexers/rdf.py new file mode 100644 index 0000000..3919a9b --- /dev/null +++ b/pygments/lexers/rdf.py @@ -0,0 +1,462 @@ +""" + pygments.lexers.rdf + ~~~~~~~~~~~~~~~~~~~ + + Lexers for semantic web and RDF query languages and markup. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, bygroups, default +from pygments.token import Keyword, Punctuation, String, Number, Operator, \ + Generic, Whitespace, Name, Literal, Comment, Text + +__all__ = ['SparqlLexer', 'TurtleLexer', 'ShExCLexer'] + + +class SparqlLexer(RegexLexer): + """ + Lexer for `SPARQL `_ query language. + + .. versionadded:: 2.0 + """ + name = 'SPARQL' + aliases = ['sparql'] + filenames = ['*.rq', '*.sparql'] + mimetypes = ['application/sparql-query'] + + # character group definitions :: + + PN_CHARS_BASE_GRP = ('a-zA-Z' + '\u00c0-\u00d6' + '\u00d8-\u00f6' + '\u00f8-\u02ff' + '\u0370-\u037d' + '\u037f-\u1fff' + '\u200c-\u200d' + '\u2070-\u218f' + '\u2c00-\u2fef' + '\u3001-\ud7ff' + '\uf900-\ufdcf' + '\ufdf0-\ufffd') + + PN_CHARS_U_GRP = (PN_CHARS_BASE_GRP + '_') + + PN_CHARS_GRP = (PN_CHARS_U_GRP + + r'\-' + + r'0-9' + + '\u00b7' + + '\u0300-\u036f' + + '\u203f-\u2040') + + HEX_GRP = '0-9A-Fa-f' + + PN_LOCAL_ESC_CHARS_GRP = r' _~.\-!$&"()*+,;=/?#@%' + + # terminal productions :: + + PN_CHARS_BASE = '[' + PN_CHARS_BASE_GRP + ']' + + PN_CHARS_U = '[' + PN_CHARS_U_GRP + ']' + + PN_CHARS = '[' + PN_CHARS_GRP + ']' + + HEX = '[' + HEX_GRP + ']' + + PN_LOCAL_ESC_CHARS = '[' + PN_LOCAL_ESC_CHARS_GRP + ']' + + IRIREF = r'<(?:[^<>"{}|^`\\\x00-\x20])*>' + + BLANK_NODE_LABEL = '_:[0-9' + PN_CHARS_U_GRP + '](?:[' + PN_CHARS_GRP + \ + '.]*' + PN_CHARS + ')?' + + PN_PREFIX = PN_CHARS_BASE + '(?:[' + PN_CHARS_GRP + '.]*' + PN_CHARS + ')?' + + VARNAME = '[0-9' + PN_CHARS_U_GRP + '][' + PN_CHARS_U_GRP + \ + '0-9\u00b7\u0300-\u036f\u203f-\u2040]*' + + PERCENT = '%' + HEX + HEX + + PN_LOCAL_ESC = r'\\' + PN_LOCAL_ESC_CHARS + + PLX = '(?:' + PERCENT + ')|(?:' + PN_LOCAL_ESC + ')' + + PN_LOCAL = ('(?:[' + PN_CHARS_U_GRP + ':0-9' + ']|' + PLX + ')' + + '(?:(?:[' + PN_CHARS_GRP + '.:]|' + PLX + ')*(?:[' + + PN_CHARS_GRP + ':]|' + PLX + '))?') + + EXPONENT = r'[eE][+-]?\d+' + + # Lexer token definitions :: + + tokens = { + 'root': [ + (r'\s+', Text), + # keywords :: + (r'(?i)(select|construct|describe|ask|where|filter|group\s+by|minus|' + r'distinct|reduced|from\s+named|from|order\s+by|desc|asc|limit|' + r'offset|values|bindings|load|into|clear|drop|create|add|move|copy|' + r'insert\s+data|delete\s+data|delete\s+where|with|delete|insert|' + r'using\s+named|using|graph|default|named|all|optional|service|' + r'silent|bind|undef|union|not\s+in|in|as|having|to|prefix|base)\b', Keyword), + (r'(a)\b', Keyword), + # IRIs :: + ('(' + IRIREF + ')', Name.Label), + # blank nodes :: + ('(' + BLANK_NODE_LABEL + ')', Name.Label), + # # variables :: + ('[?$]' + VARNAME, Name.Variable), + # prefixed names :: + (r'(' + PN_PREFIX + r')?(\:)(' + PN_LOCAL + r')?', + bygroups(Name.Namespace, Punctuation, Name.Tag)), + # function names :: + (r'(?i)(str|lang|langmatches|datatype|bound|iri|uri|bnode|rand|abs|' + r'ceil|floor|round|concat|strlen|ucase|lcase|encode_for_uri|' + r'contains|strstarts|strends|strbefore|strafter|year|month|day|' + r'hours|minutes|seconds|timezone|tz|now|uuid|struuid|md5|sha1|sha256|sha384|' + r'sha512|coalesce|if|strlang|strdt|sameterm|isiri|isuri|isblank|' + r'isliteral|isnumeric|regex|substr|replace|exists|not\s+exists|' + r'count|sum|min|max|avg|sample|group_concat|separator)\b', + Name.Function), + # boolean literals :: + (r'(true|false)', Keyword.Constant), + # double literals :: + (r'[+\-]?(\d+\.\d*' + EXPONENT + r'|\.?\d+' + EXPONENT + ')', Number.Float), + # decimal literals :: + (r'[+\-]?(\d+\.\d*|\.\d+)', Number.Float), + # integer literals :: + (r'[+\-]?\d+', Number.Integer), + # operators :: + (r'(\|\||&&|=|\*|\-|\+|/|!=|<=|>=|!|<|>)', Operator), + # punctuation characters :: + (r'[(){}.;,:^\[\]]', Punctuation), + # line comments :: + (r'#[^\n]*', Comment), + # strings :: + (r'"""', String, 'triple-double-quoted-string'), + (r'"', String, 'single-double-quoted-string'), + (r"'''", String, 'triple-single-quoted-string'), + (r"'", String, 'single-single-quoted-string'), + ], + 'triple-double-quoted-string': [ + (r'"""', String, 'end-of-string'), + (r'[^\\]+', String), + (r'\\', String, 'string-escape'), + ], + 'single-double-quoted-string': [ + (r'"', String, 'end-of-string'), + (r'[^"\\\n]+', String), + (r'\\', String, 'string-escape'), + ], + 'triple-single-quoted-string': [ + (r"'''", String, 'end-of-string'), + (r'[^\\]+', String), + (r'\\', String.Escape, 'string-escape'), + ], + 'single-single-quoted-string': [ + (r"'", String, 'end-of-string'), + (r"[^'\\\n]+", String), + (r'\\', String, 'string-escape'), + ], + 'string-escape': [ + (r'u' + HEX + '{4}', String.Escape, '#pop'), + (r'U' + HEX + '{8}', String.Escape, '#pop'), + (r'.', String.Escape, '#pop'), + ], + 'end-of-string': [ + (r'(@)([a-zA-Z]+(?:-[a-zA-Z0-9]+)*)', + bygroups(Operator, Name.Function), '#pop:2'), + (r'\^\^', Operator, '#pop:2'), + default('#pop:2'), + ], + } + + +class TurtleLexer(RegexLexer): + """ + Lexer for `Turtle `_ data language. + + .. versionadded:: 2.1 + """ + name = 'Turtle' + aliases = ['turtle'] + filenames = ['*.ttl'] + mimetypes = ['text/turtle', 'application/x-turtle'] + + # character group definitions :: + PN_CHARS_BASE_GRP = ('a-zA-Z' + '\u00c0-\u00d6' + '\u00d8-\u00f6' + '\u00f8-\u02ff' + '\u0370-\u037d' + '\u037f-\u1fff' + '\u200c-\u200d' + '\u2070-\u218f' + '\u2c00-\u2fef' + '\u3001-\ud7ff' + '\uf900-\ufdcf' + '\ufdf0-\ufffd') + + PN_CHARS_U_GRP = (PN_CHARS_BASE_GRP + '_') + + PN_CHARS_GRP = (PN_CHARS_U_GRP + + r'\-' + + r'0-9' + + '\u00b7' + + '\u0300-\u036f' + + '\u203f-\u2040') + + PN_CHARS = '[' + PN_CHARS_GRP + ']' + + PN_CHARS_BASE = '[' + PN_CHARS_BASE_GRP + ']' + + PN_PREFIX = PN_CHARS_BASE + '(?:[' + PN_CHARS_GRP + '.]*' + PN_CHARS + ')?' + + HEX_GRP = '0-9A-Fa-f' + + HEX = '[' + HEX_GRP + ']' + + PERCENT = '%' + HEX + HEX + + PN_LOCAL_ESC_CHARS_GRP = r' _~.\-!$&"()*+,;=/?#@%' + + PN_LOCAL_ESC_CHARS = '[' + PN_LOCAL_ESC_CHARS_GRP + ']' + + PN_LOCAL_ESC = r'\\' + PN_LOCAL_ESC_CHARS + + PLX = '(?:' + PERCENT + ')|(?:' + PN_LOCAL_ESC + ')' + + PN_LOCAL = ('(?:[' + PN_CHARS_U_GRP + ':0-9' + ']|' + PLX + ')' + + '(?:(?:[' + PN_CHARS_GRP + '.:]|' + PLX + ')*(?:[' + + PN_CHARS_GRP + ':]|' + PLX + '))?') + + patterns = { + 'PNAME_NS': r'((?:[a-zA-Z][\w-]*)?\:)', # Simplified character range + 'IRIREF': r'(<[^<>"{}|^`\\\x00-\x20]*>)' + } + + tokens = { + 'root': [ + (r'\s+', Text), + + # Base / prefix + (r'(@base|BASE)(\s+)%(IRIREF)s(\s*)(\.?)' % patterns, + bygroups(Keyword, Whitespace, Name.Variable, Whitespace, + Punctuation)), + (r'(@prefix|PREFIX)(\s+)%(PNAME_NS)s(\s+)%(IRIREF)s(\s*)(\.?)' % patterns, + bygroups(Keyword, Whitespace, Name.Namespace, Whitespace, + Name.Variable, Whitespace, Punctuation)), + + # The shorthand predicate 'a' + (r'(?<=\s)a(?=\s)', Keyword.Type), + + # IRIREF + (r'%(IRIREF)s' % patterns, Name.Variable), + + # PrefixedName + (r'(' + PN_PREFIX + r')?(\:)(' + PN_LOCAL + r')?', + bygroups(Name.Namespace, Punctuation, Name.Tag)), + + # Comment + (r'#[^\n]+', Comment), + + (r'\b(true|false)\b', Literal), + (r'[+\-]?\d*\.\d+', Number.Float), + (r'[+\-]?\d*(:?\.\d+)?E[+\-]?\d+', Number.Float), + (r'[+\-]?\d+', Number.Integer), + (r'[\[\](){}.;,:^]', Punctuation), + + (r'"""', String, 'triple-double-quoted-string'), + (r'"', String, 'single-double-quoted-string'), + (r"'''", String, 'triple-single-quoted-string'), + (r"'", String, 'single-single-quoted-string'), + ], + 'triple-double-quoted-string': [ + (r'"""', String, 'end-of-string'), + (r'[^\\]+', String), + (r'\\', String, 'string-escape'), + ], + 'single-double-quoted-string': [ + (r'"', String, 'end-of-string'), + (r'[^"\\\n]+', String), + (r'\\', String, 'string-escape'), + ], + 'triple-single-quoted-string': [ + (r"'''", String, 'end-of-string'), + (r'[^\\]+', String), + (r'\\', String, 'string-escape'), + ], + 'single-single-quoted-string': [ + (r"'", String, 'end-of-string'), + (r"[^'\\\n]+", String), + (r'\\', String, 'string-escape'), + ], + 'string-escape': [ + (r'.', String, '#pop'), + ], + 'end-of-string': [ + (r'(@)([a-zA-Z]+(?:-[a-zA-Z0-9]+)*)', + bygroups(Operator, Generic.Emph), '#pop:2'), + + (r'(\^\^)%(IRIREF)s' % patterns, bygroups(Operator, Generic.Emph), '#pop:2'), + + default('#pop:2'), + + ], + } + + # Turtle and Tera Term macro files share the same file extension + # but each has a recognizable and distinct syntax. + def analyse_text(text): + for t in ('@base ', 'BASE ', '@prefix ', 'PREFIX '): + if re.search(r'^\s*%s' % t, text): + return 0.80 + + +class ShExCLexer(RegexLexer): + """ + Lexer for `ShExC `_ shape expressions language syntax. + """ + name = 'ShExC' + aliases = ['shexc', 'shex'] + filenames = ['*.shex'] + mimetypes = ['text/shex'] + + # character group definitions :: + + PN_CHARS_BASE_GRP = ('a-zA-Z' + '\u00c0-\u00d6' + '\u00d8-\u00f6' + '\u00f8-\u02ff' + '\u0370-\u037d' + '\u037f-\u1fff' + '\u200c-\u200d' + '\u2070-\u218f' + '\u2c00-\u2fef' + '\u3001-\ud7ff' + '\uf900-\ufdcf' + '\ufdf0-\ufffd') + + PN_CHARS_U_GRP = (PN_CHARS_BASE_GRP + '_') + + PN_CHARS_GRP = (PN_CHARS_U_GRP + + r'\-' + + r'0-9' + + '\u00b7' + + '\u0300-\u036f' + + '\u203f-\u2040') + + HEX_GRP = '0-9A-Fa-f' + + PN_LOCAL_ESC_CHARS_GRP = r"_~.\-!$&'()*+,;=/?#@%" + + # terminal productions :: + + PN_CHARS_BASE = '[' + PN_CHARS_BASE_GRP + ']' + + PN_CHARS_U = '[' + PN_CHARS_U_GRP + ']' + + PN_CHARS = '[' + PN_CHARS_GRP + ']' + + HEX = '[' + HEX_GRP + ']' + + PN_LOCAL_ESC_CHARS = '[' + PN_LOCAL_ESC_CHARS_GRP + ']' + + UCHAR_NO_BACKSLASH = '(?:u' + HEX + '{4}|U' + HEX + '{8})' + + UCHAR = r'\\' + UCHAR_NO_BACKSLASH + + IRIREF = r'<(?:[^\x00-\x20<>"{}|^`\\]|' + UCHAR + ')*>' + + BLANK_NODE_LABEL = '_:[0-9' + PN_CHARS_U_GRP + '](?:[' + PN_CHARS_GRP + \ + '.]*' + PN_CHARS + ')?' + + PN_PREFIX = PN_CHARS_BASE + '(?:[' + PN_CHARS_GRP + '.]*' + PN_CHARS + ')?' + + PERCENT = '%' + HEX + HEX + + PN_LOCAL_ESC = r'\\' + PN_LOCAL_ESC_CHARS + + PLX = '(?:' + PERCENT + ')|(?:' + PN_LOCAL_ESC + ')' + + PN_LOCAL = ('(?:[' + PN_CHARS_U_GRP + ':0-9' + ']|' + PLX + ')' + + '(?:(?:[' + PN_CHARS_GRP + '.:]|' + PLX + ')*(?:[' + + PN_CHARS_GRP + ':]|' + PLX + '))?') + + EXPONENT = r'[eE][+-]?\d+' + + # Lexer token definitions :: + + tokens = { + 'root': [ + (r'\s+', Text), + # keywords :: + (r'(?i)(base|prefix|start|external|' + r'literal|iri|bnode|nonliteral|length|minlength|maxlength|' + r'mininclusive|minexclusive|maxinclusive|maxexclusive|' + r'totaldigits|fractiondigits|' + r'closed|extra)\b', Keyword), + (r'(a)\b', Keyword), + # IRIs :: + ('(' + IRIREF + ')', Name.Label), + # blank nodes :: + ('(' + BLANK_NODE_LABEL + ')', Name.Label), + # prefixed names :: + (r'(' + PN_PREFIX + r')?(\:)(' + PN_LOCAL + ')?', + bygroups(Name.Namespace, Punctuation, Name.Tag)), + # boolean literals :: + (r'(true|false)', Keyword.Constant), + # double literals :: + (r'[+\-]?(\d+\.\d*' + EXPONENT + r'|\.?\d+' + EXPONENT + ')', Number.Float), + # decimal literals :: + (r'[+\-]?(\d+\.\d*|\.\d+)', Number.Float), + # integer literals :: + (r'[+\-]?\d+', Number.Integer), + # operators :: + (r'[@|$&=*+?^\-~]', Operator), + # operator keywords :: + (r'(?i)(and|or|not)\b', Operator.Word), + # punctuation characters :: + (r'[(){}.;,:^\[\]]', Punctuation), + # line comments :: + (r'#[^\n]*', Comment), + # strings :: + (r'"""', String, 'triple-double-quoted-string'), + (r'"', String, 'single-double-quoted-string'), + (r"'''", String, 'triple-single-quoted-string'), + (r"'", String, 'single-single-quoted-string'), + ], + 'triple-double-quoted-string': [ + (r'"""', String, 'end-of-string'), + (r'[^\\]+', String), + (r'\\', String, 'string-escape'), + ], + 'single-double-quoted-string': [ + (r'"', String, 'end-of-string'), + (r'[^"\\\n]+', String), + (r'\\', String, 'string-escape'), + ], + 'triple-single-quoted-string': [ + (r"'''", String, 'end-of-string'), + (r'[^\\]+', String), + (r'\\', String.Escape, 'string-escape'), + ], + 'single-single-quoted-string': [ + (r"'", String, 'end-of-string'), + (r"[^'\\\n]+", String), + (r'\\', String, 'string-escape'), + ], + 'string-escape': [ + (UCHAR_NO_BACKSLASH, String.Escape, '#pop'), + (r'.', String.Escape, '#pop'), + ], + 'end-of-string': [ + (r'(@)([a-zA-Z]+(?:-[a-zA-Z0-9]+)*)', + bygroups(Operator, Name.Function), '#pop:2'), + (r'\^\^', Operator, '#pop:2'), + default('#pop:2'), + ], + } diff --git a/pygments/lexers/rebol.py b/pygments/lexers/rebol.py new file mode 100644 index 0000000..7d363dd --- /dev/null +++ b/pygments/lexers/rebol.py @@ -0,0 +1,430 @@ +""" + pygments.lexers.rebol + ~~~~~~~~~~~~~~~~~~~~~ + + Lexers for the REBOL and related languages. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, bygroups +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Generic, Whitespace + +__all__ = ['RebolLexer', 'RedLexer'] + + +class RebolLexer(RegexLexer): + """ + A `REBOL `_ lexer. + + .. versionadded:: 1.1 + """ + name = 'REBOL' + aliases = ['rebol'] + filenames = ['*.r', '*.r3', '*.reb'] + mimetypes = ['text/x-rebol'] + + flags = re.IGNORECASE | re.MULTILINE + + escape_re = r'(?:\^\([0-9a-f]{1,4}\)*)' + + def word_callback(lexer, match): + word = match.group() + + if re.match(".*:$", word): + yield match.start(), Generic.Subheading, word + elif re.match( + r'(native|alias|all|any|as-string|as-binary|bind|bound\?|case|' + r'catch|checksum|comment|debase|dehex|exclude|difference|disarm|' + r'either|else|enbase|foreach|remove-each|form|free|get|get-env|if|' + r'in|intersect|loop|minimum-of|maximum-of|mold|new-line|' + r'new-line\?|not|now|prin|print|reduce|compose|construct|repeat|' + r'reverse|save|script\?|set|shift|switch|throw|to-hex|trace|try|' + r'type\?|union|unique|unless|unprotect|unset|until|use|value\?|' + r'while|compress|decompress|secure|open|close|read|read-io|' + r'write-io|write|update|query|wait|input\?|exp|log-10|log-2|' + r'log-e|square-root|cosine|sine|tangent|arccosine|arcsine|' + r'arctangent|protect|lowercase|uppercase|entab|detab|connected\?|' + r'browse|launch|stats|get-modes|set-modes|to-local-file|' + r'to-rebol-file|encloak|decloak|create-link|do-browser|bind\?|' + r'hide|draw|show|size-text|textinfo|offset-to-caret|' + r'caret-to-offset|local-request-file|rgb-to-hsv|hsv-to-rgb|' + r'crypt-strength\?|dh-make-key|dh-generate-key|dh-compute-key|' + r'dsa-make-key|dsa-generate-key|dsa-make-signature|' + r'dsa-verify-signature|rsa-make-key|rsa-generate-key|' + r'rsa-encrypt)$', word): + yield match.start(), Name.Builtin, word + elif re.match( + r'(add|subtract|multiply|divide|remainder|power|and~|or~|xor~|' + r'minimum|maximum|negate|complement|absolute|random|head|tail|' + r'next|back|skip|at|pick|first|second|third|fourth|fifth|sixth|' + r'seventh|eighth|ninth|tenth|last|path|find|select|make|to|copy\*|' + r'insert|remove|change|poke|clear|trim|sort|min|max|abs|cp|' + r'copy)$', word): + yield match.start(), Name.Function, word + elif re.match( + r'(error|source|input|license|help|install|echo|Usage|with|func|' + r'throw-on-error|function|does|has|context|probe|\?\?|as-pair|' + r'mod|modulo|round|repend|about|set-net|append|join|rejoin|reform|' + r'remold|charset|array|replace|move|extract|forskip|forall|alter|' + r'first+|also|take|for|forever|dispatch|attempt|what-dir|' + r'change-dir|clean-path|list-dir|dirize|rename|split-path|delete|' + r'make-dir|delete-dir|in-dir|confirm|dump-obj|upgrade|what|' + r'build-tag|process-source|build-markup|decode-cgi|read-cgi|' + r'write-user|save-user|set-user-name|protect-system|parse-xml|' + r'cvs-date|cvs-version|do-boot|get-net-info|desktop|layout|' + r'scroll-para|get-face|alert|set-face|uninstall|unfocus|' + r'request-dir|center-face|do-events|net-error|decode-url|' + r'parse-header|parse-header-date|parse-email-addrs|import-email|' + r'send|build-attach-body|resend|show-popup|hide-popup|open-events|' + r'find-key-face|do-face|viewtop|confine|find-window|' + r'insert-event-func|remove-event-func|inform|dump-pane|dump-face|' + r'flag-face|deflag-face|clear-fields|read-net|vbug|path-thru|' + r'read-thru|load-thru|do-thru|launch-thru|load-image|' + r'request-download|do-face-alt|set-font|set-para|get-style|' + r'set-style|make-face|stylize|choose|hilight-text|hilight-all|' + r'unlight-text|focus|scroll-drag|clear-face|reset-face|scroll-face|' + r'resize-face|load-stock|load-stock-block|notify|request|flash|' + r'request-color|request-pass|request-text|request-list|' + r'request-date|request-file|dbug|editor|link-relative-path|' + r'emailer|parse-error)$', word): + yield match.start(), Keyword.Namespace, word + elif re.match( + r'(halt|quit|do|load|q|recycle|call|run|ask|parse|view|unview|' + r'return|exit|break)$', word): + yield match.start(), Name.Exception, word + elif re.match('REBOL$', word): + yield match.start(), Generic.Heading, word + elif re.match("to-.*", word): + yield match.start(), Keyword, word + elif re.match(r'(\+|-|\*|/|//|\*\*|and|or|xor|=\?|=|==|<>|<|>|<=|>=)$', + word): + yield match.start(), Operator, word + elif re.match(r".*\?$", word): + yield match.start(), Keyword, word + elif re.match(r".*\!$", word): + yield match.start(), Keyword.Type, word + elif re.match("'.*", word): + yield match.start(), Name.Variable.Instance, word # lit-word + elif re.match("#.*", word): + yield match.start(), Name.Label, word # issue + elif re.match("%.*", word): + yield match.start(), Name.Decorator, word # file + else: + yield match.start(), Name.Variable, word + + tokens = { + 'root': [ + (r'[^R]+', Comment), + (r'REBOL\s+\[', Generic.Strong, 'script'), + (r'R', Comment) + ], + 'script': [ + (r'\s+', Text), + (r'#"', String.Char, 'char'), + (r'#\{[0-9a-f]*\}', Number.Hex), + (r'2#\{', Number.Hex, 'bin2'), + (r'64#\{[0-9a-z+/=\s]*\}', Number.Hex), + (r'"', String, 'string'), + (r'\{', String, 'string2'), + (r';#+.*\n', Comment.Special), + (r';\*+.*\n', Comment.Preproc), + (r';.*\n', Comment), + (r'%"', Name.Decorator, 'stringFile'), + (r'%[^(^{")\s\[\]]+', Name.Decorator), + (r'[+-]?([a-z]{1,3})?\$\d+(\.\d+)?', Number.Float), # money + (r'[+-]?\d+\:\d+(\:\d+)?(\.\d+)?', String.Other), # time + (r'\d+[\-/][0-9a-z]+[\-/]\d+(\/\d+\:\d+((\:\d+)?' + r'([.\d+]?([+-]?\d+:\d+)?)?)?)?', String.Other), # date + (r'\d+(\.\d+)+\.\d+', Keyword.Constant), # tuple + (r'\d+X\d+', Keyword.Constant), # pair + (r'[+-]?\d+(\'\d+)?([.,]\d*)?E[+-]?\d+', Number.Float), + (r'[+-]?\d+(\'\d+)?[.,]\d*', Number.Float), + (r'[+-]?\d+(\'\d+)?', Number), + (r'[\[\]()]', Generic.Strong), + (r'[a-z]+[^(^{"\s:)]*://[^(^{"\s)]*', Name.Decorator), # url + (r'mailto:[^(^{"@\s)]+@[^(^{"@\s)]+', Name.Decorator), # url + (r'[^(^{"@\s)]+@[^(^{"@\s)]+', Name.Decorator), # email + (r'comment\s"', Comment, 'commentString1'), + (r'comment\s\{', Comment, 'commentString2'), + (r'comment\s\[', Comment, 'commentBlock'), + (r'comment\s[^(\s{"\[]+', Comment), + (r'/[^(^{")\s/[\]]*', Name.Attribute), + (r'([^(^{")\s/[\]]+)(?=[:({"\s/\[\]])', word_callback), + (r'<[\w:.-]*>', Name.Tag), + (r'<[^(<>\s")]+', Name.Tag, 'tag'), + (r'([^(^{")\s]+)', Text), + ], + 'string': [ + (r'[^(^")]+', String), + (escape_re, String.Escape), + (r'[(|)]+', String), + (r'\^.', String.Escape), + (r'"', String, '#pop'), + ], + 'string2': [ + (r'[^(^{})]+', String), + (escape_re, String.Escape), + (r'[(|)]+', String), + (r'\^.', String.Escape), + (r'\{', String, '#push'), + (r'\}', String, '#pop'), + ], + 'stringFile': [ + (r'[^(^")]+', Name.Decorator), + (escape_re, Name.Decorator), + (r'\^.', Name.Decorator), + (r'"', Name.Decorator, '#pop'), + ], + 'char': [ + (escape_re + '"', String.Char, '#pop'), + (r'\^."', String.Char, '#pop'), + (r'."', String.Char, '#pop'), + ], + 'tag': [ + (escape_re, Name.Tag), + (r'"', Name.Tag, 'tagString'), + (r'[^(<>\r\n")]+', Name.Tag), + (r'>', Name.Tag, '#pop'), + ], + 'tagString': [ + (r'[^(^")]+', Name.Tag), + (escape_re, Name.Tag), + (r'[(|)]+', Name.Tag), + (r'\^.', Name.Tag), + (r'"', Name.Tag, '#pop'), + ], + 'tuple': [ + (r'(\d+\.)+', Keyword.Constant), + (r'\d+', Keyword.Constant, '#pop'), + ], + 'bin2': [ + (r'\s+', Number.Hex), + (r'([01]\s*){8}', Number.Hex), + (r'\}', Number.Hex, '#pop'), + ], + 'commentString1': [ + (r'[^(^")]+', Comment), + (escape_re, Comment), + (r'[(|)]+', Comment), + (r'\^.', Comment), + (r'"', Comment, '#pop'), + ], + 'commentString2': [ + (r'[^(^{})]+', Comment), + (escape_re, Comment), + (r'[(|)]+', Comment), + (r'\^.', Comment), + (r'\{', Comment, '#push'), + (r'\}', Comment, '#pop'), + ], + 'commentBlock': [ + (r'\[', Comment, '#push'), + (r'\]', Comment, '#pop'), + (r'"', Comment, "commentString1"), + (r'\{', Comment, "commentString2"), + (r'[^(\[\]"{)]+', Comment), + ], + } + + def analyse_text(text): + """ + Check if code contains REBOL header and so it probably not R code + """ + if re.match(r'^\s*REBOL\s*\[', text, re.IGNORECASE): + # The code starts with REBOL header + return 1.0 + elif re.search(r'\s*REBOL\s*\[', text, re.IGNORECASE): + # The code contains REBOL header but also some text before it + return 0.5 + + +class RedLexer(RegexLexer): + """ + A `Red-language `_ lexer. + + .. versionadded:: 2.0 + """ + name = 'Red' + aliases = ['red', 'red/system'] + filenames = ['*.red', '*.reds'] + mimetypes = ['text/x-red', 'text/x-red-system'] + + flags = re.IGNORECASE | re.MULTILINE + + escape_re = r'(?:\^\([0-9a-f]{1,4}\)*)' + + def word_callback(lexer, match): + word = match.group() + + if re.match(".*:$", word): + yield match.start(), Generic.Subheading, word + elif re.match(r'(if|unless|either|any|all|while|until|loop|repeat|' + r'foreach|forall|func|function|does|has|switch|' + r'case|reduce|compose|get|set|print|prin|equal\?|' + r'not-equal\?|strict-equal\?|lesser\?|greater\?|lesser-or-equal\?|' + r'greater-or-equal\?|same\?|not|type\?|stats|' + r'bind|union|replace|charset|routine)$', word): + yield match.start(), Name.Builtin, word + elif re.match(r'(make|random|reflect|to|form|mold|absolute|add|divide|multiply|negate|' + r'power|remainder|round|subtract|even\?|odd\?|and~|complement|or~|xor~|' + r'append|at|back|change|clear|copy|find|head|head\?|index\?|insert|' + r'length\?|next|pick|poke|remove|reverse|select|sort|skip|swap|tail|tail\?|' + r'take|trim|create|close|delete|modify|open|open\?|query|read|rename|' + r'update|write)$', word): + yield match.start(), Name.Function, word + elif re.match(r'(yes|on|no|off|true|false|tab|cr|lf|newline|escape|slash|sp|space|null|' + r'none|crlf|dot|null-byte)$', word): + yield match.start(), Name.Builtin.Pseudo, word + elif re.match(r'(#system-global|#include|#enum|#define|#either|#if|#import|#export|' + r'#switch|#default|#get-definition)$', word): + yield match.start(), Keyword.Namespace, word + elif re.match(r'(system|halt|quit|quit-return|do|load|q|recycle|call|run|ask|parse|' + r'raise-error|return|exit|break|alias|push|pop|probe|\?\?|spec-of|body-of|' + r'quote|forever)$', word): + yield match.start(), Name.Exception, word + elif re.match(r'(action\?|block\?|char\?|datatype\?|file\?|function\?|get-path\?|zero\?|' + r'get-word\?|integer\?|issue\?|lit-path\?|lit-word\?|logic\?|native\?|' + r'op\?|paren\?|path\?|refinement\?|set-path\?|set-word\?|string\?|unset\?|' + r'any-struct\?|none\?|word\?|any-series\?)$', word): + yield match.start(), Keyword, word + elif re.match(r'(JNICALL|stdcall|cdecl|infix)$', word): + yield match.start(), Keyword.Namespace, word + elif re.match("to-.*", word): + yield match.start(), Keyword, word + elif re.match(r'(\+|-\*\*|-|\*\*|//|/|\*|and|or|xor|=\?|===|==|=|<>|<=|>=|' + r'<<<|>>>|<<|>>|<|>%)$', word): + yield match.start(), Operator, word + elif re.match(r".*\!$", word): + yield match.start(), Keyword.Type, word + elif re.match("'.*", word): + yield match.start(), Name.Variable.Instance, word # lit-word + elif re.match("#.*", word): + yield match.start(), Name.Label, word # issue + elif re.match("%.*", word): + yield match.start(), Name.Decorator, word # file + elif re.match(":.*", word): + yield match.start(), Generic.Subheading, word # get-word + else: + yield match.start(), Name.Variable, word + + tokens = { + 'root': [ + (r'[^R]+', Comment), + (r'Red/System\s+\[', Generic.Strong, 'script'), + (r'Red\s+\[', Generic.Strong, 'script'), + (r'R', Comment) + ], + 'script': [ + (r'\s+', Text), + (r'#"', String.Char, 'char'), + (r'#\{[0-9a-f\s]*\}', Number.Hex), + (r'2#\{', Number.Hex, 'bin2'), + (r'64#\{[0-9a-z+/=\s]*\}', Number.Hex), + (r'([0-9a-f]+)(h)((\s)|(?=[\[\]{}"()]))', + bygroups(Number.Hex, Name.Variable, Whitespace)), + (r'"', String, 'string'), + (r'\{', String, 'string2'), + (r';#+.*\n', Comment.Special), + (r';\*+.*\n', Comment.Preproc), + (r';.*\n', Comment), + (r'%"', Name.Decorator, 'stringFile'), + (r'%[^(^{")\s\[\]]+', Name.Decorator), + (r'[+-]?([a-z]{1,3})?\$\d+(\.\d+)?', Number.Float), # money + (r'[+-]?\d+\:\d+(\:\d+)?(\.\d+)?', String.Other), # time + (r'\d+[\-/][0-9a-z]+[\-/]\d+(/\d+:\d+((:\d+)?' + r'([\.\d+]?([+-]?\d+:\d+)?)?)?)?', String.Other), # date + (r'\d+(\.\d+)+\.\d+', Keyword.Constant), # tuple + (r'\d+X\d+', Keyword.Constant), # pair + (r'[+-]?\d+(\'\d+)?([.,]\d*)?E[+-]?\d+', Number.Float), + (r'[+-]?\d+(\'\d+)?[.,]\d*', Number.Float), + (r'[+-]?\d+(\'\d+)?', Number), + (r'[\[\]()]', Generic.Strong), + (r'[a-z]+[^(^{"\s:)]*://[^(^{"\s)]*', Name.Decorator), # url + (r'mailto:[^(^{"@\s)]+@[^(^{"@\s)]+', Name.Decorator), # url + (r'[^(^{"@\s)]+@[^(^{"@\s)]+', Name.Decorator), # email + (r'comment\s"', Comment, 'commentString1'), + (r'comment\s\{', Comment, 'commentString2'), + (r'comment\s\[', Comment, 'commentBlock'), + (r'comment\s[^(\s{"\[]+', Comment), + (r'/[^(^{^")\s/[\]]*', Name.Attribute), + (r'([^(^{^")\s/[\]]+)(?=[:({"\s/\[\]])', word_callback), + (r'<[\w:.-]*>', Name.Tag), + (r'<[^(<>\s")]+', Name.Tag, 'tag'), + (r'([^(^{")\s]+)', Text), + ], + 'string': [ + (r'[^(^")]+', String), + (escape_re, String.Escape), + (r'[(|)]+', String), + (r'\^.', String.Escape), + (r'"', String, '#pop'), + ], + 'string2': [ + (r'[^(^{})]+', String), + (escape_re, String.Escape), + (r'[(|)]+', String), + (r'\^.', String.Escape), + (r'\{', String, '#push'), + (r'\}', String, '#pop'), + ], + 'stringFile': [ + (r'[^(^")]+', Name.Decorator), + (escape_re, Name.Decorator), + (r'\^.', Name.Decorator), + (r'"', Name.Decorator, '#pop'), + ], + 'char': [ + (escape_re + '"', String.Char, '#pop'), + (r'\^."', String.Char, '#pop'), + (r'."', String.Char, '#pop'), + ], + 'tag': [ + (escape_re, Name.Tag), + (r'"', Name.Tag, 'tagString'), + (r'[^(<>\r\n")]+', Name.Tag), + (r'>', Name.Tag, '#pop'), + ], + 'tagString': [ + (r'[^(^")]+', Name.Tag), + (escape_re, Name.Tag), + (r'[(|)]+', Name.Tag), + (r'\^.', Name.Tag), + (r'"', Name.Tag, '#pop'), + ], + 'tuple': [ + (r'(\d+\.)+', Keyword.Constant), + (r'\d+', Keyword.Constant, '#pop'), + ], + 'bin2': [ + (r'\s+', Number.Hex), + (r'([01]\s*){8}', Number.Hex), + (r'\}', Number.Hex, '#pop'), + ], + 'commentString1': [ + (r'[^(^")]+', Comment), + (escape_re, Comment), + (r'[(|)]+', Comment), + (r'\^.', Comment), + (r'"', Comment, '#pop'), + ], + 'commentString2': [ + (r'[^(^{})]+', Comment), + (escape_re, Comment), + (r'[(|)]+', Comment), + (r'\^.', Comment), + (r'\{', Comment, '#push'), + (r'\}', Comment, '#pop'), + ], + 'commentBlock': [ + (r'\[', Comment, '#push'), + (r'\]', Comment, '#pop'), + (r'"', Comment, "commentString1"), + (r'\{', Comment, "commentString2"), + (r'[^(\[\]"{)]+', Comment), + ], + } diff --git a/pygments/lexers/resource.py b/pygments/lexers/resource.py new file mode 100644 index 0000000..2e6b2dd --- /dev/null +++ b/pygments/lexers/resource.py @@ -0,0 +1,84 @@ +""" + pygments.lexers.resource + ~~~~~~~~~~~~~~~~~~~~~~~~ + + Lexer for resource definition files. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, bygroups, words +from pygments.token import Comment, String, Number, Operator, Text, \ + Keyword, Name + +__all__ = ['ResourceLexer'] + + +class ResourceLexer(RegexLexer): + """Lexer for `ICU Resource bundles + `_. + + .. versionadded:: 2.0 + """ + name = 'ResourceBundle' + aliases = ['resourcebundle', 'resource'] + filenames = [] + + _types = (':table', ':array', ':string', ':bin', ':import', ':intvector', + ':int', ':alias') + + flags = re.MULTILINE | re.IGNORECASE + tokens = { + 'root': [ + (r'//.*?$', Comment), + (r'"', String, 'string'), + (r'-?\d+', Number.Integer), + (r'[,{}]', Operator), + (r'([^\s{:]+)(\s*)(%s?)' % '|'.join(_types), + bygroups(Name, Text, Keyword)), + (r'\s+', Text), + (words(_types), Keyword), + ], + 'string': [ + (r'(\\x[0-9a-f]{2}|\\u[0-9a-f]{4}|\\U00[0-9a-f]{6}|' + r'\\[0-7]{1,3}|\\c.|\\[abtnvfre\'"?\\]|\\\{|[^"{\\])+', String), + (r'\{', String.Escape, 'msgname'), + (r'"', String, '#pop') + ], + 'msgname': [ + (r'([^{},]+)(\s*)', bygroups(Name, String.Escape), ('#pop', 'message')) + ], + 'message': [ + (r'\{', String.Escape, 'msgname'), + (r'\}', String.Escape, '#pop'), + (r'(,)(\s*)([a-z]+)(\s*\})', + bygroups(Operator, String.Escape, Keyword, String.Escape), '#pop'), + (r'(,)(\s*)([a-z]+)(\s*)(,)(\s*)(offset)(\s*)(:)(\s*)(-?\d+)(\s*)', + bygroups(Operator, String.Escape, Keyword, String.Escape, Operator, + String.Escape, Operator.Word, String.Escape, Operator, + String.Escape, Number.Integer, String.Escape), 'choice'), + (r'(,)(\s*)([a-z]+)(\s*)(,)(\s*)', + bygroups(Operator, String.Escape, Keyword, String.Escape, Operator, + String.Escape), 'choice'), + (r'\s+', String.Escape) + ], + 'choice': [ + (r'(=|<|>|<=|>=|!=)(-?\d+)(\s*\{)', + bygroups(Operator, Number.Integer, String.Escape), 'message'), + (r'([a-z]+)(\s*\{)', bygroups(Keyword.Type, String.Escape), 'str'), + (r'\}', String.Escape, ('#pop', '#pop')), + (r'\s+', String.Escape) + ], + 'str': [ + (r'\}', String.Escape, '#pop'), + (r'\{', String.Escape, 'msgname'), + (r'[^{}]+', String) + ] + } + + def analyse_text(text): + if text.startswith('root:table'): + return 1.0 diff --git a/pygments/lexers/ride.py b/pygments/lexers/ride.py new file mode 100644 index 0000000..e68c396 --- /dev/null +++ b/pygments/lexers/ride.py @@ -0,0 +1,139 @@ +""" + pygments.lexers.ride + ~~~~~~~~~~~~~~~~~~~~ + + Lexer for the Ride programming language. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, words, include +from pygments.token import Comment, Keyword, Name, Number, Punctuation, \ + String, Text + +__all__ = ['RideLexer'] + + +class RideLexer(RegexLexer): + """ + For `Ride `_ + source code. + + .. versionadded:: 2.6 + """ + + name = 'Ride' + aliases = ['ride'] + filenames = ['*.ride'] + mimetypes = ['text/x-ride'] + + validName = r'[a-zA-Z_][a-zA-Z0-9_\']*' + + builtinOps = ( + '||', '|', '>=', '>', '==', '!', + '=', '<=', '<', '::', ':+', ':', '!=', '/', + '.', '=>', '-', '+', '*', '&&', '%', '++', + ) + + globalVariablesName = ( + 'NOALG', 'MD5', 'SHA1', 'SHA224', 'SHA256', 'SHA384', 'SHA512', + 'SHA3224', 'SHA3256', 'SHA3384', 'SHA3512', 'nil', 'this', 'unit', + 'height', 'lastBlock', 'Buy', 'Sell', 'CEILING', 'FLOOR', 'DOWN', + 'HALFDOWN', 'HALFEVEN', 'HALFUP', 'UP', + ) + + typesName = ( + 'Unit', 'Int', 'Boolean', 'ByteVector', 'String', 'Address', 'Alias', + 'Transfer', 'AssetPair', 'DataEntry', 'Order', 'Transaction', + 'GenesisTransaction', 'PaymentTransaction', 'ReissueTransaction', + 'BurnTransaction', 'MassTransferTransaction', 'ExchangeTransaction', + 'TransferTransaction', 'SetAssetScriptTransaction', + 'InvokeScriptTransaction', 'IssueTransaction', 'LeaseTransaction', + 'LeaseCancelTransaction', 'CreateAliasTransaction', + 'SetScriptTransaction', 'SponsorFeeTransaction', 'DataTransaction', + 'WriteSet', 'AttachedPayment', 'ScriptTransfer', 'TransferSet', + 'ScriptResult', 'Invocation', 'Asset', 'BlockInfo', 'Issue', 'Reissue', + 'Burn', 'NoAlg', 'Md5', 'Sha1', 'Sha224', 'Sha256', 'Sha384', 'Sha512', + 'Sha3224', 'Sha3256', 'Sha3384', 'Sha3512', 'BinaryEntry', + 'BooleanEntry', 'IntegerEntry', 'StringEntry', 'List', 'Ceiling', + 'Down', 'Floor', 'HalfDown', 'HalfEven', 'HalfUp', 'Up', + ) + + functionsName = ( + 'fraction', 'size', 'toBytes', 'take', 'drop', 'takeRight', 'dropRight', + 'toString', 'isDefined', 'extract', 'throw', 'getElement', 'value', + 'cons', 'toUtf8String', 'toInt', 'indexOf', 'lastIndexOf', 'split', + 'parseInt', 'parseIntValue', 'keccak256', 'blake2b256', 'sha256', + 'sigVerify', 'toBase58String', 'fromBase58String', 'toBase64String', + 'fromBase64String', 'transactionById', 'transactionHeightById', + 'getInteger', 'getBoolean', 'getBinary', 'getString', + 'addressFromPublicKey', 'addressFromString', 'addressFromRecipient', + 'assetBalance', 'wavesBalance', 'getIntegerValue', 'getBooleanValue', + 'getBinaryValue', 'getStringValue', 'addressFromStringValue', + 'assetInfo', 'rsaVerify', 'checkMerkleProof', 'median', + 'valueOrElse', 'valueOrErrorMessage', 'contains', 'log', 'pow', + 'toBase16String', 'fromBase16String', 'blockInfoByHeight', + 'transferTransactionById', + ) + + reservedWords = words(( + 'match', 'case', 'else', 'func', 'if', + 'let', 'then', '@Callable', '@Verifier', + ), suffix=r'\b') + + tokens = { + 'root': [ + # Comments + (r'#.*', Comment.Single), + # Whitespace + (r'\s+', Text), + # Strings + (r'"', String, 'doublequote'), + (r'utf8\'', String, 'utf8quote'), + (r'base(58|64|16)\'', String, 'singlequote'), + # Keywords + (reservedWords, Keyword.Reserved), + (r'\{-#.*?#-\}', Keyword.Reserved), + (r'FOLD<\d+>', Keyword.Reserved), + # Types + (words(typesName), Keyword.Type), + # Main + # (specialName, Keyword.Reserved), + # Prefix Operators + (words(builtinOps, prefix=r'\(', suffix=r'\)'), Name.Function), + # Infix Operators + (words(builtinOps), Name.Function), + (words(globalVariablesName), Name.Function), + (words(functionsName), Name.Function), + # Numbers + include('numbers'), + # Variable Names + (validName, Name.Variable), + # Parens + (r'[,()\[\]{}]', Punctuation), + ], + + 'doublequote': [ + (r'\\u[0-9a-fA-F]{4}', String.Escape), + (r'\\[nrfvb\\"]', String.Escape), + (r'[^"]', String), + (r'"', String, '#pop'), + ], + + 'utf8quote': [ + (r'\\u[0-9a-fA-F]{4}', String.Escape), + (r'\\[nrfvb\\\']', String.Escape), + (r'[^\']', String), + (r'\'', String, '#pop'), + ], + + 'singlequote': [ + (r'[^\']', String), + (r'\'', String, '#pop'), + ], + + 'numbers': [ + (r'_?\d+', Number.Integer), + ], + } diff --git a/pygments/lexers/rita.py b/pygments/lexers/rita.py new file mode 100644 index 0000000..ff742e9 --- /dev/null +++ b/pygments/lexers/rita.py @@ -0,0 +1,43 @@ +""" + pygments.lexers.rita + ~~~~~~~~~~~~~~~~~~~~ + + Lexers for RITA language + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer +from pygments.token import Comment, Operator, Keyword, Name, Literal, \ + Punctuation, Whitespace + +__all__ = ['RitaLexer'] + + +class RitaLexer(RegexLexer): + """ + Lexer for RITA. + + .. versionadded:: 2.11 + """ + name = 'Rita' + url = 'https://github.com/zaibacu/rita-dsl' + filenames = ['*.rita'] + aliases = ['rita'] + mimetypes = ['text/rita'] + + tokens = { + 'root': [ + (r'\n', Whitespace), + (r'\s+', Whitespace), + (r'#(.*?)\n', Comment.Single), + (r'@(.*?)\n', Operator), # Yes, whole line as an operator + (r'"(\w|\d|\s|(\\")|[\'_\-./,\?\!])+?"', Literal), + (r'\'(\w|\d|\s|(\\\')|["_\-./,\?\!])+?\'', Literal), + (r'([A-Z_]+)', Keyword), + (r'([a-z0-9_]+)', Name), + (r'((->)|[!?+*|=])', Operator), + (r'[\(\),\{\}]', Punctuation) + ] + } diff --git a/pygments/lexers/rnc.py b/pygments/lexers/rnc.py new file mode 100644 index 0000000..fc1c9a8 --- /dev/null +++ b/pygments/lexers/rnc.py @@ -0,0 +1,67 @@ +""" + pygments.lexers.rnc + ~~~~~~~~~~~~~~~~~~~ + + Lexer for Relax-NG Compact syntax + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Punctuation + +__all__ = ['RNCCompactLexer'] + + +class RNCCompactLexer(RegexLexer): + """ + For RelaxNG-compact syntax. + + .. versionadded:: 2.2 + """ + + name = 'Relax-NG Compact' + url = 'http://relaxng.org' + aliases = ['rng-compact', 'rnc'] + filenames = ['*.rnc'] + + tokens = { + 'root': [ + (r'namespace\b', Keyword.Namespace), + (r'(?:default|datatypes)\b', Keyword.Declaration), + (r'##.*$', Comment.Preproc), + (r'#.*$', Comment.Single), + (r'"[^"]*"', String.Double), + # TODO single quoted strings and escape sequences outside of + # double-quoted strings + (r'(?:element|attribute|mixed)\b', Keyword.Declaration, 'variable'), + (r'(text\b|xsd:[^ ]+)', Keyword.Type, 'maybe_xsdattributes'), + (r'[,?&*=|~]|>>', Operator), + (r'[(){}]', Punctuation), + (r'.', Text), + ], + + # a variable has been declared using `element` or `attribute` + 'variable': [ + (r'[^{]+', Name.Variable), + (r'\{', Punctuation, '#pop'), + ], + + # after an xsd: declaration there may be attributes + 'maybe_xsdattributes': [ + (r'\{', Punctuation, 'xsdattributes'), + (r'\}', Punctuation, '#pop'), + (r'.', Text), + ], + + # attributes take the form { key1 = value1 key2 = value2 ... } + 'xsdattributes': [ + (r'[^ =}]', Name.Attribute), + (r'=', Operator), + (r'"[^"]*"', String.Double), + (r'\}', Punctuation, '#pop'), + (r'.', Text), + ], + } diff --git a/pygments/lexers/roboconf.py b/pygments/lexers/roboconf.py new file mode 100644 index 0000000..2d8e5ff --- /dev/null +++ b/pygments/lexers/roboconf.py @@ -0,0 +1,81 @@ +""" + pygments.lexers.roboconf + ~~~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for Roboconf DSL. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, words, re +from pygments.token import Text, Operator, Keyword, Name, Comment + +__all__ = ['RoboconfGraphLexer', 'RoboconfInstancesLexer'] + + +class RoboconfGraphLexer(RegexLexer): + """ + Lexer for Roboconf graph files. + + .. versionadded:: 2.1 + """ + name = 'Roboconf Graph' + aliases = ['roboconf-graph'] + filenames = ['*.graph'] + + flags = re.IGNORECASE | re.MULTILINE + tokens = { + 'root': [ + # Skip white spaces + (r'\s+', Text), + + # There is one operator + (r'=', Operator), + + # Keywords + (words(('facet', 'import'), suffix=r'\s*\b', prefix=r'\b'), Keyword), + (words(( + 'installer', 'extends', 'exports', 'imports', 'facets', + 'children'), suffix=r'\s*:?', prefix=r'\b'), Name), + + # Comments + (r'#.*\n', Comment), + + # Default + (r'[^#]', Text), + (r'.*\n', Text) + ] + } + + +class RoboconfInstancesLexer(RegexLexer): + """ + Lexer for Roboconf instances files. + + .. versionadded:: 2.1 + """ + name = 'Roboconf Instances' + aliases = ['roboconf-instances'] + filenames = ['*.instances'] + + flags = re.IGNORECASE | re.MULTILINE + tokens = { + 'root': [ + + # Skip white spaces + (r'\s+', Text), + + # Keywords + (words(('instance of', 'import'), suffix=r'\s*\b', prefix=r'\b'), Keyword), + (words(('name', 'count'), suffix=r's*:?', prefix=r'\b'), Name), + (r'\s*[\w.-]+\s*:', Name), + + # Comments + (r'#.*\n', Comment), + + # Default + (r'[^#]', Text), + (r'.*\n', Text) + ] + } diff --git a/pygments/lexers/robotframework.py b/pygments/lexers/robotframework.py new file mode 100644 index 0000000..91794d0 --- /dev/null +++ b/pygments/lexers/robotframework.py @@ -0,0 +1,552 @@ +""" + pygments.lexers.robotframework + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Lexer for Robot Framework. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +# Copyright 2012 Nokia Siemens Networks Oyj +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +from pygments.lexer import Lexer +from pygments.token import Token + +__all__ = ['RobotFrameworkLexer'] + + +HEADING = Token.Generic.Heading +SETTING = Token.Keyword.Namespace +IMPORT = Token.Name.Namespace +TC_KW_NAME = Token.Generic.Subheading +KEYWORD = Token.Name.Function +ARGUMENT = Token.String +VARIABLE = Token.Name.Variable +COMMENT = Token.Comment +SEPARATOR = Token.Punctuation +SYNTAX = Token.Punctuation +GHERKIN = Token.Generic.Emph +ERROR = Token.Error + + +def normalize(string, remove=''): + string = string.lower() + for char in remove + ' ': + if char in string: + string = string.replace(char, '') + return string + + +class RobotFrameworkLexer(Lexer): + """ + For Robot Framework test data. + + Supports both space and pipe separated plain text formats. + + .. versionadded:: 1.6 + """ + name = 'RobotFramework' + url = 'http://robotframework.org' + aliases = ['robotframework'] + filenames = ['*.robot', '*.resource'] + mimetypes = ['text/x-robotframework'] + + def __init__(self, **options): + options['tabsize'] = 2 + options['encoding'] = 'UTF-8' + Lexer.__init__(self, **options) + + def get_tokens_unprocessed(self, text): + row_tokenizer = RowTokenizer() + var_tokenizer = VariableTokenizer() + index = 0 + for row in text.splitlines(): + for value, token in row_tokenizer.tokenize(row): + for value, token in var_tokenizer.tokenize(value, token): + if value: + yield index, token, str(value) + index += len(value) + + +class VariableTokenizer: + + def tokenize(self, string, token): + var = VariableSplitter(string, identifiers='$@%&') + if var.start < 0 or token in (COMMENT, ERROR): + yield string, token + return + for value, token in self._tokenize(var, string, token): + if value: + yield value, token + + def _tokenize(self, var, string, orig_token): + before = string[:var.start] + yield before, orig_token + yield var.identifier + '{', SYNTAX + yield from self.tokenize(var.base, VARIABLE) + yield '}', SYNTAX + if var.index is not None: + yield '[', SYNTAX + yield from self.tokenize(var.index, VARIABLE) + yield ']', SYNTAX + yield from self.tokenize(string[var.end:], orig_token) + + +class RowTokenizer: + + def __init__(self): + self._table = UnknownTable() + self._splitter = RowSplitter() + testcases = TestCaseTable() + settings = SettingTable(testcases.set_default_template) + variables = VariableTable() + keywords = KeywordTable() + self._tables = {'settings': settings, 'setting': settings, + 'metadata': settings, + 'variables': variables, 'variable': variables, + 'testcases': testcases, 'testcase': testcases, + 'tasks': testcases, 'task': testcases, + 'keywords': keywords, 'keyword': keywords, + 'userkeywords': keywords, 'userkeyword': keywords} + + def tokenize(self, row): + commented = False + heading = False + for index, value in enumerate(self._splitter.split(row)): + # First value, and every second after that, is a separator. + index, separator = divmod(index-1, 2) + if value.startswith('#'): + commented = True + elif index == 0 and value.startswith('*'): + self._table = self._start_table(value) + heading = True + yield from self._tokenize(value, index, commented, + separator, heading) + self._table.end_row() + + def _start_table(self, header): + name = normalize(header, remove='*') + return self._tables.get(name, UnknownTable()) + + def _tokenize(self, value, index, commented, separator, heading): + if commented: + yield value, COMMENT + elif separator: + yield value, SEPARATOR + elif heading: + yield value, HEADING + else: + yield from self._table.tokenize(value, index) + + +class RowSplitter: + _space_splitter = re.compile('( {2,})') + _pipe_splitter = re.compile(r'((?:^| +)\|(?: +|$))') + + def split(self, row): + splitter = (row.startswith('| ') and self._split_from_pipes + or self._split_from_spaces) + yield from splitter(row) + yield '\n' + + def _split_from_spaces(self, row): + yield '' # Start with (pseudo)separator similarly as with pipes + yield from self._space_splitter.split(row) + + def _split_from_pipes(self, row): + _, separator, rest = self._pipe_splitter.split(row, 1) + yield separator + while self._pipe_splitter.search(rest): + cell, separator, rest = self._pipe_splitter.split(rest, 1) + yield cell + yield separator + yield rest + + +class Tokenizer: + _tokens = None + + def __init__(self): + self._index = 0 + + def tokenize(self, value): + values_and_tokens = self._tokenize(value, self._index) + self._index += 1 + if isinstance(values_and_tokens, type(Token)): + values_and_tokens = [(value, values_and_tokens)] + return values_and_tokens + + def _tokenize(self, value, index): + index = min(index, len(self._tokens) - 1) + return self._tokens[index] + + def _is_assign(self, value): + if value.endswith('='): + value = value[:-1].strip() + var = VariableSplitter(value, identifiers='$@&') + return var.start == 0 and var.end == len(value) + + +class Comment(Tokenizer): + _tokens = (COMMENT,) + + +class Setting(Tokenizer): + _tokens = (SETTING, ARGUMENT) + _keyword_settings = ('suitesetup', 'suiteprecondition', 'suiteteardown', + 'suitepostcondition', 'testsetup', 'tasksetup', 'testprecondition', + 'testteardown','taskteardown', 'testpostcondition', 'testtemplate', 'tasktemplate') + _import_settings = ('library', 'resource', 'variables') + _other_settings = ('documentation', 'metadata', 'forcetags', 'defaulttags', + 'testtimeout','tasktimeout') + _custom_tokenizer = None + + def __init__(self, template_setter=None): + Tokenizer.__init__(self) + self._template_setter = template_setter + + def _tokenize(self, value, index): + if index == 1 and self._template_setter: + self._template_setter(value) + if index == 0: + normalized = normalize(value) + if normalized in self._keyword_settings: + self._custom_tokenizer = KeywordCall(support_assign=False) + elif normalized in self._import_settings: + self._custom_tokenizer = ImportSetting() + elif normalized not in self._other_settings: + return ERROR + elif self._custom_tokenizer: + return self._custom_tokenizer.tokenize(value) + return Tokenizer._tokenize(self, value, index) + + +class ImportSetting(Tokenizer): + _tokens = (IMPORT, ARGUMENT) + + +class TestCaseSetting(Setting): + _keyword_settings = ('setup', 'precondition', 'teardown', 'postcondition', + 'template') + _import_settings = () + _other_settings = ('documentation', 'tags', 'timeout') + + def _tokenize(self, value, index): + if index == 0: + type = Setting._tokenize(self, value[1:-1], index) + return [('[', SYNTAX), (value[1:-1], type), (']', SYNTAX)] + return Setting._tokenize(self, value, index) + + +class KeywordSetting(TestCaseSetting): + _keyword_settings = ('teardown',) + _other_settings = ('documentation', 'arguments', 'return', 'timeout', 'tags') + + +class Variable(Tokenizer): + _tokens = (SYNTAX, ARGUMENT) + + def _tokenize(self, value, index): + if index == 0 and not self._is_assign(value): + return ERROR + return Tokenizer._tokenize(self, value, index) + + +class KeywordCall(Tokenizer): + _tokens = (KEYWORD, ARGUMENT) + + def __init__(self, support_assign=True): + Tokenizer.__init__(self) + self._keyword_found = not support_assign + self._assigns = 0 + + def _tokenize(self, value, index): + if not self._keyword_found and self._is_assign(value): + self._assigns += 1 + return SYNTAX # VariableTokenizer tokenizes this later. + if self._keyword_found: + return Tokenizer._tokenize(self, value, index - self._assigns) + self._keyword_found = True + return GherkinTokenizer().tokenize(value, KEYWORD) + + +class GherkinTokenizer: + _gherkin_prefix = re.compile('^(Given|When|Then|And|But) ', re.IGNORECASE) + + def tokenize(self, value, token): + match = self._gherkin_prefix.match(value) + if not match: + return [(value, token)] + end = match.end() + return [(value[:end], GHERKIN), (value[end:], token)] + + +class TemplatedKeywordCall(Tokenizer): + _tokens = (ARGUMENT,) + + +class ForLoop(Tokenizer): + + def __init__(self): + Tokenizer.__init__(self) + self._in_arguments = False + + def _tokenize(self, value, index): + token = self._in_arguments and ARGUMENT or SYNTAX + if value.upper() in ('IN', 'IN RANGE'): + self._in_arguments = True + return token + + +class _Table: + _tokenizer_class = None + + def __init__(self, prev_tokenizer=None): + self._tokenizer = self._tokenizer_class() + self._prev_tokenizer = prev_tokenizer + self._prev_values_on_row = [] + + def tokenize(self, value, index): + if self._continues(value, index): + self._tokenizer = self._prev_tokenizer + yield value, SYNTAX + else: + yield from self._tokenize(value, index) + self._prev_values_on_row.append(value) + + def _continues(self, value, index): + return value == '...' and all(self._is_empty(t) + for t in self._prev_values_on_row) + + def _is_empty(self, value): + return value in ('', '\\') + + def _tokenize(self, value, index): + return self._tokenizer.tokenize(value) + + def end_row(self): + self.__init__(prev_tokenizer=self._tokenizer) + + +class UnknownTable(_Table): + _tokenizer_class = Comment + + def _continues(self, value, index): + return False + + +class VariableTable(_Table): + _tokenizer_class = Variable + + +class SettingTable(_Table): + _tokenizer_class = Setting + + def __init__(self, template_setter, prev_tokenizer=None): + _Table.__init__(self, prev_tokenizer) + self._template_setter = template_setter + + def _tokenize(self, value, index): + if index == 0 and normalize(value) == 'testtemplate': + self._tokenizer = Setting(self._template_setter) + return _Table._tokenize(self, value, index) + + def end_row(self): + self.__init__(self._template_setter, prev_tokenizer=self._tokenizer) + + +class TestCaseTable(_Table): + _setting_class = TestCaseSetting + _test_template = None + _default_template = None + + @property + def _tokenizer_class(self): + if self._test_template or (self._default_template and + self._test_template is not False): + return TemplatedKeywordCall + return KeywordCall + + def _continues(self, value, index): + return index > 0 and _Table._continues(self, value, index) + + def _tokenize(self, value, index): + if index == 0: + if value: + self._test_template = None + return GherkinTokenizer().tokenize(value, TC_KW_NAME) + if index == 1 and self._is_setting(value): + if self._is_template(value): + self._test_template = False + self._tokenizer = self._setting_class(self.set_test_template) + else: + self._tokenizer = self._setting_class() + if index == 1 and self._is_for_loop(value): + self._tokenizer = ForLoop() + if index == 1 and self._is_empty(value): + return [(value, SYNTAX)] + return _Table._tokenize(self, value, index) + + def _is_setting(self, value): + return value.startswith('[') and value.endswith(']') + + def _is_template(self, value): + return normalize(value) == '[template]' + + def _is_for_loop(self, value): + return value.startswith(':') and normalize(value, remove=':') == 'for' + + def set_test_template(self, template): + self._test_template = self._is_template_set(template) + + def set_default_template(self, template): + self._default_template = self._is_template_set(template) + + def _is_template_set(self, template): + return normalize(template) not in ('', '\\', 'none', '${empty}') + + +class KeywordTable(TestCaseTable): + _tokenizer_class = KeywordCall + _setting_class = KeywordSetting + + def _is_template(self, value): + return False + + +# Following code copied directly from Robot Framework 2.7.5. + +class VariableSplitter: + + def __init__(self, string, identifiers): + self.identifier = None + self.base = None + self.index = None + self.start = -1 + self.end = -1 + self._identifiers = identifiers + self._may_have_internal_variables = False + try: + self._split(string) + except ValueError: + pass + else: + self._finalize() + + def get_replaced_base(self, variables): + if self._may_have_internal_variables: + return variables.replace_string(self.base) + return self.base + + def _finalize(self): + self.identifier = self._variable_chars[0] + self.base = ''.join(self._variable_chars[2:-1]) + self.end = self.start + len(self._variable_chars) + if self._has_list_or_dict_variable_index(): + self.index = ''.join(self._list_and_dict_variable_index_chars[1:-1]) + self.end += len(self._list_and_dict_variable_index_chars) + + def _has_list_or_dict_variable_index(self): + return self._list_and_dict_variable_index_chars\ + and self._list_and_dict_variable_index_chars[-1] == ']' + + def _split(self, string): + start_index, max_index = self._find_variable(string) + self.start = start_index + self._open_curly = 1 + self._state = self._variable_state + self._variable_chars = [string[start_index], '{'] + self._list_and_dict_variable_index_chars = [] + self._string = string + start_index += 2 + for index, char in enumerate(string[start_index:]): + index += start_index # Giving start to enumerate only in Py 2.6+ + try: + self._state(char, index) + except StopIteration: + return + if index == max_index and not self._scanning_list_variable_index(): + return + + def _scanning_list_variable_index(self): + return self._state in [self._waiting_list_variable_index_state, + self._list_variable_index_state] + + def _find_variable(self, string): + max_end_index = string.rfind('}') + if max_end_index == -1: + raise ValueError('No variable end found') + if self._is_escaped(string, max_end_index): + return self._find_variable(string[:max_end_index]) + start_index = self._find_start_index(string, 1, max_end_index) + if start_index == -1: + raise ValueError('No variable start found') + return start_index, max_end_index + + def _find_start_index(self, string, start, end): + index = string.find('{', start, end) - 1 + if index < 0: + return -1 + if self._start_index_is_ok(string, index): + return index + return self._find_start_index(string, index+2, end) + + def _start_index_is_ok(self, string, index): + return string[index] in self._identifiers\ + and not self._is_escaped(string, index) + + def _is_escaped(self, string, index): + escaped = False + while index > 0 and string[index-1] == '\\': + index -= 1 + escaped = not escaped + return escaped + + def _variable_state(self, char, index): + self._variable_chars.append(char) + if char == '}' and not self._is_escaped(self._string, index): + self._open_curly -= 1 + if self._open_curly == 0: + if not self._is_list_or_dict_variable(): + raise StopIteration + self._state = self._waiting_list_variable_index_state + elif char in self._identifiers: + self._state = self._internal_variable_start_state + + def _is_list_or_dict_variable(self): + return self._variable_chars[0] in ('@','&') + + def _internal_variable_start_state(self, char, index): + self._state = self._variable_state + if char == '{': + self._variable_chars.append(char) + self._open_curly += 1 + self._may_have_internal_variables = True + else: + self._variable_state(char, index) + + def _waiting_list_variable_index_state(self, char, index): + if char != '[': + raise StopIteration + self._list_and_dict_variable_index_chars.append(char) + self._state = self._list_variable_index_state + + def _list_variable_index_state(self, char, index): + self._list_and_dict_variable_index_chars.append(char) + if char == ']': + raise StopIteration diff --git a/pygments/lexers/ruby.py b/pygments/lexers/ruby.py new file mode 100644 index 0000000..b71b535 --- /dev/null +++ b/pygments/lexers/ruby.py @@ -0,0 +1,523 @@ +""" + pygments.lexers.ruby + ~~~~~~~~~~~~~~~~~~~~ + + Lexers for Ruby and related languages. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import Lexer, RegexLexer, ExtendedRegexLexer, include, \ + bygroups, default, LexerContext, do_insertions, words, line_re +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Error, Generic, Whitespace +from pygments.util import shebang_matches + +__all__ = ['RubyLexer', 'RubyConsoleLexer', 'FancyLexer'] + + +RUBY_OPERATORS = ( + '*', '**', '-', '+', '-@', '+@', '/', '%', '&', '|', '^', '`', '~', + '[]', '[]=', '<<', '>>', '<', '<>', '<=>', '>', '>=', '==', '===' +) + + +class RubyLexer(ExtendedRegexLexer): + """ + For Ruby source code. + """ + + name = 'Ruby' + url = 'http://www.ruby-lang.org' + aliases = ['ruby', 'rb', 'duby'] + filenames = ['*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', + '*.rbx', '*.duby', 'Gemfile', 'Vagrantfile'] + mimetypes = ['text/x-ruby', 'application/x-ruby'] + + flags = re.DOTALL | re.MULTILINE + + def heredoc_callback(self, match, ctx): + # okay, this is the hardest part of parsing Ruby... + # match: 1 = <<[-~]?, 2 = quote? 3 = name 4 = quote? 5 = rest of line + + start = match.start(1) + yield start, Operator, match.group(1) # <<[-~]? + yield match.start(2), String.Heredoc, match.group(2) # quote ", ', ` + yield match.start(3), String.Delimiter, match.group(3) # heredoc name + yield match.start(4), String.Heredoc, match.group(4) # quote again + + heredocstack = ctx.__dict__.setdefault('heredocstack', []) + outermost = not bool(heredocstack) + heredocstack.append((match.group(1) in ('<<-', '<<~'), match.group(3))) + + ctx.pos = match.start(5) + ctx.end = match.end(5) + # this may find other heredocs, so limit the recursion depth + if len(heredocstack) < 100: + yield from self.get_tokens_unprocessed(context=ctx) + else: + yield ctx.pos, String.Heredoc, match.group(5) + ctx.pos = match.end() + + if outermost: + # this is the outer heredoc again, now we can process them all + for tolerant, hdname in heredocstack: + lines = [] + for match in line_re.finditer(ctx.text, ctx.pos): + if tolerant: + check = match.group().strip() + else: + check = match.group().rstrip() + if check == hdname: + for amatch in lines: + yield amatch.start(), String.Heredoc, amatch.group() + yield match.start(), String.Delimiter, match.group() + ctx.pos = match.end() + break + else: + lines.append(match) + else: + # end of heredoc not found -- error! + for amatch in lines: + yield amatch.start(), Error, amatch.group() + ctx.end = len(ctx.text) + del heredocstack[:] + + def gen_rubystrings_rules(): + def intp_regex_callback(self, match, ctx): + yield match.start(1), String.Regex, match.group(1) # begin + nctx = LexerContext(match.group(3), 0, ['interpolated-regex']) + for i, t, v in self.get_tokens_unprocessed(context=nctx): + yield match.start(3)+i, t, v + yield match.start(4), String.Regex, match.group(4) # end[mixounse]* + ctx.pos = match.end() + + def intp_string_callback(self, match, ctx): + yield match.start(1), String.Other, match.group(1) + nctx = LexerContext(match.group(3), 0, ['interpolated-string']) + for i, t, v in self.get_tokens_unprocessed(context=nctx): + yield match.start(3)+i, t, v + yield match.start(4), String.Other, match.group(4) # end + ctx.pos = match.end() + + states = {} + states['strings'] = [ + # easy ones + (r'\:@{0,2}[a-zA-Z_]\w*[!?]?', String.Symbol), + (words(RUBY_OPERATORS, prefix=r'\:@{0,2}'), String.Symbol), + (r":'(\\\\|\\[^\\]|[^'\\])*'", String.Symbol), + (r':"', String.Symbol, 'simple-sym'), + (r'([a-zA-Z_]\w*)(:)(?!:)', + bygroups(String.Symbol, Punctuation)), # Since Ruby 1.9 + (r'"', String.Double, 'simple-string-double'), + (r"'", String.Single, 'simple-string-single'), + (r'(?', '<>', 'ab'): + states[name+'-intp-string'] = [ + (r'\\[\\' + bracecc + ']', String.Other), + (lbrace, String.Other, '#push'), + (rbrace, String.Other, '#pop'), + include('string-intp-escaped'), + (r'[\\#' + bracecc + ']', String.Other), + (r'[^\\#' + bracecc + ']+', String.Other), + ] + states['strings'].append((r'%[QWx]?' + lbrace, String.Other, + name+'-intp-string')) + states[name+'-string'] = [ + (r'\\[\\' + bracecc + ']', String.Other), + (lbrace, String.Other, '#push'), + (rbrace, String.Other, '#pop'), + (r'[\\#' + bracecc + ']', String.Other), + (r'[^\\#' + bracecc + ']+', String.Other), + ] + states['strings'].append((r'%[qsw]' + lbrace, String.Other, + name+'-string')) + states[name+'-regex'] = [ + (r'\\[\\' + bracecc + ']', String.Regex), + (lbrace, String.Regex, '#push'), + (rbrace + '[mixounse]*', String.Regex, '#pop'), + include('string-intp'), + (r'[\\#' + bracecc + ']', String.Regex), + (r'[^\\#' + bracecc + ']+', String.Regex), + ] + states['strings'].append((r'%r' + lbrace, String.Regex, + name+'-regex')) + + # these must come after %! + states['strings'] += [ + # %r regex + (r'(%r([\W_]))((?:\\\2|(?!\2).)*)(\2[mixounse]*)', + intp_regex_callback), + # regular fancy strings with qsw + (r'%[qsw]([\W_])((?:\\\1|(?!\1).)*)\1', String.Other), + (r'(%[QWx]([\W_]))((?:\\\2|(?!\2).)*)(\2)', + intp_string_callback), + # special forms of fancy strings after operators or + # in method calls with braces + (r'(?<=[-+/*%=<>&!^|~,(])(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)', + bygroups(Whitespace, String.Other, None)), + # and because of fixed width lookbehinds the whole thing a + # second time for line startings... + (r'^(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)', + bygroups(Whitespace, String.Other, None)), + # all regular fancy strings without qsw + (r'(%([^a-zA-Z0-9\s]))((?:\\\2|(?!\2).)*)(\2)', + intp_string_callback), + ] + + return states + + tokens = { + 'root': [ + (r'\A#!.+?$', Comment.Hashbang), + (r'#.*?$', Comment.Single), + (r'=begin\s.*?\n=end.*?$', Comment.Multiline), + # keywords + (words(( + 'BEGIN', 'END', 'alias', 'begin', 'break', 'case', 'defined?', + 'do', 'else', 'elsif', 'end', 'ensure', 'for', 'if', 'in', 'next', 'redo', + 'rescue', 'raise', 'retry', 'return', 'super', 'then', 'undef', + 'unless', 'until', 'when', 'while', 'yield'), suffix=r'\b'), + Keyword), + # start of function, class and module names + (r'(module)(\s+)([a-zA-Z_]\w*' + r'(?:::[a-zA-Z_]\w*)*)', + bygroups(Keyword, Whitespace, Name.Namespace)), + (r'(def)(\s+)', bygroups(Keyword, Whitespace), 'funcname'), + (r'def(?=[*%&^`~+-/\[<>=])', Keyword, 'funcname'), + (r'(class)(\s+)', bygroups(Keyword, Whitespace), 'classname'), + # special methods + (words(( + 'initialize', 'new', 'loop', 'include', 'extend', 'raise', 'attr_reader', + 'attr_writer', 'attr_accessor', 'attr', 'catch', 'throw', 'private', + 'module_function', 'public', 'protected', 'true', 'false', 'nil'), + suffix=r'\b'), + Keyword.Pseudo), + (r'(not|and|or)\b', Operator.Word), + (words(( + 'autoload', 'block_given', 'const_defined', 'eql', 'equal', 'frozen', 'include', + 'instance_of', 'is_a', 'iterator', 'kind_of', 'method_defined', 'nil', + 'private_method_defined', 'protected_method_defined', + 'public_method_defined', 'respond_to', 'tainted'), suffix=r'\?'), + Name.Builtin), + (r'(chomp|chop|exit|gsub|sub)!', Name.Builtin), + (words(( + 'Array', 'Float', 'Integer', 'String', '__id__', '__send__', 'abort', + 'ancestors', 'at_exit', 'autoload', 'binding', 'callcc', 'caller', + 'catch', 'chomp', 'chop', 'class_eval', 'class_variables', + 'clone', 'const_defined?', 'const_get', 'const_missing', 'const_set', + 'constants', 'display', 'dup', 'eval', 'exec', 'exit', 'extend', 'fail', 'fork', + 'format', 'freeze', 'getc', 'gets', 'global_variables', 'gsub', + 'hash', 'id', 'included_modules', 'inspect', 'instance_eval', + 'instance_method', 'instance_methods', + 'instance_variable_get', 'instance_variable_set', 'instance_variables', + 'lambda', 'load', 'local_variables', 'loop', + 'method', 'method_missing', 'methods', 'module_eval', 'name', + 'object_id', 'open', 'p', 'print', 'printf', 'private_class_method', + 'private_instance_methods', + 'private_methods', 'proc', 'protected_instance_methods', + 'protected_methods', 'public_class_method', + 'public_instance_methods', 'public_methods', + 'putc', 'puts', 'raise', 'rand', 'readline', 'readlines', 'require', + 'scan', 'select', 'self', 'send', 'set_trace_func', 'singleton_methods', 'sleep', + 'split', 'sprintf', 'srand', 'sub', 'syscall', 'system', 'taint', + 'test', 'throw', 'to_a', 'to_s', 'trace_var', 'trap', 'untaint', + 'untrace_var', 'warn'), prefix=r'(?~!:])|' + r'(?<=(?:\s|;)when\s)|' + r'(?<=(?:\s|;)or\s)|' + r'(?<=(?:\s|;)and\s)|' + r'(?<=\.index\s)|' + r'(?<=\.scan\s)|' + r'(?<=\.sub\s)|' + r'(?<=\.sub!\s)|' + r'(?<=\.gsub\s)|' + r'(?<=\.gsub!\s)|' + r'(?<=\.match\s)|' + r'(?<=(?:\s|;)if\s)|' + r'(?<=(?:\s|;)elsif\s)|' + r'(?<=^when\s)|' + r'(?<=^index\s)|' + r'(?<=^scan\s)|' + r'(?<=^sub\s)|' + r'(?<=^gsub\s)|' + r'(?<=^sub!\s)|' + r'(?<=^gsub!\s)|' + r'(?<=^match\s)|' + r'(?<=^if\s)|' + r'(?<=^elsif\s)' + r')(\s*)(/)', bygroups(Text, String.Regex), 'multiline-regex'), + # multiline regex (in method calls or subscripts) + (r'(?<=\(|,|\[)/', String.Regex, 'multiline-regex'), + # multiline regex (this time the funny no whitespace rule) + (r'(\s+)(/)(?![\s=])', bygroups(Whitespace, String.Regex), + 'multiline-regex'), + # lex numbers and ignore following regular expressions which + # are division operators in fact (grrrr. i hate that. any + # better ideas?) + # since pygments 0.7 we also eat a "?" operator after numbers + # so that the char operator does not work. Chars are not allowed + # there so that you can use the ternary operator. + # stupid example: + # x>=0?n[x]:"" + (r'(0_?[0-7]+(?:_[0-7]+)*)(\s*)([/?])?', + bygroups(Number.Oct, Whitespace, Operator)), + (r'(0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*)(\s*)([/?])?', + bygroups(Number.Hex, Whitespace, Operator)), + (r'(0b[01]+(?:_[01]+)*)(\s*)([/?])?', + bygroups(Number.Bin, Whitespace, Operator)), + (r'([\d]+(?:_\d+)*)(\s*)([/?])?', + bygroups(Number.Integer, Whitespace, Operator)), + # Names + (r'@@[a-zA-Z_]\w*', Name.Variable.Class), + (r'@[a-zA-Z_]\w*', Name.Variable.Instance), + (r'\$\w+', Name.Variable.Global), + (r'\$[!@&`\'+~=/\\,;.<>_*$?:"^-]', Name.Variable.Global), + (r'\$-[0adFiIlpvw]', Name.Variable.Global), + (r'::', Operator), + include('strings'), + # chars + (r'\?(\\[MC]-)*' # modifiers + r'(\\([\\abefnrstv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})|\S)' + r'(?!\w)', + String.Char), + (r'[A-Z]\w+', Name.Constant), + # this is needed because ruby attributes can look + # like keywords (class) or like this: ` ?!? + (words(RUBY_OPERATORS, prefix=r'(\.|::)'), + bygroups(Operator, Name.Operator)), + (r'(\.|::)([a-zA-Z_]\w*[!?]?|[*%&^`~+\-/\[<>=])', + bygroups(Operator, Name)), + (r'[a-zA-Z_]\w*[!?]?', Name), + (r'(\[|\]|\*\*|<>?|>=|<=|<=>|=~|={3}|' + r'!~|&&?|\|\||\.{1,3})', Operator), + (r'[-+/*%=<>&!^|~]=?', Operator), + (r'[(){};,/?:\\]', Punctuation), + (r'\s+', Whitespace) + ], + 'funcname': [ + (r'\(', Punctuation, 'defexpr'), + (r'(?:([a-zA-Z_]\w*)(\.))?' # optional scope name, like "self." + r'(' + r'[a-zA-Z\u0080-\uffff][a-zA-Z0-9_\u0080-\uffff]*[!?=]?' # method name + r'|!=|!~|=~|\*\*?|[-+!~]@?|[/%&|^]|<=>|<[<=]?|>[>=]?|===?' # or operator override + r'|\[\]=?' # or element reference/assignment override + r'|`' # or the undocumented backtick override + r')', + bygroups(Name.Class, Operator, Name.Function), '#pop'), + default('#pop') + ], + 'classname': [ + (r'\(', Punctuation, 'defexpr'), + (r'<<', Operator, '#pop'), + (r'[A-Z_]\w*', Name.Class, '#pop'), + default('#pop') + ], + 'defexpr': [ + (r'(\))(\.|::)?', bygroups(Punctuation, Operator), '#pop'), + (r'\(', Operator, '#push'), + include('root') + ], + 'in-intp': [ + (r'\{', String.Interpol, '#push'), + (r'\}', String.Interpol, '#pop'), + include('root'), + ], + 'string-intp': [ + (r'#\{', String.Interpol, 'in-intp'), + (r'#@@?[a-zA-Z_]\w*', String.Interpol), + (r'#\$[a-zA-Z_]\w*', String.Interpol) + ], + 'string-intp-escaped': [ + include('string-intp'), + (r'\\([\\abefnrstv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})', + String.Escape) + ], + 'interpolated-regex': [ + include('string-intp'), + (r'[\\#]', String.Regex), + (r'[^\\#]+', String.Regex), + ], + 'interpolated-string': [ + include('string-intp'), + (r'[\\#]', String.Other), + (r'[^\\#]+', String.Other), + ], + 'multiline-regex': [ + include('string-intp'), + (r'\\\\', String.Regex), + (r'\\/', String.Regex), + (r'[\\#]', String.Regex), + (r'[^\\/#]+', String.Regex), + (r'/[mixounse]*', String.Regex, '#pop'), + ], + 'end-part': [ + (r'.+', Comment.Preproc, '#pop') + ] + } + tokens.update(gen_rubystrings_rules()) + + def analyse_text(text): + return shebang_matches(text, r'ruby(1\.\d)?') + + +class RubyConsoleLexer(Lexer): + """ + For Ruby interactive console (**irb**) output like: + + .. sourcecode:: rbcon + + irb(main):001:0> a = 1 + => 1 + irb(main):002:0> puts a + 1 + => nil + """ + name = 'Ruby irb session' + aliases = ['rbcon', 'irb'] + mimetypes = ['text/x-ruby-shellsession'] + + _prompt_re = re.compile(r'irb\([a-zA-Z_]\w*\):\d{3}:\d+[>*"\'] ' + r'|>> |\?> ') + + def get_tokens_unprocessed(self, text): + rblexer = RubyLexer(**self.options) + + curcode = '' + insertions = [] + for match in line_re.finditer(text): + line = match.group() + m = self._prompt_re.match(line) + if m is not None: + end = m.end() + insertions.append((len(curcode), + [(0, Generic.Prompt, line[:end])])) + curcode += line[end:] + else: + if curcode: + yield from do_insertions( + insertions, rblexer.get_tokens_unprocessed(curcode)) + curcode = '' + insertions = [] + yield match.start(), Generic.Output, line + if curcode: + yield from do_insertions( + insertions, rblexer.get_tokens_unprocessed(curcode)) + + +class FancyLexer(RegexLexer): + """ + Pygments Lexer For Fancy. + + Fancy is a self-hosted, pure object-oriented, dynamic, + class-based, concurrent general-purpose programming language + running on Rubinius, the Ruby VM. + + .. versionadded:: 1.5 + """ + name = 'Fancy' + url = 'https://github.com/bakkdoor/fancy' + filenames = ['*.fy', '*.fancypack'] + aliases = ['fancy', 'fy'] + mimetypes = ['text/x-fancysrc'] + + tokens = { + # copied from PerlLexer: + 'balanced-regex': [ + (r'/(\\\\|\\[^\\]|[^/\\])*/[egimosx]*', String.Regex, '#pop'), + (r'!(\\\\|\\[^\\]|[^!\\])*![egimosx]*', String.Regex, '#pop'), + (r'\\(\\\\|[^\\])*\\[egimosx]*', String.Regex, '#pop'), + (r'\{(\\\\|\\[^\\]|[^}\\])*\}[egimosx]*', String.Regex, '#pop'), + (r'<(\\\\|\\[^\\]|[^>\\])*>[egimosx]*', String.Regex, '#pop'), + (r'\[(\\\\|\\[^\\]|[^\]\\])*\][egimosx]*', String.Regex, '#pop'), + (r'\((\\\\|\\[^\\]|[^)\\])*\)[egimosx]*', String.Regex, '#pop'), + (r'@(\\\\|\\[^\\]|[^@\\])*@[egimosx]*', String.Regex, '#pop'), + (r'%(\\\\|\\[^\\]|[^%\\])*%[egimosx]*', String.Regex, '#pop'), + (r'\$(\\\\|\\[^\\]|[^$\\])*\$[egimosx]*', String.Regex, '#pop'), + ], + 'root': [ + (r'\s+', Whitespace), + + # balanced delimiters (copied from PerlLexer): + (r's\{(\\\\|\\[^\\]|[^}\\])*\}\s*', String.Regex, 'balanced-regex'), + (r's<(\\\\|\\[^\\]|[^>\\])*>\s*', String.Regex, 'balanced-regex'), + (r's\[(\\\\|\\[^\\]|[^\]\\])*\]\s*', String.Regex, 'balanced-regex'), + (r's\((\\\\|\\[^\\]|[^)\\])*\)\s*', String.Regex, 'balanced-regex'), + (r'm?/(\\\\|\\[^\\]|[^///\n])*/[gcimosx]*', String.Regex), + (r'm(?=[/!\\{<\[(@%$])', String.Regex, 'balanced-regex'), + + # Comments + (r'#(.*?)\n', Comment.Single), + # Symbols + (r'\'([^\'\s\[\](){}]+|\[\])', String.Symbol), + # Multi-line DoubleQuotedString + (r'"""(\\\\|\\[^\\]|[^\\])*?"""', String), + # DoubleQuotedString + (r'"(\\\\|\\[^\\]|[^"\\])*"', String), + # keywords + (r'(def|class|try|catch|finally|retry|return|return_local|match|' + r'case|->|=>)\b', Keyword), + # constants + (r'(self|super|nil|false|true)\b', Name.Constant), + (r'[(){};,/?|:\\]', Punctuation), + # names + (words(( + 'Object', 'Array', 'Hash', 'Directory', 'File', 'Class', 'String', + 'Number', 'Enumerable', 'FancyEnumerable', 'Block', 'TrueClass', + 'NilClass', 'FalseClass', 'Tuple', 'Symbol', 'Stack', 'Set', + 'FancySpec', 'Method', 'Package', 'Range'), suffix=r'\b'), + Name.Builtin), + # functions + (r'[a-zA-Z](\w|[-+?!=*/^><%])*:', Name.Function), + # operators, must be below functions + (r'[-+*/~,<>=&!?%^\[\].$]+', Operator), + (r'[A-Z]\w*', Name.Constant), + (r'@[a-zA-Z_]\w*', Name.Variable.Instance), + (r'@@[a-zA-Z_]\w*', Name.Variable.Class), + ('@@?', Operator), + (r'[a-zA-Z_]\w*', Name), + # numbers - / checks are necessary to avoid mismarking regexes, + # see comment in RubyLexer + (r'(0[oO]?[0-7]+(?:_[0-7]+)*)(\s*)([/?])?', + bygroups(Number.Oct, Whitespace, Operator)), + (r'(0[xX][0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*)(\s*)([/?])?', + bygroups(Number.Hex, Whitespace, Operator)), + (r'(0[bB][01]+(?:_[01]+)*)(\s*)([/?])?', + bygroups(Number.Bin, Whitespace, Operator)), + (r'([\d]+(?:_\d+)*)(\s*)([/?])?', + bygroups(Number.Integer, Whitespace, Operator)), + (r'\d+([eE][+-]?[0-9]+)|\d+\.\d+([eE][+-]?[0-9]+)?', Number.Float), + (r'\d+', Number.Integer) + ] + } diff --git a/pygments/lexers/rust.py b/pygments/lexers/rust.py new file mode 100644 index 0000000..500c7a5 --- /dev/null +++ b/pygments/lexers/rust.py @@ -0,0 +1,223 @@ +""" + pygments.lexers.rust + ~~~~~~~~~~~~~~~~~~~~ + + Lexers for the Rust language. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, include, bygroups, words, default +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Whitespace + +__all__ = ['RustLexer'] + + +class RustLexer(RegexLexer): + """ + Lexer for the Rust programming language (version 1.47). + + .. versionadded:: 1.6 + """ + name = 'Rust' + url = 'https://www.rust-lang.org/' + filenames = ['*.rs', '*.rs.in'] + aliases = ['rust', 'rs'] + mimetypes = ['text/rust', 'text/x-rust'] + + keyword_types = (words(( + 'u8', 'u16', 'u32', 'u64', 'u128', 'i8', 'i16', 'i32', 'i64', 'i128', + 'usize', 'isize', 'f32', 'f64', 'char', 'str', 'bool', + ), suffix=r'\b'), Keyword.Type) + + builtin_funcs_types = (words(( + 'Copy', 'Send', 'Sized', 'Sync', 'Unpin', + 'Drop', 'Fn', 'FnMut', 'FnOnce', 'drop', + 'Box', 'ToOwned', 'Clone', + 'PartialEq', 'PartialOrd', 'Eq', 'Ord', + 'AsRef', 'AsMut', 'Into', 'From', 'Default', + 'Iterator', 'Extend', 'IntoIterator', 'DoubleEndedIterator', + 'ExactSizeIterator', + 'Option', 'Some', 'None', + 'Result', 'Ok', 'Err', + 'String', 'ToString', 'Vec', + ), suffix=r'\b'), Name.Builtin) + + builtin_macros = (words(( + 'asm', 'assert', 'assert_eq', 'assert_ne', 'cfg', 'column', + 'compile_error', 'concat', 'concat_idents', 'dbg', 'debug_assert', + 'debug_assert_eq', 'debug_assert_ne', 'env', 'eprint', 'eprintln', + 'file', 'format', 'format_args', 'format_args_nl', 'global_asm', + 'include', 'include_bytes', 'include_str', + 'is_aarch64_feature_detected', + 'is_arm_feature_detected', + 'is_mips64_feature_detected', + 'is_mips_feature_detected', + 'is_powerpc64_feature_detected', + 'is_powerpc_feature_detected', + 'is_x86_feature_detected', + 'line', 'llvm_asm', 'log_syntax', 'macro_rules', 'matches', + 'module_path', 'option_env', 'panic', 'print', 'println', 'stringify', + 'thread_local', 'todo', 'trace_macros', 'unimplemented', 'unreachable', + 'vec', 'write', 'writeln', + ), suffix=r'!'), Name.Function.Magic) + + tokens = { + 'root': [ + # rust allows a file to start with a shebang, but if the first line + # starts with #![ then it's not a shebang but a crate attribute. + (r'#![^[\r\n].*$', Comment.Preproc), + default('base'), + ], + 'base': [ + # Whitespace and Comments + (r'\n', Whitespace), + (r'\s+', Whitespace), + (r'//!.*?\n', String.Doc), + (r'///(\n|[^/].*?\n)', String.Doc), + (r'//(.*?)\n', Comment.Single), + (r'/\*\*(\n|[^/*])', String.Doc, 'doccomment'), + (r'/\*!', String.Doc, 'doccomment'), + (r'/\*', Comment.Multiline, 'comment'), + + # Macro parameters + (r"""\$([a-zA-Z_]\w*|\(,?|\),?|,?)""", Comment.Preproc), + # Keywords + (words(('as', 'async', 'await', 'box', 'const', 'crate', 'dyn', + 'else', 'extern', 'for', 'if', 'impl', 'in', 'loop', + 'match', 'move', 'mut', 'pub', 'ref', 'return', 'static', + 'super', 'trait', 'unsafe', 'use', 'where', 'while'), + suffix=r'\b'), Keyword), + (words(('abstract', 'become', 'do', 'final', 'macro', 'override', + 'priv', 'typeof', 'try', 'unsized', 'virtual', 'yield'), + suffix=r'\b'), Keyword.Reserved), + (r'(true|false)\b', Keyword.Constant), + (r'self\b', Name.Builtin.Pseudo), + (r'mod\b', Keyword, 'modname'), + (r'let\b', Keyword.Declaration), + (r'fn\b', Keyword, 'funcname'), + (r'(struct|enum|type|union)\b', Keyword, 'typename'), + (r'(default)(\s+)(type|fn)\b', bygroups(Keyword, Text, Keyword)), + keyword_types, + (r'[sS]elf\b', Name.Builtin.Pseudo), + # Prelude (taken from Rust's src/libstd/prelude.rs) + builtin_funcs_types, + builtin_macros, + # Path separators, so types don't catch them. + (r'::\b', Text), + # Types in positions. + (r'(?::|->)', Text, 'typename'), + # Labels + (r'(break|continue)(\b\s*)(\'[A-Za-z_]\w*)?', + bygroups(Keyword, Text.Whitespace, Name.Label)), + + # Character literals + (r"""'(\\['"\\nrt]|\\x[0-7][0-9a-fA-F]|\\0""" + r"""|\\u\{[0-9a-fA-F]{1,6}\}|.)'""", + String.Char), + (r"""b'(\\['"\\nrt]|\\x[0-9a-fA-F]{2}|\\0""" + r"""|\\u\{[0-9a-fA-F]{1,6}\}|.)'""", + String.Char), + + # Binary literals + (r'0b[01_]+', Number.Bin, 'number_lit'), + # Octal literals + (r'0o[0-7_]+', Number.Oct, 'number_lit'), + # Hexadecimal literals + (r'0[xX][0-9a-fA-F_]+', Number.Hex, 'number_lit'), + # Decimal literals + (r'[0-9][0-9_]*(\.[0-9_]+[eE][+\-]?[0-9_]+|' + r'\.[0-9_]*(?!\.)|[eE][+\-]?[0-9_]+)', Number.Float, + 'number_lit'), + (r'[0-9][0-9_]*', Number.Integer, 'number_lit'), + + # String literals + (r'b"', String, 'bytestring'), + (r'"', String, 'string'), + (r'(?s)b?r(#*)".*?"\1', String), + + # Lifetime names + (r"'", Operator, 'lifetime'), + + # Operators and Punctuation + (r'\.\.=?', Operator), + (r'[{}()\[\],.;]', Punctuation), + (r'[+\-*/%&|<>^!~@=:?]', Operator), + + # Identifiers + (r'[a-zA-Z_]\w*', Name), + # Raw identifiers + (r'r#[a-zA-Z_]\w*', Name), + + # Attributes + (r'#!?\[', Comment.Preproc, 'attribute['), + + # Misc + # Lone hashes: not used in Rust syntax, but allowed in macro + # arguments, most famously for quote::quote!() + (r'#', Text), + ], + 'comment': [ + (r'[^*/]+', Comment.Multiline), + (r'/\*', Comment.Multiline, '#push'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[*/]', Comment.Multiline), + ], + 'doccomment': [ + (r'[^*/]+', String.Doc), + (r'/\*', String.Doc, '#push'), + (r'\*/', String.Doc, '#pop'), + (r'[*/]', String.Doc), + ], + 'modname': [ + (r'\s+', Text), + (r'[a-zA-Z_]\w*', Name.Namespace, '#pop'), + default('#pop'), + ], + 'funcname': [ + (r'\s+', Text), + (r'[a-zA-Z_]\w*', Name.Function, '#pop'), + default('#pop'), + ], + 'typename': [ + (r'\s+', Text), + (r'&', Keyword.Pseudo), + (r"'", Operator, 'lifetime'), + builtin_funcs_types, + keyword_types, + (r'[a-zA-Z_]\w*', Name.Class, '#pop'), + default('#pop'), + ], + 'lifetime': [ + (r"(static|_)", Name.Builtin), + (r"[a-zA-Z_]+\w*", Name.Attribute), + default('#pop'), + ], + 'number_lit': [ + (r'[ui](8|16|32|64|size)', Keyword, '#pop'), + (r'f(32|64)', Keyword, '#pop'), + default('#pop'), + ], + 'string': [ + (r'"', String, '#pop'), + (r"""\\['"\\nrt]|\\x[0-7][0-9a-fA-F]|\\0""" + r"""|\\u\{[0-9a-fA-F]{1,6}\}""", String.Escape), + (r'[^\\"]+', String), + (r'\\', String), + ], + 'bytestring': [ + (r"""\\x[89a-fA-F][0-9a-fA-F]""", String.Escape), + include('string'), + ], + 'attribute_common': [ + (r'"', String, 'string'), + (r'\[', Comment.Preproc, 'attribute['), + ], + 'attribute[': [ + include('attribute_common'), + (r'\]', Comment.Preproc, '#pop'), + (r'[^"\]\[]+', Comment.Preproc), + ], + } diff --git a/pygments/lexers/sas.py b/pygments/lexers/sas.py new file mode 100644 index 0000000..f2f408c --- /dev/null +++ b/pygments/lexers/sas.py @@ -0,0 +1,227 @@ +""" + pygments.lexers.sas + ~~~~~~~~~~~~~~~~~~~ + + Lexer for SAS. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re +from pygments.lexer import RegexLexer, include, words +from pygments.token import Comment, Keyword, Name, Number, String, Text, \ + Other, Generic + +__all__ = ['SASLexer'] + + +class SASLexer(RegexLexer): + """ + For SAS files. + + .. versionadded:: 2.2 + """ + # Syntax from syntax/sas.vim by James Kidd + + name = 'SAS' + aliases = ['sas'] + filenames = ['*.SAS', '*.sas'] + mimetypes = ['text/x-sas', 'text/sas', 'application/x-sas'] + flags = re.IGNORECASE | re.MULTILINE + + builtins_macros = ( + "bquote", "nrbquote", "cmpres", "qcmpres", "compstor", "datatyp", + "display", "do", "else", "end", "eval", "global", "goto", "if", + "index", "input", "keydef", "label", "left", "length", "let", + "local", "lowcase", "macro", "mend", "nrquote", + "nrstr", "put", "qleft", "qlowcase", "qscan", + "qsubstr", "qsysfunc", "qtrim", "quote", "qupcase", "scan", + "str", "substr", "superq", "syscall", "sysevalf", "sysexec", + "sysfunc", "sysget", "syslput", "sysprod", "sysrc", "sysrput", + "then", "to", "trim", "unquote", "until", "upcase", "verify", + "while", "window" + ) + + builtins_conditionals = ( + "do", "if", "then", "else", "end", "until", "while" + ) + + builtins_statements = ( + "abort", "array", "attrib", "by", "call", "cards", "cards4", + "catname", "continue", "datalines", "datalines4", "delete", "delim", + "delimiter", "display", "dm", "drop", "endsas", "error", "file", + "filename", "footnote", "format", "goto", "in", "infile", "informat", + "input", "keep", "label", "leave", "length", "libname", "link", + "list", "lostcard", "merge", "missing", "modify", "options", "output", + "out", "page", "put", "redirect", "remove", "rename", "replace", + "retain", "return", "select", "set", "skip", "startsas", "stop", + "title", "update", "waitsas", "where", "window", "x", "systask" + ) + + builtins_sql = ( + "add", "and", "alter", "as", "cascade", "check", "create", + "delete", "describe", "distinct", "drop", "foreign", "from", + "group", "having", "index", "insert", "into", "in", "key", "like", + "message", "modify", "msgtype", "not", "null", "on", "or", + "order", "primary", "references", "reset", "restrict", "select", + "set", "table", "unique", "update", "validate", "view", "where" + ) + + builtins_functions = ( + "abs", "addr", "airy", "arcos", "arsin", "atan", "attrc", + "attrn", "band", "betainv", "blshift", "bnot", "bor", + "brshift", "bxor", "byte", "cdf", "ceil", "cexist", "cinv", + "close", "cnonct", "collate", "compbl", "compound", + "compress", "cos", "cosh", "css", "curobs", "cv", "daccdb", + "daccdbsl", "daccsl", "daccsyd", "dacctab", "dairy", "date", + "datejul", "datepart", "datetime", "day", "dclose", "depdb", + "depdbsl", "depsl", "depsyd", + "deptab", "dequote", "dhms", "dif", "digamma", + "dim", "dinfo", "dnum", "dopen", "doptname", "doptnum", + "dread", "dropnote", "dsname", "erf", "erfc", "exist", "exp", + "fappend", "fclose", "fcol", "fdelete", "fetch", "fetchobs", + "fexist", "fget", "fileexist", "filename", "fileref", + "finfo", "finv", "fipname", "fipnamel", "fipstate", "floor", + "fnonct", "fnote", "fopen", "foptname", "foptnum", "fpoint", + "fpos", "fput", "fread", "frewind", "frlen", "fsep", "fuzz", + "fwrite", "gaminv", "gamma", "getoption", "getvarc", "getvarn", + "hbound", "hms", "hosthelp", "hour", "ibessel", "index", + "indexc", "indexw", "input", "inputc", "inputn", "int", + "intck", "intnx", "intrr", "irr", "jbessel", "juldate", + "kurtosis", "lag", "lbound", "left", "length", "lgamma", + "libname", "libref", "log", "log10", "log2", "logpdf", "logpmf", + "logsdf", "lowcase", "max", "mdy", "mean", "min", "minute", + "mod", "month", "mopen", "mort", "n", "netpv", "nmiss", + "normal", "note", "npv", "open", "ordinal", "pathname", + "pdf", "peek", "peekc", "pmf", "point", "poisson", "poke", + "probbeta", "probbnml", "probchi", "probf", "probgam", + "probhypr", "probit", "probnegb", "probnorm", "probt", + "put", "putc", "putn", "qtr", "quote", "ranbin", "rancau", + "ranexp", "rangam", "range", "rank", "rannor", "ranpoi", + "rantbl", "rantri", "ranuni", "repeat", "resolve", "reverse", + "rewind", "right", "round", "saving", "scan", "sdf", "second", + "sign", "sin", "sinh", "skewness", "soundex", "spedis", + "sqrt", "std", "stderr", "stfips", "stname", "stnamel", + "substr", "sum", "symget", "sysget", "sysmsg", "sysprod", + "sysrc", "system", "tan", "tanh", "time", "timepart", "tinv", + "tnonct", "today", "translate", "tranwrd", "trigamma", + "trim", "trimn", "trunc", "uniform", "upcase", "uss", "var", + "varfmt", "varinfmt", "varlabel", "varlen", "varname", + "varnum", "varray", "varrayx", "vartype", "verify", "vformat", + "vformatd", "vformatdx", "vformatn", "vformatnx", "vformatw", + "vformatwx", "vformatx", "vinarray", "vinarrayx", "vinformat", + "vinformatd", "vinformatdx", "vinformatn", "vinformatnx", + "vinformatw", "vinformatwx", "vinformatx", "vlabel", + "vlabelx", "vlength", "vlengthx", "vname", "vnamex", "vtype", + "vtypex", "weekday", "year", "yyq", "zipfips", "zipname", + "zipnamel", "zipstate" + ) + + tokens = { + 'root': [ + include('comments'), + include('proc-data'), + include('cards-datalines'), + include('logs'), + include('general'), + (r'.', Text), + ], + # SAS is multi-line regardless, but * is ended by ; + 'comments': [ + (r'^\s*\*.*?;', Comment), + (r'/\*.*?\*/', Comment), + (r'^\s*\*(.|\n)*?;', Comment.Multiline), + (r'/[*](.|\n)*?[*]/', Comment.Multiline), + ], + # Special highlight for proc, data, quit, run + 'proc-data': [ + (r'(^|;)\s*(proc \w+|data|run|quit)[\s;]', + Keyword.Reserved), + ], + # Special highlight cards and datalines + 'cards-datalines': [ + (r'^\s*(datalines|cards)\s*;\s*$', Keyword, 'data'), + ], + 'data': [ + (r'(.|\n)*^\s*;\s*$', Other, '#pop'), + ], + # Special highlight for put NOTE|ERROR|WARNING (order matters) + 'logs': [ + (r'\n?^\s*%?put ', Keyword, 'log-messages'), + ], + 'log-messages': [ + (r'NOTE(:|-).*', Generic, '#pop'), + (r'WARNING(:|-).*', Generic.Emph, '#pop'), + (r'ERROR(:|-).*', Generic.Error, '#pop'), + include('general'), + ], + 'general': [ + include('keywords'), + include('vars-strings'), + include('special'), + include('numbers'), + ], + # Keywords, statements, functions, macros + 'keywords': [ + (words(builtins_statements, + prefix = r'\b', + suffix = r'\b'), + Keyword), + (words(builtins_sql, + prefix = r'\b', + suffix = r'\b'), + Keyword), + (words(builtins_conditionals, + prefix = r'\b', + suffix = r'\b'), + Keyword), + (words(builtins_macros, + prefix = r'%', + suffix = r'\b'), + Name.Builtin), + (words(builtins_functions, + prefix = r'\b', + suffix = r'\('), + Name.Builtin), + ], + # Strings and user-defined variables and macros (order matters) + 'vars-strings': [ + (r'&[a-z_]\w{0,31}\.?', Name.Variable), + (r'%[a-z_]\w{0,31}', Name.Function), + (r'\'', String, 'string_squote'), + (r'"', String, 'string_dquote'), + ], + 'string_squote': [ + ('\'', String, '#pop'), + (r'\\\\|\\"|\\\n', String.Escape), + # AFAIK, macro variables are not evaluated in single quotes + # (r'&', Name.Variable, 'validvar'), + (r'[^$\'\\]+', String), + (r'[$\'\\]', String), + ], + 'string_dquote': [ + (r'"', String, '#pop'), + (r'\\\\|\\"|\\\n', String.Escape), + (r'&', Name.Variable, 'validvar'), + (r'[^$&"\\]+', String), + (r'[$"\\]', String), + ], + 'validvar': [ + (r'[a-z_]\w{0,31}\.?', Name.Variable, '#pop'), + ], + # SAS numbers and special variables + 'numbers': [ + (r'\b[+-]?([0-9]+(\.[0-9]+)?|\.[0-9]+|\.)(E[+-]?[0-9]+)?i?\b', + Number), + ], + 'special': [ + (r'(null|missing|_all_|_automatic_|_character_|_n_|' + r'_infile_|_name_|_null_|_numeric_|_user_|_webout_)', + Keyword.Constant), + ], + # 'operators': [ + # (r'(-|=|<=|>=|<|>|<>|&|!=|' + # r'\||\*|\+|\^|/|!|~|~=)', Operator) + # ], + } diff --git a/pygments/lexers/savi.py b/pygments/lexers/savi.py new file mode 100644 index 0000000..2397fab --- /dev/null +++ b/pygments/lexers/savi.py @@ -0,0 +1,170 @@ +""" + pygments.lexers.savi + ~~~~~~~~~~~~~~~~~~~~ + + Lexer for Savi. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups, include +from pygments.token import Whitespace, Keyword, Name, String, Number, \ + Operator, Punctuation, Comment, Generic, Error + +__all__ = ['SaviLexer'] + + +# The canonical version of this file can be found in the following repository, +# where it is kept in sync with any language changes, as well as the other +# pygments-like lexers that are maintained for use with other tools: +# - https://github.com/savi-lang/savi/blob/main/tooling/pygments/lexers/savi.py +# +# If you're changing this file in the pygments repository, please ensure that +# any changes you make are also propagated to the official Savi repository, +# in order to avoid accidental clobbering of your changes later when an update +# from the Savi repository flows forward into the pygments repository. +# +# If you're changing this file in the Savi repository, please ensure that +# any changes you make are also reflected in the other pygments-like lexers +# (rouge, vscode, etc) so that all of the lexers can be kept cleanly in sync. + +class SaviLexer(RegexLexer): + """ + For Savi source code. + + .. versionadded: 2.10 + """ + + name = 'Savi' + url = 'https://github.com/savi-lang/savi' + aliases = ['savi'] + filenames = ['*.savi'] + + tokens = { + "root": [ + # Line Comment + (r'//.*?$', Comment.Single), + + # Doc Comment + (r'::.*?$', Comment.Single), + + # Capability Operator + (r'(\')(\w+)(?=[^\'])', bygroups(Operator, Name)), + + # Double-Quote String + (r'\w?"', String.Double, "string.double"), + + # Single-Char String + (r"'", String.Char, "string.char"), + + # Type Name + (r'(_?[A-Z]\w*)', Name.Class), + + # Nested Type Name + (r'(\.)(\s*)(_?[A-Z]\w*)', bygroups(Punctuation, Whitespace, Name.Class)), + + # Declare + (r'^([ \t]*)(:\w+)', + bygroups(Whitespace, Name.Tag), + "decl"), + + # Error-Raising Calls/Names + (r'((\w+|\+|\-|\*)\!)', Generic.Deleted), + + # Numeric Values + (r'\b\d([\d_]*(\.[\d_]+)?)\b', Number), + + # Hex Numeric Values + (r'\b0x([0-9a-fA-F_]+)\b', Number.Hex), + + # Binary Numeric Values + (r'\b0b([01_]+)\b', Number.Bin), + + # Function Call (with braces) + (r'\w+(?=\()', Name.Function), + + # Function Call (with receiver) + (r'(\.)(\s*)(\w+)', bygroups(Punctuation, Whitespace, Name.Function)), + + # Function Call (with self receiver) + (r'(@)(\w+)', bygroups(Punctuation, Name.Function)), + + # Parenthesis + (r'\(', Punctuation, "root"), + (r'\)', Punctuation, "#pop"), + + # Brace + (r'\{', Punctuation, "root"), + (r'\}', Punctuation, "#pop"), + + # Bracket + (r'\[', Punctuation, "root"), + (r'(\])(\!)', bygroups(Punctuation, Generic.Deleted), "#pop"), + (r'\]', Punctuation, "#pop"), + + # Punctuation + (r'[,;:\.@]', Punctuation), + + # Piping Operators + (r'(\|\>)', Operator), + + # Branching Operators + (r'(\&\&|\|\||\?\?|\&\?|\|\?|\.\?)', Operator), + + # Comparison Operators + (r'(\<\=\>|\=\~|\=\=|\<\=|\>\=|\<|\>)', Operator), + + # Arithmetic Operators + (r'(\+|\-|\/|\*|\%)', Operator), + + # Assignment Operators + (r'(\=)', Operator), + + # Other Operators + (r'(\!|\<\<|\<|\&|\|)', Operator), + + # Identifiers + (r'\b\w+\b', Name), + + # Whitespace + (r'[ \t\r]+\n*|\n+', Whitespace), + ], + + # Declare (nested rules) + "decl": [ + (r'\b[a-z_]\w*\b(?!\!)', Keyword.Declaration), + (r':', Punctuation, "#pop"), + (r'\n', Whitespace, "#pop"), + include("root"), + ], + + # Double-Quote String (nested rules) + "string.double": [ + (r'\\\(', String.Interpol, "string.interpolation"), + (r'\\u[0-9a-fA-F]{4}', String.Escape), + (r'\\x[0-9a-fA-F]{2}', String.Escape), + (r'\\[bfnrt\\\']', String.Escape), + (r'\\"', String.Escape), + (r'"', String.Double, "#pop"), + (r'[^\\"]+', String.Double), + (r'.', Error), + ], + + # Single-Char String (nested rules) + "string.char": [ + (r'\\u[0-9a-fA-F]{4}', String.Escape), + (r'\\x[0-9a-fA-F]{2}', String.Escape), + (r'\\[bfnrt\\\']', String.Escape), + (r"\\'", String.Escape), + (r"'", String.Char, "#pop"), + (r"[^\\']+", String.Char), + (r'.', Error), + ], + + # Interpolation inside String (nested rules) + "string.interpolation": [ + (r"\)", String.Interpol, "#pop"), + include("root"), + ] + } diff --git a/pygments/lexers/scdoc.py b/pygments/lexers/scdoc.py new file mode 100644 index 0000000..f670d33 --- /dev/null +++ b/pygments/lexers/scdoc.py @@ -0,0 +1,79 @@ +""" + pygments.lexers.scdoc + ~~~~~~~~~~~~~~~~~~~~~ + + Lexer for scdoc, a simple man page generator. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, include, bygroups, using, this +from pygments.token import Text, Comment, Keyword, String, Generic + +__all__ = ['ScdocLexer'] + + +class ScdocLexer(RegexLexer): + """ + `scdoc` is a simple man page generator for POSIX systems written in C99. + + .. versionadded:: 2.5 + """ + name = 'scdoc' + url = 'https://git.sr.ht/~sircmpwn/scdoc' + aliases = ['scdoc', 'scd'] + filenames = ['*.scd', '*.scdoc'] + flags = re.MULTILINE + + tokens = { + 'root': [ + # comment + (r'^(;.+\n)', bygroups(Comment)), + + # heading with pound prefix + (r'^(#)([^#].+\n)', bygroups(Generic.Heading, Text)), + (r'^(#{2})(.+\n)', bygroups(Generic.Subheading, Text)), + # bulleted lists + (r'^(\s*)([*-])(\s)(.+\n)', + bygroups(Text, Keyword, Text, using(this, state='inline'))), + # numbered lists + (r'^(\s*)(\.+\.)( .+\n)', + bygroups(Text, Keyword, using(this, state='inline'))), + # quote + (r'^(\s*>\s)(.+\n)', bygroups(Keyword, Generic.Emph)), + # text block + (r'^(```\n)([\w\W]*?)(^```$)', bygroups(String, Text, String)), + + include('inline'), + ], + 'inline': [ + # escape + (r'\\.', Text), + # underlines + (r'(\s)(_[^_]+_)(\W|\n)', bygroups(Text, Generic.Emph, Text)), + # bold + (r'(\s)(\*[^*]+\*)(\W|\n)', bygroups(Text, Generic.Strong, Text)), + # inline code + (r'`[^`]+`', String.Backtick), + + # general text, must come last! + (r'[^\\\s]+', Text), + (r'.', Text), + ], + } + + def analyse_text(text): + """This is very similar to markdown, save for the escape characters + needed for * and _.""" + result = 0 + + if '\\*' in text: + result += 0.01 + + if '\\_' in text: + result += 0.01 + + return result diff --git a/pygments/lexers/scripting.py b/pygments/lexers/scripting.py new file mode 100644 index 0000000..25cc406 --- /dev/null +++ b/pygments/lexers/scripting.py @@ -0,0 +1,1286 @@ +""" + pygments.lexers.scripting + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + Lexer for scripting and embedded languages. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, include, bygroups, default, combined, \ + words +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Error, Whitespace, Other +from pygments.util import get_bool_opt, get_list_opt + +__all__ = ['LuaLexer', 'MoonScriptLexer', 'ChaiscriptLexer', 'LSLLexer', + 'AppleScriptLexer', 'RexxLexer', 'MOOCodeLexer', 'HybrisLexer', + 'EasytrieveLexer', 'JclLexer', 'MiniScriptLexer'] + + +class LuaLexer(RegexLexer): + """ + For Lua source code. + + Additional options accepted: + + `func_name_highlighting` + If given and ``True``, highlight builtin function names + (default: ``True``). + `disabled_modules` + If given, must be a list of module names whose function names + should not be highlighted. By default all modules are highlighted. + + To get a list of allowed modules have a look into the + `_lua_builtins` module: + + .. sourcecode:: pycon + + >>> from pygments.lexers._lua_builtins import MODULES + >>> MODULES.keys() + ['string', 'coroutine', 'modules', 'io', 'basic', ...] + """ + + name = 'Lua' + url = 'https://www.lua.org/' + aliases = ['lua'] + filenames = ['*.lua', '*.wlua'] + mimetypes = ['text/x-lua', 'application/x-lua'] + + _comment_multiline = r'(?:--\[(?P=*)\[[\w\W]*?\](?P=level)\])' + _comment_single = r'(?:--.*$)' + _space = r'(?:\s+)' + _s = r'(?:%s|%s|%s)' % (_comment_multiline, _comment_single, _space) + _name = r'(?:[^\W\d]\w*)' + + tokens = { + 'root': [ + # Lua allows a file to start with a shebang. + (r'#!.*', Comment.Preproc), + default('base'), + ], + 'ws': [ + (_comment_multiline, Comment.Multiline), + (_comment_single, Comment.Single), + (_space, Text), + ], + 'base': [ + include('ws'), + + (r'(?i)0x[\da-f]*(\.[\da-f]*)?(p[+-]?\d+)?', Number.Hex), + (r'(?i)(\d*\.\d+|\d+\.\d*)(e[+-]?\d+)?', Number.Float), + (r'(?i)\d+e[+-]?\d+', Number.Float), + (r'\d+', Number.Integer), + + # multiline strings + (r'(?s)\[(=*)\[.*?\]\1\]', String), + + (r'::', Punctuation, 'label'), + (r'\.{3}', Punctuation), + (r'[=<>|~&+\-*/%#^]+|\.\.', Operator), + (r'[\[\]{}().,:;]', Punctuation), + (r'(and|or|not)\b', Operator.Word), + + ('(break|do|else|elseif|end|for|if|in|repeat|return|then|until|' + r'while)\b', Keyword.Reserved), + (r'goto\b', Keyword.Reserved, 'goto'), + (r'(local)\b', Keyword.Declaration), + (r'(true|false|nil)\b', Keyword.Constant), + + (r'(function)\b', Keyword.Reserved, 'funcname'), + + (r'[A-Za-z_]\w*(\.[A-Za-z_]\w*)?', Name), + + ("'", String.Single, combined('stringescape', 'sqs')), + ('"', String.Double, combined('stringescape', 'dqs')) + ], + + 'funcname': [ + include('ws'), + (r'[.:]', Punctuation), + (r'%s(?=%s*[.:])' % (_name, _s), Name.Class), + (_name, Name.Function, '#pop'), + # inline function + (r'\(', Punctuation, '#pop'), + ], + + 'goto': [ + include('ws'), + (_name, Name.Label, '#pop'), + ], + + 'label': [ + include('ws'), + (r'::', Punctuation, '#pop'), + (_name, Name.Label), + ], + + 'stringescape': [ + (r'\\([abfnrtv\\"\']|[\r\n]{1,2}|z\s*|x[0-9a-fA-F]{2}|\d{1,3}|' + r'u\{[0-9a-fA-F]+\})', String.Escape), + ], + + 'sqs': [ + (r"'", String.Single, '#pop'), + (r"[^\\']+", String.Single), + ], + + 'dqs': [ + (r'"', String.Double, '#pop'), + (r'[^\\"]+', String.Double), + ] + } + + def __init__(self, **options): + self.func_name_highlighting = get_bool_opt( + options, 'func_name_highlighting', True) + self.disabled_modules = get_list_opt(options, 'disabled_modules', []) + + self._functions = set() + if self.func_name_highlighting: + from pygments.lexers._lua_builtins import MODULES + for mod, func in MODULES.items(): + if mod not in self.disabled_modules: + self._functions.update(func) + RegexLexer.__init__(self, **options) + + def get_tokens_unprocessed(self, text): + for index, token, value in \ + RegexLexer.get_tokens_unprocessed(self, text): + if token is Name: + if value in self._functions: + yield index, Name.Builtin, value + continue + elif '.' in value: + a, b = value.split('.') + yield index, Name, a + yield index + len(a), Punctuation, '.' + yield index + len(a) + 1, Name, b + continue + yield index, token, value + +class MoonScriptLexer(LuaLexer): + """ + For MoonScript source code. + + .. versionadded:: 1.5 + """ + + name = 'MoonScript' + url = 'http://moonscript.org' + aliases = ['moonscript', 'moon'] + filenames = ['*.moon'] + mimetypes = ['text/x-moonscript', 'application/x-moonscript'] + + tokens = { + 'root': [ + (r'#!(.*?)$', Comment.Preproc), + default('base'), + ], + 'base': [ + ('--.*$', Comment.Single), + (r'(?i)(\d*\.\d+|\d+\.\d*)(e[+-]?\d+)?', Number.Float), + (r'(?i)\d+e[+-]?\d+', Number.Float), + (r'(?i)0x[0-9a-f]*', Number.Hex), + (r'\d+', Number.Integer), + (r'\n', Whitespace), + (r'[^\S\n]+', Text), + (r'(?s)\[(=*)\[.*?\]\1\]', String), + (r'(->|=>)', Name.Function), + (r':[a-zA-Z_]\w*', Name.Variable), + (r'(==|!=|~=|<=|>=|\.\.\.|\.\.|[=+\-*/%^<>#!.\\:])', Operator), + (r'[;,]', Punctuation), + (r'[\[\]{}()]', Keyword.Type), + (r'[a-zA-Z_]\w*:', Name.Variable), + (words(( + 'class', 'extends', 'if', 'then', 'super', 'do', 'with', + 'import', 'export', 'while', 'elseif', 'return', 'for', 'in', + 'from', 'when', 'using', 'else', 'and', 'or', 'not', 'switch', + 'break'), suffix=r'\b'), + Keyword), + (r'(true|false|nil)\b', Keyword.Constant), + (r'(and|or|not)\b', Operator.Word), + (r'(self)\b', Name.Builtin.Pseudo), + (r'@@?([a-zA-Z_]\w*)?', Name.Variable.Class), + (r'[A-Z]\w*', Name.Class), # proper name + (r'[A-Za-z_]\w*(\.[A-Za-z_]\w*)?', Name), + ("'", String.Single, combined('stringescape', 'sqs')), + ('"', String.Double, combined('stringescape', 'dqs')) + ], + 'stringescape': [ + (r'''\\([abfnrtv\\"']|\d{1,3})''', String.Escape) + ], + 'sqs': [ + ("'", String.Single, '#pop'), + ("[^']+", String) + ], + 'dqs': [ + ('"', String.Double, '#pop'), + ('[^"]+', String) + ] + } + + def get_tokens_unprocessed(self, text): + # set . as Operator instead of Punctuation + for index, token, value in LuaLexer.get_tokens_unprocessed(self, text): + if token == Punctuation and value == ".": + token = Operator + yield index, token, value + + +class ChaiscriptLexer(RegexLexer): + """ + For ChaiScript source code. + + .. versionadded:: 2.0 + """ + + name = 'ChaiScript' + url = 'http://chaiscript.com/' + aliases = ['chaiscript', 'chai'] + filenames = ['*.chai'] + mimetypes = ['text/x-chaiscript', 'application/x-chaiscript'] + + flags = re.DOTALL | re.MULTILINE + + tokens = { + 'commentsandwhitespace': [ + (r'\s+', Text), + (r'//.*?\n', Comment.Single), + (r'/\*.*?\*/', Comment.Multiline), + (r'^\#.*?\n', Comment.Single) + ], + 'slashstartsregex': [ + include('commentsandwhitespace'), + (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/' + r'([gim]+\b|\B)', String.Regex, '#pop'), + (r'(?=/)', Text, ('#pop', 'badregex')), + default('#pop') + ], + 'badregex': [ + (r'\n', Text, '#pop') + ], + 'root': [ + include('commentsandwhitespace'), + (r'\n', Text), + (r'[^\S\n]+', Text), + (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|\.\.' + r'(<<|>>>?|==?|!=?|[-<>+*%&|^/])=?', Operator, 'slashstartsregex'), + (r'[{(\[;,]', Punctuation, 'slashstartsregex'), + (r'[})\].]', Punctuation), + (r'[=+\-*/]', Operator), + (r'(for|in|while|do|break|return|continue|if|else|' + r'throw|try|catch' + r')\b', Keyword, 'slashstartsregex'), + (r'(var)\b', Keyword.Declaration, 'slashstartsregex'), + (r'(attr|def|fun)\b', Keyword.Reserved), + (r'(true|false)\b', Keyword.Constant), + (r'(eval|throw)\b', Name.Builtin), + (r'`\S+`', Name.Builtin), + (r'[$a-zA-Z_]\w*', Name.Other), + (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), + (r'0x[0-9a-fA-F]+', Number.Hex), + (r'[0-9]+', Number.Integer), + (r'"', String.Double, 'dqstring'), + (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single), + ], + 'dqstring': [ + (r'\$\{[^"}]+?\}', String.Interpol), + (r'\$', String.Double), + (r'\\\\', String.Double), + (r'\\"', String.Double), + (r'[^\\"$]+', String.Double), + (r'"', String.Double, '#pop'), + ], + } + + +class LSLLexer(RegexLexer): + """ + For Second Life's Linden Scripting Language source code. + + .. versionadded:: 2.0 + """ + + name = 'LSL' + aliases = ['lsl'] + filenames = ['*.lsl'] + mimetypes = ['text/x-lsl'] + + flags = re.MULTILINE + + lsl_keywords = r'\b(?:do|else|for|if|jump|return|while)\b' + lsl_types = r'\b(?:float|integer|key|list|quaternion|rotation|string|vector)\b' + lsl_states = r'\b(?:(?:state)\s+\w+|default)\b' + lsl_events = r'\b(?:state_(?:entry|exit)|touch(?:_(?:start|end))?|(?:land_)?collision(?:_(?:start|end))?|timer|listen|(?:no_)?sensor|control|(?:not_)?at_(?:rot_)?target|money|email|run_time_permissions|changed|attach|dataserver|moving_(?:start|end)|link_message|(?:on|object)_rez|remote_data|http_re(?:sponse|quest)|path_update|transaction_result)\b' + lsl_functions_builtin = r'\b(?:ll(?:ReturnObjectsBy(?:ID|Owner)|Json(?:2List|[GS]etValue|ValueType)|Sin|Cos|Tan|Atan2|Sqrt|Pow|Abs|Fabs|Frand|Floor|Ceil|Round|Vec(?:Mag|Norm|Dist)|Rot(?:Between|2(?:Euler|Fwd|Left|Up))|(?:Euler|Axes)2Rot|Whisper|(?:Region|Owner)?Say|Shout|Listen(?:Control|Remove)?|Sensor(?:Repeat|Remove)?|Detected(?:Name|Key|Owner|Type|Pos|Vel|Grab|Rot|Group|LinkNumber)|Die|Ground|Wind|(?:[GS]et)(?:AnimationOverride|MemoryLimit|PrimMediaParams|ParcelMusicURL|Object(?:Desc|Name)|PhysicsMaterial|Status|Scale|Color|Alpha|Texture|Pos|Rot|Force|Torque)|ResetAnimationOverride|(?:Scale|Offset|Rotate)Texture|(?:Rot)?Target(?:Remove)?|(?:Stop)?MoveToTarget|Apply(?:Rotational)?Impulse|Set(?:KeyframedMotion|ContentType|RegionPos|(?:Angular)?Velocity|Buoyancy|HoverHeight|ForceAndTorque|TimerEvent|ScriptState|Damage|TextureAnim|Sound(?:Queueing|Radius)|Vehicle(?:Type|(?:Float|Vector|Rotation)Param)|(?:Touch|Sit)?Text|Camera(?:Eye|At)Offset|PrimitiveParams|ClickAction|Link(?:Alpha|Color|PrimitiveParams(?:Fast)?|Texture(?:Anim)?|Camera|Media)|RemoteScriptAccessPin|PayPrice|LocalRot)|ScaleByFactor|Get(?:(?:Max|Min)ScaleFactor|ClosestNavPoint|StaticPath|SimStats|Env|PrimitiveParams|Link(?:PrimitiveParams|Number(?:OfSides)?|Key|Name|Media)|HTTPHeader|FreeURLs|Object(?:Details|PermMask|PrimCount)|Parcel(?:MaxPrims|Details|Prim(?:Count|Owners))|Attached|(?:SPMax|Free|Used)Memory|Region(?:Name|TimeDilation|FPS|Corner|AgentCount)|Root(?:Position|Rotation)|UnixTime|(?:Parcel|Region)Flags|(?:Wall|GMT)clock|SimulatorHostname|BoundingBox|GeometricCenter|Creator|NumberOf(?:Prims|NotecardLines|Sides)|Animation(?:List)?|(?:Camera|Local)(?:Pos|Rot)|Vel|Accel|Omega|Time(?:stamp|OfDay)|(?:Object|CenterOf)?Mass|MassMKS|Energy|Owner|(?:Owner)?Key|SunDirection|Texture(?:Offset|Scale|Rot)|Inventory(?:Number|Name|Key|Type|Creator|PermMask)|Permissions(?:Key)?|StartParameter|List(?:Length|EntryType)|Date|Agent(?:Size|Info|Language|List)|LandOwnerAt|NotecardLine|Script(?:Name|State))|(?:Get|Reset|GetAndReset)Time|PlaySound(?:Slave)?|LoopSound(?:Master|Slave)?|(?:Trigger|Stop|Preload)Sound|(?:(?:Get|Delete)Sub|Insert)String|To(?:Upper|Lower)|Give(?:InventoryList|Money)|RezObject|(?:Stop)?LookAt|Sleep|CollisionFilter|(?:Take|Release)Controls|DetachFromAvatar|AttachToAvatar(?:Temp)?|InstantMessage|(?:GetNext)?Email|StopHover|MinEventDelay|RotLookAt|String(?:Length|Trim)|(?:Start|Stop)Animation|TargetOmega|RequestPermissions|(?:Create|Break)Link|BreakAllLinks|(?:Give|Remove)Inventory|Water|PassTouches|Request(?:Agent|Inventory)Data|TeleportAgent(?:Home|GlobalCoords)?|ModifyLand|CollisionSound|ResetScript|MessageLinked|PushObject|PassCollisions|AxisAngle2Rot|Rot2(?:Axis|Angle)|A(?:cos|sin)|AngleBetween|AllowInventoryDrop|SubStringIndex|List2(?:CSV|Integer|Json|Float|String|Key|Vector|Rot|List(?:Strided)?)|DeleteSubList|List(?:Statistics|Sort|Randomize|(?:Insert|Find|Replace)List)|EdgeOfWorld|AdjustSoundVolume|Key2Name|TriggerSoundLimited|EjectFromLand|(?:CSV|ParseString)2List|OverMyLand|SameGroup|UnSit|Ground(?:Slope|Normal|Contour)|GroundRepel|(?:Set|Remove)VehicleFlags|(?:AvatarOn)?(?:Link)?SitTarget|Script(?:Danger|Profiler)|Dialog|VolumeDetect|ResetOtherScript|RemoteLoadScriptPin|(?:Open|Close)RemoteDataChannel|SendRemoteData|RemoteDataReply|(?:Integer|String)ToBase64|XorBase64|Log(?:10)?|Base64To(?:String|Integer)|ParseStringKeepNulls|RezAtRoot|RequestSimulatorData|ForceMouselook|(?:Load|Release|(?:E|Une)scape)URL|ParcelMedia(?:CommandList|Query)|ModPow|MapDestination|(?:RemoveFrom|AddTo|Reset)Land(?:Pass|Ban)List|(?:Set|Clear)CameraParams|HTTP(?:Request|Response)|TextBox|DetectedTouch(?:UV|Face|Pos|(?:N|Bin)ormal|ST)|(?:MD5|SHA1|DumpList2)String|Request(?:Secure)?URL|Clear(?:Prim|Link)Media|(?:Link)?ParticleSystem|(?:Get|Request)(?:Username|DisplayName)|RegionSayTo|CastRay|GenerateKey|TransferLindenDollars|ManageEstateAccess|(?:Create|Delete)Character|ExecCharacterCmd|Evade|FleeFrom|NavigateTo|PatrolPoints|Pursue|UpdateCharacter|WanderWithin))\b' + lsl_constants_float = r'\b(?:DEG_TO_RAD|PI(?:_BY_TWO)?|RAD_TO_DEG|SQRT2|TWO_PI)\b' + lsl_constants_integer = r'\b(?:JSON_APPEND|STATUS_(?:PHYSICS|ROTATE_[XYZ]|PHANTOM|SANDBOX|BLOCK_GRAB(?:_OBJECT)?|(?:DIE|RETURN)_AT_EDGE|CAST_SHADOWS|OK|MALFORMED_PARAMS|TYPE_MISMATCH|BOUNDS_ERROR|NOT_(?:FOUND|SUPPORTED)|INTERNAL_ERROR|WHITELIST_FAILED)|AGENT(?:_(?:BY_(?:LEGACY_|USER)NAME|FLYING|ATTACHMENTS|SCRIPTED|MOUSELOOK|SITTING|ON_OBJECT|AWAY|WALKING|IN_AIR|TYPING|CROUCHING|BUSY|ALWAYS_RUN|AUTOPILOT|LIST_(?:PARCEL(?:_OWNER)?|REGION)))?|CAMERA_(?:PITCH|DISTANCE|BEHINDNESS_(?:ANGLE|LAG)|(?:FOCUS|POSITION)(?:_(?:THRESHOLD|LOCKED|LAG))?|FOCUS_OFFSET|ACTIVE)|ANIM_ON|LOOP|REVERSE|PING_PONG|SMOOTH|ROTATE|SCALE|ALL_SIDES|LINK_(?:ROOT|SET|ALL_(?:OTHERS|CHILDREN)|THIS)|ACTIVE|PASSIVE|SCRIPTED|CONTROL_(?:FWD|BACK|(?:ROT_)?(?:LEFT|RIGHT)|UP|DOWN|(?:ML_)?LBUTTON)|PERMISSION_(?:RETURN_OBJECTS|DEBIT|OVERRIDE_ANIMATIONS|SILENT_ESTATE_MANAGEMENT|TAKE_CONTROLS|TRIGGER_ANIMATION|ATTACH|CHANGE_LINKS|(?:CONTROL|TRACK)_CAMERA|TELEPORT)|INVENTORY_(?:TEXTURE|SOUND|OBJECT|SCRIPT|LANDMARK|CLOTHING|NOTECARD|BODYPART|ANIMATION|GESTURE|ALL|NONE)|CHANGED_(?:INVENTORY|COLOR|SHAPE|SCALE|TEXTURE|LINK|ALLOWED_DROP|OWNER|REGION(?:_START)?|TELEPORT|MEDIA)|OBJECT_(?:(?:PHYSICS|SERVER|STREAMING)_COST|UNKNOWN_DETAIL|CHARACTER_TIME|PHANTOM|PHYSICS|TEMP_ON_REZ|NAME|DESC|POS|PRIM_EQUIVALENCE|RETURN_(?:PARCEL(?:_OWNER)?|REGION)|ROO?T|VELOCITY|OWNER|GROUP|CREATOR|ATTACHED_POINT|RENDER_WEIGHT|PATHFINDING_TYPE|(?:RUNNING|TOTAL)_SCRIPT_COUNT|SCRIPT_(?:MEMORY|TIME))|TYPE_(?:INTEGER|FLOAT|STRING|KEY|VECTOR|ROTATION|INVALID)|(?:DEBUG|PUBLIC)_CHANNEL|ATTACH_(?:AVATAR_CENTER|CHEST|HEAD|BACK|PELVIS|MOUTH|CHIN|NECK|NOSE|BELLY|[LR](?:SHOULDER|HAND|FOOT|EAR|EYE|[UL](?:ARM|LEG)|HIP)|(?:LEFT|RIGHT)_PEC|HUD_(?:CENTER_[12]|TOP_(?:RIGHT|CENTER|LEFT)|BOTTOM(?:_(?:RIGHT|LEFT))?))|LAND_(?:LEVEL|RAISE|LOWER|SMOOTH|NOISE|REVERT)|DATA_(?:ONLINE|NAME|BORN|SIM_(?:POS|STATUS|RATING)|PAYINFO)|PAYMENT_INFO_(?:ON_FILE|USED)|REMOTE_DATA_(?:CHANNEL|REQUEST|REPLY)|PSYS_(?:PART_(?:BF_(?:ZERO|ONE(?:_MINUS_(?:DEST_COLOR|SOURCE_(ALPHA|COLOR)))?|DEST_COLOR|SOURCE_(ALPHA|COLOR))|BLEND_FUNC_(DEST|SOURCE)|FLAGS|(?:START|END)_(?:COLOR|ALPHA|SCALE|GLOW)|MAX_AGE|(?:RIBBON|WIND|INTERP_(?:COLOR|SCALE)|BOUNCE|FOLLOW_(?:SRC|VELOCITY)|TARGET_(?:POS|LINEAR)|EMISSIVE)_MASK)|SRC_(?:MAX_AGE|PATTERN|ANGLE_(?:BEGIN|END)|BURST_(?:RATE|PART_COUNT|RADIUS|SPEED_(?:MIN|MAX))|ACCEL|TEXTURE|TARGET_KEY|OMEGA|PATTERN_(?:DROP|EXPLODE|ANGLE(?:_CONE(?:_EMPTY)?)?)))|VEHICLE_(?:REFERENCE_FRAME|TYPE_(?:NONE|SLED|CAR|BOAT|AIRPLANE|BALLOON)|(?:LINEAR|ANGULAR)_(?:FRICTION_TIMESCALE|MOTOR_DIRECTION)|LINEAR_MOTOR_OFFSET|HOVER_(?:HEIGHT|EFFICIENCY|TIMESCALE)|BUOYANCY|(?:LINEAR|ANGULAR)_(?:DEFLECTION_(?:EFFICIENCY|TIMESCALE)|MOTOR_(?:DECAY_)?TIMESCALE)|VERTICAL_ATTRACTION_(?:EFFICIENCY|TIMESCALE)|BANKING_(?:EFFICIENCY|MIX|TIMESCALE)|FLAG_(?:NO_DEFLECTION_UP|LIMIT_(?:ROLL_ONLY|MOTOR_UP)|HOVER_(?:(?:WATER|TERRAIN|UP)_ONLY|GLOBAL_HEIGHT)|MOUSELOOK_(?:STEER|BANK)|CAMERA_DECOUPLED))|PRIM_(?:TYPE(?:_(?:BOX|CYLINDER|PRISM|SPHERE|TORUS|TUBE|RING|SCULPT))?|HOLE_(?:DEFAULT|CIRCLE|SQUARE|TRIANGLE)|MATERIAL(?:_(?:STONE|METAL|GLASS|WOOD|FLESH|PLASTIC|RUBBER))?|SHINY_(?:NONE|LOW|MEDIUM|HIGH)|BUMP_(?:NONE|BRIGHT|DARK|WOOD|BARK|BRICKS|CHECKER|CONCRETE|TILE|STONE|DISKS|GRAVEL|BLOBS|SIDING|LARGETILE|STUCCO|SUCTION|WEAVE)|TEXGEN_(?:DEFAULT|PLANAR)|SCULPT_(?:TYPE_(?:SPHERE|TORUS|PLANE|CYLINDER|MASK)|FLAG_(?:MIRROR|INVERT))|PHYSICS(?:_(?:SHAPE_(?:CONVEX|NONE|PRIM|TYPE)))?|(?:POS|ROT)_LOCAL|SLICE|TEXT|FLEXIBLE|POINT_LIGHT|TEMP_ON_REZ|PHANTOM|POSITION|SIZE|ROTATION|TEXTURE|NAME|OMEGA|DESC|LINK_TARGET|COLOR|BUMP_SHINY|FULLBRIGHT|TEXGEN|GLOW|MEDIA_(?:ALT_IMAGE_ENABLE|CONTROLS|(?:CURRENT|HOME)_URL|AUTO_(?:LOOP|PLAY|SCALE|ZOOM)|FIRST_CLICK_INTERACT|(?:WIDTH|HEIGHT)_PIXELS|WHITELIST(?:_ENABLE)?|PERMS_(?:INTERACT|CONTROL)|PARAM_MAX|CONTROLS_(?:STANDARD|MINI)|PERM_(?:NONE|OWNER|GROUP|ANYONE)|MAX_(?:URL_LENGTH|WHITELIST_(?:SIZE|COUNT)|(?:WIDTH|HEIGHT)_PIXELS)))|MASK_(?:BASE|OWNER|GROUP|EVERYONE|NEXT)|PERM_(?:TRANSFER|MODIFY|COPY|MOVE|ALL)|PARCEL_(?:MEDIA_COMMAND_(?:STOP|PAUSE|PLAY|LOOP|TEXTURE|URL|TIME|AGENT|UNLOAD|AUTO_ALIGN|TYPE|SIZE|DESC|LOOP_SET)|FLAG_(?:ALLOW_(?:FLY|(?:GROUP_)?SCRIPTS|LANDMARK|TERRAFORM|DAMAGE|CREATE_(?:GROUP_)?OBJECTS)|USE_(?:ACCESS_(?:GROUP|LIST)|BAN_LIST|LAND_PASS_LIST)|LOCAL_SOUND_ONLY|RESTRICT_PUSHOBJECT|ALLOW_(?:GROUP|ALL)_OBJECT_ENTRY)|COUNT_(?:TOTAL|OWNER|GROUP|OTHER|SELECTED|TEMP)|DETAILS_(?:NAME|DESC|OWNER|GROUP|AREA|ID|SEE_AVATARS))|LIST_STAT_(?:MAX|MIN|MEAN|MEDIAN|STD_DEV|SUM(?:_SQUARES)?|NUM_COUNT|GEOMETRIC_MEAN|RANGE)|PAY_(?:HIDE|DEFAULT)|REGION_FLAG_(?:ALLOW_DAMAGE|FIXED_SUN|BLOCK_TERRAFORM|SANDBOX|DISABLE_(?:COLLISIONS|PHYSICS)|BLOCK_FLY|ALLOW_DIRECT_TELEPORT|RESTRICT_PUSHOBJECT)|HTTP_(?:METHOD|MIMETYPE|BODY_(?:MAXLENGTH|TRUNCATED)|CUSTOM_HEADER|PRAGMA_NO_CACHE|VERBOSE_THROTTLE|VERIFY_CERT)|STRING_(?:TRIM(?:_(?:HEAD|TAIL))?)|CLICK_ACTION_(?:NONE|TOUCH|SIT|BUY|PAY|OPEN(?:_MEDIA)?|PLAY|ZOOM)|TOUCH_INVALID_FACE|PROFILE_(?:NONE|SCRIPT_MEMORY)|RC_(?:DATA_FLAGS|DETECT_PHANTOM|GET_(?:LINK_NUM|NORMAL|ROOT_KEY)|MAX_HITS|REJECT_(?:TYPES|AGENTS|(?:NON)?PHYSICAL|LAND))|RCERR_(?:CAST_TIME_EXCEEDED|SIM_PERF_LOW|UNKNOWN)|ESTATE_ACCESS_(?:ALLOWED_(?:AGENT|GROUP)_(?:ADD|REMOVE)|BANNED_AGENT_(?:ADD|REMOVE))|DENSITY|FRICTION|RESTITUTION|GRAVITY_MULTIPLIER|KFM_(?:COMMAND|CMD_(?:PLAY|STOP|PAUSE|SET_MODE)|MODE|FORWARD|LOOP|PING_PONG|REVERSE|DATA|ROTATION|TRANSLATION)|ERR_(?:GENERIC|PARCEL_PERMISSIONS|MALFORMED_PARAMS|RUNTIME_PERMISSIONS|THROTTLED)|CHARACTER_(?:CMD_(?:(?:SMOOTH_)?STOP|JUMP)|DESIRED_(?:TURN_)?SPEED|RADIUS|STAY_WITHIN_PARCEL|LENGTH|ORIENTATION|ACCOUNT_FOR_SKIPPED_FRAMES|AVOIDANCE_MODE|TYPE(?:_(?:[A-D]|NONE))?|MAX_(?:DECEL|TURN_RADIUS|(?:ACCEL|SPEED)))|PURSUIT_(?:OFFSET|FUZZ_FACTOR|GOAL_TOLERANCE|INTERCEPT)|REQUIRE_LINE_OF_SIGHT|FORCE_DIRECT_PATH|VERTICAL|HORIZONTAL|AVOID_(?:CHARACTERS|DYNAMIC_OBSTACLES|NONE)|PU_(?:EVADE_(?:HIDDEN|SPOTTED)|FAILURE_(?:DYNAMIC_PATHFINDING_DISABLED|INVALID_(?:GOAL|START)|NO_(?:NAVMESH|VALID_DESTINATION)|OTHER|TARGET_GONE|(?:PARCEL_)?UNREACHABLE)|(?:GOAL|SLOWDOWN_DISTANCE)_REACHED)|TRAVERSAL_TYPE(?:_(?:FAST|NONE|SLOW))?|CONTENT_TYPE_(?:ATOM|FORM|HTML|JSON|LLSD|RSS|TEXT|XHTML|XML)|GCNP_(?:RADIUS|STATIC)|(?:PATROL|WANDER)_PAUSE_AT_WAYPOINTS|OPT_(?:AVATAR|CHARACTER|EXCLUSION_VOLUME|LEGACY_LINKSET|MATERIAL_VOLUME|OTHER|STATIC_OBSTACLE|WALKABLE)|SIM_STAT_PCT_CHARS_STEPPED)\b' + lsl_constants_integer_boolean = r'\b(?:FALSE|TRUE)\b' + lsl_constants_rotation = r'\b(?:ZERO_ROTATION)\b' + lsl_constants_string = r'\b(?:EOF|JSON_(?:ARRAY|DELETE|FALSE|INVALID|NULL|NUMBER|OBJECT|STRING|TRUE)|NULL_KEY|TEXTURE_(?:BLANK|DEFAULT|MEDIA|PLYWOOD|TRANSPARENT)|URL_REQUEST_(?:GRANTED|DENIED))\b' + lsl_constants_vector = r'\b(?:TOUCH_INVALID_(?:TEXCOORD|VECTOR)|ZERO_VECTOR)\b' + lsl_invalid_broken = r'\b(?:LAND_(?:LARGE|MEDIUM|SMALL)_BRUSH)\b' + lsl_invalid_deprecated = r'\b(?:ATTACH_[LR]PEC|DATA_RATING|OBJECT_ATTACHMENT_(?:GEOMETRY_BYTES|SURFACE_AREA)|PRIM_(?:CAST_SHADOWS|MATERIAL_LIGHT|TYPE_LEGACY)|PSYS_SRC_(?:INNER|OUTER)ANGLE|VEHICLE_FLAG_NO_FLY_UP|ll(?:Cloud|Make(?:Explosion|Fountain|Smoke|Fire)|RemoteDataSetRegion|Sound(?:Preload)?|XorBase64Strings(?:Correct)?))\b' + lsl_invalid_illegal = r'\b(?:event)\b' + lsl_invalid_unimplemented = r'\b(?:CHARACTER_(?:MAX_ANGULAR_(?:ACCEL|SPEED)|TURN_SPEED_MULTIPLIER)|PERMISSION_(?:CHANGE_(?:JOINTS|PERMISSIONS)|RELEASE_OWNERSHIP|REMAP_CONTROLS)|PRIM_PHYSICS_MATERIAL|PSYS_SRC_OBJ_REL_MASK|ll(?:CollisionSprite|(?:Stop)?PointAt|(?:(?:Refresh|Set)Prim)URL|(?:Take|Release)Camera|RemoteLoadScript))\b' + lsl_reserved_godmode = r'\b(?:ll(?:GodLikeRezObject|Set(?:Inventory|Object)PermMask))\b' + lsl_reserved_log = r'\b(?:print)\b' + lsl_operators = r'\+\+|\-\-|<<|>>|&&?|\|\|?|\^|~|[!%<>=*+\-/]=?' + + tokens = { + 'root': + [ + (r'//.*?\n', Comment.Single), + (r'/\*', Comment.Multiline, 'comment'), + (r'"', String.Double, 'string'), + (lsl_keywords, Keyword), + (lsl_types, Keyword.Type), + (lsl_states, Name.Class), + (lsl_events, Name.Builtin), + (lsl_functions_builtin, Name.Function), + (lsl_constants_float, Keyword.Constant), + (lsl_constants_integer, Keyword.Constant), + (lsl_constants_integer_boolean, Keyword.Constant), + (lsl_constants_rotation, Keyword.Constant), + (lsl_constants_string, Keyword.Constant), + (lsl_constants_vector, Keyword.Constant), + (lsl_invalid_broken, Error), + (lsl_invalid_deprecated, Error), + (lsl_invalid_illegal, Error), + (lsl_invalid_unimplemented, Error), + (lsl_reserved_godmode, Keyword.Reserved), + (lsl_reserved_log, Keyword.Reserved), + (r'\b([a-zA-Z_]\w*)\b', Name.Variable), + (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d*', Number.Float), + (r'(\d+\.\d*|\.\d+)', Number.Float), + (r'0[xX][0-9a-fA-F]+', Number.Hex), + (r'\d+', Number.Integer), + (lsl_operators, Operator), + (r':=?', Error), + (r'[,;{}()\[\]]', Punctuation), + (r'\n+', Whitespace), + (r'\s+', Whitespace) + ], + 'comment': + [ + (r'[^*/]+', Comment.Multiline), + (r'/\*', Comment.Multiline, '#push'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[*/]', Comment.Multiline) + ], + 'string': + [ + (r'\\([nt"\\])', String.Escape), + (r'"', String.Double, '#pop'), + (r'\\.', Error), + (r'[^"\\]+', String.Double), + ] + } + + +class AppleScriptLexer(RegexLexer): + """ + For AppleScript source code, + including `AppleScript Studio + `_. + Contributed by Andreas Amann . + + .. versionadded:: 1.0 + """ + + name = 'AppleScript' + url = 'https://developer.apple.com/library/archive/documentation/AppleScript/Conceptual/AppleScriptLangGuide/introduction/ASLR_intro.html' + aliases = ['applescript'] + filenames = ['*.applescript'] + + flags = re.MULTILINE | re.DOTALL + + Identifiers = r'[a-zA-Z]\w*' + + # XXX: use words() for all of these + Literals = ('AppleScript', 'current application', 'false', 'linefeed', + 'missing value', 'pi', 'quote', 'result', 'return', 'space', + 'tab', 'text item delimiters', 'true', 'version') + Classes = ('alias ', 'application ', 'boolean ', 'class ', 'constant ', + 'date ', 'file ', 'integer ', 'list ', 'number ', 'POSIX file ', + 'real ', 'record ', 'reference ', 'RGB color ', 'script ', + 'text ', 'unit types', '(?:Unicode )?text', 'string') + BuiltIn = ('attachment', 'attribute run', 'character', 'day', 'month', + 'paragraph', 'word', 'year') + HandlerParams = ('about', 'above', 'against', 'apart from', 'around', + 'aside from', 'at', 'below', 'beneath', 'beside', + 'between', 'for', 'given', 'instead of', 'on', 'onto', + 'out of', 'over', 'since') + Commands = ('ASCII (character|number)', 'activate', 'beep', 'choose URL', + 'choose application', 'choose color', 'choose file( name)?', + 'choose folder', 'choose from list', + 'choose remote application', 'clipboard info', + 'close( access)?', 'copy', 'count', 'current date', 'delay', + 'delete', 'display (alert|dialog)', 'do shell script', + 'duplicate', 'exists', 'get eof', 'get volume settings', + 'info for', 'launch', 'list (disks|folder)', 'load script', + 'log', 'make', 'mount volume', 'new', 'offset', + 'open( (for access|location))?', 'path to', 'print', 'quit', + 'random number', 'read', 'round', 'run( script)?', + 'say', 'scripting components', + 'set (eof|the clipboard to|volume)', 'store script', + 'summarize', 'system attribute', 'system info', + 'the clipboard', 'time to GMT', 'write', 'quoted form') + References = ('(in )?back of', '(in )?front of', '[0-9]+(st|nd|rd|th)', + 'first', 'second', 'third', 'fourth', 'fifth', 'sixth', + 'seventh', 'eighth', 'ninth', 'tenth', 'after', 'back', + 'before', 'behind', 'every', 'front', 'index', 'last', + 'middle', 'some', 'that', 'through', 'thru', 'where', 'whose') + Operators = ("and", "or", "is equal", "equals", "(is )?equal to", "is not", + "isn't", "isn't equal( to)?", "is not equal( to)?", + "doesn't equal", "does not equal", "(is )?greater than", + "comes after", "is not less than or equal( to)?", + "isn't less than or equal( to)?", "(is )?less than", + "comes before", "is not greater than or equal( to)?", + "isn't greater than or equal( to)?", + "(is )?greater than or equal( to)?", "is not less than", + "isn't less than", "does not come before", + "doesn't come before", "(is )?less than or equal( to)?", + "is not greater than", "isn't greater than", + "does not come after", "doesn't come after", "starts? with", + "begins? with", "ends? with", "contains?", "does not contain", + "doesn't contain", "is in", "is contained by", "is not in", + "is not contained by", "isn't contained by", "div", "mod", + "not", "(a )?(ref( to)?|reference to)", "is", "does") + Control = ('considering', 'else', 'error', 'exit', 'from', 'if', + 'ignoring', 'in', 'repeat', 'tell', 'then', 'times', 'to', + 'try', 'until', 'using terms from', 'while', 'whith', + 'with timeout( of)?', 'with transaction', 'by', 'continue', + 'end', 'its?', 'me', 'my', 'return', 'of', 'as') + Declarations = ('global', 'local', 'prop(erty)?', 'set', 'get') + Reserved = ('but', 'put', 'returning', 'the') + StudioClasses = ('action cell', 'alert reply', 'application', 'box', + 'browser( cell)?', 'bundle', 'button( cell)?', 'cell', + 'clip view', 'color well', 'color-panel', + 'combo box( item)?', 'control', + 'data( (cell|column|item|row|source))?', 'default entry', + 'dialog reply', 'document', 'drag info', 'drawer', + 'event', 'font(-panel)?', 'formatter', + 'image( (cell|view))?', 'matrix', 'menu( item)?', 'item', + 'movie( view)?', 'open-panel', 'outline view', 'panel', + 'pasteboard', 'plugin', 'popup button', + 'progress indicator', 'responder', 'save-panel', + 'scroll view', 'secure text field( cell)?', 'slider', + 'sound', 'split view', 'stepper', 'tab view( item)?', + 'table( (column|header cell|header view|view))', + 'text( (field( cell)?|view))?', 'toolbar( item)?', + 'user-defaults', 'view', 'window') + StudioEvents = ('accept outline drop', 'accept table drop', 'action', + 'activated', 'alert ended', 'awake from nib', 'became key', + 'became main', 'begin editing', 'bounds changed', + 'cell value', 'cell value changed', 'change cell value', + 'change item value', 'changed', 'child of item', + 'choose menu item', 'clicked', 'clicked toolbar item', + 'closed', 'column clicked', 'column moved', + 'column resized', 'conclude drop', 'data representation', + 'deminiaturized', 'dialog ended', 'document nib name', + 'double clicked', 'drag( (entered|exited|updated))?', + 'drop', 'end editing', 'exposed', 'idle', 'item expandable', + 'item value', 'item value changed', 'items changed', + 'keyboard down', 'keyboard up', 'launched', + 'load data representation', 'miniaturized', 'mouse down', + 'mouse dragged', 'mouse entered', 'mouse exited', + 'mouse moved', 'mouse up', 'moved', + 'number of browser rows', 'number of items', + 'number of rows', 'open untitled', 'opened', 'panel ended', + 'parameters updated', 'plugin loaded', 'prepare drop', + 'prepare outline drag', 'prepare outline drop', + 'prepare table drag', 'prepare table drop', + 'read from file', 'resigned active', 'resigned key', + 'resigned main', 'resized( sub views)?', + 'right mouse down', 'right mouse dragged', + 'right mouse up', 'rows changed', 'scroll wheel', + 'selected tab view item', 'selection changed', + 'selection changing', 'should begin editing', + 'should close', 'should collapse item', + 'should end editing', 'should expand item', + 'should open( untitled)?', + 'should quit( after last window closed)?', + 'should select column', 'should select item', + 'should select row', 'should select tab view item', + 'should selection change', 'should zoom', 'shown', + 'update menu item', 'update parameters', + 'update toolbar item', 'was hidden', 'was miniaturized', + 'will become active', 'will close', 'will dismiss', + 'will display browser cell', 'will display cell', + 'will display item cell', 'will display outline cell', + 'will finish launching', 'will hide', 'will miniaturize', + 'will move', 'will open', 'will pop up', 'will quit', + 'will resign active', 'will resize( sub views)?', + 'will select tab view item', 'will show', 'will zoom', + 'write to file', 'zoomed') + StudioCommands = ('animate', 'append', 'call method', 'center', + 'close drawer', 'close panel', 'display', + 'display alert', 'display dialog', 'display panel', 'go', + 'hide', 'highlight', 'increment', 'item for', + 'load image', 'load movie', 'load nib', 'load panel', + 'load sound', 'localized string', 'lock focus', 'log', + 'open drawer', 'path for', 'pause', 'perform action', + 'play', 'register', 'resume', 'scroll', 'select( all)?', + 'show', 'size to fit', 'start', 'step back', + 'step forward', 'stop', 'synchronize', 'unlock focus', + 'update') + StudioProperties = ('accepts arrow key', 'action method', 'active', + 'alignment', 'allowed identifiers', + 'allows branch selection', 'allows column reordering', + 'allows column resizing', 'allows column selection', + 'allows customization', + 'allows editing text attributes', + 'allows empty selection', 'allows mixed state', + 'allows multiple selection', 'allows reordering', + 'allows undo', 'alpha( value)?', 'alternate image', + 'alternate increment value', 'alternate title', + 'animation delay', 'associated file name', + 'associated object', 'auto completes', 'auto display', + 'auto enables items', 'auto repeat', + 'auto resizes( outline column)?', + 'auto save expanded items', 'auto save name', + 'auto save table columns', 'auto saves configuration', + 'auto scroll', 'auto sizes all columns to fit', + 'auto sizes cells', 'background color', 'bezel state', + 'bezel style', 'bezeled', 'border rect', 'border type', + 'bordered', 'bounds( rotation)?', 'box type', + 'button returned', 'button type', + 'can choose directories', 'can choose files', + 'can draw', 'can hide', + 'cell( (background color|size|type))?', 'characters', + 'class', 'click count', 'clicked( data)? column', + 'clicked data item', 'clicked( data)? row', + 'closeable', 'collating', 'color( (mode|panel))', + 'command key down', 'configuration', + 'content(s| (size|view( margins)?))?', 'context', + 'continuous', 'control key down', 'control size', + 'control tint', 'control view', + 'controller visible', 'coordinate system', + 'copies( on scroll)?', 'corner view', 'current cell', + 'current column', 'current( field)? editor', + 'current( menu)? item', 'current row', + 'current tab view item', 'data source', + 'default identifiers', 'delta (x|y|z)', + 'destination window', 'directory', 'display mode', + 'displayed cell', 'document( (edited|rect|view))?', + 'double value', 'dragged column', 'dragged distance', + 'dragged items', 'draws( cell)? background', + 'draws grid', 'dynamically scrolls', 'echos bullets', + 'edge', 'editable', 'edited( data)? column', + 'edited data item', 'edited( data)? row', 'enabled', + 'enclosing scroll view', 'ending page', + 'error handling', 'event number', 'event type', + 'excluded from windows menu', 'executable path', + 'expanded', 'fax number', 'field editor', 'file kind', + 'file name', 'file type', 'first responder', + 'first visible column', 'flipped', 'floating', + 'font( panel)?', 'formatter', 'frameworks path', + 'frontmost', 'gave up', 'grid color', 'has data items', + 'has horizontal ruler', 'has horizontal scroller', + 'has parent data item', 'has resize indicator', + 'has shadow', 'has sub menu', 'has vertical ruler', + 'has vertical scroller', 'header cell', 'header view', + 'hidden', 'hides when deactivated', 'highlights by', + 'horizontal line scroll', 'horizontal page scroll', + 'horizontal ruler view', 'horizontally resizable', + 'icon image', 'id', 'identifier', + 'ignores multiple clicks', + 'image( (alignment|dims when disabled|frame style|scaling))?', + 'imports graphics', 'increment value', + 'indentation per level', 'indeterminate', 'index', + 'integer value', 'intercell spacing', 'item height', + 'key( (code|equivalent( modifier)?|window))?', + 'knob thickness', 'label', 'last( visible)? column', + 'leading offset', 'leaf', 'level', 'line scroll', + 'loaded', 'localized sort', 'location', 'loop mode', + 'main( (bunde|menu|window))?', 'marker follows cell', + 'matrix mode', 'maximum( content)? size', + 'maximum visible columns', + 'menu( form representation)?', 'miniaturizable', + 'miniaturized', 'minimized image', 'minimized title', + 'minimum column width', 'minimum( content)? size', + 'modal', 'modified', 'mouse down state', + 'movie( (controller|file|rect))?', 'muted', 'name', + 'needs display', 'next state', 'next text', + 'number of tick marks', 'only tick mark values', + 'opaque', 'open panel', 'option key down', + 'outline table column', 'page scroll', 'pages across', + 'pages down', 'palette label', 'pane splitter', + 'parent data item', 'parent window', 'pasteboard', + 'path( (names|separator))?', 'playing', + 'plays every frame', 'plays selection only', 'position', + 'preferred edge', 'preferred type', 'pressure', + 'previous text', 'prompt', 'properties', + 'prototype cell', 'pulls down', 'rate', + 'released when closed', 'repeated', + 'requested print time', 'required file type', + 'resizable', 'resized column', 'resource path', + 'returns records', 'reuses columns', 'rich text', + 'roll over', 'row height', 'rulers visible', + 'save panel', 'scripts path', 'scrollable', + 'selectable( identifiers)?', 'selected cell', + 'selected( data)? columns?', 'selected data items?', + 'selected( data)? rows?', 'selected item identifier', + 'selection by rect', 'send action on arrow key', + 'sends action when done editing', 'separates columns', + 'separator item', 'sequence number', 'services menu', + 'shared frameworks path', 'shared support path', + 'sheet', 'shift key down', 'shows alpha', + 'shows state by', 'size( mode)?', + 'smart insert delete enabled', 'sort case sensitivity', + 'sort column', 'sort order', 'sort type', + 'sorted( data rows)?', 'sound', 'source( mask)?', + 'spell checking enabled', 'starting page', 'state', + 'string value', 'sub menu', 'super menu', 'super view', + 'tab key traverses cells', 'tab state', 'tab type', + 'tab view', 'table view', 'tag', 'target( printer)?', + 'text color', 'text container insert', + 'text container origin', 'text returned', + 'tick mark position', 'time stamp', + 'title(d| (cell|font|height|position|rect))?', + 'tool tip', 'toolbar', 'trailing offset', 'transparent', + 'treat packages as directories', 'truncated labels', + 'types', 'unmodified characters', 'update views', + 'use sort indicator', 'user defaults', + 'uses data source', 'uses ruler', + 'uses threaded animation', + 'uses title from previous column', 'value wraps', + 'version', + 'vertical( (line scroll|page scroll|ruler view))?', + 'vertically resizable', 'view', + 'visible( document rect)?', 'volume', 'width', 'window', + 'windows menu', 'wraps', 'zoomable', 'zoomed') + + tokens = { + 'root': [ + (r'\s+', Text), + (r'¬\n', String.Escape), + (r"'s\s+", Text), # This is a possessive, consider moving + (r'(--|#).*?$', Comment), + (r'\(\*', Comment.Multiline, 'comment'), + (r'[(){}!,.:]', Punctuation), + (r'(«)([^»]+)(»)', + bygroups(Text, Name.Builtin, Text)), + (r'\b((?:considering|ignoring)\s*)' + r'(application responses|case|diacriticals|hyphens|' + r'numeric strings|punctuation|white space)', + bygroups(Keyword, Name.Builtin)), + (r'(-|\*|\+|&|≠|>=?|<=?|=|≥|≤|/|÷|\^)', Operator), + (r"\b(%s)\b" % '|'.join(Operators), Operator.Word), + (r'^(\s*(?:on|end)\s+)' + r'(%s)' % '|'.join(StudioEvents[::-1]), + bygroups(Keyword, Name.Function)), + (r'^(\s*)(in|on|script|to)(\s+)', bygroups(Text, Keyword, Text)), + (r'\b(as )(%s)\b' % '|'.join(Classes), + bygroups(Keyword, Name.Class)), + (r'\b(%s)\b' % '|'.join(Literals), Name.Constant), + (r'\b(%s)\b' % '|'.join(Commands), Name.Builtin), + (r'\b(%s)\b' % '|'.join(Control), Keyword), + (r'\b(%s)\b' % '|'.join(Declarations), Keyword), + (r'\b(%s)\b' % '|'.join(Reserved), Name.Builtin), + (r'\b(%s)s?\b' % '|'.join(BuiltIn), Name.Builtin), + (r'\b(%s)\b' % '|'.join(HandlerParams), Name.Builtin), + (r'\b(%s)\b' % '|'.join(StudioProperties), Name.Attribute), + (r'\b(%s)s?\b' % '|'.join(StudioClasses), Name.Builtin), + (r'\b(%s)\b' % '|'.join(StudioCommands), Name.Builtin), + (r'\b(%s)\b' % '|'.join(References), Name.Builtin), + (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double), + (r'\b(%s)\b' % Identifiers, Name.Variable), + (r'[-+]?(\d+\.\d*|\d*\.\d+)(E[-+][0-9]+)?', Number.Float), + (r'[-+]?\d+', Number.Integer), + ], + 'comment': [ + (r'\(\*', Comment.Multiline, '#push'), + (r'\*\)', Comment.Multiline, '#pop'), + ('[^*(]+', Comment.Multiline), + ('[*(]', Comment.Multiline), + ], + } + + +class RexxLexer(RegexLexer): + """ + Rexx is a scripting language available for + a wide range of different platforms with its roots found on mainframe + systems. It is popular for I/O- and data based tasks and can act as glue + language to bind different applications together. + + .. versionadded:: 2.0 + """ + name = 'Rexx' + url = 'http://www.rexxinfo.org/' + aliases = ['rexx', 'arexx'] + filenames = ['*.rexx', '*.rex', '*.rx', '*.arexx'] + mimetypes = ['text/x-rexx'] + flags = re.IGNORECASE + + tokens = { + 'root': [ + (r'\s+', Whitespace), + (r'/\*', Comment.Multiline, 'comment'), + (r'"', String, 'string_double'), + (r"'", String, 'string_single'), + (r'[0-9]+(\.[0-9]+)?(e[+-]?[0-9])?', Number), + (r'([a-z_]\w*)(\s*)(:)(\s*)(procedure)\b', + bygroups(Name.Function, Whitespace, Operator, Whitespace, + Keyword.Declaration)), + (r'([a-z_]\w*)(\s*)(:)', + bygroups(Name.Label, Whitespace, Operator)), + include('function'), + include('keyword'), + include('operator'), + (r'[a-z_]\w*', Text), + ], + 'function': [ + (words(( + 'abbrev', 'abs', 'address', 'arg', 'b2x', 'bitand', 'bitor', 'bitxor', + 'c2d', 'c2x', 'center', 'charin', 'charout', 'chars', 'compare', + 'condition', 'copies', 'd2c', 'd2x', 'datatype', 'date', 'delstr', + 'delword', 'digits', 'errortext', 'form', 'format', 'fuzz', 'insert', + 'lastpos', 'left', 'length', 'linein', 'lineout', 'lines', 'max', + 'min', 'overlay', 'pos', 'queued', 'random', 'reverse', 'right', 'sign', + 'sourceline', 'space', 'stream', 'strip', 'substr', 'subword', 'symbol', + 'time', 'trace', 'translate', 'trunc', 'value', 'verify', 'word', + 'wordindex', 'wordlength', 'wordpos', 'words', 'x2b', 'x2c', 'x2d', + 'xrange'), suffix=r'(\s*)(\()'), + bygroups(Name.Builtin, Whitespace, Operator)), + ], + 'keyword': [ + (r'(address|arg|by|call|do|drop|else|end|exit|for|forever|if|' + r'interpret|iterate|leave|nop|numeric|off|on|options|parse|' + r'pull|push|queue|return|say|select|signal|to|then|trace|until|' + r'while)\b', Keyword.Reserved), + ], + 'operator': [ + (r'(-|//|/|\(|\)|\*\*|\*|\\<<|\\<|\\==|\\=|\\>>|\\>|\\|\|\||\||' + r'&&|&|%|\+|<<=|<<|<=|<>|<|==|=|><|>=|>>=|>>|>|¬<<|¬<|¬==|¬=|' + r'¬>>|¬>|¬|\.|,)', Operator), + ], + 'string_double': [ + (r'[^"\n]+', String), + (r'""', String), + (r'"', String, '#pop'), + (r'\n', Text, '#pop'), # Stray linefeed also terminates strings. + ], + 'string_single': [ + (r'[^\'\n]+', String), + (r'\'\'', String), + (r'\'', String, '#pop'), + (r'\n', Text, '#pop'), # Stray linefeed also terminates strings. + ], + 'comment': [ + (r'[^*]+', Comment.Multiline), + (r'\*/', Comment.Multiline, '#pop'), + (r'\*', Comment.Multiline), + ] + } + + _c = lambda s: re.compile(s, re.MULTILINE) + _ADDRESS_COMMAND_PATTERN = _c(r'^\s*address\s+command\b') + _ADDRESS_PATTERN = _c(r'^\s*address\s+') + _DO_WHILE_PATTERN = _c(r'^\s*do\s+while\b') + _IF_THEN_DO_PATTERN = _c(r'^\s*if\b.+\bthen\s+do\s*$') + _PROCEDURE_PATTERN = _c(r'^\s*([a-z_]\w*)(\s*)(:)(\s*)(procedure)\b') + _ELSE_DO_PATTERN = _c(r'\belse\s+do\s*$') + _PARSE_ARG_PATTERN = _c(r'^\s*parse\s+(upper\s+)?(arg|value)\b') + PATTERNS_AND_WEIGHTS = ( + (_ADDRESS_COMMAND_PATTERN, 0.2), + (_ADDRESS_PATTERN, 0.05), + (_DO_WHILE_PATTERN, 0.1), + (_ELSE_DO_PATTERN, 0.1), + (_IF_THEN_DO_PATTERN, 0.1), + (_PROCEDURE_PATTERN, 0.5), + (_PARSE_ARG_PATTERN, 0.2), + ) + + def analyse_text(text): + """ + Check for initial comment and patterns that distinguish Rexx from other + C-like languages. + """ + if re.search(r'/\*\**\s*rexx', text, re.IGNORECASE): + # Header matches MVS Rexx requirements, this is certainly a Rexx + # script. + return 1.0 + elif text.startswith('/*'): + # Header matches general Rexx requirements; the source code might + # still be any language using C comments such as C++, C# or Java. + lowerText = text.lower() + result = sum(weight + for (pattern, weight) in RexxLexer.PATTERNS_AND_WEIGHTS + if pattern.search(lowerText)) + 0.01 + return min(result, 1.0) + + +class MOOCodeLexer(RegexLexer): + """ + For MOOCode (the MOO scripting language). + + .. versionadded:: 0.9 + """ + name = 'MOOCode' + url = 'http://www.moo.mud.org/' + filenames = ['*.moo'] + aliases = ['moocode', 'moo'] + mimetypes = ['text/x-moocode'] + + tokens = { + 'root': [ + # Numbers + (r'(0|[1-9][0-9_]*)', Number.Integer), + # Strings + (r'"(\\\\|\\[^\\]|[^"\\])*"', String), + # exceptions + (r'(E_PERM|E_DIV)', Name.Exception), + # db-refs + (r'((#[-0-9]+)|(\$\w+))', Name.Entity), + # Keywords + (r'\b(if|else|elseif|endif|for|endfor|fork|endfork|while' + r'|endwhile|break|continue|return|try' + r'|except|endtry|finally|in)\b', Keyword), + # builtins + (r'(random|length)', Name.Builtin), + # special variables + (r'(player|caller|this|args)', Name.Variable.Instance), + # skip whitespace + (r'\s+', Text), + (r'\n', Text), + # other operators + (r'([!;=,{}&|:.\[\]@()<>?]+)', Operator), + # function call + (r'(\w+)(\()', bygroups(Name.Function, Operator)), + # variables + (r'(\w+)', Text), + ] + } + + +class HybrisLexer(RegexLexer): + """ + For Hybris source code. + + .. versionadded:: 1.4 + """ + + name = 'Hybris' + aliases = ['hybris', 'hy'] + filenames = ['*.hy', '*.hyb'] + mimetypes = ['text/x-hybris', 'application/x-hybris'] + + flags = re.MULTILINE | re.DOTALL + + tokens = { + 'root': [ + # method names + (r'^(\s*(?:function|method|operator\s+)+?)' + r'([a-zA-Z_]\w*)' + r'(\s*)(\()', bygroups(Keyword, Name.Function, Text, Operator)), + (r'[^\S\n]+', Text), + (r'//.*?\n', Comment.Single), + (r'/\*.*?\*/', Comment.Multiline), + (r'@[a-zA-Z_][\w.]*', Name.Decorator), + (r'(break|case|catch|next|default|do|else|finally|for|foreach|of|' + r'unless|if|new|return|switch|me|throw|try|while)\b', Keyword), + (r'(extends|private|protected|public|static|throws|function|method|' + r'operator)\b', Keyword.Declaration), + (r'(true|false|null|__FILE__|__LINE__|__VERSION__|__LIB_PATH__|' + r'__INC_PATH__)\b', Keyword.Constant), + (r'(class|struct)(\s+)', + bygroups(Keyword.Declaration, Text), 'class'), + (r'(import|include)(\s+)', + bygroups(Keyword.Namespace, Text), 'import'), + (words(( + 'gc_collect', 'gc_mm_items', 'gc_mm_usage', 'gc_collect_threshold', + 'urlencode', 'urldecode', 'base64encode', 'base64decode', 'sha1', 'crc32', + 'sha2', 'md5', 'md5_file', 'acos', 'asin', 'atan', 'atan2', 'ceil', 'cos', + 'cosh', 'exp', 'fabs', 'floor', 'fmod', 'log', 'log10', 'pow', 'sin', + 'sinh', 'sqrt', 'tan', 'tanh', 'isint', 'isfloat', 'ischar', 'isstring', + 'isarray', 'ismap', 'isalias', 'typeof', 'sizeof', 'toint', 'tostring', + 'fromxml', 'toxml', 'binary', 'pack', 'load', 'eval', 'var_names', + 'var_values', 'user_functions', 'dyn_functions', 'methods', 'call', + 'call_method', 'mknod', 'mkfifo', 'mount', 'umount2', 'umount', 'ticks', + 'usleep', 'sleep', 'time', 'strtime', 'strdate', 'dllopen', 'dlllink', + 'dllcall', 'dllcall_argv', 'dllclose', 'env', 'exec', 'fork', 'getpid', + 'wait', 'popen', 'pclose', 'exit', 'kill', 'pthread_create', + 'pthread_create_argv', 'pthread_exit', 'pthread_join', 'pthread_kill', + 'smtp_send', 'http_get', 'http_post', 'http_download', 'socket', 'bind', + 'listen', 'accept', 'getsockname', 'getpeername', 'settimeout', 'connect', + 'server', 'recv', 'send', 'close', 'print', 'println', 'printf', 'input', + 'readline', 'serial_open', 'serial_fcntl', 'serial_get_attr', + 'serial_get_ispeed', 'serial_get_ospeed', 'serial_set_attr', + 'serial_set_ispeed', 'serial_set_ospeed', 'serial_write', 'serial_read', + 'serial_close', 'xml_load', 'xml_parse', 'fopen', 'fseek', 'ftell', + 'fsize', 'fread', 'fwrite', 'fgets', 'fclose', 'file', 'readdir', + 'pcre_replace', 'size', 'pop', 'unmap', 'has', 'keys', 'values', + 'length', 'find', 'substr', 'replace', 'split', 'trim', 'remove', + 'contains', 'join'), suffix=r'\b'), + Name.Builtin), + (words(( + 'MethodReference', 'Runner', 'Dll', 'Thread', 'Pipe', 'Process', + 'Runnable', 'CGI', 'ClientSocket', 'Socket', 'ServerSocket', + 'File', 'Console', 'Directory', 'Exception'), suffix=r'\b'), + Keyword.Type), + (r'"(\\\\|\\[^\\]|[^"\\])*"', String), + (r"'\\.'|'[^\\]'|'\\u[0-9a-f]{4}'", String.Char), + (r'(\.)([a-zA-Z_]\w*)', + bygroups(Operator, Name.Attribute)), + (r'[a-zA-Z_]\w*:', Name.Label), + (r'[a-zA-Z_$]\w*', Name), + (r'[~^*!%&\[\](){}<>|+=:;,./?\-@]+', Operator), + (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), + (r'0x[0-9a-f]+', Number.Hex), + (r'[0-9]+L?', Number.Integer), + (r'\n', Text), + ], + 'class': [ + (r'[a-zA-Z_]\w*', Name.Class, '#pop') + ], + 'import': [ + (r'[\w.]+\*?', Name.Namespace, '#pop') + ], + } + + def analyse_text(text): + """public method and private method don't seem to be quite common + elsewhere.""" + result = 0 + if re.search(r'\b(?:public|private)\s+method\b', text): + result += 0.01 + return result + + + +class EasytrieveLexer(RegexLexer): + """ + Easytrieve Plus is a programming language for extracting, filtering and + converting sequential data. Furthermore it can layout data for reports. + It is mainly used on mainframe platforms and can access several of the + mainframe's native file formats. It is somewhat comparable to awk. + + .. versionadded:: 2.1 + """ + name = 'Easytrieve' + aliases = ['easytrieve'] + filenames = ['*.ezt', '*.mac'] + mimetypes = ['text/x-easytrieve'] + flags = 0 + + # Note: We cannot use r'\b' at the start and end of keywords because + # Easytrieve Plus delimiter characters are: + # + # * space ( ) + # * apostrophe (') + # * period (.) + # * comma (,) + # * parenthesis ( and ) + # * colon (:) + # + # Additionally words end once a '*' appears, indicatins a comment. + _DELIMITERS = r' \'.,():\n' + _DELIMITERS_OR_COMENT = _DELIMITERS + '*' + _DELIMITER_PATTERN = '[' + _DELIMITERS + ']' + _DELIMITER_PATTERN_CAPTURE = '(' + _DELIMITER_PATTERN + ')' + _NON_DELIMITER_OR_COMMENT_PATTERN = '[^' + _DELIMITERS_OR_COMENT + ']' + _OPERATORS_PATTERN = '[.+\\-/=\\[\\](){}<>;,&%¬]' + _KEYWORDS = [ + 'AFTER-BREAK', 'AFTER-LINE', 'AFTER-SCREEN', 'AIM', 'AND', 'ATTR', + 'BEFORE', 'BEFORE-BREAK', 'BEFORE-LINE', 'BEFORE-SCREEN', 'BUSHU', + 'BY', 'CALL', 'CASE', 'CHECKPOINT', 'CHKP', 'CHKP-STATUS', 'CLEAR', + 'CLOSE', 'COL', 'COLOR', 'COMMIT', 'CONTROL', 'COPY', 'CURSOR', 'D', + 'DECLARE', 'DEFAULT', 'DEFINE', 'DELETE', 'DENWA', 'DISPLAY', 'DLI', + 'DO', 'DUPLICATE', 'E', 'ELSE', 'ELSE-IF', 'END', 'END-CASE', + 'END-DO', 'END-IF', 'END-PROC', 'ENDPAGE', 'ENDTABLE', 'ENTER', 'EOF', + 'EQ', 'ERROR', 'EXIT', 'EXTERNAL', 'EZLIB', 'F1', 'F10', 'F11', 'F12', + 'F13', 'F14', 'F15', 'F16', 'F17', 'F18', 'F19', 'F2', 'F20', 'F21', + 'F22', 'F23', 'F24', 'F25', 'F26', 'F27', 'F28', 'F29', 'F3', 'F30', + 'F31', 'F32', 'F33', 'F34', 'F35', 'F36', 'F4', 'F5', 'F6', 'F7', + 'F8', 'F9', 'FETCH', 'FILE-STATUS', 'FILL', 'FINAL', 'FIRST', + 'FIRST-DUP', 'FOR', 'GE', 'GET', 'GO', 'GOTO', 'GQ', 'GR', 'GT', + 'HEADING', 'HEX', 'HIGH-VALUES', 'IDD', 'IDMS', 'IF', 'IN', 'INSERT', + 'JUSTIFY', 'KANJI-DATE', 'KANJI-DATE-LONG', 'KANJI-TIME', 'KEY', + 'KEY-PRESSED', 'KOKUGO', 'KUN', 'LAST-DUP', 'LE', 'LEVEL', 'LIKE', + 'LINE', 'LINE-COUNT', 'LINE-NUMBER', 'LINK', 'LIST', 'LOW-VALUES', + 'LQ', 'LS', 'LT', 'MACRO', 'MASK', 'MATCHED', 'MEND', 'MESSAGE', + 'MOVE', 'MSTART', 'NE', 'NEWPAGE', 'NOMASK', 'NOPRINT', 'NOT', + 'NOTE', 'NOVERIFY', 'NQ', 'NULL', 'OF', 'OR', 'OTHERWISE', 'PA1', + 'PA2', 'PA3', 'PAGE-COUNT', 'PAGE-NUMBER', 'PARM-REGISTER', + 'PATH-ID', 'PATTERN', 'PERFORM', 'POINT', 'POS', 'PRIMARY', 'PRINT', + 'PROCEDURE', 'PROGRAM', 'PUT', 'READ', 'RECORD', 'RECORD-COUNT', + 'RECORD-LENGTH', 'REFRESH', 'RELEASE', 'RENUM', 'REPEAT', 'REPORT', + 'REPORT-INPUT', 'RESHOW', 'RESTART', 'RETRIEVE', 'RETURN-CODE', + 'ROLLBACK', 'ROW', 'S', 'SCREEN', 'SEARCH', 'SECONDARY', 'SELECT', + 'SEQUENCE', 'SIZE', 'SKIP', 'SOKAKU', 'SORT', 'SQL', 'STOP', 'SUM', + 'SYSDATE', 'SYSDATE-LONG', 'SYSIN', 'SYSIPT', 'SYSLST', 'SYSPRINT', + 'SYSSNAP', 'SYSTIME', 'TALLY', 'TERM-COLUMNS', 'TERM-NAME', + 'TERM-ROWS', 'TERMINATION', 'TITLE', 'TO', 'TRANSFER', 'TRC', + 'UNIQUE', 'UNTIL', 'UPDATE', 'UPPERCASE', 'USER', 'USERID', 'VALUE', + 'VERIFY', 'W', 'WHEN', 'WHILE', 'WORK', 'WRITE', 'X', 'XDM', 'XRST' + ] + + tokens = { + 'root': [ + (r'\*.*\n', Comment.Single), + (r'\n+', Whitespace), + # Macro argument + (r'&' + _NON_DELIMITER_OR_COMMENT_PATTERN + r'+\.', Name.Variable, + 'after_macro_argument'), + # Macro call + (r'%' + _NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name.Variable), + (r'(FILE|MACRO|REPORT)(\s+)', + bygroups(Keyword.Declaration, Whitespace), 'after_declaration'), + (r'(JOB|PARM)' + r'(' + _DELIMITER_PATTERN + r')', + bygroups(Keyword.Declaration, Operator)), + (words(_KEYWORDS, suffix=_DELIMITER_PATTERN_CAPTURE), + bygroups(Keyword.Reserved, Operator)), + (_OPERATORS_PATTERN, Operator), + # Procedure declaration + (r'(' + _NON_DELIMITER_OR_COMMENT_PATTERN + r'+)(\s*)(\.?)(\s*)(PROC)(\s*\n)', + bygroups(Name.Function, Whitespace, Operator, Whitespace, + Keyword.Declaration, Whitespace)), + (r'[0-9]+\.[0-9]*', Number.Float), + (r'[0-9]+', Number.Integer), + (r"'(''|[^'])*'", String), + (r'\s+', Whitespace), + # Everything else just belongs to a name + (_NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name), + ], + 'after_declaration': [ + (_NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name.Function), + default('#pop'), + ], + 'after_macro_argument': [ + (r'\*.*\n', Comment.Single, '#pop'), + (r'\s+', Whitespace, '#pop'), + (_OPERATORS_PATTERN, Operator, '#pop'), + (r"'(''|[^'])*'", String, '#pop'), + # Everything else just belongs to a name + (_NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name), + ], + } + _COMMENT_LINE_REGEX = re.compile(r'^\s*\*') + _MACRO_HEADER_REGEX = re.compile(r'^\s*MACRO') + + def analyse_text(text): + """ + Perform a structural analysis for basic Easytrieve constructs. + """ + result = 0.0 + lines = text.split('\n') + hasEndProc = False + hasHeaderComment = False + hasFile = False + hasJob = False + hasProc = False + hasParm = False + hasReport = False + + def isCommentLine(line): + return EasytrieveLexer._COMMENT_LINE_REGEX.match(lines[0]) is not None + + def isEmptyLine(line): + return not bool(line.strip()) + + # Remove possible empty lines and header comments. + while lines and (isEmptyLine(lines[0]) or isCommentLine(lines[0])): + if not isEmptyLine(lines[0]): + hasHeaderComment = True + del lines[0] + + if EasytrieveLexer._MACRO_HEADER_REGEX.match(lines[0]): + # Looks like an Easytrieve macro. + result = 0.4 + if hasHeaderComment: + result += 0.4 + else: + # Scan the source for lines starting with indicators. + for line in lines: + words = line.split() + if (len(words) >= 2): + firstWord = words[0] + if not hasReport: + if not hasJob: + if not hasFile: + if not hasParm: + if firstWord == 'PARM': + hasParm = True + if firstWord == 'FILE': + hasFile = True + if firstWord == 'JOB': + hasJob = True + elif firstWord == 'PROC': + hasProc = True + elif firstWord == 'END-PROC': + hasEndProc = True + elif firstWord == 'REPORT': + hasReport = True + + # Weight the findings. + if hasJob and (hasProc == hasEndProc): + if hasHeaderComment: + result += 0.1 + if hasParm: + if hasProc: + # Found PARM, JOB and PROC/END-PROC: + # pretty sure this is Easytrieve. + result += 0.8 + else: + # Found PARAM and JOB: probably this is Easytrieve + result += 0.5 + else: + # Found JOB and possibly other keywords: might be Easytrieve + result += 0.11 + if hasParm: + # Note: PARAM is not a proper English word, so this is + # regarded a much better indicator for Easytrieve than + # the other words. + result += 0.2 + if hasFile: + result += 0.01 + if hasReport: + result += 0.01 + assert 0.0 <= result <= 1.0 + return result + + +class JclLexer(RegexLexer): + """ + Job Control Language (JCL) + is a scripting language used on mainframe platforms to instruct the system + on how to run a batch job or start a subsystem. It is somewhat + comparable to MS DOS batch and Unix shell scripts. + + .. versionadded:: 2.1 + """ + name = 'JCL' + aliases = ['jcl'] + filenames = ['*.jcl'] + mimetypes = ['text/x-jcl'] + flags = re.IGNORECASE + + tokens = { + 'root': [ + (r'//\*.*\n', Comment.Single), + (r'//', Keyword.Pseudo, 'statement'), + (r'/\*', Keyword.Pseudo, 'jes2_statement'), + # TODO: JES3 statement + (r'.*\n', Other) # Input text or inline code in any language. + ], + 'statement': [ + (r'\s*\n', Whitespace, '#pop'), + (r'([a-z]\w*)(\s+)(exec|job)(\s*)', + bygroups(Name.Label, Whitespace, Keyword.Reserved, Whitespace), + 'option'), + (r'[a-z]\w*', Name.Variable, 'statement_command'), + (r'\s+', Whitespace, 'statement_command'), + ], + 'statement_command': [ + (r'\s+(command|cntl|dd|endctl|endif|else|include|jcllib|' + r'output|pend|proc|set|then|xmit)\s+', Keyword.Reserved, 'option'), + include('option') + ], + 'jes2_statement': [ + (r'\s*\n', Whitespace, '#pop'), + (r'\$', Keyword, 'option'), + (r'\b(jobparam|message|netacct|notify|output|priority|route|' + r'setup|signoff|xeq|xmit)\b', Keyword, 'option'), + ], + 'option': [ + # (r'\n', Text, 'root'), + (r'\*', Name.Builtin), + (r'[\[\](){}<>;,]', Punctuation), + (r'[-+*/=&%]', Operator), + (r'[a-z_]\w*', Name), + (r'\d+\.\d*', Number.Float), + (r'\.\d+', Number.Float), + (r'\d+', Number.Integer), + (r"'", String, 'option_string'), + (r'[ \t]+', Whitespace, 'option_comment'), + (r'\.', Punctuation), + ], + 'option_string': [ + (r"(\n)(//)", bygroups(Text, Keyword.Pseudo)), + (r"''", String), + (r"[^']", String), + (r"'", String, '#pop'), + ], + 'option_comment': [ + # (r'\n', Text, 'root'), + (r'.+', Comment.Single), + ] + } + + _JOB_HEADER_PATTERN = re.compile(r'^//[a-z#$@][a-z0-9#$@]{0,7}\s+job(\s+.*)?$', + re.IGNORECASE) + + def analyse_text(text): + """ + Recognize JCL job by header. + """ + result = 0.0 + lines = text.split('\n') + if len(lines) > 0: + if JclLexer._JOB_HEADER_PATTERN.match(lines[0]): + result = 1.0 + assert 0.0 <= result <= 1.0 + return result + + +class MiniScriptLexer(RegexLexer): + """ + For MiniScript source code. + + .. versionadded:: 2.6 + """ + + name = 'MiniScript' + url = 'https://miniscript.org' + aliases = ['miniscript', 'ms'] + filenames = ['*.ms'] + mimetypes = ['text/x-minicript', 'application/x-miniscript'] + + tokens = { + 'root': [ + (r'#!(.*?)$', Comment.Preproc), + default('base'), + ], + 'base': [ + ('//.*$', Comment.Single), + (r'(?i)(\d*\.\d+|\d+\.\d*)(e[+-]?\d+)?', Number), + (r'(?i)\d+e[+-]?\d+', Number), + (r'\d+', Number), + (r'\n', Text), + (r'[^\S\n]+', Text), + (r'"', String, 'string_double'), + (r'(==|!=|<=|>=|[=+\-*/%^<>.:])', Operator), + (r'[;,\[\]{}()]', Punctuation), + (words(( + 'break', 'continue', 'else', 'end', 'for', 'function', 'if', + 'in', 'isa', 'then', 'repeat', 'return', 'while'), suffix=r'\b'), + Keyword), + (words(( + 'abs', 'acos', 'asin', 'atan', 'ceil', 'char', 'cos', 'floor', + 'log', 'round', 'rnd', 'pi', 'sign', 'sin', 'sqrt', 'str', 'tan', + 'hasIndex', 'indexOf', 'len', 'val', 'code', 'remove', 'lower', + 'upper', 'replace', 'split', 'indexes', 'values', 'join', 'sum', + 'sort', 'shuffle', 'push', 'pop', 'pull', 'range', + 'print', 'input', 'time', 'wait', 'locals', 'globals', 'outer', + 'yield'), suffix=r'\b'), + Name.Builtin), + (r'(true|false|null)\b', Keyword.Constant), + (r'(and|or|not|new)\b', Operator.Word), + (r'(self|super|__isa)\b', Name.Builtin.Pseudo), + (r'[a-zA-Z_]\w*', Name.Variable) + ], + 'string_double': [ + (r'[^"\n]+', String), + (r'""', String), + (r'"', String, '#pop'), + (r'\n', Text, '#pop'), # Stray linefeed also terminates strings. + ] + } diff --git a/pygments/lexers/sgf.py b/pygments/lexers/sgf.py new file mode 100644 index 0000000..865d55c --- /dev/null +++ b/pygments/lexers/sgf.py @@ -0,0 +1,60 @@ +""" + pygments.lexers.sgf + ~~~~~~~~~~~~~~~~~~~ + + Lexer for Smart Game Format (sgf) file format. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups +from pygments.token import Name, Literal, String, Punctuation, Whitespace + +__all__ = ["SmartGameFormatLexer"] + + +class SmartGameFormatLexer(RegexLexer): + """ + Lexer for Smart Game Format (sgf) file format. + + The format is used to store game records of board games for two players + (mainly Go game). + + .. versionadded:: 2.4 + """ + name = 'SmartGameFormat' + url = 'https://www.red-bean.com/sgf/' + aliases = ['sgf'] + filenames = ['*.sgf'] + + tokens = { + 'root': [ + (r'[():;]+', Punctuation), + # tokens: + (r'(A[BW]|AE|AN|AP|AR|AS|[BW]L|BM|[BW]R|[BW]S|[BW]T|CA|CH|CP|CR|' + r'DD|DM|DO|DT|EL|EV|EX|FF|FG|G[BW]|GC|GM|GN|HA|HO|ID|IP|IT|IY|KM|' + r'KO|LB|LN|LT|L|MA|MN|M|N|OB|OM|ON|OP|OT|OV|P[BW]|PC|PL|PM|RE|RG|' + r'RO|RU|SO|SC|SE|SI|SL|SO|SQ|ST|SU|SZ|T[BW]|TC|TE|TM|TR|UC|US|VW|' + r'V|[BW]|C)', + Name.Builtin), + # number: + (r'(\[)([0-9.]+)(\])', + bygroups(Punctuation, Literal.Number, Punctuation)), + # date: + (r'(\[)([0-9]{4}-[0-9]{2}-[0-9]{2})(\])', + bygroups(Punctuation, Literal.Date, Punctuation)), + # point: + (r'(\[)([a-z]{2})(\])', + bygroups(Punctuation, String, Punctuation)), + # double points: + (r'(\[)([a-z]{2})(:)([a-z]{2})(\])', + bygroups(Punctuation, String, Punctuation, String, Punctuation)), + + (r'(\[)([\w\s#()+,\-.:?]+)(\])', + bygroups(Punctuation, String, Punctuation)), + (r'(\[)(\s.*)(\])', + bygroups(Punctuation, Whitespace, Punctuation)), + (r'\s+', Whitespace) + ], + } diff --git a/pygments/lexers/shell.py b/pygments/lexers/shell.py new file mode 100644 index 0000000..d3fb8a1 --- /dev/null +++ b/pygments/lexers/shell.py @@ -0,0 +1,918 @@ +""" + pygments.lexers.shell + ~~~~~~~~~~~~~~~~~~~~~ + + Lexers for various shells. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import Lexer, RegexLexer, do_insertions, bygroups, \ + include, default, this, using, words, line_re +from pygments.token import Punctuation, Whitespace, \ + Text, Comment, Operator, Keyword, Name, String, Number, Generic +from pygments.util import shebang_matches + +__all__ = ['BashLexer', 'BashSessionLexer', 'TcshLexer', 'BatchLexer', + 'SlurmBashLexer', 'MSDOSSessionLexer', 'PowerShellLexer', + 'PowerShellSessionLexer', 'TcshSessionLexer', 'FishShellLexer', + 'ExeclineLexer'] + + +class BashLexer(RegexLexer): + """ + Lexer for (ba|k|z|)sh shell scripts. + + .. versionadded:: 0.6 + """ + + name = 'Bash' + aliases = ['bash', 'sh', 'ksh', 'zsh', 'shell'] + filenames = ['*.sh', '*.ksh', '*.bash', '*.ebuild', '*.eclass', + '*.exheres-0', '*.exlib', '*.zsh', + '.bashrc', 'bashrc', '.bash_*', 'bash_*', 'zshrc', '.zshrc', + '.kshrc', 'kshrc', + 'PKGBUILD'] + mimetypes = ['application/x-sh', 'application/x-shellscript', 'text/x-shellscript'] + + tokens = { + 'root': [ + include('basic'), + (r'`', String.Backtick, 'backticks'), + include('data'), + include('interp'), + ], + 'interp': [ + (r'\$\(\(', Keyword, 'math'), + (r'\$\(', Keyword, 'paren'), + (r'\$\{#?', String.Interpol, 'curly'), + (r'\$[a-zA-Z_]\w*', Name.Variable), # user variable + (r'\$(?:\d+|[#$?!_*@-])', Name.Variable), # builtin + (r'\$', Text), + ], + 'basic': [ + (r'\b(if|fi|else|while|in|do|done|for|then|return|function|case|' + r'select|continue|until|esac|elif)(\s*)\b', + bygroups(Keyword, Whitespace)), + (r'\b(alias|bg|bind|break|builtin|caller|cd|command|compgen|' + r'complete|declare|dirs|disown|echo|enable|eval|exec|exit|' + r'export|false|fc|fg|getopts|hash|help|history|jobs|kill|let|' + r'local|logout|popd|printf|pushd|pwd|read|readonly|set|shift|' + r'shopt|source|suspend|test|time|times|trap|true|type|typeset|' + r'ulimit|umask|unalias|unset|wait)(?=[\s)`])', + Name.Builtin), + (r'\A#!.+\n', Comment.Hashbang), + (r'#.*\n', Comment.Single), + (r'\\[\w\W]', String.Escape), + (r'(\b\w+)(\s*)(\+?=)', bygroups(Name.Variable, Whitespace, Operator)), + (r'[\[\]{}()=]', Operator), + (r'<<<', Operator), # here-string + (r'<<-?\s*(\'?)\\?(\w+)[\w\W]+?\2', String), + (r'&&|\|\|', Operator), + ], + 'data': [ + (r'(?s)\$?"(\\.|[^"\\$])*"', String.Double), + (r'"', String.Double, 'string'), + (r"(?s)\$'(\\\\|\\[0-7]+|\\.|[^'\\])*'", String.Single), + (r"(?s)'.*?'", String.Single), + (r';', Punctuation), + (r'&', Punctuation), + (r'\|', Punctuation), + (r'\s+', Whitespace), + (r'\d+\b', Number), + (r'[^=\s\[\]{}()$"\'`\\<&|;]+', Text), + (r'<', Text), + ], + 'string': [ + (r'"', String.Double, '#pop'), + (r'(?s)(\\\\|\\[0-7]+|\\.|[^"\\$])+', String.Double), + include('interp'), + ], + 'curly': [ + (r'\}', String.Interpol, '#pop'), + (r':-', Keyword), + (r'\w+', Name.Variable), + (r'[^}:"\'`$\\]+', Punctuation), + (r':', Punctuation), + include('root'), + ], + 'paren': [ + (r'\)', Keyword, '#pop'), + include('root'), + ], + 'math': [ + (r'\)\)', Keyword, '#pop'), + (r'[-+*/%^|&]|\*\*|\|\|', Operator), + (r'\d+#\d+', Number), + (r'\d+#(?! )', Number), + (r'\d+', Number), + include('root'), + ], + 'backticks': [ + (r'`', String.Backtick, '#pop'), + include('root'), + ], + } + + def analyse_text(text): + if shebang_matches(text, r'(ba|z|)sh'): + return 1 + if text.startswith('$ '): + return 0.2 + + +class SlurmBashLexer(BashLexer): + """ + Lexer for (ba|k|z|)sh Slurm scripts. + + .. versionadded:: 2.4 + """ + + name = 'Slurm' + aliases = ['slurm', 'sbatch'] + filenames = ['*.sl'] + mimetypes = [] + EXTRA_KEYWORDS = {'srun'} + + def get_tokens_unprocessed(self, text): + for index, token, value in BashLexer.get_tokens_unprocessed(self, text): + if token is Text and value in self.EXTRA_KEYWORDS: + yield index, Name.Builtin, value + elif token is Comment.Single and 'SBATCH' in value: + yield index, Keyword.Pseudo, value + else: + yield index, token, value + + +class ShellSessionBaseLexer(Lexer): + """ + Base lexer for shell sessions. + + .. versionadded:: 2.1 + """ + + _bare_continuation = False + _venv = re.compile(r'^(\([^)]*\))(\s*)') + + def get_tokens_unprocessed(self, text): + innerlexer = self._innerLexerCls(**self.options) + + pos = 0 + curcode = '' + insertions = [] + backslash_continuation = False + + for match in line_re.finditer(text): + line = match.group() + + venv_match = self._venv.match(line) + if venv_match: + venv = venv_match.group(1) + venv_whitespace = venv_match.group(2) + insertions.append((len(curcode), + [(0, Generic.Prompt.VirtualEnv, venv)])) + if venv_whitespace: + insertions.append((len(curcode), + [(0, Text, venv_whitespace)])) + line = line[venv_match.end():] + + m = self._ps1rgx.match(line) + if m: + # To support output lexers (say diff output), the output + # needs to be broken by prompts whenever the output lexer + # changes. + if not insertions: + pos = match.start() + + insertions.append((len(curcode), + [(0, Generic.Prompt, m.group(1))])) + curcode += m.group(2) + backslash_continuation = curcode.endswith('\\\n') + elif backslash_continuation: + if line.startswith(self._ps2): + insertions.append((len(curcode), + [(0, Generic.Prompt, + line[:len(self._ps2)])])) + curcode += line[len(self._ps2):] + else: + curcode += line + backslash_continuation = curcode.endswith('\\\n') + elif self._bare_continuation and line.startswith(self._ps2): + insertions.append((len(curcode), + [(0, Generic.Prompt, + line[:len(self._ps2)])])) + curcode += line[len(self._ps2):] + else: + if insertions: + toks = innerlexer.get_tokens_unprocessed(curcode) + for i, t, v in do_insertions(insertions, toks): + yield pos+i, t, v + yield match.start(), Generic.Output, line + insertions = [] + curcode = '' + if insertions: + for i, t, v in do_insertions(insertions, + innerlexer.get_tokens_unprocessed(curcode)): + yield pos+i, t, v + + +class BashSessionLexer(ShellSessionBaseLexer): + """ + Lexer for Bash shell sessions, i.e. command lines, including a + prompt, interspersed with output. + + .. versionadded:: 1.1 + """ + + name = 'Bash Session' + aliases = ['console', 'shell-session'] + filenames = ['*.sh-session', '*.shell-session'] + mimetypes = ['application/x-shell-session', 'application/x-sh-session'] + + _innerLexerCls = BashLexer + _ps1rgx = re.compile( + r'^((?:(?:\[.*?\])|(?:\(\S+\))?(?:| |sh\S*?|\w+\S+[@:]\S+(?:\s+\S+)' \ + r'?|\[\S+[@:][^\n]+\].+))\s*[$#%]\s*)(.*\n?)') + _ps2 = '> ' + + +class BatchLexer(RegexLexer): + """ + Lexer for the DOS/Windows Batch file format. + + .. versionadded:: 0.7 + """ + name = 'Batchfile' + aliases = ['batch', 'bat', 'dosbatch', 'winbatch'] + filenames = ['*.bat', '*.cmd'] + mimetypes = ['application/x-dos-batch'] + + flags = re.MULTILINE | re.IGNORECASE + + _nl = r'\n\x1a' + _punct = r'&<>|' + _ws = r'\t\v\f\r ,;=\xa0' + _nlws = r'\s\x1a\xa0,;=' + _space = r'(?:(?:(?:\^[%s])?[%s])+)' % (_nl, _ws) + _keyword_terminator = (r'(?=(?:\^[%s]?)?[%s+./:[\\\]]|[%s%s(])' % + (_nl, _ws, _nl, _punct)) + _token_terminator = r'(?=\^?[%s]|[%s%s])' % (_ws, _punct, _nl) + _start_label = r'((?:(?<=^[^:])|^[^:]?)[%s]*)(:)' % _ws + _label = r'(?:(?:[^%s%s+:^]|\^[%s]?[\w\W])*)' % (_nlws, _punct, _nl) + _label_compound = r'(?:(?:[^%s%s+:^)]|\^[%s]?[^)])*)' % (_nlws, _punct, _nl) + _number = r'(?:-?(?:0[0-7]+|0x[\da-f]+|\d+)%s)' % _token_terminator + _opword = r'(?:equ|geq|gtr|leq|lss|neq)' + _string = r'(?:"[^%s"]*(?:"|(?=[%s])))' % (_nl, _nl) + _variable = (r'(?:(?:%%(?:\*|(?:~[a-z]*(?:\$[^:]+:)?)?\d|' + r'[^%%:%s]+(?::(?:~(?:-?\d+)?(?:,(?:-?\d+)?)?|(?:[^%%%s^]|' + r'\^[^%%%s])[^=%s]*=(?:[^%%%s^]|\^[^%%%s])*)?)?%%))|' + r'(?:\^?![^!:%s]+(?::(?:~(?:-?\d+)?(?:,(?:-?\d+)?)?|(?:' + r'[^!%s^]|\^[^!%s])[^=%s]*=(?:[^!%s^]|\^[^!%s])*)?)?\^?!))' % + (_nl, _nl, _nl, _nl, _nl, _nl, _nl, _nl, _nl, _nl, _nl, _nl)) + _core_token = r'(?:(?:(?:\^[%s]?)?[^"%s%s])+)' % (_nl, _nlws, _punct) + _core_token_compound = r'(?:(?:(?:\^[%s]?)?[^"%s%s)])+)' % (_nl, _nlws, _punct) + _token = r'(?:[%s]+|%s)' % (_punct, _core_token) + _token_compound = r'(?:[%s]+|%s)' % (_punct, _core_token_compound) + _stoken = (r'(?:[%s]+|(?:%s|%s|%s)+)' % + (_punct, _string, _variable, _core_token)) + + def _make_begin_state(compound, _core_token=_core_token, + _core_token_compound=_core_token_compound, + _keyword_terminator=_keyword_terminator, + _nl=_nl, _punct=_punct, _string=_string, + _space=_space, _start_label=_start_label, + _stoken=_stoken, _token_terminator=_token_terminator, + _variable=_variable, _ws=_ws): + rest = '(?:%s|%s|[^"%%%s%s%s])*' % (_string, _variable, _nl, _punct, + ')' if compound else '') + rest_of_line = r'(?:(?:[^%s^]|\^[%s]?[\w\W])*)' % (_nl, _nl) + rest_of_line_compound = r'(?:(?:[^%s^)]|\^[%s]?[^)])*)' % (_nl, _nl) + set_space = r'((?:(?:\^[%s]?)?[^\S\n])*)' % _nl + suffix = '' + if compound: + _keyword_terminator = r'(?:(?=\))|%s)' % _keyword_terminator + _token_terminator = r'(?:(?=\))|%s)' % _token_terminator + suffix = '/compound' + return [ + ((r'\)', Punctuation, '#pop') if compound else + (r'\)((?=\()|%s)%s' % (_token_terminator, rest_of_line), + Comment.Single)), + (r'(?=%s)' % _start_label, Text, 'follow%s' % suffix), + (_space, using(this, state='text')), + include('redirect%s' % suffix), + (r'[%s]+' % _nl, Text), + (r'\(', Punctuation, 'root/compound'), + (r'@+', Punctuation), + (r'((?:for|if|rem)(?:(?=(?:\^[%s]?)?/)|(?:(?!\^)|' + r'(?<=m))(?:(?=\()|%s)))(%s?%s?(?:\^[%s]?)?/(?:\^[%s]?)?\?)' % + (_nl, _token_terminator, _space, + _core_token_compound if compound else _core_token, _nl, _nl), + bygroups(Keyword, using(this, state='text')), + 'follow%s' % suffix), + (r'(goto%s)(%s(?:\^[%s]?)?/(?:\^[%s]?)?\?%s)' % + (_keyword_terminator, rest, _nl, _nl, rest), + bygroups(Keyword, using(this, state='text')), + 'follow%s' % suffix), + (words(('assoc', 'break', 'cd', 'chdir', 'cls', 'color', 'copy', + 'date', 'del', 'dir', 'dpath', 'echo', 'endlocal', 'erase', + 'exit', 'ftype', 'keys', 'md', 'mkdir', 'mklink', 'move', + 'path', 'pause', 'popd', 'prompt', 'pushd', 'rd', 'ren', + 'rename', 'rmdir', 'setlocal', 'shift', 'start', 'time', + 'title', 'type', 'ver', 'verify', 'vol'), + suffix=_keyword_terminator), Keyword, 'follow%s' % suffix), + (r'(call)(%s?)(:)' % _space, + bygroups(Keyword, using(this, state='text'), Punctuation), + 'call%s' % suffix), + (r'call%s' % _keyword_terminator, Keyword), + (r'(for%s(?!\^))(%s)(/f%s)' % + (_token_terminator, _space, _token_terminator), + bygroups(Keyword, using(this, state='text'), Keyword), + ('for/f', 'for')), + (r'(for%s(?!\^))(%s)(/l%s)' % + (_token_terminator, _space, _token_terminator), + bygroups(Keyword, using(this, state='text'), Keyword), + ('for/l', 'for')), + (r'for%s(?!\^)' % _token_terminator, Keyword, ('for2', 'for')), + (r'(goto%s)(%s?)(:?)' % (_keyword_terminator, _space), + bygroups(Keyword, using(this, state='text'), Punctuation), + 'label%s' % suffix), + (r'(if(?:(?=\()|%s)(?!\^))(%s?)((?:/i%s)?)(%s?)((?:not%s)?)(%s?)' % + (_token_terminator, _space, _token_terminator, _space, + _token_terminator, _space), + bygroups(Keyword, using(this, state='text'), Keyword, + using(this, state='text'), Keyword, + using(this, state='text')), ('(?', 'if')), + (r'rem(((?=\()|%s)%s?%s?.*|%s%s)' % + (_token_terminator, _space, _stoken, _keyword_terminator, + rest_of_line_compound if compound else rest_of_line), + Comment.Single, 'follow%s' % suffix), + (r'(set%s)%s(/a)' % (_keyword_terminator, set_space), + bygroups(Keyword, using(this, state='text'), Keyword), + 'arithmetic%s' % suffix), + (r'(set%s)%s((?:/p)?)%s((?:(?:(?:\^[%s]?)?[^"%s%s^=%s]|' + r'\^[%s]?[^"=])+)?)((?:(?:\^[%s]?)?=)?)' % + (_keyword_terminator, set_space, set_space, _nl, _nl, _punct, + ')' if compound else '', _nl, _nl), + bygroups(Keyword, using(this, state='text'), Keyword, + using(this, state='text'), using(this, state='variable'), + Punctuation), + 'follow%s' % suffix), + default('follow%s' % suffix) + ] + + def _make_follow_state(compound, _label=_label, + _label_compound=_label_compound, _nl=_nl, + _space=_space, _start_label=_start_label, + _token=_token, _token_compound=_token_compound, + _ws=_ws): + suffix = '/compound' if compound else '' + state = [] + if compound: + state.append((r'(?=\))', Text, '#pop')) + state += [ + (r'%s([%s]*)(%s)(.*)' % + (_start_label, _ws, _label_compound if compound else _label), + bygroups(Text, Punctuation, Text, Name.Label, Comment.Single)), + include('redirect%s' % suffix), + (r'(?=[%s])' % _nl, Text, '#pop'), + (r'\|\|?|&&?', Punctuation, '#pop'), + include('text') + ] + return state + + def _make_arithmetic_state(compound, _nl=_nl, _punct=_punct, + _string=_string, _variable=_variable, + _ws=_ws, _nlws=_nlws): + op = r'=+\-*/!~' + state = [] + if compound: + state.append((r'(?=\))', Text, '#pop')) + state += [ + (r'0[0-7]+', Number.Oct), + (r'0x[\da-f]+', Number.Hex), + (r'\d+', Number.Integer), + (r'[(),]+', Punctuation), + (r'([%s]|%%|\^\^)+' % op, Operator), + (r'(%s|%s|(\^[%s]?)?[^()%s%%\^"%s%s]|\^[%s]?%s)+' % + (_string, _variable, _nl, op, _nlws, _punct, _nlws, + r'[^)]' if compound else r'[\w\W]'), + using(this, state='variable')), + (r'(?=[\x00|&])', Text, '#pop'), + include('follow') + ] + return state + + def _make_call_state(compound, _label=_label, + _label_compound=_label_compound): + state = [] + if compound: + state.append((r'(?=\))', Text, '#pop')) + state.append((r'(:?)(%s)' % (_label_compound if compound else _label), + bygroups(Punctuation, Name.Label), '#pop')) + return state + + def _make_label_state(compound, _label=_label, + _label_compound=_label_compound, _nl=_nl, + _punct=_punct, _string=_string, _variable=_variable): + state = [] + if compound: + state.append((r'(?=\))', Text, '#pop')) + state.append((r'(%s?)((?:%s|%s|\^[%s]?%s|[^"%%^%s%s%s])*)' % + (_label_compound if compound else _label, _string, + _variable, _nl, r'[^)]' if compound else r'[\w\W]', _nl, + _punct, r')' if compound else ''), + bygroups(Name.Label, Comment.Single), '#pop')) + return state + + def _make_redirect_state(compound, + _core_token_compound=_core_token_compound, + _nl=_nl, _punct=_punct, _stoken=_stoken, + _string=_string, _space=_space, + _variable=_variable, _nlws=_nlws): + stoken_compound = (r'(?:[%s]+|(?:%s|%s|%s)+)' % + (_punct, _string, _variable, _core_token_compound)) + return [ + (r'((?:(?<=[%s])\d)?)(>>?&|<&)([%s]*)(\d)' % + (_nlws, _nlws), + bygroups(Number.Integer, Punctuation, Text, Number.Integer)), + (r'((?:(?<=[%s])(?>?|<)(%s?%s)' % + (_nlws, _nl, _space, stoken_compound if compound else _stoken), + bygroups(Number.Integer, Punctuation, using(this, state='text'))) + ] + + tokens = { + 'root': _make_begin_state(False), + 'follow': _make_follow_state(False), + 'arithmetic': _make_arithmetic_state(False), + 'call': _make_call_state(False), + 'label': _make_label_state(False), + 'redirect': _make_redirect_state(False), + 'root/compound': _make_begin_state(True), + 'follow/compound': _make_follow_state(True), + 'arithmetic/compound': _make_arithmetic_state(True), + 'call/compound': _make_call_state(True), + 'label/compound': _make_label_state(True), + 'redirect/compound': _make_redirect_state(True), + 'variable-or-escape': [ + (_variable, Name.Variable), + (r'%%%%|\^[%s]?(\^!|[\w\W])' % _nl, String.Escape) + ], + 'string': [ + (r'"', String.Double, '#pop'), + (_variable, Name.Variable), + (r'\^!|%%', String.Escape), + (r'[^"%%^%s]+|[%%^]' % _nl, String.Double), + default('#pop') + ], + 'sqstring': [ + include('variable-or-escape'), + (r'[^%]+|%', String.Single) + ], + 'bqstring': [ + include('variable-or-escape'), + (r'[^%]+|%', String.Backtick) + ], + 'text': [ + (r'"', String.Double, 'string'), + include('variable-or-escape'), + (r'[^"%%^%s%s\d)]+|.' % (_nlws, _punct), Text) + ], + 'variable': [ + (r'"', String.Double, 'string'), + include('variable-or-escape'), + (r'[^"%%^%s]+|.' % _nl, Name.Variable) + ], + 'for': [ + (r'(%s)(in)(%s)(\()' % (_space, _space), + bygroups(using(this, state='text'), Keyword, + using(this, state='text'), Punctuation), '#pop'), + include('follow') + ], + 'for2': [ + (r'\)', Punctuation), + (r'(%s)(do%s)' % (_space, _token_terminator), + bygroups(using(this, state='text'), Keyword), '#pop'), + (r'[%s]+' % _nl, Text), + include('follow') + ], + 'for/f': [ + (r'(")((?:%s|[^"])*?")([%s]*)(\))' % (_variable, _nlws), + bygroups(String.Double, using(this, state='string'), Text, + Punctuation)), + (r'"', String.Double, ('#pop', 'for2', 'string')), + (r"('(?:%%%%|%s|[\w\W])*?')([%s]*)(\))" % (_variable, _nlws), + bygroups(using(this, state='sqstring'), Text, Punctuation)), + (r'(`(?:%%%%|%s|[\w\W])*?`)([%s]*)(\))' % (_variable, _nlws), + bygroups(using(this, state='bqstring'), Text, Punctuation)), + include('for2') + ], + 'for/l': [ + (r'-?\d+', Number.Integer), + include('for2') + ], + 'if': [ + (r'((?:cmdextversion|errorlevel)%s)(%s)(\d+)' % + (_token_terminator, _space), + bygroups(Keyword, using(this, state='text'), + Number.Integer), '#pop'), + (r'(defined%s)(%s)(%s)' % (_token_terminator, _space, _stoken), + bygroups(Keyword, using(this, state='text'), + using(this, state='variable')), '#pop'), + (r'(exist%s)(%s%s)' % (_token_terminator, _space, _stoken), + bygroups(Keyword, using(this, state='text')), '#pop'), + (r'(%s%s)(%s)(%s%s)' % (_number, _space, _opword, _space, _number), + bygroups(using(this, state='arithmetic'), Operator.Word, + using(this, state='arithmetic')), '#pop'), + (_stoken, using(this, state='text'), ('#pop', 'if2')), + ], + 'if2': [ + (r'(%s?)(==)(%s?%s)' % (_space, _space, _stoken), + bygroups(using(this, state='text'), Operator, + using(this, state='text')), '#pop'), + (r'(%s)(%s)(%s%s)' % (_space, _opword, _space, _stoken), + bygroups(using(this, state='text'), Operator.Word, + using(this, state='text')), '#pop') + ], + '(?': [ + (_space, using(this, state='text')), + (r'\(', Punctuation, ('#pop', 'else?', 'root/compound')), + default('#pop') + ], + 'else?': [ + (_space, using(this, state='text')), + (r'else%s' % _token_terminator, Keyword, '#pop'), + default('#pop') + ] + } + + +class MSDOSSessionLexer(ShellSessionBaseLexer): + """ + Lexer for MS DOS shell sessions, i.e. command lines, including a + prompt, interspersed with output. + + .. versionadded:: 2.1 + """ + + name = 'MSDOS Session' + aliases = ['doscon'] + filenames = [] + mimetypes = [] + + _innerLexerCls = BatchLexer + _ps1rgx = re.compile(r'^([^>]*>)(.*\n?)') + _ps2 = 'More? ' + + +class TcshLexer(RegexLexer): + """ + Lexer for tcsh scripts. + + .. versionadded:: 0.10 + """ + + name = 'Tcsh' + aliases = ['tcsh', 'csh'] + filenames = ['*.tcsh', '*.csh'] + mimetypes = ['application/x-csh'] + + tokens = { + 'root': [ + include('basic'), + (r'\$\(', Keyword, 'paren'), + (r'\$\{#?', Keyword, 'curly'), + (r'`', String.Backtick, 'backticks'), + include('data'), + ], + 'basic': [ + (r'\b(if|endif|else|while|then|foreach|case|default|' + r'continue|goto|breaksw|end|switch|endsw)\s*\b', + Keyword), + (r'\b(alias|alloc|bg|bindkey|break|builtins|bye|caller|cd|chdir|' + r'complete|dirs|echo|echotc|eval|exec|exit|fg|filetest|getxvers|' + r'glob|getspath|hashstat|history|hup|inlib|jobs|kill|' + r'limit|log|login|logout|ls-F|migrate|newgrp|nice|nohup|notify|' + r'onintr|popd|printenv|pushd|rehash|repeat|rootnode|popd|pushd|' + r'set|shift|sched|setenv|setpath|settc|setty|setxvers|shift|' + r'source|stop|suspend|source|suspend|telltc|time|' + r'umask|unalias|uncomplete|unhash|universe|unlimit|unset|unsetenv|' + r'ver|wait|warp|watchlog|where|which)\s*\b', + Name.Builtin), + (r'#.*', Comment), + (r'\\[\w\W]', String.Escape), + (r'(\b\w+)(\s*)(=)', bygroups(Name.Variable, Text, Operator)), + (r'[\[\]{}()=]+', Operator), + (r'<<\s*(\'?)\\?(\w+)[\w\W]+?\2', String), + (r';', Punctuation), + ], + 'data': [ + (r'(?s)"(\\\\|\\[0-7]+|\\.|[^"\\])*"', String.Double), + (r"(?s)'(\\\\|\\[0-7]+|\\.|[^'\\])*'", String.Single), + (r'\s+', Text), + (r'[^=\s\[\]{}()$"\'`\\;#]+', Text), + (r'\d+(?= |\Z)', Number), + (r'\$#?(\w+|.)', Name.Variable), + ], + 'curly': [ + (r'\}', Keyword, '#pop'), + (r':-', Keyword), + (r'\w+', Name.Variable), + (r'[^}:"\'`$]+', Punctuation), + (r':', Punctuation), + include('root'), + ], + 'paren': [ + (r'\)', Keyword, '#pop'), + include('root'), + ], + 'backticks': [ + (r'`', String.Backtick, '#pop'), + include('root'), + ], + } + + +class TcshSessionLexer(ShellSessionBaseLexer): + """ + Lexer for Tcsh sessions, i.e. command lines, including a + prompt, interspersed with output. + + .. versionadded:: 2.1 + """ + + name = 'Tcsh Session' + aliases = ['tcshcon'] + filenames = [] + mimetypes = [] + + _innerLexerCls = TcshLexer + _ps1rgx = re.compile(r'^([^>]+>)(.*\n?)') + _ps2 = '? ' + + +class PowerShellLexer(RegexLexer): + """ + For Windows PowerShell code. + + .. versionadded:: 1.5 + """ + name = 'PowerShell' + aliases = ['powershell', 'pwsh', 'posh', 'ps1', 'psm1'] + filenames = ['*.ps1', '*.psm1'] + mimetypes = ['text/x-powershell'] + + flags = re.DOTALL | re.IGNORECASE | re.MULTILINE + + keywords = ( + 'while validateset validaterange validatepattern validatelength ' + 'validatecount until trap switch return ref process param parameter in ' + 'if global: local: function foreach for finally filter end elseif else ' + 'dynamicparam do default continue cmdletbinding break begin alias \\? ' + '% #script #private #local #global mandatory parametersetname position ' + 'valuefrompipeline valuefrompipelinebypropertyname ' + 'valuefromremainingarguments helpmessage try catch throw').split() + + operators = ( + 'and as band bnot bor bxor casesensitive ccontains ceq cge cgt cle ' + 'clike clt cmatch cne cnotcontains cnotlike cnotmatch contains ' + 'creplace eq exact f file ge gt icontains ieq ige igt ile ilike ilt ' + 'imatch ine inotcontains inotlike inotmatch ireplace is isnot le like ' + 'lt match ne not notcontains notlike notmatch or regex replace ' + 'wildcard').split() + + verbs = ( + 'write where watch wait use update unregister unpublish unprotect ' + 'unlock uninstall undo unblock trace test tee take sync switch ' + 'suspend submit stop step start split sort skip show set send select ' + 'search scroll save revoke resume restore restart resolve resize ' + 'reset request repair rename remove register redo receive read push ' + 'publish protect pop ping out optimize open new move mount merge ' + 'measure lock limit join invoke install initialize import hide group ' + 'grant get format foreach find export expand exit enter enable edit ' + 'dismount disconnect disable deny debug cxnew copy convertto ' + 'convertfrom convert connect confirm compress complete compare close ' + 'clear checkpoint block backup assert approve aggregate add').split() + + aliases_ = ( + 'ac asnp cat cd cfs chdir clc clear clhy cli clp cls clv cnsn ' + 'compare copy cp cpi cpp curl cvpa dbp del diff dir dnsn ebp echo epal ' + 'epcsv epsn erase etsn exsn fc fhx fl foreach ft fw gal gbp gc gci gcm ' + 'gcs gdr ghy gi gjb gl gm gmo gp gps gpv group gsn gsnp gsv gu gv gwmi ' + 'h history icm iex ihy ii ipal ipcsv ipmo ipsn irm ise iwmi iwr kill lp ' + 'ls man md measure mi mount move mp mv nal ndr ni nmo npssc nsn nv ogv ' + 'oh popd ps pushd pwd r rbp rcjb rcsn rd rdr ren ri rjb rm rmdir rmo ' + 'rni rnp rp rsn rsnp rujb rv rvpa rwmi sajb sal saps sasv sbp sc select ' + 'set shcm si sl sleep sls sort sp spjb spps spsv start sujb sv swmi tee ' + 'trcm type wget where wjb write').split() + + commenthelp = ( + 'component description example externalhelp forwardhelpcategory ' + 'forwardhelptargetname functionality inputs link ' + 'notes outputs parameter remotehelprunspace role synopsis').split() + + tokens = { + 'root': [ + # we need to count pairs of parentheses for correct highlight + # of '$(...)' blocks in strings + (r'\(', Punctuation, 'child'), + (r'\s+', Text), + (r'^(\s*#[#\s]*)(\.(?:%s))([^\n]*$)' % '|'.join(commenthelp), + bygroups(Comment, String.Doc, Comment)), + (r'#[^\n]*?$', Comment), + (r'(<|<)#', Comment.Multiline, 'multline'), + (r'@"\n', String.Heredoc, 'heredoc-double'), + (r"@'\n.*?\n'@", String.Heredoc), + # escaped syntax + (r'`[\'"$@-]', Punctuation), + (r'"', String.Double, 'string'), + (r"'([^']|'')*'", String.Single), + (r'(\$|@@|@)((global|script|private|env):)?\w+', + Name.Variable), + (r'(%s)\b' % '|'.join(keywords), Keyword), + (r'-(%s)\b' % '|'.join(operators), Operator), + (r'(%s)-[a-z_]\w*\b' % '|'.join(verbs), Name.Builtin), + (r'(%s)\s' % '|'.join(aliases_), Name.Builtin), + (r'\[[a-z_\[][\w. `,\[\]]*\]', Name.Constant), # .net [type]s + (r'-[a-z_]\w*', Name), + (r'\w+', Name), + (r'[.,;:@{}\[\]$()=+*/\\&%!~?^`|<>-]', Punctuation), + ], + 'child': [ + (r'\)', Punctuation, '#pop'), + include('root'), + ], + 'multline': [ + (r'[^#&.]+', Comment.Multiline), + (r'#(>|>)', Comment.Multiline, '#pop'), + (r'\.(%s)' % '|'.join(commenthelp), String.Doc), + (r'[#&.]', Comment.Multiline), + ], + 'string': [ + (r"`[0abfnrtv'\"$`]", String.Escape), + (r'[^$`"]+', String.Double), + (r'\$\(', Punctuation, 'child'), + (r'""', String.Double), + (r'[`$]', String.Double), + (r'"', String.Double, '#pop'), + ], + 'heredoc-double': [ + (r'\n"@', String.Heredoc, '#pop'), + (r'\$\(', Punctuation, 'child'), + (r'[^@\n]+"]', String.Heredoc), + (r".", String.Heredoc), + ] + } + + +class PowerShellSessionLexer(ShellSessionBaseLexer): + """ + Lexer for PowerShell sessions, i.e. command lines, including a + prompt, interspersed with output. + + .. versionadded:: 2.1 + """ + + name = 'PowerShell Session' + aliases = ['pwsh-session', 'ps1con'] + filenames = [] + mimetypes = [] + + _innerLexerCls = PowerShellLexer + _bare_continuation = True + _ps1rgx = re.compile(r'^((?:\[[^]]+\]: )?PS[^>]*> ?)(.*\n?)') + _ps2 = '> ' + + +class FishShellLexer(RegexLexer): + """ + Lexer for Fish shell scripts. + + .. versionadded:: 2.1 + """ + + name = 'Fish' + aliases = ['fish', 'fishshell'] + filenames = ['*.fish', '*.load'] + mimetypes = ['application/x-fish'] + + tokens = { + 'root': [ + include('basic'), + include('data'), + include('interp'), + ], + 'interp': [ + (r'\$\(\(', Keyword, 'math'), + (r'\(', Keyword, 'paren'), + (r'\$#?(\w+|.)', Name.Variable), + ], + 'basic': [ + (r'\b(begin|end|if|else|while|break|for|in|return|function|block|' + r'case|continue|switch|not|and|or|set|echo|exit|pwd|true|false|' + r'cd|count|test)(\s*)\b', + bygroups(Keyword, Text)), + (r'\b(alias|bg|bind|breakpoint|builtin|command|commandline|' + r'complete|contains|dirh|dirs|emit|eval|exec|fg|fish|fish_config|' + r'fish_indent|fish_pager|fish_prompt|fish_right_prompt|' + r'fish_update_completions|fishd|funced|funcsave|functions|help|' + r'history|isatty|jobs|math|mimedb|nextd|open|popd|prevd|psub|' + r'pushd|random|read|set_color|source|status|trap|type|ulimit|' + r'umask|vared|fc|getopts|hash|kill|printf|time|wait)\s*\b(?!\.)', + Name.Builtin), + (r'#.*\n', Comment), + (r'\\[\w\W]', String.Escape), + (r'(\b\w+)(\s*)(=)', bygroups(Name.Variable, Whitespace, Operator)), + (r'[\[\]()=]', Operator), + (r'<<-?\s*(\'?)\\?(\w+)[\w\W]+?\2', String), + ], + 'data': [ + (r'(?s)\$?"(\\\\|\\[0-7]+|\\.|[^"\\$])*"', String.Double), + (r'"', String.Double, 'string'), + (r"(?s)\$'(\\\\|\\[0-7]+|\\.|[^'\\])*'", String.Single), + (r"(?s)'.*?'", String.Single), + (r';', Punctuation), + (r'&|\||\^|<|>', Operator), + (r'\s+', Text), + (r'\d+(?= |\Z)', Number), + (r'[^=\s\[\]{}()$"\'`\\<&|;]+', Text), + ], + 'string': [ + (r'"', String.Double, '#pop'), + (r'(?s)(\\\\|\\[0-7]+|\\.|[^"\\$])+', String.Double), + include('interp'), + ], + 'paren': [ + (r'\)', Keyword, '#pop'), + include('root'), + ], + 'math': [ + (r'\)\)', Keyword, '#pop'), + (r'[-+*/%^|&]|\*\*|\|\|', Operator), + (r'\d+#\d+', Number), + (r'\d+#(?! )', Number), + (r'\d+', Number), + include('root'), + ], + } + +class ExeclineLexer(RegexLexer): + """ + Lexer for Laurent Bercot's execline language + (https://skarnet.org/software/execline). + + .. versionadded:: 2.7 + """ + + name = 'execline' + aliases = ['execline'] + filenames = ['*.exec'] + + tokens = { + 'root': [ + include('basic'), + include('data'), + include('interp') + ], + 'interp': [ + (r'\$\{', String.Interpol, 'curly'), + (r'\$[\w@#]+', Name.Variable), # user variable + (r'\$', Text), + ], + 'basic': [ + (r'\b(background|backtick|cd|define|dollarat|elgetopt|' + r'elgetpositionals|elglob|emptyenv|envfile|exec|execlineb|' + r'exit|export|fdblock|fdclose|fdmove|fdreserve|fdswap|' + r'forbacktickx|foreground|forstdin|forx|getcwd|getpid|heredoc|' + r'homeof|if|ifelse|ifte|ifthenelse|importas|loopwhilex|' + r'multidefine|multisubstitute|pipeline|piperw|posix-cd|' + r'redirfd|runblock|shift|trap|tryexec|umask|unexport|wait|' + r'withstdinas)\b', Name.Builtin), + (r'\A#!.+\n', Comment.Hashbang), + (r'#.*\n', Comment.Single), + (r'[{}]', Operator) + ], + 'data': [ + (r'(?s)"(\\.|[^"\\$])*"', String.Double), + (r'"', String.Double, 'string'), + (r'\s+', Text), + (r'[^\s{}$"\\]+', Text) + ], + 'string': [ + (r'"', String.Double, '#pop'), + (r'(?s)(\\\\|\\.|[^"\\$])+', String.Double), + include('interp'), + ], + 'curly': [ + (r'\}', String.Interpol, '#pop'), + (r'[\w#@]+', Name.Variable), + include('root') + ] + + } + + def analyse_text(text): + if shebang_matches(text, r'execlineb'): + return 1 diff --git a/pygments/lexers/sieve.py b/pygments/lexers/sieve.py new file mode 100644 index 0000000..ab43db8 --- /dev/null +++ b/pygments/lexers/sieve.py @@ -0,0 +1,78 @@ +""" + pygments.lexers.sieve + ~~~~~~~~~~~~~~~~~~~~~ + + Lexer for Sieve file format. + + https://tools.ietf.org/html/rfc5228 + https://tools.ietf.org/html/rfc5173 + https://tools.ietf.org/html/rfc5229 + https://tools.ietf.org/html/rfc5230 + https://tools.ietf.org/html/rfc5232 + https://tools.ietf.org/html/rfc5235 + https://tools.ietf.org/html/rfc5429 + https://tools.ietf.org/html/rfc8580 + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups +from pygments.token import Comment, Name, Literal, String, Text, Punctuation, \ + Keyword + +__all__ = ["SieveLexer"] + + +class SieveLexer(RegexLexer): + """ + Lexer for sieve format. + + .. versionadded:: 2.6 + """ + name = 'Sieve' + filenames = ['*.siv', '*.sieve'] + aliases = ['sieve'] + + tokens = { + 'root': [ + (r'\s+', Text), + (r'[();,{}\[\]]', Punctuation), + # import: + (r'(?i)require', + Keyword.Namespace), + # tags: + (r'(?i)(:)(addresses|all|contains|content|create|copy|comparator|' + r'count|days|detail|domain|fcc|flags|from|handle|importance|is|' + r'localpart|length|lowerfirst|lower|matches|message|mime|options|' + r'over|percent|quotewildcard|raw|regex|specialuse|subject|text|' + r'under|upperfirst|upper|value)', + bygroups(Name.Tag, Name.Tag)), + # tokens: + (r'(?i)(address|addflag|allof|anyof|body|discard|elsif|else|envelope|' + r'ereject|exists|false|fileinto|if|hasflag|header|keep|' + r'notify_method_capability|notify|not|redirect|reject|removeflag|' + r'setflag|size|spamtest|stop|string|true|vacation|virustest)', + Name.Builtin), + (r'(?i)set', + Keyword.Declaration), + # number: + (r'([0-9.]+)([kmgKMG])?', + bygroups(Literal.Number, Literal.Number)), + # comment: + (r'#.*$', + Comment.Single), + (r'/\*.*\*/', + Comment.Multiline), + # string: + (r'"[^"]*?"', + String), + # text block: + (r'text:', + Name.Tag, 'text'), + ], + 'text': [ + (r'[^.].*?\n', String), + (r'^\.', Punctuation, "#pop"), + ] + } diff --git a/pygments/lexers/slash.py b/pygments/lexers/slash.py new file mode 100644 index 0000000..e3434c5 --- /dev/null +++ b/pygments/lexers/slash.py @@ -0,0 +1,184 @@ +""" + pygments.lexers.slash + ~~~~~~~~~~~~~~~~~~~~~ + + Lexer for the `Slash `_ programming + language. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import ExtendedRegexLexer, bygroups, DelegatingLexer +from pygments.token import Name, Number, String, Comment, Punctuation, \ + Other, Keyword, Operator, Whitespace + +__all__ = ['SlashLexer'] + + +class SlashLanguageLexer(ExtendedRegexLexer): + _nkw = r'(?=[^a-zA-Z_0-9])' + + def move_state(new_state): + return ("#pop", new_state) + + def right_angle_bracket(lexer, match, ctx): + if len(ctx.stack) > 1 and ctx.stack[-2] == "string": + ctx.stack.pop() + yield match.start(), String.Interpol, '}' + ctx.pos = match.end() + pass + + tokens = { + "root": [ + (r"<%=", Comment.Preproc, move_state("slash")), + (r"<%!!", Comment.Preproc, move_state("slash")), + (r"<%#.*?%>", Comment.Multiline), + (r"<%", Comment.Preproc, move_state("slash")), + (r".|\n", Other), + ], + "string": [ + (r"\\", String.Escape, move_state("string_e")), + (r"\"", String, move_state("slash")), + (r"#\{", String.Interpol, "slash"), + (r'.|\n', String), + ], + "string_e": [ + (r'n', String.Escape, move_state("string")), + (r't', String.Escape, move_state("string")), + (r'r', String.Escape, move_state("string")), + (r'e', String.Escape, move_state("string")), + (r'x[a-fA-F0-9]{2}', String.Escape, move_state("string")), + (r'.', String.Escape, move_state("string")), + ], + "regexp": [ + (r'}[a-z]*', String.Regex, move_state("slash")), + (r'\\(.|\n)', String.Regex), + (r'{', String.Regex, "regexp_r"), + (r'.|\n', String.Regex), + ], + "regexp_r": [ + (r'}[a-z]*', String.Regex, "#pop"), + (r'\\(.|\n)', String.Regex), + (r'{', String.Regex, "regexp_r"), + ], + "slash": [ + (r"%>", Comment.Preproc, move_state("root")), + (r"\"", String, move_state("string")), + (r"'[a-zA-Z0-9_]+", String), + (r'%r{', String.Regex, move_state("regexp")), + (r'/\*.*?\*/', Comment.Multiline), + (r"(#|//).*?\n", Comment.Single), + (r'-?[0-9]+e[+-]?[0-9]+', Number.Float), + (r'-?[0-9]+\.[0-9]+(e[+-]?[0-9]+)?', Number.Float), + (r'-?[0-9]+', Number.Integer), + (r'nil'+_nkw, Name.Builtin), + (r'true'+_nkw, Name.Builtin), + (r'false'+_nkw, Name.Builtin), + (r'self'+_nkw, Name.Builtin), + (r'(class)(\s+)([A-Z][a-zA-Z0-9_\']*)', + bygroups(Keyword, Whitespace, Name.Class)), + (r'class'+_nkw, Keyword), + (r'extends'+_nkw, Keyword), + (r'(def)(\s+)(self)(\s*)(\.)(\s*)([a-z_][a-zA-Z0-9_\']*=?|<<|>>|==|<=>|<=|<|>=|>|\+|-(self)?|~(self)?|\*|/|%|^|&&|&|\||\[\]=?)', + bygroups(Keyword, Whitespace, Name.Builtin, Whitespace, Punctuation, Whitespace, Name.Function)), + (r'(def)(\s+)([a-z_][a-zA-Z0-9_\']*=?|<<|>>|==|<=>|<=|<|>=|>|\+|-(self)?|~(self)?|\*|/|%|^|&&|&|\||\[\]=?)', + bygroups(Keyword, Whitespace, Name.Function)), + (r'def'+_nkw, Keyword), + (r'if'+_nkw, Keyword), + (r'elsif'+_nkw, Keyword), + (r'else'+_nkw, Keyword), + (r'unless'+_nkw, Keyword), + (r'for'+_nkw, Keyword), + (r'in'+_nkw, Keyword), + (r'while'+_nkw, Keyword), + (r'until'+_nkw, Keyword), + (r'and'+_nkw, Keyword), + (r'or'+_nkw, Keyword), + (r'not'+_nkw, Keyword), + (r'lambda'+_nkw, Keyword), + (r'try'+_nkw, Keyword), + (r'catch'+_nkw, Keyword), + (r'return'+_nkw, Keyword), + (r'next'+_nkw, Keyword), + (r'last'+_nkw, Keyword), + (r'throw'+_nkw, Keyword), + (r'use'+_nkw, Keyword), + (r'switch'+_nkw, Keyword), + (r'\\', Keyword), + (r'λ', Keyword), + (r'__FILE__'+_nkw, Name.Builtin.Pseudo), + (r'__LINE__'+_nkw, Name.Builtin.Pseudo), + (r'[A-Z][a-zA-Z0-9_\']*'+_nkw, Name.Constant), + (r'[a-z_][a-zA-Z0-9_\']*'+_nkw, Name), + (r'@[a-z_][a-zA-Z0-9_\']*'+_nkw, Name.Variable.Instance), + (r'@@[a-z_][a-zA-Z0-9_\']*'+_nkw, Name.Variable.Class), + (r'\(', Punctuation), + (r'\)', Punctuation), + (r'\[', Punctuation), + (r'\]', Punctuation), + (r'\{', Punctuation), + (r'\}', right_angle_bracket), + (r';', Punctuation), + (r',', Punctuation), + (r'<<=', Operator), + (r'>>=', Operator), + (r'<<', Operator), + (r'>>', Operator), + (r'==', Operator), + (r'!=', Operator), + (r'=>', Operator), + (r'=', Operator), + (r'<=>', Operator), + (r'<=', Operator), + (r'>=', Operator), + (r'<', Operator), + (r'>', Operator), + (r'\+\+', Operator), + (r'\+=', Operator), + (r'-=', Operator), + (r'\*\*=', Operator), + (r'\*=', Operator), + (r'\*\*', Operator), + (r'\*', Operator), + (r'/=', Operator), + (r'\+', Operator), + (r'-', Operator), + (r'/', Operator), + (r'%=', Operator), + (r'%', Operator), + (r'^=', Operator), + (r'&&=', Operator), + (r'&=', Operator), + (r'&&', Operator), + (r'&', Operator), + (r'\|\|=', Operator), + (r'\|=', Operator), + (r'\|\|', Operator), + (r'\|', Operator), + (r'!', Operator), + (r'\.\.\.', Operator), + (r'\.\.', Operator), + (r'\.', Operator), + (r'::', Operator), + (r':', Operator), + (r'(\s|\n)+', Whitespace), + (r'[a-z_][a-zA-Z0-9_\']*', Name.Variable), + ], + } + + +class SlashLexer(DelegatingLexer): + """ + Lexer for the Slash programming language. + + .. versionadded:: 2.4 + """ + + name = 'Slash' + aliases = ['slash'] + filenames = ['*.sla'] + + def __init__(self, **options): + from pygments.lexers.web import HtmlLexer + super().__init__(HtmlLexer, SlashLanguageLexer, **options) diff --git a/pygments/lexers/smalltalk.py b/pygments/lexers/smalltalk.py new file mode 100644 index 0000000..ca41c07 --- /dev/null +++ b/pygments/lexers/smalltalk.py @@ -0,0 +1,196 @@ +""" + pygments.lexers.smalltalk + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for Smalltalk and related languages. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, include, bygroups, default +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation + +__all__ = ['SmalltalkLexer', 'NewspeakLexer'] + + +class SmalltalkLexer(RegexLexer): + """ + For Smalltalk syntax. + Contributed by Stefan Matthias Aust. + Rewritten by Nils Winter. + + .. versionadded:: 0.10 + """ + name = 'Smalltalk' + url = 'http://www.smalltalk.org/' + filenames = ['*.st'] + aliases = ['smalltalk', 'squeak', 'st'] + mimetypes = ['text/x-smalltalk'] + + tokens = { + 'root': [ + (r'(<)(\w+:)(.*?)(>)', bygroups(Text, Keyword, Text, Text)), + include('squeak fileout'), + include('whitespaces'), + include('method definition'), + (r'(\|)([\w\s]*)(\|)', bygroups(Operator, Name.Variable, Operator)), + include('objects'), + (r'\^|\:=|\_', Operator), + # temporaries + (r'[\]({}.;!]', Text), + ], + 'method definition': [ + # Not perfect can't allow whitespaces at the beginning and the + # without breaking everything + (r'([a-zA-Z]+\w*:)(\s*)(\w+)', + bygroups(Name.Function, Text, Name.Variable)), + (r'^(\b[a-zA-Z]+\w*\b)(\s*)$', bygroups(Name.Function, Text)), + (r'^([-+*/\\~<>=|&!?,@%]+)(\s*)(\w+)(\s*)$', + bygroups(Name.Function, Text, Name.Variable, Text)), + ], + 'blockvariables': [ + include('whitespaces'), + (r'(:)(\s*)(\w+)', + bygroups(Operator, Text, Name.Variable)), + (r'\|', Operator, '#pop'), + default('#pop'), # else pop + ], + 'literals': [ + (r"'(''|[^'])*'", String, 'afterobject'), + (r'\$.', String.Char, 'afterobject'), + (r'#\(', String.Symbol, 'parenth'), + (r'\)', Text, 'afterobject'), + (r'(\d+r)?-?\d+(\.\d+)?(e-?\d+)?', Number, 'afterobject'), + ], + '_parenth_helper': [ + include('whitespaces'), + (r'(\d+r)?-?\d+(\.\d+)?(e-?\d+)?', Number), + (r'[-+*/\\~<>=|&#!?,@%\w:]+', String.Symbol), + # literals + (r"'(''|[^'])*'", String), + (r'\$.', String.Char), + (r'#*\(', String.Symbol, 'inner_parenth'), + ], + 'parenth': [ + # This state is a bit tricky since + # we can't just pop this state + (r'\)', String.Symbol, ('root', 'afterobject')), + include('_parenth_helper'), + ], + 'inner_parenth': [ + (r'\)', String.Symbol, '#pop'), + include('_parenth_helper'), + ], + 'whitespaces': [ + # skip whitespace and comments + (r'\s+', Text), + (r'"(""|[^"])*"', Comment), + ], + 'objects': [ + (r'\[', Text, 'blockvariables'), + (r'\]', Text, 'afterobject'), + (r'\b(self|super|true|false|nil|thisContext)\b', + Name.Builtin.Pseudo, 'afterobject'), + (r'\b[A-Z]\w*(?!:)\b', Name.Class, 'afterobject'), + (r'\b[a-z]\w*(?!:)\b', Name.Variable, 'afterobject'), + (r'#("(""|[^"])*"|[-+*/\\~<>=|&!?,@%]+|[\w:]+)', + String.Symbol, 'afterobject'), + include('literals'), + ], + 'afterobject': [ + (r'! !$', Keyword, '#pop'), # squeak chunk delimiter + include('whitespaces'), + (r'\b(ifTrue:|ifFalse:|whileTrue:|whileFalse:|timesRepeat:)', + Name.Builtin, '#pop'), + (r'\b(new\b(?!:))', Name.Builtin), + (r'\:=|\_', Operator, '#pop'), + (r'\b[a-zA-Z]+\w*:', Name.Function, '#pop'), + (r'\b[a-zA-Z]+\w*', Name.Function), + (r'\w+:?|[-+*/\\~<>=|&!?,@%]+', Name.Function, '#pop'), + (r'\.', Punctuation, '#pop'), + (r';', Punctuation), + (r'[\])}]', Text), + (r'[\[({]', Text, '#pop'), + ], + 'squeak fileout': [ + # Squeak fileout format (optional) + (r'^"(""|[^"])*"!', Keyword), + (r"^'(''|[^'])*'!", Keyword), + (r'^(!)(\w+)( commentStamp: )(.*?)( prior: .*?!\n)(.*?)(!)', + bygroups(Keyword, Name.Class, Keyword, String, Keyword, Text, Keyword)), + (r"^(!)(\w+(?: class)?)( methodsFor: )('(?:''|[^'])*')(.*?!)", + bygroups(Keyword, Name.Class, Keyword, String, Keyword)), + (r'^(\w+)( subclass: )(#\w+)' + r'(\s+instanceVariableNames: )(.*?)' + r'(\s+classVariableNames: )(.*?)' + r'(\s+poolDictionaries: )(.*?)' + r'(\s+category: )(.*?)(!)', + bygroups(Name.Class, Keyword, String.Symbol, Keyword, String, Keyword, + String, Keyword, String, Keyword, String, Keyword)), + (r'^(\w+(?: class)?)(\s+instanceVariableNames: )(.*?)(!)', + bygroups(Name.Class, Keyword, String, Keyword)), + (r'(!\n)(\].*)(! !)$', bygroups(Keyword, Text, Keyword)), + (r'! !$', Keyword), + ], + } + + +class NewspeakLexer(RegexLexer): + """ + For Newspeak syntax. + + .. versionadded:: 1.1 + """ + name = 'Newspeak' + url = 'http://newspeaklanguage.org/' + filenames = ['*.ns2'] + aliases = ['newspeak', ] + mimetypes = ['text/x-newspeak'] + + tokens = { + 'root': [ + (r'\b(Newsqueak2)\b', Keyword.Declaration), + (r"'[^']*'", String), + (r'\b(class)(\s+)(\w+)(\s*)', + bygroups(Keyword.Declaration, Text, Name.Class, Text)), + (r'\b(mixin|self|super|private|public|protected|nil|true|false)\b', + Keyword), + (r'(\w+\:)(\s*)([a-zA-Z_]\w+)', + bygroups(Name.Function, Text, Name.Variable)), + (r'(\w+)(\s*)(=)', + bygroups(Name.Attribute, Text, Operator)), + (r'<\w+>', Comment.Special), + include('expressionstat'), + include('whitespace') + ], + + 'expressionstat': [ + (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float), + (r'\d+', Number.Integer), + (r':\w+', Name.Variable), + (r'(\w+)(::)', bygroups(Name.Variable, Operator)), + (r'\w+:', Name.Function), + (r'\w+', Name.Variable), + (r'\(|\)', Punctuation), + (r'\[|\]', Punctuation), + (r'\{|\}', Punctuation), + + (r'(\^|\+|\/|~|\*|<|>|=|@|%|\||&|\?|!|,|-|:)', Operator), + (r'\.|;', Punctuation), + include('whitespace'), + include('literals'), + ], + 'literals': [ + (r'\$.', String), + (r"'[^']*'", String), + (r"#'[^']*'", String.Symbol), + (r"#\w+:?", String.Symbol), + (r"#(\+|\/|~|\*|<|>|=|@|%|\||&|\?|!|,|-)+", String.Symbol) + ], + 'whitespace': [ + (r'\s+', Text), + (r'"[^"]*"', Comment) + ], + } diff --git a/pygments/lexers/smithy.py b/pygments/lexers/smithy.py new file mode 100644 index 0000000..69b576e --- /dev/null +++ b/pygments/lexers/smithy.py @@ -0,0 +1,78 @@ +""" + pygments.lexers.smithy + ~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for the Smithy IDL. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups, words +from pygments.token import Text, Comment, Keyword, Name, String, \ + Number, Whitespace, Punctuation + +__all__ = ['SmithyLexer'] + + +class SmithyLexer(RegexLexer): + """ + For Smithy IDL + + .. versionadded:: 2.10 + """ + name = 'Smithy' + url = 'https://awslabs.github.io/smithy/' + filenames = ['*.smithy'] + aliases = ['smithy'] + + unquoted = r'[A-Za-z0-9_\.#$-]+' + identifier = r"[A-Za-z0-9_\.#$-]+" + + simple_shapes = ( + 'use', 'byte', 'short', 'integer', 'long', 'float', 'document', + 'double', 'bigInteger', 'bigDecimal', 'boolean', 'blob', 'string', + 'timestamp', + ) + + aggregate_shapes = ( + 'apply', 'list', 'map', 'set', 'structure', 'union', 'resource', + 'operation', 'service', 'trait' + ) + + tokens = { + 'root': [ + (r'///.*$', Comment.Multiline), + (r'//.*$', Comment), + (r'@[0-9a-zA-Z\.#-]*', Name.Decorator), + (r'(=)', Name.Decorator), + (r'^(\$version)(:)(.+)', + bygroups(Keyword.Declaration, Name.Decorator, Name.Class)), + (r'^(namespace)(\s+' + identifier + r')\b', + bygroups(Keyword.Declaration, Name.Class)), + (words(simple_shapes, + prefix=r'^', suffix=r'(\s+' + identifier + r')\b'), + bygroups(Keyword.Declaration, Name.Class)), + (words(aggregate_shapes, + prefix=r'^', suffix=r'(\s+' + identifier + r')'), + bygroups(Keyword.Declaration, Name.Class)), + (r'^(metadata)(\s+)((?:\S+)|(?:\"[^"]+\"))(\s*)(=)', + bygroups(Keyword.Declaration, Whitespace, Name.Class, + Whitespace, Name.Decorator)), + (r"(true|false|null)", Keyword.Constant), + (r"(-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?)", Number), + (identifier + ":", Name.Label), + (identifier, Name.Variable.Class), + (r'\[', Text, "#push"), + (r'\]', Text, "#pop"), + (r'\(', Text, "#push"), + (r'\)', Text, "#pop"), + (r'\{', Text, "#push"), + (r'\}', Text, "#pop"), + (r'"{3}(\\\\|\n|\\")*"{3}', String.Doc), + (r'"(\\\\|\n|\\"|[^"])*"', String.Double), + (r"'(\\\\|\n|\\'|[^'])*'", String.Single), + (r'[:,]+', Punctuation), + (r'\s+', Whitespace), + ] + } diff --git a/pygments/lexers/smv.py b/pygments/lexers/smv.py new file mode 100644 index 0000000..b51e170 --- /dev/null +++ b/pygments/lexers/smv.py @@ -0,0 +1,78 @@ +""" + pygments.lexers.smv + ~~~~~~~~~~~~~~~~~~~ + + Lexers for the SMV languages. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, words +from pygments.token import Comment, Keyword, Name, Number, Operator, \ + Punctuation, Text + +__all__ = ['NuSMVLexer'] + + +class NuSMVLexer(RegexLexer): + """ + Lexer for the NuSMV language. + + .. versionadded:: 2.2 + """ + + name = 'NuSMV' + aliases = ['nusmv'] + filenames = ['*.smv'] + mimetypes = [] + + tokens = { + 'root': [ + # Comments + (r'(?s)\/\-\-.*?\-\-/', Comment), + (r'--.*\n', Comment), + + # Reserved + (words(('MODULE', 'DEFINE', 'MDEFINE', 'CONSTANTS', 'VAR', 'IVAR', + 'FROZENVAR', 'INIT', 'TRANS', 'INVAR', 'SPEC', 'CTLSPEC', + 'LTLSPEC', 'PSLSPEC', 'COMPUTE', 'NAME', 'INVARSPEC', + 'FAIRNESS', 'JUSTICE', 'COMPASSION', 'ISA', 'ASSIGN', + 'CONSTRAINT', 'SIMPWFF', 'CTLWFF', 'LTLWFF', 'PSLWFF', + 'COMPWFF', 'IN', 'MIN', 'MAX', 'MIRROR', 'PRED', + 'PREDICATES'), suffix=r'(?![\w$#-])'), + Keyword.Declaration), + (r'process(?![\w$#-])', Keyword), + (words(('array', 'of', 'boolean', 'integer', 'real', 'word'), + suffix=r'(?![\w$#-])'), Keyword.Type), + (words(('case', 'esac'), suffix=r'(?![\w$#-])'), Keyword), + (words(('word1', 'bool', 'signed', 'unsigned', 'extend', 'resize', + 'sizeof', 'uwconst', 'swconst', 'init', 'self', 'count', + 'abs', 'max', 'min'), suffix=r'(?![\w$#-])'), + Name.Builtin), + (words(('EX', 'AX', 'EF', 'AF', 'EG', 'AG', 'E', 'F', 'O', 'G', + 'H', 'X', 'Y', 'Z', 'A', 'U', 'S', 'V', 'T', 'BU', 'EBF', + 'ABF', 'EBG', 'ABG', 'next', 'mod', 'union', 'in', 'xor', + 'xnor'), suffix=r'(?![\w$#-])'), + Operator.Word), + (words(('TRUE', 'FALSE'), suffix=r'(?![\w$#-])'), Keyword.Constant), + + # Names + (r'[a-zA-Z_][\w$#-]*', Name.Variable), + + # Operators + (r':=', Operator), + (r'[-&|+*/<>!=]', Operator), + + # Literals + (r'\-?\d+\b', Number.Integer), + (r'0[su][bB]\d*_[01_]+', Number.Bin), + (r'0[su][oO]\d*_[0-7_]+', Number.Oct), + (r'0[su][dD]\d*_[\d_]+', Number.Decimal), + (r'0[su][hH]\d*_[\da-fA-F_]+', Number.Hex), + + # Whitespace, punctuation and the rest + (r'\s+', Text.Whitespace), + (r'[()\[\]{};?:.,]', Punctuation), + ], + } diff --git a/pygments/lexers/snobol.py b/pygments/lexers/snobol.py new file mode 100644 index 0000000..c336d19 --- /dev/null +++ b/pygments/lexers/snobol.py @@ -0,0 +1,82 @@ +""" + pygments.lexers.snobol + ~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for the SNOBOL language. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation + +__all__ = ['SnobolLexer'] + + +class SnobolLexer(RegexLexer): + """ + Lexer for the SNOBOL4 programming language. + + Recognizes the common ASCII equivalents of the original SNOBOL4 operators. + Does not require spaces around binary operators. + + .. versionadded:: 1.5 + """ + + name = "Snobol" + aliases = ["snobol"] + filenames = ['*.snobol'] + mimetypes = ['text/x-snobol'] + + tokens = { + # root state, start of line + # comments, continuation lines, and directives start in column 1 + # as do labels + 'root': [ + (r'\*.*\n', Comment), + (r'[+.] ', Punctuation, 'statement'), + (r'-.*\n', Comment), + (r'END\s*\n', Name.Label, 'heredoc'), + (r'[A-Za-z$][\w$]*', Name.Label, 'statement'), + (r'\s+', Text, 'statement'), + ], + # statement state, line after continuation or label + 'statement': [ + (r'\s*\n', Text, '#pop'), + (r'\s+', Text), + (r'(?<=[^\w.])(LT|LE|EQ|NE|GE|GT|INTEGER|IDENT|DIFFER|LGT|SIZE|' + r'REPLACE|TRIM|DUPL|REMDR|DATE|TIME|EVAL|APPLY|OPSYN|LOAD|UNLOAD|' + r'LEN|SPAN|BREAK|ANY|NOTANY|TAB|RTAB|REM|POS|RPOS|FAIL|FENCE|' + r'ABORT|ARB|ARBNO|BAL|SUCCEED|INPUT|OUTPUT|TERMINAL)(?=[^\w.])', + Name.Builtin), + (r'[A-Za-z][\w.]*', Name), + # ASCII equivalents of original operators + # | for the EBCDIC equivalent, ! likewise + # \ for EBCDIC negation + (r'\*\*|[?$.!%*/#+\-@|&\\=]', Operator), + (r'"[^"]*"', String), + (r"'[^']*'", String), + # Accept SPITBOL syntax for real numbers + # as well as Macro SNOBOL4 + (r'[0-9]+(?=[^.EeDd])', Number.Integer), + (r'[0-9]+(\.[0-9]*)?([EDed][-+]?[0-9]+)?', Number.Float), + # Goto + (r':', Punctuation, 'goto'), + (r'[()<>,;]', Punctuation), + ], + # Goto block + 'goto': [ + (r'\s*\n', Text, "#pop:2"), + (r'\s+', Text), + (r'F|S', Keyword), + (r'(\()([A-Za-z][\w.]*)(\))', + bygroups(Punctuation, Name.Label, Punctuation)) + ], + # everything after the END statement is basically one + # big heredoc. + 'heredoc': [ + (r'.*\n', String.Heredoc) + ] + } diff --git a/pygments/lexers/solidity.py b/pygments/lexers/solidity.py new file mode 100644 index 0000000..46a3b9e --- /dev/null +++ b/pygments/lexers/solidity.py @@ -0,0 +1,87 @@ +""" + pygments.lexers.solidity + ~~~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for Solidity. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups, include, words +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Whitespace + +__all__ = ['SolidityLexer'] + + +class SolidityLexer(RegexLexer): + """ + For Solidity source code. + + .. versionadded:: 2.5 + """ + + name = 'Solidity' + aliases = ['solidity'] + filenames = ['*.sol'] + mimetypes = [] + + datatype = ( + r'\b(address|bool|(?:(?:bytes|hash|int|string|uint)(?:8|16|24|32|40|48|56|64' + r'|72|80|88|96|104|112|120|128|136|144|152|160|168|176|184|192|200|208' + r'|216|224|232|240|248|256)?))\b' + ) + + tokens = { + 'root': [ + include('whitespace'), + include('comments'), + (r'\bpragma\s+solidity\b', Keyword, 'pragma'), + (r'\b(contract)(\s+)([a-zA-Z_]\w*)', + bygroups(Keyword, Whitespace, Name.Entity)), + (datatype + r'(\s+)((?:external|public|internal|private)\s+)?' + + r'([a-zA-Z_]\w*)', + bygroups(Keyword.Type, Whitespace, Keyword, Name.Variable)), + (r'\b(enum|event|function|struct)(\s+)([a-zA-Z_]\w*)', + bygroups(Keyword.Type, Whitespace, Name.Variable)), + (r'\b(msg|block|tx)\.([A-Za-z_][a-zA-Z0-9_]*)\b', Keyword), + (words(( + 'block', 'break', 'constant', 'constructor', 'continue', + 'contract', 'do', 'else', 'external', 'false', 'for', + 'function', 'if', 'import', 'inherited', 'internal', 'is', + 'library', 'mapping', 'memory', 'modifier', 'msg', 'new', + 'payable', 'private', 'public', 'require', 'return', + 'returns', 'struct', 'suicide', 'throw', 'this', 'true', + 'tx', 'var', 'while'), prefix=r'\b', suffix=r'\b'), + Keyword.Type), + (words(('keccak256',), prefix=r'\b', suffix=r'\b'), Name.Builtin), + (datatype, Keyword.Type), + include('constants'), + (r'[a-zA-Z_]\w*', Text), + (r'[~!%^&*+=|?:<>/-]', Operator), + (r'[.;{}(),\[\]]', Punctuation) + ], + 'comments': [ + (r'//(\n|[\w\W]*?[^\\]\n)', Comment.Single), + (r'/(\\\n)?[*][\w\W]*?[*](\\\n)?/', Comment.Multiline), + (r'/(\\\n)?[*][\w\W]*', Comment.Multiline) + ], + 'constants': [ + (r'("(\\"|.)*?")', String.Double), + (r"('(\\'|.)*?')", String.Single), + (r'\b0[xX][0-9a-fA-F]+\b', Number.Hex), + (r'\b\d+\b', Number.Decimal), + ], + 'pragma': [ + include('whitespace'), + include('comments'), + (r'(\^|>=|<)(\s*)(\d+\.\d+\.\d+)', + bygroups(Operator, Whitespace, Keyword)), + (r';', Punctuation, '#pop') + ], + 'whitespace': [ + (r'\s+', Whitespace), + (r'\n', Whitespace) + ] + } diff --git a/pygments/lexers/sophia.py b/pygments/lexers/sophia.py new file mode 100644 index 0000000..fc90181 --- /dev/null +++ b/pygments/lexers/sophia.py @@ -0,0 +1,103 @@ +""" + pygments.lexers.sophia + ~~~~~~~~~~~~~~~~~~~~~~ + + Lexer for Sophia. + + Derived from pygments/lexers/reason.py. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, include, default, words +from pygments.token import Comment, Keyword, Name, Number, Operator, \ + Punctuation, String, Text + +__all__ = ['SophiaLexer'] + +class SophiaLexer(RegexLexer): + """ + A Sophia lexer. + + .. versionadded:: 2.11 + """ + + name = 'Sophia' + aliases = ['sophia'] + filenames = ['*.aes'] + mimetypes = [] + + keywords = ( + 'contract', 'include', 'let', 'switch', 'type', 'record', 'datatype', + 'if', 'elif', 'else', 'function', 'stateful', 'payable', 'public', + 'entrypoint', 'private', 'indexed', 'namespace', 'interface', 'main', + 'using', 'as', 'for', 'hiding', + ) + + builtins = ('state', 'put', 'abort', 'require') + + word_operators = ('mod', 'band', 'bor', 'bxor', 'bnot') + + primitive_types = ('int', 'address', 'bool', 'bits', 'bytes', 'string', + 'list', 'option', 'char', 'unit', 'map', 'event', + 'hash', 'signature', 'oracle', 'oracle_query') + + tokens = { + 'escape-sequence': [ + (r'\\[\\"\'ntbr]', String.Escape), + (r'\\[0-9]{3}', String.Escape), + (r'\\x[0-9a-fA-F]{2}', String.Escape), + ], + 'root': [ + (r'\s+', Text.Whitespace), + (r'(true|false)\b', Keyword.Constant), + (r'\b([A-Z][\w\']*)(?=\s*\.)', Name.Class, 'dotted'), + (r'\b([A-Z][\w\']*)', Name.Function), + (r'//.*?\n', Comment.Single), + (r'\/\*(?!/)', Comment.Multiline, 'comment'), + + (r'0[xX][\da-fA-F][\da-fA-F_]*', Number.Hex), + (r'#[\da-fA-F][\da-fA-F_]*', Name.Label), + (r'\d[\d_]*', Number.Integer), + + (words(keywords, suffix=r'\b'), Keyword), + (words(builtins, suffix=r'\b'), Name.Builtin), + (words(word_operators, prefix=r'\b', suffix=r'\b'), Operator.Word), + (words(primitive_types, prefix=r'\b', suffix=r'\b'), Keyword.Type), + + (r'[=!<>+\\*/:&|?~@^-]', Operator.Word), + (r'[.;:{}(),\[\]]', Punctuation), + + (r"(ak_|ok_|oq_|ct_)[\w']*", Name.Label), + (r"[^\W\d][\w']*", Name), + + (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'", + String.Char), + (r"'.'", String.Char), + (r"'[a-z][\w]*", Name.Variable), + + (r'"', String.Double, 'string') + ], + 'comment': [ + (r'[^/*]+', Comment.Multiline), + (r'\/\*', Comment.Multiline, '#push'), + (r'\*\/', Comment.Multiline, '#pop'), + (r'\*', Comment.Multiline), + ], + 'string': [ + (r'[^\\"]+', String.Double), + include('escape-sequence'), + (r'\\\n', String.Double), + (r'"', String.Double, '#pop'), + ], + 'dotted': [ + (r'\s+', Text), + (r'\.', Punctuation), + (r'[A-Z][\w\']*(?=\s*\.)', Name.Function), + (r'[A-Z][\w\']*', Name.Function, '#pop'), + (r'[a-z_][\w\']*', Name, '#pop'), + default('#pop'), + ], + } + diff --git a/pygments/lexers/special.py b/pygments/lexers/special.py new file mode 100644 index 0000000..6db68a5 --- /dev/null +++ b/pygments/lexers/special.py @@ -0,0 +1,116 @@ +""" + pygments.lexers.special + ~~~~~~~~~~~~~~~~~~~~~~~ + + Special lexers. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import ast + +from pygments.lexer import Lexer, line_re +from pygments.token import Token, Error, Text, Generic +from pygments.util import get_choice_opt + + +__all__ = ['TextLexer', 'OutputLexer', 'RawTokenLexer'] + + +class TextLexer(Lexer): + """ + "Null" lexer, doesn't highlight anything. + """ + name = 'Text only' + aliases = ['text'] + filenames = ['*.txt'] + mimetypes = ['text/plain'] + priority = 0.01 + + def get_tokens_unprocessed(self, text): + yield 0, Text, text + + def analyse_text(text): + return TextLexer.priority + + +class OutputLexer(Lexer): + """ + Simple lexer that highlights everything as ``Token.Generic.Output``. + + .. versionadded:: 2.10 + """ + name = 'Text output' + aliases = ['output'] + + def get_tokens_unprocessed(self, text): + yield 0, Generic.Output, text + + +_ttype_cache = {} + + +class RawTokenLexer(Lexer): + """ + Recreate a token stream formatted with the `RawTokenFormatter`. + + Additional options accepted: + + `compress` + If set to ``"gz"`` or ``"bz2"``, decompress the token stream with + the given compression algorithm before lexing (default: ``""``). + """ + name = 'Raw token data' + aliases = [] + filenames = [] + mimetypes = ['application/x-pygments-tokens'] + + def __init__(self, **options): + self.compress = get_choice_opt(options, 'compress', + ['', 'none', 'gz', 'bz2'], '') + Lexer.__init__(self, **options) + + def get_tokens(self, text): + if self.compress: + if isinstance(text, str): + text = text.encode('latin1') + try: + if self.compress == 'gz': + import gzip + text = gzip.decompress(text) + elif self.compress == 'bz2': + import bz2 + text = bz2.decompress(text) + except OSError: + yield Error, text.decode('latin1') + if isinstance(text, bytes): + text = text.decode('latin1') + + # do not call Lexer.get_tokens() because stripping is not optional. + text = text.strip('\n') + '\n' + for i, t, v in self.get_tokens_unprocessed(text): + yield t, v + + def get_tokens_unprocessed(self, text): + length = 0 + for match in line_re.finditer(text): + try: + ttypestr, val = match.group().rstrip().split('\t', 1) + ttype = _ttype_cache.get(ttypestr) + if not ttype: + ttype = Token + ttypes = ttypestr.split('.')[1:] + for ttype_ in ttypes: + if not ttype_ or not ttype_[0].isupper(): + raise ValueError('malformed token name') + ttype = getattr(ttype, ttype_) + _ttype_cache[ttypestr] = ttype + val = ast.literal_eval(val) + if not isinstance(val, str): + raise ValueError('expected str') + except (SyntaxError, ValueError): + val = match.group() + ttype = Error + yield length, ttype, val + length += len(val) diff --git a/pygments/lexers/spice.py b/pygments/lexers/spice.py new file mode 100644 index 0000000..53f111b --- /dev/null +++ b/pygments/lexers/spice.py @@ -0,0 +1,71 @@ +""" + pygments.lexers.spice + ~~~~~~~~~~~~~~~~~~~~~ + + Lexers for the Spice programming language. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups, words +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Whitespace + +__all__ = ['SpiceLexer'] + + +class SpiceLexer(RegexLexer): + """ + For Spice source. + + .. versionadded:: 2.11 + """ + name = 'Spice' + url = 'https://www.spicelang.com' + filenames = ['*.spice'] + aliases = ['spice', 'spicelang'] + mimetypes = ['text/x-spice'] + + tokens = { + 'root': [ + (r'\n', Whitespace), + (r'\s+', Whitespace), + (r'\\\n', Text), + # comments + (r'//(.*?)\n', Comment.Single), + (r'/(\\\n)?[*]{2}(.|\n)*?[*](\\\n)?/', String.Doc), + (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline), + # keywords + (r'(import|as)\b', Keyword.Namespace), + (r'(f|p|type|struct|enum)\b', Keyword.Declaration), + (words(('if', 'else', 'for', 'foreach', 'while', 'break', + 'continue', 'return', 'assert', 'thread', 'unsafe', 'ext', + 'dll'), suffix=r'\b'), Keyword), + (words(('const', 'signed', 'unsigned', 'inline', 'public'), + suffix=r'\b'), Keyword.Pseudo), + (words(('new', 'switch', 'case', 'yield', 'stash', 'pick', 'sync', + 'class'), suffix=r'\b'), Keyword.Reserved), + (r'(true|false|nil)\b', Keyword.Constant), + (words(('double', 'int', 'short', 'long', 'byte', 'char', 'string', + 'bool', 'dyn'), suffix=r'\b'), Keyword.Type), + (words(('printf', 'sizeof', 'len', 'tid', 'join'), suffix=r'\b(\()'), + bygroups(Name.Builtin, Punctuation)), + # numeric literals + (r'[0-9]*[.][0-9]+', Number.Double), + (r'0[bB][01]+[sl]?', Number.Bin), + (r'0[oO][0-7]+[sl]?', Number.Oct), + (r'0[xXhH][0-9a-fA-F]+[sl]?', Number.Hex), + (r'(0[dD])?[0-9]+[sl]?', Number.Integer), + # string literal + (r'"(\\\\|\\[^\\]|[^"\\])*"', String), + # char literal + (r'\'(\\\\|\\[^\\]|[^\'\\])\'', String.Char), + # tokens + (r'<<=|>>=|<<|>>|<=|>=|\+=|-=|\*=|/=|\%=|\|=|&=|\^=|&&|\|\||&|\||' + r'\+\+|--|\%|\^|\~|==|!=|::|[.]{3}|[+\-*/&]', Operator), + (r'[|<>=!()\[\]{}.,;:\?]', Punctuation), + # identifiers + (r'[^\W\d]\w*', Name.Other), + ] + } diff --git a/pygments/lexers/sql.py b/pygments/lexers/sql.py new file mode 100644 index 0000000..043a3d1 --- /dev/null +++ b/pygments/lexers/sql.py @@ -0,0 +1,838 @@ +""" + pygments.lexers.sql + ~~~~~~~~~~~~~~~~~~~ + + Lexers for various SQL dialects and related interactive sessions. + + Postgres specific lexers: + + `PostgresLexer` + A SQL lexer for the PostgreSQL dialect. Differences w.r.t. the SQL + lexer are: + + - keywords and data types list parsed from the PG docs (run the + `_postgres_builtins` module to update them); + - Content of $-strings parsed using a specific lexer, e.g. the content + of a PL/Python function is parsed using the Python lexer; + - parse PG specific constructs: E-strings, $-strings, U&-strings, + different operators and punctuation. + + `PlPgsqlLexer` + A lexer for the PL/pgSQL language. Adds a few specific construct on + top of the PG SQL lexer (such as <