fix: bump to latest dclark/regexp2

Fixes #805
2025-10-08 22:52:04 +02:00 · 2023-09-09 11:46:02 +10:00
parent a242abf7af
commit 4dd2cbef84
26 changed files with 542 additions and 166 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,8 @@
 # Binaries for programs and plugins
+.git
+.idea
+.vscode
+.hermit
 *.exe
 *.dll
 *.so
--- a/24
+++ b/24
@@ -0,0 +1,24 @@
+VERSION = %(git describe --tags --dirty  --always)%
+export CGOENABLED = 0
+
+tokentype_enumer.go: types.go
+  build: go generate
+
+# Regenerate the list of lexers in the README
+README.md: lexers/*.go lexers/*/*.xml table.py
+  build: ./table.py
+  -clean
+
+implicit %{1}%{2}.min.%{3}: **/*.{css,js}
+  build: esbuild --bundle %{IN} --minify --outfile=%{OUT}
+
+implicit %{1}: cmd/*
+  cd cmd/%{1}
+  inputs: cmd/%{1}/**/* **/*.go
+  build: go build -ldflags="-X 'main.version=%{VERSION}'" -o ../../%{1} .
+
+#upload: chromad
+#  build:
+#    scp chromad root@swapoff.org:
+#    ssh root@swapoff.org 'install -m755 ./chromad /srv/http/swapoff.org/bin && service chromad restart'
+#    touch upload
--- a/README.md
+++ b/README.md
@@ -1,7 +1,6 @@
 # Chroma — A general purpose syntax highlighter in pure Go
-[![Golang Documentation](https://godoc.org/github.com/alecthomas/chroma?status.svg)](https://godoc.org/github.com/alecthomas/chroma) [![CI](https://github.com/alecthomas/chroma/actions/workflows/ci.yml/badge.svg)](https://github.com/alecthomas/chroma/actions/workflows/ci.yml) [![Slack chat](https://img.shields.io/static/v1?logo=slack&style=flat&label=slack&color=green&message=gophers)](https://invite.slack.golangbridge.org/)

-> **NOTE:** As Chroma has just been released, its API is still in flux. That said, the high-level interface should not change significantly.
+[![Golang Documentation](https://godoc.org/github.com/alecthomas/chroma?status.svg)](https://godoc.org/github.com/alecthomas/chroma) [![CI](https://github.com/alecthomas/chroma/actions/workflows/ci.yml/badge.svg)](https://github.com/alecthomas/chroma/actions/workflows/ci.yml) [![Slack chat](https://img.shields.io/static/v1?logo=slack&style=flat&label=slack&color=green&message=gophers)](https://invite.slack.golangbridge.org/)

 Chroma takes source code and other structured text and converts it into syntax
 highlighted HTML, ANSI-coloured text, etc.
@@ -10,6 +9,7 @@ Chroma is based heavily on [Pygments](http://pygments.org/), and includes
 translators for Pygments lexers and styles.

 <a id="markdown-table-of-contents" name="table-of-contents"></a>
+
 ## Table of Contents

 <!-- TOC -->
@@ -18,14 +18,14 @@ translators for Pygments lexers and styles.
 2. [Supported languages](#supported-languages)
 3. [Try it](#try-it)
 4. [Using the library](#using-the-library)
-    1. [Quick start](#quick-start)
-    2. [Identifying the language](#identifying-the-language)
-    3. [Formatting the output](#formatting-the-output)
-    4. [The HTML formatter](#the-html-formatter)
+   1. [Quick start](#quick-start)
+   2. [Identifying the language](#identifying-the-language)
+   3. [Formatting the output](#formatting-the-output)
+   4. [The HTML formatter](#the-html-formatter)
 5. [More detail](#more-detail)
-    1. [Lexers](#lexers)
-    2. [Formatters](#formatters)
-    3. [Styles](#styles)
+   1. [Lexers](#lexers)
+   2. [Formatters](#formatters)
+   3. [Styles](#styles)
 6. [Command-line interface](#command-line-interface)
 7. [Testing lexers](#testing-lexers)
 8. [What's missing compared to Pygments?](#whats-missing-compared-to-pygments)
@@ -33,46 +33,48 @@ translators for Pygments lexers and styles.
 <!-- /TOC -->

 <a id="markdown-supported-languages" name="supported-languages"></a>
+
 ## Supported languages

-Prefix | Language
-:----: | --------
-A | ABAP, ABNF, ActionScript, ActionScript 3, Ada, Angular2, ANTLR, ApacheConf, APL, AppleScript, ArangoDB AQL, Arduino, Awk
-B | Ballerina, Bash, Batchfile, BibTeX, Bicep, BlitzBasic, BNF, Brainfuck, BQN
-C | C, C#, C++, Caddyfile, Caddyfile Directives, Cap'n Proto, Cassandra CQL, Ceylon, CFEngine3, cfstatement, ChaiScript, Chapel, Cheetah, Clojure, CMake, COBOL, CoffeeScript, Common Lisp, Coq, Crystal, CSS, Cython
-D | D, Dart, Diff, Django/Jinja, Docker, DTD, Dylan
-E | EBNF, Elixir, Elm, EmacsLisp, Erlang
-F | Factor, Fish, Forth, Fortran, FSharp
-G | GAS, GDScript, Genshi, Genshi HTML, Genshi Text, Gherkin, GLSL, Gnuplot, Go, Go HTML Template, Go Text Template, GraphQL, Groff, Groovy
-H | Handlebars, Haskell, Haxe, HCL, Hexdump, HLB, HLSL, HTML, HTTP, Hy
-I | Idris, Igor, INI, Io
-J | J, Java, JavaScript, JSON, Julia, Jungle
-K | Kotlin
-L | Lighttpd configuration file, LLVM, Lua
-M | Makefile, Mako, markdown, Mason, Mathematica, Matlab, MiniZinc, MLIR, Modula-2, MonkeyC, MorrowindScript, Myghty, MySQL
-N | NASM, Newspeak, Nginx configuration file, Nim, Nix
-O | Objective-C, OCaml, Octave, OnesEnterprise, OpenEdge ABL, OpenSCAD, Org Mode
-P | PacmanConf, Perl, PHP, PHTML, Pig, PkgConfig, PL/pgSQL, plaintext, Pony, PostgreSQL SQL dialect, PostScript, POVRay, PowerShell, Prolog, PromQL, Properties, Protocol Buffer, PSL, Puppet, Python 2, Python
-Q | QBasic
-R | R, Racket, Ragel, Raku, react, ReasonML, reg, reStructuredText, Rexx, Ruby, Rust
-S | SAS, Sass, Scala, Scheme, Scilab, SCSS, Sed, Smali, Smalltalk, Smarty, Snobol, Solidity, SPARQL, SQL, SquidConf, Standard ML, stas, Stylus, Svelte, Swift, SYSTEMD, systemverilog
-T | TableGen, TASM, Tcl, Tcsh, Termcap, Terminfo, Terraform, TeX, Thrift, TOML, TradingView, Transact-SQL, Turing, Turtle, Twig, TypeScript, TypoScript, TypoScriptCssData, TypoScriptHtmlData
-V | VB.net, verilog, VHDL, VHS, VimL, vue
-W | WDTE
-X | XML, Xorg
-Y | YAML, YANG
-Z | Zig
-
+| Prefix | Language                                                                                                                                                                                                                             |
+| :----: | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+|   A    | ABAP, ABNF, ActionScript, ActionScript 3, Ada, AL, Angular2, ANTLR, ApacheConf, APL, AppleScript, ArangoDB AQL, Arduino, ArmAsm, AutoHotkey, AutoIt, Awk                                                                             |
+|   B    | Ballerina, Bash, Bash Session, Batchfile, BibTeX, Bicep, BlitzBasic, BNF, BQN, Brainfuck                                                                                                                                             |
+|   C    | C, C#, C++, Caddyfile, Caddyfile Directives, Cap'n Proto, Cassandra CQL, Ceylon, CFEngine3, cfstatement, ChaiScript, Chapel, Cheetah, Clojure, CMake, COBOL, CoffeeScript, Common Lisp, Coq, Crystal, CSS, Cython                    |
+|   D    | D, Dart, Diff, Django/Jinja, dns, Docker, DTD, Dylan                                                                                                                                                                                 |
+|   E    | EBNF, Elixir, Elm, EmacsLisp, Erlang                                                                                                                                                                                                 |
+|   F    | Factor, Fennel, Fish, Forth, Fortran, FortranFixed, FSharp                                                                                                                                                                           |
+|   G    | GAS, GDScript, Genshi, Genshi HTML, Genshi Text, Gherkin, GLSL, Gnuplot, Go, Go HTML Template, Go Text Template, GraphQL, Groff, Groovy                                                                                              |
+|   H    | Handlebars, Haskell, Haxe, HCL, Hexdump, HLB, HLSL, HolyC, HTML, HTTP, Hy                                                                                                                                                            |
+|   I    | Idris, Igor, INI, Io, ISCdhcpd                                                                                                                                                                                                       |
+|   J    | J, Java, JavaScript, JSON, Julia, Jungle                                                                                                                                                                                             |
+|   K    | Kotlin                                                                                                                                                                                                                               |
+|   L    | Lighttpd configuration file, LLVM, Lua                                                                                                                                                                                               |
+|   M    | Makefile, Mako, markdown, Mason, Mathematica, Matlab, mcfunction, Meson, Metal, MiniZinc, MLIR, Modula-2, MonkeyC, MorrowindScript, Myghty, MySQL                                                                                    |
+|   N    | NASM, Natural, Newspeak, Nginx configuration file, Nim, Nix                                                                                                                                                                          |
+|   O    | Objective-C, OCaml, Octave, Odin, OnesEnterprise, OpenEdge ABL, OpenSCAD, Org Mode                                                                                                                                                   |
+|   P    | PacmanConf, Perl, PHP, PHTML, Pig, PkgConfig, PL/pgSQL, plaintext, Plutus Core, Pony, PostgreSQL SQL dialect, PostScript, POVRay, PowerQuery, PowerShell, Prolog, PromQL, properties, Protocol Buffer, PSL, Puppet, Python, Python 2 |
+|   Q    | QBasic, QML                                                                                                                                                                                                                          |
+|   R    | R, Racket, Ragel, Raku, react, ReasonML, reg, reStructuredText, Rexx, Ruby, Rust                                                                                                                                                     |
+|   S    | SAS, Sass, Scala, Scheme, Scilab, SCSS, Sed, Sieve, Smali, Smalltalk, Smarty, Snobol, Solidity, SourcePawn, SPARQL, SQL, SquidConf, Standard ML, stas, Stylus, Svelte, Swift, SYSTEMD, systemverilog                                 |
+|   T    | TableGen, Tal, TASM, Tcl, Tcsh, Termcap, Terminfo, Terraform, TeX, Thrift, TOML, TradingView, Transact-SQL, Turing, Turtle, Twig, TypeScript, TypoScript, TypoScriptCssData, TypoScriptHtmlData                                      |
+|   V    | V, V shell, Vala, VB.net, verilog, VHDL, VHS, VimL, vue                                                                                                                                                                              |
+|   W    | WDTE, WebGPU Shading Language, Whiley                                                                                                                                                                                                |
+|   X    | XML, Xorg                                                                                                                                                                                                                            |
+|   Y    | YAML, YANG                                                                                                                                                                                                                           |
+|   Z    | Z80 Assembly, Zed, Zig                                                                                                                                                                                                               |

 _I will attempt to keep this section up to date, but an authoritative list can be
 displayed with `chroma --list`._

 <a id="markdown-try-it" name="try-it"></a>
+
 ## Try it

 Try out various languages and styles on the [Chroma Playground](https://swapoff.org/chroma/playground/).

 <a id="markdown-using-the-library" name="using-the-library"></a>
+
 ## Using the library

 Chroma, like Pygments, has the concepts of
@@ -94,6 +96,7 @@ be returned. In this situation you may want to default to the `Fallback`
 value in each respective package, which provides sane defaults.

 <a id="markdown-quick-start" name="quick-start"></a>
+
 ### Quick start

 A convenience function exists that can be used to simply format some source
@@ -104,6 +107,7 @@ err := quick.Highlight(os.Stdout, someSourceCode, "go", "html", "monokai")
 ```

 <a id="markdown-identifying-the-language" name="identifying-the-language"></a>
+
 ### Identifying the language

 To highlight code, you'll first have to identify what language the code is
@@ -111,21 +115,21 @@ written in. There are three primary ways to do that:

 1. Detect the language from its filename.

-    ```go
-    lexer := lexers.Match("foo.go")
-    ```
+   ```go
+   lexer := lexers.Match("foo.go")
+   ```

-3. Explicitly specify the language by its Chroma syntax ID (a full list is available from `lexers.Names()`).
+2. Explicitly specify the language by its Chroma syntax ID (a full list is available from `lexers.Names()`).

-    ```go
-    lexer := lexers.Get("go")
-    ```
+   ```go
+   lexer := lexers.Get("go")
+   ```

 3. Detect the language from its content.

-    ```go
-    lexer := lexers.Analyse("package main\n\nfunc main()\n{\n}\n")
-    ```
+   ```go
+   lexer := lexers.Analyse("package main\n\nfunc main()\n{\n}\n")
+   ```

 In all cases, `nil` will be returned if the language can not be identified.

@@ -144,6 +148,7 @@ lexer = chroma.Coalesce(lexer)
 ```

 <a id="markdown-formatting-the-output" name="formatting-the-output"></a>
+
 ### Formatting the output

 Once a language is identified you will need to pick a formatter and a style (theme).
@@ -173,6 +178,7 @@ err := formatter.Format(w, style, iterator)
 ```

 <a id="markdown-the-html-formatter" name="the-html-formatter"></a>
+
 ### The HTML formatter

 By default the `html` registered formatter generates standalone HTML with
@@ -198,9 +204,11 @@ err := formatter.WriteCSS(w, style)
 ```

 <a id="markdown-more-detail" name="more-detail"></a>
+
 ## More detail

 <a id="markdown-lexers" name="lexers"></a>
+
 ### Lexers

 See the [Pygments documentation](http://pygments.org/docs/lexerdevelopment/)
@@ -221,6 +229,7 @@ See notes in [pygments-lexers.txt](https://github.com/alecthomas/chroma/blob/mas
 for a list of lexers, and notes on some of the issues importing them.

 <a id="markdown-formatters" name="formatters"></a>
+
 ### Formatters

 Chroma supports HTML output, as well as terminal output in 8 colour, 256 colour, and true-colour.
@@ -229,6 +238,7 @@ A `noop` formatter is included that outputs the token text only, and a `tokens`
 formatter outputs raw tokens. The latter is useful for debugging lexers.

 <a id="markdown-styles" name="styles"></a>
+
 ### Styles

 Chroma styles are defined in XML. The style entries use the
@@ -237,7 +247,7 @@ Chroma styles are defined in XML. The style entries use the
 All Pygments styles have been converted to Chroma using the `_tools/style.py`
 script.

-When you work with one of [Chroma's styles](https://github.com/alecthomas/chroma/tree/master/styles), 
+When you work with one of [Chroma's styles](https://github.com/alecthomas/chroma/tree/master/styles),
 know that the `Background` token type provides the default style for tokens. It does so
 by defining a foreground color and background color.

@@ -253,6 +263,7 @@ Also, token types in a style file are hierarchical. For instance, when `CommentS
 For a quick overview of the available styles and how they look, check out the [Chroma Style Gallery](https://xyproto.github.io/splash/docs/).

 <a id="markdown-command-line-interface" name="command-line-interface"></a>
+
 ## Command-line interface

 A command-line interface to Chroma is included.
@@ -280,26 +291,32 @@ for that setup the `chroma` executable can be just symlinked to `~/.lessfilter`.
 <a id="markdown-whats-missing-compared-to-pygments" name="whats-missing-compared-to-pygments"></a>

 <a id="markdown-testing-lexers" name="testing-lexers"></a>
+
 ## Testing lexers
+
 If you edit some lexers and want to try it, open a shell in `cmd/chromad` and run:
+
 ```shell
 go run .
 ```
+
 A Link will be printed. Open it in your Browser. Now you can test on the Playground with your local changes.

 If you want to run the tests and the lexers, open a shell in the root directory and run:
+
 ```shell
 go test ./lexers
 ```
+
 When updating or adding a lexer, please add tests. See [lexers/README.md](lexers/README.md) for more.

 ## What's missing compared to Pygments?

 - Quite a few lexers, for various reasons (pull-requests welcome):
-    - Pygments lexers for complex languages often include custom code to
-      handle certain aspects, such as Raku's ability to nest code inside
-      regular expressions. These require time and effort to convert.
-    - I mostly only converted languages I had heard of, to reduce the porting cost.
+  - Pygments lexers for complex languages often include custom code to
+    handle certain aspects, such as Raku's ability to nest code inside
+    regular expressions. These require time and effort to convert.
+  - I mostly only converted languages I had heard of, to reduce the porting cost.
 - Some more esoteric features of Pygments are omitted for simplicity.
 - Though the Chroma API supports content detection, very few languages support them.
  I have plans to implement a statistical analyser at some point, but not enough time.
--- a/bin/.bit-0.0.1.pkg
+++ b/bin/.bit-0.0.1.pkg
@@ -0,0 +1 @@
+hermit
--- a/bin/.python3-3.11.4.pkg
+++ b/bin/.python3-3.11.4.pkg
@@ -0,0 +1 @@
+hermit
--- a/bin/bit
+++ b/bin/bit
@@ -0,0 +1 @@
+.bit-0.0.1.pkg
--- a/bin/pip
+++ b/bin/pip
@@ -0,0 +1 @@
+.python3-3.11.4.pkg
--- a/bin/pip3
+++ b/bin/pip3
@@ -0,0 +1 @@
+.python3-3.11.4.pkg
--- a/bin/pip3.11
+++ b/bin/pip3.11
@@ -0,0 +1 @@
+.python3-3.11.4.pkg
--- a/bin/pydoc3
+++ b/bin/pydoc3
@@ -0,0 +1 @@
+.python3-3.11.4.pkg
--- a/bin/pydoc3.11
+++ b/bin/pydoc3.11
@@ -0,0 +1 @@
+.python3-3.11.4.pkg
--- a/bin/python
+++ b/bin/python
@@ -0,0 +1 @@
+.python3-3.11.4.pkg
--- a/bin/python3
+++ b/bin/python3
@@ -0,0 +1 @@
+.python3-3.11.4.pkg
--- a/bin/python3-config
+++ b/bin/python3-config
@@ -0,0 +1 @@
+.python3-3.11.4.pkg
--- a/bin/python3.11
+++ b/bin/python3.11
@@ -0,0 +1 @@
+.python3-3.11.4.pkg
--- a/bin/python3.11-config
+++ b/bin/python3.11-config
@@ -0,0 +1 @@
+.python3-3.11.4.pkg
--- a/cmd/chroma/go.mod
+++ b/cmd/chroma/go.mod
@@ -12,6 +12,6 @@ require (
 )

 require (
-	github.com/dlclark/regexp2 v1.4.0 // indirect
+	github.com/dlclark/regexp2 v1.10.0 // indirect
 	golang.org/x/sys v0.6.0 // indirect
 )
--- a/cmd/chroma/go.sum
+++ b/cmd/chroma/go.sum
@@ -2,8 +2,8 @@ github.com/alecthomas/assert/v2 v2.2.1 h1:XivOgYcduV98QCahG8T5XTezV5bylXe+lBxLG2
 github.com/alecthomas/kong v0.8.0 h1:ryDCzutfIqJPnNn0omnrgHLbAggDQM2VWHikE1xqK7s=
 github.com/alecthomas/kong v0.8.0/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U=
 github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk=
-github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E=
-github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
+github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=
+github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
--- a/cmd/chromad/go.mod
+++ b/cmd/chromad/go.mod
@@ -12,7 +12,7 @@ require (
 )

 require (
-	github.com/dlclark/regexp2 v1.4.0 // indirect
+	github.com/dlclark/regexp2 v1.10.0 // indirect
 	github.com/felixge/httpsnoop v1.0.1 // indirect
 	github.com/gorilla/securecookie v1.1.1 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
--- a/cmd/chromad/go.sum
+++ b/cmd/chromad/go.sum
@@ -7,8 +7,8 @@ github.com/alecthomas/kong-hcl v1.0.1/go.mod h1:6Y+MaMTZ/KQe9Qme6aSlKtql65FJE0/1
 github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E=
-github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
+github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=
+github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ=
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/gorilla/csrf v1.7.1 h1:Ir3o2c1/Uzj6FBxMlAUB6SivgVMy1ONXwYgXn+/aHPE=
--- a/go.mod
+++ b/go.mod
@@ -5,7 +5,7 @@ go 1.19
 require (
 	github.com/alecthomas/assert/v2 v2.2.1
 	github.com/alecthomas/repr v0.2.0
-	github.com/dlclark/regexp2 v1.4.0
+	github.com/dlclark/regexp2 v1.10.0
 )

 require github.com/hexops/gotextdiff v1.0.3 // indirect
--- a/go.sum
+++ b/go.sum
@@ -2,7 +2,7 @@ github.com/alecthomas/assert/v2 v2.2.1 h1:XivOgYcduV98QCahG8T5XTezV5bylXe+lBxLG2
 github.com/alecthomas/assert/v2 v2.2.1/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ=
 github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk=
 github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
-github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E=
-github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
+github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=
+github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
--- a/lexers/embedded/dns.xml
+++ b/lexers/embedded/dns.xml
@@ -24,10 +24,10 @@
      <rule pattern="\b\d+[dhwm]?">
        <token type="LiteralStringChar"/>
      </rule>
-      <rule pattern="\b([[:alnum:].-])+\.($|\s)">
+      <rule pattern="\b([\w\d.-])+\.($|\s)">
        <token type="NameProperty"/>
      </rule>
-      <rule pattern="^(@|[[:alnum:]-]+)">
+      <rule pattern="^(@|[\w\d-]+)">
        <token type="NameClass"/>
      </rule>
      <rule pattern="^\$(TTL|GENERATE|INCLUDE|ORIGIN)">
--- a/regexp.go
+++ b/regexp.go
@@ -332,7 +332,7 @@ func (r *RegexLexer) maybeCompile() (err error) {
 					pattern = "(?" + rule.flags + ")" + pattern
 				}
 				pattern = `\G` + pattern
-				rule.Regexp, err = regexp2.Compile(pattern, regexp2.RE2)
+				rule.Regexp, err = regexp2.Compile(pattern, 0)
 				if err != nil {
 					return fmt.Errorf("failed to compile rule %s.%d: %s", state, i, err)
 				}
--- a/table.py
+++ b/table.py
@@ -5,8 +5,7 @@ from subprocess import check_output

 README_FILE = "README.md"

-
-lines = check_output(["go", "run", "./cmd/chroma/main.go", "--list"]).decode("utf-8").splitlines()
+lines = check_output(["chroma", "--list"]).decode("utf-8").splitlines()
 lines = [line.strip() for line in lines if line.startswith("  ") and not line.startswith("   ")]
 lines = sorted(lines, key=lambda l: l.lower())

--- a/tokentype_enumer.go
+++ b/tokentype_enumer.go
@@ -4,9 +4,11 @@ package chroma

 import (
 	"fmt"
+	"strings"
 )

 const _TokenTypeName = "NoneOtherErrorCodeLineLineLinkLineTableTDLineTableLineHighlightLineNumbersTableLineNumbersLinePreWrapperBackgroundEOFTypeKeywordKeywordConstantKeywordDeclarationKeywordNamespaceKeywordPseudoKeywordReservedKeywordTypeNameNameAttributeNameBuiltinNameBuiltinPseudoNameClassNameConstantNameDecoratorNameEntityNameExceptionNameFunctionNameFunctionMagicNameKeywordNameLabelNameNamespaceNameOperatorNameOtherNamePseudoNamePropertyNameTagNameVariableNameVariableAnonymousNameVariableClassNameVariableGlobalNameVariableInstanceNameVariableMagicLiteralLiteralDateLiteralOtherLiteralStringLiteralStringAffixLiteralStringAtomLiteralStringBacktickLiteralStringBooleanLiteralStringCharLiteralStringDelimiterLiteralStringDocLiteralStringDoubleLiteralStringEscapeLiteralStringHeredocLiteralStringInterpolLiteralStringNameLiteralStringOtherLiteralStringRegexLiteralStringSingleLiteralStringSymbolLiteralNumberLiteralNumberBinLiteralNumberFloatLiteralNumberHexLiteralNumberIntegerLiteralNumberIntegerLongLiteralNumberOctOperatorOperatorWordPunctuationCommentCommentHashbangCommentMultilineCommentSingleCommentSpecialCommentPreprocCommentPreprocFileGenericGenericDeletedGenericEmphGenericErrorGenericHeadingGenericInsertedGenericOutputGenericPromptGenericStrongGenericSubheadingGenericTracebackGenericUnderlineTextTextWhitespaceTextSymbolTextPunctuation"
+const _TokenTypeLowerName = "noneothererrorcodelinelinelinklinetabletdlinetablelinehighlightlinenumberstablelinenumberslineprewrapperbackgroundeoftypekeywordkeywordconstantkeyworddeclarationkeywordnamespacekeywordpseudokeywordreservedkeywordtypenamenameattributenamebuiltinnamebuiltinpseudonameclassnameconstantnamedecoratornameentitynameexceptionnamefunctionnamefunctionmagicnamekeywordnamelabelnamenamespacenameoperatornameothernamepseudonamepropertynametagnamevariablenamevariableanonymousnamevariableclassnamevariableglobalnamevariableinstancenamevariablemagicliteralliteraldateliteralotherliteralstringliteralstringaffixliteralstringatomliteralstringbacktickliteralstringbooleanliteralstringcharliteralstringdelimiterliteralstringdocliteralstringdoubleliteralstringescapeliteralstringheredocliteralstringinterpolliteralstringnameliteralstringotherliteralstringregexliteralstringsingleliteralstringsymbolliteralnumberliteralnumberbinliteralnumberfloatliteralnumberhexliteralnumberintegerliteralnumberintegerlongliteralnumberoctoperatoroperatorwordpunctuationcommentcommenthashbangcommentmultilinecommentsinglecommentspecialcommentpreproccommentpreprocfilegenericgenericdeletedgenericemphgenericerrorgenericheadinggenericinsertedgenericoutputgenericpromptgenericstronggenericsubheadinggenerictracebackgenericunderlinetexttextwhitespacetextsymboltextpunctuation"

 var _TokenTypeMap = map[TokenType]string{
 	-13:  _TokenTypeName[0:4],
@@ -117,108 +119,414 @@ func (i TokenType) String() string {
 	return fmt.Sprintf("TokenType(%d)", i)
 }

-var _TokenTypeValues = []TokenType{-13, -12, -11, -10, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0, 1000, 1001, 1002, 1003, 1004, 1005, 1006, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 3000, 3001, 3002, 3100, 3101, 3102, 3103, 3104, 3105, 3106, 3107, 3108, 3109, 3110, 3111, 3112, 3113, 3114, 3115, 3116, 3200, 3201, 3202, 3203, 3204, 3205, 3206, 4000, 4001, 5000, 6000, 6001, 6002, 6003, 6004, 6100, 6101, 7000, 7001, 7002, 7003, 7004, 7005, 7006, 7007, 7008, 7009, 7010, 7011, 8000, 8001, 8002, 8003}
+// An "invalid array index" compiler error signifies that the constant values have changed.
+// Re-run the stringer command to generate them again.
+func _TokenTypeNoOp() {
+	var x [1]struct{}
+	_ = x[None-(-13)]
+	_ = x[Other-(-12)]
+	_ = x[Error-(-11)]
+	_ = x[CodeLine-(-10)]
+	_ = x[LineLink-(-9)]
+	_ = x[LineTableTD-(-8)]
+	_ = x[LineTable-(-7)]
+	_ = x[LineHighlight-(-6)]
+	_ = x[LineNumbersTable-(-5)]
+	_ = x[LineNumbers-(-4)]
+	_ = x[Line-(-3)]
+	_ = x[PreWrapper-(-2)]
+	_ = x[Background-(-1)]
+	_ = x[EOFType-(0)]
+	_ = x[Keyword-(1000)]
+	_ = x[KeywordConstant-(1001)]
+	_ = x[KeywordDeclaration-(1002)]
+	_ = x[KeywordNamespace-(1003)]
+	_ = x[KeywordPseudo-(1004)]
+	_ = x[KeywordReserved-(1005)]
+	_ = x[KeywordType-(1006)]
+	_ = x[Name-(2000)]
+	_ = x[NameAttribute-(2001)]
+	_ = x[NameBuiltin-(2002)]
+	_ = x[NameBuiltinPseudo-(2003)]
+	_ = x[NameClass-(2004)]
+	_ = x[NameConstant-(2005)]
+	_ = x[NameDecorator-(2006)]
+	_ = x[NameEntity-(2007)]
+	_ = x[NameException-(2008)]
+	_ = x[NameFunction-(2009)]
+	_ = x[NameFunctionMagic-(2010)]
+	_ = x[NameKeyword-(2011)]
+	_ = x[NameLabel-(2012)]
+	_ = x[NameNamespace-(2013)]
+	_ = x[NameOperator-(2014)]
+	_ = x[NameOther-(2015)]
+	_ = x[NamePseudo-(2016)]
+	_ = x[NameProperty-(2017)]
+	_ = x[NameTag-(2018)]
+	_ = x[NameVariable-(2019)]
+	_ = x[NameVariableAnonymous-(2020)]
+	_ = x[NameVariableClass-(2021)]
+	_ = x[NameVariableGlobal-(2022)]
+	_ = x[NameVariableInstance-(2023)]
+	_ = x[NameVariableMagic-(2024)]
+	_ = x[Literal-(3000)]
+	_ = x[LiteralDate-(3001)]
+	_ = x[LiteralOther-(3002)]
+	_ = x[LiteralString-(3100)]
+	_ = x[LiteralStringAffix-(3101)]
+	_ = x[LiteralStringAtom-(3102)]
+	_ = x[LiteralStringBacktick-(3103)]
+	_ = x[LiteralStringBoolean-(3104)]
+	_ = x[LiteralStringChar-(3105)]
+	_ = x[LiteralStringDelimiter-(3106)]
+	_ = x[LiteralStringDoc-(3107)]
+	_ = x[LiteralStringDouble-(3108)]
+	_ = x[LiteralStringEscape-(3109)]
+	_ = x[LiteralStringHeredoc-(3110)]
+	_ = x[LiteralStringInterpol-(3111)]
+	_ = x[LiteralStringName-(3112)]
+	_ = x[LiteralStringOther-(3113)]
+	_ = x[LiteralStringRegex-(3114)]
+	_ = x[LiteralStringSingle-(3115)]
+	_ = x[LiteralStringSymbol-(3116)]
+	_ = x[LiteralNumber-(3200)]
+	_ = x[LiteralNumberBin-(3201)]
+	_ = x[LiteralNumberFloat-(3202)]
+	_ = x[LiteralNumberHex-(3203)]
+	_ = x[LiteralNumberInteger-(3204)]
+	_ = x[LiteralNumberIntegerLong-(3205)]
+	_ = x[LiteralNumberOct-(3206)]
+	_ = x[Operator-(4000)]
+	_ = x[OperatorWord-(4001)]
+	_ = x[Punctuation-(5000)]
+	_ = x[Comment-(6000)]
+	_ = x[CommentHashbang-(6001)]
+	_ = x[CommentMultiline-(6002)]
+	_ = x[CommentSingle-(6003)]
+	_ = x[CommentSpecial-(6004)]
+	_ = x[CommentPreproc-(6100)]
+	_ = x[CommentPreprocFile-(6101)]
+	_ = x[Generic-(7000)]
+	_ = x[GenericDeleted-(7001)]
+	_ = x[GenericEmph-(7002)]
+	_ = x[GenericError-(7003)]
+	_ = x[GenericHeading-(7004)]
+	_ = x[GenericInserted-(7005)]
+	_ = x[GenericOutput-(7006)]
+	_ = x[GenericPrompt-(7007)]
+	_ = x[GenericStrong-(7008)]
+	_ = x[GenericSubheading-(7009)]
+	_ = x[GenericTraceback-(7010)]
+	_ = x[GenericUnderline-(7011)]
+	_ = x[Text-(8000)]
+	_ = x[TextWhitespace-(8001)]
+	_ = x[TextSymbol-(8002)]
+	_ = x[TextPunctuation-(8003)]
+}
+
+var _TokenTypeValues = []TokenType{None, Other, Error, CodeLine, LineLink, LineTableTD, LineTable, LineHighlight, LineNumbersTable, LineNumbers, Line, PreWrapper, Background, EOFType, Keyword, KeywordConstant, KeywordDeclaration, KeywordNamespace, KeywordPseudo, KeywordReserved, KeywordType, Name, NameAttribute, NameBuiltin, NameBuiltinPseudo, NameClass, NameConstant, NameDecorator, NameEntity, NameException, NameFunction, NameFunctionMagic, NameKeyword, NameLabel, NameNamespace, NameOperator, NameOther, NamePseudo, NameProperty, NameTag, NameVariable, NameVariableAnonymous, NameVariableClass, NameVariableGlobal, NameVariableInstance, NameVariableMagic, Literal, LiteralDate, LiteralOther, LiteralString, LiteralStringAffix, LiteralStringAtom, LiteralStringBacktick, LiteralStringBoolean, LiteralStringChar, LiteralStringDelimiter, LiteralStringDoc, LiteralStringDouble, LiteralStringEscape, LiteralStringHeredoc, LiteralStringInterpol, LiteralStringName, LiteralStringOther, LiteralStringRegex, LiteralStringSingle, LiteralStringSymbol, LiteralNumber, LiteralNumberBin, LiteralNumberFloat, LiteralNumberHex, LiteralNumberInteger, LiteralNumberIntegerLong, LiteralNumberOct, Operator, OperatorWord, Punctuation, Comment, CommentHashbang, CommentMultiline, CommentSingle, CommentSpecial, CommentPreproc, CommentPreprocFile, Generic, GenericDeleted, GenericEmph, GenericError, GenericHeading, GenericInserted, GenericOutput, GenericPrompt, GenericStrong, GenericSubheading, GenericTraceback, GenericUnderline, Text, TextWhitespace, TextSymbol, TextPunctuation}

 var _TokenTypeNameToValueMap = map[string]TokenType{
-	_TokenTypeName[0:4]:       -13,
-	_TokenTypeName[4:9]:       -12,
-	_TokenTypeName[9:14]:      -11,
-	_TokenTypeName[14:22]:     -10,
-	_TokenTypeName[22:30]:     -9,
-	_TokenTypeName[30:41]:     -8,
-	_TokenTypeName[41:50]:     -7,
-	_TokenTypeName[50:63]:     -6,
-	_TokenTypeName[63:79]:     -5,
-	_TokenTypeName[79:90]:     -4,
-	_TokenTypeName[90:94]:     -3,
-	_TokenTypeName[94:104]:    -2,
-	_TokenTypeName[104:114]:   -1,
-	_TokenTypeName[114:121]:   0,
-	_TokenTypeName[121:128]:   1000,
-	_TokenTypeName[128:143]:   1001,
-	_TokenTypeName[143:161]:   1002,
-	_TokenTypeName[161:177]:   1003,
-	_TokenTypeName[177:190]:   1004,
-	_TokenTypeName[190:205]:   1005,
-	_TokenTypeName[205:216]:   1006,
-	_TokenTypeName[216:220]:   2000,
-	_TokenTypeName[220:233]:   2001,
-	_TokenTypeName[233:244]:   2002,
-	_TokenTypeName[244:261]:   2003,
-	_TokenTypeName[261:270]:   2004,
-	_TokenTypeName[270:282]:   2005,
-	_TokenTypeName[282:295]:   2006,
-	_TokenTypeName[295:305]:   2007,
-	_TokenTypeName[305:318]:   2008,
-	_TokenTypeName[318:330]:   2009,
-	_TokenTypeName[330:347]:   2010,
-	_TokenTypeName[347:358]:   2011,
-	_TokenTypeName[358:367]:   2012,
-	_TokenTypeName[367:380]:   2013,
-	_TokenTypeName[380:392]:   2014,
-	_TokenTypeName[392:401]:   2015,
-	_TokenTypeName[401:411]:   2016,
-	_TokenTypeName[411:423]:   2017,
-	_TokenTypeName[423:430]:   2018,
-	_TokenTypeName[430:442]:   2019,
-	_TokenTypeName[442:463]:   2020,
-	_TokenTypeName[463:480]:   2021,
-	_TokenTypeName[480:498]:   2022,
-	_TokenTypeName[498:518]:   2023,
-	_TokenTypeName[518:535]:   2024,
-	_TokenTypeName[535:542]:   3000,
-	_TokenTypeName[542:553]:   3001,
-	_TokenTypeName[553:565]:   3002,
-	_TokenTypeName[565:578]:   3100,
-	_TokenTypeName[578:596]:   3101,
-	_TokenTypeName[596:613]:   3102,
-	_TokenTypeName[613:634]:   3103,
-	_TokenTypeName[634:654]:   3104,
-	_TokenTypeName[654:671]:   3105,
-	_TokenTypeName[671:693]:   3106,
-	_TokenTypeName[693:709]:   3107,
-	_TokenTypeName[709:728]:   3108,
-	_TokenTypeName[728:747]:   3109,
-	_TokenTypeName[747:767]:   3110,
-	_TokenTypeName[767:788]:   3111,
-	_TokenTypeName[788:805]:   3112,
-	_TokenTypeName[805:823]:   3113,
-	_TokenTypeName[823:841]:   3114,
-	_TokenTypeName[841:860]:   3115,
-	_TokenTypeName[860:879]:   3116,
-	_TokenTypeName[879:892]:   3200,
-	_TokenTypeName[892:908]:   3201,
-	_TokenTypeName[908:926]:   3202,
-	_TokenTypeName[926:942]:   3203,
-	_TokenTypeName[942:962]:   3204,
-	_TokenTypeName[962:986]:   3205,
-	_TokenTypeName[986:1002]:  3206,
-	_TokenTypeName[1002:1010]: 4000,
-	_TokenTypeName[1010:1022]: 4001,
-	_TokenTypeName[1022:1033]: 5000,
-	_TokenTypeName[1033:1040]: 6000,
-	_TokenTypeName[1040:1055]: 6001,
-	_TokenTypeName[1055:1071]: 6002,
-	_TokenTypeName[1071:1084]: 6003,
-	_TokenTypeName[1084:1098]: 6004,
-	_TokenTypeName[1098:1112]: 6100,
-	_TokenTypeName[1112:1130]: 6101,
-	_TokenTypeName[1130:1137]: 7000,
-	_TokenTypeName[1137:1151]: 7001,
-	_TokenTypeName[1151:1162]: 7002,
-	_TokenTypeName[1162:1174]: 7003,
-	_TokenTypeName[1174:1188]: 7004,
-	_TokenTypeName[1188:1203]: 7005,
-	_TokenTypeName[1203:1216]: 7006,
-	_TokenTypeName[1216:1229]: 7007,
-	_TokenTypeName[1229:1242]: 7008,
-	_TokenTypeName[1242:1259]: 7009,
-	_TokenTypeName[1259:1275]: 7010,
-	_TokenTypeName[1275:1291]: 7011,
-	_TokenTypeName[1291:1295]: 8000,
-	_TokenTypeName[1295:1309]: 8001,
-	_TokenTypeName[1309:1319]: 8002,
-	_TokenTypeName[1319:1334]: 8003,
+	_TokenTypeName[0:4]:            None,
+	_TokenTypeLowerName[0:4]:       None,
+	_TokenTypeName[4:9]:            Other,
+	_TokenTypeLowerName[4:9]:       Other,
+	_TokenTypeName[9:14]:           Error,
+	_TokenTypeLowerName[9:14]:      Error,
+	_TokenTypeName[14:22]:          CodeLine,
+	_TokenTypeLowerName[14:22]:     CodeLine,
+	_TokenTypeName[22:30]:          LineLink,
+	_TokenTypeLowerName[22:30]:     LineLink,
+	_TokenTypeName[30:41]:          LineTableTD,
+	_TokenTypeLowerName[30:41]:     LineTableTD,
+	_TokenTypeName[41:50]:          LineTable,
+	_TokenTypeLowerName[41:50]:     LineTable,
+	_TokenTypeName[50:63]:          LineHighlight,
+	_TokenTypeLowerName[50:63]:     LineHighlight,
+	_TokenTypeName[63:79]:          LineNumbersTable,
+	_TokenTypeLowerName[63:79]:     LineNumbersTable,
+	_TokenTypeName[79:90]:          LineNumbers,
+	_TokenTypeLowerName[79:90]:     LineNumbers,
+	_TokenTypeName[90:94]:          Line,
+	_TokenTypeLowerName[90:94]:     Line,
+	_TokenTypeName[94:104]:         PreWrapper,
+	_TokenTypeLowerName[94:104]:    PreWrapper,
+	_TokenTypeName[104:114]:        Background,
+	_TokenTypeLowerName[104:114]:   Background,
+	_TokenTypeName[114:121]:        EOFType,
+	_TokenTypeLowerName[114:121]:   EOFType,
+	_TokenTypeName[121:128]:        Keyword,
+	_TokenTypeLowerName[121:128]:   Keyword,
+	_TokenTypeName[128:143]:        KeywordConstant,
+	_TokenTypeLowerName[128:143]:   KeywordConstant,
+	_TokenTypeName[143:161]:        KeywordDeclaration,
+	_TokenTypeLowerName[143:161]:   KeywordDeclaration,
+	_TokenTypeName[161:177]:        KeywordNamespace,
+	_TokenTypeLowerName[161:177]:   KeywordNamespace,
+	_TokenTypeName[177:190]:        KeywordPseudo,
+	_TokenTypeLowerName[177:190]:   KeywordPseudo,
+	_TokenTypeName[190:205]:        KeywordReserved,
+	_TokenTypeLowerName[190:205]:   KeywordReserved,
+	_TokenTypeName[205:216]:        KeywordType,
+	_TokenTypeLowerName[205:216]:   KeywordType,
+	_TokenTypeName[216:220]:        Name,
+	_TokenTypeLowerName[216:220]:   Name,
+	_TokenTypeName[220:233]:        NameAttribute,
+	_TokenTypeLowerName[220:233]:   NameAttribute,
+	_TokenTypeName[233:244]:        NameBuiltin,
+	_TokenTypeLowerName[233:244]:   NameBuiltin,
+	_TokenTypeName[244:261]:        NameBuiltinPseudo,
+	_TokenTypeLowerName[244:261]:   NameBuiltinPseudo,
+	_TokenTypeName[261:270]:        NameClass,
+	_TokenTypeLowerName[261:270]:   NameClass,
+	_TokenTypeName[270:282]:        NameConstant,
+	_TokenTypeLowerName[270:282]:   NameConstant,
+	_TokenTypeName[282:295]:        NameDecorator,
+	_TokenTypeLowerName[282:295]:   NameDecorator,
+	_TokenTypeName[295:305]:        NameEntity,
+	_TokenTypeLowerName[295:305]:   NameEntity,
+	_TokenTypeName[305:318]:        NameException,
+	_TokenTypeLowerName[305:318]:   NameException,
+	_TokenTypeName[318:330]:        NameFunction,
+	_TokenTypeLowerName[318:330]:   NameFunction,
+	_TokenTypeName[330:347]:        NameFunctionMagic,
+	_TokenTypeLowerName[330:347]:   NameFunctionMagic,
+	_TokenTypeName[347:358]:        NameKeyword,
+	_TokenTypeLowerName[347:358]:   NameKeyword,
+	_TokenTypeName[358:367]:        NameLabel,
+	_TokenTypeLowerName[358:367]:   NameLabel,
+	_TokenTypeName[367:380]:        NameNamespace,
+	_TokenTypeLowerName[367:380]:   NameNamespace,
+	_TokenTypeName[380:392]:        NameOperator,
+	_TokenTypeLowerName[380:392]:   NameOperator,
+	_TokenTypeName[392:401]:        NameOther,
+	_TokenTypeLowerName[392:401]:   NameOther,
+	_TokenTypeName[401:411]:        NamePseudo,
+	_TokenTypeLowerName[401:411]:   NamePseudo,
+	_TokenTypeName[411:423]:        NameProperty,
+	_TokenTypeLowerName[411:423]:   NameProperty,
+	_TokenTypeName[423:430]:        NameTag,
+	_TokenTypeLowerName[423:430]:   NameTag,
+	_TokenTypeName[430:442]:        NameVariable,
+	_TokenTypeLowerName[430:442]:   NameVariable,
+	_TokenTypeName[442:463]:        NameVariableAnonymous,
+	_TokenTypeLowerName[442:463]:   NameVariableAnonymous,
+	_TokenTypeName[463:480]:        NameVariableClass,
+	_TokenTypeLowerName[463:480]:   NameVariableClass,
+	_TokenTypeName[480:498]:        NameVariableGlobal,
+	_TokenTypeLowerName[480:498]:   NameVariableGlobal,
+	_TokenTypeName[498:518]:        NameVariableInstance,
+	_TokenTypeLowerName[498:518]:   NameVariableInstance,
+	_TokenTypeName[518:535]:        NameVariableMagic,
+	_TokenTypeLowerName[518:535]:   NameVariableMagic,
+	_TokenTypeName[535:542]:        Literal,
+	_TokenTypeLowerName[535:542]:   Literal,
+	_TokenTypeName[542:553]:        LiteralDate,
+	_TokenTypeLowerName[542:553]:   LiteralDate,
+	_TokenTypeName[553:565]:        LiteralOther,
+	_TokenTypeLowerName[553:565]:   LiteralOther,
+	_TokenTypeName[565:578]:        LiteralString,
+	_TokenTypeLowerName[565:578]:   LiteralString,
+	_TokenTypeName[578:596]:        LiteralStringAffix,
+	_TokenTypeLowerName[578:596]:   LiteralStringAffix,
+	_TokenTypeName[596:613]:        LiteralStringAtom,
+	_TokenTypeLowerName[596:613]:   LiteralStringAtom,
+	_TokenTypeName[613:634]:        LiteralStringBacktick,
+	_TokenTypeLowerName[613:634]:   LiteralStringBacktick,
+	_TokenTypeName[634:654]:        LiteralStringBoolean,
+	_TokenTypeLowerName[634:654]:   LiteralStringBoolean,
+	_TokenTypeName[654:671]:        LiteralStringChar,
+	_TokenTypeLowerName[654:671]:   LiteralStringChar,
+	_TokenTypeName[671:693]:        LiteralStringDelimiter,
+	_TokenTypeLowerName[671:693]:   LiteralStringDelimiter,
+	_TokenTypeName[693:709]:        LiteralStringDoc,
+	_TokenTypeLowerName[693:709]:   LiteralStringDoc,
+	_TokenTypeName[709:728]:        LiteralStringDouble,
+	_TokenTypeLowerName[709:728]:   LiteralStringDouble,
+	_TokenTypeName[728:747]:        LiteralStringEscape,
+	_TokenTypeLowerName[728:747]:   LiteralStringEscape,
+	_TokenTypeName[747:767]:        LiteralStringHeredoc,
+	_TokenTypeLowerName[747:767]:   LiteralStringHeredoc,
+	_TokenTypeName[767:788]:        LiteralStringInterpol,
+	_TokenTypeLowerName[767:788]:   LiteralStringInterpol,
+	_TokenTypeName[788:805]:        LiteralStringName,
+	_TokenTypeLowerName[788:805]:   LiteralStringName,
+	_TokenTypeName[805:823]:        LiteralStringOther,
+	_TokenTypeLowerName[805:823]:   LiteralStringOther,
+	_TokenTypeName[823:841]:        LiteralStringRegex,
+	_TokenTypeLowerName[823:841]:   LiteralStringRegex,
+	_TokenTypeName[841:860]:        LiteralStringSingle,
+	_TokenTypeLowerName[841:860]:   LiteralStringSingle,
+	_TokenTypeName[860:879]:        LiteralStringSymbol,
+	_TokenTypeLowerName[860:879]:   LiteralStringSymbol,
+	_TokenTypeName[879:892]:        LiteralNumber,
+	_TokenTypeLowerName[879:892]:   LiteralNumber,
+	_TokenTypeName[892:908]:        LiteralNumberBin,
+	_TokenTypeLowerName[892:908]:   LiteralNumberBin,
+	_TokenTypeName[908:926]:        LiteralNumberFloat,
+	_TokenTypeLowerName[908:926]:   LiteralNumberFloat,
+	_TokenTypeName[926:942]:        LiteralNumberHex,
+	_TokenTypeLowerName[926:942]:   LiteralNumberHex,
+	_TokenTypeName[942:962]:        LiteralNumberInteger,
+	_TokenTypeLowerName[942:962]:   LiteralNumberInteger,
+	_TokenTypeName[962:986]:        LiteralNumberIntegerLong,
+	_TokenTypeLowerName[962:986]:   LiteralNumberIntegerLong,
+	_TokenTypeName[986:1002]:       LiteralNumberOct,
+	_TokenTypeLowerName[986:1002]:  LiteralNumberOct,
+	_TokenTypeName[1002:1010]:      Operator,
+	_TokenTypeLowerName[1002:1010]: Operator,
+	_TokenTypeName[1010:1022]:      OperatorWord,
+	_TokenTypeLowerName[1010:1022]: OperatorWord,
+	_TokenTypeName[1022:1033]:      Punctuation,
+	_TokenTypeLowerName[1022:1033]: Punctuation,
+	_TokenTypeName[1033:1040]:      Comment,
+	_TokenTypeLowerName[1033:1040]: Comment,
+	_TokenTypeName[1040:1055]:      CommentHashbang,
+	_TokenTypeLowerName[1040:1055]: CommentHashbang,
+	_TokenTypeName[1055:1071]:      CommentMultiline,
+	_TokenTypeLowerName[1055:1071]: CommentMultiline,
+	_TokenTypeName[1071:1084]:      CommentSingle,
+	_TokenTypeLowerName[1071:1084]: CommentSingle,
+	_TokenTypeName[1084:1098]:      CommentSpecial,
+	_TokenTypeLowerName[1084:1098]: CommentSpecial,
+	_TokenTypeName[1098:1112]:      CommentPreproc,
+	_TokenTypeLowerName[1098:1112]: CommentPreproc,
+	_TokenTypeName[1112:1130]:      CommentPreprocFile,
+	_TokenTypeLowerName[1112:1130]: CommentPreprocFile,
+	_TokenTypeName[1130:1137]:      Generic,
+	_TokenTypeLowerName[1130:1137]: Generic,
+	_TokenTypeName[1137:1151]:      GenericDeleted,
+	_TokenTypeLowerName[1137:1151]: GenericDeleted,
+	_TokenTypeName[1151:1162]:      GenericEmph,
+	_TokenTypeLowerName[1151:1162]: GenericEmph,
+	_TokenTypeName[1162:1174]:      GenericError,
+	_TokenTypeLowerName[1162:1174]: GenericError,
+	_TokenTypeName[1174:1188]:      GenericHeading,
+	_TokenTypeLowerName[1174:1188]: GenericHeading,
+	_TokenTypeName[1188:1203]:      GenericInserted,
+	_TokenTypeLowerName[1188:1203]: GenericInserted,
+	_TokenTypeName[1203:1216]:      GenericOutput,
+	_TokenTypeLowerName[1203:1216]: GenericOutput,
+	_TokenTypeName[1216:1229]:      GenericPrompt,
+	_TokenTypeLowerName[1216:1229]: GenericPrompt,
+	_TokenTypeName[1229:1242]:      GenericStrong,
+	_TokenTypeLowerName[1229:1242]: GenericStrong,
+	_TokenTypeName[1242:1259]:      GenericSubheading,
+	_TokenTypeLowerName[1242:1259]: GenericSubheading,
+	_TokenTypeName[1259:1275]:      GenericTraceback,
+	_TokenTypeLowerName[1259:1275]: GenericTraceback,
+	_TokenTypeName[1275:1291]:      GenericUnderline,
+	_TokenTypeLowerName[1275:1291]: GenericUnderline,
+	_TokenTypeName[1291:1295]:      Text,
+	_TokenTypeLowerName[1291:1295]: Text,
+	_TokenTypeName[1295:1309]:      TextWhitespace,
+	_TokenTypeLowerName[1295:1309]: TextWhitespace,
+	_TokenTypeName[1309:1319]:      TextSymbol,
+	_TokenTypeLowerName[1309:1319]: TextSymbol,
+	_TokenTypeName[1319:1334]:      TextPunctuation,
+	_TokenTypeLowerName[1319:1334]: TextPunctuation,
+}
+
+var _TokenTypeNames = []string{
+	_TokenTypeName[0:4],
+	_TokenTypeName[4:9],
+	_TokenTypeName[9:14],
+	_TokenTypeName[14:22],
+	_TokenTypeName[22:30],
+	_TokenTypeName[30:41],
+	_TokenTypeName[41:50],
+	_TokenTypeName[50:63],
+	_TokenTypeName[63:79],
+	_TokenTypeName[79:90],
+	_TokenTypeName[90:94],
+	_TokenTypeName[94:104],
+	_TokenTypeName[104:114],
+	_TokenTypeName[114:121],
+	_TokenTypeName[121:128],
+	_TokenTypeName[128:143],
+	_TokenTypeName[143:161],
+	_TokenTypeName[161:177],
+	_TokenTypeName[177:190],
+	_TokenTypeName[190:205],
+	_TokenTypeName[205:216],
+	_TokenTypeName[216:220],
+	_TokenTypeName[220:233],
+	_TokenTypeName[233:244],
+	_TokenTypeName[244:261],
+	_TokenTypeName[261:270],
+	_TokenTypeName[270:282],
+	_TokenTypeName[282:295],
+	_TokenTypeName[295:305],
+	_TokenTypeName[305:318],
+	_TokenTypeName[318:330],
+	_TokenTypeName[330:347],
+	_TokenTypeName[347:358],
+	_TokenTypeName[358:367],
+	_TokenTypeName[367:380],
+	_TokenTypeName[380:392],
+	_TokenTypeName[392:401],
+	_TokenTypeName[401:411],
+	_TokenTypeName[411:423],
+	_TokenTypeName[423:430],
+	_TokenTypeName[430:442],
+	_TokenTypeName[442:463],
+	_TokenTypeName[463:480],
+	_TokenTypeName[480:498],
+	_TokenTypeName[498:518],
+	_TokenTypeName[518:535],
+	_TokenTypeName[535:542],
+	_TokenTypeName[542:553],
+	_TokenTypeName[553:565],
+	_TokenTypeName[565:578],
+	_TokenTypeName[578:596],
+	_TokenTypeName[596:613],
+	_TokenTypeName[613:634],
+	_TokenTypeName[634:654],
+	_TokenTypeName[654:671],
+	_TokenTypeName[671:693],
+	_TokenTypeName[693:709],
+	_TokenTypeName[709:728],
+	_TokenTypeName[728:747],
+	_TokenTypeName[747:767],
+	_TokenTypeName[767:788],
+	_TokenTypeName[788:805],
+	_TokenTypeName[805:823],
+	_TokenTypeName[823:841],
+	_TokenTypeName[841:860],
+	_TokenTypeName[860:879],
+	_TokenTypeName[879:892],
+	_TokenTypeName[892:908],
+	_TokenTypeName[908:926],
+	_TokenTypeName[926:942],
+	_TokenTypeName[942:962],
+	_TokenTypeName[962:986],
+	_TokenTypeName[986:1002],
+	_TokenTypeName[1002:1010],
+	_TokenTypeName[1010:1022],
+	_TokenTypeName[1022:1033],
+	_TokenTypeName[1033:1040],
+	_TokenTypeName[1040:1055],
+	_TokenTypeName[1055:1071],
+	_TokenTypeName[1071:1084],
+	_TokenTypeName[1084:1098],
+	_TokenTypeName[1098:1112],
+	_TokenTypeName[1112:1130],
+	_TokenTypeName[1130:1137],
+	_TokenTypeName[1137:1151],
+	_TokenTypeName[1151:1162],
+	_TokenTypeName[1162:1174],
+	_TokenTypeName[1174:1188],
+	_TokenTypeName[1188:1203],
+	_TokenTypeName[1203:1216],
+	_TokenTypeName[1216:1229],
+	_TokenTypeName[1229:1242],
+	_TokenTypeName[1242:1259],
+	_TokenTypeName[1259:1275],
+	_TokenTypeName[1275:1291],
+	_TokenTypeName[1291:1295],
+	_TokenTypeName[1295:1309],
+	_TokenTypeName[1309:1319],
+	_TokenTypeName[1319:1334],
 }

 // TokenTypeString retrieves an enum value from the enum constants string name.
@@ -227,6 +535,10 @@ func TokenTypeString(s string) (TokenType, error) {
 	if val, ok := _TokenTypeNameToValueMap[s]; ok {
 		return val, nil
 	}
+
+	if val, ok := _TokenTypeNameToValueMap[strings.ToLower(s)]; ok {
+		return val, nil
+	}
 	return 0, fmt.Errorf("%s does not belong to TokenType values", s)
 }

@@ -235,6 +547,13 @@ func TokenTypeValues() []TokenType {
 	return _TokenTypeValues
 }

+// TokenTypeStrings returns a slice of all String values of the enum
+func TokenTypeStrings() []string {
+	strs := make([]string, len(_TokenTypeNames))
+	copy(strs, _TokenTypeNames)
+	return strs
+}
+
 // IsATokenType returns "true" if the value is listed in the enum definition. "false" otherwise
 func (i TokenType) IsATokenType() bool {
 	_, ok := _TokenTypeMap[i]