Don't output extra whitespace in YAML multiline (#993)

This resolves a particular issue with parsing YAML multiline, for example: ```yaml a: | multiline literal line 2 ``` The regex used would capture the amount of indentation in the third capture group and then use that as a kind of "status" to know which lines are part of the indented multiline. However, because its a captured group it has to be assigned a token which was `TextWhitespace`. This meant that the indentation was outputted after the multiline, technically it should be seen as an non-captured group, but then its no longer to refer to it in the regex. Therefore I've gone with the solution to add a new token, Ignore, which will not be emitted as a token in the iterator, which can safely be used to make use of capture groups but not have them show up in the output. ## Before ![image](https://github.com/user-attachments/assets/c29353c5-9e15-4f14-a733-57a60fb51910) ## After ![image](https://github.com/user-attachments/assets/57b5d129-a9d3-4b84-ae1f-dc05182b9ad3)
2025-03-17 20:58:08 +02:00 · 2024-08-22 22:58:31 +02:00 · 2024-08-22 22:58:31 +02:00 · 4d11870090
commit 4d11870090
parent 895a0488b5
6 changed files with 448 additions and 424 deletions
--- a/lexers/embedded/yaml.xml
+++ b/lexers/embedded/yaml.xml
@ -53,7 +53,7 @@
        <bygroups>
          <token type="Punctuation"/>
          <token type="LiteralStringDoc"/>
-          <token type="TextWhitespace"/>
+          <token type="Ignore"/>
        </bygroups>
      </rule>
      <rule pattern="(false|False|FALSE|true|True|TRUE|null|Off|off|yes|Yes|YES|OFF|On|ON|no|No|on|NO|n|N|Y|y)\b">
--- a/lexers/testdata/yaml.expected
+++ b/lexers/testdata/yaml.expected
@ -191,55 +191,55 @@
  {"type":"TextWhitespace","value":" "},
  {"type":"Punctuation","value":"|"},
  {"type":"LiteralStringDoc","value":"\n    This entire block of text will be the value of the 'literal_block' key,\n    with line breaks being preserved.\n\n    The literal continues until de-dented, and the leading indentation is\n    stripped.\n\n        Any lines that are 'more-indented' keep the rest of their indentation -\n        these lines will be indented by 4 spaces."},
-  {"type":"TextWhitespace","value":"    \n"},
+  {"type":"TextWhitespace","value":"\n"},
  {"type":"NameTag","value":"folded_style"},
  {"type":"Punctuation","value":":"},
  {"type":"TextWhitespace","value":" "},
  {"type":"Punctuation","value":"\u003e"},
  {"type":"LiteralStringDoc","value":"\n    This entire block of text will be the value of 'folded_style', but this\n    time, all newlines will be replaced with a single space.\n\n    Blank lines, like above, are converted to a newline character.\n\n        'More-indented' lines keep their newlines, too -\n        this text will appear over two lines."},
-  {"type":"TextWhitespace","value":"    \n"},
+  {"type":"TextWhitespace","value":"\n"},
  {"type":"NameTag","value":"literal_block_with_strip_chomping"},
  {"type":"Punctuation","value":":"},
  {"type":"TextWhitespace","value":" "},
  {"type":"Punctuation","value":"|-"},
  {"type":"LiteralStringDoc","value":"\n    This entire block of text will be the value of the 'literal_block' key,\n    with line breaks being preserved and the strip chomping indicator.\n\n    The literal continues until de-dented, and the leading indentation is\n    stripped.\n\n        Any lines that are 'more-indented' keep the rest of their indentation -\n        these lines will be indented by 4 spaces."},
-  {"type":"TextWhitespace","value":"    \n"},
+  {"type":"TextWhitespace","value":"\n"},
  {"type":"NameTag","value":"literal_block_with_keep_chomping"},
  {"type":"Punctuation","value":":"},
  {"type":"TextWhitespace","value":" "},
  {"type":"Punctuation","value":"|+"},
  {"type":"LiteralStringDoc","value":"\n    This entire block of text will be the value of the 'literal_block' key,\n    with line breaks being preserved and the keep chomping indicator.\n\n    The literal continues until de-dented, and the leading indentation is\n    stripped."},
-  {"type":"TextWhitespace","value":"    \n\n"},
+  {"type":"TextWhitespace","value":"\n\n"},
  {"type":"NameTag","value":"a"},
  {"type":"Punctuation","value":":"},
  {"type":"TextWhitespace","value":" "},
  {"type":"Punctuation","value":"|"},
  {"type":"LiteralStringDoc","value":"\n  multiline literal\n  line 2"},
-  {"type":"TextWhitespace","value":"  \n"},
+  {"type":"TextWhitespace","value":"\n"},
  {"type":"NameTag","value":"b"},
  {"type":"Punctuation","value":":"},
  {"type":"TextWhitespace","value":" "},
  {"type":"Punctuation","value":"\u003e"},
  {"type":"LiteralStringDoc","value":"\n  multiline: folded\n  line 2"},
-  {"type":"TextWhitespace","value":"  \n"},
+  {"type":"TextWhitespace","value":"\n"},
  {"type":"NameTag","value":"c"},
  {"type":"Punctuation","value":":"},
  {"type":"TextWhitespace","value":" "},
  {"type":"Punctuation","value":"|-"},
  {"type":"LiteralStringDoc","value":"\n  multiline # literal strip\n  line 2"},
-  {"type":"TextWhitespace","value":"  \n"},
+  {"type":"TextWhitespace","value":"\n"},
  {"type":"NameTag","value":"d"},
  {"type":"Punctuation","value":":"},
  {"type":"TextWhitespace","value":" "},
  {"type":"Punctuation","value":"\u003e-"},
  {"type":"LiteralStringDoc","value":"\n  multiline folded strip\n  line 2: test\n\n  # not a comment\n   indented by 1"},
-  {"type":"TextWhitespace","value":"  \n"},
+  {"type":"TextWhitespace","value":"\n"},
  {"type":"NameTag","value":"e"},
  {"type":"Punctuation","value":":"},
  {"type":"TextWhitespace","value":" "},
  {"type":"Punctuation","value":"|+"},
  {"type":"LiteralStringDoc","value":"\n  multiline literal keep\n  line: 2"},
-  {"type":"TextWhitespace","value":"  \n"},
+  {"type":"TextWhitespace","value":"\n"},
  {"type":"Comment","value":"# this is a comment"},
  {"type":"TextWhitespace","value":"\n"},
  {"type":"NameTag","value":"f"},
@ -247,13 +247,13 @@
  {"type":"TextWhitespace","value":" "},
  {"type":"Punctuation","value":"\u003e+"},
  {"type":"LiteralStringDoc","value":"\n multiline folded keep one space\n line 2"},
-  {"type":"TextWhitespace","value":" \n"},
+  {"type":"TextWhitespace","value":"\n"},
  {"type":"NameTag","value":"g"},
  {"type":"Punctuation","value":":"},
  {"type":"TextWhitespace","value":" "},
  {"type":"Punctuation","value":"|"},
  {"type":"LiteralStringDoc","value":"\n  multiline literal with only one line"},
-  {"type":"TextWhitespace","value":"  \n"},
+  {"type":"TextWhitespace","value":"\n"},
  {"type":"NameTag","value":"h"},
  {"type":"Punctuation","value":":"},
  {"type":"TextWhitespace","value":" "},
@ -267,31 +267,31 @@
  {"type":"TextWhitespace","value":" "},
  {"type":"Punctuation","value":"|"},
  {"type":"LiteralStringDoc","value":"\n    multiline literal\n    line 2"},
-  {"type":"TextWhitespace","value":"    \n  "},
+  {"type":"TextWhitespace","value":"\n  "},
  {"type":"NameTag","value":"b"},
  {"type":"Punctuation","value":":"},
  {"type":"TextWhitespace","value":" "},
  {"type":"Punctuation","value":"\u003e"},
  {"type":"LiteralStringDoc","value":"\n    multiline: folded\n    line 2"},
-  {"type":"TextWhitespace","value":"    \n  "},
+  {"type":"TextWhitespace","value":"\n  "},
  {"type":"NameTag","value":"c"},
  {"type":"Punctuation","value":":"},
  {"type":"TextWhitespace","value":" "},
  {"type":"Punctuation","value":"|-"},
  {"type":"LiteralStringDoc","value":"\n      multiline # literal strip\n      line 2 6 leading spaces"},
-  {"type":"TextWhitespace","value":"      \n\n  "},
+  {"type":"TextWhitespace","value":"\n\n  "},
  {"type":"NameTag","value":"d"},
  {"type":"Punctuation","value":":"},
  {"type":"TextWhitespace","value":" "},
  {"type":"Punctuation","value":"\u003e-"},
  {"type":"LiteralStringDoc","value":"\n    multiline folded strip\n    line 2: test\n    # not a comment"},
-  {"type":"TextWhitespace","value":"    \n  "},
+  {"type":"TextWhitespace","value":"\n  "},
  {"type":"NameTag","value":"e"},
  {"type":"Punctuation","value":":"},
  {"type":"TextWhitespace","value":" "},
  {"type":"Punctuation","value":"|+"},
  {"type":"LiteralStringDoc","value":"\n    multiline literal keep\n    line: 2"},
-  {"type":"TextWhitespace","value":"    \n  "},
+  {"type":"TextWhitespace","value":"\n  "},
  {"type":"Comment","value":"# this is a comment"},
  {"type":"TextWhitespace","value":"\n  "},
  {"type":"NameTag","value":"f"},
@ -299,13 +299,13 @@
  {"type":"TextWhitespace","value":" "},
  {"type":"Punctuation","value":"\u003e+"},
  {"type":"LiteralStringDoc","value":"\n    multiline folded keep\n    line 2"},
-  {"type":"TextWhitespace","value":"    \n  "},
+  {"type":"TextWhitespace","value":"\n  "},
  {"type":"NameTag","value":"g"},
  {"type":"Punctuation","value":":"},
  {"type":"TextWhitespace","value":" "},
  {"type":"Punctuation","value":"|"},
  {"type":"LiteralStringDoc","value":"\n    multiline literal with only one line"},
-  {"type":"TextWhitespace","value":"    \n  "},
+  {"type":"TextWhitespace","value":"\n  "},
  {"type":"NameTag","value":"h"},
  {"type":"Punctuation","value":":"},
  {"type":"TextWhitespace","value":" "},
@ -355,7 +355,7 @@
  {"type":"TextWhitespace","value":" "},
  {"type":"Punctuation","value":"|"},
  {"type":"LiteralStringDoc","value":"\n  This is a key\n  that has multiple lines"},
-  {"type":"TextWhitespace","value":"  \n"},
+  {"type":"TextWhitespace","value":"\n"},
  {"type":"Punctuation","value":":"},
  {"type":"TextWhitespace","value":" "},
  {"type":"Literal","value":"and this is its value"},
@ -622,7 +622,7 @@
  {"type":"TextWhitespace","value":" "},
  {"type":"Punctuation","value":"|"},
  {"type":"LiteralStringDoc","value":"\n  R0lGODlhDAAMAIQAAP//9/X17unp5WZmZgAAAOfn515eXvPz7Y6OjuDg4J+fn5\n  OTk6enp56enmlpaWNjY6Ojo4SEhP/++f/++f/++f/++f/++f/++f/++f/++f/+\n  +f/++f/++f/++f/++f/++SH+Dk1hZGUgd2l0aCBHSU1QACwAAAAADAAMAAAFLC\n  AgjoEwnuNAFOhpEMTRiggcz4BNJHrv/zCFcLiwMWYNG84BwwEeECcgggoBADs="},
-  {"type":"TextWhitespace","value":"  \n\n"},
+  {"type":"TextWhitespace","value":"\n\n"},
  {"type":"Comment","value":"# YAML also has a set type, which looks like this:"},
  {"type":"TextWhitespace","value":"\n"},
  {"type":"NameTag","value":"set"},
--- a/regexp.go
+++ b/regexp.go
@ -194,6 +194,9 @@ func (l *LexerState) Iterator() Token { // nolint: gocognit
 		for len(l.iteratorStack) > 0 {
 			n := len(l.iteratorStack) - 1
 			t := l.iteratorStack[n]()
+			if t.Type == Ignore {
+				continue
+			}
 			if t == EOF {
 				l.iteratorStack = l.iteratorStack[:n]
 				continue
@ -243,6 +246,9 @@ func (l *LexerState) Iterator() Token { // nolint: gocognit
 	for len(l.iteratorStack) > 0 {
 		n := len(l.iteratorStack) - 1
 		t := l.iteratorStack[n]()
+		if t.Type == Ignore {
+			continue
+		}
 		if t == EOF {
 			l.iteratorStack = l.iteratorStack[:n]
 			continue
--- a/regexp_test.go
+++ b/regexp_test.go
@ -192,3 +192,14 @@ func TestByGroupNames(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, []Token{{Error, `abc=123`}}, it.Tokens())
 }
+
+func TestIgnoreToken(t *testing.T) {
+	l := Coalesce(mustNewLexer(t, &Config{EnsureNL: true}, Rules{ // nolint: forbidigo
+		"root": {
+			{`(\s*)(\w+)(?:\1)(\n)`, ByGroups(Ignore, Keyword, Whitespace), nil},
+		},
+	}))
+	it, err := l.Tokenise(nil, `  hello  `)
+	assert.NoError(t, err)
+	assert.Equal(t, []Token{{Keyword, "hello"}, {TextWhitespace, "\n"}}, it.Tokens())
+}
--- a/tokentype_enumer.go
+++ b/tokentype_enumer.go
@ -7,110 +7,111 @@ import (
 	"strings"
 )

-const _TokenTypeName = "NoneOtherErrorCodeLineLineLinkLineTableTDLineTableLineHighlightLineNumbersTableLineNumbersLinePreWrapperBackgroundEOFTypeKeywordKeywordConstantKeywordDeclarationKeywordNamespaceKeywordPseudoKeywordReservedKeywordTypeNameNameAttributeNameBuiltinNameBuiltinPseudoNameClassNameConstantNameDecoratorNameEntityNameExceptionNameFunctionNameFunctionMagicNameKeywordNameLabelNameNamespaceNameOperatorNameOtherNamePseudoNamePropertyNameTagNameVariableNameVariableAnonymousNameVariableClassNameVariableGlobalNameVariableInstanceNameVariableMagicLiteralLiteralDateLiteralOtherLiteralStringLiteralStringAffixLiteralStringAtomLiteralStringBacktickLiteralStringBooleanLiteralStringCharLiteralStringDelimiterLiteralStringDocLiteralStringDoubleLiteralStringEscapeLiteralStringHeredocLiteralStringInterpolLiteralStringNameLiteralStringOtherLiteralStringRegexLiteralStringSingleLiteralStringSymbolLiteralNumberLiteralNumberBinLiteralNumberFloatLiteralNumberHexLiteralNumberIntegerLiteralNumberIntegerLongLiteralNumberOctLiteralNumberByteOperatorOperatorWordPunctuationCommentCommentHashbangCommentMultilineCommentSingleCommentSpecialCommentPreprocCommentPreprocFileGenericGenericDeletedGenericEmphGenericErrorGenericHeadingGenericInsertedGenericOutputGenericPromptGenericStrongGenericSubheadingGenericTracebackGenericUnderlineTextTextWhitespaceTextSymbolTextPunctuation"
-const _TokenTypeLowerName = "noneothererrorcodelinelinelinklinetabletdlinetablelinehighlightlinenumberstablelinenumberslineprewrapperbackgroundeoftypekeywordkeywordconstantkeyworddeclarationkeywordnamespacekeywordpseudokeywordreservedkeywordtypenamenameattributenamebuiltinnamebuiltinpseudonameclassnameconstantnamedecoratornameentitynameexceptionnamefunctionnamefunctionmagicnamekeywordnamelabelnamenamespacenameoperatornameothernamepseudonamepropertynametagnamevariablenamevariableanonymousnamevariableclassnamevariableglobalnamevariableinstancenamevariablemagicliteralliteraldateliteralotherliteralstringliteralstringaffixliteralstringatomliteralstringbacktickliteralstringbooleanliteralstringcharliteralstringdelimiterliteralstringdocliteralstringdoubleliteralstringescapeliteralstringheredocliteralstringinterpolliteralstringnameliteralstringotherliteralstringregexliteralstringsingleliteralstringsymbolliteralnumberliteralnumberbinliteralnumberfloatliteralnumberhexliteralnumberintegerliteralnumberintegerlongliteralnumberoctliteralnumberbyteoperatoroperatorwordpunctuationcommentcommenthashbangcommentmultilinecommentsinglecommentspecialcommentpreproccommentpreprocfilegenericgenericdeletedgenericemphgenericerrorgenericheadinggenericinsertedgenericoutputgenericpromptgenericstronggenericsubheadinggenerictracebackgenericunderlinetexttextwhitespacetextsymboltextpunctuation"
+const _TokenTypeName = "IgnoreNoneOtherErrorCodeLineLineLinkLineTableTDLineTableLineHighlightLineNumbersTableLineNumbersLinePreWrapperBackgroundEOFTypeKeywordKeywordConstantKeywordDeclarationKeywordNamespaceKeywordPseudoKeywordReservedKeywordTypeNameNameAttributeNameBuiltinNameBuiltinPseudoNameClassNameConstantNameDecoratorNameEntityNameExceptionNameFunctionNameFunctionMagicNameKeywordNameLabelNameNamespaceNameOperatorNameOtherNamePseudoNamePropertyNameTagNameVariableNameVariableAnonymousNameVariableClassNameVariableGlobalNameVariableInstanceNameVariableMagicLiteralLiteralDateLiteralOtherLiteralStringLiteralStringAffixLiteralStringAtomLiteralStringBacktickLiteralStringBooleanLiteralStringCharLiteralStringDelimiterLiteralStringDocLiteralStringDoubleLiteralStringEscapeLiteralStringHeredocLiteralStringInterpolLiteralStringNameLiteralStringOtherLiteralStringRegexLiteralStringSingleLiteralStringSymbolLiteralNumberLiteralNumberBinLiteralNumberFloatLiteralNumberHexLiteralNumberIntegerLiteralNumberIntegerLongLiteralNumberOctLiteralNumberByteOperatorOperatorWordPunctuationCommentCommentHashbangCommentMultilineCommentSingleCommentSpecialCommentPreprocCommentPreprocFileGenericGenericDeletedGenericEmphGenericErrorGenericHeadingGenericInsertedGenericOutputGenericPromptGenericStrongGenericSubheadingGenericTracebackGenericUnderlineTextTextWhitespaceTextSymbolTextPunctuation"
+const _TokenTypeLowerName = "ignorenoneothererrorcodelinelinelinklinetabletdlinetablelinehighlightlinenumberstablelinenumberslineprewrapperbackgroundeoftypekeywordkeywordconstantkeyworddeclarationkeywordnamespacekeywordpseudokeywordreservedkeywordtypenamenameattributenamebuiltinnamebuiltinpseudonameclassnameconstantnamedecoratornameentitynameexceptionnamefunctionnamefunctionmagicnamekeywordnamelabelnamenamespacenameoperatornameothernamepseudonamepropertynametagnamevariablenamevariableanonymousnamevariableclassnamevariableglobalnamevariableinstancenamevariablemagicliteralliteraldateliteralotherliteralstringliteralstringaffixliteralstringatomliteralstringbacktickliteralstringbooleanliteralstringcharliteralstringdelimiterliteralstringdocliteralstringdoubleliteralstringescapeliteralstringheredocliteralstringinterpolliteralstringnameliteralstringotherliteralstringregexliteralstringsingleliteralstringsymbolliteralnumberliteralnumberbinliteralnumberfloatliteralnumberhexliteralnumberintegerliteralnumberintegerlongliteralnumberoctliteralnumberbyteoperatoroperatorwordpunctuationcommentcommenthashbangcommentmultilinecommentsinglecommentspecialcommentpreproccommentpreprocfilegenericgenericdeletedgenericemphgenericerrorgenericheadinggenericinsertedgenericoutputgenericpromptgenericstronggenericsubheadinggenerictracebackgenericunderlinetexttextwhitespacetextsymboltextpunctuation"

 var _TokenTypeMap = map[TokenType]string{
-	-13:  _TokenTypeName[0:4],
-	-12:  _TokenTypeName[4:9],
-	-11:  _TokenTypeName[9:14],
-	-10:  _TokenTypeName[14:22],
-	-9:   _TokenTypeName[22:30],
-	-8:   _TokenTypeName[30:41],
-	-7:   _TokenTypeName[41:50],
-	-6:   _TokenTypeName[50:63],
-	-5:   _TokenTypeName[63:79],
-	-4:   _TokenTypeName[79:90],
-	-3:   _TokenTypeName[90:94],
-	-2:   _TokenTypeName[94:104],
-	-1:   _TokenTypeName[104:114],
-	0:    _TokenTypeName[114:121],
-	1000: _TokenTypeName[121:128],
-	1001: _TokenTypeName[128:143],
-	1002: _TokenTypeName[143:161],
-	1003: _TokenTypeName[161:177],
-	1004: _TokenTypeName[177:190],
-	1005: _TokenTypeName[190:205],
-	1006: _TokenTypeName[205:216],
-	2000: _TokenTypeName[216:220],
-	2001: _TokenTypeName[220:233],
-	2002: _TokenTypeName[233:244],
-	2003: _TokenTypeName[244:261],
-	2004: _TokenTypeName[261:270],
-	2005: _TokenTypeName[270:282],
-	2006: _TokenTypeName[282:295],
-	2007: _TokenTypeName[295:305],
-	2008: _TokenTypeName[305:318],
-	2009: _TokenTypeName[318:330],
-	2010: _TokenTypeName[330:347],
-	2011: _TokenTypeName[347:358],
-	2012: _TokenTypeName[358:367],
-	2013: _TokenTypeName[367:380],
-	2014: _TokenTypeName[380:392],
-	2015: _TokenTypeName[392:401],
-	2016: _TokenTypeName[401:411],
-	2017: _TokenTypeName[411:423],
-	2018: _TokenTypeName[423:430],
-	2019: _TokenTypeName[430:442],
-	2020: _TokenTypeName[442:463],
-	2021: _TokenTypeName[463:480],
-	2022: _TokenTypeName[480:498],
-	2023: _TokenTypeName[498:518],
-	2024: _TokenTypeName[518:535],
-	3000: _TokenTypeName[535:542],
-	3001: _TokenTypeName[542:553],
-	3002: _TokenTypeName[553:565],
-	3100: _TokenTypeName[565:578],
-	3101: _TokenTypeName[578:596],
-	3102: _TokenTypeName[596:613],
-	3103: _TokenTypeName[613:634],
-	3104: _TokenTypeName[634:654],
-	3105: _TokenTypeName[654:671],
-	3106: _TokenTypeName[671:693],
-	3107: _TokenTypeName[693:709],
-	3108: _TokenTypeName[709:728],
-	3109: _TokenTypeName[728:747],
-	3110: _TokenTypeName[747:767],
-	3111: _TokenTypeName[767:788],
-	3112: _TokenTypeName[788:805],
-	3113: _TokenTypeName[805:823],
-	3114: _TokenTypeName[823:841],
-	3115: _TokenTypeName[841:860],
-	3116: _TokenTypeName[860:879],
-	3200: _TokenTypeName[879:892],
-	3201: _TokenTypeName[892:908],
-	3202: _TokenTypeName[908:926],
-	3203: _TokenTypeName[926:942],
-	3204: _TokenTypeName[942:962],
-	3205: _TokenTypeName[962:986],
-	3206: _TokenTypeName[986:1002],
-	3207: _TokenTypeName[1002:1019],
-	4000: _TokenTypeName[1019:1027],
-	4001: _TokenTypeName[1027:1039],
-	5000: _TokenTypeName[1039:1050],
-	6000: _TokenTypeName[1050:1057],
-	6001: _TokenTypeName[1057:1072],
-	6002: _TokenTypeName[1072:1088],
-	6003: _TokenTypeName[1088:1101],
-	6004: _TokenTypeName[1101:1115],
-	6100: _TokenTypeName[1115:1129],
-	6101: _TokenTypeName[1129:1147],
-	7000: _TokenTypeName[1147:1154],
-	7001: _TokenTypeName[1154:1168],
-	7002: _TokenTypeName[1168:1179],
-	7003: _TokenTypeName[1179:1191],
-	7004: _TokenTypeName[1191:1205],
-	7005: _TokenTypeName[1205:1220],
-	7006: _TokenTypeName[1220:1233],
-	7007: _TokenTypeName[1233:1246],
-	7008: _TokenTypeName[1246:1259],
-	7009: _TokenTypeName[1259:1276],
-	7010: _TokenTypeName[1276:1292],
-	7011: _TokenTypeName[1292:1308],
-	8000: _TokenTypeName[1308:1312],
-	8001: _TokenTypeName[1312:1326],
-	8002: _TokenTypeName[1326:1336],
-	8003: _TokenTypeName[1336:1351],
+	-14:  _TokenTypeName[0:6],
+	-13:  _TokenTypeName[6:10],
+	-12:  _TokenTypeName[10:15],
+	-11:  _TokenTypeName[15:20],
+	-10:  _TokenTypeName[20:28],
+	-9:   _TokenTypeName[28:36],
+	-8:   _TokenTypeName[36:47],
+	-7:   _TokenTypeName[47:56],
+	-6:   _TokenTypeName[56:69],
+	-5:   _TokenTypeName[69:85],
+	-4:   _TokenTypeName[85:96],
+	-3:   _TokenTypeName[96:100],
+	-2:   _TokenTypeName[100:110],
+	-1:   _TokenTypeName[110:120],
+	0:    _TokenTypeName[120:127],
+	1000: _TokenTypeName[127:134],
+	1001: _TokenTypeName[134:149],
+	1002: _TokenTypeName[149:167],
+	1003: _TokenTypeName[167:183],
+	1004: _TokenTypeName[183:196],
+	1005: _TokenTypeName[196:211],
+	1006: _TokenTypeName[211:222],
+	2000: _TokenTypeName[222:226],
+	2001: _TokenTypeName[226:239],
+	2002: _TokenTypeName[239:250],
+	2003: _TokenTypeName[250:267],
+	2004: _TokenTypeName[267:276],
+	2005: _TokenTypeName[276:288],
+	2006: _TokenTypeName[288:301],
+	2007: _TokenTypeName[301:311],
+	2008: _TokenTypeName[311:324],
+	2009: _TokenTypeName[324:336],
+	2010: _TokenTypeName[336:353],
+	2011: _TokenTypeName[353:364],
+	2012: _TokenTypeName[364:373],
+	2013: _TokenTypeName[373:386],
+	2014: _TokenTypeName[386:398],
+	2015: _TokenTypeName[398:407],
+	2016: _TokenTypeName[407:417],
+	2017: _TokenTypeName[417:429],
+	2018: _TokenTypeName[429:436],
+	2019: _TokenTypeName[436:448],
+	2020: _TokenTypeName[448:469],
+	2021: _TokenTypeName[469:486],
+	2022: _TokenTypeName[486:504],
+	2023: _TokenTypeName[504:524],
+	2024: _TokenTypeName[524:541],
+	3000: _TokenTypeName[541:548],
+	3001: _TokenTypeName[548:559],
+	3002: _TokenTypeName[559:571],
+	3100: _TokenTypeName[571:584],
+	3101: _TokenTypeName[584:602],
+	3102: _TokenTypeName[602:619],
+	3103: _TokenTypeName[619:640],
+	3104: _TokenTypeName[640:660],
+	3105: _TokenTypeName[660:677],
+	3106: _TokenTypeName[677:699],
+	3107: _TokenTypeName[699:715],
+	3108: _TokenTypeName[715:734],
+	3109: _TokenTypeName[734:753],
+	3110: _TokenTypeName[753:773],
+	3111: _TokenTypeName[773:794],
+	3112: _TokenTypeName[794:811],
+	3113: _TokenTypeName[811:829],
+	3114: _TokenTypeName[829:847],
+	3115: _TokenTypeName[847:866],
+	3116: _TokenTypeName[866:885],
+	3200: _TokenTypeName[885:898],
+	3201: _TokenTypeName[898:914],
+	3202: _TokenTypeName[914:932],
+	3203: _TokenTypeName[932:948],
+	3204: _TokenTypeName[948:968],
+	3205: _TokenTypeName[968:992],
+	3206: _TokenTypeName[992:1008],
+	3207: _TokenTypeName[1008:1025],
+	4000: _TokenTypeName[1025:1033],
+	4001: _TokenTypeName[1033:1045],
+	5000: _TokenTypeName[1045:1056],
+	6000: _TokenTypeName[1056:1063],
+	6001: _TokenTypeName[1063:1078],
+	6002: _TokenTypeName[1078:1094],
+	6003: _TokenTypeName[1094:1107],
+	6004: _TokenTypeName[1107:1121],
+	6100: _TokenTypeName[1121:1135],
+	6101: _TokenTypeName[1135:1153],
+	7000: _TokenTypeName[1153:1160],
+	7001: _TokenTypeName[1160:1174],
+	7002: _TokenTypeName[1174:1185],
+	7003: _TokenTypeName[1185:1197],
+	7004: _TokenTypeName[1197:1211],
+	7005: _TokenTypeName[1211:1226],
+	7006: _TokenTypeName[1226:1239],
+	7007: _TokenTypeName[1239:1252],
+	7008: _TokenTypeName[1252:1265],
+	7009: _TokenTypeName[1265:1282],
+	7010: _TokenTypeName[1282:1298],
+	7011: _TokenTypeName[1298:1314],
+	8000: _TokenTypeName[1314:1318],
+	8001: _TokenTypeName[1318:1332],
+	8002: _TokenTypeName[1332:1342],
+	8003: _TokenTypeName[1342:1357],
 }

 func (i TokenType) String() string {
@ -124,6 +125,7 @@ func (i TokenType) String() string {
 // Re-run the stringer command to generate them again.
 func _TokenTypeNoOp() {
 	var x [1]struct{}
+	_ = x[Ignore-(-14)]
 	_ = x[None-(-13)]
 	_ = x[Other-(-12)]
 	_ = x[Error-(-11)]
@ -226,312 +228,315 @@ func _TokenTypeNoOp() {
 	_ = x[TextPunctuation-(8003)]
 }

-var _TokenTypeValues = []TokenType{None, Other, Error, CodeLine, LineLink, LineTableTD, LineTable, LineHighlight, LineNumbersTable, LineNumbers, Line, PreWrapper, Background, EOFType, Keyword, KeywordConstant, KeywordDeclaration, KeywordNamespace, KeywordPseudo, KeywordReserved, KeywordType, Name, NameAttribute, NameBuiltin, NameBuiltinPseudo, NameClass, NameConstant, NameDecorator, NameEntity, NameException, NameFunction, NameFunctionMagic, NameKeyword, NameLabel, NameNamespace, NameOperator, NameOther, NamePseudo, NameProperty, NameTag, NameVariable, NameVariableAnonymous, NameVariableClass, NameVariableGlobal, NameVariableInstance, NameVariableMagic, Literal, LiteralDate, LiteralOther, LiteralString, LiteralStringAffix, LiteralStringAtom, LiteralStringBacktick, LiteralStringBoolean, LiteralStringChar, LiteralStringDelimiter, LiteralStringDoc, LiteralStringDouble, LiteralStringEscape, LiteralStringHeredoc, LiteralStringInterpol, LiteralStringName, LiteralStringOther, LiteralStringRegex, LiteralStringSingle, LiteralStringSymbol, LiteralNumber, LiteralNumberBin, LiteralNumberFloat, LiteralNumberHex, LiteralNumberInteger, LiteralNumberIntegerLong, LiteralNumberOct, LiteralNumberByte, Operator, OperatorWord, Punctuation, Comment, CommentHashbang, CommentMultiline, CommentSingle, CommentSpecial, CommentPreproc, CommentPreprocFile, Generic, GenericDeleted, GenericEmph, GenericError, GenericHeading, GenericInserted, GenericOutput, GenericPrompt, GenericStrong, GenericSubheading, GenericTraceback, GenericUnderline, Text, TextWhitespace, TextSymbol, TextPunctuation}
+var _TokenTypeValues = []TokenType{Ignore, None, Other, Error, CodeLine, LineLink, LineTableTD, LineTable, LineHighlight, LineNumbersTable, LineNumbers, Line, PreWrapper, Background, EOFType, Keyword, KeywordConstant, KeywordDeclaration, KeywordNamespace, KeywordPseudo, KeywordReserved, KeywordType, Name, NameAttribute, NameBuiltin, NameBuiltinPseudo, NameClass, NameConstant, NameDecorator, NameEntity, NameException, NameFunction, NameFunctionMagic, NameKeyword, NameLabel, NameNamespace, NameOperator, NameOther, NamePseudo, NameProperty, NameTag, NameVariable, NameVariableAnonymous, NameVariableClass, NameVariableGlobal, NameVariableInstance, NameVariableMagic, Literal, LiteralDate, LiteralOther, LiteralString, LiteralStringAffix, LiteralStringAtom, LiteralStringBacktick, LiteralStringBoolean, LiteralStringChar, LiteralStringDelimiter, LiteralStringDoc, LiteralStringDouble, LiteralStringEscape, LiteralStringHeredoc, LiteralStringInterpol, LiteralStringName, LiteralStringOther, LiteralStringRegex, LiteralStringSingle, LiteralStringSymbol, LiteralNumber, LiteralNumberBin, LiteralNumberFloat, LiteralNumberHex, LiteralNumberInteger, LiteralNumberIntegerLong, LiteralNumberOct, LiteralNumberByte, Operator, OperatorWord, Punctuation, Comment, CommentHashbang, CommentMultiline, CommentSingle, CommentSpecial, CommentPreproc, CommentPreprocFile, Generic, GenericDeleted, GenericEmph, GenericError, GenericHeading, GenericInserted, GenericOutput, GenericPrompt, GenericStrong, GenericSubheading, GenericTraceback, GenericUnderline, Text, TextWhitespace, TextSymbol, TextPunctuation}

 var _TokenTypeNameToValueMap = map[string]TokenType{
-	_TokenTypeName[0:4]:            None,
-	_TokenTypeLowerName[0:4]:       None,
-	_TokenTypeName[4:9]:            Other,
-	_TokenTypeLowerName[4:9]:       Other,
-	_TokenTypeName[9:14]:           Error,
-	_TokenTypeLowerName[9:14]:      Error,
-	_TokenTypeName[14:22]:          CodeLine,
-	_TokenTypeLowerName[14:22]:     CodeLine,
-	_TokenTypeName[22:30]:          LineLink,
-	_TokenTypeLowerName[22:30]:     LineLink,
-	_TokenTypeName[30:41]:          LineTableTD,
-	_TokenTypeLowerName[30:41]:     LineTableTD,
-	_TokenTypeName[41:50]:          LineTable,
-	_TokenTypeLowerName[41:50]:     LineTable,
-	_TokenTypeName[50:63]:          LineHighlight,
-	_TokenTypeLowerName[50:63]:     LineHighlight,
-	_TokenTypeName[63:79]:          LineNumbersTable,
-	_TokenTypeLowerName[63:79]:     LineNumbersTable,
-	_TokenTypeName[79:90]:          LineNumbers,
-	_TokenTypeLowerName[79:90]:     LineNumbers,
-	_TokenTypeName[90:94]:          Line,
-	_TokenTypeLowerName[90:94]:     Line,
-	_TokenTypeName[94:104]:         PreWrapper,
-	_TokenTypeLowerName[94:104]:    PreWrapper,
-	_TokenTypeName[104:114]:        Background,
-	_TokenTypeLowerName[104:114]:   Background,
-	_TokenTypeName[114:121]:        EOFType,
-	_TokenTypeLowerName[114:121]:   EOFType,
-	_TokenTypeName[121:128]:        Keyword,
-	_TokenTypeLowerName[121:128]:   Keyword,
-	_TokenTypeName[128:143]:        KeywordConstant,
-	_TokenTypeLowerName[128:143]:   KeywordConstant,
-	_TokenTypeName[143:161]:        KeywordDeclaration,
-	_TokenTypeLowerName[143:161]:   KeywordDeclaration,
-	_TokenTypeName[161:177]:        KeywordNamespace,
-	_TokenTypeLowerName[161:177]:   KeywordNamespace,
-	_TokenTypeName[177:190]:        KeywordPseudo,
-	_TokenTypeLowerName[177:190]:   KeywordPseudo,
-	_TokenTypeName[190:205]:        KeywordReserved,
-	_TokenTypeLowerName[190:205]:   KeywordReserved,
-	_TokenTypeName[205:216]:        KeywordType,
-	_TokenTypeLowerName[205:216]:   KeywordType,
-	_TokenTypeName[216:220]:        Name,
-	_TokenTypeLowerName[216:220]:   Name,
-	_TokenTypeName[220:233]:        NameAttribute,
-	_TokenTypeLowerName[220:233]:   NameAttribute,
-	_TokenTypeName[233:244]:        NameBuiltin,
-	_TokenTypeLowerName[233:244]:   NameBuiltin,
-	_TokenTypeName[244:261]:        NameBuiltinPseudo,
-	_TokenTypeLowerName[244:261]:   NameBuiltinPseudo,
-	_TokenTypeName[261:270]:        NameClass,
-	_TokenTypeLowerName[261:270]:   NameClass,
-	_TokenTypeName[270:282]:        NameConstant,
-	_TokenTypeLowerName[270:282]:   NameConstant,
-	_TokenTypeName[282:295]:        NameDecorator,
-	_TokenTypeLowerName[282:295]:   NameDecorator,
-	_TokenTypeName[295:305]:        NameEntity,
-	_TokenTypeLowerName[295:305]:   NameEntity,
-	_TokenTypeName[305:318]:        NameException,
-	_TokenTypeLowerName[305:318]:   NameException,
-	_TokenTypeName[318:330]:        NameFunction,
-	_TokenTypeLowerName[318:330]:   NameFunction,
-	_TokenTypeName[330:347]:        NameFunctionMagic,
-	_TokenTypeLowerName[330:347]:   NameFunctionMagic,
-	_TokenTypeName[347:358]:        NameKeyword,
-	_TokenTypeLowerName[347:358]:   NameKeyword,
-	_TokenTypeName[358:367]:        NameLabel,
-	_TokenTypeLowerName[358:367]:   NameLabel,
-	_TokenTypeName[367:380]:        NameNamespace,
-	_TokenTypeLowerName[367:380]:   NameNamespace,
-	_TokenTypeName[380:392]:        NameOperator,
-	_TokenTypeLowerName[380:392]:   NameOperator,
-	_TokenTypeName[392:401]:        NameOther,
-	_TokenTypeLowerName[392:401]:   NameOther,
-	_TokenTypeName[401:411]:        NamePseudo,
-	_TokenTypeLowerName[401:411]:   NamePseudo,
-	_TokenTypeName[411:423]:        NameProperty,
-	_TokenTypeLowerName[411:423]:   NameProperty,
-	_TokenTypeName[423:430]:        NameTag,
-	_TokenTypeLowerName[423:430]:   NameTag,
-	_TokenTypeName[430:442]:        NameVariable,
-	_TokenTypeLowerName[430:442]:   NameVariable,
-	_TokenTypeName[442:463]:        NameVariableAnonymous,
-	_TokenTypeLowerName[442:463]:   NameVariableAnonymous,
-	_TokenTypeName[463:480]:        NameVariableClass,
-	_TokenTypeLowerName[463:480]:   NameVariableClass,
-	_TokenTypeName[480:498]:        NameVariableGlobal,
-	_TokenTypeLowerName[480:498]:   NameVariableGlobal,
-	_TokenTypeName[498:518]:        NameVariableInstance,
-	_TokenTypeLowerName[498:518]:   NameVariableInstance,
-	_TokenTypeName[518:535]:        NameVariableMagic,
-	_TokenTypeLowerName[518:535]:   NameVariableMagic,
-	_TokenTypeName[535:542]:        Literal,
-	_TokenTypeLowerName[535:542]:   Literal,
-	_TokenTypeName[542:553]:        LiteralDate,
-	_TokenTypeLowerName[542:553]:   LiteralDate,
-	_TokenTypeName[553:565]:        LiteralOther,
-	_TokenTypeLowerName[553:565]:   LiteralOther,
-	_TokenTypeName[565:578]:        LiteralString,
-	_TokenTypeLowerName[565:578]:   LiteralString,
-	_TokenTypeName[578:596]:        LiteralStringAffix,
-	_TokenTypeLowerName[578:596]:   LiteralStringAffix,
-	_TokenTypeName[596:613]:        LiteralStringAtom,
-	_TokenTypeLowerName[596:613]:   LiteralStringAtom,
-	_TokenTypeName[613:634]:        LiteralStringBacktick,
-	_TokenTypeLowerName[613:634]:   LiteralStringBacktick,
-	_TokenTypeName[634:654]:        LiteralStringBoolean,
-	_TokenTypeLowerName[634:654]:   LiteralStringBoolean,
-	_TokenTypeName[654:671]:        LiteralStringChar,
-	_TokenTypeLowerName[654:671]:   LiteralStringChar,
-	_TokenTypeName[671:693]:        LiteralStringDelimiter,
-	_TokenTypeLowerName[671:693]:   LiteralStringDelimiter,
-	_TokenTypeName[693:709]:        LiteralStringDoc,
-	_TokenTypeLowerName[693:709]:   LiteralStringDoc,
-	_TokenTypeName[709:728]:        LiteralStringDouble,
-	_TokenTypeLowerName[709:728]:   LiteralStringDouble,
-	_TokenTypeName[728:747]:        LiteralStringEscape,
-	_TokenTypeLowerName[728:747]:   LiteralStringEscape,
-	_TokenTypeName[747:767]:        LiteralStringHeredoc,
-	_TokenTypeLowerName[747:767]:   LiteralStringHeredoc,
-	_TokenTypeName[767:788]:        LiteralStringInterpol,
-	_TokenTypeLowerName[767:788]:   LiteralStringInterpol,
-	_TokenTypeName[788:805]:        LiteralStringName,
-	_TokenTypeLowerName[788:805]:   LiteralStringName,
-	_TokenTypeName[805:823]:        LiteralStringOther,
-	_TokenTypeLowerName[805:823]:   LiteralStringOther,
-	_TokenTypeName[823:841]:        LiteralStringRegex,
-	_TokenTypeLowerName[823:841]:   LiteralStringRegex,
-	_TokenTypeName[841:860]:        LiteralStringSingle,
-	_TokenTypeLowerName[841:860]:   LiteralStringSingle,
-	_TokenTypeName[860:879]:        LiteralStringSymbol,
-	_TokenTypeLowerName[860:879]:   LiteralStringSymbol,
-	_TokenTypeName[879:892]:        LiteralNumber,
-	_TokenTypeLowerName[879:892]:   LiteralNumber,
-	_TokenTypeName[892:908]:        LiteralNumberBin,
-	_TokenTypeLowerName[892:908]:   LiteralNumberBin,
-	_TokenTypeName[908:926]:        LiteralNumberFloat,
-	_TokenTypeLowerName[908:926]:   LiteralNumberFloat,
-	_TokenTypeName[926:942]:        LiteralNumberHex,
-	_TokenTypeLowerName[926:942]:   LiteralNumberHex,
-	_TokenTypeName[942:962]:        LiteralNumberInteger,
-	_TokenTypeLowerName[942:962]:   LiteralNumberInteger,
-	_TokenTypeName[962:986]:        LiteralNumberIntegerLong,
-	_TokenTypeLowerName[962:986]:   LiteralNumberIntegerLong,
-	_TokenTypeName[986:1002]:       LiteralNumberOct,
-	_TokenTypeLowerName[986:1002]:  LiteralNumberOct,
-	_TokenTypeName[1002:1019]:      LiteralNumberByte,
-	_TokenTypeLowerName[1002:1019]: LiteralNumberByte,
-	_TokenTypeName[1019:1027]:      Operator,
-	_TokenTypeLowerName[1019:1027]: Operator,
-	_TokenTypeName[1027:1039]:      OperatorWord,
-	_TokenTypeLowerName[1027:1039]: OperatorWord,
-	_TokenTypeName[1039:1050]:      Punctuation,
-	_TokenTypeLowerName[1039:1050]: Punctuation,
-	_TokenTypeName[1050:1057]:      Comment,
-	_TokenTypeLowerName[1050:1057]: Comment,
-	_TokenTypeName[1057:1072]:      CommentHashbang,
-	_TokenTypeLowerName[1057:1072]: CommentHashbang,
-	_TokenTypeName[1072:1088]:      CommentMultiline,
-	_TokenTypeLowerName[1072:1088]: CommentMultiline,
-	_TokenTypeName[1088:1101]:      CommentSingle,
-	_TokenTypeLowerName[1088:1101]: CommentSingle,
-	_TokenTypeName[1101:1115]:      CommentSpecial,
-	_TokenTypeLowerName[1101:1115]: CommentSpecial,
-	_TokenTypeName[1115:1129]:      CommentPreproc,
-	_TokenTypeLowerName[1115:1129]: CommentPreproc,
-	_TokenTypeName[1129:1147]:      CommentPreprocFile,
-	_TokenTypeLowerName[1129:1147]: CommentPreprocFile,
-	_TokenTypeName[1147:1154]:      Generic,
-	_TokenTypeLowerName[1147:1154]: Generic,
-	_TokenTypeName[1154:1168]:      GenericDeleted,
-	_TokenTypeLowerName[1154:1168]: GenericDeleted,
-	_TokenTypeName[1168:1179]:      GenericEmph,
-	_TokenTypeLowerName[1168:1179]: GenericEmph,
-	_TokenTypeName[1179:1191]:      GenericError,
-	_TokenTypeLowerName[1179:1191]: GenericError,
-	_TokenTypeName[1191:1205]:      GenericHeading,
-	_TokenTypeLowerName[1191:1205]: GenericHeading,
-	_TokenTypeName[1205:1220]:      GenericInserted,
-	_TokenTypeLowerName[1205:1220]: GenericInserted,
-	_TokenTypeName[1220:1233]:      GenericOutput,
-	_TokenTypeLowerName[1220:1233]: GenericOutput,
-	_TokenTypeName[1233:1246]:      GenericPrompt,
-	_TokenTypeLowerName[1233:1246]: GenericPrompt,
-	_TokenTypeName[1246:1259]:      GenericStrong,
-	_TokenTypeLowerName[1246:1259]: GenericStrong,
-	_TokenTypeName[1259:1276]:      GenericSubheading,
-	_TokenTypeLowerName[1259:1276]: GenericSubheading,
-	_TokenTypeName[1276:1292]:      GenericTraceback,
-	_TokenTypeLowerName[1276:1292]: GenericTraceback,
-	_TokenTypeName[1292:1308]:      GenericUnderline,
-	_TokenTypeLowerName[1292:1308]: GenericUnderline,
-	_TokenTypeName[1308:1312]:      Text,
-	_TokenTypeLowerName[1308:1312]: Text,
-	_TokenTypeName[1312:1326]:      TextWhitespace,
-	_TokenTypeLowerName[1312:1326]: TextWhitespace,
-	_TokenTypeName[1326:1336]:      TextSymbol,
-	_TokenTypeLowerName[1326:1336]: TextSymbol,
-	_TokenTypeName[1336:1351]:      TextPunctuation,
-	_TokenTypeLowerName[1336:1351]: TextPunctuation,
+	_TokenTypeName[0:6]:            Ignore,
+	_TokenTypeLowerName[0:6]:       Ignore,
+	_TokenTypeName[6:10]:           None,
+	_TokenTypeLowerName[6:10]:      None,
+	_TokenTypeName[10:15]:          Other,
+	_TokenTypeLowerName[10:15]:     Other,
+	_TokenTypeName[15:20]:          Error,
+	_TokenTypeLowerName[15:20]:     Error,
+	_TokenTypeName[20:28]:          CodeLine,
+	_TokenTypeLowerName[20:28]:     CodeLine,
+	_TokenTypeName[28:36]:          LineLink,
+	_TokenTypeLowerName[28:36]:     LineLink,
+	_TokenTypeName[36:47]:          LineTableTD,
+	_TokenTypeLowerName[36:47]:     LineTableTD,
+	_TokenTypeName[47:56]:          LineTable,
+	_TokenTypeLowerName[47:56]:     LineTable,
+	_TokenTypeName[56:69]:          LineHighlight,
+	_TokenTypeLowerName[56:69]:     LineHighlight,
+	_TokenTypeName[69:85]:          LineNumbersTable,
+	_TokenTypeLowerName[69:85]:     LineNumbersTable,
+	_TokenTypeName[85:96]:          LineNumbers,
+	_TokenTypeLowerName[85:96]:     LineNumbers,
+	_TokenTypeName[96:100]:         Line,
+	_TokenTypeLowerName[96:100]:    Line,
+	_TokenTypeName[100:110]:        PreWrapper,
+	_TokenTypeLowerName[100:110]:   PreWrapper,
+	_TokenTypeName[110:120]:        Background,
+	_TokenTypeLowerName[110:120]:   Background,
+	_TokenTypeName[120:127]:        EOFType,
+	_TokenTypeLowerName[120:127]:   EOFType,
+	_TokenTypeName[127:134]:        Keyword,
+	_TokenTypeLowerName[127:134]:   Keyword,
+	_TokenTypeName[134:149]:        KeywordConstant,
+	_TokenTypeLowerName[134:149]:   KeywordConstant,
+	_TokenTypeName[149:167]:        KeywordDeclaration,
+	_TokenTypeLowerName[149:167]:   KeywordDeclaration,
+	_TokenTypeName[167:183]:        KeywordNamespace,
+	_TokenTypeLowerName[167:183]:   KeywordNamespace,
+	_TokenTypeName[183:196]:        KeywordPseudo,
+	_TokenTypeLowerName[183:196]:   KeywordPseudo,
+	_TokenTypeName[196:211]:        KeywordReserved,
+	_TokenTypeLowerName[196:211]:   KeywordReserved,
+	_TokenTypeName[211:222]:        KeywordType,
+	_TokenTypeLowerName[211:222]:   KeywordType,
+	_TokenTypeName[222:226]:        Name,
+	_TokenTypeLowerName[222:226]:   Name,
+	_TokenTypeName[226:239]:        NameAttribute,
+	_TokenTypeLowerName[226:239]:   NameAttribute,
+	_TokenTypeName[239:250]:        NameBuiltin,
+	_TokenTypeLowerName[239:250]:   NameBuiltin,
+	_TokenTypeName[250:267]:        NameBuiltinPseudo,
+	_TokenTypeLowerName[250:267]:   NameBuiltinPseudo,
+	_TokenTypeName[267:276]:        NameClass,
+	_TokenTypeLowerName[267:276]:   NameClass,
+	_TokenTypeName[276:288]:        NameConstant,
+	_TokenTypeLowerName[276:288]:   NameConstant,
+	_TokenTypeName[288:301]:        NameDecorator,
+	_TokenTypeLowerName[288:301]:   NameDecorator,
+	_TokenTypeName[301:311]:        NameEntity,
+	_TokenTypeLowerName[301:311]:   NameEntity,
+	_TokenTypeName[311:324]:        NameException,
+	_TokenTypeLowerName[311:324]:   NameException,
+	_TokenTypeName[324:336]:        NameFunction,
+	_TokenTypeLowerName[324:336]:   NameFunction,
+	_TokenTypeName[336:353]:        NameFunctionMagic,
+	_TokenTypeLowerName[336:353]:   NameFunctionMagic,
+	_TokenTypeName[353:364]:        NameKeyword,
+	_TokenTypeLowerName[353:364]:   NameKeyword,
+	_TokenTypeName[364:373]:        NameLabel,
+	_TokenTypeLowerName[364:373]:   NameLabel,
+	_TokenTypeName[373:386]:        NameNamespace,
+	_TokenTypeLowerName[373:386]:   NameNamespace,
+	_TokenTypeName[386:398]:        NameOperator,
+	_TokenTypeLowerName[386:398]:   NameOperator,
+	_TokenTypeName[398:407]:        NameOther,
+	_TokenTypeLowerName[398:407]:   NameOther,
+	_TokenTypeName[407:417]:        NamePseudo,
+	_TokenTypeLowerName[407:417]:   NamePseudo,
+	_TokenTypeName[417:429]:        NameProperty,
+	_TokenTypeLowerName[417:429]:   NameProperty,
+	_TokenTypeName[429:436]:        NameTag,
+	_TokenTypeLowerName[429:436]:   NameTag,
+	_TokenTypeName[436:448]:        NameVariable,
+	_TokenTypeLowerName[436:448]:   NameVariable,
+	_TokenTypeName[448:469]:        NameVariableAnonymous,
+	_TokenTypeLowerName[448:469]:   NameVariableAnonymous,
+	_TokenTypeName[469:486]:        NameVariableClass,
+	_TokenTypeLowerName[469:486]:   NameVariableClass,
+	_TokenTypeName[486:504]:        NameVariableGlobal,
+	_TokenTypeLowerName[486:504]:   NameVariableGlobal,
+	_TokenTypeName[504:524]:        NameVariableInstance,
+	_TokenTypeLowerName[504:524]:   NameVariableInstance,
+	_TokenTypeName[524:541]:        NameVariableMagic,
+	_TokenTypeLowerName[524:541]:   NameVariableMagic,
+	_TokenTypeName[541:548]:        Literal,
+	_TokenTypeLowerName[541:548]:   Literal,
+	_TokenTypeName[548:559]:        LiteralDate,
+	_TokenTypeLowerName[548:559]:   LiteralDate,
+	_TokenTypeName[559:571]:        LiteralOther,
+	_TokenTypeLowerName[559:571]:   LiteralOther,
+	_TokenTypeName[571:584]:        LiteralString,
+	_TokenTypeLowerName[571:584]:   LiteralString,
+	_TokenTypeName[584:602]:        LiteralStringAffix,
+	_TokenTypeLowerName[584:602]:   LiteralStringAffix,
+	_TokenTypeName[602:619]:        LiteralStringAtom,
+	_TokenTypeLowerName[602:619]:   LiteralStringAtom,
+	_TokenTypeName[619:640]:        LiteralStringBacktick,
+	_TokenTypeLowerName[619:640]:   LiteralStringBacktick,
+	_TokenTypeName[640:660]:        LiteralStringBoolean,
+	_TokenTypeLowerName[640:660]:   LiteralStringBoolean,
+	_TokenTypeName[660:677]:        LiteralStringChar,
+	_TokenTypeLowerName[660:677]:   LiteralStringChar,
+	_TokenTypeName[677:699]:        LiteralStringDelimiter,
+	_TokenTypeLowerName[677:699]:   LiteralStringDelimiter,
+	_TokenTypeName[699:715]:        LiteralStringDoc,
+	_TokenTypeLowerName[699:715]:   LiteralStringDoc,
+	_TokenTypeName[715:734]:        LiteralStringDouble,
+	_TokenTypeLowerName[715:734]:   LiteralStringDouble,
+	_TokenTypeName[734:753]:        LiteralStringEscape,
+	_TokenTypeLowerName[734:753]:   LiteralStringEscape,
+	_TokenTypeName[753:773]:        LiteralStringHeredoc,
+	_TokenTypeLowerName[753:773]:   LiteralStringHeredoc,
+	_TokenTypeName[773:794]:        LiteralStringInterpol,
+	_TokenTypeLowerName[773:794]:   LiteralStringInterpol,
+	_TokenTypeName[794:811]:        LiteralStringName,
+	_TokenTypeLowerName[794:811]:   LiteralStringName,
+	_TokenTypeName[811:829]:        LiteralStringOther,
+	_TokenTypeLowerName[811:829]:   LiteralStringOther,
+	_TokenTypeName[829:847]:        LiteralStringRegex,
+	_TokenTypeLowerName[829:847]:   LiteralStringRegex,
+	_TokenTypeName[847:866]:        LiteralStringSingle,
+	_TokenTypeLowerName[847:866]:   LiteralStringSingle,
+	_TokenTypeName[866:885]:        LiteralStringSymbol,
+	_TokenTypeLowerName[866:885]:   LiteralStringSymbol,
+	_TokenTypeName[885:898]:        LiteralNumber,
+	_TokenTypeLowerName[885:898]:   LiteralNumber,
+	_TokenTypeName[898:914]:        LiteralNumberBin,
+	_TokenTypeLowerName[898:914]:   LiteralNumberBin,
+	_TokenTypeName[914:932]:        LiteralNumberFloat,
+	_TokenTypeLowerName[914:932]:   LiteralNumberFloat,
+	_TokenTypeName[932:948]:        LiteralNumberHex,
+	_TokenTypeLowerName[932:948]:   LiteralNumberHex,
+	_TokenTypeName[948:968]:        LiteralNumberInteger,
+	_TokenTypeLowerName[948:968]:   LiteralNumberInteger,
+	_TokenTypeName[968:992]:        LiteralNumberIntegerLong,
+	_TokenTypeLowerName[968:992]:   LiteralNumberIntegerLong,
+	_TokenTypeName[992:1008]:       LiteralNumberOct,
+	_TokenTypeLowerName[992:1008]:  LiteralNumberOct,
+	_TokenTypeName[1008:1025]:      LiteralNumberByte,
+	_TokenTypeLowerName[1008:1025]: LiteralNumberByte,
+	_TokenTypeName[1025:1033]:      Operator,
+	_TokenTypeLowerName[1025:1033]: Operator,
+	_TokenTypeName[1033:1045]:      OperatorWord,
+	_TokenTypeLowerName[1033:1045]: OperatorWord,
+	_TokenTypeName[1045:1056]:      Punctuation,
+	_TokenTypeLowerName[1045:1056]: Punctuation,
+	_TokenTypeName[1056:1063]:      Comment,
+	_TokenTypeLowerName[1056:1063]: Comment,
+	_TokenTypeName[1063:1078]:      CommentHashbang,
+	_TokenTypeLowerName[1063:1078]: CommentHashbang,
+	_TokenTypeName[1078:1094]:      CommentMultiline,
+	_TokenTypeLowerName[1078:1094]: CommentMultiline,
+	_TokenTypeName[1094:1107]:      CommentSingle,
+	_TokenTypeLowerName[1094:1107]: CommentSingle,
+	_TokenTypeName[1107:1121]:      CommentSpecial,
+	_TokenTypeLowerName[1107:1121]: CommentSpecial,
+	_TokenTypeName[1121:1135]:      CommentPreproc,
+	_TokenTypeLowerName[1121:1135]: CommentPreproc,
+	_TokenTypeName[1135:1153]:      CommentPreprocFile,
+	_TokenTypeLowerName[1135:1153]: CommentPreprocFile,
+	_TokenTypeName[1153:1160]:      Generic,
+	_TokenTypeLowerName[1153:1160]: Generic,
+	_TokenTypeName[1160:1174]:      GenericDeleted,
+	_TokenTypeLowerName[1160:1174]: GenericDeleted,
+	_TokenTypeName[1174:1185]:      GenericEmph,
+	_TokenTypeLowerName[1174:1185]: GenericEmph,
+	_TokenTypeName[1185:1197]:      GenericError,
+	_TokenTypeLowerName[1185:1197]: GenericError,
+	_TokenTypeName[1197:1211]:      GenericHeading,
+	_TokenTypeLowerName[1197:1211]: GenericHeading,
+	_TokenTypeName[1211:1226]:      GenericInserted,
+	_TokenTypeLowerName[1211:1226]: GenericInserted,
+	_TokenTypeName[1226:1239]:      GenericOutput,
+	_TokenTypeLowerName[1226:1239]: GenericOutput,
+	_TokenTypeName[1239:1252]:      GenericPrompt,
+	_TokenTypeLowerName[1239:1252]: GenericPrompt,
+	_TokenTypeName[1252:1265]:      GenericStrong,
+	_TokenTypeLowerName[1252:1265]: GenericStrong,
+	_TokenTypeName[1265:1282]:      GenericSubheading,
+	_TokenTypeLowerName[1265:1282]: GenericSubheading,
+	_TokenTypeName[1282:1298]:      GenericTraceback,
+	_TokenTypeLowerName[1282:1298]: GenericTraceback,
+	_TokenTypeName[1298:1314]:      GenericUnderline,
+	_TokenTypeLowerName[1298:1314]: GenericUnderline,
+	_TokenTypeName[1314:1318]:      Text,
+	_TokenTypeLowerName[1314:1318]: Text,
+	_TokenTypeName[1318:1332]:      TextWhitespace,
+	_TokenTypeLowerName[1318:1332]: TextWhitespace,
+	_TokenTypeName[1332:1342]:      TextSymbol,
+	_TokenTypeLowerName[1332:1342]: TextSymbol,
+	_TokenTypeName[1342:1357]:      TextPunctuation,
+	_TokenTypeLowerName[1342:1357]: TextPunctuation,
 }

 var _TokenTypeNames = []string{
-	_TokenTypeName[0:4],
-	_TokenTypeName[4:9],
-	_TokenTypeName[9:14],
-	_TokenTypeName[14:22],
-	_TokenTypeName[22:30],
-	_TokenTypeName[30:41],
-	_TokenTypeName[41:50],
-	_TokenTypeName[50:63],
-	_TokenTypeName[63:79],
-	_TokenTypeName[79:90],
-	_TokenTypeName[90:94],
-	_TokenTypeName[94:104],
-	_TokenTypeName[104:114],
-	_TokenTypeName[114:121],
-	_TokenTypeName[121:128],
-	_TokenTypeName[128:143],
-	_TokenTypeName[143:161],
-	_TokenTypeName[161:177],
-	_TokenTypeName[177:190],
-	_TokenTypeName[190:205],
-	_TokenTypeName[205:216],
-	_TokenTypeName[216:220],
-	_TokenTypeName[220:233],
-	_TokenTypeName[233:244],
-	_TokenTypeName[244:261],
-	_TokenTypeName[261:270],
-	_TokenTypeName[270:282],
-	_TokenTypeName[282:295],
-	_TokenTypeName[295:305],
-	_TokenTypeName[305:318],
-	_TokenTypeName[318:330],
-	_TokenTypeName[330:347],
-	_TokenTypeName[347:358],
-	_TokenTypeName[358:367],
-	_TokenTypeName[367:380],
-	_TokenTypeName[380:392],
-	_TokenTypeName[392:401],
-	_TokenTypeName[401:411],
-	_TokenTypeName[411:423],
-	_TokenTypeName[423:430],
-	_TokenTypeName[430:442],
-	_TokenTypeName[442:463],
-	_TokenTypeName[463:480],
-	_TokenTypeName[480:498],
-	_TokenTypeName[498:518],
-	_TokenTypeName[518:535],
-	_TokenTypeName[535:542],
-	_TokenTypeName[542:553],
-	_TokenTypeName[553:565],
-	_TokenTypeName[565:578],
-	_TokenTypeName[578:596],
-	_TokenTypeName[596:613],
-	_TokenTypeName[613:634],
-	_TokenTypeName[634:654],
-	_TokenTypeName[654:671],
-	_TokenTypeName[671:693],
-	_TokenTypeName[693:709],
-	_TokenTypeName[709:728],
-	_TokenTypeName[728:747],
-	_TokenTypeName[747:767],
-	_TokenTypeName[767:788],
-	_TokenTypeName[788:805],
-	_TokenTypeName[805:823],
-	_TokenTypeName[823:841],
-	_TokenTypeName[841:860],
-	_TokenTypeName[860:879],
-	_TokenTypeName[879:892],
-	_TokenTypeName[892:908],
-	_TokenTypeName[908:926],
-	_TokenTypeName[926:942],
-	_TokenTypeName[942:962],
-	_TokenTypeName[962:986],
-	_TokenTypeName[986:1002],
-	_TokenTypeName[1002:1019],
-	_TokenTypeName[1019:1027],
-	_TokenTypeName[1027:1039],
-	_TokenTypeName[1039:1050],
-	_TokenTypeName[1050:1057],
-	_TokenTypeName[1057:1072],
-	_TokenTypeName[1072:1088],
-	_TokenTypeName[1088:1101],
-	_TokenTypeName[1101:1115],
-	_TokenTypeName[1115:1129],
-	_TokenTypeName[1129:1147],
-	_TokenTypeName[1147:1154],
-	_TokenTypeName[1154:1168],
-	_TokenTypeName[1168:1179],
-	_TokenTypeName[1179:1191],
-	_TokenTypeName[1191:1205],
-	_TokenTypeName[1205:1220],
-	_TokenTypeName[1220:1233],
-	_TokenTypeName[1233:1246],
-	_TokenTypeName[1246:1259],
-	_TokenTypeName[1259:1276],
-	_TokenTypeName[1276:1292],
-	_TokenTypeName[1292:1308],
-	_TokenTypeName[1308:1312],
-	_TokenTypeName[1312:1326],
-	_TokenTypeName[1326:1336],
-	_TokenTypeName[1336:1351],
+	_TokenTypeName[0:6],
+	_TokenTypeName[6:10],
+	_TokenTypeName[10:15],
+	_TokenTypeName[15:20],
+	_TokenTypeName[20:28],
+	_TokenTypeName[28:36],
+	_TokenTypeName[36:47],
+	_TokenTypeName[47:56],
+	_TokenTypeName[56:69],
+	_TokenTypeName[69:85],
+	_TokenTypeName[85:96],
+	_TokenTypeName[96:100],
+	_TokenTypeName[100:110],
+	_TokenTypeName[110:120],
+	_TokenTypeName[120:127],
+	_TokenTypeName[127:134],
+	_TokenTypeName[134:149],
+	_TokenTypeName[149:167],
+	_TokenTypeName[167:183],
+	_TokenTypeName[183:196],
+	_TokenTypeName[196:211],
+	_TokenTypeName[211:222],
+	_TokenTypeName[222:226],
+	_TokenTypeName[226:239],
+	_TokenTypeName[239:250],
+	_TokenTypeName[250:267],
+	_TokenTypeName[267:276],
+	_TokenTypeName[276:288],
+	_TokenTypeName[288:301],
+	_TokenTypeName[301:311],
+	_TokenTypeName[311:324],
+	_TokenTypeName[324:336],
+	_TokenTypeName[336:353],
+	_TokenTypeName[353:364],
+	_TokenTypeName[364:373],
+	_TokenTypeName[373:386],
+	_TokenTypeName[386:398],
+	_TokenTypeName[398:407],
+	_TokenTypeName[407:417],
+	_TokenTypeName[417:429],
+	_TokenTypeName[429:436],
+	_TokenTypeName[436:448],
+	_TokenTypeName[448:469],
+	_TokenTypeName[469:486],
+	_TokenTypeName[486:504],
+	_TokenTypeName[504:524],
+	_TokenTypeName[524:541],
+	_TokenTypeName[541:548],
+	_TokenTypeName[548:559],
+	_TokenTypeName[559:571],
+	_TokenTypeName[571:584],
+	_TokenTypeName[584:602],
+	_TokenTypeName[602:619],
+	_TokenTypeName[619:640],
+	_TokenTypeName[640:660],
+	_TokenTypeName[660:677],
+	_TokenTypeName[677:699],
+	_TokenTypeName[699:715],
+	_TokenTypeName[715:734],
+	_TokenTypeName[734:753],
+	_TokenTypeName[753:773],
+	_TokenTypeName[773:794],
+	_TokenTypeName[794:811],
+	_TokenTypeName[811:829],
+	_TokenTypeName[829:847],
+	_TokenTypeName[847:866],
+	_TokenTypeName[866:885],
+	_TokenTypeName[885:898],
+	_TokenTypeName[898:914],
+	_TokenTypeName[914:932],
+	_TokenTypeName[932:948],
+	_TokenTypeName[948:968],
+	_TokenTypeName[968:992],
+	_TokenTypeName[992:1008],
+	_TokenTypeName[1008:1025],
+	_TokenTypeName[1025:1033],
+	_TokenTypeName[1033:1045],
+	_TokenTypeName[1045:1056],
+	_TokenTypeName[1056:1063],
+	_TokenTypeName[1063:1078],
+	_TokenTypeName[1078:1094],
+	_TokenTypeName[1094:1107],
+	_TokenTypeName[1107:1121],
+	_TokenTypeName[1121:1135],
+	_TokenTypeName[1135:1153],
+	_TokenTypeName[1153:1160],
+	_TokenTypeName[1160:1174],
+	_TokenTypeName[1174:1185],
+	_TokenTypeName[1185:1197],
+	_TokenTypeName[1197:1211],
+	_TokenTypeName[1211:1226],
+	_TokenTypeName[1226:1239],
+	_TokenTypeName[1239:1252],
+	_TokenTypeName[1252:1265],
+	_TokenTypeName[1265:1282],
+	_TokenTypeName[1282:1298],
+	_TokenTypeName[1298:1314],
+	_TokenTypeName[1314:1318],
+	_TokenTypeName[1318:1332],
+	_TokenTypeName[1332:1342],
+	_TokenTypeName[1342:1357],
 }

 // TokenTypeString retrieves an enum value from the enum constants string name.
--- a/types.go
+++ b/types.go
@ -41,6 +41,8 @@ const (
 	Other
 	// No highlighting.
 	None
+	// Don't emit this token to the output.
+	Ignore
 	// Used as an EOF marker / nil token
 	EOFType TokenType = 0
 )