1
0
mirror of https://github.com/alecthomas/chroma.git synced 2025-03-17 20:58:08 +02:00

Don't output extra whitespace in YAML multiline (#993)

This resolves a particular issue with parsing YAML multiline, for
example:
```yaml
a: |
  multiline literal
  line 2
```

The regex used would capture the amount of indentation in the third
capture group and then use that as a kind of "status" to know which
lines are part of the indented multiline. However, because its a
captured group it has to be assigned a token which was `TextWhitespace`.
This meant that the indentation was outputted after the multiline,
technically it should be seen as an non-captured group, but then its no
longer to refer to it in the regex. Therefore I've gone with the
solution to add a new token, Ignore, which will not be emitted as a
token in the iterator, which can safely be used to make use of capture
groups but not have them show up in the output.

## Before

![image](https://github.com/user-attachments/assets/c29353c5-9e15-4f14-a733-57a60fb51910)

## After

![image](https://github.com/user-attachments/assets/57b5d129-a9d3-4b84-ae1f-dc05182b9ad3)
This commit is contained in:
Gusted 2024-08-22 22:58:31 +02:00 committed by GitHub
parent 895a0488b5
commit 4d11870090
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 448 additions and 424 deletions

View File

@ -53,7 +53,7 @@
<bygroups>
<token type="Punctuation"/>
<token type="LiteralStringDoc"/>
<token type="TextWhitespace"/>
<token type="Ignore"/>
</bygroups>
</rule>
<rule pattern="(false|False|FALSE|true|True|TRUE|null|Off|off|yes|Yes|YES|OFF|On|ON|no|No|on|NO|n|N|Y|y)\b">

View File

@ -191,55 +191,55 @@
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|"},
{"type":"LiteralStringDoc","value":"\n This entire block of text will be the value of the 'literal_block' key,\n with line breaks being preserved.\n\n The literal continues until de-dented, and the leading indentation is\n stripped.\n\n Any lines that are 'more-indented' keep the rest of their indentation -\n these lines will be indented by 4 spaces."},
{"type":"TextWhitespace","value":" \n"},
{"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"folded_style"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"\u003e"},
{"type":"LiteralStringDoc","value":"\n This entire block of text will be the value of 'folded_style', but this\n time, all newlines will be replaced with a single space.\n\n Blank lines, like above, are converted to a newline character.\n\n 'More-indented' lines keep their newlines, too -\n this text will appear over two lines."},
{"type":"TextWhitespace","value":" \n"},
{"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"literal_block_with_strip_chomping"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|-"},
{"type":"LiteralStringDoc","value":"\n This entire block of text will be the value of the 'literal_block' key,\n with line breaks being preserved and the strip chomping indicator.\n\n The literal continues until de-dented, and the leading indentation is\n stripped.\n\n Any lines that are 'more-indented' keep the rest of their indentation -\n these lines will be indented by 4 spaces."},
{"type":"TextWhitespace","value":" \n"},
{"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"literal_block_with_keep_chomping"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|+"},
{"type":"LiteralStringDoc","value":"\n This entire block of text will be the value of the 'literal_block' key,\n with line breaks being preserved and the keep chomping indicator.\n\n The literal continues until de-dented, and the leading indentation is\n stripped."},
{"type":"TextWhitespace","value":" \n\n"},
{"type":"TextWhitespace","value":"\n\n"},
{"type":"NameTag","value":"a"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|"},
{"type":"LiteralStringDoc","value":"\n multiline literal\n line 2"},
{"type":"TextWhitespace","value":" \n"},
{"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"b"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"\u003e"},
{"type":"LiteralStringDoc","value":"\n multiline: folded\n line 2"},
{"type":"TextWhitespace","value":" \n"},
{"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"c"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|-"},
{"type":"LiteralStringDoc","value":"\n multiline # literal strip\n line 2"},
{"type":"TextWhitespace","value":" \n"},
{"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"d"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"\u003e-"},
{"type":"LiteralStringDoc","value":"\n multiline folded strip\n line 2: test\n\n # not a comment\n indented by 1"},
{"type":"TextWhitespace","value":" \n"},
{"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"e"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|+"},
{"type":"LiteralStringDoc","value":"\n multiline literal keep\n line: 2"},
{"type":"TextWhitespace","value":" \n"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Comment","value":"# this is a comment"},
{"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"f"},
@ -247,13 +247,13 @@
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"\u003e+"},
{"type":"LiteralStringDoc","value":"\n multiline folded keep one space\n line 2"},
{"type":"TextWhitespace","value":" \n"},
{"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"g"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|"},
{"type":"LiteralStringDoc","value":"\n multiline literal with only one line"},
{"type":"TextWhitespace","value":" \n"},
{"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"h"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
@ -267,31 +267,31 @@
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|"},
{"type":"LiteralStringDoc","value":"\n multiline literal\n line 2"},
{"type":"TextWhitespace","value":" \n "},
{"type":"TextWhitespace","value":"\n "},
{"type":"NameTag","value":"b"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"\u003e"},
{"type":"LiteralStringDoc","value":"\n multiline: folded\n line 2"},
{"type":"TextWhitespace","value":" \n "},
{"type":"TextWhitespace","value":"\n "},
{"type":"NameTag","value":"c"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|-"},
{"type":"LiteralStringDoc","value":"\n multiline # literal strip\n line 2 6 leading spaces"},
{"type":"TextWhitespace","value":" \n\n "},
{"type":"TextWhitespace","value":"\n\n "},
{"type":"NameTag","value":"d"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"\u003e-"},
{"type":"LiteralStringDoc","value":"\n multiline folded strip\n line 2: test\n # not a comment"},
{"type":"TextWhitespace","value":" \n "},
{"type":"TextWhitespace","value":"\n "},
{"type":"NameTag","value":"e"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|+"},
{"type":"LiteralStringDoc","value":"\n multiline literal keep\n line: 2"},
{"type":"TextWhitespace","value":" \n "},
{"type":"TextWhitespace","value":"\n "},
{"type":"Comment","value":"# this is a comment"},
{"type":"TextWhitespace","value":"\n "},
{"type":"NameTag","value":"f"},
@ -299,13 +299,13 @@
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"\u003e+"},
{"type":"LiteralStringDoc","value":"\n multiline folded keep\n line 2"},
{"type":"TextWhitespace","value":" \n "},
{"type":"TextWhitespace","value":"\n "},
{"type":"NameTag","value":"g"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|"},
{"type":"LiteralStringDoc","value":"\n multiline literal with only one line"},
{"type":"TextWhitespace","value":" \n "},
{"type":"TextWhitespace","value":"\n "},
{"type":"NameTag","value":"h"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
@ -355,7 +355,7 @@
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|"},
{"type":"LiteralStringDoc","value":"\n This is a key\n that has multiple lines"},
{"type":"TextWhitespace","value":" \n"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Literal","value":"and this is its value"},
@ -622,7 +622,7 @@
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|"},
{"type":"LiteralStringDoc","value":"\n R0lGODlhDAAMAIQAAP//9/X17unp5WZmZgAAAOfn515eXvPz7Y6OjuDg4J+fn5\n OTk6enp56enmlpaWNjY6Ojo4SEhP/++f/++f/++f/++f/++f/++f/++f/++f/+\n +f/++f/++f/++f/++f/++SH+Dk1hZGUgd2l0aCBHSU1QACwAAAAADAAMAAAFLC\n AgjoEwnuNAFOhpEMTRiggcz4BNJHrv/zCFcLiwMWYNG84BwwEeECcgggoBADs="},
{"type":"TextWhitespace","value":" \n\n"},
{"type":"TextWhitespace","value":"\n\n"},
{"type":"Comment","value":"# YAML also has a set type, which looks like this:"},
{"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"set"},

View File

@ -194,6 +194,9 @@ func (l *LexerState) Iterator() Token { // nolint: gocognit
for len(l.iteratorStack) > 0 {
n := len(l.iteratorStack) - 1
t := l.iteratorStack[n]()
if t.Type == Ignore {
continue
}
if t == EOF {
l.iteratorStack = l.iteratorStack[:n]
continue
@ -243,6 +246,9 @@ func (l *LexerState) Iterator() Token { // nolint: gocognit
for len(l.iteratorStack) > 0 {
n := len(l.iteratorStack) - 1
t := l.iteratorStack[n]()
if t.Type == Ignore {
continue
}
if t == EOF {
l.iteratorStack = l.iteratorStack[:n]
continue

View File

@ -192,3 +192,14 @@ func TestByGroupNames(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, []Token{{Error, `abc=123`}}, it.Tokens())
}
func TestIgnoreToken(t *testing.T) {
l := Coalesce(mustNewLexer(t, &Config{EnsureNL: true}, Rules{ // nolint: forbidigo
"root": {
{`(\s*)(\w+)(?:\1)(\n)`, ByGroups(Ignore, Keyword, Whitespace), nil},
},
}))
it, err := l.Tokenise(nil, ` hello `)
assert.NoError(t, err)
assert.Equal(t, []Token{{Keyword, "hello"}, {TextWhitespace, "\n"}}, it.Tokens())
}

View File

@ -7,110 +7,111 @@ import (
"strings"
)
const _TokenTypeName = "NoneOtherErrorCodeLineLineLinkLineTableTDLineTableLineHighlightLineNumbersTableLineNumbersLinePreWrapperBackgroundEOFTypeKeywordKeywordConstantKeywordDeclarationKeywordNamespaceKeywordPseudoKeywordReservedKeywordTypeNameNameAttributeNameBuiltinNameBuiltinPseudoNameClassNameConstantNameDecoratorNameEntityNameExceptionNameFunctionNameFunctionMagicNameKeywordNameLabelNameNamespaceNameOperatorNameOtherNamePseudoNamePropertyNameTagNameVariableNameVariableAnonymousNameVariableClassNameVariableGlobalNameVariableInstanceNameVariableMagicLiteralLiteralDateLiteralOtherLiteralStringLiteralStringAffixLiteralStringAtomLiteralStringBacktickLiteralStringBooleanLiteralStringCharLiteralStringDelimiterLiteralStringDocLiteralStringDoubleLiteralStringEscapeLiteralStringHeredocLiteralStringInterpolLiteralStringNameLiteralStringOtherLiteralStringRegexLiteralStringSingleLiteralStringSymbolLiteralNumberLiteralNumberBinLiteralNumberFloatLiteralNumberHexLiteralNumberIntegerLiteralNumberIntegerLongLiteralNumberOctLiteralNumberByteOperatorOperatorWordPunctuationCommentCommentHashbangCommentMultilineCommentSingleCommentSpecialCommentPreprocCommentPreprocFileGenericGenericDeletedGenericEmphGenericErrorGenericHeadingGenericInsertedGenericOutputGenericPromptGenericStrongGenericSubheadingGenericTracebackGenericUnderlineTextTextWhitespaceTextSymbolTextPunctuation"
const _TokenTypeLowerName = "noneothererrorcodelinelinelinklinetabletdlinetablelinehighlightlinenumberstablelinenumberslineprewrapperbackgroundeoftypekeywordkeywordconstantkeyworddeclarationkeywordnamespacekeywordpseudokeywordreservedkeywordtypenamenameattributenamebuiltinnamebuiltinpseudonameclassnameconstantnamedecoratornameentitynameexceptionnamefunctionnamefunctionmagicnamekeywordnamelabelnamenamespacenameoperatornameothernamepseudonamepropertynametagnamevariablenamevariableanonymousnamevariableclassnamevariableglobalnamevariableinstancenamevariablemagicliteralliteraldateliteralotherliteralstringliteralstringaffixliteralstringatomliteralstringbacktickliteralstringbooleanliteralstringcharliteralstringdelimiterliteralstringdocliteralstringdoubleliteralstringescapeliteralstringheredocliteralstringinterpolliteralstringnameliteralstringotherliteralstringregexliteralstringsingleliteralstringsymbolliteralnumberliteralnumberbinliteralnumberfloatliteralnumberhexliteralnumberintegerliteralnumberintegerlongliteralnumberoctliteralnumberbyteoperatoroperatorwordpunctuationcommentcommenthashbangcommentmultilinecommentsinglecommentspecialcommentpreproccommentpreprocfilegenericgenericdeletedgenericemphgenericerrorgenericheadinggenericinsertedgenericoutputgenericpromptgenericstronggenericsubheadinggenerictracebackgenericunderlinetexttextwhitespacetextsymboltextpunctuation"
const _TokenTypeName = "IgnoreNoneOtherErrorCodeLineLineLinkLineTableTDLineTableLineHighlightLineNumbersTableLineNumbersLinePreWrapperBackgroundEOFTypeKeywordKeywordConstantKeywordDeclarationKeywordNamespaceKeywordPseudoKeywordReservedKeywordTypeNameNameAttributeNameBuiltinNameBuiltinPseudoNameClassNameConstantNameDecoratorNameEntityNameExceptionNameFunctionNameFunctionMagicNameKeywordNameLabelNameNamespaceNameOperatorNameOtherNamePseudoNamePropertyNameTagNameVariableNameVariableAnonymousNameVariableClassNameVariableGlobalNameVariableInstanceNameVariableMagicLiteralLiteralDateLiteralOtherLiteralStringLiteralStringAffixLiteralStringAtomLiteralStringBacktickLiteralStringBooleanLiteralStringCharLiteralStringDelimiterLiteralStringDocLiteralStringDoubleLiteralStringEscapeLiteralStringHeredocLiteralStringInterpolLiteralStringNameLiteralStringOtherLiteralStringRegexLiteralStringSingleLiteralStringSymbolLiteralNumberLiteralNumberBinLiteralNumberFloatLiteralNumberHexLiteralNumberIntegerLiteralNumberIntegerLongLiteralNumberOctLiteralNumberByteOperatorOperatorWordPunctuationCommentCommentHashbangCommentMultilineCommentSingleCommentSpecialCommentPreprocCommentPreprocFileGenericGenericDeletedGenericEmphGenericErrorGenericHeadingGenericInsertedGenericOutputGenericPromptGenericStrongGenericSubheadingGenericTracebackGenericUnderlineTextTextWhitespaceTextSymbolTextPunctuation"
const _TokenTypeLowerName = "ignorenoneothererrorcodelinelinelinklinetabletdlinetablelinehighlightlinenumberstablelinenumberslineprewrapperbackgroundeoftypekeywordkeywordconstantkeyworddeclarationkeywordnamespacekeywordpseudokeywordreservedkeywordtypenamenameattributenamebuiltinnamebuiltinpseudonameclassnameconstantnamedecoratornameentitynameexceptionnamefunctionnamefunctionmagicnamekeywordnamelabelnamenamespacenameoperatornameothernamepseudonamepropertynametagnamevariablenamevariableanonymousnamevariableclassnamevariableglobalnamevariableinstancenamevariablemagicliteralliteraldateliteralotherliteralstringliteralstringaffixliteralstringatomliteralstringbacktickliteralstringbooleanliteralstringcharliteralstringdelimiterliteralstringdocliteralstringdoubleliteralstringescapeliteralstringheredocliteralstringinterpolliteralstringnameliteralstringotherliteralstringregexliteralstringsingleliteralstringsymbolliteralnumberliteralnumberbinliteralnumberfloatliteralnumberhexliteralnumberintegerliteralnumberintegerlongliteralnumberoctliteralnumberbyteoperatoroperatorwordpunctuationcommentcommenthashbangcommentmultilinecommentsinglecommentspecialcommentpreproccommentpreprocfilegenericgenericdeletedgenericemphgenericerrorgenericheadinggenericinsertedgenericoutputgenericpromptgenericstronggenericsubheadinggenerictracebackgenericunderlinetexttextwhitespacetextsymboltextpunctuation"
var _TokenTypeMap = map[TokenType]string{
-13: _TokenTypeName[0:4],
-12: _TokenTypeName[4:9],
-11: _TokenTypeName[9:14],
-10: _TokenTypeName[14:22],
-9: _TokenTypeName[22:30],
-8: _TokenTypeName[30:41],
-7: _TokenTypeName[41:50],
-6: _TokenTypeName[50:63],
-5: _TokenTypeName[63:79],
-4: _TokenTypeName[79:90],
-3: _TokenTypeName[90:94],
-2: _TokenTypeName[94:104],
-1: _TokenTypeName[104:114],
0: _TokenTypeName[114:121],
1000: _TokenTypeName[121:128],
1001: _TokenTypeName[128:143],
1002: _TokenTypeName[143:161],
1003: _TokenTypeName[161:177],
1004: _TokenTypeName[177:190],
1005: _TokenTypeName[190:205],
1006: _TokenTypeName[205:216],
2000: _TokenTypeName[216:220],
2001: _TokenTypeName[220:233],
2002: _TokenTypeName[233:244],
2003: _TokenTypeName[244:261],
2004: _TokenTypeName[261:270],
2005: _TokenTypeName[270:282],
2006: _TokenTypeName[282:295],
2007: _TokenTypeName[295:305],
2008: _TokenTypeName[305:318],
2009: _TokenTypeName[318:330],
2010: _TokenTypeName[330:347],
2011: _TokenTypeName[347:358],
2012: _TokenTypeName[358:367],
2013: _TokenTypeName[367:380],
2014: _TokenTypeName[380:392],
2015: _TokenTypeName[392:401],
2016: _TokenTypeName[401:411],
2017: _TokenTypeName[411:423],
2018: _TokenTypeName[423:430],
2019: _TokenTypeName[430:442],
2020: _TokenTypeName[442:463],
2021: _TokenTypeName[463:480],
2022: _TokenTypeName[480:498],
2023: _TokenTypeName[498:518],
2024: _TokenTypeName[518:535],
3000: _TokenTypeName[535:542],
3001: _TokenTypeName[542:553],
3002: _TokenTypeName[553:565],
3100: _TokenTypeName[565:578],
3101: _TokenTypeName[578:596],
3102: _TokenTypeName[596:613],
3103: _TokenTypeName[613:634],
3104: _TokenTypeName[634:654],
3105: _TokenTypeName[654:671],
3106: _TokenTypeName[671:693],
3107: _TokenTypeName[693:709],
3108: _TokenTypeName[709:728],
3109: _TokenTypeName[728:747],
3110: _TokenTypeName[747:767],
3111: _TokenTypeName[767:788],
3112: _TokenTypeName[788:805],
3113: _TokenTypeName[805:823],
3114: _TokenTypeName[823:841],
3115: _TokenTypeName[841:860],
3116: _TokenTypeName[860:879],
3200: _TokenTypeName[879:892],
3201: _TokenTypeName[892:908],
3202: _TokenTypeName[908:926],
3203: _TokenTypeName[926:942],
3204: _TokenTypeName[942:962],
3205: _TokenTypeName[962:986],
3206: _TokenTypeName[986:1002],
3207: _TokenTypeName[1002:1019],
4000: _TokenTypeName[1019:1027],
4001: _TokenTypeName[1027:1039],
5000: _TokenTypeName[1039:1050],
6000: _TokenTypeName[1050:1057],
6001: _TokenTypeName[1057:1072],
6002: _TokenTypeName[1072:1088],
6003: _TokenTypeName[1088:1101],
6004: _TokenTypeName[1101:1115],
6100: _TokenTypeName[1115:1129],
6101: _TokenTypeName[1129:1147],
7000: _TokenTypeName[1147:1154],
7001: _TokenTypeName[1154:1168],
7002: _TokenTypeName[1168:1179],
7003: _TokenTypeName[1179:1191],
7004: _TokenTypeName[1191:1205],
7005: _TokenTypeName[1205:1220],
7006: _TokenTypeName[1220:1233],
7007: _TokenTypeName[1233:1246],
7008: _TokenTypeName[1246:1259],
7009: _TokenTypeName[1259:1276],
7010: _TokenTypeName[1276:1292],
7011: _TokenTypeName[1292:1308],
8000: _TokenTypeName[1308:1312],
8001: _TokenTypeName[1312:1326],
8002: _TokenTypeName[1326:1336],
8003: _TokenTypeName[1336:1351],
-14: _TokenTypeName[0:6],
-13: _TokenTypeName[6:10],
-12: _TokenTypeName[10:15],
-11: _TokenTypeName[15:20],
-10: _TokenTypeName[20:28],
-9: _TokenTypeName[28:36],
-8: _TokenTypeName[36:47],
-7: _TokenTypeName[47:56],
-6: _TokenTypeName[56:69],
-5: _TokenTypeName[69:85],
-4: _TokenTypeName[85:96],
-3: _TokenTypeName[96:100],
-2: _TokenTypeName[100:110],
-1: _TokenTypeName[110:120],
0: _TokenTypeName[120:127],
1000: _TokenTypeName[127:134],
1001: _TokenTypeName[134:149],
1002: _TokenTypeName[149:167],
1003: _TokenTypeName[167:183],
1004: _TokenTypeName[183:196],
1005: _TokenTypeName[196:211],
1006: _TokenTypeName[211:222],
2000: _TokenTypeName[222:226],
2001: _TokenTypeName[226:239],
2002: _TokenTypeName[239:250],
2003: _TokenTypeName[250:267],
2004: _TokenTypeName[267:276],
2005: _TokenTypeName[276:288],
2006: _TokenTypeName[288:301],
2007: _TokenTypeName[301:311],
2008: _TokenTypeName[311:324],
2009: _TokenTypeName[324:336],
2010: _TokenTypeName[336:353],
2011: _TokenTypeName[353:364],
2012: _TokenTypeName[364:373],
2013: _TokenTypeName[373:386],
2014: _TokenTypeName[386:398],
2015: _TokenTypeName[398:407],
2016: _TokenTypeName[407:417],
2017: _TokenTypeName[417:429],
2018: _TokenTypeName[429:436],
2019: _TokenTypeName[436:448],
2020: _TokenTypeName[448:469],
2021: _TokenTypeName[469:486],
2022: _TokenTypeName[486:504],
2023: _TokenTypeName[504:524],
2024: _TokenTypeName[524:541],
3000: _TokenTypeName[541:548],
3001: _TokenTypeName[548:559],
3002: _TokenTypeName[559:571],
3100: _TokenTypeName[571:584],
3101: _TokenTypeName[584:602],
3102: _TokenTypeName[602:619],
3103: _TokenTypeName[619:640],
3104: _TokenTypeName[640:660],
3105: _TokenTypeName[660:677],
3106: _TokenTypeName[677:699],
3107: _TokenTypeName[699:715],
3108: _TokenTypeName[715:734],
3109: _TokenTypeName[734:753],
3110: _TokenTypeName[753:773],
3111: _TokenTypeName[773:794],
3112: _TokenTypeName[794:811],
3113: _TokenTypeName[811:829],
3114: _TokenTypeName[829:847],
3115: _TokenTypeName[847:866],
3116: _TokenTypeName[866:885],
3200: _TokenTypeName[885:898],
3201: _TokenTypeName[898:914],
3202: _TokenTypeName[914:932],
3203: _TokenTypeName[932:948],
3204: _TokenTypeName[948:968],
3205: _TokenTypeName[968:992],
3206: _TokenTypeName[992:1008],
3207: _TokenTypeName[1008:1025],
4000: _TokenTypeName[1025:1033],
4001: _TokenTypeName[1033:1045],
5000: _TokenTypeName[1045:1056],
6000: _TokenTypeName[1056:1063],
6001: _TokenTypeName[1063:1078],
6002: _TokenTypeName[1078:1094],
6003: _TokenTypeName[1094:1107],
6004: _TokenTypeName[1107:1121],
6100: _TokenTypeName[1121:1135],
6101: _TokenTypeName[1135:1153],
7000: _TokenTypeName[1153:1160],
7001: _TokenTypeName[1160:1174],
7002: _TokenTypeName[1174:1185],
7003: _TokenTypeName[1185:1197],
7004: _TokenTypeName[1197:1211],
7005: _TokenTypeName[1211:1226],
7006: _TokenTypeName[1226:1239],
7007: _TokenTypeName[1239:1252],
7008: _TokenTypeName[1252:1265],
7009: _TokenTypeName[1265:1282],
7010: _TokenTypeName[1282:1298],
7011: _TokenTypeName[1298:1314],
8000: _TokenTypeName[1314:1318],
8001: _TokenTypeName[1318:1332],
8002: _TokenTypeName[1332:1342],
8003: _TokenTypeName[1342:1357],
}
func (i TokenType) String() string {
@ -124,6 +125,7 @@ func (i TokenType) String() string {
// Re-run the stringer command to generate them again.
func _TokenTypeNoOp() {
var x [1]struct{}
_ = x[Ignore-(-14)]
_ = x[None-(-13)]
_ = x[Other-(-12)]
_ = x[Error-(-11)]
@ -226,312 +228,315 @@ func _TokenTypeNoOp() {
_ = x[TextPunctuation-(8003)]
}
var _TokenTypeValues = []TokenType{None, Other, Error, CodeLine, LineLink, LineTableTD, LineTable, LineHighlight, LineNumbersTable, LineNumbers, Line, PreWrapper, Background, EOFType, Keyword, KeywordConstant, KeywordDeclaration, KeywordNamespace, KeywordPseudo, KeywordReserved, KeywordType, Name, NameAttribute, NameBuiltin, NameBuiltinPseudo, NameClass, NameConstant, NameDecorator, NameEntity, NameException, NameFunction, NameFunctionMagic, NameKeyword, NameLabel, NameNamespace, NameOperator, NameOther, NamePseudo, NameProperty, NameTag, NameVariable, NameVariableAnonymous, NameVariableClass, NameVariableGlobal, NameVariableInstance, NameVariableMagic, Literal, LiteralDate, LiteralOther, LiteralString, LiteralStringAffix, LiteralStringAtom, LiteralStringBacktick, LiteralStringBoolean, LiteralStringChar, LiteralStringDelimiter, LiteralStringDoc, LiteralStringDouble, LiteralStringEscape, LiteralStringHeredoc, LiteralStringInterpol, LiteralStringName, LiteralStringOther, LiteralStringRegex, LiteralStringSingle, LiteralStringSymbol, LiteralNumber, LiteralNumberBin, LiteralNumberFloat, LiteralNumberHex, LiteralNumberInteger, LiteralNumberIntegerLong, LiteralNumberOct, LiteralNumberByte, Operator, OperatorWord, Punctuation, Comment, CommentHashbang, CommentMultiline, CommentSingle, CommentSpecial, CommentPreproc, CommentPreprocFile, Generic, GenericDeleted, GenericEmph, GenericError, GenericHeading, GenericInserted, GenericOutput, GenericPrompt, GenericStrong, GenericSubheading, GenericTraceback, GenericUnderline, Text, TextWhitespace, TextSymbol, TextPunctuation}
var _TokenTypeValues = []TokenType{Ignore, None, Other, Error, CodeLine, LineLink, LineTableTD, LineTable, LineHighlight, LineNumbersTable, LineNumbers, Line, PreWrapper, Background, EOFType, Keyword, KeywordConstant, KeywordDeclaration, KeywordNamespace, KeywordPseudo, KeywordReserved, KeywordType, Name, NameAttribute, NameBuiltin, NameBuiltinPseudo, NameClass, NameConstant, NameDecorator, NameEntity, NameException, NameFunction, NameFunctionMagic, NameKeyword, NameLabel, NameNamespace, NameOperator, NameOther, NamePseudo, NameProperty, NameTag, NameVariable, NameVariableAnonymous, NameVariableClass, NameVariableGlobal, NameVariableInstance, NameVariableMagic, Literal, LiteralDate, LiteralOther, LiteralString, LiteralStringAffix, LiteralStringAtom, LiteralStringBacktick, LiteralStringBoolean, LiteralStringChar, LiteralStringDelimiter, LiteralStringDoc, LiteralStringDouble, LiteralStringEscape, LiteralStringHeredoc, LiteralStringInterpol, LiteralStringName, LiteralStringOther, LiteralStringRegex, LiteralStringSingle, LiteralStringSymbol, LiteralNumber, LiteralNumberBin, LiteralNumberFloat, LiteralNumberHex, LiteralNumberInteger, LiteralNumberIntegerLong, LiteralNumberOct, LiteralNumberByte, Operator, OperatorWord, Punctuation, Comment, CommentHashbang, CommentMultiline, CommentSingle, CommentSpecial, CommentPreproc, CommentPreprocFile, Generic, GenericDeleted, GenericEmph, GenericError, GenericHeading, GenericInserted, GenericOutput, GenericPrompt, GenericStrong, GenericSubheading, GenericTraceback, GenericUnderline, Text, TextWhitespace, TextSymbol, TextPunctuation}
var _TokenTypeNameToValueMap = map[string]TokenType{
_TokenTypeName[0:4]: None,
_TokenTypeLowerName[0:4]: None,
_TokenTypeName[4:9]: Other,
_TokenTypeLowerName[4:9]: Other,
_TokenTypeName[9:14]: Error,
_TokenTypeLowerName[9:14]: Error,
_TokenTypeName[14:22]: CodeLine,
_TokenTypeLowerName[14:22]: CodeLine,
_TokenTypeName[22:30]: LineLink,
_TokenTypeLowerName[22:30]: LineLink,
_TokenTypeName[30:41]: LineTableTD,
_TokenTypeLowerName[30:41]: LineTableTD,
_TokenTypeName[41:50]: LineTable,
_TokenTypeLowerName[41:50]: LineTable,
_TokenTypeName[50:63]: LineHighlight,
_TokenTypeLowerName[50:63]: LineHighlight,
_TokenTypeName[63:79]: LineNumbersTable,
_TokenTypeLowerName[63:79]: LineNumbersTable,
_TokenTypeName[79:90]: LineNumbers,
_TokenTypeLowerName[79:90]: LineNumbers,
_TokenTypeName[90:94]: Line,
_TokenTypeLowerName[90:94]: Line,
_TokenTypeName[94:104]: PreWrapper,
_TokenTypeLowerName[94:104]: PreWrapper,
_TokenTypeName[104:114]: Background,
_TokenTypeLowerName[104:114]: Background,
_TokenTypeName[114:121]: EOFType,
_TokenTypeLowerName[114:121]: EOFType,
_TokenTypeName[121:128]: Keyword,
_TokenTypeLowerName[121:128]: Keyword,
_TokenTypeName[128:143]: KeywordConstant,
_TokenTypeLowerName[128:143]: KeywordConstant,
_TokenTypeName[143:161]: KeywordDeclaration,
_TokenTypeLowerName[143:161]: KeywordDeclaration,
_TokenTypeName[161:177]: KeywordNamespace,
_TokenTypeLowerName[161:177]: KeywordNamespace,
_TokenTypeName[177:190]: KeywordPseudo,
_TokenTypeLowerName[177:190]: KeywordPseudo,
_TokenTypeName[190:205]: KeywordReserved,
_TokenTypeLowerName[190:205]: KeywordReserved,
_TokenTypeName[205:216]: KeywordType,
_TokenTypeLowerName[205:216]: KeywordType,
_TokenTypeName[216:220]: Name,
_TokenTypeLowerName[216:220]: Name,
_TokenTypeName[220:233]: NameAttribute,
_TokenTypeLowerName[220:233]: NameAttribute,
_TokenTypeName[233:244]: NameBuiltin,
_TokenTypeLowerName[233:244]: NameBuiltin,
_TokenTypeName[244:261]: NameBuiltinPseudo,
_TokenTypeLowerName[244:261]: NameBuiltinPseudo,
_TokenTypeName[261:270]: NameClass,
_TokenTypeLowerName[261:270]: NameClass,
_TokenTypeName[270:282]: NameConstant,
_TokenTypeLowerName[270:282]: NameConstant,
_TokenTypeName[282:295]: NameDecorator,
_TokenTypeLowerName[282:295]: NameDecorator,
_TokenTypeName[295:305]: NameEntity,
_TokenTypeLowerName[295:305]: NameEntity,
_TokenTypeName[305:318]: NameException,
_TokenTypeLowerName[305:318]: NameException,
_TokenTypeName[318:330]: NameFunction,
_TokenTypeLowerName[318:330]: NameFunction,
_TokenTypeName[330:347]: NameFunctionMagic,
_TokenTypeLowerName[330:347]: NameFunctionMagic,
_TokenTypeName[347:358]: NameKeyword,
_TokenTypeLowerName[347:358]: NameKeyword,
_TokenTypeName[358:367]: NameLabel,
_TokenTypeLowerName[358:367]: NameLabel,
_TokenTypeName[367:380]: NameNamespace,
_TokenTypeLowerName[367:380]: NameNamespace,
_TokenTypeName[380:392]: NameOperator,
_TokenTypeLowerName[380:392]: NameOperator,
_TokenTypeName[392:401]: NameOther,
_TokenTypeLowerName[392:401]: NameOther,
_TokenTypeName[401:411]: NamePseudo,
_TokenTypeLowerName[401:411]: NamePseudo,
_TokenTypeName[411:423]: NameProperty,
_TokenTypeLowerName[411:423]: NameProperty,
_TokenTypeName[423:430]: NameTag,
_TokenTypeLowerName[423:430]: NameTag,
_TokenTypeName[430:442]: NameVariable,
_TokenTypeLowerName[430:442]: NameVariable,
_TokenTypeName[442:463]: NameVariableAnonymous,
_TokenTypeLowerName[442:463]: NameVariableAnonymous,
_TokenTypeName[463:480]: NameVariableClass,
_TokenTypeLowerName[463:480]: NameVariableClass,
_TokenTypeName[480:498]: NameVariableGlobal,
_TokenTypeLowerName[480:498]: NameVariableGlobal,
_TokenTypeName[498:518]: NameVariableInstance,
_TokenTypeLowerName[498:518]: NameVariableInstance,
_TokenTypeName[518:535]: NameVariableMagic,
_TokenTypeLowerName[518:535]: NameVariableMagic,
_TokenTypeName[535:542]: Literal,
_TokenTypeLowerName[535:542]: Literal,
_TokenTypeName[542:553]: LiteralDate,
_TokenTypeLowerName[542:553]: LiteralDate,
_TokenTypeName[553:565]: LiteralOther,
_TokenTypeLowerName[553:565]: LiteralOther,
_TokenTypeName[565:578]: LiteralString,
_TokenTypeLowerName[565:578]: LiteralString,
_TokenTypeName[578:596]: LiteralStringAffix,
_TokenTypeLowerName[578:596]: LiteralStringAffix,
_TokenTypeName[596:613]: LiteralStringAtom,
_TokenTypeLowerName[596:613]: LiteralStringAtom,
_TokenTypeName[613:634]: LiteralStringBacktick,
_TokenTypeLowerName[613:634]: LiteralStringBacktick,
_TokenTypeName[634:654]: LiteralStringBoolean,
_TokenTypeLowerName[634:654]: LiteralStringBoolean,
_TokenTypeName[654:671]: LiteralStringChar,
_TokenTypeLowerName[654:671]: LiteralStringChar,
_TokenTypeName[671:693]: LiteralStringDelimiter,
_TokenTypeLowerName[671:693]: LiteralStringDelimiter,
_TokenTypeName[693:709]: LiteralStringDoc,
_TokenTypeLowerName[693:709]: LiteralStringDoc,
_TokenTypeName[709:728]: LiteralStringDouble,
_TokenTypeLowerName[709:728]: LiteralStringDouble,
_TokenTypeName[728:747]: LiteralStringEscape,
_TokenTypeLowerName[728:747]: LiteralStringEscape,
_TokenTypeName[747:767]: LiteralStringHeredoc,
_TokenTypeLowerName[747:767]: LiteralStringHeredoc,
_TokenTypeName[767:788]: LiteralStringInterpol,
_TokenTypeLowerName[767:788]: LiteralStringInterpol,
_TokenTypeName[788:805]: LiteralStringName,
_TokenTypeLowerName[788:805]: LiteralStringName,
_TokenTypeName[805:823]: LiteralStringOther,
_TokenTypeLowerName[805:823]: LiteralStringOther,
_TokenTypeName[823:841]: LiteralStringRegex,
_TokenTypeLowerName[823:841]: LiteralStringRegex,
_TokenTypeName[841:860]: LiteralStringSingle,
_TokenTypeLowerName[841:860]: LiteralStringSingle,
_TokenTypeName[860:879]: LiteralStringSymbol,
_TokenTypeLowerName[860:879]: LiteralStringSymbol,
_TokenTypeName[879:892]: LiteralNumber,
_TokenTypeLowerName[879:892]: LiteralNumber,
_TokenTypeName[892:908]: LiteralNumberBin,
_TokenTypeLowerName[892:908]: LiteralNumberBin,
_TokenTypeName[908:926]: LiteralNumberFloat,
_TokenTypeLowerName[908:926]: LiteralNumberFloat,
_TokenTypeName[926:942]: LiteralNumberHex,
_TokenTypeLowerName[926:942]: LiteralNumberHex,
_TokenTypeName[942:962]: LiteralNumberInteger,
_TokenTypeLowerName[942:962]: LiteralNumberInteger,
_TokenTypeName[962:986]: LiteralNumberIntegerLong,
_TokenTypeLowerName[962:986]: LiteralNumberIntegerLong,
_TokenTypeName[986:1002]: LiteralNumberOct,
_TokenTypeLowerName[986:1002]: LiteralNumberOct,
_TokenTypeName[1002:1019]: LiteralNumberByte,
_TokenTypeLowerName[1002:1019]: LiteralNumberByte,
_TokenTypeName[1019:1027]: Operator,
_TokenTypeLowerName[1019:1027]: Operator,
_TokenTypeName[1027:1039]: OperatorWord,
_TokenTypeLowerName[1027:1039]: OperatorWord,
_TokenTypeName[1039:1050]: Punctuation,
_TokenTypeLowerName[1039:1050]: Punctuation,
_TokenTypeName[1050:1057]: Comment,
_TokenTypeLowerName[1050:1057]: Comment,
_TokenTypeName[1057:1072]: CommentHashbang,
_TokenTypeLowerName[1057:1072]: CommentHashbang,
_TokenTypeName[1072:1088]: CommentMultiline,
_TokenTypeLowerName[1072:1088]: CommentMultiline,
_TokenTypeName[1088:1101]: CommentSingle,
_TokenTypeLowerName[1088:1101]: CommentSingle,
_TokenTypeName[1101:1115]: CommentSpecial,
_TokenTypeLowerName[1101:1115]: CommentSpecial,
_TokenTypeName[1115:1129]: CommentPreproc,
_TokenTypeLowerName[1115:1129]: CommentPreproc,
_TokenTypeName[1129:1147]: CommentPreprocFile,
_TokenTypeLowerName[1129:1147]: CommentPreprocFile,
_TokenTypeName[1147:1154]: Generic,
_TokenTypeLowerName[1147:1154]: Generic,
_TokenTypeName[1154:1168]: GenericDeleted,
_TokenTypeLowerName[1154:1168]: GenericDeleted,
_TokenTypeName[1168:1179]: GenericEmph,
_TokenTypeLowerName[1168:1179]: GenericEmph,
_TokenTypeName[1179:1191]: GenericError,
_TokenTypeLowerName[1179:1191]: GenericError,
_TokenTypeName[1191:1205]: GenericHeading,
_TokenTypeLowerName[1191:1205]: GenericHeading,
_TokenTypeName[1205:1220]: GenericInserted,
_TokenTypeLowerName[1205:1220]: GenericInserted,
_TokenTypeName[1220:1233]: GenericOutput,
_TokenTypeLowerName[1220:1233]: GenericOutput,
_TokenTypeName[1233:1246]: GenericPrompt,
_TokenTypeLowerName[1233:1246]: GenericPrompt,
_TokenTypeName[1246:1259]: GenericStrong,
_TokenTypeLowerName[1246:1259]: GenericStrong,
_TokenTypeName[1259:1276]: GenericSubheading,
_TokenTypeLowerName[1259:1276]: GenericSubheading,
_TokenTypeName[1276:1292]: GenericTraceback,
_TokenTypeLowerName[1276:1292]: GenericTraceback,
_TokenTypeName[1292:1308]: GenericUnderline,
_TokenTypeLowerName[1292:1308]: GenericUnderline,
_TokenTypeName[1308:1312]: Text,
_TokenTypeLowerName[1308:1312]: Text,
_TokenTypeName[1312:1326]: TextWhitespace,
_TokenTypeLowerName[1312:1326]: TextWhitespace,
_TokenTypeName[1326:1336]: TextSymbol,
_TokenTypeLowerName[1326:1336]: TextSymbol,
_TokenTypeName[1336:1351]: TextPunctuation,
_TokenTypeLowerName[1336:1351]: TextPunctuation,
_TokenTypeName[0:6]: Ignore,
_TokenTypeLowerName[0:6]: Ignore,
_TokenTypeName[6:10]: None,
_TokenTypeLowerName[6:10]: None,
_TokenTypeName[10:15]: Other,
_TokenTypeLowerName[10:15]: Other,
_TokenTypeName[15:20]: Error,
_TokenTypeLowerName[15:20]: Error,
_TokenTypeName[20:28]: CodeLine,
_TokenTypeLowerName[20:28]: CodeLine,
_TokenTypeName[28:36]: LineLink,
_TokenTypeLowerName[28:36]: LineLink,
_TokenTypeName[36:47]: LineTableTD,
_TokenTypeLowerName[36:47]: LineTableTD,
_TokenTypeName[47:56]: LineTable,
_TokenTypeLowerName[47:56]: LineTable,
_TokenTypeName[56:69]: LineHighlight,
_TokenTypeLowerName[56:69]: LineHighlight,
_TokenTypeName[69:85]: LineNumbersTable,
_TokenTypeLowerName[69:85]: LineNumbersTable,
_TokenTypeName[85:96]: LineNumbers,
_TokenTypeLowerName[85:96]: LineNumbers,
_TokenTypeName[96:100]: Line,
_TokenTypeLowerName[96:100]: Line,
_TokenTypeName[100:110]: PreWrapper,
_TokenTypeLowerName[100:110]: PreWrapper,
_TokenTypeName[110:120]: Background,
_TokenTypeLowerName[110:120]: Background,
_TokenTypeName[120:127]: EOFType,
_TokenTypeLowerName[120:127]: EOFType,
_TokenTypeName[127:134]: Keyword,
_TokenTypeLowerName[127:134]: Keyword,
_TokenTypeName[134:149]: KeywordConstant,
_TokenTypeLowerName[134:149]: KeywordConstant,
_TokenTypeName[149:167]: KeywordDeclaration,
_TokenTypeLowerName[149:167]: KeywordDeclaration,
_TokenTypeName[167:183]: KeywordNamespace,
_TokenTypeLowerName[167:183]: KeywordNamespace,
_TokenTypeName[183:196]: KeywordPseudo,
_TokenTypeLowerName[183:196]: KeywordPseudo,
_TokenTypeName[196:211]: KeywordReserved,
_TokenTypeLowerName[196:211]: KeywordReserved,
_TokenTypeName[211:222]: KeywordType,
_TokenTypeLowerName[211:222]: KeywordType,
_TokenTypeName[222:226]: Name,
_TokenTypeLowerName[222:226]: Name,
_TokenTypeName[226:239]: NameAttribute,
_TokenTypeLowerName[226:239]: NameAttribute,
_TokenTypeName[239:250]: NameBuiltin,
_TokenTypeLowerName[239:250]: NameBuiltin,
_TokenTypeName[250:267]: NameBuiltinPseudo,
_TokenTypeLowerName[250:267]: NameBuiltinPseudo,
_TokenTypeName[267:276]: NameClass,
_TokenTypeLowerName[267:276]: NameClass,
_TokenTypeName[276:288]: NameConstant,
_TokenTypeLowerName[276:288]: NameConstant,
_TokenTypeName[288:301]: NameDecorator,
_TokenTypeLowerName[288:301]: NameDecorator,
_TokenTypeName[301:311]: NameEntity,
_TokenTypeLowerName[301:311]: NameEntity,
_TokenTypeName[311:324]: NameException,
_TokenTypeLowerName[311:324]: NameException,
_TokenTypeName[324:336]: NameFunction,
_TokenTypeLowerName[324:336]: NameFunction,
_TokenTypeName[336:353]: NameFunctionMagic,
_TokenTypeLowerName[336:353]: NameFunctionMagic,
_TokenTypeName[353:364]: NameKeyword,
_TokenTypeLowerName[353:364]: NameKeyword,
_TokenTypeName[364:373]: NameLabel,
_TokenTypeLowerName[364:373]: NameLabel,
_TokenTypeName[373:386]: NameNamespace,
_TokenTypeLowerName[373:386]: NameNamespace,
_TokenTypeName[386:398]: NameOperator,
_TokenTypeLowerName[386:398]: NameOperator,
_TokenTypeName[398:407]: NameOther,
_TokenTypeLowerName[398:407]: NameOther,
_TokenTypeName[407:417]: NamePseudo,
_TokenTypeLowerName[407:417]: NamePseudo,
_TokenTypeName[417:429]: NameProperty,
_TokenTypeLowerName[417:429]: NameProperty,
_TokenTypeName[429:436]: NameTag,
_TokenTypeLowerName[429:436]: NameTag,
_TokenTypeName[436:448]: NameVariable,
_TokenTypeLowerName[436:448]: NameVariable,
_TokenTypeName[448:469]: NameVariableAnonymous,
_TokenTypeLowerName[448:469]: NameVariableAnonymous,
_TokenTypeName[469:486]: NameVariableClass,
_TokenTypeLowerName[469:486]: NameVariableClass,
_TokenTypeName[486:504]: NameVariableGlobal,
_TokenTypeLowerName[486:504]: NameVariableGlobal,
_TokenTypeName[504:524]: NameVariableInstance,
_TokenTypeLowerName[504:524]: NameVariableInstance,
_TokenTypeName[524:541]: NameVariableMagic,
_TokenTypeLowerName[524:541]: NameVariableMagic,
_TokenTypeName[541:548]: Literal,
_TokenTypeLowerName[541:548]: Literal,
_TokenTypeName[548:559]: LiteralDate,
_TokenTypeLowerName[548:559]: LiteralDate,
_TokenTypeName[559:571]: LiteralOther,
_TokenTypeLowerName[559:571]: LiteralOther,
_TokenTypeName[571:584]: LiteralString,
_TokenTypeLowerName[571:584]: LiteralString,
_TokenTypeName[584:602]: LiteralStringAffix,
_TokenTypeLowerName[584:602]: LiteralStringAffix,
_TokenTypeName[602:619]: LiteralStringAtom,
_TokenTypeLowerName[602:619]: LiteralStringAtom,
_TokenTypeName[619:640]: LiteralStringBacktick,
_TokenTypeLowerName[619:640]: LiteralStringBacktick,
_TokenTypeName[640:660]: LiteralStringBoolean,
_TokenTypeLowerName[640:660]: LiteralStringBoolean,
_TokenTypeName[660:677]: LiteralStringChar,
_TokenTypeLowerName[660:677]: LiteralStringChar,
_TokenTypeName[677:699]: LiteralStringDelimiter,
_TokenTypeLowerName[677:699]: LiteralStringDelimiter,
_TokenTypeName[699:715]: LiteralStringDoc,
_TokenTypeLowerName[699:715]: LiteralStringDoc,
_TokenTypeName[715:734]: LiteralStringDouble,
_TokenTypeLowerName[715:734]: LiteralStringDouble,
_TokenTypeName[734:753]: LiteralStringEscape,
_TokenTypeLowerName[734:753]: LiteralStringEscape,
_TokenTypeName[753:773]: LiteralStringHeredoc,
_TokenTypeLowerName[753:773]: LiteralStringHeredoc,
_TokenTypeName[773:794]: LiteralStringInterpol,
_TokenTypeLowerName[773:794]: LiteralStringInterpol,
_TokenTypeName[794:811]: LiteralStringName,
_TokenTypeLowerName[794:811]: LiteralStringName,
_TokenTypeName[811:829]: LiteralStringOther,
_TokenTypeLowerName[811:829]: LiteralStringOther,
_TokenTypeName[829:847]: LiteralStringRegex,
_TokenTypeLowerName[829:847]: LiteralStringRegex,
_TokenTypeName[847:866]: LiteralStringSingle,
_TokenTypeLowerName[847:866]: LiteralStringSingle,
_TokenTypeName[866:885]: LiteralStringSymbol,
_TokenTypeLowerName[866:885]: LiteralStringSymbol,
_TokenTypeName[885:898]: LiteralNumber,
_TokenTypeLowerName[885:898]: LiteralNumber,
_TokenTypeName[898:914]: LiteralNumberBin,
_TokenTypeLowerName[898:914]: LiteralNumberBin,
_TokenTypeName[914:932]: LiteralNumberFloat,
_TokenTypeLowerName[914:932]: LiteralNumberFloat,
_TokenTypeName[932:948]: LiteralNumberHex,
_TokenTypeLowerName[932:948]: LiteralNumberHex,
_TokenTypeName[948:968]: LiteralNumberInteger,
_TokenTypeLowerName[948:968]: LiteralNumberInteger,
_TokenTypeName[968:992]: LiteralNumberIntegerLong,
_TokenTypeLowerName[968:992]: LiteralNumberIntegerLong,
_TokenTypeName[992:1008]: LiteralNumberOct,
_TokenTypeLowerName[992:1008]: LiteralNumberOct,
_TokenTypeName[1008:1025]: LiteralNumberByte,
_TokenTypeLowerName[1008:1025]: LiteralNumberByte,
_TokenTypeName[1025:1033]: Operator,
_TokenTypeLowerName[1025:1033]: Operator,
_TokenTypeName[1033:1045]: OperatorWord,
_TokenTypeLowerName[1033:1045]: OperatorWord,
_TokenTypeName[1045:1056]: Punctuation,
_TokenTypeLowerName[1045:1056]: Punctuation,
_TokenTypeName[1056:1063]: Comment,
_TokenTypeLowerName[1056:1063]: Comment,
_TokenTypeName[1063:1078]: CommentHashbang,
_TokenTypeLowerName[1063:1078]: CommentHashbang,
_TokenTypeName[1078:1094]: CommentMultiline,
_TokenTypeLowerName[1078:1094]: CommentMultiline,
_TokenTypeName[1094:1107]: CommentSingle,
_TokenTypeLowerName[1094:1107]: CommentSingle,
_TokenTypeName[1107:1121]: CommentSpecial,
_TokenTypeLowerName[1107:1121]: CommentSpecial,
_TokenTypeName[1121:1135]: CommentPreproc,
_TokenTypeLowerName[1121:1135]: CommentPreproc,
_TokenTypeName[1135:1153]: CommentPreprocFile,
_TokenTypeLowerName[1135:1153]: CommentPreprocFile,
_TokenTypeName[1153:1160]: Generic,
_TokenTypeLowerName[1153:1160]: Generic,
_TokenTypeName[1160:1174]: GenericDeleted,
_TokenTypeLowerName[1160:1174]: GenericDeleted,
_TokenTypeName[1174:1185]: GenericEmph,
_TokenTypeLowerName[1174:1185]: GenericEmph,
_TokenTypeName[1185:1197]: GenericError,
_TokenTypeLowerName[1185:1197]: GenericError,
_TokenTypeName[1197:1211]: GenericHeading,
_TokenTypeLowerName[1197:1211]: GenericHeading,
_TokenTypeName[1211:1226]: GenericInserted,
_TokenTypeLowerName[1211:1226]: GenericInserted,
_TokenTypeName[1226:1239]: GenericOutput,
_TokenTypeLowerName[1226:1239]: GenericOutput,
_TokenTypeName[1239:1252]: GenericPrompt,
_TokenTypeLowerName[1239:1252]: GenericPrompt,
_TokenTypeName[1252:1265]: GenericStrong,
_TokenTypeLowerName[1252:1265]: GenericStrong,
_TokenTypeName[1265:1282]: GenericSubheading,
_TokenTypeLowerName[1265:1282]: GenericSubheading,
_TokenTypeName[1282:1298]: GenericTraceback,
_TokenTypeLowerName[1282:1298]: GenericTraceback,
_TokenTypeName[1298:1314]: GenericUnderline,
_TokenTypeLowerName[1298:1314]: GenericUnderline,
_TokenTypeName[1314:1318]: Text,
_TokenTypeLowerName[1314:1318]: Text,
_TokenTypeName[1318:1332]: TextWhitespace,
_TokenTypeLowerName[1318:1332]: TextWhitespace,
_TokenTypeName[1332:1342]: TextSymbol,
_TokenTypeLowerName[1332:1342]: TextSymbol,
_TokenTypeName[1342:1357]: TextPunctuation,
_TokenTypeLowerName[1342:1357]: TextPunctuation,
}
var _TokenTypeNames = []string{
_TokenTypeName[0:4],
_TokenTypeName[4:9],
_TokenTypeName[9:14],
_TokenTypeName[14:22],
_TokenTypeName[22:30],
_TokenTypeName[30:41],
_TokenTypeName[41:50],
_TokenTypeName[50:63],
_TokenTypeName[63:79],
_TokenTypeName[79:90],
_TokenTypeName[90:94],
_TokenTypeName[94:104],
_TokenTypeName[104:114],
_TokenTypeName[114:121],
_TokenTypeName[121:128],
_TokenTypeName[128:143],
_TokenTypeName[143:161],
_TokenTypeName[161:177],
_TokenTypeName[177:190],
_TokenTypeName[190:205],
_TokenTypeName[205:216],
_TokenTypeName[216:220],
_TokenTypeName[220:233],
_TokenTypeName[233:244],
_TokenTypeName[244:261],
_TokenTypeName[261:270],
_TokenTypeName[270:282],
_TokenTypeName[282:295],
_TokenTypeName[295:305],
_TokenTypeName[305:318],
_TokenTypeName[318:330],
_TokenTypeName[330:347],
_TokenTypeName[347:358],
_TokenTypeName[358:367],
_TokenTypeName[367:380],
_TokenTypeName[380:392],
_TokenTypeName[392:401],
_TokenTypeName[401:411],
_TokenTypeName[411:423],
_TokenTypeName[423:430],
_TokenTypeName[430:442],
_TokenTypeName[442:463],
_TokenTypeName[463:480],
_TokenTypeName[480:498],
_TokenTypeName[498:518],
_TokenTypeName[518:535],
_TokenTypeName[535:542],
_TokenTypeName[542:553],
_TokenTypeName[553:565],
_TokenTypeName[565:578],
_TokenTypeName[578:596],
_TokenTypeName[596:613],
_TokenTypeName[613:634],
_TokenTypeName[634:654],
_TokenTypeName[654:671],
_TokenTypeName[671:693],
_TokenTypeName[693:709],
_TokenTypeName[709:728],
_TokenTypeName[728:747],
_TokenTypeName[747:767],
_TokenTypeName[767:788],
_TokenTypeName[788:805],
_TokenTypeName[805:823],
_TokenTypeName[823:841],
_TokenTypeName[841:860],
_TokenTypeName[860:879],
_TokenTypeName[879:892],
_TokenTypeName[892:908],
_TokenTypeName[908:926],
_TokenTypeName[926:942],
_TokenTypeName[942:962],
_TokenTypeName[962:986],
_TokenTypeName[986:1002],
_TokenTypeName[1002:1019],
_TokenTypeName[1019:1027],
_TokenTypeName[1027:1039],
_TokenTypeName[1039:1050],
_TokenTypeName[1050:1057],
_TokenTypeName[1057:1072],
_TokenTypeName[1072:1088],
_TokenTypeName[1088:1101],
_TokenTypeName[1101:1115],
_TokenTypeName[1115:1129],
_TokenTypeName[1129:1147],
_TokenTypeName[1147:1154],
_TokenTypeName[1154:1168],
_TokenTypeName[1168:1179],
_TokenTypeName[1179:1191],
_TokenTypeName[1191:1205],
_TokenTypeName[1205:1220],
_TokenTypeName[1220:1233],
_TokenTypeName[1233:1246],
_TokenTypeName[1246:1259],
_TokenTypeName[1259:1276],
_TokenTypeName[1276:1292],
_TokenTypeName[1292:1308],
_TokenTypeName[1308:1312],
_TokenTypeName[1312:1326],
_TokenTypeName[1326:1336],
_TokenTypeName[1336:1351],
_TokenTypeName[0:6],
_TokenTypeName[6:10],
_TokenTypeName[10:15],
_TokenTypeName[15:20],
_TokenTypeName[20:28],
_TokenTypeName[28:36],
_TokenTypeName[36:47],
_TokenTypeName[47:56],
_TokenTypeName[56:69],
_TokenTypeName[69:85],
_TokenTypeName[85:96],
_TokenTypeName[96:100],
_TokenTypeName[100:110],
_TokenTypeName[110:120],
_TokenTypeName[120:127],
_TokenTypeName[127:134],
_TokenTypeName[134:149],
_TokenTypeName[149:167],
_TokenTypeName[167:183],
_TokenTypeName[183:196],
_TokenTypeName[196:211],
_TokenTypeName[211:222],
_TokenTypeName[222:226],
_TokenTypeName[226:239],
_TokenTypeName[239:250],
_TokenTypeName[250:267],
_TokenTypeName[267:276],
_TokenTypeName[276:288],
_TokenTypeName[288:301],
_TokenTypeName[301:311],
_TokenTypeName[311:324],
_TokenTypeName[324:336],
_TokenTypeName[336:353],
_TokenTypeName[353:364],
_TokenTypeName[364:373],
_TokenTypeName[373:386],
_TokenTypeName[386:398],
_TokenTypeName[398:407],
_TokenTypeName[407:417],
_TokenTypeName[417:429],
_TokenTypeName[429:436],
_TokenTypeName[436:448],
_TokenTypeName[448:469],
_TokenTypeName[469:486],
_TokenTypeName[486:504],
_TokenTypeName[504:524],
_TokenTypeName[524:541],
_TokenTypeName[541:548],
_TokenTypeName[548:559],
_TokenTypeName[559:571],
_TokenTypeName[571:584],
_TokenTypeName[584:602],
_TokenTypeName[602:619],
_TokenTypeName[619:640],
_TokenTypeName[640:660],
_TokenTypeName[660:677],
_TokenTypeName[677:699],
_TokenTypeName[699:715],
_TokenTypeName[715:734],
_TokenTypeName[734:753],
_TokenTypeName[753:773],
_TokenTypeName[773:794],
_TokenTypeName[794:811],
_TokenTypeName[811:829],
_TokenTypeName[829:847],
_TokenTypeName[847:866],
_TokenTypeName[866:885],
_TokenTypeName[885:898],
_TokenTypeName[898:914],
_TokenTypeName[914:932],
_TokenTypeName[932:948],
_TokenTypeName[948:968],
_TokenTypeName[968:992],
_TokenTypeName[992:1008],
_TokenTypeName[1008:1025],
_TokenTypeName[1025:1033],
_TokenTypeName[1033:1045],
_TokenTypeName[1045:1056],
_TokenTypeName[1056:1063],
_TokenTypeName[1063:1078],
_TokenTypeName[1078:1094],
_TokenTypeName[1094:1107],
_TokenTypeName[1107:1121],
_TokenTypeName[1121:1135],
_TokenTypeName[1135:1153],
_TokenTypeName[1153:1160],
_TokenTypeName[1160:1174],
_TokenTypeName[1174:1185],
_TokenTypeName[1185:1197],
_TokenTypeName[1197:1211],
_TokenTypeName[1211:1226],
_TokenTypeName[1226:1239],
_TokenTypeName[1239:1252],
_TokenTypeName[1252:1265],
_TokenTypeName[1265:1282],
_TokenTypeName[1282:1298],
_TokenTypeName[1298:1314],
_TokenTypeName[1314:1318],
_TokenTypeName[1318:1332],
_TokenTypeName[1332:1342],
_TokenTypeName[1342:1357],
}
// TokenTypeString retrieves an enum value from the enum constants string name.

View File

@ -41,6 +41,8 @@ const (
Other
// No highlighting.
None
// Don't emit this token to the output.
Ignore
// Used as an EOF marker / nil token
EOFType TokenType = 0
)