go/chroma
1
0
Fork 0
mirror of https://github.com/alecthomas/chroma.git synced 2025-07-15 01:14:21 +02:00
Code Issues Releases Activity

Don't output extra whitespace in YAML multiline (#993)

Browse Source 
This resolves a particular issue with parsing YAML multiline, for
example:
```yaml
a: |
  multiline literal
  line 2
```

The regex used would capture the amount of indentation in the third
capture group and then use that as a kind of "status" to know which
lines are part of the indented multiline. However, because its a
captured group it has to be assigned a token which was `TextWhitespace`.
This meant that the indentation was outputted after the multiline,
technically it should be seen as an non-captured group, but then its no
longer to refer to it in the regex. Therefore I've gone with the
solution to add a new token, Ignore, which will not be emitted as a
token in the iterator, which can safely be used to make use of capture
groups but not have them show up in the output.

## Before

![image](https://github.com/user-attachments/assets/c29353c5-9e15-4f14-a733-57a60fb51910)

## After

![image](https://github.com/user-attachments/assets/57b5d129-a9d3-4b84-ae1f-dc05182b9ad3)
This commit is contained in:
Gusted
2024-08-22 22:58:31 +02:00
committed by GitHub
parent 895a0488b5
commit 4d11870090
6 changed files with 448 additions and 424 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lexers/embedded/yaml.xml
View File

@ -53,7 +53,7 @@
<bygroups> <bygroups>
<token type="Punctuation"/> <token type="Punctuation"/>
<token type="LiteralStringDoc"/> <token type="LiteralStringDoc"/>
<token type="TextWhitespace"/> <token type="Ignore"/>
</bygroups> </bygroups>
</rule> </rule>
<rule pattern="(false|False|FALSE|true|True|TRUE|null|Off|off|yes|Yes|YES|OFF|On|ON|no|No|on|NO|n|N|Y|y)\b"> <rule pattern="(false|False|FALSE|true|True|TRUE|null|Off|off|yes|Yes|YES|OFF|On|ON|no|No|on|NO|n|N|Y|y)\b">

40
lexers/testdata/yaml.expected vendored
View File

@ -191,55 +191,55 @@
{"type":"TextWhitespace","value":" "}, {"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|"}, {"type":"Punctuation","value":"|"},
{"type":"LiteralStringDoc","value":"\n This entire block of text will be the value of the 'literal_block' key,\n with line breaks being preserved.\n\n The literal continues until de-dented, and the leading indentation is\n stripped.\n\n Any lines that are 'more-indented' keep the rest of their indentation -\n these lines will be indented by 4 spaces."}, {"type":"LiteralStringDoc","value":"\n This entire block of text will be the value of the 'literal_block' key,\n with line breaks being preserved.\n\n The literal continues until de-dented, and the leading indentation is\n stripped.\n\n Any lines that are 'more-indented' keep the rest of their indentation -\n these lines will be indented by 4 spaces."},
{"type":"TextWhitespace","value":" \n"}, {"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"folded_style"}, {"type":"NameTag","value":"folded_style"},
{"type":"Punctuation","value":":"}, {"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "}, {"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"\u003e"}, {"type":"Punctuation","value":"\u003e"},
{"type":"LiteralStringDoc","value":"\n This entire block of text will be the value of 'folded_style', but this\n time, all newlines will be replaced with a single space.\n\n Blank lines, like above, are converted to a newline character.\n\n 'More-indented' lines keep their newlines, too -\n this text will appear over two lines."}, {"type":"LiteralStringDoc","value":"\n This entire block of text will be the value of 'folded_style', but this\n time, all newlines will be replaced with a single space.\n\n Blank lines, like above, are converted to a newline character.\n\n 'More-indented' lines keep their newlines, too -\n this text will appear over two lines."},
{"type":"TextWhitespace","value":" \n"}, {"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"literal_block_with_strip_chomping"}, {"type":"NameTag","value":"literal_block_with_strip_chomping"},
{"type":"Punctuation","value":":"}, {"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "}, {"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|-"}, {"type":"Punctuation","value":"|-"},
{"type":"LiteralStringDoc","value":"\n This entire block of text will be the value of the 'literal_block' key,\n with line breaks being preserved and the strip chomping indicator.\n\n The literal continues until de-dented, and the leading indentation is\n stripped.\n\n Any lines that are 'more-indented' keep the rest of their indentation -\n these lines will be indented by 4 spaces."}, {"type":"LiteralStringDoc","value":"\n This entire block of text will be the value of the 'literal_block' key,\n with line breaks being preserved and the strip chomping indicator.\n\n The literal continues until de-dented, and the leading indentation is\n stripped.\n\n Any lines that are 'more-indented' keep the rest of their indentation -\n these lines will be indented by 4 spaces."},
{"type":"TextWhitespace","value":" \n"}, {"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"literal_block_with_keep_chomping"}, {"type":"NameTag","value":"literal_block_with_keep_chomping"},
{"type":"Punctuation","value":":"}, {"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "}, {"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|+"}, {"type":"Punctuation","value":"|+"},
{"type":"LiteralStringDoc","value":"\n This entire block of text will be the value of the 'literal_block' key,\n with line breaks being preserved and the keep chomping indicator.\n\n The literal continues until de-dented, and the leading indentation is\n stripped."}, {"type":"LiteralStringDoc","value":"\n This entire block of text will be the value of the 'literal_block' key,\n with line breaks being preserved and the keep chomping indicator.\n\n The literal continues until de-dented, and the leading indentation is\n stripped."},
{"type":"TextWhitespace","value":" \n\n"}, {"type":"TextWhitespace","value":"\n\n"},
{"type":"NameTag","value":"a"}, {"type":"NameTag","value":"a"},
{"type":"Punctuation","value":":"}, {"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "}, {"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|"}, {"type":"Punctuation","value":"|"},
{"type":"LiteralStringDoc","value":"\n multiline literal\n line 2"}, {"type":"LiteralStringDoc","value":"\n multiline literal\n line 2"},
{"type":"TextWhitespace","value":" \n"}, {"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"b"}, {"type":"NameTag","value":"b"},
{"type":"Punctuation","value":":"}, {"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "}, {"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"\u003e"}, {"type":"Punctuation","value":"\u003e"},
{"type":"LiteralStringDoc","value":"\n multiline: folded\n line 2"}, {"type":"LiteralStringDoc","value":"\n multiline: folded\n line 2"},
{"type":"TextWhitespace","value":" \n"}, {"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"c"}, {"type":"NameTag","value":"c"},
{"type":"Punctuation","value":":"}, {"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "}, {"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|-"}, {"type":"Punctuation","value":"|-"},
{"type":"LiteralStringDoc","value":"\n multiline # literal strip\n line 2"}, {"type":"LiteralStringDoc","value":"\n multiline # literal strip\n line 2"},
{"type":"TextWhitespace","value":" \n"}, {"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"d"}, {"type":"NameTag","value":"d"},
{"type":"Punctuation","value":":"}, {"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "}, {"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"\u003e-"}, {"type":"Punctuation","value":"\u003e-"},
{"type":"LiteralStringDoc","value":"\n multiline folded strip\n line 2: test\n\n # not a comment\n indented by 1"}, {"type":"LiteralStringDoc","value":"\n multiline folded strip\n line 2: test\n\n # not a comment\n indented by 1"},
{"type":"TextWhitespace","value":" \n"}, {"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"e"}, {"type":"NameTag","value":"e"},
{"type":"Punctuation","value":":"}, {"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "}, {"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|+"}, {"type":"Punctuation","value":"|+"},
{"type":"LiteralStringDoc","value":"\n multiline literal keep\n line: 2"}, {"type":"LiteralStringDoc","value":"\n multiline literal keep\n line: 2"},
{"type":"TextWhitespace","value":" \n"}, {"type":"TextWhitespace","value":"\n"},
{"type":"Comment","value":"# this is a comment"}, {"type":"Comment","value":"# this is a comment"},
{"type":"TextWhitespace","value":"\n"}, {"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"f"}, {"type":"NameTag","value":"f"},
@ -247,13 +247,13 @@
{"type":"TextWhitespace","value":" "}, {"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"\u003e+"}, {"type":"Punctuation","value":"\u003e+"},
{"type":"LiteralStringDoc","value":"\n multiline folded keep one space\n line 2"}, {"type":"LiteralStringDoc","value":"\n multiline folded keep one space\n line 2"},
{"type":"TextWhitespace","value":" \n"}, {"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"g"}, {"type":"NameTag","value":"g"},
{"type":"Punctuation","value":":"}, {"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "}, {"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|"}, {"type":"Punctuation","value":"|"},
{"type":"LiteralStringDoc","value":"\n multiline literal with only one line"}, {"type":"LiteralStringDoc","value":"\n multiline literal with only one line"},
{"type":"TextWhitespace","value":" \n"}, {"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"h"}, {"type":"NameTag","value":"h"},
{"type":"Punctuation","value":":"}, {"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "}, {"type":"TextWhitespace","value":" "},
@ -267,31 +267,31 @@
{"type":"TextWhitespace","value":" "}, {"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|"}, {"type":"Punctuation","value":"|"},
{"type":"LiteralStringDoc","value":"\n multiline literal\n line 2"}, {"type":"LiteralStringDoc","value":"\n multiline literal\n line 2"},
{"type":"TextWhitespace","value":" \n "}, {"type":"TextWhitespace","value":"\n "},
{"type":"NameTag","value":"b"}, {"type":"NameTag","value":"b"},
{"type":"Punctuation","value":":"}, {"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "}, {"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"\u003e"}, {"type":"Punctuation","value":"\u003e"},
{"type":"LiteralStringDoc","value":"\n multiline: folded\n line 2"}, {"type":"LiteralStringDoc","value":"\n multiline: folded\n line 2"},
{"type":"TextWhitespace","value":" \n "}, {"type":"TextWhitespace","value":"\n "},
{"type":"NameTag","value":"c"}, {"type":"NameTag","value":"c"},
{"type":"Punctuation","value":":"}, {"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "}, {"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|-"}, {"type":"Punctuation","value":"|-"},
{"type":"LiteralStringDoc","value":"\n multiline # literal strip\n line 2 6 leading spaces"}, {"type":"LiteralStringDoc","value":"\n multiline # literal strip\n line 2 6 leading spaces"},
{"type":"TextWhitespace","value":" \n\n "}, {"type":"TextWhitespace","value":"\n\n "},
{"type":"NameTag","value":"d"}, {"type":"NameTag","value":"d"},
{"type":"Punctuation","value":":"}, {"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "}, {"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"\u003e-"}, {"type":"Punctuation","value":"\u003e-"},
{"type":"LiteralStringDoc","value":"\n multiline folded strip\n line 2: test\n # not a comment"}, {"type":"LiteralStringDoc","value":"\n multiline folded strip\n line 2: test\n # not a comment"},
{"type":"TextWhitespace","value":" \n "}, {"type":"TextWhitespace","value":"\n "},
{"type":"NameTag","value":"e"}, {"type":"NameTag","value":"e"},
{"type":"Punctuation","value":":"}, {"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "}, {"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|+"}, {"type":"Punctuation","value":"|+"},
{"type":"LiteralStringDoc","value":"\n multiline literal keep\n line: 2"}, {"type":"LiteralStringDoc","value":"\n multiline literal keep\n line: 2"},
{"type":"TextWhitespace","value":" \n "}, {"type":"TextWhitespace","value":"\n "},
{"type":"Comment","value":"# this is a comment"}, {"type":"Comment","value":"# this is a comment"},
{"type":"TextWhitespace","value":"\n "}, {"type":"TextWhitespace","value":"\n "},
{"type":"NameTag","value":"f"}, {"type":"NameTag","value":"f"},
@ -299,13 +299,13 @@
{"type":"TextWhitespace","value":" "}, {"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"\u003e+"}, {"type":"Punctuation","value":"\u003e+"},
{"type":"LiteralStringDoc","value":"\n multiline folded keep\n line 2"}, {"type":"LiteralStringDoc","value":"\n multiline folded keep\n line 2"},
{"type":"TextWhitespace","value":" \n "}, {"type":"TextWhitespace","value":"\n "},
{"type":"NameTag","value":"g"}, {"type":"NameTag","value":"g"},
{"type":"Punctuation","value":":"}, {"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "}, {"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|"}, {"type":"Punctuation","value":"|"},
{"type":"LiteralStringDoc","value":"\n multiline literal with only one line"}, {"type":"LiteralStringDoc","value":"\n multiline literal with only one line"},
{"type":"TextWhitespace","value":" \n "}, {"type":"TextWhitespace","value":"\n "},
{"type":"NameTag","value":"h"}, {"type":"NameTag","value":"h"},
{"type":"Punctuation","value":":"}, {"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "}, {"type":"TextWhitespace","value":" "},
@ -355,7 +355,7 @@
{"type":"TextWhitespace","value":" "}, {"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|"}, {"type":"Punctuation","value":"|"},
{"type":"LiteralStringDoc","value":"\n This is a key\n that has multiple lines"}, {"type":"LiteralStringDoc","value":"\n This is a key\n that has multiple lines"},
{"type":"TextWhitespace","value":" \n"}, {"type":"TextWhitespace","value":"\n"},
{"type":"Punctuation","value":":"}, {"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "}, {"type":"TextWhitespace","value":" "},
{"type":"Literal","value":"and this is its value"}, {"type":"Literal","value":"and this is its value"},
@ -622,7 +622,7 @@
{"type":"TextWhitespace","value":" "}, {"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"|"}, {"type":"Punctuation","value":"|"},
{"type":"LiteralStringDoc","value":"\n R0lGODlhDAAMAIQAAP//9/X17unp5WZmZgAAAOfn515eXvPz7Y6OjuDg4J+fn5\n OTk6enp56enmlpaWNjY6Ojo4SEhP/++f/++f/++f/++f/++f/++f/++f/++f/+\n +f/++f/++f/++f/++f/++SH+Dk1hZGUgd2l0aCBHSU1QACwAAAAADAAMAAAFLC\n AgjoEwnuNAFOhpEMTRiggcz4BNJHrv/zCFcLiwMWYNG84BwwEeECcgggoBADs="}, {"type":"LiteralStringDoc","value":"\n R0lGODlhDAAMAIQAAP//9/X17unp5WZmZgAAAOfn515eXvPz7Y6OjuDg4J+fn5\n OTk6enp56enmlpaWNjY6Ojo4SEhP/++f/++f/++f/++f/++f/++f/++f/++f/+\n +f/++f/++f/++f/++f/++SH+Dk1hZGUgd2l0aCBHSU1QACwAAAAADAAMAAAFLC\n AgjoEwnuNAFOhpEMTRiggcz4BNJHrv/zCFcLiwMWYNG84BwwEeECcgggoBADs="},
{"type":"TextWhitespace","value":" \n\n"}, {"type":"TextWhitespace","value":"\n\n"},
{"type":"Comment","value":"# YAML also has a set type, which looks like this:"}, {"type":"Comment","value":"# YAML also has a set type, which looks like this:"},
{"type":"TextWhitespace","value":"\n"}, {"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"set"}, {"type":"NameTag","value":"set"},

6
regexp.go
View File

@ -194,6 +194,9 @@ func (l *LexerState) Iterator() Token { // nolint: gocognit
for len(l.iteratorStack) > 0 { for len(l.iteratorStack) > 0 {
n := len(l.iteratorStack) - 1 n := len(l.iteratorStack) - 1
t := l.iteratorStack[n]() t := l.iteratorStack[n]()
if t.Type == Ignore {
continue
}
if t == EOF { if t == EOF {
l.iteratorStack = l.iteratorStack[:n] l.iteratorStack = l.iteratorStack[:n]
continue continue
@ -243,6 +246,9 @@ func (l *LexerState) Iterator() Token { // nolint: gocognit
for len(l.iteratorStack) > 0 { for len(l.iteratorStack) > 0 {
n := len(l.iteratorStack) - 1 n := len(l.iteratorStack) - 1
t := l.iteratorStack[n]() t := l.iteratorStack[n]()
if t.Type == Ignore {
continue
}
if t == EOF { if t == EOF {
l.iteratorStack = l.iteratorStack[:n] l.iteratorStack = l.iteratorStack[:n]
continue continue

11
regexp_test.go
View File

@ -192,3 +192,14 @@ func TestByGroupNames(t *testing.T) {
assert.NoError(t, err) assert.NoError(t, err)
assert.Equal(t, []Token{{Error, `abc=123`}}, it.Tokens()) assert.Equal(t, []Token{{Error, `abc=123`}}, it.Tokens())
} }
func TestIgnoreToken(t *testing.T) {
l := Coalesce(mustNewLexer(t, &Config{EnsureNL: true}, Rules{ // nolint: forbidigo
"root": {
{`(\s*)(\w+)(?:\1)(\n)`, ByGroups(Ignore, Keyword, Whitespace), nil},
},
}))
it, err := l.Tokenise(nil, ` hello `)
assert.NoError(t, err)
assert.Equal(t, []Token{{Keyword, "hello"}, {TextWhitespace, "\n"}}, it.Tokens())
}

811
tokentype_enumer.go
View File

@ -7,110 +7,111 @@ import (
"strings" "strings"
) )
const _TokenTypeName = "NoneOtherErrorCodeLineLineLinkLineTableTDLineTableLineHighlightLineNumbersTableLineNumbersLinePreWrapperBackgroundEOFTypeKeywordKeywordConstantKeywordDeclarationKeywordNamespaceKeywordPseudoKeywordReservedKeywordTypeNameNameAttributeNameBuiltinNameBuiltinPseudoNameClassNameConstantNameDecoratorNameEntityNameExceptionNameFunctionNameFunctionMagicNameKeywordNameLabelNameNamespaceNameOperatorNameOtherNamePseudoNamePropertyNameTagNameVariableNameVariableAnonymousNameVariableClassNameVariableGlobalNameVariableInstanceNameVariableMagicLiteralLiteralDateLiteralOtherLiteralStringLiteralStringAffixLiteralStringAtomLiteralStringBacktickLiteralStringBooleanLiteralStringCharLiteralStringDelimiterLiteralStringDocLiteralStringDoubleLiteralStringEscapeLiteralStringHeredocLiteralStringInterpolLiteralStringNameLiteralStringOtherLiteralStringRegexLiteralStringSingleLiteralStringSymbolLiteralNumberLiteralNumberBinLiteralNumberFloatLiteralNumberHexLiteralNumberIntegerLiteralNumberIntegerLongLiteralNumberOctLiteralNumberByteOperatorOperatorWordPunctuationCommentCommentHashbangCommentMultilineCommentSingleCommentSpecialCommentPreprocCommentPreprocFileGenericGenericDeletedGenericEmphGenericErrorGenericHeadingGenericInsertedGenericOutputGenericPromptGenericStrongGenericSubheadingGenericTracebackGenericUnderlineTextTextWhitespaceTextSymbolTextPunctuation" const _TokenTypeName = "IgnoreNoneOtherErrorCodeLineLineLinkLineTableTDLineTableLineHighlightLineNumbersTableLineNumbersLinePreWrapperBackgroundEOFTypeKeywordKeywordConstantKeywordDeclarationKeywordNamespaceKeywordPseudoKeywordReservedKeywordTypeNameNameAttributeNameBuiltinNameBuiltinPseudoNameClassNameConstantNameDecoratorNameEntityNameExceptionNameFunctionNameFunctionMagicNameKeywordNameLabelNameNamespaceNameOperatorNameOtherNamePseudoNamePropertyNameTagNameVariableNameVariableAnonymousNameVariableClassNameVariableGlobalNameVariableInstanceNameVariableMagicLiteralLiteralDateLiteralOtherLiteralStringLiteralStringAffixLiteralStringAtomLiteralStringBacktickLiteralStringBooleanLiteralStringCharLiteralStringDelimiterLiteralStringDocLiteralStringDoubleLiteralStringEscapeLiteralStringHeredocLiteralStringInterpolLiteralStringNameLiteralStringOtherLiteralStringRegexLiteralStringSingleLiteralStringSymbolLiteralNumberLiteralNumberBinLiteralNumberFloatLiteralNumberHexLiteralNumberIntegerLiteralNumberIntegerLongLiteralNumberOctLiteralNumberByteOperatorOperatorWordPunctuationCommentCommentHashbangCommentMultilineCommentSingleCommentSpecialCommentPreprocCommentPreprocFileGenericGenericDeletedGenericEmphGenericErrorGenericHeadingGenericInsertedGenericOutputGenericPromptGenericStrongGenericSubheadingGenericTracebackGenericUnderlineTextTextWhitespaceTextSymbolTextPunctuation"
const _TokenTypeLowerName = "noneothererrorcodelinelinelinklinetabletdlinetablelinehighlightlinenumberstablelinenumberslineprewrapperbackgroundeoftypekeywordkeywordconstantkeyworddeclarationkeywordnamespacekeywordpseudokeywordreservedkeywordtypenamenameattributenamebuiltinnamebuiltinpseudonameclassnameconstantnamedecoratornameentitynameexceptionnamefunctionnamefunctionmagicnamekeywordnamelabelnamenamespacenameoperatornameothernamepseudonamepropertynametagnamevariablenamevariableanonymousnamevariableclassnamevariableglobalnamevariableinstancenamevariablemagicliteralliteraldateliteralotherliteralstringliteralstringaffixliteralstringatomliteralstringbacktickliteralstringbooleanliteralstringcharliteralstringdelimiterliteralstringdocliteralstringdoubleliteralstringescapeliteralstringheredocliteralstringinterpolliteralstringnameliteralstringotherliteralstringregexliteralstringsingleliteralstringsymbolliteralnumberliteralnumberbinliteralnumberfloatliteralnumberhexliteralnumberintegerliteralnumberintegerlongliteralnumberoctliteralnumberbyteoperatoroperatorwordpunctuationcommentcommenthashbangcommentmultilinecommentsinglecommentspecialcommentpreproccommentpreprocfilegenericgenericdeletedgenericemphgenericerrorgenericheadinggenericinsertedgenericoutputgenericpromptgenericstronggenericsubheadinggenerictracebackgenericunderlinetexttextwhitespacetextsymboltextpunctuation" const _TokenTypeLowerName = "ignorenoneothererrorcodelinelinelinklinetabletdlinetablelinehighlightlinenumberstablelinenumberslineprewrapperbackgroundeoftypekeywordkeywordconstantkeyworddeclarationkeywordnamespacekeywordpseudokeywordreservedkeywordtypenamenameattributenamebuiltinnamebuiltinpseudonameclassnameconstantnamedecoratornameentitynameexceptionnamefunctionnamefunctionmagicnamekeywordnamelabelnamenamespacenameoperatornameothernamepseudonamepropertynametagnamevariablenamevariableanonymousnamevariableclassnamevariableglobalnamevariableinstancenamevariablemagicliteralliteraldateliteralotherliteralstringliteralstringaffixliteralstringatomliteralstringbacktickliteralstringbooleanliteralstringcharliteralstringdelimiterliteralstringdocliteralstringdoubleliteralstringescapeliteralstringheredocliteralstringinterpolliteralstringnameliteralstringotherliteralstringregexliteralstringsingleliteralstringsymbolliteralnumberliteralnumberbinliteralnumberfloatliteralnumberhexliteralnumberintegerliteralnumberintegerlongliteralnumberoctliteralnumberbyteoperatoroperatorwordpunctuationcommentcommenthashbangcommentmultilinecommentsinglecommentspecialcommentpreproccommentpreprocfilegenericgenericdeletedgenericemphgenericerrorgenericheadinggenericinsertedgenericoutputgenericpromptgenericstronggenericsubheadinggenerictracebackgenericunderlinetexttextwhitespacetextsymboltextpunctuation"
var _TokenTypeMap = map[TokenType]string{ var _TokenTypeMap = map[TokenType]string{
-13: _TokenTypeName[0:4], -14: _TokenTypeName[0:6],
-12: _TokenTypeName[4:9], -13: _TokenTypeName[6:10],
-11: _TokenTypeName[9:14], -12: _TokenTypeName[10:15],
-10: _TokenTypeName[14:22], -11: _TokenTypeName[15:20],
-9: _TokenTypeName[22:30], -10: _TokenTypeName[20:28],
-8: _TokenTypeName[30:41], -9: _TokenTypeName[28:36],
-7: _TokenTypeName[41:50], -8: _TokenTypeName[36:47],
-6: _TokenTypeName[50:63], -7: _TokenTypeName[47:56],
-5: _TokenTypeName[63:79], -6: _TokenTypeName[56:69],
-4: _TokenTypeName[79:90], -5: _TokenTypeName[69:85],
-3: _TokenTypeName[90:94], -4: _TokenTypeName[85:96],
-2: _TokenTypeName[94:104], -3: _TokenTypeName[96:100],
-1: _TokenTypeName[104:114], -2: _TokenTypeName[100:110],
0: _TokenTypeName[114:121], -1: _TokenTypeName[110:120],
1000: _TokenTypeName[121:128], 0: _TokenTypeName[120:127],
1001: _TokenTypeName[128:143], 1000: _TokenTypeName[127:134],
1002: _TokenTypeName[143:161], 1001: _TokenTypeName[134:149],
1003: _TokenTypeName[161:177], 1002: _TokenTypeName[149:167],
1004: _TokenTypeName[177:190], 1003: _TokenTypeName[167:183],
1005: _TokenTypeName[190:205], 1004: _TokenTypeName[183:196],
1006: _TokenTypeName[205:216], 1005: _TokenTypeName[196:211],
2000: _TokenTypeName[216:220], 1006: _TokenTypeName[211:222],
2001: _TokenTypeName[220:233], 2000: _TokenTypeName[222:226],
2002: _TokenTypeName[233:244], 2001: _TokenTypeName[226:239],
2003: _TokenTypeName[244:261], 2002: _TokenTypeName[239:250],
2004: _TokenTypeName[261:270], 2003: _TokenTypeName[250:267],
2005: _TokenTypeName[270:282], 2004: _TokenTypeName[267:276],
2006: _TokenTypeName[282:295], 2005: _TokenTypeName[276:288],
2007: _TokenTypeName[295:305], 2006: _TokenTypeName[288:301],
2008: _TokenTypeName[305:318], 2007: _TokenTypeName[301:311],
2009: _TokenTypeName[318:330], 2008: _TokenTypeName[311:324],
2010: _TokenTypeName[330:347], 2009: _TokenTypeName[324:336],
2011: _TokenTypeName[347:358], 2010: _TokenTypeName[336:353],
2012: _TokenTypeName[358:367], 2011: _TokenTypeName[353:364],
2013: _TokenTypeName[367:380], 2012: _TokenTypeName[364:373],
2014: _TokenTypeName[380:392], 2013: _TokenTypeName[373:386],
2015: _TokenTypeName[392:401], 2014: _TokenTypeName[386:398],
2016: _TokenTypeName[401:411], 2015: _TokenTypeName[398:407],
2017: _TokenTypeName[411:423], 2016: _TokenTypeName[407:417],
2018: _TokenTypeName[423:430], 2017: _TokenTypeName[417:429],
2019: _TokenTypeName[430:442], 2018: _TokenTypeName[429:436],
2020: _TokenTypeName[442:463], 2019: _TokenTypeName[436:448],
2021: _TokenTypeName[463:480], 2020: _TokenTypeName[448:469],
2022: _TokenTypeName[480:498], 2021: _TokenTypeName[469:486],
2023: _TokenTypeName[498:518], 2022: _TokenTypeName[486:504],
2024: _TokenTypeName[518:535], 2023: _TokenTypeName[504:524],
3000: _TokenTypeName[535:542], 2024: _TokenTypeName[524:541],
3001: _TokenTypeName[542:553], 3000: _TokenTypeName[541:548],
3002: _TokenTypeName[553:565], 3001: _TokenTypeName[548:559],
3100: _TokenTypeName[565:578], 3002: _TokenTypeName[559:571],
3101: _TokenTypeName[578:596], 3100: _TokenTypeName[571:584],
3102: _TokenTypeName[596:613], 3101: _TokenTypeName[584:602],
3103: _TokenTypeName[613:634], 3102: _TokenTypeName[602:619],
3104: _TokenTypeName[634:654], 3103: _TokenTypeName[619:640],
3105: _TokenTypeName[654:671], 3104: _TokenTypeName[640:660],
3106: _TokenTypeName[671:693], 3105: _TokenTypeName[660:677],
3107: _TokenTypeName[693:709], 3106: _TokenTypeName[677:699],
3108: _TokenTypeName[709:728], 3107: _TokenTypeName[699:715],
3109: _TokenTypeName[728:747], 3108: _TokenTypeName[715:734],
3110: _TokenTypeName[747:767], 3109: _TokenTypeName[734:753],
3111: _TokenTypeName[767:788], 3110: _TokenTypeName[753:773],
3112: _TokenTypeName[788:805], 3111: _TokenTypeName[773:794],
3113: _TokenTypeName[805:823], 3112: _TokenTypeName[794:811],
3114: _TokenTypeName[823:841], 3113: _TokenTypeName[811:829],
3115: _TokenTypeName[841:860], 3114: _TokenTypeName[829:847],
3116: _TokenTypeName[860:879], 3115: _TokenTypeName[847:866],
3200: _TokenTypeName[879:892], 3116: _TokenTypeName[866:885],
3201: _TokenTypeName[892:908], 3200: _TokenTypeName[885:898],
3202: _TokenTypeName[908:926], 3201: _TokenTypeName[898:914],
3203: _TokenTypeName[926:942], 3202: _TokenTypeName[914:932],
3204: _TokenTypeName[942:962], 3203: _TokenTypeName[932:948],
3205: _TokenTypeName[962:986], 3204: _TokenTypeName[948:968],
3206: _TokenTypeName[986:1002], 3205: _TokenTypeName[968:992],
3207: _TokenTypeName[1002:1019], 3206: _TokenTypeName[992:1008],
4000: _TokenTypeName[1019:1027], 3207: _TokenTypeName[1008:1025],
4001: _TokenTypeName[1027:1039], 4000: _TokenTypeName[1025:1033],
5000: _TokenTypeName[1039:1050], 4001: _TokenTypeName[1033:1045],
6000: _TokenTypeName[1050:1057], 5000: _TokenTypeName[1045:1056],
6001: _TokenTypeName[1057:1072], 6000: _TokenTypeName[1056:1063],
6002: _TokenTypeName[1072:1088], 6001: _TokenTypeName[1063:1078],
6003: _TokenTypeName[1088:1101], 6002: _TokenTypeName[1078:1094],
6004: _TokenTypeName[1101:1115], 6003: _TokenTypeName[1094:1107],
6100: _TokenTypeName[1115:1129], 6004: _TokenTypeName[1107:1121],
6101: _TokenTypeName[1129:1147], 6100: _TokenTypeName[1121:1135],
7000: _TokenTypeName[1147:1154], 6101: _TokenTypeName[1135:1153],
7001: _TokenTypeName[1154:1168], 7000: _TokenTypeName[1153:1160],
7002: _TokenTypeName[1168:1179], 7001: _TokenTypeName[1160:1174],
7003: _TokenTypeName[1179:1191], 7002: _TokenTypeName[1174:1185],
7004: _TokenTypeName[1191:1205], 7003: _TokenTypeName[1185:1197],
7005: _TokenTypeName[1205:1220], 7004: _TokenTypeName[1197:1211],
7006: _TokenTypeName[1220:1233], 7005: _TokenTypeName[1211:1226],
7007: _TokenTypeName[1233:1246], 7006: _TokenTypeName[1226:1239],
7008: _TokenTypeName[1246:1259], 7007: _TokenTypeName[1239:1252],
7009: _TokenTypeName[1259:1276], 7008: _TokenTypeName[1252:1265],
7010: _TokenTypeName[1276:1292], 7009: _TokenTypeName[1265:1282],
7011: _TokenTypeName[1292:1308], 7010: _TokenTypeName[1282:1298],
8000: _TokenTypeName[1308:1312], 7011: _TokenTypeName[1298:1314],
8001: _TokenTypeName[1312:1326], 8000: _TokenTypeName[1314:1318],
8002: _TokenTypeName[1326:1336], 8001: _TokenTypeName[1318:1332],
8003: _TokenTypeName[1336:1351], 8002: _TokenTypeName[1332:1342],
8003: _TokenTypeName[1342:1357],
} }
func (i TokenType) String() string { func (i TokenType) String() string {
@ -124,6 +125,7 @@ func (i TokenType) String() string {
// Re-run the stringer command to generate them again. // Re-run the stringer command to generate them again.
func _TokenTypeNoOp() { func _TokenTypeNoOp() {
var x [1]struct{} var x [1]struct{}
_ = x[Ignore-(-14)]
_ = x[None-(-13)] _ = x[None-(-13)]
_ = x[Other-(-12)] _ = x[Other-(-12)]
_ = x[Error-(-11)] _ = x[Error-(-11)]
@ -226,312 +228,315 @@ func _TokenTypeNoOp() {
_ = x[TextPunctuation-(8003)] _ = x[TextPunctuation-(8003)]
} }
var _TokenTypeValues = []TokenType{None, Other, Error, CodeLine, LineLink, LineTableTD, LineTable, LineHighlight, LineNumbersTable, LineNumbers, Line, PreWrapper, Background, EOFType, Keyword, KeywordConstant, KeywordDeclaration, KeywordNamespace, KeywordPseudo, KeywordReserved, KeywordType, Name, NameAttribute, NameBuiltin, NameBuiltinPseudo, NameClass, NameConstant, NameDecorator, NameEntity, NameException, NameFunction, NameFunctionMagic, NameKeyword, NameLabel, NameNamespace, NameOperator, NameOther, NamePseudo, NameProperty, NameTag, NameVariable, NameVariableAnonymous, NameVariableClass, NameVariableGlobal, NameVariableInstance, NameVariableMagic, Literal, LiteralDate, LiteralOther, LiteralString, LiteralStringAffix, LiteralStringAtom, LiteralStringBacktick, LiteralStringBoolean, LiteralStringChar, LiteralStringDelimiter, LiteralStringDoc, LiteralStringDouble, LiteralStringEscape, LiteralStringHeredoc, LiteralStringInterpol, LiteralStringName, LiteralStringOther, LiteralStringRegex, LiteralStringSingle, LiteralStringSymbol, LiteralNumber, LiteralNumberBin, LiteralNumberFloat, LiteralNumberHex, LiteralNumberInteger, LiteralNumberIntegerLong, LiteralNumberOct, LiteralNumberByte, Operator, OperatorWord, Punctuation, Comment, CommentHashbang, CommentMultiline, CommentSingle, CommentSpecial, CommentPreproc, CommentPreprocFile, Generic, GenericDeleted, GenericEmph, GenericError, GenericHeading, GenericInserted, GenericOutput, GenericPrompt, GenericStrong, GenericSubheading, GenericTraceback, GenericUnderline, Text, TextWhitespace, TextSymbol, TextPunctuation} var _TokenTypeValues = []TokenType{Ignore, None, Other, Error, CodeLine, LineLink, LineTableTD, LineTable, LineHighlight, LineNumbersTable, LineNumbers, Line, PreWrapper, Background, EOFType, Keyword, KeywordConstant, KeywordDeclaration, KeywordNamespace, KeywordPseudo, KeywordReserved, KeywordType, Name, NameAttribute, NameBuiltin, NameBuiltinPseudo, NameClass, NameConstant, NameDecorator, NameEntity, NameException, NameFunction, NameFunctionMagic, NameKeyword, NameLabel, NameNamespace, NameOperator, NameOther, NamePseudo, NameProperty, NameTag, NameVariable, NameVariableAnonymous, NameVariableClass, NameVariableGlobal, NameVariableInstance, NameVariableMagic, Literal, LiteralDate, LiteralOther, LiteralString, LiteralStringAffix, LiteralStringAtom, LiteralStringBacktick, LiteralStringBoolean, LiteralStringChar, LiteralStringDelimiter, LiteralStringDoc, LiteralStringDouble, LiteralStringEscape, LiteralStringHeredoc, LiteralStringInterpol, LiteralStringName, LiteralStringOther, LiteralStringRegex, LiteralStringSingle, LiteralStringSymbol, LiteralNumber, LiteralNumberBin, LiteralNumberFloat, LiteralNumberHex, LiteralNumberInteger, LiteralNumberIntegerLong, LiteralNumberOct, LiteralNumberByte, Operator, OperatorWord, Punctuation, Comment, CommentHashbang, CommentMultiline, CommentSingle, CommentSpecial, CommentPreproc, CommentPreprocFile, Generic, GenericDeleted, GenericEmph, GenericError, GenericHeading, GenericInserted, GenericOutput, GenericPrompt, GenericStrong, GenericSubheading, GenericTraceback, GenericUnderline, Text, TextWhitespace, TextSymbol, TextPunctuation}
var _TokenTypeNameToValueMap = map[string]TokenType{ var _TokenTypeNameToValueMap = map[string]TokenType{
_TokenTypeName[0:4]: None, _TokenTypeName[0:6]: Ignore,
_TokenTypeLowerName[0:4]: None, _TokenTypeLowerName[0:6]: Ignore,
_TokenTypeName[4:9]: Other, _TokenTypeName[6:10]: None,
_TokenTypeLowerName[4:9]: Other, _TokenTypeLowerName[6:10]: None,
_TokenTypeName[9:14]: Error, _TokenTypeName[10:15]: Other,
_TokenTypeLowerName[9:14]: Error, _TokenTypeLowerName[10:15]: Other,
_TokenTypeName[14:22]: CodeLine, _TokenTypeName[15:20]: Error,
_TokenTypeLowerName[14:22]: CodeLine, _TokenTypeLowerName[15:20]: Error,
_TokenTypeName[22:30]: LineLink, _TokenTypeName[20:28]: CodeLine,
_TokenTypeLowerName[22:30]: LineLink, _TokenTypeLowerName[20:28]: CodeLine,
_TokenTypeName[30:41]: LineTableTD, _TokenTypeName[28:36]: LineLink,
_TokenTypeLowerName[30:41]: LineTableTD, _TokenTypeLowerName[28:36]: LineLink,
_TokenTypeName[41:50]: LineTable, _TokenTypeName[36:47]: LineTableTD,
_TokenTypeLowerName[41:50]: LineTable, _TokenTypeLowerName[36:47]: LineTableTD,
_TokenTypeName[50:63]: LineHighlight, _TokenTypeName[47:56]: LineTable,
_TokenTypeLowerName[50:63]: LineHighlight, _TokenTypeLowerName[47:56]: LineTable,
_TokenTypeName[63:79]: LineNumbersTable, _TokenTypeName[56:69]: LineHighlight,
_TokenTypeLowerName[63:79]: LineNumbersTable, _TokenTypeLowerName[56:69]: LineHighlight,
_TokenTypeName[79:90]: LineNumbers, _TokenTypeName[69:85]: LineNumbersTable,
_TokenTypeLowerName[79:90]: LineNumbers, _TokenTypeLowerName[69:85]: LineNumbersTable,
_TokenTypeName[90:94]: Line, _TokenTypeName[85:96]: LineNumbers,
_TokenTypeLowerName[90:94]: Line, _TokenTypeLowerName[85:96]: LineNumbers,
_TokenTypeName[94:104]: PreWrapper, _TokenTypeName[96:100]: Line,
_TokenTypeLowerName[94:104]: PreWrapper, _TokenTypeLowerName[96:100]: Line,
_TokenTypeName[104:114]: Background, _TokenTypeName[100:110]: PreWrapper,
_TokenTypeLowerName[104:114]: Background, _TokenTypeLowerName[100:110]: PreWrapper,
_TokenTypeName[114:121]: EOFType, _TokenTypeName[110:120]: Background,
_TokenTypeLowerName[114:121]: EOFType, _TokenTypeLowerName[110:120]: Background,
_TokenTypeName[121:128]: Keyword, _TokenTypeName[120:127]: EOFType,
_TokenTypeLowerName[121:128]: Keyword, _TokenTypeLowerName[120:127]: EOFType,
_TokenTypeName[128:143]: KeywordConstant, _TokenTypeName[127:134]: Keyword,
_TokenTypeLowerName[128:143]: KeywordConstant, _TokenTypeLowerName[127:134]: Keyword,
_TokenTypeName[143:161]: KeywordDeclaration, _TokenTypeName[134:149]: KeywordConstant,
_TokenTypeLowerName[143:161]: KeywordDeclaration, _TokenTypeLowerName[134:149]: KeywordConstant,
_TokenTypeName[161:177]: KeywordNamespace, _TokenTypeName[149:167]: KeywordDeclaration,
_TokenTypeLowerName[161:177]: KeywordNamespace, _TokenTypeLowerName[149:167]: KeywordDeclaration,
_TokenTypeName[177:190]: KeywordPseudo, _TokenTypeName[167:183]: KeywordNamespace,
_TokenTypeLowerName[177:190]: KeywordPseudo, _TokenTypeLowerName[167:183]: KeywordNamespace,
_TokenTypeName[190:205]: KeywordReserved, _TokenTypeName[183:196]: KeywordPseudo,
_TokenTypeLowerName[190:205]: KeywordReserved, _TokenTypeLowerName[183:196]: KeywordPseudo,
_TokenTypeName[205:216]: KeywordType, _TokenTypeName[196:211]: KeywordReserved,
_TokenTypeLowerName[205:216]: KeywordType, _TokenTypeLowerName[196:211]: KeywordReserved,
_TokenTypeName[216:220]: Name, _TokenTypeName[211:222]: KeywordType,
_TokenTypeLowerName[216:220]: Name, _TokenTypeLowerName[211:222]: KeywordType,
_TokenTypeName[220:233]: NameAttribute, _TokenTypeName[222:226]: Name,
_TokenTypeLowerName[220:233]: NameAttribute, _TokenTypeLowerName[222:226]: Name,
_TokenTypeName[233:244]: NameBuiltin, _TokenTypeName[226:239]: NameAttribute,
_TokenTypeLowerName[233:244]: NameBuiltin, _TokenTypeLowerName[226:239]: NameAttribute,
_TokenTypeName[244:261]: NameBuiltinPseudo, _TokenTypeName[239:250]: NameBuiltin,
_TokenTypeLowerName[244:261]: NameBuiltinPseudo, _TokenTypeLowerName[239:250]: NameBuiltin,
_TokenTypeName[261:270]: NameClass, _TokenTypeName[250:267]: NameBuiltinPseudo,
_TokenTypeLowerName[261:270]: NameClass, _TokenTypeLowerName[250:267]: NameBuiltinPseudo,
_TokenTypeName[270:282]: NameConstant, _TokenTypeName[267:276]: NameClass,
_TokenTypeLowerName[270:282]: NameConstant, _TokenTypeLowerName[267:276]: NameClass,
_TokenTypeName[282:295]: NameDecorator, _TokenTypeName[276:288]: NameConstant,
_TokenTypeLowerName[282:295]: NameDecorator, _TokenTypeLowerName[276:288]: NameConstant,
_TokenTypeName[295:305]: NameEntity, _TokenTypeName[288:301]: NameDecorator,
_TokenTypeLowerName[295:305]: NameEntity, _TokenTypeLowerName[288:301]: NameDecorator,
_TokenTypeName[305:318]: NameException, _TokenTypeName[301:311]: NameEntity,
_TokenTypeLowerName[305:318]: NameException, _TokenTypeLowerName[301:311]: NameEntity,
_TokenTypeName[318:330]: NameFunction, _TokenTypeName[311:324]: NameException,
_TokenTypeLowerName[318:330]: NameFunction, _TokenTypeLowerName[311:324]: NameException,
_TokenTypeName[330:347]: NameFunctionMagic, _TokenTypeName[324:336]: NameFunction,
_TokenTypeLowerName[330:347]: NameFunctionMagic, _TokenTypeLowerName[324:336]: NameFunction,
_TokenTypeName[347:358]: NameKeyword, _TokenTypeName[336:353]: NameFunctionMagic,
_TokenTypeLowerName[347:358]: NameKeyword, _TokenTypeLowerName[336:353]: NameFunctionMagic,
_TokenTypeName[358:367]: NameLabel, _TokenTypeName[353:364]: NameKeyword,
_TokenTypeLowerName[358:367]: NameLabel, _TokenTypeLowerName[353:364]: NameKeyword,
_TokenTypeName[367:380]: NameNamespace, _TokenTypeName[364:373]: NameLabel,
_TokenTypeLowerName[367:380]: NameNamespace, _TokenTypeLowerName[364:373]: NameLabel,
_TokenTypeName[380:392]: NameOperator, _TokenTypeName[373:386]: NameNamespace,
_TokenTypeLowerName[380:392]: NameOperator, _TokenTypeLowerName[373:386]: NameNamespace,
_TokenTypeName[392:401]: NameOther, _TokenTypeName[386:398]: NameOperator,
_TokenTypeLowerName[392:401]: NameOther, _TokenTypeLowerName[386:398]: NameOperator,
_TokenTypeName[401:411]: NamePseudo, _TokenTypeName[398:407]: NameOther,
_TokenTypeLowerName[401:411]: NamePseudo, _TokenTypeLowerName[398:407]: NameOther,
_TokenTypeName[411:423]: NameProperty, _TokenTypeName[407:417]: NamePseudo,
_TokenTypeLowerName[411:423]: NameProperty, _TokenTypeLowerName[407:417]: NamePseudo,
_TokenTypeName[423:430]: NameTag, _TokenTypeName[417:429]: NameProperty,
_TokenTypeLowerName[423:430]: NameTag, _TokenTypeLowerName[417:429]: NameProperty,
_TokenTypeName[430:442]: NameVariable, _TokenTypeName[429:436]: NameTag,
_TokenTypeLowerName[430:442]: NameVariable, _TokenTypeLowerName[429:436]: NameTag,
_TokenTypeName[442:463]: NameVariableAnonymous, _TokenTypeName[436:448]: NameVariable,
_TokenTypeLowerName[442:463]: NameVariableAnonymous, _TokenTypeLowerName[436:448]: NameVariable,
_TokenTypeName[463:480]: NameVariableClass, _TokenTypeName[448:469]: NameVariableAnonymous,
_TokenTypeLowerName[463:480]: NameVariableClass, _TokenTypeLowerName[448:469]: NameVariableAnonymous,
_TokenTypeName[480:498]: NameVariableGlobal, _TokenTypeName[469:486]: NameVariableClass,
_TokenTypeLowerName[480:498]: NameVariableGlobal, _TokenTypeLowerName[469:486]: NameVariableClass,
_TokenTypeName[498:518]: NameVariableInstance, _TokenTypeName[486:504]: NameVariableGlobal,
_TokenTypeLowerName[498:518]: NameVariableInstance, _TokenTypeLowerName[486:504]: NameVariableGlobal,
_TokenTypeName[518:535]: NameVariableMagic, _TokenTypeName[504:524]: NameVariableInstance,
_TokenTypeLowerName[518:535]: NameVariableMagic, _TokenTypeLowerName[504:524]: NameVariableInstance,
_TokenTypeName[535:542]: Literal, _TokenTypeName[524:541]: NameVariableMagic,
_TokenTypeLowerName[535:542]: Literal, _TokenTypeLowerName[524:541]: NameVariableMagic,
_TokenTypeName[542:553]: LiteralDate, _TokenTypeName[541:548]: Literal,
_TokenTypeLowerName[542:553]: LiteralDate, _TokenTypeLowerName[541:548]: Literal,
_TokenTypeName[553:565]: LiteralOther, _TokenTypeName[548:559]: LiteralDate,
_TokenTypeLowerName[553:565]: LiteralOther, _TokenTypeLowerName[548:559]: LiteralDate,
_TokenTypeName[565:578]: LiteralString, _TokenTypeName[559:571]: LiteralOther,
_TokenTypeLowerName[565:578]: LiteralString, _TokenTypeLowerName[559:571]: LiteralOther,
_TokenTypeName[578:596]: LiteralStringAffix, _TokenTypeName[571:584]: LiteralString,
_TokenTypeLowerName[578:596]: LiteralStringAffix, _TokenTypeLowerName[571:584]: LiteralString,
_TokenTypeName[596:613]: LiteralStringAtom, _TokenTypeName[584:602]: LiteralStringAffix,
_TokenTypeLowerName[596:613]: LiteralStringAtom, _TokenTypeLowerName[584:602]: LiteralStringAffix,
_TokenTypeName[613:634]: LiteralStringBacktick, _TokenTypeName[602:619]: LiteralStringAtom,
_TokenTypeLowerName[613:634]: LiteralStringBacktick, _TokenTypeLowerName[602:619]: LiteralStringAtom,
_TokenTypeName[634:654]: LiteralStringBoolean, _TokenTypeName[619:640]: LiteralStringBacktick,
_TokenTypeLowerName[634:654]: LiteralStringBoolean, _TokenTypeLowerName[619:640]: LiteralStringBacktick,
_TokenTypeName[654:671]: LiteralStringChar, _TokenTypeName[640:660]: LiteralStringBoolean,
_TokenTypeLowerName[654:671]: LiteralStringChar, _TokenTypeLowerName[640:660]: LiteralStringBoolean,
_TokenTypeName[671:693]: LiteralStringDelimiter, _TokenTypeName[660:677]: LiteralStringChar,
_TokenTypeLowerName[671:693]: LiteralStringDelimiter, _TokenTypeLowerName[660:677]: LiteralStringChar,
_TokenTypeName[693:709]: LiteralStringDoc, _TokenTypeName[677:699]: LiteralStringDelimiter,
_TokenTypeLowerName[693:709]: LiteralStringDoc, _TokenTypeLowerName[677:699]: LiteralStringDelimiter,
_TokenTypeName[709:728]: LiteralStringDouble, _TokenTypeName[699:715]: LiteralStringDoc,
_TokenTypeLowerName[709:728]: LiteralStringDouble, _TokenTypeLowerName[699:715]: LiteralStringDoc,
_TokenTypeName[728:747]: LiteralStringEscape, _TokenTypeName[715:734]: LiteralStringDouble,
_TokenTypeLowerName[728:747]: LiteralStringEscape, _TokenTypeLowerName[715:734]: LiteralStringDouble,
_TokenTypeName[747:767]: LiteralStringHeredoc, _TokenTypeName[734:753]: LiteralStringEscape,
_TokenTypeLowerName[747:767]: LiteralStringHeredoc, _TokenTypeLowerName[734:753]: LiteralStringEscape,
_TokenTypeName[767:788]: LiteralStringInterpol, _TokenTypeName[753:773]: LiteralStringHeredoc,
_TokenTypeLowerName[767:788]: LiteralStringInterpol, _TokenTypeLowerName[753:773]: LiteralStringHeredoc,
_TokenTypeName[788:805]: LiteralStringName, _TokenTypeName[773:794]: LiteralStringInterpol,
_TokenTypeLowerName[788:805]: LiteralStringName, _TokenTypeLowerName[773:794]: LiteralStringInterpol,
_TokenTypeName[805:823]: LiteralStringOther, _TokenTypeName[794:811]: LiteralStringName,
_TokenTypeLowerName[805:823]: LiteralStringOther, _TokenTypeLowerName[794:811]: LiteralStringName,
_TokenTypeName[823:841]: LiteralStringRegex, _TokenTypeName[811:829]: LiteralStringOther,
_TokenTypeLowerName[823:841]: LiteralStringRegex, _TokenTypeLowerName[811:829]: LiteralStringOther,
_TokenTypeName[841:860]: LiteralStringSingle, _TokenTypeName[829:847]: LiteralStringRegex,
_TokenTypeLowerName[841:860]: LiteralStringSingle, _TokenTypeLowerName[829:847]: LiteralStringRegex,
_TokenTypeName[860:879]: LiteralStringSymbol, _TokenTypeName[847:866]: LiteralStringSingle,
_TokenTypeLowerName[860:879]: LiteralStringSymbol, _TokenTypeLowerName[847:866]: LiteralStringSingle,
_TokenTypeName[879:892]: LiteralNumber, _TokenTypeName[866:885]: LiteralStringSymbol,
_TokenTypeLowerName[879:892]: LiteralNumber, _TokenTypeLowerName[866:885]: LiteralStringSymbol,
_TokenTypeName[892:908]: LiteralNumberBin, _TokenTypeName[885:898]: LiteralNumber,
_TokenTypeLowerName[892:908]: LiteralNumberBin, _TokenTypeLowerName[885:898]: LiteralNumber,
_TokenTypeName[908:926]: LiteralNumberFloat, _TokenTypeName[898:914]: LiteralNumberBin,
_TokenTypeLowerName[908:926]: LiteralNumberFloat, _TokenTypeLowerName[898:914]: LiteralNumberBin,
_TokenTypeName[926:942]: LiteralNumberHex, _TokenTypeName[914:932]: LiteralNumberFloat,
_TokenTypeLowerName[926:942]: LiteralNumberHex, _TokenTypeLowerName[914:932]: LiteralNumberFloat,
_TokenTypeName[942:962]: LiteralNumberInteger, _TokenTypeName[932:948]: LiteralNumberHex,
_TokenTypeLowerName[942:962]: LiteralNumberInteger, _TokenTypeLowerName[932:948]: LiteralNumberHex,
_TokenTypeName[962:986]: LiteralNumberIntegerLong, _TokenTypeName[948:968]: LiteralNumberInteger,
_TokenTypeLowerName[962:986]: LiteralNumberIntegerLong, _TokenTypeLowerName[948:968]: LiteralNumberInteger,
_TokenTypeName[986:1002]: LiteralNumberOct, _TokenTypeName[968:992]: LiteralNumberIntegerLong,
_TokenTypeLowerName[986:1002]: LiteralNumberOct, _TokenTypeLowerName[968:992]: LiteralNumberIntegerLong,
_TokenTypeName[1002:1019]: LiteralNumberByte, _TokenTypeName[992:1008]: LiteralNumberOct,
_TokenTypeLowerName[1002:1019]: LiteralNumberByte, _TokenTypeLowerName[992:1008]: LiteralNumberOct,
_TokenTypeName[1019:1027]: Operator, _TokenTypeName[1008:1025]: LiteralNumberByte,
_TokenTypeLowerName[1019:1027]: Operator, _TokenTypeLowerName[1008:1025]: LiteralNumberByte,
_TokenTypeName[1027:1039]: OperatorWord, _TokenTypeName[1025:1033]: Operator,
_TokenTypeLowerName[1027:1039]: OperatorWord, _TokenTypeLowerName[1025:1033]: Operator,
_TokenTypeName[1039:1050]: Punctuation, _TokenTypeName[1033:1045]: OperatorWord,
_TokenTypeLowerName[1039:1050]: Punctuation, _TokenTypeLowerName[1033:1045]: OperatorWord,
_TokenTypeName[1050:1057]: Comment, _TokenTypeName[1045:1056]: Punctuation,
_TokenTypeLowerName[1050:1057]: Comment, _TokenTypeLowerName[1045:1056]: Punctuation,
_TokenTypeName[1057:1072]: CommentHashbang, _TokenTypeName[1056:1063]: Comment,
_TokenTypeLowerName[1057:1072]: CommentHashbang, _TokenTypeLowerName[1056:1063]: Comment,
_TokenTypeName[1072:1088]: CommentMultiline, _TokenTypeName[1063:1078]: CommentHashbang,
_TokenTypeLowerName[1072:1088]: CommentMultiline, _TokenTypeLowerName[1063:1078]: CommentHashbang,
_TokenTypeName[1088:1101]: CommentSingle, _TokenTypeName[1078:1094]: CommentMultiline,
_TokenTypeLowerName[1088:1101]: CommentSingle, _TokenTypeLowerName[1078:1094]: CommentMultiline,
_TokenTypeName[1101:1115]: CommentSpecial, _TokenTypeName[1094:1107]: CommentSingle,
_TokenTypeLowerName[1101:1115]: CommentSpecial, _TokenTypeLowerName[1094:1107]: CommentSingle,
_TokenTypeName[1115:1129]: CommentPreproc, _TokenTypeName[1107:1121]: CommentSpecial,
_TokenTypeLowerName[1115:1129]: CommentPreproc, _TokenTypeLowerName[1107:1121]: CommentSpecial,
_TokenTypeName[1129:1147]: CommentPreprocFile, _TokenTypeName[1121:1135]: CommentPreproc,
_TokenTypeLowerName[1129:1147]: CommentPreprocFile, _TokenTypeLowerName[1121:1135]: CommentPreproc,
_TokenTypeName[1147:1154]: Generic, _TokenTypeName[1135:1153]: CommentPreprocFile,
_TokenTypeLowerName[1147:1154]: Generic, _TokenTypeLowerName[1135:1153]: CommentPreprocFile,
_TokenTypeName[1154:1168]: GenericDeleted, _TokenTypeName[1153:1160]: Generic,
_TokenTypeLowerName[1154:1168]: GenericDeleted, _TokenTypeLowerName[1153:1160]: Generic,
_TokenTypeName[1168:1179]: GenericEmph, _TokenTypeName[1160:1174]: GenericDeleted,
_TokenTypeLowerName[1168:1179]: GenericEmph, _TokenTypeLowerName[1160:1174]: GenericDeleted,
_TokenTypeName[1179:1191]: GenericError, _TokenTypeName[1174:1185]: GenericEmph,
_TokenTypeLowerName[1179:1191]: GenericError, _TokenTypeLowerName[1174:1185]: GenericEmph,
_TokenTypeName[1191:1205]: GenericHeading, _TokenTypeName[1185:1197]: GenericError,
_TokenTypeLowerName[1191:1205]: GenericHeading, _TokenTypeLowerName[1185:1197]: GenericError,
_TokenTypeName[1205:1220]: GenericInserted, _TokenTypeName[1197:1211]: GenericHeading,
_TokenTypeLowerName[1205:1220]: GenericInserted, _TokenTypeLowerName[1197:1211]: GenericHeading,
_TokenTypeName[1220:1233]: GenericOutput, _TokenTypeName[1211:1226]: GenericInserted,
_TokenTypeLowerName[1220:1233]: GenericOutput, _TokenTypeLowerName[1211:1226]: GenericInserted,
_TokenTypeName[1233:1246]: GenericPrompt, _TokenTypeName[1226:1239]: GenericOutput,
_TokenTypeLowerName[1233:1246]: GenericPrompt, _TokenTypeLowerName[1226:1239]: GenericOutput,
_TokenTypeName[1246:1259]: GenericStrong, _TokenTypeName[1239:1252]: GenericPrompt,
_TokenTypeLowerName[1246:1259]: GenericStrong, _TokenTypeLowerName[1239:1252]: GenericPrompt,
_TokenTypeName[1259:1276]: GenericSubheading, _TokenTypeName[1252:1265]: GenericStrong,
_TokenTypeLowerName[1259:1276]: GenericSubheading, _TokenTypeLowerName[1252:1265]: GenericStrong,
_TokenTypeName[1276:1292]: GenericTraceback, _TokenTypeName[1265:1282]: GenericSubheading,
_TokenTypeLowerName[1276:1292]: GenericTraceback, _TokenTypeLowerName[1265:1282]: GenericSubheading,
_TokenTypeName[1292:1308]: GenericUnderline, _TokenTypeName[1282:1298]: GenericTraceback,
_TokenTypeLowerName[1292:1308]: GenericUnderline, _TokenTypeLowerName[1282:1298]: GenericTraceback,
_TokenTypeName[1308:1312]: Text, _TokenTypeName[1298:1314]: GenericUnderline,
_TokenTypeLowerName[1308:1312]: Text, _TokenTypeLowerName[1298:1314]: GenericUnderline,
_TokenTypeName[1312:1326]: TextWhitespace, _TokenTypeName[1314:1318]: Text,
_TokenTypeLowerName[1312:1326]: TextWhitespace, _TokenTypeLowerName[1314:1318]: Text,
_TokenTypeName[1326:1336]: TextSymbol, _TokenTypeName[1318:1332]: TextWhitespace,
_TokenTypeLowerName[1326:1336]: TextSymbol, _TokenTypeLowerName[1318:1332]: TextWhitespace,
_TokenTypeName[1336:1351]: TextPunctuation, _TokenTypeName[1332:1342]: TextSymbol,
_TokenTypeLowerName[1336:1351]: TextPunctuation, _TokenTypeLowerName[1332:1342]: TextSymbol,
_TokenTypeName[1342:1357]: TextPunctuation,
_TokenTypeLowerName[1342:1357]: TextPunctuation,
} }
var _TokenTypeNames = []string{ var _TokenTypeNames = []string{
_TokenTypeName[0:4], _TokenTypeName[0:6],
_TokenTypeName[4:9], _TokenTypeName[6:10],
_TokenTypeName[9:14], _TokenTypeName[10:15],
_TokenTypeName[14:22], _TokenTypeName[15:20],
_TokenTypeName[22:30], _TokenTypeName[20:28],
_TokenTypeName[30:41], _TokenTypeName[28:36],
_TokenTypeName[41:50], _TokenTypeName[36:47],
_TokenTypeName[50:63], _TokenTypeName[47:56],
_TokenTypeName[63:79], _TokenTypeName[56:69],
_TokenTypeName[79:90], _TokenTypeName[69:85],
_TokenTypeName[90:94], _TokenTypeName[85:96],
_TokenTypeName[94:104], _TokenTypeName[96:100],
_TokenTypeName[104:114], _TokenTypeName[100:110],
_TokenTypeName[114:121], _TokenTypeName[110:120],
_TokenTypeName[121:128], _TokenTypeName[120:127],
_TokenTypeName[128:143], _TokenTypeName[127:134],
_TokenTypeName[143:161], _TokenTypeName[134:149],
_TokenTypeName[161:177], _TokenTypeName[149:167],
_TokenTypeName[177:190], _TokenTypeName[167:183],
_TokenTypeName[190:205], _TokenTypeName[183:196],
_TokenTypeName[205:216], _TokenTypeName[196:211],
_TokenTypeName[216:220], _TokenTypeName[211:222],
_TokenTypeName[220:233], _TokenTypeName[222:226],
_TokenTypeName[233:244], _TokenTypeName[226:239],
_TokenTypeName[244:261], _TokenTypeName[239:250],
_TokenTypeName[261:270], _TokenTypeName[250:267],
_TokenTypeName[270:282], _TokenTypeName[267:276],
_TokenTypeName[282:295], _TokenTypeName[276:288],
_TokenTypeName[295:305], _TokenTypeName[288:301],
_TokenTypeName[305:318], _TokenTypeName[301:311],
_TokenTypeName[318:330], _TokenTypeName[311:324],
_TokenTypeName[330:347], _TokenTypeName[324:336],
_TokenTypeName[347:358], _TokenTypeName[336:353],
_TokenTypeName[358:367], _TokenTypeName[353:364],
_TokenTypeName[367:380], _TokenTypeName[364:373],
_TokenTypeName[380:392], _TokenTypeName[373:386],
_TokenTypeName[392:401], _TokenTypeName[386:398],
_TokenTypeName[401:411], _TokenTypeName[398:407],
_TokenTypeName[411:423], _TokenTypeName[407:417],
_TokenTypeName[423:430], _TokenTypeName[417:429],
_TokenTypeName[430:442], _TokenTypeName[429:436],
_TokenTypeName[442:463], _TokenTypeName[436:448],
_TokenTypeName[463:480], _TokenTypeName[448:469],
_TokenTypeName[480:498], _TokenTypeName[469:486],
_TokenTypeName[498:518], _TokenTypeName[486:504],
_TokenTypeName[518:535], _TokenTypeName[504:524],
_TokenTypeName[535:542], _TokenTypeName[524:541],
_TokenTypeName[542:553], _TokenTypeName[541:548],
_TokenTypeName[553:565], _TokenTypeName[548:559],
_TokenTypeName[565:578], _TokenTypeName[559:571],
_TokenTypeName[578:596], _TokenTypeName[571:584],
_TokenTypeName[596:613], _TokenTypeName[584:602],
_TokenTypeName[613:634], _TokenTypeName[602:619],
_TokenTypeName[634:654], _TokenTypeName[619:640],
_TokenTypeName[654:671], _TokenTypeName[640:660],
_TokenTypeName[671:693], _TokenTypeName[660:677],
_TokenTypeName[693:709], _TokenTypeName[677:699],
_TokenTypeName[709:728], _TokenTypeName[699:715],
_TokenTypeName[728:747], _TokenTypeName[715:734],
_TokenTypeName[747:767], _TokenTypeName[734:753],
_TokenTypeName[767:788], _TokenTypeName[753:773],
_TokenTypeName[788:805], _TokenTypeName[773:794],
_TokenTypeName[805:823], _TokenTypeName[794:811],
_TokenTypeName[823:841], _TokenTypeName[811:829],
_TokenTypeName[841:860], _TokenTypeName[829:847],
_TokenTypeName[860:879], _TokenTypeName[847:866],
_TokenTypeName[879:892], _TokenTypeName[866:885],
_TokenTypeName[892:908], _TokenTypeName[885:898],
_TokenTypeName[908:926], _TokenTypeName[898:914],
_TokenTypeName[926:942], _TokenTypeName[914:932],
_TokenTypeName[942:962], _TokenTypeName[932:948],
_TokenTypeName[962:986], _TokenTypeName[948:968],
_TokenTypeName[986:1002], _TokenTypeName[968:992],
_TokenTypeName[1002:1019], _TokenTypeName[992:1008],
_TokenTypeName[1019:1027], _TokenTypeName[1008:1025],
_TokenTypeName[1027:1039], _TokenTypeName[1025:1033],
_TokenTypeName[1039:1050], _TokenTypeName[1033:1045],
_TokenTypeName[1050:1057], _TokenTypeName[1045:1056],
_TokenTypeName[1057:1072], _TokenTypeName[1056:1063],
_TokenTypeName[1072:1088], _TokenTypeName[1063:1078],
_TokenTypeName[1088:1101], _TokenTypeName[1078:1094],
_TokenTypeName[1101:1115], _TokenTypeName[1094:1107],
_TokenTypeName[1115:1129], _TokenTypeName[1107:1121],
_TokenTypeName[1129:1147], _TokenTypeName[1121:1135],
_TokenTypeName[1147:1154], _TokenTypeName[1135:1153],
_TokenTypeName[1154:1168], _TokenTypeName[1153:1160],
_TokenTypeName[1168:1179], _TokenTypeName[1160:1174],
_TokenTypeName[1179:1191], _TokenTypeName[1174:1185],
_TokenTypeName[1191:1205], _TokenTypeName[1185:1197],
_TokenTypeName[1205:1220], _TokenTypeName[1197:1211],
_TokenTypeName[1220:1233], _TokenTypeName[1211:1226],
_TokenTypeName[1233:1246], _TokenTypeName[1226:1239],
_TokenTypeName[1246:1259], _TokenTypeName[1239:1252],
_TokenTypeName[1259:1276], _TokenTypeName[1252:1265],
_TokenTypeName[1276:1292], _TokenTypeName[1265:1282],
_TokenTypeName[1292:1308], _TokenTypeName[1282:1298],
_TokenTypeName[1308:1312], _TokenTypeName[1298:1314],
_TokenTypeName[1312:1326], _TokenTypeName[1314:1318],
_TokenTypeName[1326:1336], _TokenTypeName[1318:1332],
_TokenTypeName[1336:1351], _TokenTypeName[1332:1342],
_TokenTypeName[1342:1357],
} }
// TokenTypeString retrieves an enum value from the enum constants string name. // TokenTypeString retrieves an enum value from the enum constants string name.

2
types.go
View File

@ -41,6 +41,8 @@ const (
Other Other
// No highlighting. // No highlighting.
None None
// Don't emit this token to the output.
Ignore
// Used as an EOF marker / nil token // Used as an EOF marker / nil token
EOFType TokenType = 0 EOFType TokenType = 0
) )