diff --git a/lexers/embedded/yaml.xml b/lexers/embedded/yaml.xml index 97a0b6e..0707554 100644 --- a/lexers/embedded/yaml.xml +++ b/lexers/embedded/yaml.xml @@ -53,7 +53,7 @@ - + diff --git a/lexers/testdata/yaml.expected b/lexers/testdata/yaml.expected index a73b48a..9cb7372 100644 --- a/lexers/testdata/yaml.expected +++ b/lexers/testdata/yaml.expected @@ -191,55 +191,55 @@ {"type":"TextWhitespace","value":" "}, {"type":"Punctuation","value":"|"}, {"type":"LiteralStringDoc","value":"\n This entire block of text will be the value of the 'literal_block' key,\n with line breaks being preserved.\n\n The literal continues until de-dented, and the leading indentation is\n stripped.\n\n Any lines that are 'more-indented' keep the rest of their indentation -\n these lines will be indented by 4 spaces."}, - {"type":"TextWhitespace","value":" \n"}, + {"type":"TextWhitespace","value":"\n"}, {"type":"NameTag","value":"folded_style"}, {"type":"Punctuation","value":":"}, {"type":"TextWhitespace","value":" "}, {"type":"Punctuation","value":"\u003e"}, {"type":"LiteralStringDoc","value":"\n This entire block of text will be the value of 'folded_style', but this\n time, all newlines will be replaced with a single space.\n\n Blank lines, like above, are converted to a newline character.\n\n 'More-indented' lines keep their newlines, too -\n this text will appear over two lines."}, - {"type":"TextWhitespace","value":" \n"}, + {"type":"TextWhitespace","value":"\n"}, {"type":"NameTag","value":"literal_block_with_strip_chomping"}, {"type":"Punctuation","value":":"}, {"type":"TextWhitespace","value":" "}, {"type":"Punctuation","value":"|-"}, {"type":"LiteralStringDoc","value":"\n This entire block of text will be the value of the 'literal_block' key,\n with line breaks being preserved and the strip chomping indicator.\n\n The literal continues until de-dented, and the leading indentation is\n stripped.\n\n Any lines that are 'more-indented' keep the rest of their indentation -\n these lines will be indented by 4 spaces."}, - {"type":"TextWhitespace","value":" \n"}, + {"type":"TextWhitespace","value":"\n"}, {"type":"NameTag","value":"literal_block_with_keep_chomping"}, {"type":"Punctuation","value":":"}, {"type":"TextWhitespace","value":" "}, {"type":"Punctuation","value":"|+"}, {"type":"LiteralStringDoc","value":"\n This entire block of text will be the value of the 'literal_block' key,\n with line breaks being preserved and the keep chomping indicator.\n\n The literal continues until de-dented, and the leading indentation is\n stripped."}, - {"type":"TextWhitespace","value":" \n\n"}, + {"type":"TextWhitespace","value":"\n\n"}, {"type":"NameTag","value":"a"}, {"type":"Punctuation","value":":"}, {"type":"TextWhitespace","value":" "}, {"type":"Punctuation","value":"|"}, {"type":"LiteralStringDoc","value":"\n multiline literal\n line 2"}, - {"type":"TextWhitespace","value":" \n"}, + {"type":"TextWhitespace","value":"\n"}, {"type":"NameTag","value":"b"}, {"type":"Punctuation","value":":"}, {"type":"TextWhitespace","value":" "}, {"type":"Punctuation","value":"\u003e"}, {"type":"LiteralStringDoc","value":"\n multiline: folded\n line 2"}, - {"type":"TextWhitespace","value":" \n"}, + {"type":"TextWhitespace","value":"\n"}, {"type":"NameTag","value":"c"}, {"type":"Punctuation","value":":"}, {"type":"TextWhitespace","value":" "}, {"type":"Punctuation","value":"|-"}, {"type":"LiteralStringDoc","value":"\n multiline # literal strip\n line 2"}, - {"type":"TextWhitespace","value":" \n"}, + {"type":"TextWhitespace","value":"\n"}, {"type":"NameTag","value":"d"}, {"type":"Punctuation","value":":"}, {"type":"TextWhitespace","value":" "}, {"type":"Punctuation","value":"\u003e-"}, {"type":"LiteralStringDoc","value":"\n multiline folded strip\n line 2: test\n\n # not a comment\n indented by 1"}, - {"type":"TextWhitespace","value":" \n"}, + {"type":"TextWhitespace","value":"\n"}, {"type":"NameTag","value":"e"}, {"type":"Punctuation","value":":"}, {"type":"TextWhitespace","value":" "}, {"type":"Punctuation","value":"|+"}, {"type":"LiteralStringDoc","value":"\n multiline literal keep\n line: 2"}, - {"type":"TextWhitespace","value":" \n"}, + {"type":"TextWhitespace","value":"\n"}, {"type":"Comment","value":"# this is a comment"}, {"type":"TextWhitespace","value":"\n"}, {"type":"NameTag","value":"f"}, @@ -247,13 +247,13 @@ {"type":"TextWhitespace","value":" "}, {"type":"Punctuation","value":"\u003e+"}, {"type":"LiteralStringDoc","value":"\n multiline folded keep one space\n line 2"}, - {"type":"TextWhitespace","value":" \n"}, + {"type":"TextWhitespace","value":"\n"}, {"type":"NameTag","value":"g"}, {"type":"Punctuation","value":":"}, {"type":"TextWhitespace","value":" "}, {"type":"Punctuation","value":"|"}, {"type":"LiteralStringDoc","value":"\n multiline literal with only one line"}, - {"type":"TextWhitespace","value":" \n"}, + {"type":"TextWhitespace","value":"\n"}, {"type":"NameTag","value":"h"}, {"type":"Punctuation","value":":"}, {"type":"TextWhitespace","value":" "}, @@ -267,31 +267,31 @@ {"type":"TextWhitespace","value":" "}, {"type":"Punctuation","value":"|"}, {"type":"LiteralStringDoc","value":"\n multiline literal\n line 2"}, - {"type":"TextWhitespace","value":" \n "}, + {"type":"TextWhitespace","value":"\n "}, {"type":"NameTag","value":"b"}, {"type":"Punctuation","value":":"}, {"type":"TextWhitespace","value":" "}, {"type":"Punctuation","value":"\u003e"}, {"type":"LiteralStringDoc","value":"\n multiline: folded\n line 2"}, - {"type":"TextWhitespace","value":" \n "}, + {"type":"TextWhitespace","value":"\n "}, {"type":"NameTag","value":"c"}, {"type":"Punctuation","value":":"}, {"type":"TextWhitespace","value":" "}, {"type":"Punctuation","value":"|-"}, {"type":"LiteralStringDoc","value":"\n multiline # literal strip\n line 2 6 leading spaces"}, - {"type":"TextWhitespace","value":" \n\n "}, + {"type":"TextWhitespace","value":"\n\n "}, {"type":"NameTag","value":"d"}, {"type":"Punctuation","value":":"}, {"type":"TextWhitespace","value":" "}, {"type":"Punctuation","value":"\u003e-"}, {"type":"LiteralStringDoc","value":"\n multiline folded strip\n line 2: test\n # not a comment"}, - {"type":"TextWhitespace","value":" \n "}, + {"type":"TextWhitespace","value":"\n "}, {"type":"NameTag","value":"e"}, {"type":"Punctuation","value":":"}, {"type":"TextWhitespace","value":" "}, {"type":"Punctuation","value":"|+"}, {"type":"LiteralStringDoc","value":"\n multiline literal keep\n line: 2"}, - {"type":"TextWhitespace","value":" \n "}, + {"type":"TextWhitespace","value":"\n "}, {"type":"Comment","value":"# this is a comment"}, {"type":"TextWhitespace","value":"\n "}, {"type":"NameTag","value":"f"}, @@ -299,13 +299,13 @@ {"type":"TextWhitespace","value":" "}, {"type":"Punctuation","value":"\u003e+"}, {"type":"LiteralStringDoc","value":"\n multiline folded keep\n line 2"}, - {"type":"TextWhitespace","value":" \n "}, + {"type":"TextWhitespace","value":"\n "}, {"type":"NameTag","value":"g"}, {"type":"Punctuation","value":":"}, {"type":"TextWhitespace","value":" "}, {"type":"Punctuation","value":"|"}, {"type":"LiteralStringDoc","value":"\n multiline literal with only one line"}, - {"type":"TextWhitespace","value":" \n "}, + {"type":"TextWhitespace","value":"\n "}, {"type":"NameTag","value":"h"}, {"type":"Punctuation","value":":"}, {"type":"TextWhitespace","value":" "}, @@ -355,7 +355,7 @@ {"type":"TextWhitespace","value":" "}, {"type":"Punctuation","value":"|"}, {"type":"LiteralStringDoc","value":"\n This is a key\n that has multiple lines"}, - {"type":"TextWhitespace","value":" \n"}, + {"type":"TextWhitespace","value":"\n"}, {"type":"Punctuation","value":":"}, {"type":"TextWhitespace","value":" "}, {"type":"Literal","value":"and this is its value"}, @@ -622,7 +622,7 @@ {"type":"TextWhitespace","value":" "}, {"type":"Punctuation","value":"|"}, {"type":"LiteralStringDoc","value":"\n R0lGODlhDAAMAIQAAP//9/X17unp5WZmZgAAAOfn515eXvPz7Y6OjuDg4J+fn5\n OTk6enp56enmlpaWNjY6Ojo4SEhP/++f/++f/++f/++f/++f/++f/++f/++f/+\n +f/++f/++f/++f/++f/++SH+Dk1hZGUgd2l0aCBHSU1QACwAAAAADAAMAAAFLC\n AgjoEwnuNAFOhpEMTRiggcz4BNJHrv/zCFcLiwMWYNG84BwwEeECcgggoBADs="}, - {"type":"TextWhitespace","value":" \n\n"}, + {"type":"TextWhitespace","value":"\n\n"}, {"type":"Comment","value":"# YAML also has a set type, which looks like this:"}, {"type":"TextWhitespace","value":"\n"}, {"type":"NameTag","value":"set"}, diff --git a/regexp.go b/regexp.go index 0dcb077..5796726 100644 --- a/regexp.go +++ b/regexp.go @@ -194,6 +194,9 @@ func (l *LexerState) Iterator() Token { // nolint: gocognit for len(l.iteratorStack) > 0 { n := len(l.iteratorStack) - 1 t := l.iteratorStack[n]() + if t.Type == Ignore { + continue + } if t == EOF { l.iteratorStack = l.iteratorStack[:n] continue @@ -243,6 +246,9 @@ func (l *LexerState) Iterator() Token { // nolint: gocognit for len(l.iteratorStack) > 0 { n := len(l.iteratorStack) - 1 t := l.iteratorStack[n]() + if t.Type == Ignore { + continue + } if t == EOF { l.iteratorStack = l.iteratorStack[:n] continue diff --git a/regexp_test.go b/regexp_test.go index 3cb746d..01e3a50 100644 --- a/regexp_test.go +++ b/regexp_test.go @@ -192,3 +192,14 @@ func TestByGroupNames(t *testing.T) { assert.NoError(t, err) assert.Equal(t, []Token{{Error, `abc=123`}}, it.Tokens()) } + +func TestIgnoreToken(t *testing.T) { + l := Coalesce(mustNewLexer(t, &Config{EnsureNL: true}, Rules{ // nolint: forbidigo + "root": { + {`(\s*)(\w+)(?:\1)(\n)`, ByGroups(Ignore, Keyword, Whitespace), nil}, + }, + })) + it, err := l.Tokenise(nil, ` hello `) + assert.NoError(t, err) + assert.Equal(t, []Token{{Keyword, "hello"}, {TextWhitespace, "\n"}}, it.Tokens()) +} diff --git a/tokentype_enumer.go b/tokentype_enumer.go index e3bfbea..3024264 100644 --- a/tokentype_enumer.go +++ b/tokentype_enumer.go @@ -7,110 +7,111 @@ import ( "strings" ) -const _TokenTypeName = "NoneOtherErrorCodeLineLineLinkLineTableTDLineTableLineHighlightLineNumbersTableLineNumbersLinePreWrapperBackgroundEOFTypeKeywordKeywordConstantKeywordDeclarationKeywordNamespaceKeywordPseudoKeywordReservedKeywordTypeNameNameAttributeNameBuiltinNameBuiltinPseudoNameClassNameConstantNameDecoratorNameEntityNameExceptionNameFunctionNameFunctionMagicNameKeywordNameLabelNameNamespaceNameOperatorNameOtherNamePseudoNamePropertyNameTagNameVariableNameVariableAnonymousNameVariableClassNameVariableGlobalNameVariableInstanceNameVariableMagicLiteralLiteralDateLiteralOtherLiteralStringLiteralStringAffixLiteralStringAtomLiteralStringBacktickLiteralStringBooleanLiteralStringCharLiteralStringDelimiterLiteralStringDocLiteralStringDoubleLiteralStringEscapeLiteralStringHeredocLiteralStringInterpolLiteralStringNameLiteralStringOtherLiteralStringRegexLiteralStringSingleLiteralStringSymbolLiteralNumberLiteralNumberBinLiteralNumberFloatLiteralNumberHexLiteralNumberIntegerLiteralNumberIntegerLongLiteralNumberOctLiteralNumberByteOperatorOperatorWordPunctuationCommentCommentHashbangCommentMultilineCommentSingleCommentSpecialCommentPreprocCommentPreprocFileGenericGenericDeletedGenericEmphGenericErrorGenericHeadingGenericInsertedGenericOutputGenericPromptGenericStrongGenericSubheadingGenericTracebackGenericUnderlineTextTextWhitespaceTextSymbolTextPunctuation" -const _TokenTypeLowerName = "noneothererrorcodelinelinelinklinetabletdlinetablelinehighlightlinenumberstablelinenumberslineprewrapperbackgroundeoftypekeywordkeywordconstantkeyworddeclarationkeywordnamespacekeywordpseudokeywordreservedkeywordtypenamenameattributenamebuiltinnamebuiltinpseudonameclassnameconstantnamedecoratornameentitynameexceptionnamefunctionnamefunctionmagicnamekeywordnamelabelnamenamespacenameoperatornameothernamepseudonamepropertynametagnamevariablenamevariableanonymousnamevariableclassnamevariableglobalnamevariableinstancenamevariablemagicliteralliteraldateliteralotherliteralstringliteralstringaffixliteralstringatomliteralstringbacktickliteralstringbooleanliteralstringcharliteralstringdelimiterliteralstringdocliteralstringdoubleliteralstringescapeliteralstringheredocliteralstringinterpolliteralstringnameliteralstringotherliteralstringregexliteralstringsingleliteralstringsymbolliteralnumberliteralnumberbinliteralnumberfloatliteralnumberhexliteralnumberintegerliteralnumberintegerlongliteralnumberoctliteralnumberbyteoperatoroperatorwordpunctuationcommentcommenthashbangcommentmultilinecommentsinglecommentspecialcommentpreproccommentpreprocfilegenericgenericdeletedgenericemphgenericerrorgenericheadinggenericinsertedgenericoutputgenericpromptgenericstronggenericsubheadinggenerictracebackgenericunderlinetexttextwhitespacetextsymboltextpunctuation" +const _TokenTypeName = "IgnoreNoneOtherErrorCodeLineLineLinkLineTableTDLineTableLineHighlightLineNumbersTableLineNumbersLinePreWrapperBackgroundEOFTypeKeywordKeywordConstantKeywordDeclarationKeywordNamespaceKeywordPseudoKeywordReservedKeywordTypeNameNameAttributeNameBuiltinNameBuiltinPseudoNameClassNameConstantNameDecoratorNameEntityNameExceptionNameFunctionNameFunctionMagicNameKeywordNameLabelNameNamespaceNameOperatorNameOtherNamePseudoNamePropertyNameTagNameVariableNameVariableAnonymousNameVariableClassNameVariableGlobalNameVariableInstanceNameVariableMagicLiteralLiteralDateLiteralOtherLiteralStringLiteralStringAffixLiteralStringAtomLiteralStringBacktickLiteralStringBooleanLiteralStringCharLiteralStringDelimiterLiteralStringDocLiteralStringDoubleLiteralStringEscapeLiteralStringHeredocLiteralStringInterpolLiteralStringNameLiteralStringOtherLiteralStringRegexLiteralStringSingleLiteralStringSymbolLiteralNumberLiteralNumberBinLiteralNumberFloatLiteralNumberHexLiteralNumberIntegerLiteralNumberIntegerLongLiteralNumberOctLiteralNumberByteOperatorOperatorWordPunctuationCommentCommentHashbangCommentMultilineCommentSingleCommentSpecialCommentPreprocCommentPreprocFileGenericGenericDeletedGenericEmphGenericErrorGenericHeadingGenericInsertedGenericOutputGenericPromptGenericStrongGenericSubheadingGenericTracebackGenericUnderlineTextTextWhitespaceTextSymbolTextPunctuation" +const _TokenTypeLowerName = "ignorenoneothererrorcodelinelinelinklinetabletdlinetablelinehighlightlinenumberstablelinenumberslineprewrapperbackgroundeoftypekeywordkeywordconstantkeyworddeclarationkeywordnamespacekeywordpseudokeywordreservedkeywordtypenamenameattributenamebuiltinnamebuiltinpseudonameclassnameconstantnamedecoratornameentitynameexceptionnamefunctionnamefunctionmagicnamekeywordnamelabelnamenamespacenameoperatornameothernamepseudonamepropertynametagnamevariablenamevariableanonymousnamevariableclassnamevariableglobalnamevariableinstancenamevariablemagicliteralliteraldateliteralotherliteralstringliteralstringaffixliteralstringatomliteralstringbacktickliteralstringbooleanliteralstringcharliteralstringdelimiterliteralstringdocliteralstringdoubleliteralstringescapeliteralstringheredocliteralstringinterpolliteralstringnameliteralstringotherliteralstringregexliteralstringsingleliteralstringsymbolliteralnumberliteralnumberbinliteralnumberfloatliteralnumberhexliteralnumberintegerliteralnumberintegerlongliteralnumberoctliteralnumberbyteoperatoroperatorwordpunctuationcommentcommenthashbangcommentmultilinecommentsinglecommentspecialcommentpreproccommentpreprocfilegenericgenericdeletedgenericemphgenericerrorgenericheadinggenericinsertedgenericoutputgenericpromptgenericstronggenericsubheadinggenerictracebackgenericunderlinetexttextwhitespacetextsymboltextpunctuation" var _TokenTypeMap = map[TokenType]string{ - -13: _TokenTypeName[0:4], - -12: _TokenTypeName[4:9], - -11: _TokenTypeName[9:14], - -10: _TokenTypeName[14:22], - -9: _TokenTypeName[22:30], - -8: _TokenTypeName[30:41], - -7: _TokenTypeName[41:50], - -6: _TokenTypeName[50:63], - -5: _TokenTypeName[63:79], - -4: _TokenTypeName[79:90], - -3: _TokenTypeName[90:94], - -2: _TokenTypeName[94:104], - -1: _TokenTypeName[104:114], - 0: _TokenTypeName[114:121], - 1000: _TokenTypeName[121:128], - 1001: _TokenTypeName[128:143], - 1002: _TokenTypeName[143:161], - 1003: _TokenTypeName[161:177], - 1004: _TokenTypeName[177:190], - 1005: _TokenTypeName[190:205], - 1006: _TokenTypeName[205:216], - 2000: _TokenTypeName[216:220], - 2001: _TokenTypeName[220:233], - 2002: _TokenTypeName[233:244], - 2003: _TokenTypeName[244:261], - 2004: _TokenTypeName[261:270], - 2005: _TokenTypeName[270:282], - 2006: _TokenTypeName[282:295], - 2007: _TokenTypeName[295:305], - 2008: _TokenTypeName[305:318], - 2009: _TokenTypeName[318:330], - 2010: _TokenTypeName[330:347], - 2011: _TokenTypeName[347:358], - 2012: _TokenTypeName[358:367], - 2013: _TokenTypeName[367:380], - 2014: _TokenTypeName[380:392], - 2015: _TokenTypeName[392:401], - 2016: _TokenTypeName[401:411], - 2017: _TokenTypeName[411:423], - 2018: _TokenTypeName[423:430], - 2019: _TokenTypeName[430:442], - 2020: _TokenTypeName[442:463], - 2021: _TokenTypeName[463:480], - 2022: _TokenTypeName[480:498], - 2023: _TokenTypeName[498:518], - 2024: _TokenTypeName[518:535], - 3000: _TokenTypeName[535:542], - 3001: _TokenTypeName[542:553], - 3002: _TokenTypeName[553:565], - 3100: _TokenTypeName[565:578], - 3101: _TokenTypeName[578:596], - 3102: _TokenTypeName[596:613], - 3103: _TokenTypeName[613:634], - 3104: _TokenTypeName[634:654], - 3105: _TokenTypeName[654:671], - 3106: _TokenTypeName[671:693], - 3107: _TokenTypeName[693:709], - 3108: _TokenTypeName[709:728], - 3109: _TokenTypeName[728:747], - 3110: _TokenTypeName[747:767], - 3111: _TokenTypeName[767:788], - 3112: _TokenTypeName[788:805], - 3113: _TokenTypeName[805:823], - 3114: _TokenTypeName[823:841], - 3115: _TokenTypeName[841:860], - 3116: _TokenTypeName[860:879], - 3200: _TokenTypeName[879:892], - 3201: _TokenTypeName[892:908], - 3202: _TokenTypeName[908:926], - 3203: _TokenTypeName[926:942], - 3204: _TokenTypeName[942:962], - 3205: _TokenTypeName[962:986], - 3206: _TokenTypeName[986:1002], - 3207: _TokenTypeName[1002:1019], - 4000: _TokenTypeName[1019:1027], - 4001: _TokenTypeName[1027:1039], - 5000: _TokenTypeName[1039:1050], - 6000: _TokenTypeName[1050:1057], - 6001: _TokenTypeName[1057:1072], - 6002: _TokenTypeName[1072:1088], - 6003: _TokenTypeName[1088:1101], - 6004: _TokenTypeName[1101:1115], - 6100: _TokenTypeName[1115:1129], - 6101: _TokenTypeName[1129:1147], - 7000: _TokenTypeName[1147:1154], - 7001: _TokenTypeName[1154:1168], - 7002: _TokenTypeName[1168:1179], - 7003: _TokenTypeName[1179:1191], - 7004: _TokenTypeName[1191:1205], - 7005: _TokenTypeName[1205:1220], - 7006: _TokenTypeName[1220:1233], - 7007: _TokenTypeName[1233:1246], - 7008: _TokenTypeName[1246:1259], - 7009: _TokenTypeName[1259:1276], - 7010: _TokenTypeName[1276:1292], - 7011: _TokenTypeName[1292:1308], - 8000: _TokenTypeName[1308:1312], - 8001: _TokenTypeName[1312:1326], - 8002: _TokenTypeName[1326:1336], - 8003: _TokenTypeName[1336:1351], + -14: _TokenTypeName[0:6], + -13: _TokenTypeName[6:10], + -12: _TokenTypeName[10:15], + -11: _TokenTypeName[15:20], + -10: _TokenTypeName[20:28], + -9: _TokenTypeName[28:36], + -8: _TokenTypeName[36:47], + -7: _TokenTypeName[47:56], + -6: _TokenTypeName[56:69], + -5: _TokenTypeName[69:85], + -4: _TokenTypeName[85:96], + -3: _TokenTypeName[96:100], + -2: _TokenTypeName[100:110], + -1: _TokenTypeName[110:120], + 0: _TokenTypeName[120:127], + 1000: _TokenTypeName[127:134], + 1001: _TokenTypeName[134:149], + 1002: _TokenTypeName[149:167], + 1003: _TokenTypeName[167:183], + 1004: _TokenTypeName[183:196], + 1005: _TokenTypeName[196:211], + 1006: _TokenTypeName[211:222], + 2000: _TokenTypeName[222:226], + 2001: _TokenTypeName[226:239], + 2002: _TokenTypeName[239:250], + 2003: _TokenTypeName[250:267], + 2004: _TokenTypeName[267:276], + 2005: _TokenTypeName[276:288], + 2006: _TokenTypeName[288:301], + 2007: _TokenTypeName[301:311], + 2008: _TokenTypeName[311:324], + 2009: _TokenTypeName[324:336], + 2010: _TokenTypeName[336:353], + 2011: _TokenTypeName[353:364], + 2012: _TokenTypeName[364:373], + 2013: _TokenTypeName[373:386], + 2014: _TokenTypeName[386:398], + 2015: _TokenTypeName[398:407], + 2016: _TokenTypeName[407:417], + 2017: _TokenTypeName[417:429], + 2018: _TokenTypeName[429:436], + 2019: _TokenTypeName[436:448], + 2020: _TokenTypeName[448:469], + 2021: _TokenTypeName[469:486], + 2022: _TokenTypeName[486:504], + 2023: _TokenTypeName[504:524], + 2024: _TokenTypeName[524:541], + 3000: _TokenTypeName[541:548], + 3001: _TokenTypeName[548:559], + 3002: _TokenTypeName[559:571], + 3100: _TokenTypeName[571:584], + 3101: _TokenTypeName[584:602], + 3102: _TokenTypeName[602:619], + 3103: _TokenTypeName[619:640], + 3104: _TokenTypeName[640:660], + 3105: _TokenTypeName[660:677], + 3106: _TokenTypeName[677:699], + 3107: _TokenTypeName[699:715], + 3108: _TokenTypeName[715:734], + 3109: _TokenTypeName[734:753], + 3110: _TokenTypeName[753:773], + 3111: _TokenTypeName[773:794], + 3112: _TokenTypeName[794:811], + 3113: _TokenTypeName[811:829], + 3114: _TokenTypeName[829:847], + 3115: _TokenTypeName[847:866], + 3116: _TokenTypeName[866:885], + 3200: _TokenTypeName[885:898], + 3201: _TokenTypeName[898:914], + 3202: _TokenTypeName[914:932], + 3203: _TokenTypeName[932:948], + 3204: _TokenTypeName[948:968], + 3205: _TokenTypeName[968:992], + 3206: _TokenTypeName[992:1008], + 3207: _TokenTypeName[1008:1025], + 4000: _TokenTypeName[1025:1033], + 4001: _TokenTypeName[1033:1045], + 5000: _TokenTypeName[1045:1056], + 6000: _TokenTypeName[1056:1063], + 6001: _TokenTypeName[1063:1078], + 6002: _TokenTypeName[1078:1094], + 6003: _TokenTypeName[1094:1107], + 6004: _TokenTypeName[1107:1121], + 6100: _TokenTypeName[1121:1135], + 6101: _TokenTypeName[1135:1153], + 7000: _TokenTypeName[1153:1160], + 7001: _TokenTypeName[1160:1174], + 7002: _TokenTypeName[1174:1185], + 7003: _TokenTypeName[1185:1197], + 7004: _TokenTypeName[1197:1211], + 7005: _TokenTypeName[1211:1226], + 7006: _TokenTypeName[1226:1239], + 7007: _TokenTypeName[1239:1252], + 7008: _TokenTypeName[1252:1265], + 7009: _TokenTypeName[1265:1282], + 7010: _TokenTypeName[1282:1298], + 7011: _TokenTypeName[1298:1314], + 8000: _TokenTypeName[1314:1318], + 8001: _TokenTypeName[1318:1332], + 8002: _TokenTypeName[1332:1342], + 8003: _TokenTypeName[1342:1357], } func (i TokenType) String() string { @@ -124,6 +125,7 @@ func (i TokenType) String() string { // Re-run the stringer command to generate them again. func _TokenTypeNoOp() { var x [1]struct{} + _ = x[Ignore-(-14)] _ = x[None-(-13)] _ = x[Other-(-12)] _ = x[Error-(-11)] @@ -226,312 +228,315 @@ func _TokenTypeNoOp() { _ = x[TextPunctuation-(8003)] } -var _TokenTypeValues = []TokenType{None, Other, Error, CodeLine, LineLink, LineTableTD, LineTable, LineHighlight, LineNumbersTable, LineNumbers, Line, PreWrapper, Background, EOFType, Keyword, KeywordConstant, KeywordDeclaration, KeywordNamespace, KeywordPseudo, KeywordReserved, KeywordType, Name, NameAttribute, NameBuiltin, NameBuiltinPseudo, NameClass, NameConstant, NameDecorator, NameEntity, NameException, NameFunction, NameFunctionMagic, NameKeyword, NameLabel, NameNamespace, NameOperator, NameOther, NamePseudo, NameProperty, NameTag, NameVariable, NameVariableAnonymous, NameVariableClass, NameVariableGlobal, NameVariableInstance, NameVariableMagic, Literal, LiteralDate, LiteralOther, LiteralString, LiteralStringAffix, LiteralStringAtom, LiteralStringBacktick, LiteralStringBoolean, LiteralStringChar, LiteralStringDelimiter, LiteralStringDoc, LiteralStringDouble, LiteralStringEscape, LiteralStringHeredoc, LiteralStringInterpol, LiteralStringName, LiteralStringOther, LiteralStringRegex, LiteralStringSingle, LiteralStringSymbol, LiteralNumber, LiteralNumberBin, LiteralNumberFloat, LiteralNumberHex, LiteralNumberInteger, LiteralNumberIntegerLong, LiteralNumberOct, LiteralNumberByte, Operator, OperatorWord, Punctuation, Comment, CommentHashbang, CommentMultiline, CommentSingle, CommentSpecial, CommentPreproc, CommentPreprocFile, Generic, GenericDeleted, GenericEmph, GenericError, GenericHeading, GenericInserted, GenericOutput, GenericPrompt, GenericStrong, GenericSubheading, GenericTraceback, GenericUnderline, Text, TextWhitespace, TextSymbol, TextPunctuation} +var _TokenTypeValues = []TokenType{Ignore, None, Other, Error, CodeLine, LineLink, LineTableTD, LineTable, LineHighlight, LineNumbersTable, LineNumbers, Line, PreWrapper, Background, EOFType, Keyword, KeywordConstant, KeywordDeclaration, KeywordNamespace, KeywordPseudo, KeywordReserved, KeywordType, Name, NameAttribute, NameBuiltin, NameBuiltinPseudo, NameClass, NameConstant, NameDecorator, NameEntity, NameException, NameFunction, NameFunctionMagic, NameKeyword, NameLabel, NameNamespace, NameOperator, NameOther, NamePseudo, NameProperty, NameTag, NameVariable, NameVariableAnonymous, NameVariableClass, NameVariableGlobal, NameVariableInstance, NameVariableMagic, Literal, LiteralDate, LiteralOther, LiteralString, LiteralStringAffix, LiteralStringAtom, LiteralStringBacktick, LiteralStringBoolean, LiteralStringChar, LiteralStringDelimiter, LiteralStringDoc, LiteralStringDouble, LiteralStringEscape, LiteralStringHeredoc, LiteralStringInterpol, LiteralStringName, LiteralStringOther, LiteralStringRegex, LiteralStringSingle, LiteralStringSymbol, LiteralNumber, LiteralNumberBin, LiteralNumberFloat, LiteralNumberHex, LiteralNumberInteger, LiteralNumberIntegerLong, LiteralNumberOct, LiteralNumberByte, Operator, OperatorWord, Punctuation, Comment, CommentHashbang, CommentMultiline, CommentSingle, CommentSpecial, CommentPreproc, CommentPreprocFile, Generic, GenericDeleted, GenericEmph, GenericError, GenericHeading, GenericInserted, GenericOutput, GenericPrompt, GenericStrong, GenericSubheading, GenericTraceback, GenericUnderline, Text, TextWhitespace, TextSymbol, TextPunctuation} var _TokenTypeNameToValueMap = map[string]TokenType{ - _TokenTypeName[0:4]: None, - _TokenTypeLowerName[0:4]: None, - _TokenTypeName[4:9]: Other, - _TokenTypeLowerName[4:9]: Other, - _TokenTypeName[9:14]: Error, - _TokenTypeLowerName[9:14]: Error, - _TokenTypeName[14:22]: CodeLine, - _TokenTypeLowerName[14:22]: CodeLine, - _TokenTypeName[22:30]: LineLink, - _TokenTypeLowerName[22:30]: LineLink, - _TokenTypeName[30:41]: LineTableTD, - _TokenTypeLowerName[30:41]: LineTableTD, - _TokenTypeName[41:50]: LineTable, - _TokenTypeLowerName[41:50]: LineTable, - _TokenTypeName[50:63]: LineHighlight, - _TokenTypeLowerName[50:63]: LineHighlight, - _TokenTypeName[63:79]: LineNumbersTable, - _TokenTypeLowerName[63:79]: LineNumbersTable, - _TokenTypeName[79:90]: LineNumbers, - _TokenTypeLowerName[79:90]: LineNumbers, - _TokenTypeName[90:94]: Line, - _TokenTypeLowerName[90:94]: Line, - _TokenTypeName[94:104]: PreWrapper, - _TokenTypeLowerName[94:104]: PreWrapper, - _TokenTypeName[104:114]: Background, - _TokenTypeLowerName[104:114]: Background, - _TokenTypeName[114:121]: EOFType, - _TokenTypeLowerName[114:121]: EOFType, - _TokenTypeName[121:128]: Keyword, - _TokenTypeLowerName[121:128]: Keyword, - _TokenTypeName[128:143]: KeywordConstant, - _TokenTypeLowerName[128:143]: KeywordConstant, - _TokenTypeName[143:161]: KeywordDeclaration, - _TokenTypeLowerName[143:161]: KeywordDeclaration, - _TokenTypeName[161:177]: KeywordNamespace, - _TokenTypeLowerName[161:177]: KeywordNamespace, - _TokenTypeName[177:190]: KeywordPseudo, - _TokenTypeLowerName[177:190]: KeywordPseudo, - _TokenTypeName[190:205]: KeywordReserved, - _TokenTypeLowerName[190:205]: KeywordReserved, - _TokenTypeName[205:216]: KeywordType, - _TokenTypeLowerName[205:216]: KeywordType, - _TokenTypeName[216:220]: Name, - _TokenTypeLowerName[216:220]: Name, - _TokenTypeName[220:233]: NameAttribute, - _TokenTypeLowerName[220:233]: NameAttribute, - _TokenTypeName[233:244]: NameBuiltin, - _TokenTypeLowerName[233:244]: NameBuiltin, - _TokenTypeName[244:261]: NameBuiltinPseudo, - _TokenTypeLowerName[244:261]: NameBuiltinPseudo, - _TokenTypeName[261:270]: NameClass, - _TokenTypeLowerName[261:270]: NameClass, - _TokenTypeName[270:282]: NameConstant, - _TokenTypeLowerName[270:282]: NameConstant, - _TokenTypeName[282:295]: NameDecorator, - _TokenTypeLowerName[282:295]: NameDecorator, - _TokenTypeName[295:305]: NameEntity, - _TokenTypeLowerName[295:305]: NameEntity, - _TokenTypeName[305:318]: NameException, - _TokenTypeLowerName[305:318]: NameException, - _TokenTypeName[318:330]: NameFunction, - _TokenTypeLowerName[318:330]: NameFunction, - _TokenTypeName[330:347]: NameFunctionMagic, - _TokenTypeLowerName[330:347]: NameFunctionMagic, - _TokenTypeName[347:358]: NameKeyword, - _TokenTypeLowerName[347:358]: NameKeyword, - _TokenTypeName[358:367]: NameLabel, - _TokenTypeLowerName[358:367]: NameLabel, - _TokenTypeName[367:380]: NameNamespace, - _TokenTypeLowerName[367:380]: NameNamespace, - _TokenTypeName[380:392]: NameOperator, - _TokenTypeLowerName[380:392]: NameOperator, - _TokenTypeName[392:401]: NameOther, - _TokenTypeLowerName[392:401]: NameOther, - _TokenTypeName[401:411]: NamePseudo, - _TokenTypeLowerName[401:411]: NamePseudo, - _TokenTypeName[411:423]: NameProperty, - _TokenTypeLowerName[411:423]: NameProperty, - _TokenTypeName[423:430]: NameTag, - _TokenTypeLowerName[423:430]: NameTag, - _TokenTypeName[430:442]: NameVariable, - _TokenTypeLowerName[430:442]: NameVariable, - _TokenTypeName[442:463]: NameVariableAnonymous, - _TokenTypeLowerName[442:463]: NameVariableAnonymous, - _TokenTypeName[463:480]: NameVariableClass, - _TokenTypeLowerName[463:480]: NameVariableClass, - _TokenTypeName[480:498]: NameVariableGlobal, - _TokenTypeLowerName[480:498]: NameVariableGlobal, - _TokenTypeName[498:518]: NameVariableInstance, - _TokenTypeLowerName[498:518]: NameVariableInstance, - _TokenTypeName[518:535]: NameVariableMagic, - _TokenTypeLowerName[518:535]: NameVariableMagic, - _TokenTypeName[535:542]: Literal, - _TokenTypeLowerName[535:542]: Literal, - _TokenTypeName[542:553]: LiteralDate, - _TokenTypeLowerName[542:553]: LiteralDate, - _TokenTypeName[553:565]: LiteralOther, - _TokenTypeLowerName[553:565]: LiteralOther, - _TokenTypeName[565:578]: LiteralString, - _TokenTypeLowerName[565:578]: LiteralString, - _TokenTypeName[578:596]: LiteralStringAffix, - _TokenTypeLowerName[578:596]: LiteralStringAffix, - _TokenTypeName[596:613]: LiteralStringAtom, - _TokenTypeLowerName[596:613]: LiteralStringAtom, - _TokenTypeName[613:634]: LiteralStringBacktick, - _TokenTypeLowerName[613:634]: LiteralStringBacktick, - _TokenTypeName[634:654]: LiteralStringBoolean, - _TokenTypeLowerName[634:654]: LiteralStringBoolean, - _TokenTypeName[654:671]: LiteralStringChar, - _TokenTypeLowerName[654:671]: LiteralStringChar, - _TokenTypeName[671:693]: LiteralStringDelimiter, - _TokenTypeLowerName[671:693]: LiteralStringDelimiter, - _TokenTypeName[693:709]: LiteralStringDoc, - _TokenTypeLowerName[693:709]: LiteralStringDoc, - _TokenTypeName[709:728]: LiteralStringDouble, - _TokenTypeLowerName[709:728]: LiteralStringDouble, - _TokenTypeName[728:747]: LiteralStringEscape, - _TokenTypeLowerName[728:747]: LiteralStringEscape, - _TokenTypeName[747:767]: LiteralStringHeredoc, - _TokenTypeLowerName[747:767]: LiteralStringHeredoc, - _TokenTypeName[767:788]: LiteralStringInterpol, - _TokenTypeLowerName[767:788]: LiteralStringInterpol, - _TokenTypeName[788:805]: LiteralStringName, - _TokenTypeLowerName[788:805]: LiteralStringName, - _TokenTypeName[805:823]: LiteralStringOther, - _TokenTypeLowerName[805:823]: LiteralStringOther, - _TokenTypeName[823:841]: LiteralStringRegex, - _TokenTypeLowerName[823:841]: LiteralStringRegex, - _TokenTypeName[841:860]: LiteralStringSingle, - _TokenTypeLowerName[841:860]: LiteralStringSingle, - _TokenTypeName[860:879]: LiteralStringSymbol, - _TokenTypeLowerName[860:879]: LiteralStringSymbol, - _TokenTypeName[879:892]: LiteralNumber, - _TokenTypeLowerName[879:892]: LiteralNumber, - _TokenTypeName[892:908]: LiteralNumberBin, - _TokenTypeLowerName[892:908]: LiteralNumberBin, - _TokenTypeName[908:926]: LiteralNumberFloat, - _TokenTypeLowerName[908:926]: LiteralNumberFloat, - _TokenTypeName[926:942]: LiteralNumberHex, - _TokenTypeLowerName[926:942]: LiteralNumberHex, - _TokenTypeName[942:962]: LiteralNumberInteger, - _TokenTypeLowerName[942:962]: LiteralNumberInteger, - _TokenTypeName[962:986]: LiteralNumberIntegerLong, - _TokenTypeLowerName[962:986]: LiteralNumberIntegerLong, - _TokenTypeName[986:1002]: LiteralNumberOct, - _TokenTypeLowerName[986:1002]: LiteralNumberOct, - _TokenTypeName[1002:1019]: LiteralNumberByte, - _TokenTypeLowerName[1002:1019]: LiteralNumberByte, - _TokenTypeName[1019:1027]: Operator, - _TokenTypeLowerName[1019:1027]: Operator, - _TokenTypeName[1027:1039]: OperatorWord, - _TokenTypeLowerName[1027:1039]: OperatorWord, - _TokenTypeName[1039:1050]: Punctuation, - _TokenTypeLowerName[1039:1050]: Punctuation, - _TokenTypeName[1050:1057]: Comment, - _TokenTypeLowerName[1050:1057]: Comment, - _TokenTypeName[1057:1072]: CommentHashbang, - _TokenTypeLowerName[1057:1072]: CommentHashbang, - _TokenTypeName[1072:1088]: CommentMultiline, - _TokenTypeLowerName[1072:1088]: CommentMultiline, - _TokenTypeName[1088:1101]: CommentSingle, - _TokenTypeLowerName[1088:1101]: CommentSingle, - _TokenTypeName[1101:1115]: CommentSpecial, - _TokenTypeLowerName[1101:1115]: CommentSpecial, - _TokenTypeName[1115:1129]: CommentPreproc, - _TokenTypeLowerName[1115:1129]: CommentPreproc, - _TokenTypeName[1129:1147]: CommentPreprocFile, - _TokenTypeLowerName[1129:1147]: CommentPreprocFile, - _TokenTypeName[1147:1154]: Generic, - _TokenTypeLowerName[1147:1154]: Generic, - _TokenTypeName[1154:1168]: GenericDeleted, - _TokenTypeLowerName[1154:1168]: GenericDeleted, - _TokenTypeName[1168:1179]: GenericEmph, - _TokenTypeLowerName[1168:1179]: GenericEmph, - _TokenTypeName[1179:1191]: GenericError, - _TokenTypeLowerName[1179:1191]: GenericError, - _TokenTypeName[1191:1205]: GenericHeading, - _TokenTypeLowerName[1191:1205]: GenericHeading, - _TokenTypeName[1205:1220]: GenericInserted, - _TokenTypeLowerName[1205:1220]: GenericInserted, - _TokenTypeName[1220:1233]: GenericOutput, - _TokenTypeLowerName[1220:1233]: GenericOutput, - _TokenTypeName[1233:1246]: GenericPrompt, - _TokenTypeLowerName[1233:1246]: GenericPrompt, - _TokenTypeName[1246:1259]: GenericStrong, - _TokenTypeLowerName[1246:1259]: GenericStrong, - _TokenTypeName[1259:1276]: GenericSubheading, - _TokenTypeLowerName[1259:1276]: GenericSubheading, - _TokenTypeName[1276:1292]: GenericTraceback, - _TokenTypeLowerName[1276:1292]: GenericTraceback, - _TokenTypeName[1292:1308]: GenericUnderline, - _TokenTypeLowerName[1292:1308]: GenericUnderline, - _TokenTypeName[1308:1312]: Text, - _TokenTypeLowerName[1308:1312]: Text, - _TokenTypeName[1312:1326]: TextWhitespace, - _TokenTypeLowerName[1312:1326]: TextWhitespace, - _TokenTypeName[1326:1336]: TextSymbol, - _TokenTypeLowerName[1326:1336]: TextSymbol, - _TokenTypeName[1336:1351]: TextPunctuation, - _TokenTypeLowerName[1336:1351]: TextPunctuation, + _TokenTypeName[0:6]: Ignore, + _TokenTypeLowerName[0:6]: Ignore, + _TokenTypeName[6:10]: None, + _TokenTypeLowerName[6:10]: None, + _TokenTypeName[10:15]: Other, + _TokenTypeLowerName[10:15]: Other, + _TokenTypeName[15:20]: Error, + _TokenTypeLowerName[15:20]: Error, + _TokenTypeName[20:28]: CodeLine, + _TokenTypeLowerName[20:28]: CodeLine, + _TokenTypeName[28:36]: LineLink, + _TokenTypeLowerName[28:36]: LineLink, + _TokenTypeName[36:47]: LineTableTD, + _TokenTypeLowerName[36:47]: LineTableTD, + _TokenTypeName[47:56]: LineTable, + _TokenTypeLowerName[47:56]: LineTable, + _TokenTypeName[56:69]: LineHighlight, + _TokenTypeLowerName[56:69]: LineHighlight, + _TokenTypeName[69:85]: LineNumbersTable, + _TokenTypeLowerName[69:85]: LineNumbersTable, + _TokenTypeName[85:96]: LineNumbers, + _TokenTypeLowerName[85:96]: LineNumbers, + _TokenTypeName[96:100]: Line, + _TokenTypeLowerName[96:100]: Line, + _TokenTypeName[100:110]: PreWrapper, + _TokenTypeLowerName[100:110]: PreWrapper, + _TokenTypeName[110:120]: Background, + _TokenTypeLowerName[110:120]: Background, + _TokenTypeName[120:127]: EOFType, + _TokenTypeLowerName[120:127]: EOFType, + _TokenTypeName[127:134]: Keyword, + _TokenTypeLowerName[127:134]: Keyword, + _TokenTypeName[134:149]: KeywordConstant, + _TokenTypeLowerName[134:149]: KeywordConstant, + _TokenTypeName[149:167]: KeywordDeclaration, + _TokenTypeLowerName[149:167]: KeywordDeclaration, + _TokenTypeName[167:183]: KeywordNamespace, + _TokenTypeLowerName[167:183]: KeywordNamespace, + _TokenTypeName[183:196]: KeywordPseudo, + _TokenTypeLowerName[183:196]: KeywordPseudo, + _TokenTypeName[196:211]: KeywordReserved, + _TokenTypeLowerName[196:211]: KeywordReserved, + _TokenTypeName[211:222]: KeywordType, + _TokenTypeLowerName[211:222]: KeywordType, + _TokenTypeName[222:226]: Name, + _TokenTypeLowerName[222:226]: Name, + _TokenTypeName[226:239]: NameAttribute, + _TokenTypeLowerName[226:239]: NameAttribute, + _TokenTypeName[239:250]: NameBuiltin, + _TokenTypeLowerName[239:250]: NameBuiltin, + _TokenTypeName[250:267]: NameBuiltinPseudo, + _TokenTypeLowerName[250:267]: NameBuiltinPseudo, + _TokenTypeName[267:276]: NameClass, + _TokenTypeLowerName[267:276]: NameClass, + _TokenTypeName[276:288]: NameConstant, + _TokenTypeLowerName[276:288]: NameConstant, + _TokenTypeName[288:301]: NameDecorator, + _TokenTypeLowerName[288:301]: NameDecorator, + _TokenTypeName[301:311]: NameEntity, + _TokenTypeLowerName[301:311]: NameEntity, + _TokenTypeName[311:324]: NameException, + _TokenTypeLowerName[311:324]: NameException, + _TokenTypeName[324:336]: NameFunction, + _TokenTypeLowerName[324:336]: NameFunction, + _TokenTypeName[336:353]: NameFunctionMagic, + _TokenTypeLowerName[336:353]: NameFunctionMagic, + _TokenTypeName[353:364]: NameKeyword, + _TokenTypeLowerName[353:364]: NameKeyword, + _TokenTypeName[364:373]: NameLabel, + _TokenTypeLowerName[364:373]: NameLabel, + _TokenTypeName[373:386]: NameNamespace, + _TokenTypeLowerName[373:386]: NameNamespace, + _TokenTypeName[386:398]: NameOperator, + _TokenTypeLowerName[386:398]: NameOperator, + _TokenTypeName[398:407]: NameOther, + _TokenTypeLowerName[398:407]: NameOther, + _TokenTypeName[407:417]: NamePseudo, + _TokenTypeLowerName[407:417]: NamePseudo, + _TokenTypeName[417:429]: NameProperty, + _TokenTypeLowerName[417:429]: NameProperty, + _TokenTypeName[429:436]: NameTag, + _TokenTypeLowerName[429:436]: NameTag, + _TokenTypeName[436:448]: NameVariable, + _TokenTypeLowerName[436:448]: NameVariable, + _TokenTypeName[448:469]: NameVariableAnonymous, + _TokenTypeLowerName[448:469]: NameVariableAnonymous, + _TokenTypeName[469:486]: NameVariableClass, + _TokenTypeLowerName[469:486]: NameVariableClass, + _TokenTypeName[486:504]: NameVariableGlobal, + _TokenTypeLowerName[486:504]: NameVariableGlobal, + _TokenTypeName[504:524]: NameVariableInstance, + _TokenTypeLowerName[504:524]: NameVariableInstance, + _TokenTypeName[524:541]: NameVariableMagic, + _TokenTypeLowerName[524:541]: NameVariableMagic, + _TokenTypeName[541:548]: Literal, + _TokenTypeLowerName[541:548]: Literal, + _TokenTypeName[548:559]: LiteralDate, + _TokenTypeLowerName[548:559]: LiteralDate, + _TokenTypeName[559:571]: LiteralOther, + _TokenTypeLowerName[559:571]: LiteralOther, + _TokenTypeName[571:584]: LiteralString, + _TokenTypeLowerName[571:584]: LiteralString, + _TokenTypeName[584:602]: LiteralStringAffix, + _TokenTypeLowerName[584:602]: LiteralStringAffix, + _TokenTypeName[602:619]: LiteralStringAtom, + _TokenTypeLowerName[602:619]: LiteralStringAtom, + _TokenTypeName[619:640]: LiteralStringBacktick, + _TokenTypeLowerName[619:640]: LiteralStringBacktick, + _TokenTypeName[640:660]: LiteralStringBoolean, + _TokenTypeLowerName[640:660]: LiteralStringBoolean, + _TokenTypeName[660:677]: LiteralStringChar, + _TokenTypeLowerName[660:677]: LiteralStringChar, + _TokenTypeName[677:699]: LiteralStringDelimiter, + _TokenTypeLowerName[677:699]: LiteralStringDelimiter, + _TokenTypeName[699:715]: LiteralStringDoc, + _TokenTypeLowerName[699:715]: LiteralStringDoc, + _TokenTypeName[715:734]: LiteralStringDouble, + _TokenTypeLowerName[715:734]: LiteralStringDouble, + _TokenTypeName[734:753]: LiteralStringEscape, + _TokenTypeLowerName[734:753]: LiteralStringEscape, + _TokenTypeName[753:773]: LiteralStringHeredoc, + _TokenTypeLowerName[753:773]: LiteralStringHeredoc, + _TokenTypeName[773:794]: LiteralStringInterpol, + _TokenTypeLowerName[773:794]: LiteralStringInterpol, + _TokenTypeName[794:811]: LiteralStringName, + _TokenTypeLowerName[794:811]: LiteralStringName, + _TokenTypeName[811:829]: LiteralStringOther, + _TokenTypeLowerName[811:829]: LiteralStringOther, + _TokenTypeName[829:847]: LiteralStringRegex, + _TokenTypeLowerName[829:847]: LiteralStringRegex, + _TokenTypeName[847:866]: LiteralStringSingle, + _TokenTypeLowerName[847:866]: LiteralStringSingle, + _TokenTypeName[866:885]: LiteralStringSymbol, + _TokenTypeLowerName[866:885]: LiteralStringSymbol, + _TokenTypeName[885:898]: LiteralNumber, + _TokenTypeLowerName[885:898]: LiteralNumber, + _TokenTypeName[898:914]: LiteralNumberBin, + _TokenTypeLowerName[898:914]: LiteralNumberBin, + _TokenTypeName[914:932]: LiteralNumberFloat, + _TokenTypeLowerName[914:932]: LiteralNumberFloat, + _TokenTypeName[932:948]: LiteralNumberHex, + _TokenTypeLowerName[932:948]: LiteralNumberHex, + _TokenTypeName[948:968]: LiteralNumberInteger, + _TokenTypeLowerName[948:968]: LiteralNumberInteger, + _TokenTypeName[968:992]: LiteralNumberIntegerLong, + _TokenTypeLowerName[968:992]: LiteralNumberIntegerLong, + _TokenTypeName[992:1008]: LiteralNumberOct, + _TokenTypeLowerName[992:1008]: LiteralNumberOct, + _TokenTypeName[1008:1025]: LiteralNumberByte, + _TokenTypeLowerName[1008:1025]: LiteralNumberByte, + _TokenTypeName[1025:1033]: Operator, + _TokenTypeLowerName[1025:1033]: Operator, + _TokenTypeName[1033:1045]: OperatorWord, + _TokenTypeLowerName[1033:1045]: OperatorWord, + _TokenTypeName[1045:1056]: Punctuation, + _TokenTypeLowerName[1045:1056]: Punctuation, + _TokenTypeName[1056:1063]: Comment, + _TokenTypeLowerName[1056:1063]: Comment, + _TokenTypeName[1063:1078]: CommentHashbang, + _TokenTypeLowerName[1063:1078]: CommentHashbang, + _TokenTypeName[1078:1094]: CommentMultiline, + _TokenTypeLowerName[1078:1094]: CommentMultiline, + _TokenTypeName[1094:1107]: CommentSingle, + _TokenTypeLowerName[1094:1107]: CommentSingle, + _TokenTypeName[1107:1121]: CommentSpecial, + _TokenTypeLowerName[1107:1121]: CommentSpecial, + _TokenTypeName[1121:1135]: CommentPreproc, + _TokenTypeLowerName[1121:1135]: CommentPreproc, + _TokenTypeName[1135:1153]: CommentPreprocFile, + _TokenTypeLowerName[1135:1153]: CommentPreprocFile, + _TokenTypeName[1153:1160]: Generic, + _TokenTypeLowerName[1153:1160]: Generic, + _TokenTypeName[1160:1174]: GenericDeleted, + _TokenTypeLowerName[1160:1174]: GenericDeleted, + _TokenTypeName[1174:1185]: GenericEmph, + _TokenTypeLowerName[1174:1185]: GenericEmph, + _TokenTypeName[1185:1197]: GenericError, + _TokenTypeLowerName[1185:1197]: GenericError, + _TokenTypeName[1197:1211]: GenericHeading, + _TokenTypeLowerName[1197:1211]: GenericHeading, + _TokenTypeName[1211:1226]: GenericInserted, + _TokenTypeLowerName[1211:1226]: GenericInserted, + _TokenTypeName[1226:1239]: GenericOutput, + _TokenTypeLowerName[1226:1239]: GenericOutput, + _TokenTypeName[1239:1252]: GenericPrompt, + _TokenTypeLowerName[1239:1252]: GenericPrompt, + _TokenTypeName[1252:1265]: GenericStrong, + _TokenTypeLowerName[1252:1265]: GenericStrong, + _TokenTypeName[1265:1282]: GenericSubheading, + _TokenTypeLowerName[1265:1282]: GenericSubheading, + _TokenTypeName[1282:1298]: GenericTraceback, + _TokenTypeLowerName[1282:1298]: GenericTraceback, + _TokenTypeName[1298:1314]: GenericUnderline, + _TokenTypeLowerName[1298:1314]: GenericUnderline, + _TokenTypeName[1314:1318]: Text, + _TokenTypeLowerName[1314:1318]: Text, + _TokenTypeName[1318:1332]: TextWhitespace, + _TokenTypeLowerName[1318:1332]: TextWhitespace, + _TokenTypeName[1332:1342]: TextSymbol, + _TokenTypeLowerName[1332:1342]: TextSymbol, + _TokenTypeName[1342:1357]: TextPunctuation, + _TokenTypeLowerName[1342:1357]: TextPunctuation, } var _TokenTypeNames = []string{ - _TokenTypeName[0:4], - _TokenTypeName[4:9], - _TokenTypeName[9:14], - _TokenTypeName[14:22], - _TokenTypeName[22:30], - _TokenTypeName[30:41], - _TokenTypeName[41:50], - _TokenTypeName[50:63], - _TokenTypeName[63:79], - _TokenTypeName[79:90], - _TokenTypeName[90:94], - _TokenTypeName[94:104], - _TokenTypeName[104:114], - _TokenTypeName[114:121], - _TokenTypeName[121:128], - _TokenTypeName[128:143], - _TokenTypeName[143:161], - _TokenTypeName[161:177], - _TokenTypeName[177:190], - _TokenTypeName[190:205], - _TokenTypeName[205:216], - _TokenTypeName[216:220], - _TokenTypeName[220:233], - _TokenTypeName[233:244], - _TokenTypeName[244:261], - _TokenTypeName[261:270], - _TokenTypeName[270:282], - _TokenTypeName[282:295], - _TokenTypeName[295:305], - _TokenTypeName[305:318], - _TokenTypeName[318:330], - _TokenTypeName[330:347], - _TokenTypeName[347:358], - _TokenTypeName[358:367], - _TokenTypeName[367:380], - _TokenTypeName[380:392], - _TokenTypeName[392:401], - _TokenTypeName[401:411], - _TokenTypeName[411:423], - _TokenTypeName[423:430], - _TokenTypeName[430:442], - _TokenTypeName[442:463], - _TokenTypeName[463:480], - _TokenTypeName[480:498], - _TokenTypeName[498:518], - _TokenTypeName[518:535], - _TokenTypeName[535:542], - _TokenTypeName[542:553], - _TokenTypeName[553:565], - _TokenTypeName[565:578], - _TokenTypeName[578:596], - _TokenTypeName[596:613], - _TokenTypeName[613:634], - _TokenTypeName[634:654], - _TokenTypeName[654:671], - _TokenTypeName[671:693], - _TokenTypeName[693:709], - _TokenTypeName[709:728], - _TokenTypeName[728:747], - _TokenTypeName[747:767], - _TokenTypeName[767:788], - _TokenTypeName[788:805], - _TokenTypeName[805:823], - _TokenTypeName[823:841], - _TokenTypeName[841:860], - _TokenTypeName[860:879], - _TokenTypeName[879:892], - _TokenTypeName[892:908], - _TokenTypeName[908:926], - _TokenTypeName[926:942], - _TokenTypeName[942:962], - _TokenTypeName[962:986], - _TokenTypeName[986:1002], - _TokenTypeName[1002:1019], - _TokenTypeName[1019:1027], - _TokenTypeName[1027:1039], - _TokenTypeName[1039:1050], - _TokenTypeName[1050:1057], - _TokenTypeName[1057:1072], - _TokenTypeName[1072:1088], - _TokenTypeName[1088:1101], - _TokenTypeName[1101:1115], - _TokenTypeName[1115:1129], - _TokenTypeName[1129:1147], - _TokenTypeName[1147:1154], - _TokenTypeName[1154:1168], - _TokenTypeName[1168:1179], - _TokenTypeName[1179:1191], - _TokenTypeName[1191:1205], - _TokenTypeName[1205:1220], - _TokenTypeName[1220:1233], - _TokenTypeName[1233:1246], - _TokenTypeName[1246:1259], - _TokenTypeName[1259:1276], - _TokenTypeName[1276:1292], - _TokenTypeName[1292:1308], - _TokenTypeName[1308:1312], - _TokenTypeName[1312:1326], - _TokenTypeName[1326:1336], - _TokenTypeName[1336:1351], + _TokenTypeName[0:6], + _TokenTypeName[6:10], + _TokenTypeName[10:15], + _TokenTypeName[15:20], + _TokenTypeName[20:28], + _TokenTypeName[28:36], + _TokenTypeName[36:47], + _TokenTypeName[47:56], + _TokenTypeName[56:69], + _TokenTypeName[69:85], + _TokenTypeName[85:96], + _TokenTypeName[96:100], + _TokenTypeName[100:110], + _TokenTypeName[110:120], + _TokenTypeName[120:127], + _TokenTypeName[127:134], + _TokenTypeName[134:149], + _TokenTypeName[149:167], + _TokenTypeName[167:183], + _TokenTypeName[183:196], + _TokenTypeName[196:211], + _TokenTypeName[211:222], + _TokenTypeName[222:226], + _TokenTypeName[226:239], + _TokenTypeName[239:250], + _TokenTypeName[250:267], + _TokenTypeName[267:276], + _TokenTypeName[276:288], + _TokenTypeName[288:301], + _TokenTypeName[301:311], + _TokenTypeName[311:324], + _TokenTypeName[324:336], + _TokenTypeName[336:353], + _TokenTypeName[353:364], + _TokenTypeName[364:373], + _TokenTypeName[373:386], + _TokenTypeName[386:398], + _TokenTypeName[398:407], + _TokenTypeName[407:417], + _TokenTypeName[417:429], + _TokenTypeName[429:436], + _TokenTypeName[436:448], + _TokenTypeName[448:469], + _TokenTypeName[469:486], + _TokenTypeName[486:504], + _TokenTypeName[504:524], + _TokenTypeName[524:541], + _TokenTypeName[541:548], + _TokenTypeName[548:559], + _TokenTypeName[559:571], + _TokenTypeName[571:584], + _TokenTypeName[584:602], + _TokenTypeName[602:619], + _TokenTypeName[619:640], + _TokenTypeName[640:660], + _TokenTypeName[660:677], + _TokenTypeName[677:699], + _TokenTypeName[699:715], + _TokenTypeName[715:734], + _TokenTypeName[734:753], + _TokenTypeName[753:773], + _TokenTypeName[773:794], + _TokenTypeName[794:811], + _TokenTypeName[811:829], + _TokenTypeName[829:847], + _TokenTypeName[847:866], + _TokenTypeName[866:885], + _TokenTypeName[885:898], + _TokenTypeName[898:914], + _TokenTypeName[914:932], + _TokenTypeName[932:948], + _TokenTypeName[948:968], + _TokenTypeName[968:992], + _TokenTypeName[992:1008], + _TokenTypeName[1008:1025], + _TokenTypeName[1025:1033], + _TokenTypeName[1033:1045], + _TokenTypeName[1045:1056], + _TokenTypeName[1056:1063], + _TokenTypeName[1063:1078], + _TokenTypeName[1078:1094], + _TokenTypeName[1094:1107], + _TokenTypeName[1107:1121], + _TokenTypeName[1121:1135], + _TokenTypeName[1135:1153], + _TokenTypeName[1153:1160], + _TokenTypeName[1160:1174], + _TokenTypeName[1174:1185], + _TokenTypeName[1185:1197], + _TokenTypeName[1197:1211], + _TokenTypeName[1211:1226], + _TokenTypeName[1226:1239], + _TokenTypeName[1239:1252], + _TokenTypeName[1252:1265], + _TokenTypeName[1265:1282], + _TokenTypeName[1282:1298], + _TokenTypeName[1298:1314], + _TokenTypeName[1314:1318], + _TokenTypeName[1318:1332], + _TokenTypeName[1332:1342], + _TokenTypeName[1342:1357], } // TokenTypeString retrieves an enum value from the enum constants string name. diff --git a/types.go b/types.go index d65d390..a576293 100644 --- a/types.go +++ b/types.go @@ -41,6 +41,8 @@ const ( Other // No highlighting. None + // Don't emit this token to the output. + Ignore // Used as an EOF marker / nil token EOFType TokenType = 0 )