From 895a0488b55e5771ba4cec39239aec1f8828f4f9 Mon Sep 17 00:00:00 2001 From: Koki Fukuda Date: Fri, 23 Aug 2024 05:04:41 +0900 Subject: [PATCH] Port Minecraft lexers from Pygments (#992) Ported lexers for mcfuntion, snbt from Pygments using `pygments2chroma_xml.py` script. While doing so, I encountered lack of `LiteralNumberByte` in TokenType, so I've added the type and regenerated tokentype_enumer.go. --- README.md | 4 +- lexers/embedded/mcfunction.xml | 296 ++++++++++++---------------- lexers/embedded/snbt.xml | 58 ++++++ lexers/testdata/mcfunction.expected | 105 +++++----- lexers/testdata/snbt.actual | 13 ++ lexers/testdata/snbt.expected | 127 ++++++++++++ tokentype_enumer.go | 219 ++++++++++---------- types.go | 1 + 8 files changed, 494 insertions(+), 329 deletions(-) create mode 100644 lexers/embedded/snbt.xml create mode 100644 lexers/testdata/snbt.actual create mode 100644 lexers/testdata/snbt.expected diff --git a/README.md b/README.md index 7060eb7..bbe7f36 100644 --- a/README.md +++ b/README.md @@ -45,13 +45,13 @@ translators for Pygments lexers and styles. | J | J, Java, JavaScript, JSON, Julia, Jungle | | K | Kotlin | | L | Lighttpd configuration file, LLVM, Lua | -| M | Makefile, Mako, markdown, Mason, Materialize SQL dialect, Mathematica, Matlab, mcfunction, Meson, Metal, MiniZinc, MLIR, Modula-2, MonkeyC, MorrowindScript, Myghty, MySQL | +| M | Makefile, Mako, markdown, Mason, Materialize SQL dialect, Mathematica, Matlab, MCFunction, Meson, Metal, MiniZinc, MLIR, Modula-2, MonkeyC, MorrowindScript, Myghty, MySQL | | N | NASM, Natural, Newspeak, Nginx configuration file, Nim, Nix | | O | Objective-C, OCaml, Octave, Odin, OnesEnterprise, OpenEdge ABL, OpenSCAD, Org Mode | | P | PacmanConf, Perl, PHP, PHTML, Pig, PkgConfig, PL/pgSQL, plaintext, Plutus Core, Pony, PostgreSQL SQL dialect, PostScript, POVRay, PowerQuery, PowerShell, Prolog, PromQL, Promela, properties, Protocol Buffer, PRQL, PSL, Puppet, Python, Python 2 | | Q | QBasic, QML | | R | R, Racket, Ragel, Raku, react, ReasonML, reg, Rego, reStructuredText, Rexx, RPMSpec, Ruby, Rust | -| S | SAS, Sass, Scala, Scheme, Scilab, SCSS, Sed, Sieve, Smali, Smalltalk, Smarty, Snobol, Solidity, SourcePawn, SPARQL, SQL, SquidConf, Standard ML, stas, Stylus, Svelte, Swift, SYSTEMD, systemverilog | +| S | SAS, Sass, Scala, Scheme, Scilab, SCSS, Sed, Sieve, Smali, Smalltalk, Smarty, SNBT, Snobol, Solidity, SourcePawn, SPARQL, SQL, SquidConf, Standard ML, stas, Stylus, Svelte, Swift, SYSTEMD, systemverilog | | T | TableGen, Tal, TASM, Tcl, Tcsh, Termcap, Terminfo, Terraform, TeX, Thrift, TOML, TradingView, Transact-SQL, Turing, Turtle, Twig, TypeScript, TypoScript, TypoScriptCssData, TypoScriptHtmlData | | V | V, V shell, Vala, VB.net, verilog, VHDL, VHS, VimL, vue | | W | WDTE, WebGPU Shading Language, Whiley | diff --git a/lexers/embedded/mcfunction.xml b/lexers/embedded/mcfunction.xml index 3310520..a6aa6db 100644 --- a/lexers/embedded/mcfunction.xml +++ b/lexers/embedded/mcfunction.xml @@ -1,182 +1,138 @@ + - mcfunction + MCFunction mcfunction + mcf *.mcfunction - true - true + text/mcfunction - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lexers/embedded/snbt.xml b/lexers/embedded/snbt.xml new file mode 100644 index 0000000..fdb12d0 --- /dev/null +++ b/lexers/embedded/snbt.xml @@ -0,0 +1,58 @@ + + + + SNBT + snbt + *.snbt + text/snbt + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lexers/testdata/mcfunction.expected b/lexers/testdata/mcfunction.expected index 3639656..84c241e 100644 --- a/lexers/testdata/mcfunction.expected +++ b/lexers/testdata/mcfunction.expected @@ -1,101 +1,106 @@ [ - {"type":"KeywordReserved","value":"fill"}, + {"type":"NameBuiltin","value":"fill"}, {"type":"TextWhitespace","value":" "}, - {"type":"LiteralNumberInteger","value":"29"}, + {"type":"LiteralNumberFloat","value":"29"}, {"type":"TextWhitespace","value":" "}, - {"type":"LiteralNumberInteger","value":"4"}, + {"type":"LiteralNumberFloat","value":"4"}, {"type":"TextWhitespace","value":" "}, - {"type":"LiteralNumberInteger","value":"26"}, + {"type":"LiteralNumberFloat","value":"26"}, {"type":"TextWhitespace","value":" "}, - {"type":"NameBuiltin","value":"~"}, + {"type":"Operator","value":"~"}, {"type":"TextWhitespace","value":" "}, - {"type":"NameBuiltin","value":"~"}, + {"type":"Operator","value":"~"}, {"type":"TextWhitespace","value":" "}, - {"type":"NameBuiltin","value":"~"}, + {"type":"Operator","value":"~"}, {"type":"TextWhitespace","value":" "}, - {"type":"Text","value":"minecraft:air"}, + {"type":"NameFunction","value":"minecraft:air"}, {"type":"TextWhitespace","value":" "}, - {"type":"Text","value":"replace"}, + {"type":"KeywordConstant","value":"replace"}, {"type":"TextWhitespace","value":" "}, - {"type":"Text","value":"minecraft:water"}, + {"type":"NameFunction","value":"minecraft:water"}, {"type":"TextWhitespace","value":"\n\n"}, - {"type":"CommentSingle","value":"# NBT (JSON-like) value.\n"}, - {"type":"KeywordReserved","value":"summon"}, + {"type":"CommentSingle","value":"# NBT (JSON-like) value."}, + {"type":"TextWhitespace","value":"\n"}, + {"type":"NameBuiltin","value":"summon"}, {"type":"TextWhitespace","value":" "}, - {"type":"Text","value":"zombie"}, + {"type":"KeywordConstant","value":"zombie"}, {"type":"TextWhitespace","value":" "}, - {"type":"NameBuiltin","value":"~"}, + {"type":"Operator","value":"~"}, {"type":"TextWhitespace","value":" "}, - {"type":"NameBuiltin","value":"~"}, + {"type":"Operator","value":"~"}, {"type":"TextWhitespace","value":" "}, - {"type":"NameBuiltin","value":"~"}, + {"type":"Operator","value":"~"}, {"type":"TextWhitespace","value":" "}, {"type":"Punctuation","value":"{"}, - {"type":"NameTag","value":"NoAI"}, + {"type":"NameAttribute","value":"NoAI"}, {"type":"Punctuation","value":":"}, - {"type":"LiteralNumber","value":"1b"}, + {"type":"LiteralNumberByte","value":"1b"}, {"type":"Punctuation","value":"}"}, {"type":"TextWhitespace","value":"\n\n"}, - {"type":"CommentSingle","value":"# Target selector\n"}, - {"type":"KeywordReserved","value":"gamemode"}, + {"type":"CommentSingle","value":"# Target selector"}, + {"type":"TextWhitespace","value":"\n"}, + {"type":"NameBuiltin","value":"gamemode"}, {"type":"TextWhitespace","value":" "}, - {"type":"Text","value":"creative"}, + {"type":"KeywordConstant","value":"creative"}, {"type":"TextWhitespace","value":" "}, - {"type":"KeywordConstant","value":"@a"}, + {"type":"NameVariable","value":"@a"}, {"type":"TextWhitespace","value":"\n\n"}, - {"type":"CommentSingle","value":"# With argument\n"}, - {"type":"KeywordReserved","value":"kill"}, + {"type":"CommentSingle","value":"# With argument"}, + {"type":"TextWhitespace","value":"\n"}, + {"type":"NameBuiltin","value":"kill"}, {"type":"TextWhitespace","value":" "}, - {"type":"KeywordConstant","value":"@e"}, + {"type":"NameVariable","value":"@e"}, {"type":"Punctuation","value":"["}, {"type":"NameAttribute","value":"type"}, {"type":"Punctuation","value":"="}, - {"type":"Text","value":"chicken"}, + {"type":"NameTag","value":"chicken"}, {"type":"Punctuation","value":","}, {"type":"NameAttribute","value":"nested"}, {"type":"Punctuation","value":"={"}, {"type":"NameAttribute","value":"a0"}, {"type":"Punctuation","value":"="}, - {"type":"LiteralNumberInteger","value":"0"}, + {"type":"LiteralNumberFloat","value":"0"}, {"type":"Punctuation","value":","}, {"type":"NameAttribute","value":"a1"}, {"type":"Punctuation","value":"="}, - {"type":"LiteralNumberInteger","value":"1"}, + {"type":"LiteralNumberFloat","value":"1"}, {"type":"Punctuation","value":"}]"}, {"type":"TextWhitespace","value":"\n\n"}, - {"type":"CommentSingle","value":"# Range\n"}, - {"type":"KeywordReserved","value":"kill"}, + {"type":"CommentSingle","value":"# Range"}, + {"type":"TextWhitespace","value":"\n"}, + {"type":"NameBuiltin","value":"kill"}, {"type":"TextWhitespace","value":" "}, - {"type":"KeywordConstant","value":"@e"}, + {"type":"NameVariable","value":"@e"}, {"type":"Punctuation","value":"["}, {"type":"NameAttribute","value":"y_rotation"}, {"type":"Punctuation","value":"="}, - {"type":"LiteralNumberInteger","value":"0"}, - {"type":"Punctuation","value":".."}, - {"type":"LiteralNumberInteger","value":"180"}, + {"type":"LiteralNumberFloat","value":"0"}, + {"type":"Literal","value":".."}, + {"type":"LiteralNumberFloat","value":"180"}, {"type":"Punctuation","value":"]"}, {"type":"TextWhitespace","value":"\n\n"}, - {"type":"CommentSingle","value":"# Legacy and new style\n"}, - {"type":"Text","value":"placefeature"}, - {"type":"TextWhitespace","value":" "}, - {"type":"Text","value":"minecraft:bonus_chest"}, - {"type":"TextWhitespace","value":" "}, - {"type":"NameBuiltin","value":"~"}, - {"type":"TextWhitespace","value":" "}, - {"type":"NameBuiltin","value":"~"}, - {"type":"TextWhitespace","value":" "}, - {"type":"NameBuiltin","value":"~"}, + {"type":"CommentSingle","value":"# Legacy and new style"}, {"type":"TextWhitespace","value":"\n"}, - {"type":"KeywordReserved","value":"place"}, + {"type":"NameBuiltin","value":"placefeature"}, {"type":"TextWhitespace","value":" "}, - {"type":"Text","value":"feature"}, + {"type":"NameFunction","value":"minecraft:bonus_chest"}, {"type":"TextWhitespace","value":" "}, - {"type":"Text","value":"minecraft:bonus_chest"}, + {"type":"Operator","value":"~"}, {"type":"TextWhitespace","value":" "}, - {"type":"NameBuiltin","value":"~"}, + {"type":"Operator","value":"~"}, {"type":"TextWhitespace","value":" "}, - {"type":"NameBuiltin","value":"~"}, + {"type":"Operator","value":"~"}, + {"type":"TextWhitespace","value":"\n"}, + {"type":"NameBuiltin","value":"place"}, {"type":"TextWhitespace","value":" "}, - {"type":"NameBuiltin","value":"~"}, + {"type":"KeywordConstant","value":"feature"}, + {"type":"TextWhitespace","value":" "}, + {"type":"NameFunction","value":"minecraft:bonus_chest"}, + {"type":"TextWhitespace","value":" "}, + {"type":"Operator","value":"~"}, + {"type":"TextWhitespace","value":" "}, + {"type":"Operator","value":"~"}, + {"type":"TextWhitespace","value":" "}, + {"type":"Operator","value":"~"}, {"type":"TextWhitespace","value":"\n"} ] diff --git a/lexers/testdata/snbt.actual b/lexers/testdata/snbt.actual new file mode 100644 index 0000000..0e1e3c1 --- /dev/null +++ b/lexers/testdata/snbt.actual @@ -0,0 +1,13 @@ +{ + Byte: 40b, + Boolean: true, + Short: 40s, + Int: 40, + Long: 40l, + Float: 40.0f, + Double: 40.0, + String: "foo" +} +{List: [10, 20, 30]} +{TypedArrays: {Byte: [B; 1b, 2b, 3b], Int: [I; 1, 2, 3], Long: [L; 1l, 2l, 3l]}} +{"foo bar": "baz"} diff --git a/lexers/testdata/snbt.expected b/lexers/testdata/snbt.expected new file mode 100644 index 0000000..c1aaeba --- /dev/null +++ b/lexers/testdata/snbt.expected @@ -0,0 +1,127 @@ +[ + {"type":"Punctuation","value":"{"}, + {"type":"TextWhitespace","value":"\n "}, + {"type":"NameAttribute","value":"Byte"}, + {"type":"Punctuation","value":":"}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralNumberInteger","value":"40b"}, + {"type":"Punctuation","value":","}, + {"type":"TextWhitespace","value":"\n "}, + {"type":"NameAttribute","value":"Boolean"}, + {"type":"Punctuation","value":":"}, + {"type":"TextWhitespace","value":" "}, + {"type":"NameAttribute","value":"true"}, + {"type":"Punctuation","value":","}, + {"type":"TextWhitespace","value":"\n "}, + {"type":"NameAttribute","value":"Short"}, + {"type":"Punctuation","value":":"}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralNumberInteger","value":"40s"}, + {"type":"Punctuation","value":","}, + {"type":"TextWhitespace","value":"\n "}, + {"type":"NameAttribute","value":"Int"}, + {"type":"Punctuation","value":":"}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralNumberInteger","value":"40"}, + {"type":"Punctuation","value":","}, + {"type":"TextWhitespace","value":"\n "}, + {"type":"NameAttribute","value":"Long"}, + {"type":"Punctuation","value":":"}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralNumberInteger","value":"40l"}, + {"type":"Punctuation","value":","}, + {"type":"TextWhitespace","value":"\n "}, + {"type":"NameAttribute","value":"Float"}, + {"type":"Punctuation","value":":"}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralNumberFloat","value":"40.0f"}, + {"type":"Punctuation","value":","}, + {"type":"TextWhitespace","value":"\n "}, + {"type":"NameAttribute","value":"Double"}, + {"type":"Punctuation","value":":"}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralNumberFloat","value":"40.0"}, + {"type":"Punctuation","value":","}, + {"type":"TextWhitespace","value":"\n "}, + {"type":"NameAttribute","value":"String"}, + {"type":"Punctuation","value":":"}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralStringDouble","value":"\"foo\""}, + {"type":"TextWhitespace","value":"\n"}, + {"type":"Punctuation","value":"}"}, + {"type":"Text","value":"\n"}, + {"type":"Punctuation","value":"{"}, + {"type":"NameAttribute","value":"List"}, + {"type":"Punctuation","value":":"}, + {"type":"TextWhitespace","value":" "}, + {"type":"Punctuation","value":"["}, + {"type":"LiteralNumberInteger","value":"10"}, + {"type":"Punctuation","value":","}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralNumberInteger","value":"20"}, + {"type":"Punctuation","value":","}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralNumberInteger","value":"30"}, + {"type":"Punctuation","value":"]}"}, + {"type":"Text","value":"\n"}, + {"type":"Punctuation","value":"{"}, + {"type":"NameAttribute","value":"TypedArrays"}, + {"type":"Punctuation","value":":"}, + {"type":"TextWhitespace","value":" "}, + {"type":"Punctuation","value":"{"}, + {"type":"NameAttribute","value":"Byte"}, + {"type":"Punctuation","value":":"}, + {"type":"TextWhitespace","value":" "}, + {"type":"Punctuation","value":"["}, + {"type":"NameAttribute","value":"B"}, + {"type":"Punctuation","value":";"}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralNumberInteger","value":"1b"}, + {"type":"Punctuation","value":","}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralNumberInteger","value":"2b"}, + {"type":"Punctuation","value":","}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralNumberInteger","value":"3b"}, + {"type":"Punctuation","value":"],"}, + {"type":"TextWhitespace","value":" "}, + {"type":"NameAttribute","value":"Int"}, + {"type":"Punctuation","value":":"}, + {"type":"TextWhitespace","value":" "}, + {"type":"Punctuation","value":"["}, + {"type":"NameAttribute","value":"I"}, + {"type":"Punctuation","value":";"}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralNumberInteger","value":"1"}, + {"type":"Punctuation","value":","}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralNumberInteger","value":"2"}, + {"type":"Punctuation","value":","}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralNumberInteger","value":"3"}, + {"type":"Punctuation","value":"],"}, + {"type":"TextWhitespace","value":" "}, + {"type":"NameAttribute","value":"Long"}, + {"type":"Punctuation","value":":"}, + {"type":"TextWhitespace","value":" "}, + {"type":"Punctuation","value":"["}, + {"type":"NameAttribute","value":"L"}, + {"type":"Punctuation","value":";"}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralNumberInteger","value":"1l"}, + {"type":"Punctuation","value":","}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralNumberInteger","value":"2l"}, + {"type":"Punctuation","value":","}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralNumberInteger","value":"3l"}, + {"type":"Punctuation","value":"]}}"}, + {"type":"Text","value":"\n"}, + {"type":"Punctuation","value":"{"}, + {"type":"LiteralStringDouble","value":"\"foo bar\""}, + {"type":"Punctuation","value":":"}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralStringDouble","value":"\"baz\""}, + {"type":"Punctuation","value":"}"}, + {"type":"Text","value":"\n"} +] diff --git a/tokentype_enumer.go b/tokentype_enumer.go index 696e9ce..e3bfbea 100644 --- a/tokentype_enumer.go +++ b/tokentype_enumer.go @@ -7,8 +7,8 @@ import ( "strings" ) -const _TokenTypeName = "NoneOtherErrorCodeLineLineLinkLineTableTDLineTableLineHighlightLineNumbersTableLineNumbersLinePreWrapperBackgroundEOFTypeKeywordKeywordConstantKeywordDeclarationKeywordNamespaceKeywordPseudoKeywordReservedKeywordTypeNameNameAttributeNameBuiltinNameBuiltinPseudoNameClassNameConstantNameDecoratorNameEntityNameExceptionNameFunctionNameFunctionMagicNameKeywordNameLabelNameNamespaceNameOperatorNameOtherNamePseudoNamePropertyNameTagNameVariableNameVariableAnonymousNameVariableClassNameVariableGlobalNameVariableInstanceNameVariableMagicLiteralLiteralDateLiteralOtherLiteralStringLiteralStringAffixLiteralStringAtomLiteralStringBacktickLiteralStringBooleanLiteralStringCharLiteralStringDelimiterLiteralStringDocLiteralStringDoubleLiteralStringEscapeLiteralStringHeredocLiteralStringInterpolLiteralStringNameLiteralStringOtherLiteralStringRegexLiteralStringSingleLiteralStringSymbolLiteralNumberLiteralNumberBinLiteralNumberFloatLiteralNumberHexLiteralNumberIntegerLiteralNumberIntegerLongLiteralNumberOctOperatorOperatorWordPunctuationCommentCommentHashbangCommentMultilineCommentSingleCommentSpecialCommentPreprocCommentPreprocFileGenericGenericDeletedGenericEmphGenericErrorGenericHeadingGenericInsertedGenericOutputGenericPromptGenericStrongGenericSubheadingGenericTracebackGenericUnderlineTextTextWhitespaceTextSymbolTextPunctuation" -const _TokenTypeLowerName = "noneothererrorcodelinelinelinklinetabletdlinetablelinehighlightlinenumberstablelinenumberslineprewrapperbackgroundeoftypekeywordkeywordconstantkeyworddeclarationkeywordnamespacekeywordpseudokeywordreservedkeywordtypenamenameattributenamebuiltinnamebuiltinpseudonameclassnameconstantnamedecoratornameentitynameexceptionnamefunctionnamefunctionmagicnamekeywordnamelabelnamenamespacenameoperatornameothernamepseudonamepropertynametagnamevariablenamevariableanonymousnamevariableclassnamevariableglobalnamevariableinstancenamevariablemagicliteralliteraldateliteralotherliteralstringliteralstringaffixliteralstringatomliteralstringbacktickliteralstringbooleanliteralstringcharliteralstringdelimiterliteralstringdocliteralstringdoubleliteralstringescapeliteralstringheredocliteralstringinterpolliteralstringnameliteralstringotherliteralstringregexliteralstringsingleliteralstringsymbolliteralnumberliteralnumberbinliteralnumberfloatliteralnumberhexliteralnumberintegerliteralnumberintegerlongliteralnumberoctoperatoroperatorwordpunctuationcommentcommenthashbangcommentmultilinecommentsinglecommentspecialcommentpreproccommentpreprocfilegenericgenericdeletedgenericemphgenericerrorgenericheadinggenericinsertedgenericoutputgenericpromptgenericstronggenericsubheadinggenerictracebackgenericunderlinetexttextwhitespacetextsymboltextpunctuation" +const _TokenTypeName = "NoneOtherErrorCodeLineLineLinkLineTableTDLineTableLineHighlightLineNumbersTableLineNumbersLinePreWrapperBackgroundEOFTypeKeywordKeywordConstantKeywordDeclarationKeywordNamespaceKeywordPseudoKeywordReservedKeywordTypeNameNameAttributeNameBuiltinNameBuiltinPseudoNameClassNameConstantNameDecoratorNameEntityNameExceptionNameFunctionNameFunctionMagicNameKeywordNameLabelNameNamespaceNameOperatorNameOtherNamePseudoNamePropertyNameTagNameVariableNameVariableAnonymousNameVariableClassNameVariableGlobalNameVariableInstanceNameVariableMagicLiteralLiteralDateLiteralOtherLiteralStringLiteralStringAffixLiteralStringAtomLiteralStringBacktickLiteralStringBooleanLiteralStringCharLiteralStringDelimiterLiteralStringDocLiteralStringDoubleLiteralStringEscapeLiteralStringHeredocLiteralStringInterpolLiteralStringNameLiteralStringOtherLiteralStringRegexLiteralStringSingleLiteralStringSymbolLiteralNumberLiteralNumberBinLiteralNumberFloatLiteralNumberHexLiteralNumberIntegerLiteralNumberIntegerLongLiteralNumberOctLiteralNumberByteOperatorOperatorWordPunctuationCommentCommentHashbangCommentMultilineCommentSingleCommentSpecialCommentPreprocCommentPreprocFileGenericGenericDeletedGenericEmphGenericErrorGenericHeadingGenericInsertedGenericOutputGenericPromptGenericStrongGenericSubheadingGenericTracebackGenericUnderlineTextTextWhitespaceTextSymbolTextPunctuation" +const _TokenTypeLowerName = "noneothererrorcodelinelinelinklinetabletdlinetablelinehighlightlinenumberstablelinenumberslineprewrapperbackgroundeoftypekeywordkeywordconstantkeyworddeclarationkeywordnamespacekeywordpseudokeywordreservedkeywordtypenamenameattributenamebuiltinnamebuiltinpseudonameclassnameconstantnamedecoratornameentitynameexceptionnamefunctionnamefunctionmagicnamekeywordnamelabelnamenamespacenameoperatornameothernamepseudonamepropertynametagnamevariablenamevariableanonymousnamevariableclassnamevariableglobalnamevariableinstancenamevariablemagicliteralliteraldateliteralotherliteralstringliteralstringaffixliteralstringatomliteralstringbacktickliteralstringbooleanliteralstringcharliteralstringdelimiterliteralstringdocliteralstringdoubleliteralstringescapeliteralstringheredocliteralstringinterpolliteralstringnameliteralstringotherliteralstringregexliteralstringsingleliteralstringsymbolliteralnumberliteralnumberbinliteralnumberfloatliteralnumberhexliteralnumberintegerliteralnumberintegerlongliteralnumberoctliteralnumberbyteoperatoroperatorwordpunctuationcommentcommenthashbangcommentmultilinecommentsinglecommentspecialcommentpreproccommentpreprocfilegenericgenericdeletedgenericemphgenericerrorgenericheadinggenericinsertedgenericoutputgenericpromptgenericstronggenericsubheadinggenerictracebackgenericunderlinetexttextwhitespacetextsymboltextpunctuation" var _TokenTypeMap = map[TokenType]string{ -13: _TokenTypeName[0:4], @@ -84,32 +84,33 @@ var _TokenTypeMap = map[TokenType]string{ 3204: _TokenTypeName[942:962], 3205: _TokenTypeName[962:986], 3206: _TokenTypeName[986:1002], - 4000: _TokenTypeName[1002:1010], - 4001: _TokenTypeName[1010:1022], - 5000: _TokenTypeName[1022:1033], - 6000: _TokenTypeName[1033:1040], - 6001: _TokenTypeName[1040:1055], - 6002: _TokenTypeName[1055:1071], - 6003: _TokenTypeName[1071:1084], - 6004: _TokenTypeName[1084:1098], - 6100: _TokenTypeName[1098:1112], - 6101: _TokenTypeName[1112:1130], - 7000: _TokenTypeName[1130:1137], - 7001: _TokenTypeName[1137:1151], - 7002: _TokenTypeName[1151:1162], - 7003: _TokenTypeName[1162:1174], - 7004: _TokenTypeName[1174:1188], - 7005: _TokenTypeName[1188:1203], - 7006: _TokenTypeName[1203:1216], - 7007: _TokenTypeName[1216:1229], - 7008: _TokenTypeName[1229:1242], - 7009: _TokenTypeName[1242:1259], - 7010: _TokenTypeName[1259:1275], - 7011: _TokenTypeName[1275:1291], - 8000: _TokenTypeName[1291:1295], - 8001: _TokenTypeName[1295:1309], - 8002: _TokenTypeName[1309:1319], - 8003: _TokenTypeName[1319:1334], + 3207: _TokenTypeName[1002:1019], + 4000: _TokenTypeName[1019:1027], + 4001: _TokenTypeName[1027:1039], + 5000: _TokenTypeName[1039:1050], + 6000: _TokenTypeName[1050:1057], + 6001: _TokenTypeName[1057:1072], + 6002: _TokenTypeName[1072:1088], + 6003: _TokenTypeName[1088:1101], + 6004: _TokenTypeName[1101:1115], + 6100: _TokenTypeName[1115:1129], + 6101: _TokenTypeName[1129:1147], + 7000: _TokenTypeName[1147:1154], + 7001: _TokenTypeName[1154:1168], + 7002: _TokenTypeName[1168:1179], + 7003: _TokenTypeName[1179:1191], + 7004: _TokenTypeName[1191:1205], + 7005: _TokenTypeName[1205:1220], + 7006: _TokenTypeName[1220:1233], + 7007: _TokenTypeName[1233:1246], + 7008: _TokenTypeName[1246:1259], + 7009: _TokenTypeName[1259:1276], + 7010: _TokenTypeName[1276:1292], + 7011: _TokenTypeName[1292:1308], + 8000: _TokenTypeName[1308:1312], + 8001: _TokenTypeName[1312:1326], + 8002: _TokenTypeName[1326:1336], + 8003: _TokenTypeName[1336:1351], } func (i TokenType) String() string { @@ -196,6 +197,7 @@ func _TokenTypeNoOp() { _ = x[LiteralNumberInteger-(3204)] _ = x[LiteralNumberIntegerLong-(3205)] _ = x[LiteralNumberOct-(3206)] + _ = x[LiteralNumberByte-(3207)] _ = x[Operator-(4000)] _ = x[OperatorWord-(4001)] _ = x[Punctuation-(5000)] @@ -224,7 +226,7 @@ func _TokenTypeNoOp() { _ = x[TextPunctuation-(8003)] } -var _TokenTypeValues = []TokenType{None, Other, Error, CodeLine, LineLink, LineTableTD, LineTable, LineHighlight, LineNumbersTable, LineNumbers, Line, PreWrapper, Background, EOFType, Keyword, KeywordConstant, KeywordDeclaration, KeywordNamespace, KeywordPseudo, KeywordReserved, KeywordType, Name, NameAttribute, NameBuiltin, NameBuiltinPseudo, NameClass, NameConstant, NameDecorator, NameEntity, NameException, NameFunction, NameFunctionMagic, NameKeyword, NameLabel, NameNamespace, NameOperator, NameOther, NamePseudo, NameProperty, NameTag, NameVariable, NameVariableAnonymous, NameVariableClass, NameVariableGlobal, NameVariableInstance, NameVariableMagic, Literal, LiteralDate, LiteralOther, LiteralString, LiteralStringAffix, LiteralStringAtom, LiteralStringBacktick, LiteralStringBoolean, LiteralStringChar, LiteralStringDelimiter, LiteralStringDoc, LiteralStringDouble, LiteralStringEscape, LiteralStringHeredoc, LiteralStringInterpol, LiteralStringName, LiteralStringOther, LiteralStringRegex, LiteralStringSingle, LiteralStringSymbol, LiteralNumber, LiteralNumberBin, LiteralNumberFloat, LiteralNumberHex, LiteralNumberInteger, LiteralNumberIntegerLong, LiteralNumberOct, Operator, OperatorWord, Punctuation, Comment, CommentHashbang, CommentMultiline, CommentSingle, CommentSpecial, CommentPreproc, CommentPreprocFile, Generic, GenericDeleted, GenericEmph, GenericError, GenericHeading, GenericInserted, GenericOutput, GenericPrompt, GenericStrong, GenericSubheading, GenericTraceback, GenericUnderline, Text, TextWhitespace, TextSymbol, TextPunctuation} +var _TokenTypeValues = []TokenType{None, Other, Error, CodeLine, LineLink, LineTableTD, LineTable, LineHighlight, LineNumbersTable, LineNumbers, Line, PreWrapper, Background, EOFType, Keyword, KeywordConstant, KeywordDeclaration, KeywordNamespace, KeywordPseudo, KeywordReserved, KeywordType, Name, NameAttribute, NameBuiltin, NameBuiltinPseudo, NameClass, NameConstant, NameDecorator, NameEntity, NameException, NameFunction, NameFunctionMagic, NameKeyword, NameLabel, NameNamespace, NameOperator, NameOther, NamePseudo, NameProperty, NameTag, NameVariable, NameVariableAnonymous, NameVariableClass, NameVariableGlobal, NameVariableInstance, NameVariableMagic, Literal, LiteralDate, LiteralOther, LiteralString, LiteralStringAffix, LiteralStringAtom, LiteralStringBacktick, LiteralStringBoolean, LiteralStringChar, LiteralStringDelimiter, LiteralStringDoc, LiteralStringDouble, LiteralStringEscape, LiteralStringHeredoc, LiteralStringInterpol, LiteralStringName, LiteralStringOther, LiteralStringRegex, LiteralStringSingle, LiteralStringSymbol, LiteralNumber, LiteralNumberBin, LiteralNumberFloat, LiteralNumberHex, LiteralNumberInteger, LiteralNumberIntegerLong, LiteralNumberOct, LiteralNumberByte, Operator, OperatorWord, Punctuation, Comment, CommentHashbang, CommentMultiline, CommentSingle, CommentSpecial, CommentPreproc, CommentPreprocFile, Generic, GenericDeleted, GenericEmph, GenericError, GenericHeading, GenericInserted, GenericOutput, GenericPrompt, GenericStrong, GenericSubheading, GenericTraceback, GenericUnderline, Text, TextWhitespace, TextSymbol, TextPunctuation} var _TokenTypeNameToValueMap = map[string]TokenType{ _TokenTypeName[0:4]: None, @@ -373,58 +375,60 @@ var _TokenTypeNameToValueMap = map[string]TokenType{ _TokenTypeLowerName[962:986]: LiteralNumberIntegerLong, _TokenTypeName[986:1002]: LiteralNumberOct, _TokenTypeLowerName[986:1002]: LiteralNumberOct, - _TokenTypeName[1002:1010]: Operator, - _TokenTypeLowerName[1002:1010]: Operator, - _TokenTypeName[1010:1022]: OperatorWord, - _TokenTypeLowerName[1010:1022]: OperatorWord, - _TokenTypeName[1022:1033]: Punctuation, - _TokenTypeLowerName[1022:1033]: Punctuation, - _TokenTypeName[1033:1040]: Comment, - _TokenTypeLowerName[1033:1040]: Comment, - _TokenTypeName[1040:1055]: CommentHashbang, - _TokenTypeLowerName[1040:1055]: CommentHashbang, - _TokenTypeName[1055:1071]: CommentMultiline, - _TokenTypeLowerName[1055:1071]: CommentMultiline, - _TokenTypeName[1071:1084]: CommentSingle, - _TokenTypeLowerName[1071:1084]: CommentSingle, - _TokenTypeName[1084:1098]: CommentSpecial, - _TokenTypeLowerName[1084:1098]: CommentSpecial, - _TokenTypeName[1098:1112]: CommentPreproc, - _TokenTypeLowerName[1098:1112]: CommentPreproc, - _TokenTypeName[1112:1130]: CommentPreprocFile, - _TokenTypeLowerName[1112:1130]: CommentPreprocFile, - _TokenTypeName[1130:1137]: Generic, - _TokenTypeLowerName[1130:1137]: Generic, - _TokenTypeName[1137:1151]: GenericDeleted, - _TokenTypeLowerName[1137:1151]: GenericDeleted, - _TokenTypeName[1151:1162]: GenericEmph, - _TokenTypeLowerName[1151:1162]: GenericEmph, - _TokenTypeName[1162:1174]: GenericError, - _TokenTypeLowerName[1162:1174]: GenericError, - _TokenTypeName[1174:1188]: GenericHeading, - _TokenTypeLowerName[1174:1188]: GenericHeading, - _TokenTypeName[1188:1203]: GenericInserted, - _TokenTypeLowerName[1188:1203]: GenericInserted, - _TokenTypeName[1203:1216]: GenericOutput, - _TokenTypeLowerName[1203:1216]: GenericOutput, - _TokenTypeName[1216:1229]: GenericPrompt, - _TokenTypeLowerName[1216:1229]: GenericPrompt, - _TokenTypeName[1229:1242]: GenericStrong, - _TokenTypeLowerName[1229:1242]: GenericStrong, - _TokenTypeName[1242:1259]: GenericSubheading, - _TokenTypeLowerName[1242:1259]: GenericSubheading, - _TokenTypeName[1259:1275]: GenericTraceback, - _TokenTypeLowerName[1259:1275]: GenericTraceback, - _TokenTypeName[1275:1291]: GenericUnderline, - _TokenTypeLowerName[1275:1291]: GenericUnderline, - _TokenTypeName[1291:1295]: Text, - _TokenTypeLowerName[1291:1295]: Text, - _TokenTypeName[1295:1309]: TextWhitespace, - _TokenTypeLowerName[1295:1309]: TextWhitespace, - _TokenTypeName[1309:1319]: TextSymbol, - _TokenTypeLowerName[1309:1319]: TextSymbol, - _TokenTypeName[1319:1334]: TextPunctuation, - _TokenTypeLowerName[1319:1334]: TextPunctuation, + _TokenTypeName[1002:1019]: LiteralNumberByte, + _TokenTypeLowerName[1002:1019]: LiteralNumberByte, + _TokenTypeName[1019:1027]: Operator, + _TokenTypeLowerName[1019:1027]: Operator, + _TokenTypeName[1027:1039]: OperatorWord, + _TokenTypeLowerName[1027:1039]: OperatorWord, + _TokenTypeName[1039:1050]: Punctuation, + _TokenTypeLowerName[1039:1050]: Punctuation, + _TokenTypeName[1050:1057]: Comment, + _TokenTypeLowerName[1050:1057]: Comment, + _TokenTypeName[1057:1072]: CommentHashbang, + _TokenTypeLowerName[1057:1072]: CommentHashbang, + _TokenTypeName[1072:1088]: CommentMultiline, + _TokenTypeLowerName[1072:1088]: CommentMultiline, + _TokenTypeName[1088:1101]: CommentSingle, + _TokenTypeLowerName[1088:1101]: CommentSingle, + _TokenTypeName[1101:1115]: CommentSpecial, + _TokenTypeLowerName[1101:1115]: CommentSpecial, + _TokenTypeName[1115:1129]: CommentPreproc, + _TokenTypeLowerName[1115:1129]: CommentPreproc, + _TokenTypeName[1129:1147]: CommentPreprocFile, + _TokenTypeLowerName[1129:1147]: CommentPreprocFile, + _TokenTypeName[1147:1154]: Generic, + _TokenTypeLowerName[1147:1154]: Generic, + _TokenTypeName[1154:1168]: GenericDeleted, + _TokenTypeLowerName[1154:1168]: GenericDeleted, + _TokenTypeName[1168:1179]: GenericEmph, + _TokenTypeLowerName[1168:1179]: GenericEmph, + _TokenTypeName[1179:1191]: GenericError, + _TokenTypeLowerName[1179:1191]: GenericError, + _TokenTypeName[1191:1205]: GenericHeading, + _TokenTypeLowerName[1191:1205]: GenericHeading, + _TokenTypeName[1205:1220]: GenericInserted, + _TokenTypeLowerName[1205:1220]: GenericInserted, + _TokenTypeName[1220:1233]: GenericOutput, + _TokenTypeLowerName[1220:1233]: GenericOutput, + _TokenTypeName[1233:1246]: GenericPrompt, + _TokenTypeLowerName[1233:1246]: GenericPrompt, + _TokenTypeName[1246:1259]: GenericStrong, + _TokenTypeLowerName[1246:1259]: GenericStrong, + _TokenTypeName[1259:1276]: GenericSubheading, + _TokenTypeLowerName[1259:1276]: GenericSubheading, + _TokenTypeName[1276:1292]: GenericTraceback, + _TokenTypeLowerName[1276:1292]: GenericTraceback, + _TokenTypeName[1292:1308]: GenericUnderline, + _TokenTypeLowerName[1292:1308]: GenericUnderline, + _TokenTypeName[1308:1312]: Text, + _TokenTypeLowerName[1308:1312]: Text, + _TokenTypeName[1312:1326]: TextWhitespace, + _TokenTypeLowerName[1312:1326]: TextWhitespace, + _TokenTypeName[1326:1336]: TextSymbol, + _TokenTypeLowerName[1326:1336]: TextSymbol, + _TokenTypeName[1336:1351]: TextPunctuation, + _TokenTypeLowerName[1336:1351]: TextPunctuation, } var _TokenTypeNames = []string{ @@ -501,32 +505,33 @@ var _TokenTypeNames = []string{ _TokenTypeName[942:962], _TokenTypeName[962:986], _TokenTypeName[986:1002], - _TokenTypeName[1002:1010], - _TokenTypeName[1010:1022], - _TokenTypeName[1022:1033], - _TokenTypeName[1033:1040], - _TokenTypeName[1040:1055], - _TokenTypeName[1055:1071], - _TokenTypeName[1071:1084], - _TokenTypeName[1084:1098], - _TokenTypeName[1098:1112], - _TokenTypeName[1112:1130], - _TokenTypeName[1130:1137], - _TokenTypeName[1137:1151], - _TokenTypeName[1151:1162], - _TokenTypeName[1162:1174], - _TokenTypeName[1174:1188], - _TokenTypeName[1188:1203], - _TokenTypeName[1203:1216], - _TokenTypeName[1216:1229], - _TokenTypeName[1229:1242], - _TokenTypeName[1242:1259], - _TokenTypeName[1259:1275], - _TokenTypeName[1275:1291], - _TokenTypeName[1291:1295], - _TokenTypeName[1295:1309], - _TokenTypeName[1309:1319], - _TokenTypeName[1319:1334], + _TokenTypeName[1002:1019], + _TokenTypeName[1019:1027], + _TokenTypeName[1027:1039], + _TokenTypeName[1039:1050], + _TokenTypeName[1050:1057], + _TokenTypeName[1057:1072], + _TokenTypeName[1072:1088], + _TokenTypeName[1088:1101], + _TokenTypeName[1101:1115], + _TokenTypeName[1115:1129], + _TokenTypeName[1129:1147], + _TokenTypeName[1147:1154], + _TokenTypeName[1154:1168], + _TokenTypeName[1168:1179], + _TokenTypeName[1179:1191], + _TokenTypeName[1191:1205], + _TokenTypeName[1205:1220], + _TokenTypeName[1220:1233], + _TokenTypeName[1233:1246], + _TokenTypeName[1246:1259], + _TokenTypeName[1259:1276], + _TokenTypeName[1276:1292], + _TokenTypeName[1292:1308], + _TokenTypeName[1308:1312], + _TokenTypeName[1312:1326], + _TokenTypeName[1326:1336], + _TokenTypeName[1336:1351], } // TokenTypeString retrieves an enum value from the enum constants string name. diff --git a/types.go b/types.go index 3d12310..d65d390 100644 --- a/types.go +++ b/types.go @@ -122,6 +122,7 @@ const ( LiteralNumberInteger LiteralNumberIntegerLong LiteralNumberOct + LiteralNumberByte ) // Operators.