diff --git a/lexers/r/raku.go b/lexers/r/raku.go index 71353b8..6eb7e90 100644 --- a/lexers/r/raku.go +++ b/lexers/r/raku.go @@ -12,7 +12,7 @@ import ( ) // Raku lexer. -var Raku Lexer = internal.Register(MustNewLexer( +var Raku Lexer = internal.Register(MustNewLazyLexer( &Config{ Name: "Raku", Aliases: []string{"perl6", "pl6", "raku"}, @@ -26,921 +26,1156 @@ var Raku Lexer = internal.Register(MustNewLexer( }, DotAll: true, }, - RakuRules, + rakuRules, )) -// Raku rules -// Empty capture groups are placeholders and will be replaced by bracketsFinder. -// DO NOT REMOVE THEM! -var RakuRules = Rules{ - "root": { - Include("common"), - {`\{|\}|\(|\)|\[|\]`, Punctuation, nil}, - {`.+?`, Text, nil}, - }, - "common": { - {`^#![^\n]*$`, CommentHashbang, nil}, - Include("pod"), - // Multi-line, Embedded comment - { - "#`((" + bracketsPattern + `)(\2)*)`, - CommentMultiline, - bracketsFinder(rakuMultilineComment), +func rakuRules() Rules { + type RakuToken int + + const ( + rakuQuote RakuToken = iota + rakuName + rakuNameAttribute + rakuPod + rakuPodFormatter + rakuPodDeclaration + rakuMultilineComment + rakuSlashRegex + rakuMatchRegex + rakuSubstitutionRegex + rakuSubstitutionSingleRegex + rakuRegexInsideToken + ) + + const ( + colonPairOpeningBrackets = `(?:<<|<|«|\(|\[|\{)` + colonPairClosingBrackets = `(?:>>|>|»|\)|\]|\})` + colonPairPattern = `(:)(\w[\w'-]*)(` + colonPairOpeningBrackets + `)()()` + namePattern = `((?:(?!` + colonPairPattern + `)[\w':-])+)` + variablePattern = `[$@%&]+[.^:?=!~]?` + namePattern + globalVariablePattern = `[$@%&]+\*` + namePattern + ) + + keywords := []string{ + `BEGIN`, `CATCH`, `CHECK`, `CLOSE`, `CONTROL`, `DOC`, `END`, `ENTER`, `FIRST`, `INIT`, + `KEEP`, `LAST`, `LEAVE`, `NEXT`, `POST`, `PRE`, `QUIT`, `UNDO`, `anon`, `augment`, `but`, + `class`, `constant`, `default`, `does`, `else`, `elsif`, `enum`, `for`, `gather`, `given`, + `grammar`, `has`, `if`, `import`, `is`, `of`, `let`, `loop`, `made`, `make`, `method`, + `module`, `multi`, `my`, `need`, `orwith`, `our`, `proceed`, `proto`, `repeat`, `require`, + `where`, `return`, `return-rw`, `returns`, `->`, `-->`, `role`, `state`, `sub`, `no`, + `submethod`, `subset`, `succeed`, `supersede`, `try`, `unit`, `unless`, `until`, + `use`, `when`, `while`, `with`, `without`, `export`, `native`, `repr`, `required`, `rw`, + `symbol`, `default`, `cached`, `DEPRECATED`, `dynamic`, `hidden-from-backtrace`, `nodal`, + `pure`, `raw`, `start`, `react`, `supply`, `whenever`, `also`, `rule`, `token`, `regex`, + `dynamic-scope`, `built`, `temp`, + } + + keywordsPattern := Words(`(?)`, `(>=)`, `minmax`, `notandthen`, `S`, + } + + wordOperatorsPattern := Words(`(?<=^|\b|\s)`, `(?=$|\b|\s)`, sortWords(wordOperators)...) + + operators := []string{ + `++`, `--`, `-`, `**`, `!`, `+`, `~`, `?`, `+^`, `~^`, `?^`, `^`, `*`, `/`, `%`, `%%`, `+&`, + `+<`, `+>`, `~&`, `~<`, `~>`, `?&`, `+|`, `+^`, `~|`, `~^`, `?`, `?|`, `?^`, `&`, `^`, + `<=>`, `^…^`, `^…`, `…^`, `…`, `...`, `...^`, `^...`, `^...^`, `..`, `..^`, `^..`, `^..^`, + `::=`, `:=`, `!=`, `==`, `<=`, `<`, `>=`, `>`, `~~`, `===`, `&&`, `||`, `|`, `^^`, `//`, + `??`, `!!`, `^fff^`, `^ff^`, `<==`, `==>`, `<<==`, `==>>`, `=>`, `=`, `<<`, `«`, `>>`, `»`, + `,`, `>>.`, `».`, `.&`, `.=`, `.^`, `.?`, `.+`, `.*`, `.`, `∘`, `∩`, `⊍`, `∪`, `⊎`, `∖`, + `⊖`, `≠`, `≤`, `≥`, `=:=`, `=~=`, `≅`, `∈`, `∉`, `≡`, `≢`, `∋`, `∌`, `⊂`, `⊄`, `⊆`, `⊈`, + `⊃`, `⊅`, `⊇`, `⊉`, `:`, `!!!`, `???`, `¯`, `×`, `÷`, `−`, `⁺`, `⁻`, + } + + operatorsPattern := Words(``, ``, sortWords(operators)...) + + builtinTypes := []string{ + `False`, `True`, `Order`, `More`, `Less`, `Same`, `Any`, `Array`, `Associative`, `AST`, + `atomicint`, `Attribute`, `Backtrace`, `Backtrace::Frame`, `Bag`, `Baggy`, `BagHash`, + `Blob`, `Block`, `Bool`, `Buf`, `Callable`, `CallFrame`, `Cancellation`, `Capture`, + `CArray`, `Channel`, `Code`, `compiler`, `Complex`, `ComplexStr`, `CompUnit`, + `CompUnit::PrecompilationRepository`, `CompUnit::Repository`, `Empty`, + `CompUnit::Repository::FileSystem`, `CompUnit::Repository::Installation`, `Cool`, + `CurrentThreadScheduler`, `CX::Warn`, `CX::Take`, `CX::Succeed`, `CX::Return`, `CX::Redo`, + `CX::Proceed`, `CX::Next`, `CX::Last`, `CX::Emit`, `CX::Done`, `Cursor`, `Date`, `Dateish`, + `DateTime`, `Distribution`, `Distribution::Hash`, `Distribution::Locally`, + `Distribution::Path`, `Distribution::Resource`, `Distro`, `Duration`, `Encoding`, + `Encoding::Registry`, `Endian`, `Enumeration`, `Exception`, `Failure`, `FatRat`, `Grammar`, + `Hash`, `HyperWhatever`, `Instant`, `Int`, `int`, `int16`, `int32`, `int64`, `int8`, `str`, + `IntStr`, `IO`, `IO::ArgFiles`, `IO::CatHandle`, `IO::Handle`, `IO::Notification`, + `IO::Notification::Change`, `IO::Path`, `IO::Path::Cygwin`, `IO::Path::Parts`, + `IO::Path::QNX`, `IO::Path::Unix`, `IO::Path::Win32`, `IO::Pipe`, `IO::Socket`, + `IO::Socket::Async`, `IO::Socket::Async::ListenSocket`, `IO::Socket::INET`, `IO::Spec`, + `IO::Spec::Cygwin`, `IO::Spec::QNX`, `IO::Spec::Unix`, `IO::Spec::Win32`, `IO::Special`, + `Iterable`, `Iterator`, `Junction`, `Kernel`, `Label`, `List`, `Lock`, `Lock::Async`, + `Lock::ConditionVariable`, `long`, `longlong`, `Macro`, `Map`, `Match`, + `Metamodel::AttributeContainer`, `Metamodel::C3MRO`, `Metamodel::ClassHOW`, + `Metamodel::ConcreteRoleHOW`, `Metamodel::CurriedRoleHOW`, `Metamodel::DefiniteHOW`, + `Metamodel::Documenting`, `Metamodel::EnumHOW`, `Metamodel::Finalization`, + `Metamodel::MethodContainer`, `Metamodel::Mixins`, `Metamodel::MROBasedMethodDispatch`, + `Metamodel::MultipleInheritance`, `Metamodel::Naming`, `Metamodel::Primitives`, + `Metamodel::PrivateMethodContainer`, `Metamodel::RoleContainer`, `Metamodel::RolePunning`, + `Metamodel::Stashing`, `Metamodel::Trusting`, `Metamodel::Versioning`, `Method`, `Mix`, + `MixHash`, `Mixy`, `Mu`, `NFC`, `NFD`, `NFKC`, `NFKD`, `Nil`, `Num`, `num32`, `num64`, + `Numeric`, `NumStr`, `ObjAt`, `Order`, `Pair`, `Parameter`, `Perl`, `Pod::Block`, + `Pod::Block::Code`, `Pod::Block::Comment`, `Pod::Block::Declarator`, `Pod::Block::Named`, + `Pod::Block::Para`, `Pod::Block::Table`, `Pod::Heading`, `Pod::Item`, `Pointer`, + `Positional`, `PositionalBindFailover`, `Proc`, `Proc::Async`, `Promise`, `Proxy`, + `PseudoStash`, `QuantHash`, `RaceSeq`, `Raku`, `Range`, `Rat`, `Rational`, `RatStr`, + `Real`, `Regex`, `Routine`, `Routine::WrapHandle`, `Scalar`, `Scheduler`, `Semaphore`, + `Seq`, `Sequence`, `Set`, `SetHash`, `Setty`, `Signature`, `size_t`, `Slip`, `Stash`, + `Str`, `StrDistance`, `Stringy`, `Sub`, `Submethod`, `Supplier`, `Supplier::Preserving`, + `Supply`, `Systemic`, `Tap`, `Telemetry`, `Telemetry::Instrument::Thread`, + `Telemetry::Instrument::ThreadPool`, `Telemetry::Instrument::Usage`, `Telemetry::Period`, + `Telemetry::Sampler`, `Thread`, `Test`, `ThreadPoolScheduler`, `UInt`, `uint16`, `uint32`, + `uint64`, `uint8`, `Uni`, `utf8`, `ValueObjAt`, `Variable`, `Version`, `VM`, `Whatever`, + `WhateverCode`, `WrapHandle`, `NativeCall`, + // Pragmas + `precompilation`, `experimental`, `worries`, `MONKEY-TYPING`, `MONKEY-SEE-NO-EVAL`, + `MONKEY-GUTS`, `fatal`, `lib`, `isms`, `newline`, `nqp`, `soft`, + `strict`, `trace`, `variables`, + } + + builtinTypesPattern := Words(`(? 0 { + nextOpenPos := indexAt(text, openingChars, searchPos+nChars) + nextClosePos = indexAt(text, closingChars, searchPos+nChars) + + switch { + case nextClosePos == -1: + nextClosePos = len(text) + nestingLevel = 0 + case nextOpenPos != -1 && nextOpenPos < nextClosePos: + nestingLevel++ + nChars = len(openingChars) + searchPos = nextOpenPos + default: // next_close_pos < next_open_pos + nestingLevel-- + nChars = len(closingChars) + searchPos = nextClosePos + } + } + + endPos = nextClosePos + } + + if endPos < 0 { + // if we didn't find a closer, just highlight the + // rest of the text in this class + endPos = len(text) + } + + adverbre := regexp.MustCompile(`:to\b|:heredoc\b`) + var heredocTerminator []rune + if adverbre.MatchString(string(adverbs)) { + heredocTerminator = text[state.Pos-1+nChars : endPos] + if len(heredocTerminator) > 0 { + endHeredocPos := indexAt(text[endPos:], heredocTerminator, 0) + nChars = len(heredocTerminator) + endPos += endHeredocPos + } else { + endPos = len(text) + } + } + + textBetweenBrackets := string(text[state.Pos:endPos]) + switch tokenClass { + case rakuPodDeclaration: + state.Groups[3] = "" + state.Groups[4] = "" + state.Groups[5] = textBetweenBrackets + state.Groups[6] = string(closingChars) + case rakuPod: + state.Groups[6] = textBetweenBrackets + state.Groups[7] = string(closingChars) + case rakuQuote: + state.Groups[7] = "" + state.Groups[8] = "" + if len(heredocTerminator) > 0 { + // Length of heredoc terminator + closing chars + `;` + heredocFristPunctuationLen := len(heredocTerminator) + len(openingChars) + 1 + + state.Groups[6] = string(openingChars) + + string(text[state.Pos:state.Pos+heredocFristPunctuationLen]) + + state.Groups[9] = + string(text[state.Pos+heredocFristPunctuationLen : endPos]) + + state.Groups[10] = string(heredocTerminator) + } else { + state.Groups[9] = textBetweenBrackets + state.Groups[10] = string(closingChars) + } + case rakuNameAttribute: + state.Groups[4] = textBetweenBrackets + state.Groups[5] = string(closingChars) + default: + state.Groups = []string{state.Groups[0] + string(text[state.Pos:endPos+nChars])} + } + + state.Pos = endPos + nChars + + return nil + } + } + + // Raku rules + // Empty capture groups are placeholders and will be replaced by bracketsFinder. + // DO NOT REMOVE THEM! + return Rules{ + "root": { + Include("common"), + {`\{|\}|\(|\)|\[|\]`, Punctuation, nil}, + {`.+?`, Text, nil}, }, - {`#[^\n]*$`, CommentSingle, nil}, - // /regex/ - { - `(?<=(?:^|\(|=|:|~~|\[|,|=>)\s*)(/)(?!\]|\))((?:\\\\|\\/|.)*?)((?)\s*)(/)(?!\]|\))((?:\\\\|\\/|.)*?)((?>)(\S+?)(<<)`, ByGroups(Operator, UsingSelf("root"), Operator), nil}, + {`(»)(\S+?)(«)`, ByGroups(Operator, UsingSelf("root"), Operator), nil}, + // <> + {`(?>)[^\n])+?[},;] *\n)(?!(?:(?!>>).)+?>>\S+?>>)`, Punctuation, Push("<<")}, + // «quoted words» + {`(? operators | something < onething > something + { + `(?<=[$@%&]?\w[\w':-]* +)(<=?)( *[^ ]+? *)(>=?)(?= *[$@%&]?\w[\w':-]*)`, + ByGroups(Operator, UsingSelf("root"), Operator), + nil, + }, + // + { + `(?])+?)(>)(?!\s*(?:\d+|\.(?:Int|Numeric)|[$@%]\w[\w':-]*[^(]|\[))`, + ByGroups(Punctuation, String, Punctuation), + nil, + }, + {`C?X::['\w:-]+`, NameException, nil}, + Include("metaoperator"), + // Pair | (key) => value + { + `(\([^)]+\))(\s*)(=>)(\s*)([^,\n]+)(,?\n*)`, + ByGroups(UsingSelf("root"), Text, Operator, Text, UsingSelf("root"), Text), + nil, + }, + // Pair | key => value + { + `(\w[\w'-]*)(\s*)(=>)(\s*)([^,\n]+)(,?\n*)`, + ByGroups(String, Text, Operator, Text, UsingSelf("root"), Text), + nil, + }, + Include("colon-pair"), + // Token + { + // Token with adverbs + `(?<=(?:^|\s)(?:regex|token|rule)(\s+))(['\w:-]+)(?=:['\w-]+` + + colonPairOpeningBrackets + `.+?` + colonPairClosingBrackets + `[({])`, + NameFunction, + Push("token", "name-adverb"), + }, + { + // Token without adverbs + `(?<=(?:^|\s)(?:regex|token|rule)(?:\s+))(['\w:-]+)`, + NameFunction, + Push("token"), + }, + // Substitution + {`(?<=^|\b|\s)(?>)(\S+?)(<<)`, ByGroups(Operator, UsingSelf("root"), Operator), nil}, - {`(»)(\S+?)(«)`, ByGroups(Operator, UsingSelf("root"), Operator), nil}, - // <> - {`(?>)[^\n])+?[},;] *\n)(?!(?:(?!>>).)+?>>\S+?>>)`, Punctuation, Push("<<")}, - // «quoted words» - {`(? operators | something < onething > something - { - `(?<=[$@%&]?\w[\w':-]* +)(<=?)( *[^ ]+? *)(>=?)(?= *[$@%&]?\w[\w':-]*)`, - ByGroups(Operator, UsingSelf("root"), Operator), - nil, + "substitution": { + Include("colon-pair-attribute"), + // Substitution | s{regex} = value + { + `(\s*)((` + bracketsPattern + `)(\3)*)`, + ByGroups(Text, Punctuation, Punctuation, Punctuation), + Mutators(Pop(1), bracketsFinder(rakuSubstitutionSingleRegex)), + }, + // s/regex/string/ + { + `(\s*)([^\w:\s])((?:\\\\|\\/|.)*?)(\2)((?:\\\\|\\/|.)*?)(\2)`, + ByGroups( + Text, Punctuation, UsingSelf("regex"), Punctuation, UsingSelf("qq"), Punctuation, + ), + Mutators(Pop(1), bracketsFinder(rakuSubstitutionRegex)), + }, }, - // - { - `(?])+?)(>)(?!\s*(?:\d+|\.(?:Int|Numeric)|[$@%]\w[\w':-]*[^(]|\[))`, - ByGroups(Punctuation, String, Punctuation), - nil, + "number": { + {`0_?[0-7]+(_[0-7]+)*`, LiteralNumberOct, nil}, + {`0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*`, LiteralNumberHex, nil}, + {`0b[01]+(_[01]+)*`, LiteralNumberBin, nil}, + { + `(?i)(\d*(_\d*)*\.\d+(_\d*)*|\d+(_\d*)*\.\d+(_\d*)*)(e[+-]?\d+)?`, + LiteralNumberFloat, + nil, + }, + {`(?i)\d+(_\d*)*e[+-]?\d+(_\d*)*`, LiteralNumberFloat, nil}, + {`(?<=\d+)i`, NameConstant, nil}, + {`\d+(_\d+)*`, LiteralNumberInteger, nil}, }, - {`C?X::['\w:-]+`, NameException, nil}, - Include("metaoperator"), - // Pair | (key) => value - { - `(\([^)]+\))(\s*)(=>)(\s*)([^,\n]+)(,?\n*)`, - ByGroups(UsingSelf("root"), Text, Operator, Text, UsingSelf("root"), Text), - nil, + "name-adverb": { + Include("colon-pair-attribute-keyvalue"), + Default(Pop(1)), }, - // Pair | key => value - { - `(\w[\w'-]*)(\s*)(=>)(\s*)([^,\n]+)(,?\n*)`, - ByGroups(String, Text, Operator, Text, UsingSelf("root"), Text), - nil, + "colon-pair": { + // :key(value) + {colonPairPattern, colonPair(String), bracketsFinder(rakuNameAttribute)}, + // :key + {`(:!?)(\w[\w'-]*)`, ByGroups(Punctuation, String), nil}, + // :123abc + { + `(:)(\d+)(\w[\w'-]*)(\s*[,;)]?\s*$)`, + ByGroups(Punctuation, UsingSelf("number"), String, Text), + nil, + }, + {`\s+`, Text, nil}, }, - Include("colon-pair"), - // Token - { - // Token with adverbs - `(?<=(?:^|\s)(?:regex|token|rule)(\s+))(['\w:-]+)(?=:['\w-]+` + - colonPairOpeningBrackets + `.+?` + colonPairClosingBrackets + `[({])`, - NameFunction, - Push("token", "name-adverb"), + "colon-pair-attribute": { + // :key(value) + {colonPairPattern, colonPair(NameAttribute), bracketsFinder(rakuNameAttribute)}, + // :key + {`(:!?)(\w[\w'-]*)`, ByGroups(Punctuation, NameAttribute), nil}, + // :123abc + { + `(:)(\d+)(\w+)(\s*[,;)]?\s*$)`, + ByGroups(Punctuation, UsingSelf("number"), NameAttribute, Text), + nil, + }, + {`\s+`, Text, nil}, }, - { - // Token without adverbs - `(?<=(?:^|\s)(?:regex|token|rule)(?:\s+))(['\w:-]+)`, - NameFunction, - Push("token"), + "colon-pair-attribute-keyvalue": { + // :key(value) + {colonPairPattern, colonPair(NameAttribute), bracketsFinder(rakuNameAttribute)}, }, - // Substitution - {`(?<=^|\b|\s)(? + { + `(?)`, + ByGroups(Punctuation, Operator, Punctuation, UsingSelf("root"), Punctuation), + nil, + }, + // {code} + {`(?>|»|\+|\*\*|\*|\?|=|~|<~~>`, Operator, nil}, + // Anchors + {`\^\^|\^|\$\$|\$`, NameEntity, nil}, + {`\.`, NameEntity, nil}, + {`#[^\n]*\n`, CommentSingle, nil}, + // Lookaround + { + `(?)`, + ByGroups(Punctuation, Operator, OperatorWord, Punctuation), + nil, + }, + // <$variable> + { + `(?)`, + ByGroups(Punctuation, NameVariable, Punctuation), + nil, + }, + // Capture markers + {`(?`, Punctuation, nil}, + {`(?)`, Punctuation, Pop(1)}, + {`(?`, Punctuation, Pop(1)}, + Include("regex-class-builtin"), + Include("variable"), + // | | + { + `(?:(\w[\w-:]*)(=\.?))?(&?\w[\w'-:]+?)(\(.+?\))?(?=>)`, + ByGroups( + NameVariable, Operator, NameFunction, UsingSelf("root"), + ), + nil, + }, + // + { + `(&?\w[\w':-]*?)(:)((?:.*?(?:\$<\w[\w':-]*>)?.*?)*?)(?=>)`, + ByGroups( + NameFunction, Punctuation, UsingSelf("root"), + ), + nil, + }, + Include("colon-pair-attribute"), + {`(?] + { + `\b([RZX]+)\b(\[)([^\s\]]+?)(\])`, + ByGroups(OperatorWord, Punctuation, UsingSelf("root"), Punctuation), + nil, + }, + // Z=> + {`\b([RZX]+)\b([^\s\]]+)`, ByGroups(OperatorWord, UsingSelf("operator")), nil}, }, - // s/regex/string/ - { - `(\s*)([^\w:\s])((?:\\\\|\\/|.)*?)(\2)((?:\\\\|\\/|.)*?)(\2)`, - ByGroups( - Text, Punctuation, UsingSelf("regex"), Punctuation, UsingSelf("qq"), Punctuation, - ), - Mutators(Pop(1), bracketsFinder(rakuSubstitutionRegex)), + "operator": { + // Word Operator + {wordOperatorsPattern, OperatorWord, nil}, + // Operator + {operatorsPattern, Operator, nil}, }, - }, - "number": { - {`0_?[0-7]+(_[0-7]+)*`, LiteralNumberOct, nil}, - {`0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*`, LiteralNumberHex, nil}, - {`0b[01]+(_[01]+)*`, LiteralNumberBin, nil}, - { - `(?i)(\d*(_\d*)*\.\d+(_\d*)*|\d+(_\d*)*\.\d+(_\d*)*)(e[+-]?\d+)?`, - LiteralNumberFloat, - nil, + "pod": { + // Single-line pod declaration + {`(#[|=])\s`, Keyword, Push("pod-single")}, + // Multi-line pod declaration + { + "(#[|=])((" + bracketsPattern + `)(\3)*)()()`, + ByGroups( + Keyword, Punctuation, Punctuation, Punctuation, UsingSelf("pod-begin"), + Punctuation, + ), + bracketsFinder(rakuPodDeclaration), + }, + Include("pod-blocks"), }, - {`(?i)\d+(_\d*)*e[+-]?\d+(_\d*)*`, LiteralNumberFloat, nil}, - {`(?<=\d+)i`, NameConstant, nil}, - {`\d+(_\d+)*`, LiteralNumberInteger, nil}, - }, - "name-adverb": { - Include("colon-pair-attribute-keyvalue"), - Default(Pop(1)), - }, - "colon-pair": { - // :key(value) - {colonPairPattern, colonPair(String), bracketsFinder(rakuNameAttribute)}, - // :key - {`(:!?)(\w[\w'-]*)`, ByGroups(Punctuation, String), nil}, - // :123abc - { - `(:)(\d+)(\w[\w'-]*)(\s*[,;)]?\s*$)`, - ByGroups(Punctuation, UsingSelf("number"), String, Text), - nil, + "pod-blocks": { + // =begin code + { + `(?<=^ *)( *)(=begin)( +)(code)([^\n]*)(.*?)(^\1=)(end)( +)(\4)`, + EmitterFunc(podCode), + nil, + }, + // =begin + { + `(?<=^ *)( *)(=begin)( +)(?!code)(\w[\w'-]*)([^\n]*)()()`, + ByGroups( + Comment, Keyword, Comment, Keyword, EmitterFunc(podConfig), + UsingSelf("pod-begin"), Keyword, + ), + bracketsFinder(rakuPod), + }, + // =for ... + { + `(?<=^ *)( *=)(for|defn)( +)(\w[\w'-]*)([^\n]*\n)`, + ByGroups(Keyword, Keyword, StringDoc, Keyword, EmitterFunc(podConfig)), + Push("pod-paragraph"), + }, + // =config + { + `(?<=^ *)( *=)(config)( +)(\w[\w'-]*)([^\n]*\n)`, + ByGroups(Keyword, Keyword, StringDoc, Keyword, EmitterFunc(podConfig)), + nil, + }, + // =alias + { + `(?<=^ *)( *=)(alias)( +)(\w[\w'-]*)([^\n]*\n)`, + ByGroups(Keyword, Keyword, StringDoc, Keyword, StringDoc), + nil, + }, + // =encoding + { + `(?<=^ *)( *=)(encoding)( +)([^\n]+)`, + ByGroups(Keyword, Keyword, StringDoc, Name), + nil, + }, + // =para ... + { + `(?<=^ *)( *=)(para|table|pod)((? - { - `(?)`, - ByGroups(Punctuation, Operator, Punctuation, UsingSelf("root"), Punctuation), - nil, + "pre-pod-formatter": { + // C, B, ... + { + `([CBIUDTKRPAELZVMSXN])(<+|«)`, + ByGroups(Keyword, Punctuation), + Mutators( + bracketsFinder(rakuPodFormatter), + Push("pod-formatter"), MutatorFunc(podFormatter), + ), + }, }, - // {code} - {`(?`, Punctuation, Pop(1)}, + Include("pre-pod-formatter"), + // Placeholder rule, will be replaced by podFormatter. DO NOT REMOVE! + {`.+?`, StringOther, nil}, }, - // Properties - {`(:)(\w+)`, ByGroups(Punctuation, NameAttribute), nil}, - // Operator - {`\|\||\||&&|&|\.\.|\*\*|%%|%|:|!|<<|«|>>|»|\+|\*\*|\*|\?|=|~|<~~>`, Operator, nil}, - // Anchors - {`\^\^|\^|\$\$|\$`, NameEntity, nil}, - {`\.`, NameEntity, nil}, - {`#[^\n]*\n`, CommentSingle, nil}, - // Lookaround - { - `(?)+`, NameVariable, nil}, + {`\$/`, NameVariable, nil}, + {`\$!`, NameVariable, nil}, + {`[$@%]`, NameVariable, nil}, }, - { - `(?)`, - ByGroups(Punctuation, Operator, OperatorWord, Punctuation), - nil, + "single-quote": { + {`(? - { - `(?)`, - ByGroups(Punctuation, NameVariable, Punctuation), - nil, + "single-quote-inner": { + {`(?`, Punctuation, nil}, - {`(?)`, Punctuation, Pop(1)}, - {`(?`, Punctuation, Pop(1)}, - Include("regex-class-builtin"), - Include("variable"), - // | | - { - `(?:(\w[\w-:]*)(=\.?))?(&?\w[\w'-:]+?)(\(.+?\))?(?=>)`, - ByGroups( - NameVariable, Operator, NameFunction, UsingSelf("root"), - ), - nil, + "<<": { + {`>>(?!\s*(?:\d+|\.(?:Int|Numeric)|[$@%][\w':-]+|\[))`, Punctuation, Pop(1)}, + Include("ww"), }, - // - { - `(&?\w[\w':-]*?)(:)((?:.*?(?:\$<\w[\w':-]*>)?.*?)*?)(?=>)`, - ByGroups( - NameFunction, Punctuation, UsingSelf("root"), - ), - nil, + "«": { + {`»(?!\s*(?:\d+|\.(?:Int|Numeric)|[$@%][\w':-]+|\[))`, Punctuation, Pop(1)}, + Include("ww"), }, - Include("colon-pair-attribute"), - {`(?] - { - `\b([RZX]+)\b(\[)([^\s\]]+?)(\])`, - ByGroups(OperatorWord, Punctuation, UsingSelf("root"), Punctuation), - nil, + "ww": { + Include("single-quote"), + Include("qq"), }, - // Z=> - {`\b([RZX]+)\b([^\s\]]+)`, ByGroups(OperatorWord, UsingSelf("operator")), nil}, - }, - "operator": { - // Word Operator - {wordOperatorsPattern, OperatorWord, nil}, - // Operator - {operatorsPattern, Operator, nil}, - }, - "pod": { - // Single-line pod declaration - {`(#[|=])\s`, Keyword, Push("pod-single")}, - // Multi-line pod declaration - { - "(#[|=])((" + bracketsPattern + `)(\3)*)()()`, - ByGroups( - Keyword, Punctuation, Punctuation, Punctuation, UsingSelf("pod-begin"), - Punctuation, - ), - bracketsFinder(rakuPodDeclaration), + "qq": { + { + `(?>|<.*?>|«*?»)+`, UsingSelf("root"), nil}, + // Method + { + `(\.)([^(\s]+)(\([^"]*?\))`, + ByGroups(Operator, NameFunction, UsingSelf("root")), + nil, + }, + Default(Pop(1)), }, - // =for ... - { - `(?<=^ *)( *=)(for|defn)( +)(\w[\w'-]*)([^\n]*\n)`, - ByGroups(Keyword, Keyword, StringDoc, Keyword, EmitterFunc(podConfig)), - Push("pod-paragraph"), + "Q": { + Include("escape-qq"), + {`.+?`, String, nil}, }, - // =config - { - `(?<=^ *)( *=)(config)( +)(\w[\w'-]*)([^\n]*\n)`, - ByGroups(Keyword, Keyword, StringDoc, Keyword, EmitterFunc(podConfig)), - nil, + "Q-closure": { + Include("escape-qq"), + Include("closure"), + {`.+?`, String, nil}, }, - // =alias - { - `(?<=^ *)( *=)(alias)( +)(\w[\w'-]*)([^\n]*\n)`, - ByGroups(Keyword, Keyword, StringDoc, Keyword, StringDoc), - nil, + "Q-variable": { + Include("escape-qq"), + Include("variable"), + {`.+?`, String, nil}, }, - // =encoding - { - `(?<=^ *)( *=)(encoding)( +)([^\n]+)`, - ByGroups(Keyword, Keyword, StringDoc, Name), - nil, + "closure": { + {`(?, B, ... - { - `([CBIUDTKRPAELZVMSXN])(<+|«)`, - ByGroups(Keyword, Punctuation), - Mutators( - bracketsFinder(rakuPodFormatter), - Push("pod-formatter"), MutatorFunc(podFormatter), - ), - }, - }, - "pod-formatter": { - // Placeholder rule, will be replaced by podFormatter. DO NOT REMOVE! - {`>`, Punctuation, Pop(1)}, - Include("pre-pod-formatter"), - // Placeholder rule, will be replaced by podFormatter. DO NOT REMOVE! - {`.+?`, StringOther, nil}, - }, - "variable": { - {variablePattern, NameVariable, Push("name-adverb")}, - {globalVariablePattern, NameVariableGlobal, Push("name-adverb")}, - {`[$@](?:<.*?>)+`, NameVariable, nil}, - {`\$/`, NameVariable, nil}, - {`\$!`, NameVariable, nil}, - {`[$@%]`, NameVariable, nil}, - }, - "single-quote": { - {`(?>(?!\s*(?:\d+|\.(?:Int|Numeric)|[$@%][\w':-]+|\[))`, Punctuation, Pop(1)}, - Include("ww"), - }, - "«": { - {`»(?!\s*(?:\d+|\.(?:Int|Numeric)|[$@%][\w':-]+|\[))`, Punctuation, Pop(1)}, - Include("ww"), - }, - "ww": { - Include("single-quote"), - Include("qq"), - }, - "qq": { - { - `(?>|<.*?>|«*?»)+`, UsingSelf("root"), nil}, - // Method - { - `(\.)([^(\s]+)(\([^"]*?\))`, - ByGroups(Operator, NameFunction, UsingSelf("root")), - nil, - }, - Default(Pop(1)), - }, - "Q": { - Include("escape-qq"), - {`.+?`, String, nil}, - }, - "Q-closure": { - Include("escape-qq"), - Include("closure"), - {`.+?`, String, nil}, - }, - "Q-variable": { - Include("escape-qq"), - Include("variable"), - {`.+?`, String, nil}, - }, - "closure": { - {`(?>|>|»|\)|\]|\})` -const colonPairPattern = `(:)(\w[\w'-]*)(` + colonPairOpeningBrackets + `)()()` - -const variablePattern = `[$@%&]+[.^:?=!~]?` + namePattern -const globalVariablePattern = `[$@%&]+\*` + namePattern - -var keywords = []string{ - `BEGIN`, `CATCH`, `CHECK`, `CLOSE`, `CONTROL`, `DOC`, `END`, `ENTER`, `FIRST`, `INIT`, - `KEEP`, `LAST`, `LEAVE`, `NEXT`, `POST`, `PRE`, `QUIT`, `UNDO`, `anon`, `augment`, `but`, - `class`, `constant`, `default`, `does`, `else`, `elsif`, `enum`, `for`, `gather`, `given`, - `grammar`, `has`, `if`, `import`, `is`, `of`, `let`, `loop`, `made`, `make`, `method`, - `module`, `multi`, `my`, `need`, `orwith`, `our`, `proceed`, `proto`, `repeat`, `require`, - `where`, `return`, `return-rw`, `returns`, `->`, `-->`, `role`, `state`, `sub`, `no`, - `submethod`, `subset`, `succeed`, `supersede`, `try`, `unit`, `unless`, `until`, - `use`, `when`, `while`, `with`, `without`, `export`, `native`, `repr`, `required`, `rw`, - `symbol`, `default`, `cached`, `DEPRECATED`, `dynamic`, `hidden-from-backtrace`, `nodal`, - `pure`, `raw`, `start`, `react`, `supply`, `whenever`, `also`, `rule`, `token`, `regex`, - `dynamic-scope`, `built`, `temp`, -} - -var keywordsPattern = Words(`(?)`, `(>=)`, `minmax`, `notandthen`, `S`, -} - -var wordOperatorsPattern = Words(`(?<=^|\b|\s)`, `(?=$|\b|\s)`, sortWords(wordOperators)...) - -var operators = []string{ - `++`, `--`, `-`, `**`, `!`, `+`, `~`, `?`, `+^`, `~^`, `?^`, `^`, `*`, `/`, `%`, `%%`, `+&`, - `+<`, `+>`, `~&`, `~<`, `~>`, `?&`, `+|`, `+^`, `~|`, `~^`, `?`, `?|`, `?^`, `&`, `^`, - `<=>`, `^…^`, `^…`, `…^`, `…`, `...`, `...^`, `^...`, `^...^`, `..`, `..^`, `^..`, `^..^`, - `::=`, `:=`, `!=`, `==`, `<=`, `<`, `>=`, `>`, `~~`, `===`, `&&`, `||`, `|`, `^^`, `//`, - `??`, `!!`, `^fff^`, `^ff^`, `<==`, `==>`, `<<==`, `==>>`, `=>`, `=`, `<<`, `«`, `>>`, `»`, - `,`, `>>.`, `».`, `.&`, `.=`, `.^`, `.?`, `.+`, `.*`, `.`, `∘`, `∩`, `⊍`, `∪`, `⊎`, `∖`, - `⊖`, `≠`, `≤`, `≥`, `=:=`, `=~=`, `≅`, `∈`, `∉`, `≡`, `≢`, `∋`, `∌`, `⊂`, `⊄`, `⊆`, `⊈`, - `⊃`, `⊅`, `⊇`, `⊉`, `:`, `!!!`, `???`, `¯`, `×`, `÷`, `−`, `⁺`, `⁻`, -} - -var operatorsPattern = Words(``, ``, sortWords(operators)...) - -var builtinTypes = []string{ - `False`, `True`, `Order`, `More`, `Less`, `Same`, `Any`, `Array`, `Associative`, `AST`, - `atomicint`, `Attribute`, `Backtrace`, `Backtrace::Frame`, `Bag`, `Baggy`, `BagHash`, - `Blob`, `Block`, `Bool`, `Buf`, `Callable`, `CallFrame`, `Cancellation`, `Capture`, - `CArray`, `Channel`, `Code`, `compiler`, `Complex`, `ComplexStr`, `CompUnit`, - `CompUnit::PrecompilationRepository`, `CompUnit::Repository`, `Empty`, - `CompUnit::Repository::FileSystem`, `CompUnit::Repository::Installation`, `Cool`, - `CurrentThreadScheduler`, `CX::Warn`, `CX::Take`, `CX::Succeed`, `CX::Return`, `CX::Redo`, - `CX::Proceed`, `CX::Next`, `CX::Last`, `CX::Emit`, `CX::Done`, `Cursor`, `Date`, `Dateish`, - `DateTime`, `Distribution`, `Distribution::Hash`, `Distribution::Locally`, - `Distribution::Path`, `Distribution::Resource`, `Distro`, `Duration`, `Encoding`, - `Encoding::Registry`, `Endian`, `Enumeration`, `Exception`, `Failure`, `FatRat`, `Grammar`, - `Hash`, `HyperWhatever`, `Instant`, `Int`, `int`, `int16`, `int32`, `int64`, `int8`, `str`, - `IntStr`, `IO`, `IO::ArgFiles`, `IO::CatHandle`, `IO::Handle`, `IO::Notification`, - `IO::Notification::Change`, `IO::Path`, `IO::Path::Cygwin`, `IO::Path::Parts`, - `IO::Path::QNX`, `IO::Path::Unix`, `IO::Path::Win32`, `IO::Pipe`, `IO::Socket`, - `IO::Socket::Async`, `IO::Socket::Async::ListenSocket`, `IO::Socket::INET`, `IO::Spec`, - `IO::Spec::Cygwin`, `IO::Spec::QNX`, `IO::Spec::Unix`, `IO::Spec::Win32`, `IO::Special`, - `Iterable`, `Iterator`, `Junction`, `Kernel`, `Label`, `List`, `Lock`, `Lock::Async`, - `Lock::ConditionVariable`, `long`, `longlong`, `Macro`, `Map`, `Match`, - `Metamodel::AttributeContainer`, `Metamodel::C3MRO`, `Metamodel::ClassHOW`, - `Metamodel::ConcreteRoleHOW`, `Metamodel::CurriedRoleHOW`, `Metamodel::DefiniteHOW`, - `Metamodel::Documenting`, `Metamodel::EnumHOW`, `Metamodel::Finalization`, - `Metamodel::MethodContainer`, `Metamodel::Mixins`, `Metamodel::MROBasedMethodDispatch`, - `Metamodel::MultipleInheritance`, `Metamodel::Naming`, `Metamodel::Primitives`, - `Metamodel::PrivateMethodContainer`, `Metamodel::RoleContainer`, `Metamodel::RolePunning`, - `Metamodel::Stashing`, `Metamodel::Trusting`, `Metamodel::Versioning`, `Method`, `Mix`, - `MixHash`, `Mixy`, `Mu`, `NFC`, `NFD`, `NFKC`, `NFKD`, `Nil`, `Num`, `num32`, `num64`, - `Numeric`, `NumStr`, `ObjAt`, `Order`, `Pair`, `Parameter`, `Perl`, `Pod::Block`, - `Pod::Block::Code`, `Pod::Block::Comment`, `Pod::Block::Declarator`, `Pod::Block::Named`, - `Pod::Block::Para`, `Pod::Block::Table`, `Pod::Heading`, `Pod::Item`, `Pointer`, - `Positional`, `PositionalBindFailover`, `Proc`, `Proc::Async`, `Promise`, `Proxy`, - `PseudoStash`, `QuantHash`, `RaceSeq`, `Raku`, `Range`, `Rat`, `Rational`, `RatStr`, - `Real`, `Regex`, `Routine`, `Routine::WrapHandle`, `Scalar`, `Scheduler`, `Semaphore`, - `Seq`, `Sequence`, `Set`, `SetHash`, `Setty`, `Signature`, `size_t`, `Slip`, `Stash`, - `Str`, `StrDistance`, `Stringy`, `Sub`, `Submethod`, `Supplier`, `Supplier::Preserving`, - `Supply`, `Systemic`, `Tap`, `Telemetry`, `Telemetry::Instrument::Thread`, - `Telemetry::Instrument::ThreadPool`, `Telemetry::Instrument::Usage`, `Telemetry::Period`, - `Telemetry::Sampler`, `Thread`, `Test`, `ThreadPoolScheduler`, `UInt`, `uint16`, `uint32`, - `uint64`, `uint8`, `Uni`, `utf8`, `ValueObjAt`, `Variable`, `Version`, `VM`, `Whatever`, - `WhateverCode`, `WrapHandle`, `NativeCall`, - // Pragmas - `precompilation`, `experimental`, `worries`, `MONKEY-TYPING`, `MONKEY-SEE-NO-EVAL`, - `MONKEY-GUTS`, `fatal`, `lib`, `isms`, `newline`, `nqp`, `soft`, - `strict`, `trace`, `variables`, -} - -var builtinTypesPattern = Words(`(? -1 { @@ -984,246 +1219,13 @@ func contains(s []string, e string) bool { return false } -// Finds opening brackets and their closing counterparts (including pod and heredoc) -// and modifies state groups and position accordingly -func bracketsFinder(tokenClass RakuToken) MutatorFunc { - return func(state *LexerState) error { - var openingChars []rune - var adverbs []rune - switch tokenClass { - case rakuPodDeclaration: - openingChars = []rune(state.Groups[2]) - case rakuPod: - openingChars = []rune(strings.Join(state.Groups[1:5], ``)) - case rakuPodFormatter: - openingChars = []rune(state.Groups[2]) - case rakuMultilineComment: - openingChars = []rune(state.Groups[1]) - case rakuQuote: - adverbs = []rune(state.Groups[4]) - openingChars = []rune(state.Groups[6]) - case rakuSlashRegex: - openingChars = []rune(state.Groups[1]) - case rakuSubstitutionSingleRegex, rakuSubstitutionRegex: - openingChars = []rune(state.Groups[2]) - case rakuMatchRegex: - openingChars = []rune(state.Groups[2]) - case rakuRegexInsideToken: - openingChars = []rune("{") - case rakuNameAttribute: - openingChars = []rune(state.Groups[3]) - case rakuName: - openingChars = []rune(state.Groups[1]) - } - - var openingChar = openingChars[0] - - var nChars = len(openingChars) - - var closingChar rune - var closingCharExists bool - var closingChars []rune - - switch tokenClass { - case rakuPod: - closingChars = []rune(state.Groups[1] + `=end ` + state.Groups[4]) - closingCharExists = true - default: - closingChar, closingCharExists = brackets[openingChar] - } - - switch tokenClass { - case rakuPodFormatter: - var stack, ok = state.Get("pod_formatter_stack").([]RakuFormatterRules) - if !ok { - stack = []RakuFormatterRules{} - } - var popRule = makeRuleAndPushMaybe(RuleMakingConfig{ - delimiter: []rune{closingChar}, - numberOfDelimiterChars: nChars, - tokenType: Punctuation, - mutator: Mutators(Pop(1), MutatorFunc(podFormatterPopper)), - }) - var formatter TokenType = StringOther - switch state.Groups[1] { - case "B": - formatter = GenericStrong - case "I": - formatter = GenericEmph - case "U": - formatter = GenericUnderline - } - var formattingRule = makeRuleAndPushMaybe(RuleMakingConfig{ - pattern: `.+?`, - tokenType: formatter, - mutator: nil, - }) - state.Set("pod_formatter_stack", - append(stack, RakuFormatterRules{popRule, formattingRule})) - - return nil - case rakuSlashRegex, rakuMatchRegex, rakuSubstitutionRegex, - rakuSubstitutionSingleRegex, rakuRegexInsideToken: - // We're inside a regex! - - switch tokenClass { - // If the regex knows its own delimiter and uses `UsingSelf("regex")`, then we only - // put a placeholder rule at the top of "regex" state and return - case rakuSlashRegex, rakuSubstitutionRegex: - makeRuleAndPushMaybe(RuleMakingConfig{ - pattern: `^$`, - rulePosition: topRule, - state: state, - stateName: "regex", - }) - - return nil - default: - // While matching a regex, the closing chars may have been used inside the regex - // so we have to push to regex state and pop on the matched closing chars - // and return - var delimiter []rune - if closingCharExists { - delimiter = []rune{closingChar} - } else { - delimiter = openingChars - } - - makeRuleAndPushMaybe(RuleMakingConfig{ - delimiter: delimiter, - tokenType: Punctuation, - mutator: Pop(1), - rulePosition: topRule, - state: state, - stateName: "regex", - pushToStack: true, - numberOfDelimiterChars: nChars, - }) - - // Remove inner punctuation matches - switch tokenClass { - case rakuMatchRegex, rakuSubstitutionSingleRegex: - state.Groups[3] = "" - state.Groups[4] = "" - } - - return nil - } - } - - var text = state.Text - - var endPos int - - var nonMirroredOpeningCharPosition int - - if !closingCharExists { - // it's not a mirrored character, which means we - // just need to look for the next occurrence - nonMirroredOpeningCharPosition = indexAt(text, openingChars, state.Pos) - endPos = nonMirroredOpeningCharPosition - } else { - if tokenClass != rakuPod { - closingChars = []rune(strings.Repeat(string(closingChar), nChars)) - } - - // we need to look for the corresponding closing character, - // keep nesting in mind - var nestingLevel = 1 - - var searchPos = state.Pos - nChars - - var nextClosePos int - - for nestingLevel > 0 { - var nextOpenPos = indexAt(text, openingChars, searchPos+nChars) - nextClosePos = indexAt(text, closingChars, searchPos+nChars) - - switch { - case nextClosePos == -1: - nextClosePos = len(text) - nestingLevel = 0 - case nextOpenPos != -1 && nextOpenPos < nextClosePos: - nestingLevel++ - nChars = len(openingChars) - searchPos = nextOpenPos - default: // next_close_pos < next_open_pos - nestingLevel-- - nChars = len(closingChars) - searchPos = nextClosePos - } - } - - endPos = nextClosePos - } - - if endPos < 0 { - // if we didn't find a closer, just highlight the - // rest of the text in this class - endPos = len(text) - } - - var adverbre = regexp.MustCompile(`:to\b|:heredoc\b`) - var heredocTerminator []rune - if adverbre.MatchString(string(adverbs)) { - heredocTerminator = text[state.Pos-1+nChars : endPos] - if len(heredocTerminator) > 0 { - var endHeredocPos = indexAt(text[endPos:], heredocTerminator, 0) - nChars = len(heredocTerminator) - endPos += endHeredocPos - } else { - endPos = len(text) - } - } - - var textBetweenBrackets = string(text[state.Pos:endPos]) - switch tokenClass { - case rakuPodDeclaration: - state.Groups[3] = "" - state.Groups[4] = "" - state.Groups[5] = textBetweenBrackets - state.Groups[6] = string(closingChars) - case rakuPod: - state.Groups[6] = textBetweenBrackets - state.Groups[7] = string(closingChars) - case rakuQuote: - state.Groups[7] = "" - state.Groups[8] = "" - if len(heredocTerminator) > 0 { - // Length of heredoc terminator + closing chars + `;` - var heredocFristPunctuationLen = len(heredocTerminator) + len(openingChars) + 1 - - state.Groups[6] = string(openingChars) + - string(text[state.Pos:state.Pos+heredocFristPunctuationLen]) - - state.Groups[9] = - string(text[state.Pos+heredocFristPunctuationLen : endPos]) - - state.Groups[10] = string(heredocTerminator) - } else { - state.Groups[9] = textBetweenBrackets - state.Groups[10] = string(closingChars) - } - case rakuNameAttribute: - state.Groups[4] = textBetweenBrackets - state.Groups[5] = string(closingChars) - default: - state.Groups = []string{state.Groups[0] + string(text[state.Pos:endPos+nChars])} - } - - state.Pos = endPos + nChars - - return nil - } -} - type RakuFormatterRules struct { pop, formatter *CompiledRule } // Pop from the pod_formatter_stack and reformat the pod code func podFormatterPopper(state *LexerState) error { - var stack, ok = state.Get("pod_formatter_stack").([]RakuFormatterRules) + stack, ok := state.Get("pod_formatter_stack").([]RakuFormatterRules) if ok && len(stack) > 0 { // Pop from stack @@ -1241,9 +1243,9 @@ func podFormatterPopper(state *LexerState) error { // Use the rules from pod_formatter_stack to format the pod code func podFormatter(state *LexerState) error { - var stack, ok = state.Get("pod_formatter_stack").([]RakuFormatterRules) + stack, ok := state.Get("pod_formatter_stack").([]RakuFormatterRules) if ok && len(stack) > 0 { - var rules = stack[len(stack)-1] + rules := stack[len(stack)-1] state.Rules["pod-formatter"][0] = rules.pop state.Rules["pod-formatter"][len(state.Rules["pod-formatter"])-1] = rules.formatter } @@ -1275,19 +1277,19 @@ type RuleMakingConfig struct { func makeRuleAndPushMaybe(config RuleMakingConfig) *CompiledRule { var rePattern string if len(config.delimiter) > 0 { - var delimiter = strings.Repeat(string(config.delimiter), config.numberOfDelimiterChars) + delimiter := strings.Repeat(string(config.delimiter), config.numberOfDelimiterChars) rePattern = regexp2.Escape(delimiter) } else { rePattern = config.pattern } - var regex = regexp2.MustCompile(rePattern, regexp2.None) + regex := regexp2.MustCompile(rePattern, regexp2.None) - var cRule = &CompiledRule{ + cRule := &CompiledRule{ Rule: Rule{rePattern, config.tokenType, config.mutator}, Regexp: regex, } - var state = config.state - var stateName = config.stateName + state := config.state + stateName := config.stateName switch config.rulePosition { case topRule: state.Rules[stateName] = @@ -1322,7 +1324,7 @@ func colonPair(tokenClass TokenType) Emitter { iterators = append(iterators, Literator(Token{NameAttribute, groups[2]})) } else { var keyTokenState string - var keyre = regexp.MustCompile(`^\d+$`) + keyre := regexp.MustCompile(`^\d+$`) if keyre.MatchString(groups[2]) { keyTokenState = "common" } else { @@ -1331,7 +1333,7 @@ func colonPair(tokenClass TokenType) Emitter { // Use token state to Tokenise key if keyTokenState != "" { - var iterator, err = lexer.Tokenise( + iterator, err := lexer.Tokenise( &TokeniseOptions{ State: keyTokenState, Nested: true, @@ -1362,7 +1364,7 @@ func colonPair(tokenClass TokenType) Emitter { // Use token state to Tokenise value if valueTokenState != "" { - var iterator, err = lexer.Tokenise( + iterator, err := lexer.Tokenise( &TokeniseOptions{ State: valueTokenState, Nested: true, @@ -1397,7 +1399,7 @@ func quote(groups []string, lexer Lexer) Iterator { var tokenStates []string // Set tokenStates based on adverbs - var adverbs = strings.Split(groups[4], ":") + adverbs := strings.Split(groups[4], ":") for _, adverb := range adverbs { switch adverb { case "c", "closure": @@ -1431,7 +1433,7 @@ func quote(groups []string, lexer Lexer) Iterator { tokenState = "Q" } - var iterator, err = lexer.Tokenise( + iterator, err := lexer.Tokenise( &TokeniseOptions{ State: tokenState, Nested: true, @@ -1452,7 +1454,7 @@ func quote(groups []string, lexer Lexer) Iterator { // Emitter for pod config, tokenises the properties with "colon-pair-attribute" state func podConfig(groups []string, lexer Lexer) Iterator { // Tokenise pod config - var iterator, err = lexer.Tokenise( + iterator, err := lexer.Tokenise( &TokeniseOptions{ State: "colon-pair-attribute", Nested: true, @@ -1486,16 +1488,16 @@ func podCode(groups []string, lexer Lexer) Iterator { // Tokenise pod config iterators = append(iterators, podConfig([]string{groups[5]}, lexer)) - var langMatch = regexp.MustCompile(`:lang\W+(\w+)`).FindStringSubmatch(groups[5]) + langMatch := regexp.MustCompile(`:lang\W+(\w+)`).FindStringSubmatch(groups[5]) var lang string if len(langMatch) > 1 { lang = langMatch[1] } // Tokenise code based on lang property - var sublexer = internal.Get(lang) + sublexer := internal.Get(lang) if sublexer != nil { - var codeIterator, codeIteratorError = sublexer.Tokenise(nil, groups[6]) + codeIterator, codeIteratorError := sublexer.Tokenise(nil, groups[6]) if codeIteratorError != nil { panic(codeIteratorError)