mirror of
https://github.com/alecthomas/chroma.git
synced 2025-03-17 20:58:08 +02:00
Fixed a fundamental bug where ^ would always match.
The engine was always passing a string sliced to the current position, resulting in ^ always matching. Switched to use FindRunesMatchStartingAt. Fixes #242.
This commit is contained in:
parent
6e1e5df187
commit
ea14dd8660
1
go.mod
1
go.mod
@ -10,5 +10,6 @@ require (
|
||||
github.com/mattn/go-colorable v0.0.9
|
||||
github.com/mattn/go-isatty v0.0.4
|
||||
github.com/sergi/go-diff v1.0.0 // indirect
|
||||
github.com/stretchr/testify v1.2.2
|
||||
golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35 // indirect
|
||||
)
|
||||
|
7
lexers/testdata/ballerina.expected
vendored
7
lexers/testdata/ballerina.expected
vendored
@ -7,13 +7,16 @@
|
||||
{"type":"Text","value":"\n\n"},
|
||||
{"type":"KeywordDeclaration","value":"endpoint"},
|
||||
{"type":"Text","value":" "},
|
||||
{"type":"NameLabel","value":"http:"},
|
||||
{"type":"Name","value":"http"},
|
||||
{"type":"Operator","value":":"},
|
||||
{"type":"Name","value":"Listener"},
|
||||
{"type":"Text","value":" "},
|
||||
{"type":"Name","value":"listener"},
|
||||
{"type":"Text","value":" "},
|
||||
{"type":"Operator","value":"{"},
|
||||
{"type":"NameLabel","value":"\n port:"},
|
||||
{"type":"Text","value":"\n "},
|
||||
{"type":"Name","value":"port"},
|
||||
{"type":"Operator","value":":"},
|
||||
{"type":"Text","value":" "},
|
||||
{"type":"Name","value":"9090"},
|
||||
{"type":"Text","value":"\n"},
|
||||
|
2
lexers/testdata/csharp.expected
vendored
2
lexers/testdata/csharp.expected
vendored
@ -1,6 +1,6 @@
|
||||
[
|
||||
{"type":"Name","value":"DriveInfo"},
|
||||
{"type":"NameAttribute","value":"[]"},
|
||||
{"type":"Punctuation","value":"[]"},
|
||||
{"type":"Text","value":" "},
|
||||
{"type":"Name","value":"drives"},
|
||||
{"type":"Text","value":" "},
|
||||
|
15
lexers/testdata/erlang.actual
vendored
Normal file
15
lexers/testdata/erlang.actual
vendored
Normal file
@ -0,0 +1,15 @@
|
||||
-module(repl).
|
||||
|
||||
-export([run/0]).
|
||||
|
||||
run() ->
|
||||
read_eval_process().
|
||||
|
||||
read_eval_process() ->
|
||||
Line = io:get_line("> "),
|
||||
Out = process_line(Line),
|
||||
io:format("< ~s~n~n", [Out]),
|
||||
read_eval_process().
|
||||
|
||||
process_line(Line) ->
|
||||
string:uppercase(Line).
|
79
lexers/testdata/erlang.expected
vendored
Normal file
79
lexers/testdata/erlang.expected
vendored
Normal file
@ -0,0 +1,79 @@
|
||||
[
|
||||
{"type":"Punctuation","value":"-"},
|
||||
{"type":"NameEntity","value":"module"},
|
||||
{"type":"Punctuation","value":"("},
|
||||
{"type":"Name","value":"repl"},
|
||||
{"type":"Punctuation","value":")."},
|
||||
{"type":"Text","value":"\n\n"},
|
||||
{"type":"Punctuation","value":"-"},
|
||||
{"type":"NameEntity","value":"export"},
|
||||
{"type":"Punctuation","value":"(["},
|
||||
{"type":"Name","value":"run"},
|
||||
{"type":"Operator","value":"/"},
|
||||
{"type":"LiteralNumberInteger","value":"0"},
|
||||
{"type":"Punctuation","value":"])."},
|
||||
{"type":"Text","value":"\n\n"},
|
||||
{"type":"NameFunction","value":"run"},
|
||||
{"type":"Punctuation","value":"()"},
|
||||
{"type":"Text","value":" "},
|
||||
{"type":"Operator","value":"-\u003e"},
|
||||
{"type":"Text","value":"\n "},
|
||||
{"type":"Name","value":"read_eval_process"},
|
||||
{"type":"Punctuation","value":"()."},
|
||||
{"type":"Text","value":"\n\n"},
|
||||
{"type":"NameFunction","value":"read_eval_process"},
|
||||
{"type":"Punctuation","value":"()"},
|
||||
{"type":"Text","value":" "},
|
||||
{"type":"Operator","value":"-\u003e"},
|
||||
{"type":"Text","value":"\n "},
|
||||
{"type":"NameVariable","value":"Line"},
|
||||
{"type":"Text","value":" "},
|
||||
{"type":"Operator","value":"="},
|
||||
{"type":"Text","value":" "},
|
||||
{"type":"NameNamespace","value":"io"},
|
||||
{"type":"Punctuation","value":":"},
|
||||
{"type":"NameFunction","value":"get_line"},
|
||||
{"type":"Punctuation","value":"("},
|
||||
{"type":"LiteralString","value":"\"\u003e \""},
|
||||
{"type":"Punctuation","value":"),"},
|
||||
{"type":"Text","value":"\n "},
|
||||
{"type":"NameVariable","value":"Out"},
|
||||
{"type":"Text","value":" "},
|
||||
{"type":"Operator","value":"="},
|
||||
{"type":"Text","value":" "},
|
||||
{"type":"Name","value":"process_line"},
|
||||
{"type":"Punctuation","value":"("},
|
||||
{"type":"NameVariable","value":"Line"},
|
||||
{"type":"Punctuation","value":"),"},
|
||||
{"type":"Text","value":"\n "},
|
||||
{"type":"NameNamespace","value":"io"},
|
||||
{"type":"Punctuation","value":":"},
|
||||
{"type":"NameFunction","value":"format"},
|
||||
{"type":"Punctuation","value":"("},
|
||||
{"type":"LiteralString","value":"\"\u003c "},
|
||||
{"type":"LiteralStringInterpol","value":"~s~n~n"},
|
||||
{"type":"LiteralString","value":"\""},
|
||||
{"type":"Punctuation","value":","},
|
||||
{"type":"Text","value":" "},
|
||||
{"type":"Punctuation","value":"["},
|
||||
{"type":"NameVariable","value":"Out"},
|
||||
{"type":"Punctuation","value":"]),"},
|
||||
{"type":"Text","value":"\n "},
|
||||
{"type":"Name","value":"read_eval_process"},
|
||||
{"type":"Punctuation","value":"()."},
|
||||
{"type":"Text","value":"\n\n"},
|
||||
{"type":"NameFunction","value":"process_line"},
|
||||
{"type":"Punctuation","value":"("},
|
||||
{"type":"NameVariable","value":"Line"},
|
||||
{"type":"Punctuation","value":")"},
|
||||
{"type":"Text","value":" "},
|
||||
{"type":"Operator","value":"-\u003e"},
|
||||
{"type":"Text","value":"\n "},
|
||||
{"type":"NameNamespace","value":"string"},
|
||||
{"type":"Punctuation","value":":"},
|
||||
{"type":"NameFunction","value":"uppercase"},
|
||||
{"type":"Punctuation","value":"("},
|
||||
{"type":"NameVariable","value":"Line"},
|
||||
{"type":"Punctuation","value":")."},
|
||||
{"type":"Text","value":"\n"}
|
||||
]
|
15
regexp.go
15
regexp.go
@ -276,7 +276,7 @@ func (l *LexerState) Iterator() Token {
|
||||
if !ok {
|
||||
panic("unknown state " + l.State)
|
||||
}
|
||||
ruleIndex, rule, groups := matchRules(l.Text[l.Pos:], selectedRule)
|
||||
ruleIndex, rule, groups := matchRules(l.Text, l.Pos, selectedRule)
|
||||
// No match.
|
||||
if groups == nil {
|
||||
// From Pygments :\
|
||||
@ -363,7 +363,12 @@ func (r *RegexLexer) maybeCompile() (err error) {
|
||||
for state, rules := range r.rules {
|
||||
for i, rule := range rules {
|
||||
if rule.Regexp == nil {
|
||||
rule.Regexp, err = regexp2.Compile("^(?"+rule.flags+")(?:"+rule.Pattern+")", 0)
|
||||
pattern := "(?:" + rule.Pattern + ")"
|
||||
if rule.flags != "" {
|
||||
pattern = "(?" + rule.flags + ")" + pattern
|
||||
}
|
||||
pattern = `\G` + pattern
|
||||
rule.Regexp, err = regexp2.Compile(pattern, 0)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to compile rule %s.%d: %s", state, i, err)
|
||||
}
|
||||
@ -415,10 +420,10 @@ func (r *RegexLexer) Tokenise(options *TokeniseOptions, text string) (Iterator,
|
||||
return state.Iterator, nil
|
||||
}
|
||||
|
||||
func matchRules(text []rune, rules []*CompiledRule) (int, *CompiledRule, []string) {
|
||||
func matchRules(text []rune, pos int, rules []*CompiledRule) (int, *CompiledRule, []string) {
|
||||
for i, rule := range rules {
|
||||
match, err := rule.Regexp.FindRunesMatch(text)
|
||||
if match != nil && err == nil {
|
||||
match, err := rule.Regexp.FindRunesMatchStartingAt(text, pos)
|
||||
if match != nil && err == nil && match.Index == pos {
|
||||
groups := []string{}
|
||||
for _, g := range match.Groups() {
|
||||
groups = append(groups, g.String())
|
||||
|
@ -4,6 +4,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/alecthomas/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestNewlineAtEndOfFile(t *testing.T) {
|
||||
@ -25,3 +26,21 @@ func TestNewlineAtEndOfFile(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, []Token{{Error, "hello"}}, it.Tokens())
|
||||
}
|
||||
|
||||
func TestMatchingAtStart(t *testing.T) {
|
||||
l := Coalesce(MustNewLexer(&Config{}, Rules{
|
||||
"root": {
|
||||
{`\s+`, Whitespace, nil},
|
||||
{`^-`, Punctuation, Push("directive")},
|
||||
{`->`, Operator, nil},
|
||||
},
|
||||
"directive": {
|
||||
{"module", NameEntity, Pop(1)},
|
||||
},
|
||||
}))
|
||||
it, err := l.Tokenise(nil, `-module ->`)
|
||||
assert.NoError(t, err)
|
||||
require.Equal(t,
|
||||
[]Token{{Punctuation, "-"}, {NameEntity, "module"}, {Whitespace, " "}, {Operator, "->"}},
|
||||
it.Tokens())
|
||||
}
|
||||
|
Loading…
x
Reference in New Issue
Block a user