1
0
mirror of https://github.com/alecthomas/chroma.git synced 2025-03-17 20:58:08 +02:00

Fixed a fundamental bug where ^ would always match.

The engine was always passing a string sliced to the current position,
resulting in ^ always matching. Switched to use
FindRunesMatchStartingAt.

Fixes #242.
This commit is contained in:
Alec Thomas 2019-06-09 21:43:16 +10:00
parent 6e1e5df187
commit ea14dd8660
7 changed files with 130 additions and 8 deletions

1
go.mod
View File

@ -10,5 +10,6 @@ require (
github.com/mattn/go-colorable v0.0.9
github.com/mattn/go-isatty v0.0.4
github.com/sergi/go-diff v1.0.0 // indirect
github.com/stretchr/testify v1.2.2
golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35 // indirect
)

View File

@ -7,13 +7,16 @@
{"type":"Text","value":"\n\n"},
{"type":"KeywordDeclaration","value":"endpoint"},
{"type":"Text","value":" "},
{"type":"NameLabel","value":"http:"},
{"type":"Name","value":"http"},
{"type":"Operator","value":":"},
{"type":"Name","value":"Listener"},
{"type":"Text","value":" "},
{"type":"Name","value":"listener"},
{"type":"Text","value":" "},
{"type":"Operator","value":"{"},
{"type":"NameLabel","value":"\n port:"},
{"type":"Text","value":"\n "},
{"type":"Name","value":"port"},
{"type":"Operator","value":":"},
{"type":"Text","value":" "},
{"type":"Name","value":"9090"},
{"type":"Text","value":"\n"},

View File

@ -1,6 +1,6 @@
[
{"type":"Name","value":"DriveInfo"},
{"type":"NameAttribute","value":"[]"},
{"type":"Punctuation","value":"[]"},
{"type":"Text","value":" "},
{"type":"Name","value":"drives"},
{"type":"Text","value":" "},

15
lexers/testdata/erlang.actual vendored Normal file
View File

@ -0,0 +1,15 @@
-module(repl).
-export([run/0]).
run() ->
read_eval_process().
read_eval_process() ->
Line = io:get_line("> "),
Out = process_line(Line),
io:format("< ~s~n~n", [Out]),
read_eval_process().
process_line(Line) ->
string:uppercase(Line).

79
lexers/testdata/erlang.expected vendored Normal file
View File

@ -0,0 +1,79 @@
[
{"type":"Punctuation","value":"-"},
{"type":"NameEntity","value":"module"},
{"type":"Punctuation","value":"("},
{"type":"Name","value":"repl"},
{"type":"Punctuation","value":")."},
{"type":"Text","value":"\n\n"},
{"type":"Punctuation","value":"-"},
{"type":"NameEntity","value":"export"},
{"type":"Punctuation","value":"(["},
{"type":"Name","value":"run"},
{"type":"Operator","value":"/"},
{"type":"LiteralNumberInteger","value":"0"},
{"type":"Punctuation","value":"])."},
{"type":"Text","value":"\n\n"},
{"type":"NameFunction","value":"run"},
{"type":"Punctuation","value":"()"},
{"type":"Text","value":" "},
{"type":"Operator","value":"-\u003e"},
{"type":"Text","value":"\n "},
{"type":"Name","value":"read_eval_process"},
{"type":"Punctuation","value":"()."},
{"type":"Text","value":"\n\n"},
{"type":"NameFunction","value":"read_eval_process"},
{"type":"Punctuation","value":"()"},
{"type":"Text","value":" "},
{"type":"Operator","value":"-\u003e"},
{"type":"Text","value":"\n "},
{"type":"NameVariable","value":"Line"},
{"type":"Text","value":" "},
{"type":"Operator","value":"="},
{"type":"Text","value":" "},
{"type":"NameNamespace","value":"io"},
{"type":"Punctuation","value":":"},
{"type":"NameFunction","value":"get_line"},
{"type":"Punctuation","value":"("},
{"type":"LiteralString","value":"\"\u003e \""},
{"type":"Punctuation","value":"),"},
{"type":"Text","value":"\n "},
{"type":"NameVariable","value":"Out"},
{"type":"Text","value":" "},
{"type":"Operator","value":"="},
{"type":"Text","value":" "},
{"type":"Name","value":"process_line"},
{"type":"Punctuation","value":"("},
{"type":"NameVariable","value":"Line"},
{"type":"Punctuation","value":"),"},
{"type":"Text","value":"\n "},
{"type":"NameNamespace","value":"io"},
{"type":"Punctuation","value":":"},
{"type":"NameFunction","value":"format"},
{"type":"Punctuation","value":"("},
{"type":"LiteralString","value":"\"\u003c "},
{"type":"LiteralStringInterpol","value":"~s~n~n"},
{"type":"LiteralString","value":"\""},
{"type":"Punctuation","value":","},
{"type":"Text","value":" "},
{"type":"Punctuation","value":"["},
{"type":"NameVariable","value":"Out"},
{"type":"Punctuation","value":"]),"},
{"type":"Text","value":"\n "},
{"type":"Name","value":"read_eval_process"},
{"type":"Punctuation","value":"()."},
{"type":"Text","value":"\n\n"},
{"type":"NameFunction","value":"process_line"},
{"type":"Punctuation","value":"("},
{"type":"NameVariable","value":"Line"},
{"type":"Punctuation","value":")"},
{"type":"Text","value":" "},
{"type":"Operator","value":"-\u003e"},
{"type":"Text","value":"\n "},
{"type":"NameNamespace","value":"string"},
{"type":"Punctuation","value":":"},
{"type":"NameFunction","value":"uppercase"},
{"type":"Punctuation","value":"("},
{"type":"NameVariable","value":"Line"},
{"type":"Punctuation","value":")."},
{"type":"Text","value":"\n"}
]

View File

@ -276,7 +276,7 @@ func (l *LexerState) Iterator() Token {
if !ok {
panic("unknown state " + l.State)
}
ruleIndex, rule, groups := matchRules(l.Text[l.Pos:], selectedRule)
ruleIndex, rule, groups := matchRules(l.Text, l.Pos, selectedRule)
// No match.
if groups == nil {
// From Pygments :\
@ -363,7 +363,12 @@ func (r *RegexLexer) maybeCompile() (err error) {
for state, rules := range r.rules {
for i, rule := range rules {
if rule.Regexp == nil {
rule.Regexp, err = regexp2.Compile("^(?"+rule.flags+")(?:"+rule.Pattern+")", 0)
pattern := "(?:" + rule.Pattern + ")"
if rule.flags != "" {
pattern = "(?" + rule.flags + ")" + pattern
}
pattern = `\G` + pattern
rule.Regexp, err = regexp2.Compile(pattern, 0)
if err != nil {
return fmt.Errorf("failed to compile rule %s.%d: %s", state, i, err)
}
@ -415,10 +420,10 @@ func (r *RegexLexer) Tokenise(options *TokeniseOptions, text string) (Iterator,
return state.Iterator, nil
}
func matchRules(text []rune, rules []*CompiledRule) (int, *CompiledRule, []string) {
func matchRules(text []rune, pos int, rules []*CompiledRule) (int, *CompiledRule, []string) {
for i, rule := range rules {
match, err := rule.Regexp.FindRunesMatch(text)
if match != nil && err == nil {
match, err := rule.Regexp.FindRunesMatchStartingAt(text, pos)
if match != nil && err == nil && match.Index == pos {
groups := []string{}
for _, g := range match.Groups() {
groups = append(groups, g.String())

View File

@ -4,6 +4,7 @@ import (
"testing"
"github.com/alecthomas/assert"
"github.com/stretchr/testify/require"
)
func TestNewlineAtEndOfFile(t *testing.T) {
@ -25,3 +26,21 @@ func TestNewlineAtEndOfFile(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, []Token{{Error, "hello"}}, it.Tokens())
}
func TestMatchingAtStart(t *testing.T) {
l := Coalesce(MustNewLexer(&Config{}, Rules{
"root": {
{`\s+`, Whitespace, nil},
{`^-`, Punctuation, Push("directive")},
{`->`, Operator, nil},
},
"directive": {
{"module", NameEntity, Pop(1)},
},
}))
it, err := l.Tokenise(nil, `-module ->`)
assert.NoError(t, err)
require.Equal(t,
[]Token{{Punctuation, "-"}, {NameEntity, "module"}, {Whitespace, " "}, {Operator, "->"}},
it.Tokens())
}