1
0
mirror of https://github.com/alecthomas/chroma.git synced 2025-07-05 00:38:52 +02:00

Add new TokeniseOption EnsureLF (#336)

* Add new TokeniseOption EnsureLF

ref #329

* Use efficient process suggested by @chmike
This commit is contained in:
satotake
2020-03-04 16:56:47 +09:00
committed by GitHub
parent e5d9650a20
commit 34d9c7143b
3 changed files with 84 additions and 1 deletions

View File

@ -43,3 +43,59 @@ func TestMatchingAtStart(t *testing.T) {
[]Token{{Punctuation, "-"}, {NameEntity, "module"}, {Whitespace, " "}, {Operator, "->"}},
it.Tokens())
}
func TestEnsureLFOption(t *testing.T) {
l := Coalesce(MustNewLexer(&Config{}, Rules{
"root": {
{`(\w+)(\r?\n|\r)`, ByGroups(Keyword, Whitespace), nil},
},
}))
it, err := l.Tokenise(&TokeniseOptions{
State: "root",
EnsureLF: true,
}, "hello\r\nworld\r")
assert.NoError(t, err)
assert.Equal(t, []Token{
{Keyword, "hello"},
{Whitespace, "\n"},
{Keyword, "world"},
{Whitespace, "\n"},
}, it.Tokens())
l = Coalesce(MustNewLexer(nil, Rules{
"root": {
{`(\w+)(\r?\n|\r)`, ByGroups(Keyword, Whitespace), nil},
},
}))
it, err = l.Tokenise(&TokeniseOptions{
State: "root",
EnsureLF: false,
}, "hello\r\nworld\r")
assert.NoError(t, err)
assert.Equal(t, []Token{
{Keyword, "hello"},
{Whitespace, "\r\n"},
{Keyword, "world"},
{Whitespace, "\r"},
}, it.Tokens())
}
func TestEnsureLFFunc(t *testing.T) {
tests := []struct{ in, out string }{
{in: "", out: ""},
{in: "abc", out: "abc"},
{in: "\r", out: "\n"},
{in: "a\r", out: "a\n"},
{in: "\rb", out: "\nb"},
{in: "a\rb", out: "a\nb"},
{in: "\r\n", out: "\n"},
{in: "a\r\n", out: "a\n"},
{in: "\r\nb", out: "\nb"},
{in: "a\r\nb", out: "a\nb"},
{in: "\r\r\r\n\r", out: "\n\n\n\n"},
}
for _, test := range tests {
out := ensureLF(test.in)
assert.Equal(t, out, test.out)
}
}