mirror of
https://github.com/alecthomas/chroma.git
synced 2025-10-08 22:52:04 +02:00
fix: don't emit empty tail tokens (#1121)
When tokens are split into lines, tokens that end with a newline are
emitted again as an empty token, which is confusing and doesn't have any
benefit. This conversion shouldn't emit a empty tail token.
Adds a test.
I noticed this issue by a CI failure for the output of a Go program that
was changed because of
d0ad679444
and the new output contained a
empty whitespace token at the beginning of most lines.
This commit is contained in:
@@ -49,6 +49,48 @@ func TestSplitTokensIntoLines(t *testing.T) {
|
||||
}
|
||||
actual := chroma.SplitTokensIntoLines(in)
|
||||
assert.Equal(t, expected, actual)
|
||||
|
||||
in = []chroma.Token{
|
||||
{Value: "func", Type: chroma.KeywordDeclaration},
|
||||
{Value: " ", Type: chroma.TextWhitespace},
|
||||
{Value: "main", Type: chroma.NameFunction},
|
||||
{Value: "()", Type: chroma.Punctuation},
|
||||
{Value: " ", Type: chroma.TextWhitespace},
|
||||
{Value: "{", Type: chroma.Punctuation},
|
||||
{Value: "\n\t", Type: chroma.TextWhitespace},
|
||||
{Value: "println", Type: chroma.NameBuiltin},
|
||||
{Value: "(", Type: chroma.Punctuation},
|
||||
{Value: `"mark this"`, Type: chroma.LiteralString},
|
||||
{Value: ")", Type: chroma.Punctuation},
|
||||
{Value: "\n", Type: chroma.TextWhitespace},
|
||||
{Value: "}", Type: chroma.Punctuation},
|
||||
{Value: "\n", Type: chroma.TextWhitespace},
|
||||
}
|
||||
expected = [][]chroma.Token{
|
||||
{
|
||||
{Type: chroma.KeywordDeclaration, Value: "func"},
|
||||
{Type: chroma.TextWhitespace, Value: " "},
|
||||
{Type: chroma.NameFunction, Value: "main"},
|
||||
{Type: chroma.Punctuation, Value: "()"},
|
||||
{Type: chroma.TextWhitespace, Value: " "},
|
||||
{Type: chroma.Punctuation, Value: "{"},
|
||||
{Type: chroma.TextWhitespace, Value: "\n"},
|
||||
},
|
||||
{
|
||||
{Type: chroma.TextWhitespace, Value: "\t"},
|
||||
{Type: chroma.NameBuiltin, Value: "println"},
|
||||
{Type: chroma.Punctuation, Value: "("},
|
||||
{Type: chroma.LiteralString, Value: `"mark this"`},
|
||||
{Type: chroma.Punctuation, Value: ")"},
|
||||
{Type: chroma.TextWhitespace, Value: "\n"},
|
||||
},
|
||||
{
|
||||
{Type: chroma.Punctuation, Value: "}"},
|
||||
{Type: chroma.TextWhitespace, Value: "\n"},
|
||||
},
|
||||
}
|
||||
actual = chroma.SplitTokensIntoLines(in)
|
||||
assert.Equal(t, expected, actual)
|
||||
}
|
||||
|
||||
func TestFormatterStyleToCSS(t *testing.T) {
|
||||
|
Reference in New Issue
Block a user