go/chroma
1
0
Fork 0
mirror of https://github.com/alecthomas/chroma.git synced 2025-11-29 22:47:29 +02:00
Code Issues Releases Activity

refactor: Iterator -> iter.Seq[Token]

Browse Source
This commit is contained in:
Alec Thomas
2025-09-21 20:08:46 +10:00
parent 26ea2fc245
commit ca57a0bae1
28 changed files with 96 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
_tools/exercise/main.go
View File

@@ -5,10 +5,11 @@ import (
"io/ioutil"
"os"
"gopkg.in/alecthomas/kingpin.v3-unstable"
"github.com/alecthomas/chroma/v2/formatters"
"github.com/alecthomas/chroma/v2/lexers"
"github.com/alecthomas/chroma/v2/styles"
"gopkg.in/alecthomas/kingpin.v3-unstable"
)
var (

1
bin/.gosimports-0.3.8.pkg Symbolic link
View File

@@ -0,0 +1 @@
hermit

1
bin/gosimports Symbolic link
View File

@@ -0,0 +1 @@
.gosimports-0.3.8.pkg

4
coalesce.go
View File

@@ -1,11 +1,13 @@
package chroma
import "iter"
// Coalesce is a Lexer interceptor that collapses runs of common types into a single token.
func Coalesce(lexer Lexer) Lexer { return &coalescer{lexer} }
type coalescer struct{ Lexer }
func (d *coalescer) Tokenise(options *TokeniseOptions, text string) (Iterator, error) {
func (d *coalescer) Tokenise(options *TokeniseOptions, text string) (iter.Seq[Token], error) {
it, err := d.Lexer.Tokenise(options, text)
if err != nil {
return nil, err

3
delegate.go
View File

@@ -2,6 +2,7 @@ package chroma
import (
"bytes"
"iter"
)
type delegatingLexer struct {
@@ -58,7 +59,7 @@ type insertion struct {
tokens []Token
}
func (d *delegatingLexer) Tokenise(options *TokeniseOptions, text string) (Iterator, error) { // nolint: gocognit
func (d *delegatingLexer) Tokenise(options *TokeniseOptions, text string) (iter.Seq[Token], error) { // nolint: gocognit
tokens, err := Tokenise(Coalesce(d.language), options, text)
if err != nil {
return nil, err

3
delegate_test.go
View File

@@ -1,6 +1,7 @@
package chroma
import (
"slices"
"testing"
assert "github.com/alecthomas/assert/v2"
@@ -104,7 +105,7 @@ func TestDelegate(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
it, err := delegate.Tokenise(nil, test.source)
assert.NoError(t, err)
actual := it.Tokens()
actual := slices.Collect(it)
assert.Equal(t, test.expected, actual)
})
}

25
emitters.go
View File

@@ -2,12 +2,13 @@ package chroma
import (
"fmt"
"iter"
)
// An Emitter takes group matches and returns tokens.
type Emitter interface {
// Emit tokens for the given regex groups.
Emit(groups []string, state *LexerState) Iterator
Emit(groups []string, state *LexerState) iter.Seq[Token]
}
// ValidatingEmitter is an Emitter that can validate against a compiled rule.
@@ -23,10 +24,10 @@ type SerialisableEmitter interface {
}
// EmitterFunc is a function that is an Emitter.
type EmitterFunc func(groups []string, state *LexerState) Iterator
type EmitterFunc func(groups []string, state *LexerState) iter.Seq[Token]
// Emit tokens for groups.
func (e EmitterFunc) Emit(groups []string, state *LexerState) Iterator {
func (e EmitterFunc) Emit(groups []string, state *LexerState) iter.Seq[Token] {
return e(groups, state)
}
@@ -52,8 +53,8 @@ func (b *byGroupsEmitter) ValidateEmitter(rule *CompiledRule) error {
return nil
}
func (b *byGroupsEmitter) Emit(groups []string, state *LexerState) Iterator {
iterators := make([]Iterator, 0, len(groups)-1)
func (b *byGroupsEmitter) Emit(groups []string, state *LexerState) iter.Seq[Token] {
iterators := make([]iter.Seq[Token], 0, len(groups)-1)
if len(b.Emitters) != len(groups)-1 {
iterators = append(iterators, Error.Emit(groups, state))
// panic(errors.Errorf("number of groups %q does not match number of emitters %v", groups, emitters))
@@ -69,8 +70,8 @@ func (b *byGroupsEmitter) Emit(groups []string, state *LexerState) Iterator {
// ByGroupNames emits a token for each named matching group in the rule's regex.
func ByGroupNames(emitters map[string]Emitter) Emitter {
return EmitterFunc(func(groups []string, state *LexerState) Iterator {
iterators := make([]Iterator, 0, len(state.NamedGroups)-1)
return EmitterFunc(func(groups []string, state *LexerState) iter.Seq[Token] {
iterators := make([]iter.Seq[Token], 0, len(state.NamedGroups)-1)
if len(state.NamedGroups)-1 == 0 {
if emitter, ok := emitters[`0`]; ok {
iterators = append(iterators, emitter.Emit(groups, state))
@@ -147,7 +148,7 @@ type usingByGroup struct {
}
func (u *usingByGroup) EmitterKind() string { return "usingbygroup" }
func (u *usingByGroup) Emit(groups []string, state *LexerState) Iterator {
func (u *usingByGroup) Emit(groups []string, state *LexerState) iter.Seq[Token] {
// bounds check
if len(u.Emitters) != len(groups)-1 {
panic("UsingByGroup expects number of emitters to be the same as len(groups)-1")
@@ -157,7 +158,7 @@ func (u *usingByGroup) Emit(groups []string, state *LexerState) Iterator {
sublexer := state.Registry.Get(groups[u.SublexerNameGroup])
// build iterators
iterators := make([]Iterator, len(groups)-1)
iterators := make([]iter.Seq[Token], len(groups)-1)
for i, group := range groups[1:] {
if i == u.CodeGroup-1 && sublexer != nil {
var err error
@@ -176,7 +177,7 @@ func (u *usingByGroup) Emit(groups []string, state *LexerState) Iterator {
//
// This Emitter is not serialisable.
func UsingLexer(lexer Lexer) Emitter {
return EmitterFunc(func(groups []string, _ *LexerState) Iterator {
return EmitterFunc(func(groups []string, _ *LexerState) iter.Seq[Token] {
it, err := lexer.Tokenise(&TokeniseOptions{State: "root", Nested: true}, groups[0])
if err != nil {
panic(err)
@@ -191,7 +192,7 @@ type usingEmitter struct {
func (u *usingEmitter) EmitterKind() string { return "using" }
func (u *usingEmitter) Emit(groups []string, state *LexerState) Iterator {
func (u *usingEmitter) Emit(groups []string, state *LexerState) iter.Seq[Token] {
if state.Registry == nil {
panic(fmt.Sprintf("no LexerRegistry available for Using(%q)", u.Lexer))
}
@@ -219,7 +220,7 @@ type usingSelfEmitter struct {
func (u *usingSelfEmitter) EmitterKind() string { return "usingself" }
func (u *usingSelfEmitter) Emit(groups []string, state *LexerState) Iterator {
func (u *usingSelfEmitter) Emit(groups []string, state *LexerState) iter.Seq[Token] {
it, err := state.Lexer.Tokenise(&TokeniseOptions{State: u.State, Nested: true}, groups[0])
if err != nil {
panic(err)

9
formatter.go
View File

@@ -2,6 +2,7 @@ package chroma
import (
"io"
"iter"
)
// A Formatter for Chroma lexers.
@@ -9,15 +10,15 @@ type Formatter interface {
// Format returns a formatting function for tokens.
//
// If the iterator panics, the Formatter should recover.
Format(w io.Writer, style *Style, iterator Iterator) error
Format(w io.Writer, style *Style, iterator iter.Seq[Token]) error
}
// A FormatterFunc is a Formatter implemented as a function.
//
// Guards against iterator panics.
type FormatterFunc func(w io.Writer, style *Style, iterator Iterator) error
type FormatterFunc func(w io.Writer, style *Style, iterator iter.Seq[Token]) error
func (f FormatterFunc) Format(w io.Writer, s *Style, it Iterator) (err error) { // nolint
func (f FormatterFunc) Format(w io.Writer, s *Style, it iter.Seq[Token]) (err error) { // nolint
defer func() {
if perr := recover(); perr != nil {
err = perr.(error)
@@ -30,7 +31,7 @@ type recoveringFormatter struct {
Formatter
}
func (r recoveringFormatter) Format(w io.Writer, s *Style, it Iterator) (err error) {
func (r recoveringFormatter) Format(w io.Writer, s *Style, it iter.Seq[Token]) (err error) {
defer func() {
if perr := recover(); perr != nil {
err = perr.(error)

3
formatters/api.go
View File

@@ -2,6 +2,7 @@ package formatters
import (
"io"
"iter"
"sort"
"github.com/alecthomas/chroma/v2"
@@ -11,7 +12,7 @@ import (
var (
// NoOp formatter.
NoOp = Register("noop", chroma.FormatterFunc(func(w io.Writer, s *chroma.Style, iterator chroma.Iterator) error {
NoOp = Register("noop", chroma.FormatterFunc(func(w io.Writer, s *chroma.Style, iterator iter.Seq[chroma.Token]) error {
for t := range iterator {
if t == chroma.EOF {
break

6
formatters/html/html.go
View File

@@ -4,6 +4,8 @@ import (
"fmt"
"html"
"io"
"iter"
"slices"
"sort"
"strconv"
"strings"
@@ -221,8 +223,8 @@ func (h highlightRanges) Len() int { return len(h) }
func (h highlightRanges) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
func (h highlightRanges) Less(i, j int) bool { return h[i][0] < h[j][0] }
func (f *Formatter) Format(w io.Writer, style *chroma.Style, iterator chroma.Iterator) (err error) {
return f.writeHTML(w, style, iterator.Tokens())
func (f *Formatter) Format(w io.Writer, style *chroma.Style, iterator iter.Seq[chroma.Token]) (err error) {
return f.writeHTML(w, style, slices.Collect(iterator))
}
// We deliberately don't use html/template here because it is two orders of magnitude slower (benchmarked).

3
formatters/json.go
View File

@@ -4,12 +4,13 @@ import (
"encoding/json"
"fmt"
"io"
"iter"
"github.com/alecthomas/chroma/v2"
)
// JSON formatter outputs the raw token structures as JSON.
var JSON = Register("json", chroma.FormatterFunc(func(w io.Writer, s *chroma.Style, it chroma.Iterator) error {
var JSON = Register("json", chroma.FormatterFunc(func(w io.Writer, s *chroma.Style, it iter.Seq[chroma.Token]) error {
if _, err := fmt.Fprintln(w, "["); err != nil {
return err
}

6
formatters/svg/svg.go
View File

@@ -6,8 +6,10 @@ import (
"errors"
"fmt"
"io"
"iter"
"os"
"path"
"slices"
"strings"
"github.com/alecthomas/chroma/v2"
@@ -61,8 +63,8 @@ type Formatter struct {
fontFormat FontFormat
}
func (f *Formatter) Format(w io.Writer, style *chroma.Style, iterator chroma.Iterator) (err error) {
f.writeSVG(w, style, iterator.Tokens())
func (f *Formatter) Format(w io.Writer, style *chroma.Style, iterator iter.Seq[chroma.Token]) (err error) {
f.writeSVG(w, style, slices.Collect(iterator))
return err
}

3
formatters/tokens.go
View File

@@ -3,12 +3,13 @@ package formatters
import (
"fmt"
"io"
"iter"
"github.com/alecthomas/chroma/v2"
)
// Tokens formatter outputs the raw token structures.
var Tokens = Register("tokens", chroma.FormatterFunc(func(w io.Writer, s *chroma.Style, it chroma.Iterator) error {
var Tokens = Register("tokens", chroma.FormatterFunc(func(w io.Writer, s *chroma.Style, it iter.Seq[chroma.Token]) error {
for t := range it {
if t == chroma.EOF {
break

3
formatters/tty_indexed.go
View File

@@ -2,6 +2,7 @@ package formatters
import (
"io"
"iter"
"math"
"github.com/alecthomas/chroma/v2"
@@ -237,7 +238,7 @@ type indexedTTYFormatter struct {
table *ttyTable
}
func (c *indexedTTYFormatter) Format(w io.Writer, style *chroma.Style, it chroma.Iterator) (err error) {
func (c *indexedTTYFormatter) Format(w io.Writer, style *chroma.Style, it iter.Seq[chroma.Token]) (err error) {
theme := styleToEscapeSequence(c.table, style)
for token := range it {
if token == chroma.EOF {

1
formatters/tty_indexed_test.go
View File

@@ -5,6 +5,7 @@ import (
"testing"
assert "github.com/alecthomas/assert/v2"
"github.com/alecthomas/chroma/v2"
)

3
formatters/tty_truecolour.go
View File

@@ -3,6 +3,7 @@ package formatters
import (
"fmt"
"io"
"iter"
"regexp"
"github.com/alecthomas/chroma/v2"
@@ -44,7 +45,7 @@ func writeToken(w io.Writer, formatting string, text string) {
}
}
func trueColourFormatter(w io.Writer, style *chroma.Style, it chroma.Iterator) error {
func trueColourFormatter(w io.Writer, style *chroma.Style, it iter.Seq[chroma.Token]) error {
style = clearBackground(style)
for token := range it {
if token == chroma.EOF {

25
iterator.go
View File

@@ -5,27 +5,8 @@ import (
"strings"
)
// An Iterator across tokens.
//
// EOF will be returned at the end of the Token stream.
//
// If an error occurs within an Iterator, it may propagate this in a panic. Formatters should recover.
type Iterator iter.Seq[Token]
// Tokens consumes all tokens from the iterator and returns them as a slice.
func (i Iterator) Tokens() []Token {
var out []Token
for t := range i {
if t == EOF {
break
}
out = append(out, t)
}
return out
}
// Concaterator concatenates tokens from a series of iterators.
func Concaterator(iterators ...Iterator) Iterator {
func Concaterator(iterators ...iter.Seq[Token]) iter.Seq[Token] {
return func(yield func(Token) bool) {
for _, it := range iterators {
for t := range it {
@@ -40,8 +21,8 @@ func Concaterator(iterators ...Iterator) Iterator {
}
}
// Literator converts a sequence of literal Tokens into an Iterator.
func Literator(tokens ...Token) Iterator {
// Literator converts a sequence of literal Tokens into an iter.Seq[Token].
func Literator(tokens ...Token) iter.Seq[Token] {
return func(yield func(Token) bool) {
for _, token := range tokens {
if !yield(token) {

5
lexer.go
View File

@@ -2,6 +2,7 @@ package chroma
import (
"fmt"
"iter"
"strings"
)
@@ -112,8 +113,8 @@ type TokeniseOptions struct {
type Lexer interface {
// Config describing the features of the Lexer.
Config() *Config
// Tokenise returns an Iterator over tokens in text.
Tokenise(options *TokeniseOptions, text string) (Iterator, error)
// Tokenise returns an iter.Seq[Token] over tokens in text.
Tokenise(options *TokeniseOptions, text string) (iter.Seq[Token], error)
// SetRegistry sets the registry this Lexer is associated with.
//
// The registry should be used by the Lexer if it needs to look up other

1
lexers/go_test.go
View File

@@ -4,6 +4,7 @@ import (
"testing"
assert "github.com/alecthomas/assert/v2"
"github.com/alecthomas/chroma/v2"
)

11
lexers/http.go
View File

@@ -1,6 +1,7 @@
package lexers
import (
"iter"
"strings"
. "github.com/alecthomas/chroma/v2" // nolint
@@ -36,14 +37,14 @@ func httpRules() Rules {
}
}
func httpContentBlock(groups []string, state *LexerState) Iterator {
func httpContentBlock(groups []string, state *LexerState) iter.Seq[Token] {
tokens := []Token{
{Generic, groups[0]},
}
return Literator(tokens...)
}
func httpHeaderBlock(groups []string, state *LexerState) Iterator {
func httpHeaderBlock(groups []string, state *LexerState) iter.Seq[Token] {
tokens := []Token{
{Name, groups[1]},
{Text, groups[2]},
@@ -55,7 +56,7 @@ func httpHeaderBlock(groups []string, state *LexerState) Iterator {
return Literator(tokens...)
}
func httpContinuousHeaderBlock(groups []string, state *LexerState) Iterator {
func httpContinuousHeaderBlock(groups []string, state *LexerState) iter.Seq[Token] {
tokens := []Token{
{Text, groups[1]},
{Literal, groups[2]},
@@ -68,7 +69,7 @@ func httpBodyContentTypeLexer(lexer Lexer) Lexer { return &httpBodyContentTyper{
type httpBodyContentTyper struct{ Lexer }
func (d *httpBodyContentTyper) Tokenise(options *TokeniseOptions, text string) (Iterator, error) { // nolint: gocognit
func (d *httpBodyContentTyper) Tokenise(options *TokeniseOptions, text string) (iter.Seq[Token], error) { // nolint: gocognit
it, err := d.Lexer.Tokenise(options, text)
if err != nil {
return nil, err
@@ -77,7 +78,7 @@ func (d *httpBodyContentTyper) Tokenise(options *TokeniseOptions, text string) (
return func(yield func(Token) bool) {
var contentType string
var isContentType bool
var subIterator Iterator
var subIterator iter.Seq[Token]
for token := range it {
if token == EOF {

15
lexers/raku.go
View File

@@ -1,6 +1,7 @@
package lexers
import (
"iter"
"regexp"
"slices"
"strings"
@@ -1505,8 +1506,8 @@ func makeRule(config ruleMakingConfig) *CompiledRule {
// Emitter for colon pairs, changes token state based on key and brackets
func colonPair(tokenClass TokenType) Emitter {
return EmitterFunc(func(groups []string, state *LexerState) Iterator {
iterators := []Iterator{}
return EmitterFunc(func(groups []string, state *LexerState) iter.Seq[Token] {
iterators := []iter.Seq[Token]{}
tokens := []Token{
{Punctuation, state.NamedGroups[`colon`]},
{Punctuation, state.NamedGroups[`opening_delimiters`]},
@@ -1581,10 +1582,10 @@ func colonPair(tokenClass TokenType) Emitter {
}
// Emitter for quoting constructs, changes token state based on quote name and adverbs
func quote(groups []string, state *LexerState) Iterator {
func quote(groups []string, state *LexerState) iter.Seq[Token] {
keyword := state.NamedGroups[`keyword`]
adverbsStr := state.NamedGroups[`adverbs`]
iterators := []Iterator{}
iterators := []iter.Seq[Token]{}
tokens := []Token{
{Keyword, keyword},
{StringAffix, adverbsStr},
@@ -1649,7 +1650,7 @@ func quote(groups []string, state *LexerState) Iterator {
}
// Emitter for pod config, tokenises the properties with "colon-pair-attribute" state
func podConfig(groups []string, state *LexerState) Iterator {
func podConfig(groups []string, state *LexerState) iter.Seq[Token] {
// Tokenise pod config
iterator, err := state.Lexer.Tokenise(
&TokeniseOptions{
@@ -1665,8 +1666,8 @@ func podConfig(groups []string, state *LexerState) Iterator {
}
// Emitter for pod code, tokenises the code based on the lang specified
func podCode(groups []string, state *LexerState) Iterator {
iterators := []Iterator{}
func podCode(groups []string, state *LexerState) iter.Seq[Token] {
iterators := []iter.Seq[Token]{}
tokens := []Token{
{Comment, state.NamedGroups[`ws`]},
{Keyword, state.NamedGroups[`keyword`]},

5
lexers/rst.go
View File

@@ -1,6 +1,7 @@
package lexers
import (
"iter"
"strings"
. "github.com/alecthomas/chroma/v2" // nolint
@@ -62,8 +63,8 @@ func restructuredtextRules() Rules {
}
}
func rstCodeBlock(groups []string, state *LexerState) Iterator {
iterators := []Iterator{}
func rstCodeBlock(groups []string, state *LexerState) iter.Seq[Token] {
iterators := []iter.Seq[Token]{}
tokens := []Token{
{Punctuation, groups[1]},
{Text, groups[2]},

3
mutators_test.go
View File

@@ -1,6 +1,7 @@
package chroma
import (
"slices"
"testing"
assert "github.com/alecthomas/assert/v2"
@@ -53,5 +54,5 @@ func TestCombine(t *testing.T) {
it, err := l.Tokenise(nil, "hello world")
assert.NoError(t, err)
expected := []Token{{String, `hello`}, {Whitespace, ` `}, {Name, `world`}}
assert.Equal(t, expected, it.Tokens())
assert.Equal(t, expected, slices.Collect(it))
}

3
regexp.go
View File

@@ -3,6 +3,7 @@ package chroma
import (
"encoding/json"
"fmt"
"iter"
"os"
"path/filepath"
"regexp"
@@ -491,7 +492,7 @@ func (r *RegexLexer) needRules() error {
}
// Tokenise text using lexer, returning an iterator.
func (r *RegexLexer) Tokenise(options *TokeniseOptions, text string) (Iterator, error) {
func (r *RegexLexer) Tokenise(options *TokeniseOptions, text string) (iter.Seq[Token], error) {
err := r.needRules()
if err != nil {
return nil, err

23
regexp_test.go
View File

@@ -1,6 +1,7 @@
package chroma
import (
"slices"
"testing"
assert "github.com/alecthomas/assert/v2"
@@ -22,7 +23,7 @@ func TestNewlineAtEndOfFile(t *testing.T) {
}))
it, err := l.Tokenise(nil, `hello`)
assert.NoError(t, err)
assert.Equal(t, []Token{{Keyword, "hello"}, {Whitespace, "\n"}}, it.Tokens())
assert.Equal(t, []Token{{Keyword, "hello"}, {Whitespace, "\n"}}, slices.Collect(it))
l = Coalesce(mustNewLexer(t, nil, Rules{ // nolint: forbidigo
"root": {
@@ -31,7 +32,7 @@ func TestNewlineAtEndOfFile(t *testing.T) {
}))
it, err = l.Tokenise(nil, `hello`)
assert.NoError(t, err)
assert.Equal(t, []Token{{Error, "hello"}}, it.Tokens())
assert.Equal(t, []Token{{Error, "hello"}}, slices.Collect(it))
}
func TestMatchingAtStart(t *testing.T) {
@@ -49,7 +50,7 @@ func TestMatchingAtStart(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t,
[]Token{{Punctuation, "-"}, {NameEntity, "module"}, {Whitespace, " "}, {Operator, "->"}},
it.Tokens())
slices.Collect(it))
}
func TestEnsureLFOption(t *testing.T) {
@@ -68,7 +69,7 @@ func TestEnsureLFOption(t *testing.T) {
{Whitespace, "\n"},
{Keyword, "world"},
{Whitespace, "\n"},
}, it.Tokens())
}, slices.Collect(it))
l = Coalesce(mustNewLexer(t, nil, Rules{ // nolint: forbidigo
"root": {
@@ -85,7 +86,7 @@ func TestEnsureLFOption(t *testing.T) {
{Whitespace, "\r\n"},
{Keyword, "world"},
{Whitespace, "\r"},
}, it.Tokens())
}, slices.Collect(it))
}
func TestEnsureLFFunc(t *testing.T) {
@@ -124,7 +125,7 @@ func TestByGroupNames(t *testing.T) {
}))
it, err := l.Tokenise(nil, `abc=123`)
assert.NoError(t, err)
assert.Equal(t, []Token{{String, `abc`}, {Operator, `=`}, {String, `123`}}, it.Tokens())
assert.Equal(t, []Token{{String, `abc`}, {Operator, `=`}, {String, `123`}}, slices.Collect(it))
l = Coalesce(mustNewLexer(t, nil, Rules{ // nolint: forbidigo
"root": {
@@ -140,7 +141,7 @@ func TestByGroupNames(t *testing.T) {
}))
it, err = l.Tokenise(nil, `abc=123`)
assert.NoError(t, err)
assert.Equal(t, []Token{{String, `abc`}, {Error, `=`}, {String, `123`}}, it.Tokens())
assert.Equal(t, []Token{{String, `abc`}, {Error, `=`}, {String, `123`}}, slices.Collect(it))
l = Coalesce(mustNewLexer(t, nil, Rules{ // nolint: forbidigo
"root": {
@@ -156,7 +157,7 @@ func TestByGroupNames(t *testing.T) {
}))
it, err = l.Tokenise(nil, `abc=123`)
assert.NoError(t, err)
assert.Equal(t, []Token{{String, `abc123`}}, it.Tokens())
assert.Equal(t, []Token{{String, `abc123`}}, slices.Collect(it))
l = Coalesce(mustNewLexer(t, nil, Rules{ // nolint: forbidigo
"root": {
@@ -173,7 +174,7 @@ func TestByGroupNames(t *testing.T) {
}))
it, err = l.Tokenise(nil, `abc=123`)
assert.NoError(t, err)
assert.Equal(t, []Token{{String, `abc`}, {Error, `=`}, {String, `123`}}, it.Tokens())
assert.Equal(t, []Token{{String, `abc`}, {Error, `=`}, {String, `123`}}, slices.Collect(it))
l = Coalesce(mustNewLexer(t, nil, Rules{ // nolint: forbidigo
"root": {
@@ -190,7 +191,7 @@ func TestByGroupNames(t *testing.T) {
}))
it, err = l.Tokenise(nil, `abc=123`)
assert.NoError(t, err)
assert.Equal(t, []Token{{Error, `abc=123`}}, it.Tokens())
assert.Equal(t, []Token{{Error, `abc=123`}}, slices.Collect(it))
}
func TestIgnoreToken(t *testing.T) {
@@ -201,5 +202,5 @@ func TestIgnoreToken(t *testing.T) {
}))
it, err := l.Tokenise(nil, ` hello `)
assert.NoError(t, err)
assert.Equal(t, []Token{{Keyword, "hello"}, {TextWhitespace, "\n"}}, it.Tokens())
assert.Equal(t, []Token{{Keyword, "hello"}, {TextWhitespace, "\n"}}, slices.Collect(it))
}

4
remap.go
View File

@@ -1,5 +1,7 @@
package chroma
import "iter"
type remappingLexer struct {
lexer Lexer
mapper func(Token) []Token
@@ -28,7 +30,7 @@ func (r *remappingLexer) Config() *Config {
return r.lexer.Config()
}
func (r *remappingLexer) Tokenise(options *TokeniseOptions, text string) (Iterator, error) {
func (r *remappingLexer) Tokenise(options *TokeniseOptions, text string) (iter.Seq[Token], error) {
it, err := r.lexer.Tokenise(options, text)
if err != nil {
return nil, err

3
remap_test.go
View File

@@ -1,6 +1,7 @@
package chroma
import (
"slices"
"testing"
assert "github.com/alecthomas/assert/v2"
@@ -24,6 +25,6 @@ func TestRemappingLexer(t *testing.T) {
{TextWhitespace, " "}, {Name, "print"}, {TextWhitespace, " "}, {Keyword, "else"},
{TextWhitespace, " "}, {Name, "end"},
}
actual := it.Tokens()
actual := slices.Collect(it)
assert.Equal(t, expected, actual)
}

4
types.go
View File

@@ -1,5 +1,7 @@
package chroma
import "iter"
//go:generate enumer -text -type TokenType
// TokenType is the type of token to highlight.
@@ -348,7 +350,7 @@ func (t TokenType) InSubCategory(other TokenType) bool {
return t/100 == other/100
}
func (t TokenType) Emit(groups []string, _ *LexerState) Iterator {
func (t TokenType) Emit(groups []string, _ *LexerState) iter.Seq[Token] {
return Literator(Token{Type: t, Value: groups[0]})
}