Add RemappingLexer.

This pattern is used heavily in Pygments for certain lexers,
particularly Lisp variants. See #43.

remap.go

@@ -0,0 +1,76 @@
+package chroma
+
+type remappingLexer struct {
+	lexer  Lexer
+	mapper func(*Token) []*Token
+}
+
+// RemappingLexer remaps a token to a set of, potentially empty, tokens.
+func RemappingLexer(lexer Lexer, mapper func(*Token) []*Token) Lexer {
+	return &remappingLexer{lexer, mapper}
+}
+
+func (r *remappingLexer) Config() *Config {
+	return r.lexer.Config()
+}
+
+func (r *remappingLexer) Tokenise(options *TokeniseOptions, text string) (Iterator, error) {
+	it, err := r.lexer.Tokenise(options, text)
+	if err != nil {
+		return nil, err
+	}
+	buffer := []*Token{}
+	return func() *Token {
+		for {
+			if len(buffer) > 0 {
+				t := buffer[0]
+				buffer = buffer[1:]
+				return t
+			}
+			t := it()
+			if t == nil {
+				return t
+			}
+			buffer = r.mapper(t)
+		}
+	}, nil
+}
+
+type TypeMapping struct {
+	From TokenType
+	To   TokenType
+}
+type TypeRemappingMap map[TypeMapping][]string
+
+// TypeRemappingLexer remaps types of tokens coming from a parent Lexer.
+//
+// eg. Map "defvaralias" tokens of type NameVariable to NameFunction:
+//
+// 		mapping := TypeRemappingMap{
+// 			{NameVariable, NameFunction}: {"defvaralias"},
+// 		}
+// 		lexer = TypeRemappingLexer(lexer, mapping)
+func TypeRemappingLexer(lexer Lexer, mapping TypeRemappingMap) Lexer {
+	// Lookup table for fast remapping.
+	lut := map[TokenType]map[string]TokenType{}
+	for rt, kl := range mapping {
+		km, ok := lut[rt.From]
+		if !ok {
+			km = map[string]TokenType{}
+			lut[rt.From] = km
+		}
+		for _, k := range kl {
+			km[k] = rt.To
+		}
+
+	}
+	return RemappingLexer(lexer, func(t *Token) []*Token {
+		if k, ok := lut[t.Type]; ok {
+			if tt, ok := k[t.Value]; ok {
+				t = t.Clone()
+				t.Type = tt
+			}
+		}
+		return []*Token{t}
+	})
+}

remap_test.go

@@ -0,0 +1,29 @@
+package chroma
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRemappingLexer(t *testing.T) {
+	var lexer Lexer = MustNewLexer(nil, Rules{
+		"root": {
+			{`\s+`, Whitespace, nil},
+			{`\w+`, Name, nil},
+		},
+	})
+	lexer = TypeRemappingLexer(lexer, TypeRemappingMap{
+		{Name, Keyword}: {"if", "else"},
+	})
+
+	it, err := lexer.Tokenise(nil, `if true then print else end`)
+	assert.NoError(t, err)
+	expected := []*Token{
+		{Keyword, "if"}, {TextWhitespace, " "}, {Name, "true"}, {TextWhitespace, " "}, {Name, "then"},
+		{TextWhitespace, " "}, {Name, "print"}, {TextWhitespace, " "}, {Keyword, "else"},
+		{TextWhitespace, " "}, {Name, "end"},
+	}
+	actual := it.Tokens()
+	assert.Equal(t, expected, actual)
+}