Add tracing + better error recovery.

cmd/chroma/main.go

@@ -28,6 +28,7 @@ var (
 	profileFlag    = kingpin.Flag("profile", "Enable profiling to file.").Hidden().String()
 	listFlag       = kingpin.Flag("list", "List lexers, styles and formatters.").Bool()
 	unbufferedFlag = kingpin.Flag("unbuffered", "Do not buffer output.").Bool()
+	traceFlag      = kingpin.Flag("trace", "Trace lexer states as they are traversed.").Bool()
 
 	lexerFlag     = kingpin.Flag("lexer", "Lexer to use when formatting.").PlaceHolder("autodetect").Short('l').Enum(lexers.Names(true)...)
 	styleFlag     = kingpin.Flag("style", "Style to use for formatting.").Short('s').Default("swapoff").Enum(styles.Names()...)
@@ -198,6 +199,9 @@ func lex(path string, contents string) chroma.Iterator {
 	if lexer == nil {
 		lexer = lexers.Fallback
 	}
+	if rel, ok := lexer.(*chroma.RegexLexer); ok {
+		rel.Trace(*traceFlag)
+	}
 	lexer = chroma.Coalesce(lexer)
 	it, err := lexer.Tokenise(nil, string(contents))
 	kingpin.FatalIfError(err, "")

lexers/xml.go

@@ -4,8 +4,8 @@ import (
 	. "github.com/alecthomas/chroma" // nolint
 )
 
-// Xml lexer.
-var Xml = Register(MustNewLexer(
+// XML lexer.
+var XML = Register(MustNewLexer(
 	&Config{
 		Name:      "XML",
 		Aliases:   []string{"xml"},

mutators.go

@@ -86,6 +86,9 @@ func Push(states ...string) MutatorFunc {
 // Pop state from the stack when rule matches.
 func Pop(n int) MutatorFunc {
 	return func(state *LexerState) error {
+		if len(state.Stack) == 0 {
+			return fmt.Errorf("nothing to pop")
+		}
 		state.Stack = state.Stack[:len(state.Stack)-n]
 		return nil
 	}

regexp.go

@@ -2,6 +2,7 @@ package chroma
 
 import (
 	"fmt"
+	"os"
 	"regexp"
 	"strings"
 	"sync"
@@ -129,6 +130,11 @@ func NewLexer(config *Config, rules Rules) (*RegexLexer, error) {
 	}, nil
 }
 
+func (r *RegexLexer) Trace(trace bool) *RegexLexer {
+	r.trace = trace
+	return r
+}
+
 // A CompiledRule is a Rule with a pre-compiled regex.
 //
 // Note that regular expressions are lazily compiled on first use of the lexer.
@@ -166,7 +172,7 @@ func (l *LexerState) Iterator() Iterator {
 	iteratorStack := []Iterator{}
 	return func() *Token {
 		for l.Pos < len(l.Text) && len(l.Stack) > 0 {
-			// Exhaust the IteratorStack, if any.
+			// Exhaust the iterator stack, if any.
 			for len(iteratorStack) > 0 {
 				n := len(iteratorStack) - 1
 				t := iteratorStack[n]()
@@ -178,6 +184,9 @@ func (l *LexerState) Iterator() Iterator {
 			}
 
 			l.State = l.Stack[len(l.Stack)-1]
+			if l.Lexer.trace {
+				fmt.Fprintf(os.Stderr, "%s: pos=%d, text=%q\n", l.State, l.Pos, string(l.Text[l.Pos:]))
+			}
 			ruleIndex, rule, groups := matchRules(l.Text[l.Pos:], l.Rules[l.State])
 			// No match.
 			if groups == nil {
@@ -207,6 +216,13 @@ func (l *LexerState) Iterator() Iterator {
 			}
 			return t
 		}
+
+		// If we get to here and we still have text, return it as an error.
+		if l.Pos != len(l.Text) && len(l.Stack) == 0 {
+			value := string(l.Text[l.Pos:])
+			l.Pos = len(l.Text)
+			return &Token{Type: Error, Value: value}
+		}
 		return nil
 	}
 }
@@ -214,6 +230,7 @@ func (l *LexerState) Iterator() Iterator {
 type RegexLexer struct {
 	config   *Config
 	analyser func(text string) float32
+	trace    bool
 
 	mu       sync.Mutex
 	compiled bool