Add test to pre-compile all regexes.

_tools/exercise/main.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+
+	"github.com/alecthomas/chroma/formatters"
+	"github.com/alecthomas/chroma/lexers"
+	"github.com/alecthomas/chroma/styles"
+	"gopkg.in/alecthomas/kingpin.v3-unstable"
+)
+
+var (
+	filesArgs = kingpin.Arg("file", "Files to use to exercise lexers.").Required().ExistingFiles()
+)
+
+func main() {
+	kingpin.CommandLine.Help = "Exercise linters against a list of files."
+	kingpin.Parse()
+
+	writer, err := formatters.NoOp.Format(ioutil.Discard, styles.SwapOff)
+	kingpin.FatalIfError(err, "")
+
+	for _, file := range *filesArgs {
+		lexer := lexers.Match(file)
+		if lexer == nil {
+			fmt.Printf("warning: could not find lexer for %q\n", file)
+			continue
+		}
+		text, err := ioutil.ReadFile(file)
+		kingpin.FatalIfError(err, "")
+		err = lexer.Tokenise(nil, string(text), writer)
+		kingpin.FatalIfError(err, "%s failed to tokenise %q", lexer.Config().Name, file)
+		fmt.Printf("ok: %q\n", file)
+	}
+}

lexer.go

@@ -133,6 +133,8 @@ func (e EmitterFunc) Emit(groups []string, lexer Lexer, out func(*Token)) { e(gr
 // ByGroups emits a token for each matching group in the rule's regex.
 func ByGroups(emitters ...Emitter) Emitter {
 	return EmitterFunc(func(groups []string, lexer Lexer, out func(*Token)) {
+		// NOTE: If this line panics, there is a mismatch with groups. Uncomment the following line to debug.
+		// fmt.Printf("%s %#v\n", emitters, groups[1:])
 		for i, group := range groups[1:] {
 			emitters[i].Emit([]string{group}, lexer, out)
 		}
@@ -212,6 +214,8 @@ func NewLexer(config *Config, rules Rules) (*RegexLexer, error) {
 }
 
 // A CompiledRule is a Rule with a pre-compiled regex.
+//
+// Note that regular expressions are lazily compiled on first use of the lexer.
 type CompiledRule struct {
 	Rule
 	Regexp *regexp2.Regexp
@@ -274,12 +278,12 @@ func (r *RegexLexer) maybeCompile() (err error) {
 	if r.compiled {
 		return nil
 	}
-	for _, rules := range r.rules {
+	for state, rules := range r.rules {
 		for i, rule := range rules {
 			if rule.Regexp == nil {
 				rule.Regexp, err = regexp2.Compile("^(?"+rule.flags+")(?:"+rule.Pattern+")", 0)
 				if err != nil {
-					return err
+					return fmt.Errorf("failed to compile rule %s.%d: %s", state, i, err)
 				}
 			}
 			rules[i] = rule

lexers/api_test.go

@@ -0,0 +1,21 @@
+package lexers_test
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/alecthomas/chroma/formatters"
+	"github.com/alecthomas/chroma/lexers"
+	"github.com/alecthomas/chroma/styles"
+)
+
+func TestCompileAllRegexes(t *testing.T) {
+	writer, err := formatters.NoOp.Format(ioutil.Discard, styles.SwapOff)
+	assert.NoError(t, err)
+	for _, lexer := range lexers.Registry.Lexers {
+		err = lexer.Tokenise(nil, "", writer)
+		assert.NoError(t, err, "%s failed", lexer.Config().Name)
+	}
+}

lexers/handlebars.go

@@ -9,7 +9,7 @@ var Handlebars = Register(MustNewLexer(
 	&Config{
 		Name:      "Handlebars",
 		Aliases:   []string{"handlebars"},
-		Filenames: []string{},
+		Filenames: []string{"*.handlebars"},
 		MimeTypes: []string{},
 	},
 	Rules{
@@ -23,7 +23,7 @@ var Handlebars = Register(MustNewLexer(
 			{`\s+`, Text, nil},
 			{`\}\}\}`, CommentSpecial, Pop(1)},
 			{`\}\}`, CommentPreproc, Pop(1)},
-			{`([#/]*)(each|if|unless|else|with|log|in(line)?)`, ByGroups(Keyword, Keyword), nil},
+			{`([#/]*)(each|if|unless|else|with|log|in(?:line)?)`, ByGroups(Keyword, Keyword), nil},
 			{`#\*inline`, Keyword, nil},
 			{`([#/])([\w-]+)`, ByGroups(NameFunction, NameFunction), nil},
 			{`([\w-]+)(=)`, ByGroups(NameAttribute, Operator), nil},

lexers/smalltalk.go

@@ -20,7 +20,7 @@ var Smalltalk = Register(MustNewLexer(
 			Include("method definition"),
 			{`(\|)([\w\s]*)(\|)`, ByGroups(Operator, NameVariable, Operator), nil},
 			Include("objects"),
-			{`\^|\:=|\_`, Operator, nil},
+			{`\^|:=|_`, Operator, nil},
 			{`[\]({}.;!]`, Text, nil},
 		},
 		"method definition": {
@@ -75,7 +75,7 @@ var Smalltalk = Register(MustNewLexer(
 			Include("whitespaces"),
 			{`\b(ifTrue:|ifFalse:|whileTrue:|whileFalse:|timesRepeat:)`, NameBuiltin, Pop(1)},
 			{`\b(new\b(?!:))`, NameBuiltin, nil},
-			{`\:=|\_`, Operator, Pop(1)},
+			{`:=|_`, Operator, Pop(1)},
 			{`\b[a-zA-Z]+\w*:`, NameFunction, Pop(1)},
 			{`\b[a-zA-Z]+\w*`, NameFunction, nil},
 			{`\w+:?|[-+*/\\~<>=|&!?,@%]+`, NameFunction, Pop(1)},

styles/default.go

@@ -1,51 +0,0 @@
-
-package styles
-
-import (
-    "github.com/alecthomas/chroma"
-)
-
-// Default style.
-var Default = Register(chroma.NewStyle("default", chroma.StyleEntries{
-    chroma.TextWhitespace: "#bbbbbb",
-    chroma.Comment: "italic #408080",
-    chroma.CommentPreproc: "noitalic #BC7A00",
-    chroma.Keyword: "bold #008000",
-    chroma.KeywordPseudo: "nobold",
-    chroma.KeywordType: "nobold #B00040",
-    chroma.Operator: "#666666",
-    chroma.OperatorWord: "bold #AA22FF",
-    chroma.NameBuiltin: "#008000",
-    chroma.NameFunction: "#0000FF",
-    chroma.NameClass: "bold #0000FF",
-    chroma.NameNamespace: "bold #0000FF",
-    chroma.NameException: "bold #D2413A",
-    chroma.NameVariable: "#19177C",
-    chroma.NameConstant: "#880000",
-    chroma.NameLabel: "#A0A000",
-    chroma.NameEntity: "bold #999999",
-    chroma.NameAttribute: "#7D9029",
-    chroma.NameTag: "bold #008000",
-    chroma.NameDecorator: "#AA22FF",
-    chroma.LiteralString: "#BA2121",
-    chroma.LiteralStringDoc: "italic",
-    chroma.LiteralStringInterpol: "bold #BB6688",
-    chroma.LiteralStringEscape: "bold #BB6622",
-    chroma.LiteralStringRegex: "#BB6688",
-    chroma.LiteralStringSymbol: "#19177C",
-    chroma.LiteralStringOther: "#008000",
-    chroma.LiteralNumber: "#666666",
-    chroma.GenericHeading: "bold #000080",
-    chroma.GenericSubheading: "bold #800080",
-    chroma.GenericDeleted: "#A00000",
-    chroma.GenericInserted: "#00A000",
-    chroma.GenericError: "#FF0000",
-    chroma.GenericEmph: "italic",
-    chroma.GenericStrong: "bold",
-    chroma.GenericPrompt: "bold #000080",
-    chroma.GenericOutput: "#888",
-    chroma.GenericTraceback: "#04D",
-    chroma.Error: "border:#FF0000",
-    chroma.Background: " bg:#f8f8f8",
-}))
-