1
0
mirror of https://github.com/alecthomas/chroma.git synced 2025-03-17 20:58:08 +02:00

Add test to pre-compile all regexes.

This commit is contained in:
Alec Thomas 2017-09-19 14:15:33 +10:00
parent 87183b3633
commit a72960340e
7 changed files with 77 additions and 67 deletions

36
_tools/exercise/main.go Normal file
View File

@ -0,0 +1,36 @@
package main
import (
"fmt"
"io/ioutil"
"github.com/alecthomas/chroma/formatters"
"github.com/alecthomas/chroma/lexers"
"github.com/alecthomas/chroma/styles"
"gopkg.in/alecthomas/kingpin.v3-unstable"
)
var (
filesArgs = kingpin.Arg("file", "Files to use to exercise lexers.").Required().ExistingFiles()
)
func main() {
kingpin.CommandLine.Help = "Exercise linters against a list of files."
kingpin.Parse()
writer, err := formatters.NoOp.Format(ioutil.Discard, styles.SwapOff)
kingpin.FatalIfError(err, "")
for _, file := range *filesArgs {
lexer := lexers.Match(file)
if lexer == nil {
fmt.Printf("warning: could not find lexer for %q\n", file)
continue
}
text, err := ioutil.ReadFile(file)
kingpin.FatalIfError(err, "")
err = lexer.Tokenise(nil, string(text), writer)
kingpin.FatalIfError(err, "%s failed to tokenise %q", lexer.Config().Name, file)
fmt.Printf("ok: %q\n", file)
}
}

View File

@ -133,6 +133,8 @@ func (e EmitterFunc) Emit(groups []string, lexer Lexer, out func(*Token)) { e(gr
// ByGroups emits a token for each matching group in the rule's regex.
func ByGroups(emitters ...Emitter) Emitter {
return EmitterFunc(func(groups []string, lexer Lexer, out func(*Token)) {
// NOTE: If this line panics, there is a mismatch with groups. Uncomment the following line to debug.
// fmt.Printf("%s %#v\n", emitters, groups[1:])
for i, group := range groups[1:] {
emitters[i].Emit([]string{group}, lexer, out)
}
@ -212,6 +214,8 @@ func NewLexer(config *Config, rules Rules) (*RegexLexer, error) {
}
// A CompiledRule is a Rule with a pre-compiled regex.
//
// Note that regular expressions are lazily compiled on first use of the lexer.
type CompiledRule struct {
Rule
Regexp *regexp2.Regexp
@ -274,12 +278,12 @@ func (r *RegexLexer) maybeCompile() (err error) {
if r.compiled {
return nil
}
for _, rules := range r.rules {
for state, rules := range r.rules {
for i, rule := range rules {
if rule.Regexp == nil {
rule.Regexp, err = regexp2.Compile("^(?"+rule.flags+")(?:"+rule.Pattern+")", 0)
if err != nil {
return err
return fmt.Errorf("failed to compile rule %s.%d: %s", state, i, err)
}
}
rules[i] = rule

21
lexers/api_test.go Normal file
View File

@ -0,0 +1,21 @@
package lexers_test
import (
"io/ioutil"
"testing"
"github.com/stretchr/testify/assert"
"github.com/alecthomas/chroma/formatters"
"github.com/alecthomas/chroma/lexers"
"github.com/alecthomas/chroma/styles"
)
func TestCompileAllRegexes(t *testing.T) {
writer, err := formatters.NoOp.Format(ioutil.Discard, styles.SwapOff)
assert.NoError(t, err)
for _, lexer := range lexers.Registry.Lexers {
err = lexer.Tokenise(nil, "", writer)
assert.NoError(t, err, "%s failed", lexer.Config().Name)
}
}

View File

@ -9,7 +9,7 @@ var Handlebars = Register(MustNewLexer(
&Config{
Name: "Handlebars",
Aliases: []string{"handlebars"},
Filenames: []string{},
Filenames: []string{"*.handlebars"},
MimeTypes: []string{},
},
Rules{
@ -23,7 +23,7 @@ var Handlebars = Register(MustNewLexer(
{`\s+`, Text, nil},
{`\}\}\}`, CommentSpecial, Pop(1)},
{`\}\}`, CommentPreproc, Pop(1)},
{`([#/]*)(each|if|unless|else|with|log|in(line)?)`, ByGroups(Keyword, Keyword), nil},
{`([#/]*)(each|if|unless|else|with|log|in(?:line)?)`, ByGroups(Keyword, Keyword), nil},
{`#\*inline`, Keyword, nil},
{`([#/])([\w-]+)`, ByGroups(NameFunction, NameFunction), nil},
{`([\w-]+)(=)`, ByGroups(NameAttribute, Operator), nil},

File diff suppressed because one or more lines are too long

View File

@ -20,7 +20,7 @@ var Smalltalk = Register(MustNewLexer(
Include("method definition"),
{`(\|)([\w\s]*)(\|)`, ByGroups(Operator, NameVariable, Operator), nil},
Include("objects"),
{`\^|\:=|\_`, Operator, nil},
{`\^|:=|_`, Operator, nil},
{`[\]({}.;!]`, Text, nil},
},
"method definition": {
@ -75,7 +75,7 @@ var Smalltalk = Register(MustNewLexer(
Include("whitespaces"),
{`\b(ifTrue:|ifFalse:|whileTrue:|whileFalse:|timesRepeat:)`, NameBuiltin, Pop(1)},
{`\b(new\b(?!:))`, NameBuiltin, nil},
{`\:=|\_`, Operator, Pop(1)},
{`:=|_`, Operator, Pop(1)},
{`\b[a-zA-Z]+\w*:`, NameFunction, Pop(1)},
{`\b[a-zA-Z]+\w*`, NameFunction, nil},
{`\w+:?|[-+*/\\~<>=|&!?,@%]+`, NameFunction, Pop(1)},

View File

@ -1,51 +0,0 @@
package styles
import (
"github.com/alecthomas/chroma"
)
// Default style.
var Default = Register(chroma.NewStyle("default", chroma.StyleEntries{
chroma.TextWhitespace: "#bbbbbb",
chroma.Comment: "italic #408080",
chroma.CommentPreproc: "noitalic #BC7A00",
chroma.Keyword: "bold #008000",
chroma.KeywordPseudo: "nobold",
chroma.KeywordType: "nobold #B00040",
chroma.Operator: "#666666",
chroma.OperatorWord: "bold #AA22FF",
chroma.NameBuiltin: "#008000",
chroma.NameFunction: "#0000FF",
chroma.NameClass: "bold #0000FF",
chroma.NameNamespace: "bold #0000FF",
chroma.NameException: "bold #D2413A",
chroma.NameVariable: "#19177C",
chroma.NameConstant: "#880000",
chroma.NameLabel: "#A0A000",
chroma.NameEntity: "bold #999999",
chroma.NameAttribute: "#7D9029",
chroma.NameTag: "bold #008000",
chroma.NameDecorator: "#AA22FF",
chroma.LiteralString: "#BA2121",
chroma.LiteralStringDoc: "italic",
chroma.LiteralStringInterpol: "bold #BB6688",
chroma.LiteralStringEscape: "bold #BB6622",
chroma.LiteralStringRegex: "#BB6688",
chroma.LiteralStringSymbol: "#19177C",
chroma.LiteralStringOther: "#008000",
chroma.LiteralNumber: "#666666",
chroma.GenericHeading: "bold #000080",
chroma.GenericSubheading: "bold #800080",
chroma.GenericDeleted: "#A00000",
chroma.GenericInserted: "#00A000",
chroma.GenericError: "#FF0000",
chroma.GenericEmph: "italic",
chroma.GenericStrong: "bold",
chroma.GenericPrompt: "bold #000080",
chroma.GenericOutput: "#888",
chroma.GenericTraceback: "#04D",
chroma.Error: "border:#FF0000",
chroma.Background: " bg:#f8f8f8",
}))