mirror of
https://github.com/alecthomas/chroma.git
synced 2025-03-17 20:58:08 +02:00
Add test to pre-compile all regexes.
This commit is contained in:
parent
87183b3633
commit
a72960340e
36
_tools/exercise/main.go
Normal file
36
_tools/exercise/main.go
Normal file
@ -0,0 +1,36 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
|
||||
"github.com/alecthomas/chroma/formatters"
|
||||
"github.com/alecthomas/chroma/lexers"
|
||||
"github.com/alecthomas/chroma/styles"
|
||||
"gopkg.in/alecthomas/kingpin.v3-unstable"
|
||||
)
|
||||
|
||||
var (
|
||||
filesArgs = kingpin.Arg("file", "Files to use to exercise lexers.").Required().ExistingFiles()
|
||||
)
|
||||
|
||||
func main() {
|
||||
kingpin.CommandLine.Help = "Exercise linters against a list of files."
|
||||
kingpin.Parse()
|
||||
|
||||
writer, err := formatters.NoOp.Format(ioutil.Discard, styles.SwapOff)
|
||||
kingpin.FatalIfError(err, "")
|
||||
|
||||
for _, file := range *filesArgs {
|
||||
lexer := lexers.Match(file)
|
||||
if lexer == nil {
|
||||
fmt.Printf("warning: could not find lexer for %q\n", file)
|
||||
continue
|
||||
}
|
||||
text, err := ioutil.ReadFile(file)
|
||||
kingpin.FatalIfError(err, "")
|
||||
err = lexer.Tokenise(nil, string(text), writer)
|
||||
kingpin.FatalIfError(err, "%s failed to tokenise %q", lexer.Config().Name, file)
|
||||
fmt.Printf("ok: %q\n", file)
|
||||
}
|
||||
}
|
8
lexer.go
8
lexer.go
@ -133,6 +133,8 @@ func (e EmitterFunc) Emit(groups []string, lexer Lexer, out func(*Token)) { e(gr
|
||||
// ByGroups emits a token for each matching group in the rule's regex.
|
||||
func ByGroups(emitters ...Emitter) Emitter {
|
||||
return EmitterFunc(func(groups []string, lexer Lexer, out func(*Token)) {
|
||||
// NOTE: If this line panics, there is a mismatch with groups. Uncomment the following line to debug.
|
||||
// fmt.Printf("%s %#v\n", emitters, groups[1:])
|
||||
for i, group := range groups[1:] {
|
||||
emitters[i].Emit([]string{group}, lexer, out)
|
||||
}
|
||||
@ -212,6 +214,8 @@ func NewLexer(config *Config, rules Rules) (*RegexLexer, error) {
|
||||
}
|
||||
|
||||
// A CompiledRule is a Rule with a pre-compiled regex.
|
||||
//
|
||||
// Note that regular expressions are lazily compiled on first use of the lexer.
|
||||
type CompiledRule struct {
|
||||
Rule
|
||||
Regexp *regexp2.Regexp
|
||||
@ -274,12 +278,12 @@ func (r *RegexLexer) maybeCompile() (err error) {
|
||||
if r.compiled {
|
||||
return nil
|
||||
}
|
||||
for _, rules := range r.rules {
|
||||
for state, rules := range r.rules {
|
||||
for i, rule := range rules {
|
||||
if rule.Regexp == nil {
|
||||
rule.Regexp, err = regexp2.Compile("^(?"+rule.flags+")(?:"+rule.Pattern+")", 0)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("failed to compile rule %s.%d: %s", state, i, err)
|
||||
}
|
||||
}
|
||||
rules[i] = rule
|
||||
|
21
lexers/api_test.go
Normal file
21
lexers/api_test.go
Normal file
@ -0,0 +1,21 @@
|
||||
package lexers_test
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/alecthomas/chroma/formatters"
|
||||
"github.com/alecthomas/chroma/lexers"
|
||||
"github.com/alecthomas/chroma/styles"
|
||||
)
|
||||
|
||||
func TestCompileAllRegexes(t *testing.T) {
|
||||
writer, err := formatters.NoOp.Format(ioutil.Discard, styles.SwapOff)
|
||||
assert.NoError(t, err)
|
||||
for _, lexer := range lexers.Registry.Lexers {
|
||||
err = lexer.Tokenise(nil, "", writer)
|
||||
assert.NoError(t, err, "%s failed", lexer.Config().Name)
|
||||
}
|
||||
}
|
@ -9,7 +9,7 @@ var Handlebars = Register(MustNewLexer(
|
||||
&Config{
|
||||
Name: "Handlebars",
|
||||
Aliases: []string{"handlebars"},
|
||||
Filenames: []string{},
|
||||
Filenames: []string{"*.handlebars"},
|
||||
MimeTypes: []string{},
|
||||
},
|
||||
Rules{
|
||||
@ -23,7 +23,7 @@ var Handlebars = Register(MustNewLexer(
|
||||
{`\s+`, Text, nil},
|
||||
{`\}\}\}`, CommentSpecial, Pop(1)},
|
||||
{`\}\}`, CommentPreproc, Pop(1)},
|
||||
{`([#/]*)(each|if|unless|else|with|log|in(line)?)`, ByGroups(Keyword, Keyword), nil},
|
||||
{`([#/]*)(each|if|unless|else|with|log|in(?:line)?)`, ByGroups(Keyword, Keyword), nil},
|
||||
{`#\*inline`, Keyword, nil},
|
||||
{`([#/])([\w-]+)`, ByGroups(NameFunction, NameFunction), nil},
|
||||
{`([\w-]+)(=)`, ByGroups(NameAttribute, Operator), nil},
|
||||
|
File diff suppressed because one or more lines are too long
@ -20,7 +20,7 @@ var Smalltalk = Register(MustNewLexer(
|
||||
Include("method definition"),
|
||||
{`(\|)([\w\s]*)(\|)`, ByGroups(Operator, NameVariable, Operator), nil},
|
||||
Include("objects"),
|
||||
{`\^|\:=|\_`, Operator, nil},
|
||||
{`\^|:=|_`, Operator, nil},
|
||||
{`[\]({}.;!]`, Text, nil},
|
||||
},
|
||||
"method definition": {
|
||||
@ -75,7 +75,7 @@ var Smalltalk = Register(MustNewLexer(
|
||||
Include("whitespaces"),
|
||||
{`\b(ifTrue:|ifFalse:|whileTrue:|whileFalse:|timesRepeat:)`, NameBuiltin, Pop(1)},
|
||||
{`\b(new\b(?!:))`, NameBuiltin, nil},
|
||||
{`\:=|\_`, Operator, Pop(1)},
|
||||
{`:=|_`, Operator, Pop(1)},
|
||||
{`\b[a-zA-Z]+\w*:`, NameFunction, Pop(1)},
|
||||
{`\b[a-zA-Z]+\w*`, NameFunction, nil},
|
||||
{`\w+:?|[-+*/\\~<>=|&!?,@%]+`, NameFunction, Pop(1)},
|
||||
|
@ -1,51 +0,0 @@
|
||||
|
||||
package styles
|
||||
|
||||
import (
|
||||
"github.com/alecthomas/chroma"
|
||||
)
|
||||
|
||||
// Default style.
|
||||
var Default = Register(chroma.NewStyle("default", chroma.StyleEntries{
|
||||
chroma.TextWhitespace: "#bbbbbb",
|
||||
chroma.Comment: "italic #408080",
|
||||
chroma.CommentPreproc: "noitalic #BC7A00",
|
||||
chroma.Keyword: "bold #008000",
|
||||
chroma.KeywordPseudo: "nobold",
|
||||
chroma.KeywordType: "nobold #B00040",
|
||||
chroma.Operator: "#666666",
|
||||
chroma.OperatorWord: "bold #AA22FF",
|
||||
chroma.NameBuiltin: "#008000",
|
||||
chroma.NameFunction: "#0000FF",
|
||||
chroma.NameClass: "bold #0000FF",
|
||||
chroma.NameNamespace: "bold #0000FF",
|
||||
chroma.NameException: "bold #D2413A",
|
||||
chroma.NameVariable: "#19177C",
|
||||
chroma.NameConstant: "#880000",
|
||||
chroma.NameLabel: "#A0A000",
|
||||
chroma.NameEntity: "bold #999999",
|
||||
chroma.NameAttribute: "#7D9029",
|
||||
chroma.NameTag: "bold #008000",
|
||||
chroma.NameDecorator: "#AA22FF",
|
||||
chroma.LiteralString: "#BA2121",
|
||||
chroma.LiteralStringDoc: "italic",
|
||||
chroma.LiteralStringInterpol: "bold #BB6688",
|
||||
chroma.LiteralStringEscape: "bold #BB6622",
|
||||
chroma.LiteralStringRegex: "#BB6688",
|
||||
chroma.LiteralStringSymbol: "#19177C",
|
||||
chroma.LiteralStringOther: "#008000",
|
||||
chroma.LiteralNumber: "#666666",
|
||||
chroma.GenericHeading: "bold #000080",
|
||||
chroma.GenericSubheading: "bold #800080",
|
||||
chroma.GenericDeleted: "#A00000",
|
||||
chroma.GenericInserted: "#00A000",
|
||||
chroma.GenericError: "#FF0000",
|
||||
chroma.GenericEmph: "italic",
|
||||
chroma.GenericStrong: "bold",
|
||||
chroma.GenericPrompt: "bold #000080",
|
||||
chroma.GenericOutput: "#888",
|
||||
chroma.GenericTraceback: "#04D",
|
||||
chroma.Error: "border:#FF0000",
|
||||
chroma.Background: " bg:#f8f8f8",
|
||||
}))
|
||||
|
Loading…
x
Reference in New Issue
Block a user