go/chroma
1
0
Fork 0
mirror of https://github.com/alecthomas/chroma.git synced 2025-03-17 20:58:08 +02:00
Code Issues Releases Activity

Implemented a weird little Pygments rule that I missed.

Browse Source 
> If the RegexLexer encounters a newline that is flagged as an error
> token, the stack is emptied and the lexer continues scanning in the
> 'root' state. This can help producing error-tolerant highlighting for
> erroneous input, e.g. when a single-line string is not closed.

Fixes #246.
This commit is contained in:
Alec Thomas 2019-04-22 18:22:58 +10:00
parent 26f03cb449
commit 2105c68ed2
1 changed files with 12 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
regexp.go
View File

@ -241,6 +241,7 @@ type LexerState struct {
// Custum context for mutators.
MutatorContext map[interface{}]interface{}
iteratorStack []Iterator
options *TokeniseOptions
}
// Set mutator context.
@ -278,6 +279,16 @@ func (l *LexerState) Iterator() Token {
ruleIndex, rule, groups := matchRules(l.Text[l.Pos:], selectedRule)
// No match.
if groups == nil {
// From Pygments :\
//
// If the RegexLexer encounters a newline that is flagged as an error token, the stack is
// emptied and the lexer continues scanning in the 'root' state. This can help producing
// error-tolerant highlighting for erroneous input, e.g. when a single-line string is not
// closed.
if l.Text[l.Pos] == '\n' && l.State != l.options.State {
l.Stack = []string{l.options.State}
continue
}
l.Pos++
return Token{Error, string(l.Text[l.Pos-1 : l.Pos])}
}
@ -394,6 +405,7 @@ func (r *RegexLexer) Tokenise(options *TokeniseOptions, text string) (Iterator,
text += "\n"
}
state := &LexerState{
options: options,
Lexer: r,
Text: []rune(text),
Stack: []string{options.State},