From 2105c68ed28e72b9b68f812edbaf95c748ee6e59 Mon Sep 17 00:00:00 2001
From: Alec Thomas <alec@swapoff.org>
Date: Mon, 22 Apr 2019 18:22:58 +1000
Subject: [PATCH] Implemented a weird little Pygments rule that I missed.

> If the RegexLexer encounters a newline that is flagged as an error
> token, the stack is emptied and the lexer continues scanning in the
> 'root' state. This can help producing error-tolerant highlighting for
> erroneous input, e.g. when a single-line string is not closed.

Fixes #246.
---
 regexp.go | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/regexp.go b/regexp.go
index be69ab2..2e7897c 100644
--- a/regexp.go
+++ b/regexp.go
@@ -241,6 +241,7 @@ type LexerState struct {
 	// Custum context for mutators.
 	MutatorContext map[interface{}]interface{}
 	iteratorStack  []Iterator
+	options        *TokeniseOptions
 }
 
 // Set mutator context.
@@ -278,6 +279,16 @@ func (l *LexerState) Iterator() Token {
 		ruleIndex, rule, groups := matchRules(l.Text[l.Pos:], selectedRule)
 		// No match.
 		if groups == nil {
+			// From Pygments :\
+			//
+			// If the RegexLexer encounters a newline that is flagged as an error token, the stack is
+			// emptied and the lexer continues scanning in the 'root' state. This can help producing
+			// error-tolerant highlighting for erroneous input, e.g. when a single-line string is not
+			// closed.
+			if l.Text[l.Pos] == '\n' && l.State != l.options.State {
+				l.Stack = []string{l.options.State}
+				continue
+			}
 			l.Pos++
 			return Token{Error, string(l.Text[l.Pos-1 : l.Pos])}
 		}
@@ -394,6 +405,7 @@ func (r *RegexLexer) Tokenise(options *TokeniseOptions, text string) (Iterator,
 		text += "\n"
 	}
 	state := &LexerState{
+		options:        options,
 		Lexer:          r,
 		Text:           []rune(text),
 		Stack:          []string{options.State},