Fix DelegatingLexer.

Fixes #126.

coalesce.go

@@ -13,6 +13,9 @@ func (d *coalescer) Tokenise(options *TokeniseOptions, text string) (Iterator, e
 	}
 	return func() *Token {
 		for token := it(); token != nil; token = it() {
+			if len(token.Value) == 0 {
+				continue
+			}
 			if prev == nil {
 				prev = token
 			} else {

delegate.go

@@ -67,64 +67,66 @@ func (d *delegatingLexer) Tokenise(options *TokeniseOptions, text string) (Itera
 	}
 
 	// Lex the other tokens.
-	rootTokens, err := Tokenise(d.root, options, others.String())
+	rootTokens, err := Tokenise(Coalesce(d.root), options, others.String())
 	if err != nil {
 		return nil, err
 	}
 
 	// Interleave the two sets of tokens.
 	out := []*Token{}
-	offset = 0
-	index := 0
-	next := func() *Token {
-		if index >= len(rootTokens) {
+	offset = 0 // Offset into text.
+	tokenIndex := 0
+	nextToken := func() *Token {
+		if tokenIndex >= len(rootTokens) {
 			return nil
 		}
-		t := rootTokens[index]
-		index++
+		t := rootTokens[tokenIndex]
+		tokenIndex++
 		return t
 	}
-	t := next()
-	for _, insert := range insertions {
-		// Consume tokens until insertion point.
-		for t != nil && offset+len(t.Value) <= insert.start {
+	insertionIndex := 0
+	nextInsertion := func() *insertion {
+		if insertionIndex >= len(insertions) {
+			return nil
+		}
+		i := insertions[insertionIndex]
+		insertionIndex++
+		return i
+	}
+	t := nextToken()
+	i := nextInsertion()
+	for t != nil || i != nil {
+		// fmt.Printf("%d->%d:%q   %d->%d:%q\n", offset, offset+len(t.Value), t.Value, i.start, i.end, Stringify(i.tokens...))
+		if t == nil || (i != nil && i.start < offset+len(t.Value)) {
+			var l *Token
+			l, t = splitToken(t, i.start-offset)
+			if l != nil {
+				out = append(out, l)
+				offset += len(l.Value)
+			}
+			out = append(out, i.tokens...)
+			offset += i.end - i.start
+			if t == nil {
+				t = nextToken()
+			}
+			i = nextInsertion()
+		} else {
 			out = append(out, t)
 			offset += len(t.Value)
-			t = next()
+			t = nextToken()
 		}
-		// End of root tokens, append insertion point.
-		if t == nil {
-			out = append(out, insert.tokens...)
-			break
-		}
-
-		// Split and insert.
-		l, r := splitToken(t, insert.start-offset)
-		if l != nil {
-			out = append(out, l)
-			offset += len(l.Value)
-		}
-		out = append(out, insert.tokens...)
-		offset += insert.end - insert.start
-		if r != nil {
-			out = append(out, r)
-			offset += len(r.Value)
-		}
-		t = next()
 	}
-	if t != nil {
-		out = append(out, t)
-	}
-	// Remainder.
-	out = append(out, rootTokens[index:]...)
 	return Literator(out...), nil
 }
 
 func splitToken(t *Token, offset int) (l *Token, r *Token) {
+	if t == nil {
+		return nil, nil
+	}
 	if offset == 0 {
 		return nil, t
 	}
-	if offset >= len(t.Value) {
+	if offset == len(t.Value) {
 		return t, nil
 	}
 	l = t.Clone()

lexers/g/go_test.go

@@ -0,0 +1,50 @@
+package g
+
+import (
+  "testing"
+
+  "github.com/alecthomas/assert"
+  "github.com/alecthomas/chroma"
+)
+
+func TestGoHTMLTemplateIssue126(t *testing.T) {
+  for _, source := range []string{
+    `<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
+  <channel>
+    <title>{{ if eq  .Title  .Site.Title }}{{ .Site.Title }}{{ else }}{{ with .Title }}{{.}} on {{ end }}{{ .Site.Title }}{{ end }}</title>
+    <link>{{ .Permalink }}</link>
+    <description>Recent content {{ if ne  .Title  .Site.Title }}{{ with .Title }}in {{.}} {{ end }}{{ end }}on {{ .Site.Title }}</description>
+    <generator>Hugo -- gohugo.io</generator>{{ with .Site.LanguageCode }}
+    <language>{{.}}</language>{{end}}{{ with .Site.Author.email }}
+    <managingEditor>{{.}}{{ with $.Site.Author.name }} ({{.}}){{end}}</managingEditor>{{end}}{{ with .Site.Author.email }}
+    <webMaster>{{.}}{{ with $.Site.Author.name }} ({{.}}){{end}}</webMaster>{{end}}{{ with .Site.Copyright }}
+    <copyright>{{.}}</copyright>{{end}}{{ if not .Date.IsZero }}
+    <lastBuildDate>{{ .Date.Format "Mon, 02 Jan 2006 15:04:05 -0700" | safeHTML }}</lastBuildDate>{{ end }}
+    {{ with .OutputFormats.Get "RSS" }}
+        {{ printf "<atom:link href=%q rel=\"self\" type=%q />" .Permalink .MediaType | safeHTML }}
+    {{ end }}
+    {{ range .Data.Pages }}
+    <item>
+      <title>{{ .Title }}</title>
+      <link>{{ .Permalink }}</link>
+      <pubDate>{{ .Date.Format "Mon, 02 Jan 2006 15:04:05 -0700" | safeHTML }}</pubDate>
+      {{ with .Site.Author.email }}<author>{{.}}{{ with $.Site.Author.name }} ({{.}}){{end}}</author>{{end}}
+      <guid>{{ .Permalink }}</guid>
+      <description>{{ .Summary | html }}</description>
+    </item>
+    {{ end }}
+  </channel>
+</rss>
+`,
+    `{{ $headless := .Site.GetPage "page" "some-headless-bundle" }}
+{{ $reusablePages := $headless.Resources.Match "author*" }}
+<h2>Authors</h2>
+{{ range $reusablePages }}
+    <h3>{{ .Title }}</h3>
+    {{ .Content }}
+{{ end }}`} {
+    tokens, err := chroma.Tokenise(GoHTMLTemplate, nil, source)
+    assert.NoError(t, err)
+    assert.Equal(t, source, chroma.Stringify(tokens...))
+  }
+}

lexers/lexers_test.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
+	"github.com/alecthomas/assert"
 
 	"github.com/alecthomas/chroma"
 	"github.com/alecthomas/chroma/formatters"

lexers/testdata/css.expected

@@ -5,7 +5,6 @@
   {"type":"Punctuation","value":"{"},
   {"type":"Text","value":"\n  "},
   {"type":"NameVariable","value":"--variable-name"},
-  {"type":"Text","value":""},
   {"type":"Punctuation","value":":"},
   {"type":"Text","value":" "},
   {"type":"LiteralNumberHex","value":"#fff"},

lexers/testdata/jsx.expected

@@ -44,9 +44,7 @@
   {"type":"Punctuation","value":"\u003c"},
   {"type":"NameTag","value":"App"},
   {"type":"Text","value":" "},
-  {"type":"Punctuation","value":"/"},
-  {"type":"Text","value":""},
-  {"type":"Punctuation","value":"\u003e,"},
+  {"type":"Punctuation","value":"/\u003e,"},
   {"type":"Text","value":"\n  "},
   {"type":"NameBuiltin","value":"document"},
   {"type":"Punctuation","value":"."},

lexers/testdata/php.expected

@@ -2,32 +2,19 @@
   {"type":"CommentPreproc","value":"\u003c!DOCTYPE html\u003e"},
   {"type":"Text","value":"\n"},
   {"type":"Punctuation","value":"\u003c"},
-  {"type":"Text","value":""},
   {"type":"NameTag","value":"html"},
-  {"type":"Punctuation","value":""},
-  {"type":"Text","value":""},
   {"type":"Punctuation","value":"\u003e"},
   {"type":"Text","value":"\n"},
   {"type":"Punctuation","value":"\u003c"},
-  {"type":"Text","value":""},
   {"type":"NameTag","value":"body"},
-  {"type":"Punctuation","value":""},
-  {"type":"Text","value":""},
   {"type":"Punctuation","value":"\u003e"},
   {"type":"Text","value":"\n\n"},
   {"type":"Punctuation","value":"\u003c"},
-  {"type":"Text","value":""},
   {"type":"NameTag","value":"h1"},
-  {"type":"Punctuation","value":""},
-  {"type":"Text","value":""},
   {"type":"Punctuation","value":"\u003e"},
   {"type":"Text","value":"My first PHP page"},
-  {"type":"Punctuation","value":"\u003c"},
-  {"type":"Text","value":""},
-  {"type":"Punctuation","value":"/"},
-  {"type":"Text","value":""},
+  {"type":"Punctuation","value":"\u003c/"},
   {"type":"NameTag","value":"h1"},
-  {"type":"Text","value":""},
   {"type":"Punctuation","value":"\u003e"},
   {"type":"Text","value":"\n"},
   {"type":"CommentPreproc","value":"\u003c?php"},
@@ -38,7 +25,6 @@
   {"type":"Text","value":" "},
   {"type":"NameVariable","value":"$modx"},
   {"type":"Operator","value":"-\u003e"},
-  {"type":"Text","value":""},
   {"type":"NameAttribute","value":"getIterator"},
   {"type":"Punctuation","value":"("},
   {"type":"LiteralStringSingle","value":"'modResource'"},
@@ -65,13 +51,11 @@
   {"type":"Operator","value":"="},
   {"type":"NameVariable","value":"$doc"},
   {"type":"Operator","value":"-\u003e"},
-  {"type":"Text","value":""},
   {"type":"NameAttribute","value":"content"},
   {"type":"Punctuation","value":";"},
   {"type":"Text","value":"\n    "},
   {"type":"NameVariable","value":"$doc"},
   {"type":"Operator","value":"-\u003e"},
-  {"type":"Text","value":""},
   {"type":"NameAttribute","value":"set"},
   {"type":"Punctuation","value":"("},
   {"type":"LiteralStringDouble","value":"\"content\""},
@@ -92,7 +76,6 @@
   {"type":"Punctuation","value":"("},
   {"type":"NameVariable","value":"$doc"},
   {"type":"Operator","value":"-\u003e"},
-  {"type":"Text","value":""},
   {"type":"NameAttribute","value":"content"},
   {"type":"Punctuation","value":");"},
   {"type":"Text","value":"\n    "},
@@ -102,20 +85,12 @@
   {"type":"CommentSingle","value":"// some comment\n"},
   {"type":"CommentPreproc","value":"?\u003e"},
   {"type":"Text","value":"\n"},
-  {"type":"Punctuation","value":"\u003c"},
-  {"type":"Text","value":""},
-  {"type":"Punctuation","value":"/"},
-  {"type":"Text","value":""},
+  {"type":"Punctuation","value":"\u003c/"},
   {"type":"NameTag","value":"body"},
-  {"type":"Text","value":""},
   {"type":"Punctuation","value":"\u003e"},
   {"type":"Text","value":"\n"},
-  {"type":"Punctuation","value":"\u003c"},
-  {"type":"Text","value":""},
-  {"type":"Punctuation","value":"/"},
-  {"type":"Text","value":""},
+  {"type":"Punctuation","value":"\u003c/"},
   {"type":"NameTag","value":"html"},
-  {"type":"Text","value":""},
   {"type":"Punctuation","value":"\u003e"},
   {"type":"Text","value":"\n"}
 ]

lexers/testdata/scala.expected

@@ -50,7 +50,6 @@
   {"type":"Keyword","value":":"},
   {"type":"Text","value":" "},
   {"type":"KeywordType","value":"Seq"},
-  {"type":"Text","value":""},
   {"type":"Operator","value":"["},
   {"type":"KeywordType","value":"T"},
   {"type":"Operator","value":"];"},

mutators.go

@@ -120,3 +120,12 @@ func Pop(n int) MutatorFunc {
 func Default(mutators ...Mutator) Rule {
 	return Rule{Mutator: Mutators(mutators...)}
 }
+
+// Stringify returns the raw string for a set of tokens.
+func Stringify(tokens ...*Token) string {
+	out := []string{}
+	for _, t := range tokens {
+		out = append(out, t.Value)
+	}
+	return strings.Join(out, "")
+}