mirror of
https://github.com/alecthomas/chroma.git
synced 2025-03-17 20:58:08 +02:00
parent
5ef2d37c8c
commit
59f554950e
@ -13,6 +13,9 @@ func (d *coalescer) Tokenise(options *TokeniseOptions, text string) (Iterator, e
|
||||
}
|
||||
return func() *Token {
|
||||
for token := it(); token != nil; token = it() {
|
||||
if len(token.Value) == 0 {
|
||||
continue
|
||||
}
|
||||
if prev == nil {
|
||||
prev = token
|
||||
} else {
|
||||
|
76
delegate.go
76
delegate.go
@ -67,64 +67,66 @@ func (d *delegatingLexer) Tokenise(options *TokeniseOptions, text string) (Itera
|
||||
}
|
||||
|
||||
// Lex the other tokens.
|
||||
rootTokens, err := Tokenise(d.root, options, others.String())
|
||||
rootTokens, err := Tokenise(Coalesce(d.root), options, others.String())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Interleave the two sets of tokens.
|
||||
out := []*Token{}
|
||||
offset = 0
|
||||
index := 0
|
||||
next := func() *Token {
|
||||
if index >= len(rootTokens) {
|
||||
offset = 0 // Offset into text.
|
||||
tokenIndex := 0
|
||||
nextToken := func() *Token {
|
||||
if tokenIndex >= len(rootTokens) {
|
||||
return nil
|
||||
}
|
||||
t := rootTokens[index]
|
||||
index++
|
||||
t := rootTokens[tokenIndex]
|
||||
tokenIndex++
|
||||
return t
|
||||
}
|
||||
t := next()
|
||||
for _, insert := range insertions {
|
||||
// Consume tokens until insertion point.
|
||||
for t != nil && offset+len(t.Value) <= insert.start {
|
||||
insertionIndex := 0
|
||||
nextInsertion := func() *insertion {
|
||||
if insertionIndex >= len(insertions) {
|
||||
return nil
|
||||
}
|
||||
i := insertions[insertionIndex]
|
||||
insertionIndex++
|
||||
return i
|
||||
}
|
||||
t := nextToken()
|
||||
i := nextInsertion()
|
||||
for t != nil || i != nil {
|
||||
// fmt.Printf("%d->%d:%q %d->%d:%q\n", offset, offset+len(t.Value), t.Value, i.start, i.end, Stringify(i.tokens...))
|
||||
if t == nil || (i != nil && i.start < offset+len(t.Value)) {
|
||||
var l *Token
|
||||
l, t = splitToken(t, i.start-offset)
|
||||
if l != nil {
|
||||
out = append(out, l)
|
||||
offset += len(l.Value)
|
||||
}
|
||||
out = append(out, i.tokens...)
|
||||
offset += i.end - i.start
|
||||
if t == nil {
|
||||
t = nextToken()
|
||||
}
|
||||
i = nextInsertion()
|
||||
} else {
|
||||
out = append(out, t)
|
||||
offset += len(t.Value)
|
||||
t = next()
|
||||
t = nextToken()
|
||||
}
|
||||
// End of root tokens, append insertion point.
|
||||
if t == nil {
|
||||
out = append(out, insert.tokens...)
|
||||
break
|
||||
}
|
||||
|
||||
// Split and insert.
|
||||
l, r := splitToken(t, insert.start-offset)
|
||||
if l != nil {
|
||||
out = append(out, l)
|
||||
offset += len(l.Value)
|
||||
}
|
||||
out = append(out, insert.tokens...)
|
||||
offset += insert.end - insert.start
|
||||
if r != nil {
|
||||
out = append(out, r)
|
||||
offset += len(r.Value)
|
||||
}
|
||||
t = next()
|
||||
}
|
||||
if t != nil {
|
||||
out = append(out, t)
|
||||
}
|
||||
// Remainder.
|
||||
out = append(out, rootTokens[index:]...)
|
||||
return Literator(out...), nil
|
||||
}
|
||||
|
||||
func splitToken(t *Token, offset int) (l *Token, r *Token) {
|
||||
if t == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if offset == 0 {
|
||||
return nil, t
|
||||
}
|
||||
if offset >= len(t.Value) {
|
||||
if offset == len(t.Value) {
|
||||
return t, nil
|
||||
}
|
||||
l = t.Clone()
|
||||
|
50
lexers/g/go_test.go
Normal file
50
lexers/g/go_test.go
Normal file
@ -0,0 +1,50 @@
|
||||
package g
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/alecthomas/assert"
|
||||
"github.com/alecthomas/chroma"
|
||||
)
|
||||
|
||||
func TestGoHTMLTemplateIssue126(t *testing.T) {
|
||||
for _, source := range []string{
|
||||
`<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
|
||||
<channel>
|
||||
<title>{{ if eq .Title .Site.Title }}{{ .Site.Title }}{{ else }}{{ with .Title }}{{.}} on {{ end }}{{ .Site.Title }}{{ end }}</title>
|
||||
<link>{{ .Permalink }}</link>
|
||||
<description>Recent content {{ if ne .Title .Site.Title }}{{ with .Title }}in {{.}} {{ end }}{{ end }}on {{ .Site.Title }}</description>
|
||||
<generator>Hugo -- gohugo.io</generator>{{ with .Site.LanguageCode }}
|
||||
<language>{{.}}</language>{{end}}{{ with .Site.Author.email }}
|
||||
<managingEditor>{{.}}{{ with $.Site.Author.name }} ({{.}}){{end}}</managingEditor>{{end}}{{ with .Site.Author.email }}
|
||||
<webMaster>{{.}}{{ with $.Site.Author.name }} ({{.}}){{end}}</webMaster>{{end}}{{ with .Site.Copyright }}
|
||||
<copyright>{{.}}</copyright>{{end}}{{ if not .Date.IsZero }}
|
||||
<lastBuildDate>{{ .Date.Format "Mon, 02 Jan 2006 15:04:05 -0700" | safeHTML }}</lastBuildDate>{{ end }}
|
||||
{{ with .OutputFormats.Get "RSS" }}
|
||||
{{ printf "<atom:link href=%q rel=\"self\" type=%q />" .Permalink .MediaType | safeHTML }}
|
||||
{{ end }}
|
||||
{{ range .Data.Pages }}
|
||||
<item>
|
||||
<title>{{ .Title }}</title>
|
||||
<link>{{ .Permalink }}</link>
|
||||
<pubDate>{{ .Date.Format "Mon, 02 Jan 2006 15:04:05 -0700" | safeHTML }}</pubDate>
|
||||
{{ with .Site.Author.email }}<author>{{.}}{{ with $.Site.Author.name }} ({{.}}){{end}}</author>{{end}}
|
||||
<guid>{{ .Permalink }}</guid>
|
||||
<description>{{ .Summary | html }}</description>
|
||||
</item>
|
||||
{{ end }}
|
||||
</channel>
|
||||
</rss>
|
||||
`,
|
||||
`{{ $headless := .Site.GetPage "page" "some-headless-bundle" }}
|
||||
{{ $reusablePages := $headless.Resources.Match "author*" }}
|
||||
<h2>Authors</h2>
|
||||
{{ range $reusablePages }}
|
||||
<h3>{{ .Title }}</h3>
|
||||
{{ .Content }}
|
||||
{{ end }}`} {
|
||||
tokens, err := chroma.Tokenise(GoHTMLTemplate, nil, source)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, source, chroma.Stringify(tokens...))
|
||||
}
|
||||
}
|
@ -8,7 +8,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/alecthomas/assert"
|
||||
|
||||
"github.com/alecthomas/chroma"
|
||||
"github.com/alecthomas/chroma/formatters"
|
||||
|
1
lexers/testdata/css.expected
vendored
1
lexers/testdata/css.expected
vendored
@ -5,7 +5,6 @@
|
||||
{"type":"Punctuation","value":"{"},
|
||||
{"type":"Text","value":"\n "},
|
||||
{"type":"NameVariable","value":"--variable-name"},
|
||||
{"type":"Text","value":""},
|
||||
{"type":"Punctuation","value":":"},
|
||||
{"type":"Text","value":" "},
|
||||
{"type":"LiteralNumberHex","value":"#fff"},
|
||||
|
4
lexers/testdata/jsx.expected
vendored
4
lexers/testdata/jsx.expected
vendored
@ -44,9 +44,7 @@
|
||||
{"type":"Punctuation","value":"\u003c"},
|
||||
{"type":"NameTag","value":"App"},
|
||||
{"type":"Text","value":" "},
|
||||
{"type":"Punctuation","value":"/"},
|
||||
{"type":"Text","value":""},
|
||||
{"type":"Punctuation","value":"\u003e,"},
|
||||
{"type":"Punctuation","value":"/\u003e,"},
|
||||
{"type":"Text","value":"\n "},
|
||||
{"type":"NameBuiltin","value":"document"},
|
||||
{"type":"Punctuation","value":"."},
|
||||
|
31
lexers/testdata/php.expected
vendored
31
lexers/testdata/php.expected
vendored
@ -2,32 +2,19 @@
|
||||
{"type":"CommentPreproc","value":"\u003c!DOCTYPE html\u003e"},
|
||||
{"type":"Text","value":"\n"},
|
||||
{"type":"Punctuation","value":"\u003c"},
|
||||
{"type":"Text","value":""},
|
||||
{"type":"NameTag","value":"html"},
|
||||
{"type":"Punctuation","value":""},
|
||||
{"type":"Text","value":""},
|
||||
{"type":"Punctuation","value":"\u003e"},
|
||||
{"type":"Text","value":"\n"},
|
||||
{"type":"Punctuation","value":"\u003c"},
|
||||
{"type":"Text","value":""},
|
||||
{"type":"NameTag","value":"body"},
|
||||
{"type":"Punctuation","value":""},
|
||||
{"type":"Text","value":""},
|
||||
{"type":"Punctuation","value":"\u003e"},
|
||||
{"type":"Text","value":"\n\n"},
|
||||
{"type":"Punctuation","value":"\u003c"},
|
||||
{"type":"Text","value":""},
|
||||
{"type":"NameTag","value":"h1"},
|
||||
{"type":"Punctuation","value":""},
|
||||
{"type":"Text","value":""},
|
||||
{"type":"Punctuation","value":"\u003e"},
|
||||
{"type":"Text","value":"My first PHP page"},
|
||||
{"type":"Punctuation","value":"\u003c"},
|
||||
{"type":"Text","value":""},
|
||||
{"type":"Punctuation","value":"/"},
|
||||
{"type":"Text","value":""},
|
||||
{"type":"Punctuation","value":"\u003c/"},
|
||||
{"type":"NameTag","value":"h1"},
|
||||
{"type":"Text","value":""},
|
||||
{"type":"Punctuation","value":"\u003e"},
|
||||
{"type":"Text","value":"\n"},
|
||||
{"type":"CommentPreproc","value":"\u003c?php"},
|
||||
@ -38,7 +25,6 @@
|
||||
{"type":"Text","value":" "},
|
||||
{"type":"NameVariable","value":"$modx"},
|
||||
{"type":"Operator","value":"-\u003e"},
|
||||
{"type":"Text","value":""},
|
||||
{"type":"NameAttribute","value":"getIterator"},
|
||||
{"type":"Punctuation","value":"("},
|
||||
{"type":"LiteralStringSingle","value":"'modResource'"},
|
||||
@ -65,13 +51,11 @@
|
||||
{"type":"Operator","value":"="},
|
||||
{"type":"NameVariable","value":"$doc"},
|
||||
{"type":"Operator","value":"-\u003e"},
|
||||
{"type":"Text","value":""},
|
||||
{"type":"NameAttribute","value":"content"},
|
||||
{"type":"Punctuation","value":";"},
|
||||
{"type":"Text","value":"\n "},
|
||||
{"type":"NameVariable","value":"$doc"},
|
||||
{"type":"Operator","value":"-\u003e"},
|
||||
{"type":"Text","value":""},
|
||||
{"type":"NameAttribute","value":"set"},
|
||||
{"type":"Punctuation","value":"("},
|
||||
{"type":"LiteralStringDouble","value":"\"content\""},
|
||||
@ -92,7 +76,6 @@
|
||||
{"type":"Punctuation","value":"("},
|
||||
{"type":"NameVariable","value":"$doc"},
|
||||
{"type":"Operator","value":"-\u003e"},
|
||||
{"type":"Text","value":""},
|
||||
{"type":"NameAttribute","value":"content"},
|
||||
{"type":"Punctuation","value":");"},
|
||||
{"type":"Text","value":"\n "},
|
||||
@ -102,20 +85,12 @@
|
||||
{"type":"CommentSingle","value":"// some comment\n"},
|
||||
{"type":"CommentPreproc","value":"?\u003e"},
|
||||
{"type":"Text","value":"\n"},
|
||||
{"type":"Punctuation","value":"\u003c"},
|
||||
{"type":"Text","value":""},
|
||||
{"type":"Punctuation","value":"/"},
|
||||
{"type":"Text","value":""},
|
||||
{"type":"Punctuation","value":"\u003c/"},
|
||||
{"type":"NameTag","value":"body"},
|
||||
{"type":"Text","value":""},
|
||||
{"type":"Punctuation","value":"\u003e"},
|
||||
{"type":"Text","value":"\n"},
|
||||
{"type":"Punctuation","value":"\u003c"},
|
||||
{"type":"Text","value":""},
|
||||
{"type":"Punctuation","value":"/"},
|
||||
{"type":"Text","value":""},
|
||||
{"type":"Punctuation","value":"\u003c/"},
|
||||
{"type":"NameTag","value":"html"},
|
||||
{"type":"Text","value":""},
|
||||
{"type":"Punctuation","value":"\u003e"},
|
||||
{"type":"Text","value":"\n"}
|
||||
]
|
||||
|
1
lexers/testdata/scala.expected
vendored
1
lexers/testdata/scala.expected
vendored
@ -50,7 +50,6 @@
|
||||
{"type":"Keyword","value":":"},
|
||||
{"type":"Text","value":" "},
|
||||
{"type":"KeywordType","value":"Seq"},
|
||||
{"type":"Text","value":""},
|
||||
{"type":"Operator","value":"["},
|
||||
{"type":"KeywordType","value":"T"},
|
||||
{"type":"Operator","value":"];"},
|
||||
|
@ -120,3 +120,12 @@ func Pop(n int) MutatorFunc {
|
||||
func Default(mutators ...Mutator) Rule {
|
||||
return Rule{Mutator: Mutators(mutators...)}
|
||||
}
|
||||
|
||||
// Stringify returns the raw string for a set of tokens.
|
||||
func Stringify(tokens ...*Token) string {
|
||||
out := []string{}
|
||||
for _, t := range tokens {
|
||||
out = append(out, t.Value)
|
||||
}
|
||||
return strings.Join(out, "")
|
||||
}
|
||||
|
Loading…
x
Reference in New Issue
Block a user