go/chroma
1
0
Fork 0
mirror of https://github.com/alecthomas/chroma.git synced 2025-11-27 22:38:42 +02:00
Code Issues Releases Activity

Split PHP into two lexers - PHP and PHTML.

Browse Source 
The former is pure PHP code while the latter is PHP code in <? ?> tags,
within HTML.

Fixes #210.
This commit is contained in:
Alec Thomas
2020-06-30 20:56:49 +10:00
parent 11501493c9
commit 2b9ea60d89
10 changed files with 229 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.golangci.yml
View File

@@ -20,6 +20,11 @@ linters:
- wsl - wsl
- gomnd - gomnd
- gocognit - gocognit
- goerr113
- nolintlint
- testpackage
- godot
- nestif
linters-settings: linters-settings:
govet: govet:

2
.travis.yml
View File

@@ -4,7 +4,7 @@ go:
- "1.13.x" - "1.13.x"
script: script:
- go test -v ./... - go test -v ./...
- curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s v1.22.2 - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s 1.26.0
- ./bin/golangci-lint run - ./bin/golangci-lint run
- git clean -fdx . - git clean -fdx .
after_success: after_success:

23
lexers/circular/php.go
View File

@@ -1,15 +1,12 @@
package circular package circular
import ( import (
"strings"
. "github.com/alecthomas/chroma" // nolint . "github.com/alecthomas/chroma" // nolint
"github.com/alecthomas/chroma/lexers/h"
"github.com/alecthomas/chroma/lexers/internal" "github.com/alecthomas/chroma/lexers/internal"
) )
// PHP lexer. // PHP lexer for pure PHP code (not embedded in HTML).
var PHP = internal.Register(DelegatingLexer(h.HTML, MustNewLexer( var PHP = internal.Register(MustNewLexer(
&Config{ &Config{
Name: "PHP", Name: "PHP",
Aliases: []string{"php", "php3", "php4", "php5"}, Aliases: []string{"php", "php3", "php4", "php5"},
@@ -19,12 +16,10 @@ var PHP = internal.Register(DelegatingLexer(h.HTML, MustNewLexer(
CaseInsensitive: true, CaseInsensitive: true,
EnsureNL: true, EnsureNL: true,
}, },
Rules{ phpCommonRules.Rename("php", "root"),
"root": { ))
{`<\?(php)?`, CommentPreproc, Push("php")},
{`[^<]+`, Other, nil}, var phpCommonRules = Rules{
{`<`, Other, nil},
},
"php": { "php": {
{`\?>`, CommentPreproc, Pop(1)}, {`\?>`, CommentPreproc, Pop(1)},
{`(<<<)([\'"]?)((?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*)(\2\n.*?\n\s*)(\3)(;?)(\n)`, ByGroups(LiteralString, LiteralString, LiteralStringDelimiter, LiteralString, LiteralStringDelimiter, Punctuation, Text), nil}, {`(<<<)([\'"]?)((?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*)(\2\n.*?\n\s*)(\3)(;?)(\n)`, ByGroups(LiteralString, LiteralString, LiteralStringDelimiter, LiteralString, LiteralStringDelimiter, Punctuation, Text), nil},
@@ -82,10 +77,4 @@ var PHP = internal.Register(DelegatingLexer(h.HTML, MustNewLexer(
{`(\$\{)(\S+)(\})`, ByGroups(LiteralStringInterpol, NameVariable, LiteralStringInterpol), nil}, {`(\$\{)(\S+)(\})`, ByGroups(LiteralStringInterpol, NameVariable, LiteralStringInterpol), nil},
{`[${\\]`, LiteralStringDouble, nil}, {`[${\\]`, LiteralStringDouble, nil},
}, },
},
).SetAnalyser(func(text string) float32 {
if strings.Contains(text, "<?php") {
return 0.5
} }
return 0.0
})))

34
lexers/circular/phtml.go Normal file
View File

@@ -0,0 +1,34 @@
package circular
import (
"strings"
. "github.com/alecthomas/chroma" // nolint
"github.com/alecthomas/chroma/lexers/h"
"github.com/alecthomas/chroma/lexers/internal"
)
// PHTML lexer is PHP in HTML.
var PHTML = internal.Register(DelegatingLexer(h.HTML, MustNewLexer(
&Config{
Name: "PHTML",
Aliases: []string{"phtml"},
Filenames: []string{"*.phtml"},
MimeTypes: []string{"application/x-php", "application/x-httpd-php", "application/x-httpd-php3", "application/x-httpd-php4", "application/x-httpd-php5"},
DotAll: true,
CaseInsensitive: true,
EnsureNL: true,
},
Rules{
"root": {
{`<\?(php)?`, CommentPreproc, Push("php")},
{`[^<]+`, Other, nil},
{`<`, Other, nil},
},
}.Merge(phpCommonRules),
).SetAnalyser(func(text string) float32 {
if strings.Contains(text, "<?php") {
return 0.5
}
return 0.0
})))

2
lexers/lexers_test.go
View File

@@ -73,7 +73,7 @@ func TestLexers(t *testing.T) {
if os.Getenv("RECORD") == "true" { if os.Getenv("RECORD") == "true" {
// Update the expected file with the generated output of this lexer // Update the expected file with the generated output of this lexer
f, err := os.Create(expectedFilename) f, err := os.Create(expectedFilename)
defer f.Close() defer f.Close() // nolint: gosec
assert.NoError(t, err) assert.NoError(t, err)
assert.NoError(t, formatters.JSON.Format(f, nil, chroma.Literator(actual...))) assert.NoError(t, formatters.JSON.Format(f, nil, chroma.Literator(actual...)))
} else { } else {

10
lexers/testdata/php.actual vendored
View File

@@ -1,10 +1,3 @@
<!DOCTYPE html>
<html>
<body>
<h1>My first PHP page</h1>
<?php
$docs = $modx->getIterator('modResource', ["parent" => 84]); $docs = $modx->getIterator('modResource', ["parent" => 84]);
foreach($docs as $doc){ foreach($docs as $doc){
@@ -14,6 +7,3 @@ foreach($docs as $doc){
// $doc->save(); // $doc->save();
} }
// some comment // some comment
?>
</body>
</html>

32
lexers/testdata/php.expected vendored
View File

@@ -1,24 +1,4 @@
[ [
{"type":"CommentPreproc","value":"\u003c!DOCTYPE html\u003e"},
{"type":"Text","value":"\n"},
{"type":"Punctuation","value":"\u003c"},
{"type":"NameTag","value":"html"},
{"type":"Punctuation","value":"\u003e"},
{"type":"Text","value":"\n"},
{"type":"Punctuation","value":"\u003c"},
{"type":"NameTag","value":"body"},
{"type":"Punctuation","value":"\u003e"},
{"type":"Text","value":"\n\n"},
{"type":"Punctuation","value":"\u003c"},
{"type":"NameTag","value":"h1"},
{"type":"Punctuation","value":"\u003e"},
{"type":"Text","value":"My first PHP page"},
{"type":"Punctuation","value":"\u003c/"},
{"type":"NameTag","value":"h1"},
{"type":"Punctuation","value":"\u003e"},
{"type":"Text","value":"\n"},
{"type":"CommentPreproc","value":"\u003c?php"},
{"type":"Text","value":"\n\n"},
{"type":"NameVariable","value":"$docs"}, {"type":"NameVariable","value":"$docs"},
{"type":"Text","value":" "}, {"type":"Text","value":" "},
{"type":"Operator","value":"="}, {"type":"Operator","value":"="},
@@ -82,15 +62,5 @@
{"type":"CommentSingle","value":"// $doc-\u003esave();\n"}, {"type":"CommentSingle","value":"// $doc-\u003esave();\n"},
{"type":"Punctuation","value":"}"}, {"type":"Punctuation","value":"}"},
{"type":"Text","value":"\n"}, {"type":"Text","value":"\n"},
{"type":"CommentSingle","value":"// some comment\n"}, {"type":"CommentSingle","value":"// some comment\n"}
{"type":"CommentPreproc","value":"?\u003e"},
{"type":"Text","value":"\n"},
{"type":"Punctuation","value":"\u003c/"},
{"type":"NameTag","value":"body"},
{"type":"Punctuation","value":"\u003e"},
{"type":"Text","value":"\n"},
{"type":"Punctuation","value":"\u003c/"},
{"type":"NameTag","value":"html"},
{"type":"Punctuation","value":"\u003e"},
{"type":"Text","value":"\n"}
] ]

19
lexers/testdata/phtml.actual vendored Normal file
View File

@@ -0,0 +1,19 @@
<!DOCTYPE html>
<html>
<body>
<h1>My first PHP page</h1>
<?php
$docs = $modx->getIterator('modResource', ["parent" => 84]);
foreach($docs as $doc){
$q=$doc->content;
$doc->set("content", preg_replace("/Some value/i", "Replacement", $q));
print_r($doc->content);
// $doc->save();
}
// some comment
?>
</body>
</html>

96
lexers/testdata/phtml.expected vendored Normal file
View File

@@ -0,0 +1,96 @@
[
{"type":"CommentPreproc","value":"\u003c!DOCTYPE html\u003e"},
{"type":"Text","value":"\n"},
{"type":"Punctuation","value":"\u003c"},
{"type":"NameTag","value":"html"},
{"type":"Punctuation","value":"\u003e"},
{"type":"Text","value":"\n"},
{"type":"Punctuation","value":"\u003c"},
{"type":"NameTag","value":"body"},
{"type":"Punctuation","value":"\u003e"},
{"type":"Text","value":"\n\n"},
{"type":"Punctuation","value":"\u003c"},
{"type":"NameTag","value":"h1"},
{"type":"Punctuation","value":"\u003e"},
{"type":"Text","value":"My first PHP page"},
{"type":"Punctuation","value":"\u003c/"},
{"type":"NameTag","value":"h1"},
{"type":"Punctuation","value":"\u003e"},
{"type":"Text","value":"\n"},
{"type":"CommentPreproc","value":"\u003c?php"},
{"type":"Text","value":"\n\n"},
{"type":"NameVariable","value":"$docs"},
{"type":"Text","value":" "},
{"type":"Operator","value":"="},
{"type":"Text","value":" "},
{"type":"NameVariable","value":"$modx"},
{"type":"Operator","value":"-\u003e"},
{"type":"NameAttribute","value":"getIterator"},
{"type":"Punctuation","value":"("},
{"type":"LiteralStringSingle","value":"'modResource'"},
{"type":"Punctuation","value":","},
{"type":"Text","value":" "},
{"type":"Punctuation","value":"["},
{"type":"LiteralStringDouble","value":"\"parent\""},
{"type":"Text","value":" "},
{"type":"Operator","value":"=\u003e"},
{"type":"Text","value":" "},
{"type":"LiteralNumberInteger","value":"84"},
{"type":"Punctuation","value":"]);"},
{"type":"Text","value":"\n\n"},
{"type":"Keyword","value":"foreach"},
{"type":"Punctuation","value":"("},
{"type":"NameVariable","value":"$docs"},
{"type":"Text","value":" "},
{"type":"Keyword","value":"as"},
{"type":"Text","value":" "},
{"type":"NameVariable","value":"$doc"},
{"type":"Punctuation","value":"){"},
{"type":"Text","value":"\n "},
{"type":"NameVariable","value":"$q"},
{"type":"Operator","value":"="},
{"type":"NameVariable","value":"$doc"},
{"type":"Operator","value":"-\u003e"},
{"type":"NameAttribute","value":"content"},
{"type":"Punctuation","value":";"},
{"type":"Text","value":"\n "},
{"type":"NameVariable","value":"$doc"},
{"type":"Operator","value":"-\u003e"},
{"type":"NameAttribute","value":"set"},
{"type":"Punctuation","value":"("},
{"type":"LiteralStringDouble","value":"\"content\""},
{"type":"Punctuation","value":","},
{"type":"Text","value":" "},
{"type":"NameOther","value":"preg_replace"},
{"type":"Punctuation","value":"("},
{"type":"LiteralStringDouble","value":"\"/Some value/i\""},
{"type":"Punctuation","value":","},
{"type":"Text","value":" "},
{"type":"LiteralStringDouble","value":"\"Replacement\""},
{"type":"Punctuation","value":","},
{"type":"Text","value":" "},
{"type":"NameVariable","value":"$q"},
{"type":"Punctuation","value":"));"},
{"type":"Text","value":"\n "},
{"type":"NameOther","value":"print_r"},
{"type":"Punctuation","value":"("},
{"type":"NameVariable","value":"$doc"},
{"type":"Operator","value":"-\u003e"},
{"type":"NameAttribute","value":"content"},
{"type":"Punctuation","value":");"},
{"type":"Text","value":"\n "},
{"type":"CommentSingle","value":"// $doc-\u003esave();\n"},
{"type":"Punctuation","value":"}"},
{"type":"Text","value":" \n"},
{"type":"CommentSingle","value":"// some comment\n"},
{"type":"CommentPreproc","value":"?\u003e"},
{"type":"Text","value":"\n"},
{"type":"Punctuation","value":"\u003c/"},
{"type":"NameTag","value":"body"},
{"type":"Punctuation","value":"\u003e"},
{"type":"Text","value":"\n"},
{"type":"Punctuation","value":"\u003c/"},
{"type":"NameTag","value":"html"},
{"type":"Punctuation","value":"\u003e"},
{"type":"Text","value":"\n"}
]

8
regexp.go
View File

@@ -160,6 +160,14 @@ func Tokenise(lexer Lexer, options *TokeniseOptions, text string) ([]Token, erro
// Rules maps from state to a sequence of Rules. // Rules maps from state to a sequence of Rules.
type Rules map[string][]Rule type Rules map[string][]Rule
// Rename clones rules then a rule.
func (r Rules) Rename(old, new string) Rules {
r = r.Clone()
r[new] = r[old]
delete(r, old)
return r
}
// Clone returns a clone of the Rules. // Clone returns a clone of the Rules.
func (r Rules) Clone() Rules { func (r Rules) Clone() Rules {
out := map[string][]Rule{} out := map[string][]Rule{}