Update Caddyfile lexer to fix edgecases and improve colouring

lexers/c/caddyfile.go

@@ -5,15 +5,20 @@ import (
 	"github.com/alecthomas/chroma/lexers/internal"
 )
 
-// CaddyfileCommon are the rules common to both of the lexer variants
+// caddyfileCommon are the rules common to both of the lexer variants
-var CaddyfileCommon = Rules{
+var caddyfileCommon = Rules{
 	"site_block_common": {
 		// Import keyword
 		{`(import)(\s+)([^\s]+)`, ByGroups(Keyword, Text, NameVariableMagic), nil},
 		// Matcher definition
-		{`@[^\s]+\s+`, NameDecorator, Push("matcher")},
+		{`@[^\s]+(?=\s)`, NameDecorator, Push("matcher")},
+		// Matcher token stub for docs
+		{`\[\<matcher\>\]`, NameDecorator, Push("matcher")},
+		// These cannot have matchers but may have things that look like
+		// matchers in their arguments, so we just parse as a subdirective.
+		{`try_files`, Keyword, Push("subdirective")},
 		// These are special, they can nest more directives
-		{`handle|route|handle_path|not`, Keyword, Push("nested_directive")},
+		{`handle_errors|handle|route|handle_path|not`, Keyword, Push("nested_directive")},
 		// Any other directive
 		{`[^\s#]+`, Keyword, Push("directive")},
 		Include("base"),
@@ -40,7 +45,9 @@ var CaddyfileCommon = Rules{
 	"nested_block": {
 		{`\}`, Punctuation, Pop(2)},
 		// Matcher definition
-		{`@[^\s]+\s+`, NameDecorator, Push("matcher")},
+		{`@[^\s]+(?=\s)`, NameDecorator, Push("matcher")},
+		// Something that starts with literally < is probably a docs stub
+		{`\<[^#]+\>`, Keyword, Push("nested_directive")},
 		// Any other directive
 		{`[^\s#]+`, Keyword, Push("nested_directive")},
 		Include("base"),
@@ -60,32 +67,33 @@ var CaddyfileCommon = Rules{
 	"directive": {
 		{`\{(?=\s)`, Punctuation, Push("block")},
 		Include("matcher_token"),
-		Include("comments_pop"),
+		Include("comments_pop_1"),
 		{`\n`, Text, Pop(1)},
 		Include("base"),
 	},
 	"nested_directive": {
 		{`\{(?=\s)`, Punctuation, Push("nested_block")},
 		Include("matcher_token"),
-		Include("comments_pop"),
+		Include("comments_pop_1"),
 		{`\n`, Text, Pop(1)},
 		Include("base"),
 	},
 	"subdirective": {
 		{`\{(?=\s)`, Punctuation, Push("block")},
-		Include("comments_pop"),
+		Include("comments_pop_1"),
 		{`\n`, Text, Pop(1)},
 		Include("base"),
 	},
 	"arguments": {
 		{`\{(?=\s)`, Punctuation, Push("block")},
-		Include("comments_pop"),
+		Include("comments_pop_2"),
+		{`\\\n`, Text, nil}, // Skip escaped newlines
 		{`\n`, Text, Pop(2)},
 		Include("base"),
 	},
 	"deep_subdirective": {
 		{`\{(?=\s)`, Punctuation, Push("block")},
-		Include("comments_pop"),
+		Include("comments_pop_3"),
 		{`\n`, Text, Pop(3)},
 		Include("base"),
 	},
@@ -99,89 +107,33 @@ var CaddyfileCommon = Rules{
 		{`^#.*\n`, CommentSingle, nil},   // Comment at start of line
 		{`\s+#.*\n`, CommentSingle, nil}, // Comment preceded by whitespace
 	},
-	"comments_pop": {
+	"comments_pop_1": {
 		{`^#.*\n`, CommentSingle, Pop(1)},   // Comment at start of line
 		{`\s+#.*\n`, CommentSingle, Pop(1)}, // Comment preceded by whitespace
 	},
+	"comments_pop_2": {
+		{`^#.*\n`, CommentSingle, Pop(2)},   // Comment at start of line
+		{`\s+#.*\n`, CommentSingle, Pop(2)}, // Comment preceded by whitespace
+	},
+	"comments_pop_3": {
+		{`^#.*\n`, CommentSingle, Pop(3)},   // Comment at start of line
+		{`\s+#.*\n`, CommentSingle, Pop(3)}, // Comment preceded by whitespace
+	},
 	"base": {
 		Include("comments"),
-		{`on|off`, NameConstant, nil},
+		{`on|off|first|last|before|after|internal|strip_prefix|strip_suffix|replace`, NameConstant, nil},
 		{`(https?://)?([a-z0-9.-]+)(:)([0-9]+)`, ByGroups(Name, Name, Punctuation, LiteralNumberInteger), nil},
 		{`[a-z-]+/[a-z-+]+`, LiteralString, nil},
 		{`[0-9]+[km]?\b`, LiteralNumberInteger, nil},
-		{`\{[\w+.-]+\}`, NameAttribute, nil}, // Placeholder
+		{`\{[\w+.-]+\}`, LiteralStringEscape, nil}, // Placeholder
-		{`[^\s#{}$]+`, LiteralString, nil},
+		{`\[(?=[^#{}$]+\])`, Punctuation, nil},
+		{`\]|\|`, Punctuation, nil},
+		{`[^\s#{}$\]]+`, LiteralString, nil},
 		{`/[^\s#]*`, Name, nil},
 		{`\s+`, Text, nil},
 	},
 }
 
-// CaddyfileRules are the merged rules for the main Caddyfile lexer
-var CaddyfileRules = (func(a Rules, b Rules) Rules {
-	for k, v := range b {
-		a[k] = v
-	}
-	return a
-})(
-	Rules{
-		"root": {
-			Include("comments"),
-			// Global options block
-			{`^\s*(\{)\s*$`, ByGroups(Punctuation), Push("globals")},
-			// Snippets
-			{`(\([^\s#]+\))(\s*)(\{)`, ByGroups(NameVariableAnonymous, Text, Punctuation), Push("snippet")},
-			// Site label
-			{`[^#{(\s,]+`, NameLabel, Push("label")},
-			// Site label with placeholder
-			{`\{[\w+.-]+\}`, NameAttribute, Push("label")},
-			{`\s+`, Text, nil},
-		},
-		"globals": {
-			{`\}`, Punctuation, Pop(1)},
-			{`[^\s#]+`, KeywordType, Push("directive")},
-			Include("base"),
-		},
-		"snippet": {
-			{`\}`, Punctuation, Pop(1)},
-			// Matcher definition
-			{`@[^\s]+\s+`, NameDecorator, Push("matcher")},
-			// Any directive
-			{`[^\s#]+`, KeywordType, Push("directive")},
-			Include("base"),
-		},
-		"label": {
-			{`,`, Text, nil},
-			{` `, Text, nil},
-			{`[^#{(\s,]+`, NameLabel, nil},
-			// Comment after non-block label (hack because comments end in \n)
-			{`#.*\n`, CommentSingle, Push("site_block")},
-			// Note: if \n, we'll never pop out of the site_block, it's valid
-			{`\{(?=\s)|\n`, Punctuation, Push("site_block")},
-		},
-		"site_block": {
-			{`\}`, Punctuation, Pop(2)},
-			Include("site_block_common"),
-		},
-	},
-	CaddyfileCommon,
-)
-
-// CaddyfileDirectiveRules are the merged rules for the secondary lexer
-var CaddyfileDirectiveRules = (func(a Rules, b Rules) Rules {
-	for k, v := range b {
-		a[k] = v
-	}
-	return a
-})(
-	Rules{
-		// Same as "site_block" in Caddyfile
-		"root": {
-			Include("site_block_common"),
-		},
-	},
-	CaddyfileCommon,
-)
-
 // Caddyfile lexer.
 var Caddyfile = internal.Register(MustNewLexer(
 	&Config{
@@ -190,7 +142,48 @@ var Caddyfile = internal.Register(MustNewLexer(
 		Filenames: []string{"Caddyfile*"},
 		MimeTypes: []string{},
 	},
-	CaddyfileRules,
+	Rules{
+		"root": {
+			Include("comments"),
+			// Global options block
+			{`^\s*(\{)\s*$`, ByGroups(Punctuation), Push("globals")},
+			// Snippets
+			{`(\([^\s#]+\))(\s*)(\{)`, ByGroups(NameVariableAnonymous, Text, Punctuation), Push("snippet")},
+			// Site label
+			{`[^#{(\s,]+`, GenericHeading, Push("label")},
+			// Site label with placeholder
+			{`\{[\w+.-]+\}`, NameAttribute, Push("label")},
+			{`\s+`, Text, nil},
+		},
+		"globals": {
+			{`\}`, Punctuation, Pop(1)},
+			{`[^\s#]+`, Keyword, Push("directive")},
+			Include("base"),
+		},
+		"snippet": {
+			{`\}`, Punctuation, Pop(1)},
+			// Matcher definition
+			{`@[^\s]+(?=\s)`, NameDecorator, Push("matcher")},
+			// Any directive
+			{`[^\s#]+`, Keyword, Push("directive")},
+			Include("base"),
+		},
+		"label": {
+			// Allow multiple labels, comma separated, newlines after
+			// a comma means another label is coming
+			{`,\s*\n?`, Text, nil},
+			{` `, Text, nil},
+			{`[^#{(\s,]+`, GenericHeading, nil},
+			// Comment after non-block label (hack because comments end in \n)
+			{`#.*\n`, CommentSingle, Push("site_block")},
+			// Note: if \n, we'll never pop out of the site_block, it's valid
+			{`\{(?=\s)|\n`, Punctuation, Push("site_block")},
+		},
+		"site_block": {
+			{`\}`, Punctuation, Pop(2)},
+			Include("site_block_common"),
+		},
+	}.Merge(caddyfileCommon),
 ))
 
 // Caddyfile directive-only lexer.
@@ -201,5 +194,10 @@ var CaddyfileDirectives = internal.Register(MustNewLexer(
 		Filenames: []string{},
 		MimeTypes: []string{},
 	},
-	CaddyfileDirectiveRules,
+	Rules{
+		// Same as "site_block" in Caddyfile
+		"root": {
+			Include("site_block_common"),
+		},
+	}.Merge(caddyfileCommon),
 ))

lexers/testdata/caddyfile.expected

@@ -1,13 +1,13 @@
 [
   {"type":"Punctuation","value":"{"},
   {"type":"Text","value":"\n\t"},
-  {"type":"KeywordType","value":"debug"},
+  {"type":"Keyword","value":"debug"},
   {"type":"Text","value":"\n\t"},
-  {"type":"KeywordType","value":"admin"},
+  {"type":"Keyword","value":"admin"},
   {"type":"Text","value":" "},
   {"type":"NameConstant","value":"off"},
   {"type":"Text","value":"\n\t"},
-  {"type":"KeywordType","value":"on_demand_tls"},
+  {"type":"Keyword","value":"on_demand_tls"},
   {"type":"Text","value":" "},
   {"type":"Punctuation","value":"{"},
   {"type":"Text","value":"\n\t\t"},
@@ -24,6 +24,7 @@
   {"type":"Punctuation","value":"{"},
   {"type":"Text","value":"  \n\t"},
   {"type":"NameDecorator","value":"@blocked"},
+  {"type":"Text","value":" "},
   {"type":"Punctuation","value":"{"},
   {"type":"Text","value":"\n\t\t"},
   {"type":"Keyword","value":"path"},
@@ -38,7 +39,7 @@
   {"type":"Text","value":"\n\t"},
   {"type":"Punctuation","value":"}"},
   {"type":"Text","value":"\n\t"},
-  {"type":"KeywordType","value":"redir"},
+  {"type":"Keyword","value":"redir"},
   {"type":"Text","value":" "},
   {"type":"NameDecorator","value":"@blocked"},
   {"type":"Text","value":" "},
@@ -46,9 +47,9 @@
   {"type":"Text","value":"\n"},
   {"type":"Punctuation","value":"}"},
   {"type":"Text","value":"\n\n"},
-  {"type":"NameLabel","value":"example.com"},
+  {"type":"GenericHeading","value":"example.com"},
   {"type":"Text","value":", "},
-  {"type":"NameLabel","value":"fake.org"},
+  {"type":"GenericHeading","value":"fake.org"},
   {"type":"Text","value":" "},
   {"type":"Punctuation","value":"{"},
   {"type":"Text","value":"\n\t"},
@@ -68,6 +69,7 @@
   {"type":"CommentSingle","value":"\n\t\t# Add trailing slash for directory requests\n"},
   {"type":"Text","value":"\t\t"},
   {"type":"NameDecorator","value":"@canonicalPath"},
+  {"type":"Text","value":" "},
   {"type":"Punctuation","value":"{"},
   {"type":"Text","value":"\n\t\t\t"},
   {"type":"Keyword","value":"file"},
@@ -76,7 +78,7 @@
   {"type":"Text","value":"\n\t\t\t\t"},
   {"type":"Keyword","value":"try_files"},
   {"type":"Text","value":" "},
-  {"type":"NameAttribute","value":"{path}"},
+  {"type":"LiteralStringEscape","value":"{path}"},
   {"type":"LiteralString","value":"/index.php"},
   {"type":"Text","value":"\n\t\t\t"},
   {"type":"Punctuation","value":"}"},
@@ -93,7 +95,7 @@
   {"type":"Text","value":" "},
   {"type":"NameDecorator","value":"@canonicalPath"},
   {"type":"Text","value":" "},
-  {"type":"NameAttribute","value":"{path}"},
+  {"type":"LiteralStringEscape","value":"{path}"},
   {"type":"LiteralString","value":"/"},
   {"type":"Text","value":" "},
   {"type":"LiteralNumberInteger","value":"308"},
@@ -109,9 +111,9 @@
   {"type":"Text","value":"\n\t\t\t\t"},
   {"type":"Keyword","value":"try_files"},
   {"type":"Text","value":" "},
-  {"type":"NameAttribute","value":"{path}"},
+  {"type":"LiteralStringEscape","value":"{path}"},
   {"type":"Text","value":" "},
-  {"type":"NameAttribute","value":"{path}"},
+  {"type":"LiteralStringEscape","value":"{path}"},
   {"type":"LiteralString","value":"/index.php"},
   {"type":"Text","value":" "},
   {"type":"LiteralString","value":"index.php"},
@@ -128,7 +130,7 @@
   {"type":"Text","value":" "},
   {"type":"NameDecorator","value":"@indexFiles"},
   {"type":"Text","value":" "},
-  {"type":"NameAttribute","value":"{http.matchers.file.relative}"},
+  {"type":"LiteralStringEscape","value":"{http.matchers.file.relative}"},
   {"type":"CommentSingle","value":"\n\n\t\t# Proxy PHP files to the FastCGI responder\n"},
   {"type":"Text","value":"\t\t"},
   {"type":"NameDecorator","value":"@phpFiles"},
@@ -166,6 +168,7 @@
   {"type":"Punctuation","value":"}"},
   {"type":"Text","value":"\n\n\t"},
   {"type":"NameDecorator","value":"@encode_exts"},
+  {"type":"Text","value":" "},
   {"type":"Punctuation","value":"{"},
   {"type":"Text","value":"\n\t\t"},
   {"type":"Keyword","value":"path"},
@@ -217,6 +220,7 @@
   {"type":"Punctuation","value":"}"},
   {"type":"Text","value":"\n\n\t"},
   {"type":"NameDecorator","value":"@singleLine"},
+  {"type":"Text","value":" "},
   {"type":"Keyword","value":"not"},
   {"type":"Text","value":" "},
   {"type":"Keyword","value":"path"},