1
0
mirror of https://github.com/alecthomas/chroma.git synced 2025-02-15 13:33:12 +02:00

yaml parse plan scalar values as literals rather than text.

This commit is contained in:
thomas 2020-05-11 02:42:58 +01:00 committed by Alec Thomas
parent 705c61c909
commit 684c267d74
2 changed files with 54 additions and 221 deletions

View File

@ -18,18 +18,12 @@
{"type":"NameTag","value":"key"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"value"},
{"type":"Literal","value":"value"},
{"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"another_key"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"Another"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"value"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"goes"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"here."},
{"type":"Literal","value":"Another value goes here."},
{"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"a_number_value"},
{"type":"Punctuation","value":":"},
@ -58,7 +52,7 @@
{"type":"NameTag","value":"key with spaces"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"value"},
{"type":"Literal","value":"value"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Comment","value":"# Notice that strings don't need to be quoted. However, they can be."},
{"type":"TextWhitespace","value":"\n"},
@ -87,7 +81,7 @@
{"type":"NameTag","value":"Superscript two"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"\\u00B2"},
{"type":"Literal","value":"\\u00B2"},
{"type":"TextWhitespace","value":"\n\n"},
{"type":"Comment","value":"# Multiple-line strings can be written either as a 'literal block' (using |),"},
{"type":"TextWhitespace","value":"\n"},
@ -98,125 +92,30 @@
{"type":"TextWhitespace","value":" "},
{"type":"LiteralStringDoc","value":"|\n This entire block of text will be the value of the 'literal_block' key,"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Text","value":"with"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"line"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"breaks"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"being"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"preserved."},
{"type":"Literal","value":"with line breaks being preserved."},
{"type":"TextWhitespace","value":"\n\n "},
{"type":"Text","value":"The"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"literal"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"continues"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"until"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"de-dented"},
{"type":"Punctuation","value":","},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"and"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"the"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"leading"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"indentation"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"is"},
{"type":"Literal","value":"The literal continues until de-dented, and the leading indentation is"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Text","value":"stripped."},
{"type":"Literal","value":"stripped."},
{"type":"TextWhitespace","value":"\n\n "},
{"type":"Text","value":"Any"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"lines"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"that"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"are"},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralStringSingle","value":"'more-indented'"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"keep"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"the"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"rest"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"of"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"their"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"indentation"},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralStringDoc","value":"-\n these lines will be indented by 4 spaces."},
{"type":"Literal","value":"Any lines that are 'more-indented' keep the rest of their indentation -"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Literal","value":"these lines will be indented by 4 spaces."},
{"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"folded_style"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralStringDoc","value":"\u003e\n This entire block of text will be the value of 'folded_style', but this"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Text","value":"time"},
{"type":"Punctuation","value":","},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"all"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"newlines"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"will"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"be"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"replaced"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"with"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"a"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"single"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"space."},
{"type":"Literal","value":"time, all newlines will be replaced with a single space."},
{"type":"TextWhitespace","value":"\n\n "},
{"type":"Text","value":"Blank"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"lines"},
{"type":"Punctuation","value":","},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"like"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"above"},
{"type":"Punctuation","value":","},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"are"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"converted"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"to"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"a"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"newline"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"character."},
{"type":"Literal","value":"Blank lines, like above, are converted to a newline character."},
{"type":"TextWhitespace","value":"\n\n "},
{"type":"LiteralStringSingle","value":"'More-indented'"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"lines"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"keep"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"their"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"newlines"},
{"type":"Punctuation","value":","},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"too"},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralStringDoc","value":"-\n this text will appear over two lines."},
{"type":"Literal","value":"lines keep their newlines, too -"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Literal","value":"this text will appear over two lines."},
{"type":"TextWhitespace","value":"\n\n"},
{"type":"Comment","value":"####################"},
{"type":"TextWhitespace","value":"\n"},
@ -232,14 +131,12 @@
{"type":"NameTag","value":"key"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"value"},
{"type":"Literal","value":"value"},
{"type":"TextWhitespace","value":"\n "},
{"type":"NameTag","value":"another_key"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"Another"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"Value"},
{"type":"Literal","value":"Another Value"},
{"type":"TextWhitespace","value":"\n "},
{"type":"NameTag","value":"another_nested_map"},
{"type":"Punctuation","value":":"},
@ -247,18 +144,14 @@
{"type":"NameTag","value":"hello"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"hello"},
{"type":"Literal","value":"hello"},
{"type":"TextWhitespace","value":"\n\n"},
{"type":"Comment","value":"# Maps don't have to have string keys."},
{"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"0.25"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"a"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"float"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"key"},
{"type":"Literal","value":"a float key"},
{"type":"TextWhitespace","value":"\n\n"},
{"type":"Comment","value":"# Keys can also be complex, like multi-line objects"},
{"type":"TextWhitespace","value":"\n"},
@ -268,25 +161,11 @@
{"type":"TextWhitespace","value":" "},
{"type":"LiteralStringDoc","value":"|\n This is a key"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Text","value":"that"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"has"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"multiple"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"lines"},
{"type":"Literal","value":"that has multiple lines"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"and"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"this"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"is"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"its"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"value"},
{"type":"Literal","value":"and this is its value"},
{"type":"TextWhitespace","value":"\n\n"},
{"type":"Comment","value":"# YAML also allows mapping between sequences with the complex key syntax"},
{"type":"TextWhitespace","value":"\n"},
@ -294,13 +173,11 @@
{"type":"TextWhitespace","value":"\n"},
{"type":"Comment","value":"# An example"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Text","value":"? - Manchester"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"United"},
{"type":"Text","value":"? - "},
{"type":"Literal","value":"Manchester United"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Text","value":"- Real"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"Madrid"},
{"type":"Text","value":"- "},
{"type":"Literal","value":"Real Madrid"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
@ -318,53 +195,41 @@
{"type":"NameTag","value":"a_sequence"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Text","value":"- Item"},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralNumber","value":"1"},
{"type":"Text","value":"- "},
{"type":"Literal","value":"Item 1"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Text","value":"- Item"},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralNumber","value":"2"},
{"type":"Text","value":"- "},
{"type":"Literal","value":"Item 2"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Text","value":"- "},
{"type":"LiteralNumber","value":"0.5"},
{"type":"TextWhitespace","value":" "},
{"type":"Comment","value":"# sequences can contain disparate types."},
{"type":"TextWhitespace","value":"\n "},
{"type":"Text","value":"- Item"},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralNumber","value":"4"},
{"type":"Text","value":"- "},
{"type":"Literal","value":"Item 4"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Text","value":"- "},
{"type":"NameTag","value":"key"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"value"},
{"type":"Literal","value":"value"},
{"type":"TextWhitespace","value":"\n "},
{"type":"NameTag","value":"another_key"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"another_value"},
{"type":"Literal","value":"another_value"},
{"type":"TextWhitespace","value":"\n "},
{"type":"LiteralStringDoc","value":"-\n - This is a sequence"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Text","value":"- inside"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"another"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"sequence"},
{"type":"Text","value":"- "},
{"type":"Literal","value":"inside another sequence"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Text","value":"- - - Nested"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"sequence"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"indicators"},
{"type":"Text","value":"- - - "},
{"type":"Literal","value":"Nested sequence indicators"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Text","value":"- can"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"be"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"collapsed"},
{"type":"Text","value":"- "},
{"type":"Literal","value":"can be collapsed"},
{"type":"TextWhitespace","value":"\n\n"},
{"type":"Comment","value":"# Since YAML is a superset of JSON, you can also write JSON-style maps and"},
{"type":"TextWhitespace","value":"\n"},
@ -411,9 +276,7 @@
{"type":"LiteralNumber","value":"1"},
{"type":"Punctuation","value":","},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"takeoff"},
{"type":"Punctuation","value":"]"},
{"type":"Text","value":"}"},
{"type":"Literal","value":"takeoff]}"},
{"type":"TextWhitespace","value":"\n\n"},
{"type":"Comment","value":"#######################"},
{"type":"TextWhitespace","value":"\n"},
@ -430,25 +293,7 @@
{"type":"TextWhitespace","value":" "},
{"type":"CommentPreproc","value":"\u0026anchor_name"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"This"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"string"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"will"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"appear"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"as"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"the"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"value"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"of"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"two"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"keys."},
{"type":"Literal","value":"This string will appear as the value of two keys."},
{"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"other_anchor"},
{"type":"Punctuation","value":":"},
@ -465,13 +310,7 @@
{"type":"NameTag","value":"name"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"Everyone"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"has"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"same"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"name"},
{"type":"Literal","value":"Everyone has same name"},
{"type":"TextWhitespace","value":"\n\n"},
{"type":"Comment","value":"# The regexp \u003c\u003c is called Merge Key Language-Independent Type. It is used to"},
{"type":"TextWhitespace","value":"\n"},
@ -530,7 +369,7 @@
{"type":"CommentPreproc","value":"!!python/complex"},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralNumber","value":"1"},
{"type":"Text","value":"+2j"},
{"type":"Literal","value":"+2j"},
{"type":"TextWhitespace","value":"\n\n"},
{"type":"Comment","value":"# We can also use yaml complex keys with language specific tags"},
{"type":"TextWhitespace","value":"\n"},
@ -547,9 +386,7 @@
{"type":"TextWhitespace","value":"\n"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"Fifty"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"Seven"},
{"type":"Literal","value":"Fifty Seven"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Comment","value":"# Would be {(5, 7): 'Fifty Seven'} in Python"},
{"type":"TextWhitespace","value":"\n\n"},
@ -589,10 +426,11 @@
{"type":"TextWhitespace","value":" "},
{"type":"LiteralStringDoc","value":"|\n R0lGODlhDAAMAIQAAP//9/X17unp5WZmZgAAAOfn515eXvPz7Y6OjuDg4J+fn5"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Text","value":"OTk6enp56enmlpaWNjY6Ojo4SEhP/++f/++f/++f/++f/++f/++f/++f/++f/"},
{"type":"LiteralStringDoc","value":"+\n +f/++f/++f/++f/++f/++SH+Dk1hZGUgd2l0aCBHSU1QACwAAAAADAAMAAAFLC"},
{"type":"Literal","value":"OTk6enp56enmlpaWNjY6Ojo4SEhP/++f/++f/++f/++f/++f/++f/++f/++f/+"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Text","value":"AgjoEwnuNAFOhpEMTRiggcz4BNJHrv/zCFcLiwMWYNG84BwwEeECcgggoBADs="},
{"type":"Literal","value":"+f/++f/++f/++f/++f/++SH+Dk1hZGUgd2l0aCBHSU1QACwAAAAADAAMAAAFLC"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Literal","value":"AgjoEwnuNAFOhpEMTRiggcz4BNJHrv/zCFcLiwMWYNG84BwwEeECcgggoBADs="},
{"type":"TextWhitespace","value":"\n\n"},
{"type":"Comment","value":"# YAML also has a set type, which looks like this:"},
{"type":"TextWhitespace","value":"\n"},
@ -601,26 +439,21 @@
{"type":"TextWhitespace","value":"\n "},
{"type":"Punctuation","value":"?"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"item1"},
{"type":"Literal","value":"item1"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Punctuation","value":"?"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"item2"},
{"type":"Literal","value":"item2"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Punctuation","value":"?"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"item3"},
{"type":"Literal","value":"item3"},
{"type":"TextWhitespace","value":"\n"},
{"type":"NameTag","value":"or"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"{item1"},
{"type":"Punctuation","value":","},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"item2"},
{"type":"Punctuation","value":","},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"item3}"},
{"type":"Text","value":"{"},
{"type":"Literal","value":"item1, item2, item3}"},
{"type":"TextWhitespace","value":"\n\n"},
{"type":"Comment","value":"# Sets are just maps with null values; the above is equivalent to:"},
{"type":"TextWhitespace","value":"\n"},

View File

@ -35,7 +35,7 @@ var YAML = internal.Register(MustNewLexer(
{`'(?:\\.|[^'])*'`, StringSingle, nil},
{`\d\d\d\d-\d\d-\d\d([T ]\d\d:\d\d:\d\d(\.\d+)?(Z|\s+[-+]\d+)?)?`, LiteralDate, nil},
{`\b[+\-]?(0x[\da-f]+|0o[0-7]+|(\d+\.?\d*|\.?\d+)(e[\+\-]?\d+)?|\.inf|\.nan)\b`, Number, nil},
{`\b[\w]+\b`, Text, nil},
{`[^\{\}\[\]\?,\:\!\-\*&\@].*`, Literal, nil},
},
"key": {
{`"[^"\n].*": `, NameTag, nil},