1
0
mirror of https://github.com/alecthomas/chroma.git synced 2025-03-17 20:58:08 +02:00

Add Zig lexer

Add a lexer for the Zig language (https://ziglang.org) based on the
pygments Zig lexer.
This commit is contained in:
Robin Thrift 2020-05-16 13:08:14 +02:00 committed by Alec Thomas
parent 68968043c3
commit bd463054be
4 changed files with 574 additions and 0 deletions

View File

@ -32,6 +32,7 @@ import (
_ "github.com/alecthomas/chroma/lexers/w"
_ "github.com/alecthomas/chroma/lexers/x"
_ "github.com/alecthomas/chroma/lexers/y"
_ "github.com/alecthomas/chroma/lexers/z"
)
// Registry of Lexers.

66
lexers/testdata/zig.actual vendored Normal file
View File

@ -0,0 +1,66 @@
const std = @import("std.zig");
const builtin = std.builtin;
const testing = std.testing;
pub fn once(comptime f: fn () void) Once(f) {
return Once(f){};
}
/// An object that executes the function `f` just once.
pub fn Once(comptime f: fn () void) type {
return struct {
done: bool = false,
mutex: std.Mutex = std.Mutex.init(),
/// Call the function `f`.
/// If `call` is invoked multiple times `f` will be executed only the
/// first time.
/// The invocations are thread-safe.
pub fn call(self: *@This()) void {
if (@atomicLoad(bool, &self.done, .Acquire))
return;
return self.callSlow();
}
fn callSlow(self: *@This()) void {
@setCold(true);
const T = self.mutex.acquire();
defer T.release();
// The first thread to acquire the mutex gets to run the initializer
if (!self.done) {
f();
@atomicStore(bool, &self.done, true, .Release);
}
}
};
}
var global_number: i32 = 0;
var global_once = once(incr);
fn incr() void {
global_number += 1;
}
test "Once executes its function just once" {
if (builtin.single_threaded) {
global_once.call();
global_once.call();
} else {
var threads: [10]*std.Thread = undefined;
defer for (threads) |handle| handle.wait();
for (threads) |*handle| {
handle.* = try std.Thread.spawn(@as(u8, 0), struct {
fn thread_fn(x: u8) void {
global_once.call();
}
}.thread_fn);
}
}
testing.expectEqual(@as(i32, 1), global_number);
}

453
lexers/testdata/zig.expected vendored Normal file
View File

@ -0,0 +1,453 @@
[
{"type":"KeywordReserved","value":"const"},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"std"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
{"type":"TextWhitespace","value":" "},
{"type":"NameBuiltin","value":"@import"},
{"type":"Punctuation","value":"("},
{"type":"LiteralString","value":"\"std.zig\""},
{"type":"Punctuation","value":");"},
{"type":"TextWhitespace","value":"\n"},
{"type":"KeywordReserved","value":"const"},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"builtin"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"std"},
{"type":"Punctuation","value":"."},
{"type":"Name","value":"builtin"},
{"type":"Punctuation","value":";"},
{"type":"TextWhitespace","value":"\n"},
{"type":"KeywordReserved","value":"const"},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"testing"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"std"},
{"type":"Punctuation","value":"."},
{"type":"Name","value":"testing"},
{"type":"Punctuation","value":";"},
{"type":"TextWhitespace","value":"\n\n"},
{"type":"KeywordReserved","value":"pub"},
{"type":"TextWhitespace","value":" "},
{"type":"Keyword","value":"fn"},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"once"},
{"type":"Punctuation","value":"("},
{"type":"KeywordReserved","value":"comptime"},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"f"},
{"type":"Operator","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Keyword","value":"fn"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"()"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordType","value":"void"},
{"type":"Punctuation","value":")"},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"Once"},
{"type":"Punctuation","value":"("},
{"type":"Name","value":"f"},
{"type":"Punctuation","value":")"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"{"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Keyword","value":"return"},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"Once"},
{"type":"Punctuation","value":"("},
{"type":"Name","value":"f"},
{"type":"Punctuation","value":"){};"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Punctuation","value":"}"},
{"type":"TextWhitespace","value":"\n\n"},
{"type":"CommentSingle","value":"/// An object that executes the function `f` just once.\n"},
{"type":"KeywordReserved","value":"pub"},
{"type":"TextWhitespace","value":" "},
{"type":"Keyword","value":"fn"},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"Once"},
{"type":"Punctuation","value":"("},
{"type":"KeywordReserved","value":"comptime"},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"f"},
{"type":"Operator","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Keyword","value":"fn"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"()"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordType","value":"void"},
{"type":"Punctuation","value":")"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordType","value":"type"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"{"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Keyword","value":"return"},
{"type":"TextWhitespace","value":" "},
{"type":"Keyword","value":"struct"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"{"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Name","value":"done"},
{"type":"Operator","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordType","value":"bool"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordConstant","value":"false"},
{"type":"Punctuation","value":","},
{"type":"TextWhitespace","value":"\n "},
{"type":"Name","value":"mutex"},
{"type":"Operator","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"std"},
{"type":"Punctuation","value":"."},
{"type":"Name","value":"Mutex"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"std"},
{"type":"Punctuation","value":"."},
{"type":"Name","value":"Mutex"},
{"type":"Punctuation","value":"."},
{"type":"Name","value":"init"},
{"type":"Punctuation","value":"(),"},
{"type":"TextWhitespace","value":"\n\n "},
{"type":"CommentSingle","value":"/// Call the function `f`.\n"},
{"type":"TextWhitespace","value":" "},
{"type":"CommentSingle","value":"/// If `call` is invoked multiple times `f` will be executed only the\n"},
{"type":"TextWhitespace","value":" "},
{"type":"CommentSingle","value":"/// first time.\n"},
{"type":"TextWhitespace","value":" "},
{"type":"CommentSingle","value":"/// The invocations are thread-safe.\n"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordReserved","value":"pub"},
{"type":"TextWhitespace","value":" "},
{"type":"Keyword","value":"fn"},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"call"},
{"type":"Punctuation","value":"("},
{"type":"Name","value":"self"},
{"type":"Operator","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"*"},
{"type":"NameBuiltin","value":"@This"},
{"type":"Punctuation","value":"())"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordType","value":"void"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"{"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Keyword","value":"if"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"("},
{"type":"NameBuiltin","value":"@atomicLoad"},
{"type":"Punctuation","value":"("},
{"type":"KeywordType","value":"bool"},
{"type":"Punctuation","value":","},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"\u0026"},
{"type":"Name","value":"self"},
{"type":"Punctuation","value":"."},
{"type":"Name","value":"done"},
{"type":"Punctuation","value":","},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"."},
{"type":"Name","value":"Acquire"},
{"type":"Punctuation","value":"))"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Keyword","value":"return"},
{"type":"Punctuation","value":";"},
{"type":"TextWhitespace","value":"\n\n "},
{"type":"Keyword","value":"return"},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"self"},
{"type":"Punctuation","value":"."},
{"type":"Name","value":"callSlow"},
{"type":"Punctuation","value":"();"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Punctuation","value":"}"},
{"type":"TextWhitespace","value":"\n\n "},
{"type":"Keyword","value":"fn"},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"callSlow"},
{"type":"Punctuation","value":"("},
{"type":"Name","value":"self"},
{"type":"Operator","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"*"},
{"type":"NameBuiltin","value":"@This"},
{"type":"Punctuation","value":"())"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordType","value":"void"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"{"},
{"type":"TextWhitespace","value":"\n "},
{"type":"NameBuiltin","value":"@setCold"},
{"type":"Punctuation","value":"("},
{"type":"KeywordConstant","value":"true"},
{"type":"Punctuation","value":");"},
{"type":"TextWhitespace","value":"\n\n "},
{"type":"KeywordReserved","value":"const"},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"T"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"self"},
{"type":"Punctuation","value":"."},
{"type":"Name","value":"mutex"},
{"type":"Punctuation","value":"."},
{"type":"Name","value":"acquire"},
{"type":"Punctuation","value":"();"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Keyword","value":"defer"},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"T"},
{"type":"Punctuation","value":"."},
{"type":"Name","value":"release"},
{"type":"Punctuation","value":"();"},
{"type":"TextWhitespace","value":"\n\n "},
{"type":"CommentSingle","value":"// The first thread to acquire the mutex gets to run the initializer\n"},
{"type":"TextWhitespace","value":" "},
{"type":"Keyword","value":"if"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"("},
{"type":"Operator","value":"!"},
{"type":"Name","value":"self"},
{"type":"Punctuation","value":"."},
{"type":"Name","value":"done"},
{"type":"Punctuation","value":")"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"{"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Name","value":"f"},
{"type":"Punctuation","value":"();"},
{"type":"TextWhitespace","value":"\n "},
{"type":"NameBuiltin","value":"@atomicStore"},
{"type":"Punctuation","value":"("},
{"type":"KeywordType","value":"bool"},
{"type":"Punctuation","value":","},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"\u0026"},
{"type":"Name","value":"self"},
{"type":"Punctuation","value":"."},
{"type":"Name","value":"done"},
{"type":"Punctuation","value":","},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordConstant","value":"true"},
{"type":"Punctuation","value":","},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"."},
{"type":"Name","value":"Release"},
{"type":"Punctuation","value":");"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Punctuation","value":"}"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Punctuation","value":"}"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Punctuation","value":"};"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Punctuation","value":"}"},
{"type":"TextWhitespace","value":"\n\n"},
{"type":"KeywordReserved","value":"var"},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"global_number"},
{"type":"Operator","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordType","value":"i32"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralNumberInteger","value":"0"},
{"type":"Punctuation","value":";"},
{"type":"TextWhitespace","value":"\n"},
{"type":"KeywordReserved","value":"var"},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"global_once"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"once"},
{"type":"Punctuation","value":"("},
{"type":"Name","value":"incr"},
{"type":"Punctuation","value":");"},
{"type":"TextWhitespace","value":"\n\n"},
{"type":"Keyword","value":"fn"},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"incr"},
{"type":"Punctuation","value":"()"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordType","value":"void"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"{"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Name","value":"global_number"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"+="},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralNumberInteger","value":"1"},
{"type":"Punctuation","value":";"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Punctuation","value":"}"},
{"type":"TextWhitespace","value":"\n\n"},
{"type":"Keyword","value":"test"},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralString","value":"\"Once executes its function just once\""},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"{"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Keyword","value":"if"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"("},
{"type":"Name","value":"builtin"},
{"type":"Punctuation","value":"."},
{"type":"Name","value":"single_threaded"},
{"type":"Punctuation","value":")"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"{"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Name","value":"global_once"},
{"type":"Punctuation","value":"."},
{"type":"Name","value":"call"},
{"type":"Punctuation","value":"();"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Name","value":"global_once"},
{"type":"Punctuation","value":"."},
{"type":"Name","value":"call"},
{"type":"Punctuation","value":"();"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Punctuation","value":"}"},
{"type":"TextWhitespace","value":" "},
{"type":"Keyword","value":"else"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"{"},
{"type":"TextWhitespace","value":"\n "},
{"type":"KeywordReserved","value":"var"},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"threads"},
{"type":"Operator","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"["},
{"type":"LiteralNumberInteger","value":"10"},
{"type":"Punctuation","value":"]"},
{"type":"Operator","value":"*"},
{"type":"Name","value":"std"},
{"type":"Punctuation","value":"."},
{"type":"Name","value":"Thread"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordConstant","value":"undefined"},
{"type":"Punctuation","value":";"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Keyword","value":"defer"},
{"type":"TextWhitespace","value":" "},
{"type":"Keyword","value":"for"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"("},
{"type":"Name","value":"threads"},
{"type":"Punctuation","value":")"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"|"},
{"type":"Name","value":"handle"},
{"type":"Operator","value":"|"},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"handle"},
{"type":"Punctuation","value":"."},
{"type":"Name","value":"wait"},
{"type":"Punctuation","value":"();"},
{"type":"TextWhitespace","value":"\n\n "},
{"type":"Keyword","value":"for"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"("},
{"type":"Name","value":"threads"},
{"type":"Punctuation","value":")"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"|*"},
{"type":"Name","value":"handle"},
{"type":"Operator","value":"|"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"{"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Name","value":"handle"},
{"type":"Punctuation","value":"."},
{"type":"Operator","value":"*"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
{"type":"TextWhitespace","value":" "},
{"type":"Keyword","value":"try"},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"std"},
{"type":"Punctuation","value":"."},
{"type":"Name","value":"Thread"},
{"type":"Punctuation","value":"."},
{"type":"Name","value":"spawn"},
{"type":"Punctuation","value":"("},
{"type":"NameBuiltin","value":"@as"},
{"type":"Punctuation","value":"("},
{"type":"KeywordType","value":"u8"},
{"type":"Punctuation","value":","},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralNumberInteger","value":"0"},
{"type":"Punctuation","value":"),"},
{"type":"TextWhitespace","value":" "},
{"type":"Keyword","value":"struct"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"{"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Keyword","value":"fn"},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"thread_fn"},
{"type":"Punctuation","value":"("},
{"type":"Name","value":"x"},
{"type":"Operator","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordType","value":"u8"},
{"type":"Punctuation","value":")"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordType","value":"void"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"{"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Name","value":"global_once"},
{"type":"Punctuation","value":"."},
{"type":"Name","value":"call"},
{"type":"Punctuation","value":"();"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Punctuation","value":"}"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Punctuation","value":"}."},
{"type":"Name","value":"thread_fn"},
{"type":"Punctuation","value":");"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Punctuation","value":"}"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Punctuation","value":"}"},
{"type":"TextWhitespace","value":"\n\n "},
{"type":"Name","value":"testing"},
{"type":"Punctuation","value":"."},
{"type":"Name","value":"expectEqual"},
{"type":"Punctuation","value":"("},
{"type":"NameBuiltin","value":"@as"},
{"type":"Punctuation","value":"("},
{"type":"KeywordType","value":"i32"},
{"type":"Punctuation","value":","},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralNumberInteger","value":"1"},
{"type":"Punctuation","value":"),"},
{"type":"TextWhitespace","value":" "},
{"type":"Name","value":"global_number"},
{"type":"Punctuation","value":");"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Punctuation","value":"}"},
{"type":"TextWhitespace","value":"\n"}
]

54
lexers/z/zig.go Normal file
View File

@ -0,0 +1,54 @@
package z
import (
. "github.com/alecthomas/chroma" // nolint
"github.com/alecthomas/chroma/lexers/internal"
)
// Zig lexer.
var Zig = internal.Register(MustNewLexer(
&Config{
Name: "Zig",
Aliases: []string{"zig"},
Filenames: []string{"*.zig"},
MimeTypes: []string{"text/zig"},
},
Rules{
"root": {
{`\n`, TextWhitespace, nil},
{`\s+`, TextWhitespace, nil},
{`//.*?\n`, CommentSingle, nil},
{Words(``, `\b`, `break`, `return`, `continue`, `asm`, `defer`, `errdefer`, `unreachable`, `try`, `catch`, `async`, `await`, `suspend`, `resume`, `cancel`), Keyword, nil},
{Words(``, `\b`, `const`, `var`, `extern`, `packed`, `export`, `pub`, `noalias`, `inline`, `comptime`, `nakedcc`, `stdcallcc`, `volatile`, `allowzero`, `align`, `linksection`, `threadlocal`), KeywordReserved, nil},
{Words(``, `\b`, `struct`, `enum`, `union`, `error`), Keyword, nil},
{Words(``, `\b`, `while`, `for`), Keyword, nil},
{Words(``, `\b`, `bool`, `f16`, `f32`, `f64`, `f128`, `void`, `noreturn`, `type`, `anyerror`, `promise`, `i0`, `u0`, `isize`, `usize`, `comptime_int`, `comptime_float`, `c_short`, `c_ushort`, `c_int`, `c_uint`, `c_long`, `c_ulong`, `c_longlong`, `c_ulonglong`, `c_longdouble`, `c_voidi8`, `u8`, `i16`, `u16`, `i32`, `u32`, `i64`, `u64`, `i128`, `u128`), KeywordType, nil},
{Words(``, `\b`, `true`, `false`, `null`, `undefined`), KeywordConstant, nil},
{Words(``, `\b`, `if`, `else`, `switch`, `and`, `or`, `orelse`), Keyword, nil},
{Words(``, `\b`, `fn`, `usingnamespace`, `test`), Keyword, nil},
{`0x[0-9a-fA-F]+\.[0-9a-fA-F]+([pP][\-+]?[0-9a-fA-F]+)?`, LiteralNumberFloat, nil},
{`0x[0-9a-fA-F]+\.?[pP][\-+]?[0-9a-fA-F]+`, LiteralNumberFloat, nil},
{`[0-9]+\.[0-9]+([eE][-+]?[0-9]+)?`, LiteralNumberFloat, nil},
{`[0-9]+\.?[eE][-+]?[0-9]+`, LiteralNumberFloat, nil},
{`0b[01]+`, LiteralNumberBin, nil},
{`0o[0-7]+`, LiteralNumberOct, nil},
{`0x[0-9a-fA-F]+`, LiteralNumberHex, nil},
{`[0-9]+`, LiteralNumberInteger, nil},
{`@[a-zA-Z_]\w*`, NameBuiltin, nil},
{`[a-zA-Z_]\w*`, Name, nil},
{`\'\\\'\'`, LiteralStringEscape, nil},
{`\'\\(|x[a-fA-F0-9]{2}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{6}|[nr\\t\'"])\'`, LiteralStringEscape, nil},
{`\'[^\\\']\'`, LiteralString, nil},
{`\\\\[^\n]*`, LiteralStringHeredoc, nil},
{`c\\\\[^\n]*`, LiteralStringHeredoc, nil},
{`c?"`, LiteralString, Push("string")},
{`[+%=><|^!?/\-*&~:]`, Operator, nil},
{`[{}()\[\],.;]`, Punctuation, nil},
},
"string": {
{`\\(x[a-fA-F0-9]{2}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{6}|[nr\\t\'"])`, LiteralStringEscape, nil},
{`[^\\"\n]+`, LiteralString, nil},
{`"`, LiteralString, Pop(1)},
},
},
))