From ec2ba48433edd67558af250beb124338cf32af02 Mon Sep 17 00:00:00 2001 From: jpienaar Date: Sun, 1 Dec 2019 20:15:03 -0800 Subject: [PATCH] Add lexer for LLVM TableGen (#311) --- lexers/t/tablegen.go | 42 +++++++++++++ lexers/testdata/tablegen.actual | 29 +++++++++ lexers/testdata/tablegen.expected | 100 ++++++++++++++++++++++++++++++ 3 files changed, 171 insertions(+) create mode 100644 lexers/t/tablegen.go create mode 100644 lexers/testdata/tablegen.actual create mode 100644 lexers/testdata/tablegen.expected diff --git a/lexers/t/tablegen.go b/lexers/t/tablegen.go new file mode 100644 index 0000000..18c6978 --- /dev/null +++ b/lexers/t/tablegen.go @@ -0,0 +1,42 @@ +package t + +import ( + . "github.com/alecthomas/chroma" // nolint + "github.com/alecthomas/chroma/lexers/internal" +) + +// TableGen lexer. +var Tablegen = internal.Register(MustNewLexer( + &Config{ + Name: "TableGen", + Aliases: []string{"tablegen"}, + Filenames: []string{"*.td"}, + MimeTypes: []string{"text/x-tablegen"}, + }, + Rules{ + "root": { + Include("macro"), + Include("whitespace"), + {`c?"[^"]*?"`, LiteralString, nil}, + Include("keyword"), + {`\$[_a-zA-Z][_\w]*`, NameVariable, nil}, + {`\d*[_a-zA-Z][_\w]*`, NameVariable, nil}, + {`\[\{[\w\W]*?\}\]`, LiteralString, nil}, + {`[+-]?\d+|0x[\da-fA-F]+|0b[01]+`, LiteralNumber, nil}, + {`[=<>{}\[\]()*.,!:;]`, Punctuation, nil}, + }, + "macro": { + {`(#include\s+)("[^"]*")`, ByGroups(CommentPreproc, LiteralString), nil}, + {`^\s*#(ifdef|ifndef)\s+[_\w][_\w\d]*`, CommentPreproc, nil}, + {`^\s*#define\s+[_\w][_\w\d]*`, CommentPreproc, nil}, + {`^\s*#endif`, CommentPreproc, nil}, + }, + "whitespace": { + {`(\n|\s)+`, Text, nil}, + {`//.*?\n`, Comment, nil}, + }, + "keyword": { + {Words(``, `\b`, `bit`, `bits`, `class`, `code`, `dag`, `def`, `defm`, `field`, `foreach`, `in`, `int`, `let`, `list`, `multiclass`, `string`), Keyword, nil}, + }, + }, +)) diff --git a/lexers/testdata/tablegen.actual b/lexers/testdata/tablegen.actual new file mode 100644 index 0000000..bf563df --- /dev/null +++ b/lexers/testdata/tablegen.actual @@ -0,0 +1,29 @@ +#include "main" + +#ifndef SHOULD_DEF +#define SHOULD_DEF +def ConstantOp : Toy_Op<"constant", [NoSideEffect]> { + // Provide a summary and description for this operation. This can be used to + // auto-generate documenatation of the operations within our dialect. + let summary = "constant operation"; + let description = [{ + Constant operation turns a literal into an SSA value. The data is attached + to the operation as an attribute. For example: + + %0 = "toy.constant"() + { value = dense<[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]> : tensor<2x3xf64> } + : () -> tensor<2x3xf64> + }]; + + // The constant operation takes an attribute as the only input. + // `F64ElementsAttr` corresponds to a 64-bit floating-point ElementsAttr. + let arguments = (ins F64ElementsAttr:$value); + + // The generic call operation returns a single value of TensorType. + // F64Tensor corresponds to a 64-bit floating-point TensorType. + let results = (outs F64Tensor<10>); + + // Add additional verification logic to the constant operation. + let verifier = [{ return ::verify(*this); }]; +} +#endif // SHOULD_DEF diff --git a/lexers/testdata/tablegen.expected b/lexers/testdata/tablegen.expected new file mode 100644 index 0000000..1b6e797 --- /dev/null +++ b/lexers/testdata/tablegen.expected @@ -0,0 +1,100 @@ +[ + {"type":"CommentPreproc","value":"#include "}, + {"type":"LiteralString","value":"\"main\""}, + {"type":"Text","value":"\n\n"}, + {"type":"CommentPreproc","value":"#ifndef SHOULD_DEF"}, + {"type":"Text","value":"\n"}, + {"type":"CommentPreproc","value":"#define SHOULD_DEF"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"def"}, + {"type":"Text","value":" "}, + {"type":"NameVariable","value":"ConstantOp"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":":"}, + {"type":"Text","value":" "}, + {"type":"NameVariable","value":"Toy_Op"}, + {"type":"Punctuation","value":"\u003c"}, + {"type":"LiteralString","value":"\"constant\""}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"["}, + {"type":"NameVariable","value":"NoSideEffect"}, + {"type":"Punctuation","value":"]\u003e"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"{"}, + {"type":"Text","value":"\n "}, + {"type":"Comment","value":"// Provide a summary and description for this operation. This can be used to\n"}, + {"type":"Text","value":" "}, + {"type":"Comment","value":"// auto-generate documenatation of the operations within our dialect.\n"}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"let"}, + {"type":"Text","value":" "}, + {"type":"NameVariable","value":"summary"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"="}, + {"type":"Text","value":" "}, + {"type":"LiteralString","value":"\"constant operation\""}, + {"type":"Punctuation","value":";"}, + {"type":"Text","value":"\n "}, + {"type":"Keyword","value":"let"}, + {"type":"Text","value":" "}, + {"type":"NameVariable","value":"description"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"="}, + {"type":"Text","value":" "}, + {"type":"LiteralString","value":"[{\n Constant operation turns a literal into an SSA value. The data is attached\n to the operation as an attribute. For example:\n\n %0 = \"toy.constant\"()\n { value = dense\u003c[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]\u003e : tensor\u003c2x3xf64\u003e }\n : () -\u003e tensor\u003c2x3xf64\u003e\n }]"}, + {"type":"Punctuation","value":";"}, + {"type":"Text","value":"\n\n "}, + {"type":"Comment","value":"// The constant operation takes an attribute as the only input.\n"}, + {"type":"Text","value":" "}, + {"type":"Comment","value":"// `F64ElementsAttr` corresponds to a 64-bit floating-point ElementsAttr.\n"}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"let"}, + {"type":"Text","value":" "}, + {"type":"NameVariable","value":"arguments"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"="}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"("}, + {"type":"NameVariable","value":"ins"}, + {"type":"Text","value":" "}, + {"type":"NameVariable","value":"F64ElementsAttr"}, + {"type":"Punctuation","value":":"}, + {"type":"NameVariable","value":"$value"}, + {"type":"Punctuation","value":");"}, + {"type":"Text","value":"\n\n "}, + {"type":"Comment","value":"// The generic call operation returns a single value of TensorType.\n"}, + {"type":"Text","value":" "}, + {"type":"Comment","value":"// F64Tensor corresponds to a 64-bit floating-point TensorType.\n"}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"let"}, + {"type":"Text","value":" "}, + {"type":"NameVariable","value":"results"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"="}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"("}, + {"type":"NameVariable","value":"outs"}, + {"type":"Text","value":" "}, + {"type":"NameVariable","value":"F64Tensor"}, + {"type":"Punctuation","value":"\u003c"}, + {"type":"LiteralNumber","value":"10"}, + {"type":"Punctuation","value":"\u003e);"}, + {"type":"Text","value":"\n\n "}, + {"type":"Comment","value":"// Add additional verification logic to the constant operation.\n"}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"let"}, + {"type":"Text","value":" "}, + {"type":"NameVariable","value":"verifier"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"="}, + {"type":"Text","value":" "}, + {"type":"LiteralString","value":"[{ return ::verify(*this); }]"}, + {"type":"Punctuation","value":";"}, + {"type":"Text","value":"\n"}, + {"type":"Punctuation","value":"}"}, + {"type":"Text","value":"\n"}, + {"type":"CommentPreproc","value":"#endif"}, + {"type":"Text","value":" "}, + {"type":"Comment","value":"// SHOULD_DEF\n"} +]