From eb8e868cceb487cec9be75446b711ffcbaa4000f Mon Sep 17 00:00:00 2001 From: Alec Thomas Date: Sun, 9 Jun 2019 17:39:46 +1000 Subject: [PATCH] Support for array transpose operator in Julia. Fixes #253. --- lexers/j/julia.go | 9 ++--- lexers/testdata/d.expected | 13 ------- lexers/testdata/julia.actual | 6 ++++ lexers/testdata/julia.expected | 64 ++++++++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 17 deletions(-) create mode 100644 lexers/testdata/julia.actual create mode 100644 lexers/testdata/julia.expected diff --git a/lexers/j/julia.go b/lexers/j/julia.go index 5d186df..ad83fcf 100644 --- a/lexers/j/julia.go +++ b/lexers/j/julia.go @@ -21,12 +21,13 @@ var Julia = internal.Register(MustNewLexer( {`#.*$`, Comment, nil}, {`[\[\]{}(),;]`, Punctuation, nil}, {`in\b`, KeywordPseudo, nil}, + {`isa\b`, KeywordPseudo, nil}, {`(true|false)\b`, KeywordConstant, nil}, {`(local|global|const)\b`, KeywordDeclaration, nil}, - {Words(``, `\b`, `function`, `type`, `typealias`, `abstract`, `immutable`, `mutable`, `struct`, `baremodule`, `begin`, `bitstype`, `break`, `catch`, `ccall`, `continue`, `do`, `else`, `elseif`, `end`, `export`, `finally`, `for`, `if`, `import`, `importall`, `let`, `macro`, `module`, `quote`, `return`, `try`, `using`, `while`), Keyword, nil}, + {Words(``, `\b`, `function`, `type`, `typealias`, `abstract`, `immutable`, `baremodule`, `begin`, `bitstype`, `break`, `catch`, `ccall`, `continue`, `do`, `else`, `elseif`, `end`, `export`, `finally`, `for`, `if`, `import`, `importall`, `let`, `macro`, `module`, `mutable`, `primitive`, `quote`, `return`, `struct`, `try`, `using`, `while`), Keyword, nil}, {Words(``, `\b`, `ANY`, `ASCIIString`, `AbstractArray`, `AbstractChannel`, `AbstractFloat`, `AbstractMatrix`, `AbstractRNG`, `AbstractSparseArray`, `AbstractSparseMatrix`, `AbstractSparseVector`, `AbstractString`, `AbstractVecOrMat`, `AbstractVector`, `Any`, `ArgumentError`, `Array`, `AssertionError`, `Associative`, `Base64DecodePipe`, `Base64EncodePipe`, `Bidiagonal`, `BigFloat`, `BigInt`, `BitArray`, `BitMatrix`, `BitVector`, `Bool`, `BoundsError`, `Box`, `BufferStream`, `CapturedException`, `CartesianIndex`, `CartesianRange`, `Cchar`, `Cdouble`, `Cfloat`, `Channel`, `Char`, `Cint`, `Cintmax_t`, `Clong`, `Clonglong`, `ClusterManager`, `Cmd`, `Coff_t`, `Colon`, `Complex`, `Complex128`, `Complex32`, `Complex64`, `CompositeException`, `Condition`, `Cptrdiff_t`, `Cshort`, `Csize_t`, `Cssize_t`, `Cstring`, `Cuchar`, `Cuint`, `Cuintmax_t`, `Culong`, `Culonglong`, `Cushort`, `Cwchar_t`, `Cwstring`, `DataType`, `Date`, `DateTime`, `DenseArray`, `DenseMatrix`, `DenseVecOrMat`, `DenseVector`, `Diagonal`, `Dict`, `DimensionMismatch`, `Dims`, `DirectIndexString`, `Display`, `DivideError`, `DomainError`, `EOFError`, `EachLine`, `Enum`, `Enumerate`, `ErrorException`, `Exception`, `Expr`, `Factorization`, `FileMonitor`, `FileOffset`, `Filter`, `Float16`, `Float32`, `Float64`, `FloatRange`, `Function`, `GenSym`, `GlobalRef`, `GotoNode`, `HTML`, `Hermitian`, `IO`, `IOBuffer`, `IOStream`, `IPv4`, `IPv6`, `InexactError`, `InitError`, `Int`, `Int128`, `Int16`, `Int32`, `Int64`, `Int8`, `IntSet`, `Integer`, `InterruptException`, `IntrinsicFunction`, `InvalidStateException`, `Irrational`, `KeyError`, `LabelNode`, `LambdaStaticData`, `LinSpace`, `LineNumberNode`, `LoadError`, `LocalProcess`, `LowerTriangular`, `MIME`, `Matrix`, `MersenneTwister`, `Method`, `MethodError`, `MethodTable`, `Module`, `NTuple`, `NewvarNode`, `NullException`, `Nullable`, `Number`, `ObjectIdDict`, `OrdinalRange`, `OutOfMemoryError`, `OverflowError`, `Pair`, `ParseError`, `PartialQuickSort`, `Pipe`, `PollingFileWatcher`, `ProcessExitedException`, `ProcessGroup`, `Ptr`, `QuoteNode`, `RandomDevice`, `Range`, `Rational`, `RawFD`, `ReadOnlyMemoryError`, `Real`, `ReentrantLock`, `Ref`, `Regex`, `RegexMatch`, `RemoteException`, `RemoteRef`, `RepString`, `RevString`, `RopeString`, `RoundingMode`, `SegmentationFault`, `SerializationState`, `Set`, `SharedArray`, `SharedMatrix`, `SharedVector`, `Signed`, `SimpleVector`, `SparseMatrixCSC`, `StackOverflowError`, `StatStruct`, `StepRange`, `StridedArray`, `StridedMatrix`, `StridedVecOrMat`, `StridedVector`, `SubArray`, `SubString`, `SymTridiagonal`, `Symbol`, `SymbolNode`, `Symmetric`, `SystemError`, `TCPSocket`, `Task`, `Text`, `TextDisplay`, `Timer`, `TopNode`, `Tridiagonal`, `Tuple`, `Type`, `TypeConstructor`, `TypeError`, `TypeName`, `TypeVar`, `UDPSocket`, `UInt`, `UInt128`, `UInt16`, `UInt32`, `UInt64`, `UInt8`, `UTF16String`, `UTF32String`, `UTF8String`, `UndefRefError`, `UndefVarError`, `UnicodeError`, `UniformScaling`, `Union`, `UnitRange`, `Unsigned`, `UpperTriangular`, `Val`, `Vararg`, `VecOrMat`, `Vector`, `VersionNumber`, `Void`, `WString`, `WeakKeyDict`, `WeakRef`, `WorkerConfig`, `Zip`), KeywordType, nil}, {Words(``, `\b`, `ARGS`, `CPU_CORES`, `C_NULL`, `DevNull`, `ENDIAN_BOM`, `ENV`, `I`, `Inf`, `Inf16`, `Inf32`, `Inf64`, `InsertionSort`, `JULIA_HOME`, `LOAD_PATH`, `MergeSort`, `NaN`, `NaN16`, `NaN32`, `NaN64`, `OS_NAME`, `QuickSort`, `RoundDown`, `RoundFromZero`, `RoundNearest`, `RoundNearestTiesAway`, `RoundNearestTiesUp`, `RoundToZero`, `RoundUp`, `STDERR`, `STDIN`, `STDOUT`, `VERSION`, `WORD_SIZE`, `catalan`, `e`, `eu`, `eulergamma`, `golden`, `im`, `nothing`, `pi`, `γ`, `π`, `φ`), NameBuiltin, nil}, - {Words(``, ``, `=`, `:=`, `+=`, `-=`, `*=`, `/=`, `//=`, `.//=`, `.*=`, `./=`, `\=`, `.\=`, `^=`, `.^=`, `÷=`, `.÷=`, `%=`, `.%=`, `|=`, `&=`, `$=`, `=>`, `<<=`, `>>=`, `>>>=`, `~`, `.+=`, `.-=`, `?`, `--`, `-->`, `||`, `&&`, `>`, `<`, `>=`, `≥`, `<=`, `≤`, `==`, `===`, `≡`, `!=`, `≠`, `!==`, `≢`, `.>`, `.<`, `.>=`, `.≥`, `.<=`, `.≤`, `.==`, `.!=`, `.≠`, `.=`, `.!`, `<:`, `>:`, `∈`, `∉`, `∋`, `∌`, `⊆`, `⊈`, `⊂`, `⊄`, `⊊`, `|>`, `<|`, `:`, `+`, `-`, `.+`, `.-`, `|`, `∪`, `$`, `<<`, `>>`, `>>>`, `.<<`, `.>>`, `.>>>`, `*`, `/`, `./`, `÷`, `.÷`, `%`, `⋅`, `.%`, `.*`, `\`, `.\`, `&`, `∩`, `//`, `.//`, `^`, `.^`, `::`, `.`, `+`, `-`, `!`, `~`, `√`, `∛`, `∜`), Operator, nil}, + {Words(``, ``, `=`, `:=`, `+=`, `-=`, `*=`, `/=`, `//=`, `.//=`, `.*=`, `./=`, `\=`, `.\=`, `^=`, `.^=`, `÷=`, `.÷=`, `%=`, `.%=`, `|=`, `&=`, `$=`, `=>`, `<<=`, `>>=`, `>>>=`, `~`, `.+=`, `.-=`, `?`, `--`, `-->`, `||`, `&&`, `>`, `<`, `>=`, `≥`, `<=`, `≤`, `==`, `===`, `≡`, `!=`, `≠`, `!==`, `≢`, `.>`, `.<`, `.>=`, `.≥`, `.<=`, `.≤`, `.==`, `.!=`, `.≠`, `.=`, `.!`, `<:`, `>:`, `∈`, `∉`, `∋`, `∌`, `⊆`, `⊈`, `⊂`, `⊄`, `⊊`, `|>`, `<|`, `:`, `+`, `-`, `.+`, `.-`, `|`, `∪`, `$`, `<<`, `>>`, `>>>`, `.<<`, `.>>`, `.>>>`, `*`, `/`, `./`, `÷`, `.÷`, `%`, `⋅`, `.%`, `.*`, `\`, `.\`, `&`, `∩`, `//`, `.//`, `^`, `.^`, `::`, `.`, `+`, `-`, `!`, `√`, `∛`, `∜`), Operator, nil}, {`'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,3}|\\u[a-fA-F0-9]{1,4}|\\U[a-fA-F0-9]{1,6}|[^\\\'\n])'`, LiteralStringChar, nil}, {`(?<=[.\w)\]])\'+`, Operator, nil}, {`"""`, LiteralString, Push("tqstring")}, @@ -34,8 +35,8 @@ var Julia = internal.Register(MustNewLexer( {`r"""`, LiteralStringRegex, Push("tqregex")}, {`r"`, LiteralStringRegex, Push("regex")}, {"`", LiteralStringBacktick, Push("command")}, - {`(?:[a-zA-Z_¡-￿]|[𐀀-􏿿])(?:[a-zA-Z_0-9¡-￿]|[𐀀-􏿿])*!*`, Name, nil}, - {`@(?:[a-zA-Z_¡-￿]|[𐀀-􏿿])(?:[a-zA-Z_0-9¡-￿]|[𐀀-􏿿])*!*`, NameDecorator, nil}, + {`((?:[a-zA-Z_¡-￿]|[𐀀-􏿿])(?:[a-zA-Z_0-9¡-￿]|[𐀀-􏿿])*!*)(')?`, ByGroups(Name, Operator), nil}, + {`(@(?:[a-zA-Z_¡-￿]|[𐀀-􏿿])(?:[a-zA-Z_0-9¡-￿]|[𐀀-􏿿])*!*)(')?`, ByGroups(NameDecorator, Operator), nil}, {`(\d+(_\d+)+\.\d*|\d*\.\d+(_\d+)+)([eEf][+-]?[0-9]+)?`, LiteralNumberFloat, nil}, {`(\d+\.\d*|\d*\.\d+)([eEf][+-]?[0-9]+)?`, LiteralNumberFloat, nil}, {`\d+(_\d+)+[eEf][+-]?[0-9]+`, LiteralNumberFloat, nil}, diff --git a/lexers/testdata/d.expected b/lexers/testdata/d.expected index 907ce17..2ae1873 100644 --- a/lexers/testdata/d.expected +++ b/lexers/testdata/d.expected @@ -4,34 +4,28 @@ {"type":"NameNamespace","value":"foo"}, {"type":"Operator","value":";"}, {"type":"Text","value":"\n\n"}, - {"type":"KeywordNamespace","value":"import"}, {"type":"Text","value":" "}, {"type":"NameNamespace","value":"std.stdio"}, {"type":"Operator","value":";"}, {"type":"Text","value":"\n\n"}, - {"type":"KeywordDeclaration","value":"class"}, {"type":"Text","value":" "}, {"type":"NameClass","value":"C"}, {"type":"Text","value":" "}, {"type":"Operator","value":"{}"}, {"type":"Text","value":"\n\n"}, - {"type":"CommentMultiline","value":"/* comment1 */"}, {"type":"Text","value":"\n"}, {"type":"CommentMultiline","value":"/+ comment2 +/"}, {"type":"Text","value":"\n"}, - {"type":"NameDecorator","value":"@"}, {"type":"Operator","value":"("}, {"type":"LiteralNumberInteger","value":"1"}, {"type":"Operator","value":")"}, {"type":"Text","value":"\n"}, - {"type":"NameDecorator","value":"@nogc"}, {"type":"Text","value":"\n"}, - {"type":"KeywordType","value":"int"}, {"type":"Text","value":" "}, {"type":"NameFunction","value":"main"}, @@ -39,13 +33,11 @@ {"type":"Text","value":" "}, {"type":"Operator","value":"{"}, {"type":"Text","value":"\n "}, - {"type":"Name","value":"writeln"}, {"type":"Operator","value":"("}, {"type":"NameBuiltin","value":"__FILE__"}, {"type":"Operator","value":");"}, {"type":"Text","value":"\n "}, - {"type":"KeywordDeclaration","value":"auto"}, {"type":"Text","value":" "}, {"type":"Name","value":"s"}, @@ -55,7 +47,6 @@ {"type":"LiteralString","value":"r\"hi\"d"}, {"type":"Operator","value":";"}, {"type":"Text","value":"\n "}, - {"type":"KeywordDeclaration","value":"auto"}, {"type":"Text","value":" "}, {"type":"Name","value":"w"}, @@ -65,7 +56,6 @@ {"type":"LiteralString","value":"`hi`w"}, {"type":"Operator","value":";"}, {"type":"Text","value":"\n "}, - {"type":"KeywordDeclaration","value":"auto"}, {"type":"Text","value":" "}, {"type":"Name","value":"q"}, @@ -85,7 +75,6 @@ {"type":"Text","value":"\n "}, {"type":"Operator","value":"};"}, {"type":"Text","value":"\n "}, - {"type":"KeywordDeclaration","value":"enum"}, {"type":"Text","value":" "}, {"type":"Name","value":"f"}, @@ -95,7 +84,6 @@ {"type":"LiteralNumberFloat","value":"1.2fi"}, {"type":"Operator","value":";"}, {"type":"Text","value":"\n "}, - {"type":"KeywordDeclaration","value":"enum"}, {"type":"Text","value":" "}, {"type":"Name","value":"d"}, @@ -105,7 +93,6 @@ {"type":"LiteralNumberFloat","value":"0x1.FFFFFFFFFFFFFp1023"}, {"type":"Operator","value":";"}, {"type":"Text","value":"\n "}, - {"type":"Keyword","value":"return"}, {"type":"Text","value":" "}, {"type":"LiteralNumberInteger","value":"0"}, diff --git a/lexers/testdata/julia.actual b/lexers/testdata/julia.actual new file mode 100644 index 0000000..cdddca4 --- /dev/null +++ b/lexers/testdata/julia.actual @@ -0,0 +1,6 @@ +% Implementing Feed Forward to calculate H + +m = size(X, 1); +X = [ones(m, 1) X]; +Z2 = Theta1 * X'; +A2 = sigmoid(Z2); \ No newline at end of file diff --git a/lexers/testdata/julia.expected b/lexers/testdata/julia.expected new file mode 100644 index 0000000..c8f2026 --- /dev/null +++ b/lexers/testdata/julia.expected @@ -0,0 +1,64 @@ +[ + {"type":"Operator","value":"%"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"Implementing"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"Feed"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"Forward"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"to"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"calculate"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"H"}, + {"type":"Text","value":"\n\n"}, + {"type":"Name","value":"m"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"="}, + {"type":"Text","value":" "}, + {"type":"Name","value":"size"}, + {"type":"Punctuation","value":"("}, + {"type":"Name","value":"X"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":" "}, + {"type":"LiteralNumberInteger","value":"1"}, + {"type":"Punctuation","value":");"}, + {"type":"Text","value":"\n"}, + {"type":"Name","value":"X"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"="}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"["}, + {"type":"Name","value":"ones"}, + {"type":"Punctuation","value":"("}, + {"type":"Name","value":"m"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":" "}, + {"type":"LiteralNumberInteger","value":"1"}, + {"type":"Punctuation","value":")"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"X"}, + {"type":"Punctuation","value":"];"}, + {"type":"Text","value":"\n"}, + {"type":"Name","value":"Z2"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"="}, + {"type":"Text","value":" "}, + {"type":"Name","value":"Theta1"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"*"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"X"}, + {"type":"Operator","value":"'"}, + {"type":"Punctuation","value":";"}, + {"type":"Text","value":"\n"}, + {"type":"Name","value":"A2"}, + {"type":"Text","value":" "}, + {"type":"Operator","value":"="}, + {"type":"Text","value":" "}, + {"type":"Name","value":"sigmoid"}, + {"type":"Punctuation","value":"("}, + {"type":"Name","value":"Z2"}, + {"type":"Punctuation","value":");"} +]