diff --git a/lexers/lexers_test.go b/lexers/lexers_test.go index f7f16dd..840d67e 100644 --- a/lexers/lexers_test.go +++ b/lexers/lexers_test.go @@ -9,7 +9,7 @@ import ( "strings" "testing" - assert "github.com/alecthomas/assert/v2" + "github.com/alecthomas/assert/v2" "github.com/alecthomas/repr" "github.com/alecthomas/chroma/v2" @@ -73,11 +73,16 @@ func FileTest(t *testing.T, lexer chroma.Lexer, actualFilename, expectedFilename actual, err := chroma.Tokenise(lexer, nil, string(actualText)) assert.NoError(t, err) - if os.Getenv("RECORD") == "true" { + if os.Getenv("RECORD") != "" { // Update the expected file with the generated output of this lexer f, err := os.Create(expectedFilename) defer f.Close() // nolint: gosec assert.NoError(t, err) + for _, token := range actual { + if token.Type == chroma.Error { + t.Logf("Found Error token in lexer %s output: %s", lexer.Config().Name, repr.String(token)) + } + } assert.NoError(t, formatters.JSON.Format(f, nil, chroma.Literator(actual...))) } else { // Read expected JSON into token slice. @@ -87,10 +92,14 @@ func FileTest(t *testing.T, lexer chroma.Lexer, actualFilename, expectedFilename err = json.NewDecoder(r).Decode(&expected) assert.NoError(t, err) - // Equal? - assert.Equal(t, - repr.String(expected, repr.Indent(" ")), - repr.String(actual, repr.Indent(" "))) + assert.Equal(t, expected, actual) + + // Check for error tokens. + for _, token := range actual { + if token.Type == chroma.Error { + t.Logf("Found Error token in lexer %s output: %s", lexer.Config().Name, repr.String(token)) + } + } } }) } diff --git a/lexers/testdata/hlsl.expected b/lexers/testdata/hlsl.expected index dcd0491..5a296ea 100644 --- a/lexers/testdata/hlsl.expected +++ b/lexers/testdata/hlsl.expected @@ -1,40 +1,40 @@ -[ - {"type":"KeywordType","value":"float4"}, - {"type":"Text","value":" "}, - {"type":"Name","value":"vertex"}, - {"type":"Punctuation","value":"("}, - {"type":"KeywordType","value":"float3"}, - {"type":"Text","value":" "}, - {"type":"Name","value":"position"}, - {"type":"Punctuation","value":")"}, - {"type":"Text","value":" "}, - {"type":"Punctuation","value":"{"}, - {"type":"Text","value":"\n "}, - {"type":"Keyword","value":"return"}, - {"type":"Text","value":" "}, - {"type":"NameBuiltin","value":"mul"}, - {"type":"Punctuation","value":"("}, - {"type":"Name","value":"projection"}, - {"type":"Punctuation","value":","}, - {"type":"Text","value":" "}, - {"type":"NameBuiltin","value":"mul"}, - {"type":"Punctuation","value":"("}, - {"type":"Name","value":"view"}, - {"type":"Punctuation","value":","}, - {"type":"Text","value":" "}, - {"type":"NameBuiltin","value":"mul"}, - {"type":"Punctuation","value":"("}, - {"type":"Name","value":"model"}, - {"type":"Punctuation","value":","}, - {"type":"Text","value":" "}, - {"type":"KeywordType","value":"float4"}, - {"type":"Punctuation","value":"("}, - {"type":"Name","value":"position"}, - {"type":"Punctuation","value":","}, - {"type":"Text","value":" "}, - {"type":"LiteralNumberFloat","value":"1.0"}, - {"type":"Punctuation","value":"))));"}, - {"type":"Text","value":"\n"}, - {"type":"Punctuation","value":"}"}, - {"type":"Text","value":"\n"} -] \ No newline at end of file +[ + {"type":"KeywordType","value":"float4"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"vertex"}, + {"type":"Punctuation","value":"("}, + {"type":"KeywordType","value":"float3"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"position"}, + {"type":"Punctuation","value":")"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"{"}, + {"type":"Text","value":"\n "}, + {"type":"Keyword","value":"return"}, + {"type":"Text","value":" "}, + {"type":"NameBuiltin","value":"mul"}, + {"type":"Punctuation","value":"("}, + {"type":"Name","value":"projection"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":" "}, + {"type":"NameBuiltin","value":"mul"}, + {"type":"Punctuation","value":"("}, + {"type":"Name","value":"view"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":" "}, + {"type":"NameBuiltin","value":"mul"}, + {"type":"Punctuation","value":"("}, + {"type":"Name","value":"model"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":" "}, + {"type":"KeywordType","value":"float4"}, + {"type":"Punctuation","value":"("}, + {"type":"Name","value":"position"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":" "}, + {"type":"LiteralNumberFloat","value":"1.0"}, + {"type":"Punctuation","value":"))));"}, + {"type":"Text","value":"\n"}, + {"type":"Punctuation","value":"}"}, + {"type":"Text","value":"\n"} +] diff --git a/lexers/testdata/natural.expected b/lexers/testdata/natural.expected index 7f1bc80..0e88688 100644 --- a/lexers/testdata/natural.expected +++ b/lexers/testdata/natural.expected @@ -16,9 +16,9 @@ {"type":"Other","value":"(A10)"}, {"type":"Text","value":" "}, {"type":"Keyword","value":"INIT"}, - {"type":"Operator","value":"<"}, + {"type":"Operator","value":"\u003c"}, {"type":"LiteralStringSingle","value":"'Hello'"}, - {"type":"Operator","value":">"}, + {"type":"Operator","value":"\u003e"}, {"type":"Text","value":"\n"}, {"type":"LiteralNumberInteger","value":"1"}, {"type":"Text","value":" "}, @@ -48,14 +48,12 @@ {"type":"Text","value":" "}, {"type":"NameVariable","value":"#VAR-NAME-1"}, {"type":"Text","value":"\n\n"}, - {"type":"Keyword","value":"DEFINE"}, {"type":"Text","value":" "}, {"type":"Keyword","value":"SUBROUTINE"}, {"type":"Text","value":" "}, {"type":"NameFunction","value":"MY-SUBROUTINE"}, {"type":"Text","value":"\n"}, - {"type":"Keyword","value":"FOR"}, {"type":"Text","value":" "}, {"type":"NameVariable","value":"#I"}, @@ -68,17 +66,13 @@ {"type":"Text","value":" "}, {"type":"LiteralNumberInteger","value":"10"}, {"type":"Text","value":"\n"}, - {"type":"Keyword","value":"ESCAPE"}, {"type":"Text","value":" "}, {"type":"Keyword","value":"MODULE"}, {"type":"Text","value":"\n"}, - {"type":"Keyword","value":"END-FOR"}, {"type":"Text","value":"\n"}, - {"type":"Keyword","value":"END-SUBROUTINE"}, - {"type":"Text","value":"\n\n"}, {"type":"Keyword","value":"END"}, {"type":"Text","value":"\n\n"} diff --git a/lexers/testdata/vhs.expected b/lexers/testdata/vhs.expected index 3c305ef..b91cf58 100644 --- a/lexers/testdata/vhs.expected +++ b/lexers/testdata/vhs.expected @@ -1,54 +1,54 @@ [ - {"type":"Comment","value":"# Where should we write the GIF?"}, - {"type":"TextWhitespace","value":"\n"}, - {"type":"Keyword","value":"Output"}, - {"type":"TextWhitespace","value":" "}, - {"type":"LiteralString","value":"demo.gif"}, - {"type":"TextWhitespace","value":"\n\n"}, - {"type":"Comment","value":"# Set up a 1200x600 terminal with 46px font."}, - {"type":"TextWhitespace","value":"\n"}, - {"type":"Keyword","value":"Set"}, - {"type":"TextWhitespace","value":" "}, - {"type":"NameBuiltin","value":"FontSize"}, - {"type":"TextWhitespace","value":" "}, - {"type":"LiteralNumber","value":"46"}, - {"type":"TextWhitespace","value":"\n"}, - {"type":"Keyword","value":"Set"}, - {"type":"TextWhitespace","value":" "}, - {"type":"NameBuiltin","value":"Width"}, - {"type":"TextWhitespace","value":" "}, - {"type":"LiteralNumber","value":"1200"}, - {"type":"TextWhitespace","value":"\n"}, - {"type":"Keyword","value":"Set"}, - {"type":"TextWhitespace","value":" "}, - {"type":"NameBuiltin","value":"Height"}, - {"type":"TextWhitespace","value":" "}, - {"type":"LiteralNumber","value":"600"}, - {"type":"TextWhitespace","value":"\n\n"}, - {"type":"Comment","value":"# Type a command in the terminal."}, - {"type":"TextWhitespace","value":"\n"}, - {"type":"Keyword","value":"Type"}, - {"type":"Punctuation", "value": "@"}, - {"type":"LiteralNumber", "value": "100"}, - {"type":"Text", "value": "ms"}, - {"type":"TextWhitespace","value":" "}, - {"type":"LiteralStringDouble","value":"\"echo 'Welcome to VHS!'\""}, - {"type":"TextWhitespace","value":"\n\n"}, - {"type":"Comment","value":"# Pause for dramatic effect..."}, - {"type":"TextWhitespace","value":"\n"}, - {"type":"Keyword","value":"Sleep"}, - {"type":"TextWhitespace","value":" "}, - {"type":"LiteralNumber","value":"500"}, - {"type":"Text","value":"ms"}, - {"type":"TextWhitespace","value":"\n\n"}, - {"type":"Comment","value":"# Run the command by pressing enter."}, - {"type":"TextWhitespace","value":"\n"}, - {"type":"Keyword","value":"Enter"}, - {"type":"TextWhitespace","value":"\n\n"}, - {"type":"Comment","value":"# Admire the output for a bit."}, - {"type":"TextWhitespace","value":"\n"}, - {"type":"Keyword","value":"Sleep"}, - {"type":"TextWhitespace","value":" "}, - {"type":"LiteralNumber","value":"5"}, - {"type":"Text","value":"s"} + {"type":"Comment","value":"# Where should we write the GIF?"}, + {"type":"TextWhitespace","value":"\n"}, + {"type":"Keyword","value":"Output"}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralString","value":"demo.gif"}, + {"type":"TextWhitespace","value":"\n\n"}, + {"type":"Comment","value":"# Set up a 1200x600 terminal with 46px font."}, + {"type":"TextWhitespace","value":"\n"}, + {"type":"Keyword","value":"Set"}, + {"type":"TextWhitespace","value":" "}, + {"type":"NameBuiltin","value":"FontSize"}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralNumber","value":"46"}, + {"type":"TextWhitespace","value":"\n"}, + {"type":"Keyword","value":"Set"}, + {"type":"TextWhitespace","value":" "}, + {"type":"NameBuiltin","value":"Width"}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralNumber","value":"1200"}, + {"type":"TextWhitespace","value":"\n"}, + {"type":"Keyword","value":"Set"}, + {"type":"TextWhitespace","value":" "}, + {"type":"NameBuiltin","value":"Height"}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralNumber","value":"600"}, + {"type":"TextWhitespace","value":"\n\n"}, + {"type":"Comment","value":"# Type a command in the terminal."}, + {"type":"TextWhitespace","value":"\n"}, + {"type":"Keyword","value":"Type"}, + {"type":"Punctuation","value":"@"}, + {"type":"LiteralNumber","value":"100"}, + {"type":"Text","value":"ms"}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralStringDouble","value":"\"echo 'Welcome to VHS!'\""}, + {"type":"TextWhitespace","value":"\n\n"}, + {"type":"Comment","value":"# Pause for dramatic effect..."}, + {"type":"TextWhitespace","value":"\n"}, + {"type":"Keyword","value":"Sleep"}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralNumber","value":"500"}, + {"type":"Text","value":"ms"}, + {"type":"TextWhitespace","value":"\n\n"}, + {"type":"Comment","value":"# Run the command by pressing enter."}, + {"type":"TextWhitespace","value":"\n"}, + {"type":"Keyword","value":"Enter"}, + {"type":"TextWhitespace","value":"\n\n"}, + {"type":"Comment","value":"# Admire the output for a bit."}, + {"type":"TextWhitespace","value":"\n"}, + {"type":"Keyword","value":"Sleep"}, + {"type":"TextWhitespace","value":" "}, + {"type":"LiteralNumber","value":"5"}, + {"type":"Text","value":"s"} ] diff --git a/lexers/testdata/vim.actual b/lexers/testdata/vim.actual new file mode 100644 index 0000000..eb7d152 --- /dev/null +++ b/lexers/testdata/vim.actual @@ -0,0 +1,88 @@ +execute pathogen#infect() + +syntax on +colorscheme swapoff + +" Turn off status line. +set laststatus=0 + +" Turn off annoying beeps +set vb +set noeb +set vb t_vb= + +set mouse=r +set visualbell + +" Force latin1 terminal encoding? +set fileencoding=utf-8 +set encoding=utf-8 +set termencoding=utf-8 + +" Allows ~ +set tildeop + +" Stupid backspace +set backspace=eol,indent,start + +" 99% of the time, wrap is shit +set nowrap + +" Map U to 'redo', which I like better than U being 'undo changes to this line' +map U + +" Indentation defaults +set ts=2 sts=2 sw=2 et + +" Abbreviate status messages +set shortmess=aTI + +" No 'Press ENTER...' message when using man +map K K + +" Filetypes that need real tabs +autocmd FileType make :set ts=4 noet nolist + +" Set some formatoptions +set formatoptions=tcrqn + +" Incremental search is good +set incsearch +set hlsearch +hi MatchParen cterm=NONE ctermfg=LightGreen ctermbg=NONE + +" Disable highlighted search on redraw +map :nohlsearch:redraw! + +autocmd Syntax * syntax sync fromstart + +filetype plugin on +filetype indent on + +" Always show cursor position +set ruler + +" Fucking visual-select copying to clipboard! WTF! +set clipboard= + +" C-d when file-completing +set wildmenu + +highlight SpellBad cterm=NONE ctermfg=darkred ctermbg=none + +" Show trailing whitepace and spaces before a tab: +highlight ExtraWhitespace cterm=NONE ctermbg=red guibg=red +autocmd Syntax * syn match ExtraWhitespace /\s\+$\| \+\ze\t/ + +" Round shifting to nearest shiftwidth +set shiftround + +" Insert blanks according to shiftwidth +set smarttab + +" Disable toolbar +set go-=T + +set wildignore=*.o,*.a,*.so,*.pyc,*~,*.class,build/*,build-*,tags,cscope*,third_party/*,java/*,users/*,data/*,3rdparty/*,*.jar,target/*,dist/* + +autocmd BufRead,BufNewFile *.bk set filetype=python diff --git a/lexers/testdata/vim.expected b/lexers/testdata/vim.expected new file mode 100644 index 0000000..78b9255 --- /dev/null +++ b/lexers/testdata/vim.expected @@ -0,0 +1,384 @@ +[ + {"type":"Keyword","value":"execute"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"pathogen"}, + {"type":"Text","value":"#"}, + {"type":"NameOther","value":"infect"}, + {"type":"Punctuation","value":"()"}, + {"type":"Text","value":"\n\n"}, + {"type":"Keyword","value":"syntax"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"on"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"colorscheme"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"swapoff"}, + {"type":"Text","value":"\n"}, + {"type":"Comment","value":"\n\" Turn off status line."}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"laststatus"}, + {"type":"Punctuation","value":"="}, + {"type":"LiteralNumber","value":"0"}, + {"type":"Text","value":"\n"}, + {"type":"Comment","value":"\n\" Turn off annoying beeps"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"vb"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"noeb"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"vb"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"t_vb"}, + {"type":"Punctuation","value":"="}, + {"type":"Text","value":"\n\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"mouse"}, + {"type":"Punctuation","value":"="}, + {"type":"NameOther","value":"r"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"visualbell"}, + {"type":"Text","value":"\n"}, + {"type":"Comment","value":"\n\" Force latin1 terminal encoding?"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"fileencoding"}, + {"type":"Punctuation","value":"="}, + {"type":"NameOther","value":"utf"}, + {"type":"LiteralNumber","value":"-8"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"encoding"}, + {"type":"Punctuation","value":"="}, + {"type":"NameOther","value":"utf"}, + {"type":"LiteralNumber","value":"-8"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"termencoding"}, + {"type":"Punctuation","value":"="}, + {"type":"NameOther","value":"utf"}, + {"type":"LiteralNumber","value":"-8"}, + {"type":"Text","value":"\n"}, + {"type":"Comment","value":"\n\" Allows ~\u003cmotion\u003e"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"tildeop"}, + {"type":"Text","value":"\n"}, + {"type":"Comment","value":"\n\" Stupid backspace"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"backspace"}, + {"type":"Punctuation","value":"="}, + {"type":"NameOther","value":"eol"}, + {"type":"Punctuation","value":","}, + {"type":"NameOther","value":"indent"}, + {"type":"Punctuation","value":","}, + {"type":"NameOther","value":"start"}, + {"type":"Text","value":"\n"}, + {"type":"Comment","value":"\n\" 99% of the time, wrap is shit"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"nowrap"}, + {"type":"Text","value":"\n"}, + {"type":"Comment","value":"\n\" Map U to 'redo', which I like better than U being 'undo changes to this line'"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"map"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"U"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"\u003c"}, + {"type":"NameOther","value":"C"}, + {"type":"Punctuation","value":"-"}, + {"type":"NameOther","value":"R"}, + {"type":"Punctuation","value":"\u003e"}, + {"type":"Text","value":"\n"}, + {"type":"Comment","value":"\n\" Indentation defaults"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"ts"}, + {"type":"Punctuation","value":"="}, + {"type":"LiteralNumber","value":"2"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"sts"}, + {"type":"Punctuation","value":"="}, + {"type":"LiteralNumber","value":"2"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"sw"}, + {"type":"Punctuation","value":"="}, + {"type":"LiteralNumber","value":"2"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"et"}, + {"type":"Text","value":"\n"}, + {"type":"Comment","value":"\n\" Abbreviate status messages"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"shortmess"}, + {"type":"Punctuation","value":"="}, + {"type":"NameOther","value":"aTI"}, + {"type":"Text","value":"\n"}, + {"type":"Comment","value":"\n\" No 'Press ENTER...' message when using man"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"map"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"K"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"K"}, + {"type":"Punctuation","value":"\u003c"}, + {"type":"NameOther","value":"CR"}, + {"type":"Punctuation","value":"\u003e"}, + {"type":"Text","value":"\n"}, + {"type":"Comment","value":"\n\" Filetypes that need real tabs"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"autocmd"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"FileType"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"make"}, + {"type":"Text","value":" :"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"ts"}, + {"type":"Punctuation","value":"="}, + {"type":"LiteralNumber","value":"4"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"noet"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"nolist"}, + {"type":"Text","value":"\n"}, + {"type":"Comment","value":"\n\" Set some formatoptions"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"formatoptions"}, + {"type":"Punctuation","value":"="}, + {"type":"NameOther","value":"tcrqn"}, + {"type":"Text","value":"\n"}, + {"type":"Comment","value":"\n\" Incremental search is good"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"incsearch"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"hlsearch"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"hi"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"MatchParen"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"cterm"}, + {"type":"Punctuation","value":"="}, + {"type":"NameBuiltin","value":"NONE"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"ctermfg"}, + {"type":"Punctuation","value":"="}, + {"type":"NameOther","value":"LightGreen"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"ctermbg"}, + {"type":"Punctuation","value":"="}, + {"type":"NameBuiltin","value":"NONE"}, + {"type":"Text","value":"\n"}, + {"type":"Comment","value":"\n\" Disable highlighted search on redraw"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"map"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"\u003c"}, + {"type":"NameOther","value":"silent"}, + {"type":"Punctuation","value":"\u003e"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"\u003c"}, + {"type":"NameOther","value":"C"}, + {"type":"Punctuation","value":"-"}, + {"type":"NameOther","value":"l"}, + {"type":"Punctuation","value":"\u003e"}, + {"type":"Text","value":" :"}, + {"type":"NameOther","value":"nohlsearch"}, + {"type":"Punctuation","value":"\u003c"}, + {"type":"NameOther","value":"CR"}, + {"type":"Punctuation","value":"\u003e"}, + {"type":"Text","value":":"}, + {"type":"NameOther","value":"redraw"}, + {"type":"Punctuation","value":"!\u003c"}, + {"type":"NameOther","value":"CR"}, + {"type":"Punctuation","value":"\u003e"}, + {"type":"Text","value":"\n\n"}, + {"type":"Keyword","value":"autocmd"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"Syntax"}, + {"type":"Text","value":" * "}, + {"type":"Keyword","value":"syntax"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"sync"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"fromstart"}, + {"type":"Text","value":"\n\n"}, + {"type":"Keyword","value":"filetype"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"plugin"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"on"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"filetype"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"indent"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"on"}, + {"type":"Text","value":"\n"}, + {"type":"Comment","value":"\n\" Always show cursor position"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"ruler"}, + {"type":"Text","value":"\n"}, + {"type":"Comment","value":"\n\" Fucking visual-select copying to clipboard! WTF!"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"clipboard"}, + {"type":"Punctuation","value":"="}, + {"type":"Text","value":"\n"}, + {"type":"Comment","value":"\n\" C-d when file-completing"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"wildmenu"}, + {"type":"Text","value":"\n\n"}, + {"type":"Keyword","value":"highlight"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"SpellBad"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"cterm"}, + {"type":"Punctuation","value":"="}, + {"type":"NameBuiltin","value":"NONE"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"ctermfg"}, + {"type":"Punctuation","value":"="}, + {"type":"NameOther","value":"darkred"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"ctermbg"}, + {"type":"Punctuation","value":"="}, + {"type":"NameOther","value":"none"}, + {"type":"Text","value":"\n"}, + {"type":"Comment","value":"\n\" Show trailing whitepace and spaces before a tab:"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"highlight"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"ExtraWhitespace"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"cterm"}, + {"type":"Punctuation","value":"="}, + {"type":"NameBuiltin","value":"NONE"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"ctermbg"}, + {"type":"Punctuation","value":"="}, + {"type":"NameOther","value":"red"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"guibg"}, + {"type":"Punctuation","value":"="}, + {"type":"NameOther","value":"red"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"autocmd"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"Syntax"}, + {"type":"Text","value":" * "}, + {"type":"NameOther","value":"syn"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"match"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"ExtraWhitespace"}, + {"type":"Text","value":" "}, + {"type":"LiteralStringRegex","value":"/\\s\\+$\\| \\+\\ze\\t/"}, + {"type":"Text","value":"\n"}, + {"type":"Comment","value":"\n\" Round shifting to nearest shiftwidth"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"shiftround"}, + {"type":"Text","value":"\n"}, + {"type":"Comment","value":"\n\" Insert blanks according to shiftwidth"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"smarttab"}, + {"type":"Text","value":"\n"}, + {"type":"Comment","value":"\n\" Disable toolbar"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"go"}, + {"type":"Punctuation","value":"-="}, + {"type":"NameOther","value":"T"}, + {"type":"Text","value":"\n\n"}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"wildignore"}, + {"type":"Punctuation","value":"="}, + {"type":"Text","value":"*."}, + {"type":"NameOther","value":"o"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":"*."}, + {"type":"NameOther","value":"a"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":"*."}, + {"type":"NameOther","value":"so"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":"*."}, + {"type":"NameOther","value":"pyc"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":"*"}, + {"type":"Punctuation","value":"~,"}, + {"type":"Text","value":"*."}, + {"type":"NameOther","value":"class"}, + {"type":"Punctuation","value":","}, + {"type":"NameOther","value":"build"}, + {"type":"LiteralStringRegex","value":"/*,build-*,tags,cscope*,third_party/"}, + {"type":"Text","value":"*"}, + {"type":"Punctuation","value":","}, + {"type":"NameOther","value":"java"}, + {"type":"LiteralStringRegex","value":"/*,users/"}, + {"type":"Text","value":"*"}, + {"type":"Punctuation","value":","}, + {"type":"NameOther","value":"data"}, + {"type":"LiteralStringRegex","value":"/*,3rdparty/"}, + {"type":"Text","value":"*"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":"*."}, + {"type":"NameOther","value":"jar"}, + {"type":"Punctuation","value":","}, + {"type":"NameOther","value":"target"}, + {"type":"LiteralStringRegex","value":"/*,dist/"}, + {"type":"Text","value":"*\n\n"}, + {"type":"Keyword","value":"autocmd"}, + {"type":"Text","value":" "}, + {"type":"NameOther","value":"BufRead"}, + {"type":"Punctuation","value":","}, + {"type":"NameOther","value":"BufNewFile"}, + {"type":"Text","value":" *."}, + {"type":"NameOther","value":"bk"}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"set"}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"filetype"}, + {"type":"Punctuation","value":"="}, + {"type":"NameOther","value":"python"}, + {"type":"Text","value":"\n"} +] diff --git a/lexers/vim.go b/lexers/vim.go index b0059ea..5113062 100644 --- a/lexers/vim.go +++ b/lexers/vim.go @@ -30,9 +30,10 @@ func vimlRules() Rules { {`#[0-9a-f]{6}`, LiteralNumberHex, nil}, {`^:`, Punctuation, nil}, {`[()<>+=!|,~-]`, Punctuation, nil}, - {`\b(let|if|else|endif|elseif|fun|function|endfunction)\b`, Keyword, nil}, + {`\b(let|if|else|endif|elseif|fun|function|endfunction|set|map|autocmd|filetype|hi(ghlight)?|execute|syntax|colorscheme)\b`, Keyword, nil}, {`\b(NONE|bold|italic|underline|dark|light)\b`, NameBuiltin, nil}, {`\b\w+\b`, NameOther, nil}, + {`\n`, Text, nil}, {`.`, Text, nil}, }, } diff --git a/regexp.go b/regexp.go index 1794662..3e0de6b 100644 --- a/regexp.go +++ b/regexp.go @@ -76,8 +76,8 @@ func (r Rules) Merge(rules Rules) Rules { } // MustNewLexer creates a new Lexer with deferred rules generation or panics. -func MustNewLexer(config *Config, rulesFunc func() Rules) *RegexLexer { - lexer, err := NewLexer(config, rulesFunc) +func MustNewLexer(config *Config, rules func() Rules) *RegexLexer { + lexer, err := NewLexer(config, rules) if err != nil { panic(err) }