Update documentation. Include "quick" package.

README.md

@@ -1,7 +1,152 @@
 # Chroma - A general purpose syntax highlighter for Go
 
-Chroma is inspired by [Pygments](http://pygments.org/).
+Chroma is based heavily on [Pygments](http://pygments.org/), and includes
+translaters for Pygments lexers and styles.
 
-## Unsupported Pygments features
+## Using the library
 
-- Autodetection from content.
+Chroma, like Pygments, has the concept of
+[lexers](https://github.com/alecthomas/chroma/tree/master/lexers),
+[formatters](https://github.com/alecthomas/chroma/tree/master/formatters) and
+[styles](https://github.com/alecthomas/chroma/tree/master/styles).
+
+Lexers convert source text into a stream of tokens, styles specify how token
+types are mapped to colours, and formatters convert tokens and styles into
+formatted output.
+
+A package exists for each of these, with a global `Registry` variable
+containing all of the registered implementations. There are also helper
+functions for using the registry in each package, such as looking up lexers by
+name or matching filenames, etc.
+
+In all cases, if a lexer, formatter or style can not be determined, `nil` will
+be returned. In this situation you may want to default to the `Fallback`
+value in each respective package, which provides sane defaults.
+
+## Quick start
+
+A convenience function exists that can be used to simply format some source
+text, without any effort:
+
+```go
+err := quick.Highlight(os.Stdout, someSourceCode, "go", "html", "monokai")
+```
+
+### Identifying the language
+
+To highlight code, you'll first have to identify what language the code is
+written in. There are three primary ways to do that:
+
+1. Detect the language from its filename.
+
+    ```go
+    lexer := lexers.Match("foo.go")
+    ```
+
+3. Explicitly specify the language by its Chroma syntax ID (a full list is available from `lexers.Names()`).
+
+    ```go
+    lexer := lexers.Get("go")
+    ```
+
+3. Detect the language from its content.
+
+    ```go
+    lexer := lexers.Analyse("package main\n\nfunc main()\n{\n}\n")
+    ```
+
+In all cases, `nil` will be returned if the langauge can not be identified.
+
+```go
+if lexer == nil {
+  lexer = lexers.Fallback
+}
+```
+
+At this point, it should be noted that some lexers can be extremely chatty. To
+mitigate this, you can use the coalescing lexer to coalesce runs of identical
+token types into a single token:
+
+```go
+lexer = chroma.Coalesce(lexer)
+```
+
+### Formatting the output
+
+Once a language is identified you will need to pick a formatter and a style (theme).
+
+```go
+style := styles.Get("swapoff")
+if style == nil {
+  style = styles.Fallback
+}
+formatter := formatters.Get("html")
+if formatter == nil {
+  formatter = formatters.Fallback
+}
+```
+
+Then obtain a formatting function from the formatter:
+
+```go
+writer, err := formatter.Format(w, style)
+```
+
+And finally, lex the source code and write the output:
+
+```go
+contents, err := ioutil.ReadAll(r)
+err := lexer.Tokenise(nil, string(contents), writer)
+```
+
+### Lexers
+
+See the [Pygments documentation](http://pygments.org/docs/lexerdevelopment/)
+for details on implementing lexers. Most concepts apply directly to Chroma,
+but see existing lexer implementations for real examples.
+
+In many cases lexers can be automatically converted directly from Pygments by
+using the included Python 3 script `pygments2chroma.py`. I use something like
+the following:
+
+```
+python3 ~/Projects/chroma/_tools/pygments2chroma.py \
+  pygments.lexers.jvm.KotlinLexer \
+  > ~/Projects/chroma/lexers/kotlin.go \
+  && gofmt -s -w ~/Projects/chroma/lexers/*.go
+```
+
+See notes in [pygments-lexers.go](https://github.com/alecthomas/chroma/blob/master/pygments-lexers.txt)
+for a list of lexers, and notes on some of the issues importing them.
+
+### Formatters
+
+Chroma supports HTML output, as well as terminal output in 8 colour, 256 colour, and true-colour.
+
+A `noop` formatter is included that outputs the token text only, and a `raw`
+formatter outputs raw token structs. The latter is useful for debugging lexers.
+
+### Styles
+
+Chroma styles use the [same syntax](http://pygments.org/docs/styles/) as Pygments.
+
+All Pygments styles have been converted to Chroma using the `_tools/style.py` script.
+
+## Command-line interface
+
+A command-line interface to Chroma is included. It can be installed with:
+
+```
+go get -u github.com/alecthomas/chroma/cmd/chroma
+```
+
+## What's missing compared to Pygments?
+
+- Quite a few lexers, for various reasons (pull-requests welcome):
+    - Pygments lexers for complex languages often include custom code to
+      handle certain aspects, such as Perl6's ability to nest code inside
+      regular expressions. These require time and effort to convert.
+    - I mostly only converted languages I had heard of, to reduce the porting cost.
+- Some more esoteric features of Pygments are omitted for simplicity.
+- Though the Chroma API supports content detection, very few languages support them.
+  I have plans to implement a statistical analyser at some point, but not enough time.

cmd/chroma/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bufio"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -26,6 +27,7 @@ var (
 	styleFlag     = kingpin.Flag("style", "Style to use for formatting.").Short('s').Default("swapoff").Enum(styles.Names()...)
 	formatterFlag = kingpin.Flag("formatter", "Formatter to use.").Default("terminal").Short('f').Enum(formatters.Names()...)
 
+	htmlFlag            = kingpin.Flag("html", "Enable HTML mode (equivalent to '--formatter html').").Bool()
 	htmlPrefixFlag      = kingpin.Flag("html-prefix", "HTML CSS class prefix.").PlaceHolder("PREFIX").String()
 	htmlStylesFlag      = kingpin.Flag("html-styles", "Output HTML CSS styles.").Bool()
 	htmlOnlyFlag        = kingpin.Flag("html-only", "Output HTML fragment.").Bool()
@@ -35,6 +37,10 @@ var (
 )
 
 func main() {
+	kingpin.CommandLine.Help = `
+Chroma is a general purpose syntax highlighting library and corresponding
+command, for Go.
+`
 	kingpin.Parse()
 	if *listFlag {
 		listAll()
@@ -53,9 +59,11 @@ func main() {
 		}()
 		defer pprof.StopCPUProfile()
 	}
-	// w := bufio.NewWriterSize(os.Stdout, 16384)
-	w := os.Stdout
-	// defer w.Flush()
+	w := bufio.NewWriterSize(os.Stdout, 16384)
+	defer w.Flush()
+	if *htmlFlag {
+		*formatterFlag = "html"
+	}
 	if *formatterFlag == "html" {
 		options := []html.Option{}
 		if *htmlPrefixFlag != "" {

colour.go

@@ -74,14 +74,6 @@ func (c Colour) Green() uint8 { return uint8(((c - 1) >> 8) & 0xff) }
 // Blue component of colour.
 func (c Colour) Blue() uint8 { return uint8((c - 1) & 0xff) }
 
-// Distance squared between two colours.
-func (c Colour) Distance(other Colour) int {
-	rd := int(c.Red() - other.Red())
-	gd := int(c.Green() - other.Green())
-	bd := int(c.Blue() - other.Blue())
-	return rd*rd + gd*gd + bd*bd
-}
-
 // Colours is an orderable set of colours.
 type Colours []Colour
 

formatters/api.go

@@ -7,11 +7,12 @@ import (
 	"github.com/alecthomas/chroma/formatters/html"
 )
 
-// NoOp formatter.
 var (
+	// NoOp formatter.
 	NoOp = Register("noop", chroma.FormatterFunc(func(w io.Writer, s *chroma.Style) (func(*chroma.Token), error) {
 		return func(t *chroma.Token) { io.WriteString(w, t.Value) }, nil
 	}))
+	// Default HTML formatter outputs self-contained HTML.
 	htmlFull = Register("html", html.New(html.Standalone(), html.WithClasses()))
 )
 

formatters/tty_indexed.go

@@ -259,7 +259,11 @@ func (c *indexedTTYFormatter) Format(w io.Writer, style *chroma.Style) (func(*ch
 }
 
 // TTY8 is an 8-colour terminal formatter.
+//
+// The Lab colour space is used to map RGB values to the most appropriate index colour.
 var TTY8 = Register("terminal", &indexedTTYFormatter{ttyTables[8]})
 
 // TTY256 is a 256-colour terminal formatter.
+//
+// The Lab colour space is used to map RGB values to the most appropriate index colour.
 var TTY256 = Register("terminal256", &indexedTTYFormatter{ttyTables[256]})

lexer.go

@@ -79,7 +79,8 @@ type Lexer interface {
 	// Config describing the features of the Lexer.
 	Config() *Config
 	// Tokenise text and call out for each generated token.
-	// nil will be passed to out to signify the end of the stream.
+	//
+	// A token of type EOF will be passed to out() to signify the end of the stream.
 	Tokenise(options *TokeniseOptions, text string, out func(*Token)) error
 }
 

lexers/api.go

@@ -78,3 +78,14 @@ func Register(lexer chroma.Lexer) chroma.Lexer {
 	Registry.Lexers = append(Registry.Lexers, lexer)
 	return lexer
 }
+
+// Fallback lexer if no other is found.
+var Fallback chroma.Lexer = chroma.MustNewLexer(&chroma.Config{
+	Name:      "fallback",
+	Filenames: []string{"*"},
+}, chroma.Rules{
+	"root": []chroma.Rule{
+		{`.+`, chroma.Text, nil},
+		{`\n`, chroma.Text, nil},
+	},
+})

lexers/fallback.go

@@ -1,16 +0,0 @@
-package lexers
-
-import (
-	"github.com/alecthomas/chroma"
-)
-
-// Fallback lexer if no other is found.
-var Fallback chroma.Lexer = chroma.MustNewLexer(&chroma.Config{
-	Name:      "fallback",
-	Filenames: []string{"*"},
-}, chroma.Rules{
-	"root": []chroma.Rule{
-		{`.+`, chroma.Text, nil},
-		{`\n`, chroma.Text, nil},
-	},
-})

lexers/go.go

@@ -10,7 +10,7 @@ import (
 var Go = Register(MustNewLexer(
 	&Config{
 		Name:      "Go",
-		Aliases:   []string{"go"},
+		Aliases:   []string{"go", "golang"},
 		Filenames: []string{"*.go"},
 		MimeTypes: []string{"text/x-gosrc"},
 	},

lexers/lexer_benchmark_test.go

@@ -27,6 +27,7 @@ func (f FormatterFunc) Format(w io.Writer, s *Style) (func(*Token), error) {
 `
 
 func Benchmark(b *testing.B) {
+	b.ReportAllocs()
 	for i := 0; i < b.N; i++ {
 		Go.Tokenise(nil, lexerBenchSource, func(t *chroma.Token) {})
 	}

lexers/php.go

@@ -4,8 +4,8 @@ import (
 	. "github.com/alecthomas/chroma" // nolint
 )
 
-// Php lexer.
-var Php = Register(MustNewLexer(
+// PHP lexer.
+var PHP = Register(MustNewLexer(
 	&Config{
 		Name:            "PHP",
 		Aliases:         []string{"php", "php3", "php4", "php5"},

lexers/smarty.go

@@ -17,7 +17,7 @@ var Smarty = Register(MustNewLexer(
 		"root": {
 			{`[^{]+`, Other, nil},
 			{`(\{)(\*.*?\*)(\})`, ByGroups(CommentPreproc, Comment, CommentPreproc), nil},
-			{`(\{php\})(.*?)(\{/php\})`, ByGroups(CommentPreproc, Using(Php, nil), CommentPreproc), nil},
+			{`(\{php\})(.*?)(\{/php\})`, ByGroups(CommentPreproc, Using(PHP, nil), CommentPreproc), nil},
 			{`(\{)(/?[a-zA-Z_]\w*)(\s*)`, ByGroups(CommentPreproc, NameFunction, Text), Push("smarty")},
 			{`\{`, CommentPreproc, Push("smarty")},
 		},

quick/quick.go

@@ -0,0 +1,44 @@
+package quick
+
+import (
+	"io"
+
+	"github.com/alecthomas/chroma"
+	"github.com/alecthomas/chroma/formatters"
+	"github.com/alecthomas/chroma/lexers"
+	"github.com/alecthomas/chroma/styles"
+)
+
+// Highlight some text.
+//
+// Lexer, formatter and style may be empty, in which case a best-effort is made.
+func Highlight(w io.Writer, source, lexer, formatter, style string) error {
+	// Determine lexer.
+	l := lexers.Get(lexer)
+	if l == nil {
+		l = lexers.Analyse(source)
+	}
+	if l == nil {
+		l = lexers.Fallback
+	}
+	l = chroma.Coalesce(l)
+
+	// Determine formatter.
+	f := formatters.Get(formatter)
+	if f == nil {
+		f = formatters.Fallback
+	}
+
+	// Determine style.
+	s := styles.Get(style)
+	if s == nil {
+		s = styles.Fallback
+	}
+
+	writer, err := f.Format(w, s)
+	if err != nil {
+		return err
+	}
+
+	return l.Tokenise(nil, source, writer)
+}