chroma/lexers/lexers_test.go

package lexers_test

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"testing"

	"github.com/alecthomas/assert/v2"
	"github.com/alecthomas/repr"

	"github.com/alecthomas/chroma/v2"
	"github.com/alecthomas/chroma/v2/formatters"
	"github.com/alecthomas/chroma/v2/lexers"
	"github.com/alecthomas/chroma/v2/styles"
)

func TestCompileAllRegexes(t *testing.T) {
	for _, lexer := range lexers.GlobalLexerRegistry.Lexers {
		it, err := lexer.Tokenise(nil, "")
		assert.NoError(t, err, "%s failed", lexer.Config().Name)
		err = formatters.NoOp.Format(io.Discard, styles.SwapOff, it)
		assert.NoError(t, err, "%s failed", lexer.Config().Name)
	}
}

func TestGet(t *testing.T) {
	t.Run("ByName", func(t *testing.T) {
		assert.True(t, lexers.Get("xml") == lexers.GlobalLexerRegistry.Get("XML"))
	})
	t.Run("ByAlias", func(t *testing.T) {
		assert.True(t, lexers.Get("as") == lexers.GlobalLexerRegistry.Get("Actionscript"))
	})
	t.Run("ViaFilename", func(t *testing.T) {
		expected := lexers.Get("XML")
		actual := lexers.GlobalLexerRegistry.Get("test.svg")
		assert.Equal(t,
			repr.String(expected.Config(), repr.Indent("  ")),
			repr.String(actual.Config(), repr.Indent("  ")))
	})
}

func TestGlobs(t *testing.T) {
	filename := "main.go"
	for _, lexer := range lexers.GlobalLexerRegistry.Lexers {
		config := lexer.Config()
		for _, glob := range config.Filenames {
			_, err := filepath.Match(glob, filename)
			assert.NoError(t, err)
		}
		for _, glob := range config.AliasFilenames {
			_, err := filepath.Match(glob, filename)
			assert.NoError(t, err)
		}
	}
}

func BenchmarkGet(b *testing.B) {
	for i := 0; i < b.N; i++ {
		lexers.Get("go")
	}
}

func FileTest(t *testing.T, lexer chroma.Lexer, sourceFile, expectedFilename string) {
	t.Helper()
	t.Run(lexer.Config().Name+"/"+sourceFile, func(t *testing.T) {
		// Read and tokenise source text.
		sourceBytes, err := os.ReadFile(sourceFile)
		assert.NoError(t, err)
		actualTokens, err := chroma.Tokenise(lexer, nil, string(sourceBytes))
		assert.NoError(t, err)

		// Check for error tokens early
		for _, token := range actualTokens {
			if token.Type == chroma.Error {
				t.Logf("Found Error token in lexer %s output: %s", lexer.Config().Name, repr.String(token))
			}
		}

		// Use a bytes.Buffer to "render" the actual bytes
		var actualBytes bytes.Buffer
		err = formatters.JSON.Format(&actualBytes, nil, chroma.Literator(actualTokens...))
		assert.NoError(t, err)

		expectedBytes, err := os.ReadFile(expectedFilename)
		assert.NoError(t, err)

		// Check that the expected bytes are identical
		if !bytes.Equal(actualBytes.Bytes(), expectedBytes) {
			if os.Getenv("RECORD") == "true" {
				f, err := os.Create(expectedFilename)
				assert.NoError(t, err)
				_, err = f.Write(actualBytes.Bytes())
				assert.NoError(t, err)
				assert.NoError(t, f.Close())
			} else {
				// fail via an assertion of string values for diff output
				assert.Equal(t, string(expectedBytes), actualBytes.String())
			}
		}
	})
}

// Test source files are in the form <key>.<key> and validation data is in the form <key>.<key>.expected.
func TestLexers(t *testing.T) {
	files, err := os.ReadDir("testdata")
	assert.NoError(t, err)

	for _, file := range files {
		// skip text analysis test files
		if file.Name() == "analysis" {
			continue
		}

		if file.IsDir() {
			dirname := filepath.Join("testdata", file.Name())
			lexer := lexers.Get(file.Name())
			assert.NotZero(t, lexer)

			subFiles, err := os.ReadDir(dirname)
			assert.NoError(t, err)

			for _, subFile := range subFiles {
				ext := filepath.Ext(subFile.Name())[1:]
				if ext != "actual" {
					continue
				}

				filename := filepath.Join(dirname, subFile.Name())
				expectedFilename := strings.TrimSuffix(filename, filepath.Ext(filename)) + ".expected"

				lexer = chroma.Coalesce(lexer)
				FileTest(t, lexer, filename, expectedFilename)
			}
		} else {
			ext := filepath.Ext(file.Name())[1:]
			if ext != "actual" {
				continue
			}

			base := strings.Split(strings.TrimSuffix(file.Name(), filepath.Ext(file.Name())), ".")[0]
			lexer := lexers.Get(base)
			assert.NotZero(t, lexer, base)

			filename := filepath.Join("testdata", file.Name())
			expectedFilename := strings.TrimSuffix(filename, filepath.Ext(filename)) + ".expected"

			lexer = chroma.Coalesce(lexer)
			FileTest(t, lexer, filename, expectedFilename)
		}
	}
}

func FileTestAnalysis(t *testing.T, lexer chroma.Lexer, actualFilepath, expectedFilepath string) {
	t.Helper()
	t.Run(lexer.Config().Name+"/"+actualFilepath, func(t *testing.T) {
		expectedData, err := os.ReadFile(expectedFilepath)
		assert.NoError(t, err)

		analyser, ok := lexer.(chroma.Analyser)
		assert.True(t, ok, "lexer %q does not set analyser", lexer.Config().Name)

		data, err := os.ReadFile(actualFilepath)
		assert.NoError(t, err)

		actual := analyser.AnalyseText(string(data))
		var actualData bytes.Buffer
		fmt.Fprintf(&actualData, "%s\n", strconv.FormatFloat(float64(actual), 'f', -1, 32))

		if !bytes.Equal(expectedData, actualData.Bytes()) {
			if os.Getenv("RECORD") == "true" {
				f, err := os.Create(expectedFilepath)
				assert.NoError(t, err)
				_, err = f.Write(actualData.Bytes())
				assert.NoError(t, err)
				assert.NoError(t, f.Close())
			} else {
				assert.Equal(t, string(expectedData), actualData.String())
			}
		}
	})
}

func TestLexersTextAnalyser(t *testing.T) {
	files, err := filepath.Glob("testdata/analysis/*.actual")
	assert.NoError(t, err)

	for _, actualFilepath := range files {
		filename := filepath.Base(actualFilepath)
		baseFilename := strings.TrimSuffix(filename, filepath.Ext(filename))
		lexerName := strings.Split(baseFilename, ".")[0]

		lexer := lexers.Get(lexerName)
		assert.NotZero(t, lexer, "no lexer found for name %q", lexerName)

		expectedFilepath := "testdata/analysis/" + baseFilename + ".expected"

		FileTestAnalysis(t, lexer, actualFilepath, expectedFilepath)
	}
}