go/pocketbase
1
0
Fork 0
mirror of https://github.com/pocketbase/pocketbase.git synced 2025-03-18 13:47:47 +02:00
Code Issues Releases Activity

updated default tokenizer separators

Browse Source
This commit is contained in:
Gani Georgiev 2023-02-06 16:30:47 +02:00
parent 23dfa9c634
commit f475967a4a
2 changed files with 20 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tools/tokenizer/tokenizer.go
View File

@ -19,7 +19,7 @@ import (
const eof = rune(0)
// DefaultSeparators is a list with the default token separator characters.
var DefaultSeparators = []rune{',', ' ', '\t', '\n'}
var DefaultSeparators = []rune{','}
// NewFromString creates new Tokenizer from the provided string.
func NewFromString(str string) *Tokenizer {
@ -31,7 +31,7 @@ func NewFromBytes(b []byte) *Tokenizer {
return New(bytes.NewReader(b))
}
// New creates new Tokenizer from the provided reader.
// New creates new Tokenizer from the provided reader with DefaultSeparators.
func New(r io.Reader) *Tokenizer {
return &Tokenizer{
r: bufio.NewReader(r),

36
tools/tokenizer/tokenizer_test.go
View File

@ -54,7 +54,7 @@ func TestFactories(t *testing.T) {
}
func TestScan(t *testing.T) {
tk := NewFromString("abc 123.456 (abc)")
tk := NewFromString("abc, 123.456, (abc)")
expectedTokens := []string{"abc", "123.456", "(abc)"}
@ -79,7 +79,7 @@ func TestScan(t *testing.T) {
}
}
func TestScanAllWithDefaultSeparators(t *testing.T) {
func TestScanAll(t *testing.T) {
scenarios := []struct {
name string
content string
@ -119,10 +119,24 @@ func TestScanAllWithDefaultSeparators(t *testing.T) {
},
{
"default separators",
`a, b, c, d e, "a,b, c ", (123, 456)`,
DefaultSeparators,
false,
[]string{
"a",
"b",
"c",
"d e",
`"a,b, c "`,
`(123, 456)`,
},
},
{
"custom separators",
` a , 123.456, b, c d, (
test (a,b,c) " 123 "
),"(abc d", "abc) d", "(abc) d \" " 'abc "'`,
DefaultSeparators,
[]rune{',', ' ', '\t', '\n'},
false,
[]string{
"a",
@ -137,20 +151,6 @@ func TestScanAllWithDefaultSeparators(t *testing.T) {
`'abc "'`,
},
},
{
"custom separators",
`a, b, c, d e, "a,b, c ", (123, 456)`,
[]rune{','},
false,
[]string{
"a",
"b",
"c",
"d e",
`"a,b, c "`,
`(123, 456)`,
},
},
}
for _, s := range scenarios {
@ -166,7 +166,7 @@ func TestScanAllWithDefaultSeparators(t *testing.T) {
}
if len(tokens) != len(s.expectTokens) {
t.Fatalf("[%s] Expected \n%v, \ngot \n%v", s.name, s.expectTokens, tokens)
t.Fatalf("[%s] Expected \n%v (%d), \ngot \n%v (%d)", s.name, s.expectTokens, len(s.expectTokens), tokens, len(tokens))
}
for _, tok := range tokens {