From ce5b193569721dc6662974c5e1b16c577e255bec Mon Sep 17 00:00:00 2001 From: Tao Wen Date: Fri, 2 Dec 2016 11:22:20 +0800 Subject: [PATCH] support null/true/false --- jsoniter.go | 186 +++++++++++++++++++++++++++++++++++- jsoniter_bool_test.go | 17 ++++ jsoniter_large_file_test.go | 50 ++++++++++ jsoniter_nested_test.go | 34 ++++--- jsoniter_null_test.go | 58 +++++++++++ jsoniter_skip_test.go | 44 ++++++--- 6 files changed, 358 insertions(+), 31 deletions(-) create mode 100644 jsoniter_bool_test.go create mode 100644 jsoniter_large_file_test.go create mode 100644 jsoniter_null_test.go diff --git a/jsoniter.go b/jsoniter.go index 67209bc..829992f 100644 --- a/jsoniter.go +++ b/jsoniter.go @@ -50,7 +50,12 @@ func (iter *Iterator) skipWhitespaces() { } func (iter *Iterator) ReportError(operation string, msg string) { - iter.Error = fmt.Errorf("%s: %s, parsing %v at %s", operation, msg, iter.head, string(iter.buf[0:iter.tail])) + peekStart := iter.head - 10 + if peekStart < 0 { + peekStart = 0 + } + iter.Error = fmt.Errorf("%s: %s, parsing %v ...%s... at %s", operation, msg, iter.head, + string(iter.buf[peekStart: iter.head]), string(iter.buf[0:iter.tail])) } func (iter *Iterator) readByte() (ret byte) { @@ -150,8 +155,17 @@ func (iter *Iterator) ReadString() (ret string) { if iter.Error != nil { return } - if c != '"' { - iter.ReportError("ReadString", "expects quote") + switch c { + case 'n': + iter.skipNull() + if iter.Error != nil { + return + } + return "" + case '"': + // nothing + default: + iter.ReportError("ReadString", `expects " or n`) return } for { @@ -312,6 +326,13 @@ func (iter *Iterator) ReadArray() (ret bool) { return } switch c { + case 'n': { + iter.skipNull() + if iter.Error != nil { + return + } + return false // null + } case '[': { iter.skipWhitespaces() c = iter.readByte() @@ -330,7 +351,7 @@ func (iter *Iterator) ReadArray() (ret bool) { iter.skipWhitespaces() return true default: - iter.ReportError("ReadArray", "expect [ or , or ]") + iter.ReportError("ReadArray", "expect [ or , or ] or n") return } } @@ -342,6 +363,13 @@ func (iter *Iterator) ReadObject() (ret string) { return } switch c { + case 'n': { + iter.skipNull() + if iter.Error != nil { + return + } + return "" // null + } case '{': { iter.skipWhitespaces() c = iter.readByte() @@ -373,7 +401,7 @@ func (iter *Iterator) ReadObject() (ret string) { case '}': return "" // end of object default: - iter.ReportError("ReadObject", `expect { or , or }`) + iter.ReportError("ReadObject", `expect { or , or } or n`) return } } @@ -423,6 +451,135 @@ func (iter *Iterator) ReadFloat64() (ret float64) { return } +func (iter *Iterator) ReadBool() (ret bool) { + c := iter.readByte() + if iter.Error != nil { + return + } + switch c { + case 't': + iter.skipTrue() + if iter.Error != nil { + return + } + return true + case 'f': + iter.skipFalse() + if iter.Error != nil { + return + } + return false + default: + iter.ReportError("ReadBool", "expect t or f") + return + } +} + +func (iter *Iterator) skipTrue() { + c := iter.readByte() + if iter.Error != nil { + return + } + if c != 'r' { + iter.ReportError("skipTrue", "expect r of true") + return + } + c = iter.readByte() + if iter.Error != nil { + return + } + if c != 'u' { + iter.ReportError("skipTrue", "expect u of true") + return + } + c = iter.readByte() + if iter.Error != nil { + return + } + if c != 'e' { + iter.ReportError("skipTrue", "expect e of true") + return + } +} + +func (iter *Iterator) skipFalse() { + c := iter.readByte() + if iter.Error != nil { + return + } + if c != 'a' { + iter.ReportError("skipFalse", "expect a of false") + return + } + c = iter.readByte() + if iter.Error != nil { + return + } + if c != 'l' { + iter.ReportError("skipFalse", "expect l of false") + return + } + c = iter.readByte() + if iter.Error != nil { + return + } + if c != 's' { + iter.ReportError("skipFalse", "expect s of false") + return + } + c = iter.readByte() + if iter.Error != nil { + return + } + if c != 'e' { + iter.ReportError("skipFalse", "expect e of false") + return + } +} + +func (iter *Iterator) ReadNull() (ret bool) { + c := iter.readByte() + if iter.Error != nil { + return + } + if c == 'n' { + iter.skipNull() + if iter.Error != nil { + return + } + return true + } + iter.unreadByte() + return false +} + +func (iter *Iterator) skipNull() { + c := iter.readByte() + if iter.Error != nil { + return + } + if c != 'u' { + iter.ReportError("skipNull", "expect u of null") + return + } + c = iter.readByte() + if iter.Error != nil { + return + } + if c != 'l' { + iter.ReportError("skipNull", "expect l of null") + return + } + c = iter.readByte() + if iter.Error != nil { + return + } + if c != 'l' { + iter.ReportError("skipNull", "expect l of null") + return + } +} + func (iter *Iterator) Skip() { c := iter.readByte() if iter.Error != nil { @@ -437,6 +594,12 @@ func (iter *Iterator) Skip() { iter.skipArray() case '{': iter.skipObject() + case 't': + iter.skipTrue() + case 'f': + iter.skipFalse() + case 'n': + iter.skipNull() default: iter.ReportError("Skip", fmt.Sprintf("do not know how to skip: %v", c)) return @@ -500,9 +663,22 @@ func (iter *Iterator) skipArray() { } func (iter *Iterator) skipObject() { + iter.skipWhitespaces() + c := iter.readByte() + if iter.Error != nil { + return + } + if c == '}' { + return // end of object + } else { + iter.unreadByte() + } for { iter.skipWhitespaces() c := iter.readByte() + if iter.Error != nil { + return + } if c != '"' { iter.ReportError("skipObject", `expects "`) return diff --git a/jsoniter_bool_test.go b/jsoniter_bool_test.go new file mode 100644 index 0000000..0c974ab --- /dev/null +++ b/jsoniter_bool_test.go @@ -0,0 +1,17 @@ +package jsoniter + +import "testing" + +func Test_true(t *testing.T) { + iter := ParseString(`true`) + if iter.ReadBool() != true { + t.FailNow() + } +} + +func Test_false(t *testing.T) { + iter := ParseString(`false`) + if iter.ReadBool() != false { + t.FailNow() + } +} diff --git a/jsoniter_large_file_test.go b/jsoniter_large_file_test.go new file mode 100644 index 0000000..d1b9480 --- /dev/null +++ b/jsoniter_large_file_test.go @@ -0,0 +1,50 @@ +package jsoniter + +import ( + "testing" + "os" + "encoding/json" + "io/ioutil" +) + +func Test_large_file(t *testing.T) { + file, err := os.Open("/tmp/large-file.json") + if err != nil { + t.Fatal(err) + } + iter := Parse(file, 4096) + count := 0 + for iter.ReadArray() { + iter.Skip() + count++ + } + if count != 11351 { + t.Fatal(count) + } +} + + +func Benchmark_jsoniter_large_file(b *testing.B) { + b.ReportAllocs() + for n := 0; n < b.N; n++ { + file, _ := os.Open("/tmp/large-file.json") + iter := Parse(file, 4096) + count := 0 + for iter.ReadArray() { + iter.Skip() + count++ + } + file.Close() + } +} + +func Benchmark_json_large_file(b *testing.B) { + b.ReportAllocs() + for n := 0; n < b.N; n++ { + file, _ := os.Open("/tmp/large-file.json") + bytes, _ := ioutil.ReadAll(file) + file.Close() + result := []struct{}{} + json.Unmarshal(bytes, &result) + } +} diff --git a/jsoniter_nested_test.go b/jsoniter_nested_test.go index a918f1b..dd5de8b 100644 --- a/jsoniter_nested_test.go +++ b/jsoniter_nested_test.go @@ -55,27 +55,31 @@ func Benchmark_jsoniter_nested(b *testing.B) { for l1Field := iter.ReadObject(); l1Field != ""; l1Field = iter.ReadObject() { switch l1Field { case "hello": - l2Array := make([]Level2, 0, 2) - for iter.ReadArray() { - l2 := Level2{} - for l2Field := iter.ReadObject(); l2Field != ""; l2Field = iter.ReadObject() { - switch l2Field { - case "world": - l2.World = iter.ReadString() - default: - iter.ReportError("bind l2", "unexpected field: " + l2Field) - } - } - l2Array = append(l2Array, l2) - } - l1.Hello = l2Array + l1.Hello = readLevel1Hello(iter) default: - iter.ReportError("bind l1", "unexpected field: " + l1Field) + iter.Skip() } } } } +func readLevel1Hello(iter *Iterator) []Level2 { + l2Array := make([]Level2, 0, 2) + for iter.ReadArray() { + l2 := Level2{} + for l2Field := iter.ReadObject(); l2Field != ""; l2Field = iter.ReadObject() { + switch l2Field { + case "world": + l2.World = iter.ReadString() + default: + iter.Skip() + } + } + l2Array = append(l2Array, l2) + } + return l2Array +} + func Benchmark_json_nested(b *testing.B) { for n := 0; n < b.N; n++ { l1 := Level1{} diff --git a/jsoniter_null_test.go b/jsoniter_null_test.go new file mode 100644 index 0000000..cb27d41 --- /dev/null +++ b/jsoniter_null_test.go @@ -0,0 +1,58 @@ +package jsoniter + +import ( + "testing" +) + +func Test_null(t *testing.T) { + iter := ParseString(`null`) + if iter.ReadNull() != true { + t.FailNow() + } +} + +func Test_null_object(t *testing.T) { + iter := ParseString(`[null,"a"]`) + iter.ReadArray() + if iter.ReadObject() != "" { + t.FailNow() + } + iter.ReadArray() + if iter.ReadString() != "a" { + t.FailNow() + } +} + +func Test_null_array(t *testing.T) { + iter := ParseString(`[null,"a"]`) + iter.ReadArray() + if iter.ReadArray() != false { + t.FailNow() + } + iter.ReadArray() + if iter.ReadString() != "a" { + t.FailNow() + } +} + +func Test_null_string(t *testing.T) { + iter := ParseString(`[null,"a"]`) + iter.ReadArray() + if iter.ReadString() != "" { + t.FailNow() + } + iter.ReadArray() + if iter.ReadString() != "a" { + t.FailNow() + } +} + +func Test_null_skip(t *testing.T) { + iter := ParseString(`[null,"a"]`) + iter.ReadArray() + iter.Skip() + iter.ReadArray() + if iter.ReadString() != "a" { + t.FailNow() + } +} \ No newline at end of file diff --git a/jsoniter_skip_test.go b/jsoniter_skip_test.go index 9b68068..b5eb8cd 100644 --- a/jsoniter_skip_test.go +++ b/jsoniter_skip_test.go @@ -45,6 +45,16 @@ func Test_skip_array(t *testing.T) { } } +func Test_skip_empty_array(t *testing.T) { + iter := ParseString(`[ [ ], "b"]`) + iter.ReadArray() + iter.Skip() + iter.ReadArray() + if iter.ReadString() != "b" { + t.FailNow() + } +} + func Test_skip_object(t *testing.T) { iter := ParseString(`[ {"a" : {"b": "c"}, "d": 102 }, "b"]`) iter.ReadArray() @@ -55,6 +65,16 @@ func Test_skip_object(t *testing.T) { } } +func Test_skip_empty_object(t *testing.T) { + iter := ParseString(`[ { }, "b"]`) + iter.ReadArray() + iter.Skip() + iter.ReadArray() + if iter.ReadString() != "b" { + t.FailNow() + } +} + func Test_skip_nested(t *testing.T) { iter := ParseString(`[ {"a" : [{"b": "c"}], "d": 102 }, "b"]`) iter.ReadArray() @@ -70,16 +90,13 @@ type TestResp struct { } func Benchmark_jsoniter_skip(b *testing.B) { - for n := 0; n < b.N; n++ { - result := TestResp{} - iter := ParseString(` + input := []byte(` { "_shards":{ "total" : 5, "successful" : 5, "failed" : 0 }, - "code": 200, "hits":{ "total" : 1, "hits" : [ @@ -94,8 +111,12 @@ func Benchmark_jsoniter_skip(b *testing.B) { } } ] - } + }, + "code": 200 }`) + for n := 0; n < b.N; n++ { + result := TestResp{} + iter := ParseBytes(input) for field := iter.ReadObject(); field != ""; field = iter.ReadObject() { switch field { case "code": @@ -108,16 +129,13 @@ func Benchmark_jsoniter_skip(b *testing.B) { } func Benchmark_json_skip(b *testing.B) { - for n := 0; n < b.N; n++ { - result := TestResp{} - json.Unmarshal([]byte(` + input := []byte(` { "_shards":{ "total" : 5, "successful" : 5, "failed" : 0 }, - "code": 200, "hits":{ "total" : 1, "hits" : [ @@ -132,7 +150,11 @@ func Benchmark_json_skip(b *testing.B) { } } ] - } -}`), &result) + }, + "code": 200 +}`) + for n := 0; n < b.N; n++ { + result := TestResp{} + json.Unmarshal(input, &result) } } \ No newline at end of file