support null/true/false

jsoniter.go

@@ -50,7 +50,12 @@ func (iter *Iterator) skipWhitespaces() {
 }
 
 func (iter *Iterator) ReportError(operation string, msg string) {
-	iter.Error = fmt.Errorf("%s: %s, parsing %v at %s", operation, msg, iter.head, string(iter.buf[0:iter.tail]))
+	peekStart := iter.head - 10
+	if peekStart < 0 {
+		peekStart = 0
+	}
+	iter.Error = fmt.Errorf("%s: %s, parsing %v ...%s... at %s", operation, msg, iter.head,
+		string(iter.buf[peekStart: iter.head]), string(iter.buf[0:iter.tail]))
 }
 
 func (iter *Iterator) readByte() (ret byte) {
@@ -150,8 +155,17 @@ func (iter *Iterator) ReadString() (ret string) {
 	if iter.Error != nil {
 		return
 	}
-	if c != '"' {
+	switch c {
-		iter.ReportError("ReadString", "expects quote")
+	case 'n':
+		iter.skipNull()
+		if iter.Error != nil {
+			return
+		}
+		return ""
+	case '"':
+		// nothing
+	default:
+		iter.ReportError("ReadString", `expects " or n`)
 		return
 	}
 	for {
@@ -312,6 +326,13 @@ func (iter *Iterator) ReadArray() (ret bool) {
 		return
 	}
 	switch c {
+	case 'n': {
+		iter.skipNull()
+		if iter.Error != nil {
+			return
+		}
+		return false // null
+	}
 	case '[': {
 		iter.skipWhitespaces()
 		c = iter.readByte()
@@ -330,7 +351,7 @@ func (iter *Iterator) ReadArray() (ret bool) {
 		iter.skipWhitespaces()
 		return true
 	default:
-		iter.ReportError("ReadArray", "expect [ or , or ]")
+		iter.ReportError("ReadArray", "expect [ or , or ] or n")
 		return
 	}
 }
@@ -342,6 +363,13 @@ func (iter *Iterator) ReadObject() (ret string) {
 		return
 	}
 	switch c {
+	case 'n': {
+		iter.skipNull()
+		if iter.Error != nil {
+			return
+		}
+		return "" // null
+	}
 	case '{': {
 		iter.skipWhitespaces()
 		c = iter.readByte()
@@ -373,7 +401,7 @@ func (iter *Iterator) ReadObject() (ret string) {
 	case '}':
 		return "" // end of object
 	default:
-		iter.ReportError("ReadObject", `expect { or , or }`)
+		iter.ReportError("ReadObject", `expect { or , or } or n`)
 		return
 	}
 }
@@ -423,6 +451,135 @@ func (iter *Iterator) ReadFloat64() (ret float64) {
 	return
 }
 
+func (iter *Iterator) ReadBool() (ret bool) {
+	c := iter.readByte()
+	if iter.Error != nil {
+		return
+	}
+	switch c {
+	case 't':
+		iter.skipTrue()
+		if iter.Error != nil {
+			return
+		}
+		return true
+	case 'f':
+		iter.skipFalse()
+		if iter.Error != nil {
+			return
+		}
+		return false
+	default:
+		iter.ReportError("ReadBool", "expect t or f")
+		return
+	}
+}
+
+func (iter *Iterator) skipTrue() {
+	c := iter.readByte()
+	if iter.Error != nil {
+		return
+	}
+	if c != 'r' {
+		iter.ReportError("skipTrue", "expect r of true")
+		return
+	}
+	c = iter.readByte()
+	if iter.Error != nil {
+		return
+	}
+	if c != 'u' {
+		iter.ReportError("skipTrue", "expect u of true")
+		return
+	}
+	c = iter.readByte()
+	if iter.Error != nil {
+		return
+	}
+	if c != 'e' {
+		iter.ReportError("skipTrue", "expect e of true")
+		return
+	}
+}
+
+func (iter *Iterator) skipFalse() {
+	c := iter.readByte()
+	if iter.Error != nil {
+		return
+	}
+	if c != 'a' {
+		iter.ReportError("skipFalse", "expect a of false")
+		return
+	}
+	c = iter.readByte()
+	if iter.Error != nil {
+		return
+	}
+	if c != 'l' {
+		iter.ReportError("skipFalse", "expect l of false")
+		return
+	}
+	c = iter.readByte()
+	if iter.Error != nil {
+		return
+	}
+	if c != 's' {
+		iter.ReportError("skipFalse", "expect s of false")
+		return
+	}
+	c = iter.readByte()
+	if iter.Error != nil {
+		return
+	}
+	if c != 'e' {
+		iter.ReportError("skipFalse", "expect e of false")
+		return
+	}
+}
+
+func (iter *Iterator) ReadNull() (ret bool) {
+	c := iter.readByte()
+	if iter.Error != nil {
+		return
+	}
+	if c == 'n' {
+		iter.skipNull()
+		if iter.Error != nil {
+			return
+		}
+		return true
+	}
+	iter.unreadByte()
+	return false
+}
+
+func (iter *Iterator) skipNull() {
+	c := iter.readByte()
+	if iter.Error != nil {
+		return
+	}
+	if c != 'u' {
+		iter.ReportError("skipNull", "expect u of null")
+		return
+	}
+	c = iter.readByte()
+	if iter.Error != nil {
+		return
+	}
+	if c != 'l' {
+		iter.ReportError("skipNull", "expect l of null")
+		return
+	}
+	c = iter.readByte()
+	if iter.Error != nil {
+		return
+	}
+	if c != 'l' {
+		iter.ReportError("skipNull", "expect l of null")
+		return
+	}
+}
+
 func (iter *Iterator) Skip() {
 	c := iter.readByte()
 	if iter.Error != nil {
@@ -437,6 +594,12 @@ func (iter *Iterator) Skip() {
 		iter.skipArray()
 	case '{':
 		iter.skipObject()
+	case 't':
+		iter.skipTrue()
+	case 'f':
+		iter.skipFalse()
+	case 'n':
+		iter.skipNull()
 	default:
 		iter.ReportError("Skip", fmt.Sprintf("do not know how to skip: %v", c))
 		return
@@ -500,9 +663,22 @@ func (iter *Iterator) skipArray() {
 }
 
 func (iter *Iterator) skipObject() {
+	iter.skipWhitespaces()
+	c := iter.readByte()
+	if iter.Error != nil {
+		return
+	}
+	if c == '}' {
+		return // end of object
+	} else {
+		iter.unreadByte()
+	}
 	for {
 		iter.skipWhitespaces()
 		c := iter.readByte()
+		if iter.Error != nil {
+			return
+		}
 		if c != '"' {
 			iter.ReportError("skipObject", `expects "`)
 			return

jsoniter_bool_test.go

@@ -0,0 +1,17 @@
+package jsoniter
+
+import "testing"
+
+func Test_true(t *testing.T) {
+	iter := ParseString(`true`)
+	if iter.ReadBool() != true {
+		t.FailNow()
+	}
+}
+
+func Test_false(t *testing.T) {
+	iter := ParseString(`false`)
+	if iter.ReadBool() != false {
+		t.FailNow()
+	}
+}

jsoniter_large_file_test.go

@@ -0,0 +1,50 @@
+package jsoniter
+
+import (
+	"testing"
+	"os"
+	"encoding/json"
+	"io/ioutil"
+)
+
+func Test_large_file(t *testing.T) {
+	file, err := os.Open("/tmp/large-file.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	iter := Parse(file, 4096)
+	count := 0
+	for iter.ReadArray() {
+		iter.Skip()
+		count++
+	}
+	if count != 11351 {
+		t.Fatal(count)
+	}
+}
+
+
+func Benchmark_jsoniter_large_file(b *testing.B) {
+	b.ReportAllocs()
+	for n := 0; n < b.N; n++ {
+		file, _ := os.Open("/tmp/large-file.json")
+		iter := Parse(file, 4096)
+		count := 0
+		for iter.ReadArray() {
+			iter.Skip()
+			count++
+		}
+		file.Close()
+	}
+}
+
+func Benchmark_json_large_file(b *testing.B) {
+	b.ReportAllocs()
+	for n := 0; n < b.N; n++ {
+		file, _ := os.Open("/tmp/large-file.json")
+		bytes, _ := ioutil.ReadAll(file)
+		file.Close()
+		result := []struct{}{}
+		json.Unmarshal(bytes, &result)
+	}
+}

jsoniter_nested_test.go

@@ -55,27 +55,31 @@ func Benchmark_jsoniter_nested(b *testing.B) {
 		for l1Field := iter.ReadObject(); l1Field != ""; l1Field = iter.ReadObject() {
 			switch l1Field {
 			case "hello":
-				l2Array := make([]Level2, 0, 2)
+				l1.Hello = readLevel1Hello(iter)
-				for iter.ReadArray() {
-					l2 := Level2{}
-					for l2Field := iter.ReadObject(); l2Field != ""; l2Field = iter.ReadObject() {
-						switch l2Field {
-						case "world":
-							l2.World = iter.ReadString()
-						default:
-							iter.ReportError("bind l2", "unexpected field: " + l2Field)
-						}
-					}
-					l2Array = append(l2Array, l2)
-				}
-				l1.Hello = l2Array
 			default:
-				iter.ReportError("bind l1", "unexpected field: " + l1Field)
+				iter.Skip()
 			}
 		}
 	}
 }
 
+func readLevel1Hello(iter *Iterator) []Level2 {
+	l2Array := make([]Level2, 0, 2)
+	for iter.ReadArray() {
+		l2 := Level2{}
+		for l2Field := iter.ReadObject(); l2Field != ""; l2Field = iter.ReadObject() {
+			switch l2Field {
+			case "world":
+				l2.World = iter.ReadString()
+			default:
+				iter.Skip()
+			}
+		}
+		l2Array = append(l2Array, l2)
+	}
+	return l2Array
+}
+
 func Benchmark_json_nested(b *testing.B) {
 	for n := 0; n < b.N; n++ {
 		l1 := Level1{}

jsoniter_null_test.go

@@ -0,0 +1,58 @@
+package jsoniter
+
+import (
+	"testing"
+)
+
+func Test_null(t *testing.T) {
+	iter := ParseString(`null`)
+	if iter.ReadNull() != true {
+		t.FailNow()
+	}
+}
+
+func Test_null_object(t *testing.T) {
+	iter := ParseString(`[null,"a"]`)
+	iter.ReadArray()
+	if iter.ReadObject() != "" {
+		t.FailNow()
+	}
+	iter.ReadArray()
+	if iter.ReadString() != "a" {
+		t.FailNow()
+	}
+}
+
+func Test_null_array(t *testing.T) {
+	iter := ParseString(`[null,"a"]`)
+	iter.ReadArray()
+	if iter.ReadArray() != false {
+		t.FailNow()
+	}
+	iter.ReadArray()
+	if iter.ReadString() != "a" {
+		t.FailNow()
+	}
+}
+
+func Test_null_string(t *testing.T) {
+	iter := ParseString(`[null,"a"]`)
+	iter.ReadArray()
+	if iter.ReadString() != "" {
+		t.FailNow()
+	}
+	iter.ReadArray()
+	if iter.ReadString() != "a" {
+		t.FailNow()
+	}
+}
+
+func Test_null_skip(t *testing.T) {
+	iter := ParseString(`[null,"a"]`)
+	iter.ReadArray()
+	iter.Skip()
+	iter.ReadArray()
+	if iter.ReadString() != "a" {
+		t.FailNow()
+	}
+}

jsoniter_skip_test.go

@@ -45,6 +45,16 @@ func Test_skip_array(t *testing.T) {
 	}
 }
 
+func Test_skip_empty_array(t *testing.T) {
+	iter := ParseString(`[ [ ], "b"]`)
+	iter.ReadArray()
+	iter.Skip()
+	iter.ReadArray()
+	if iter.ReadString() != "b" {
+		t.FailNow()
+	}
+}
+
 func Test_skip_object(t *testing.T) {
 	iter := ParseString(`[ {"a" : {"b": "c"}, "d": 102 }, "b"]`)
 	iter.ReadArray()
@@ -55,6 +65,16 @@ func Test_skip_object(t *testing.T) {
 	}
 }
 
+func Test_skip_empty_object(t *testing.T) {
+	iter := ParseString(`[ { }, "b"]`)
+	iter.ReadArray()
+	iter.Skip()
+	iter.ReadArray()
+	if iter.ReadString() != "b" {
+		t.FailNow()
+	}
+}
+
 func Test_skip_nested(t *testing.T) {
 	iter := ParseString(`[ {"a" : [{"b": "c"}], "d": 102 }, "b"]`)
 	iter.ReadArray()
@@ -70,16 +90,13 @@ type TestResp struct {
 }
 
 func Benchmark_jsoniter_skip(b *testing.B) {
-	for n := 0; n < b.N; n++ {
+	input := []byte(`
-		result := TestResp{}
-		iter := ParseString(`
 {
     "_shards":{
         "total" : 5,
         "successful" : 5,
         "failed" : 0
     },
-    "code": 200,
     "hits":{
         "total" : 1,
         "hits" : [
@@ -94,8 +111,12 @@ func Benchmark_jsoniter_skip(b *testing.B) {
                 }
             }
         ]
-    }
+    },
+    "code": 200
 }`)
+	for n := 0; n < b.N; n++ {
+		result := TestResp{}
+		iter := ParseBytes(input)
 		for field := iter.ReadObject(); field != ""; field = iter.ReadObject() {
 			switch field {
 			case "code":
@@ -108,16 +129,13 @@ func Benchmark_jsoniter_skip(b *testing.B) {
 }
 
 func Benchmark_json_skip(b *testing.B) {
-	for n := 0; n < b.N; n++ {
+	input := []byte(`
-		result := TestResp{}
-		json.Unmarshal([]byte(`
 {
     "_shards":{
         "total" : 5,
         "successful" : 5,
         "failed" : 0
     },
-    "code": 200,
     "hits":{
         "total" : 1,
         "hits" : [
@@ -132,7 +150,11 @@ func Benchmark_json_skip(b *testing.B) {
                 }
             }
         ]
-    }
+    },
-}`), &result)
+    "code": 200
+}`)
+	for n := 0; n < b.N; n++ {
+		result := TestResp{}
+		json.Unmarshal(input, &result)
 	}
 }