Merge pull request #635 from molon/fix-tests

encoding/json: Compatibility optimization

README.md

@@ -8,8 +8,6 @@
 
 A high-performance 100% compatible drop-in replacement of "encoding/json"
 
-You can also use thrift like JSON using [thrift-iterator](https://github.com/thrift-iterator/go)
-
 # Benchmark
 
 ![benchmark](http://jsoniter.com/benchmarks/go-benchmark.png)

extra/binary_as_string_codec.go

@@ -180,9 +180,9 @@ func readHex(iter *jsoniter.Iterator, b1, b2 byte) byte {
 	}
 	ret *= 16
 	if b2 >= '0' && b2 <= '9' {
-		ret = b2 - '0'
+		ret += b2 - '0'
 	} else if b2 >= 'a' && b2 <= 'f' {
-		ret = b2 - 'a' + 10
+		ret += b2 - 'a' + 10
 	} else {
 		iter.ReportError("read hex", "expects 0~9 or a~f, but found "+string([]byte{b2}))
 		return 0

extra/binary_as_string_codec_test.go

@@ -22,11 +22,11 @@ func TestBinaryAsStringCodec(t *testing.T) {
 	})
 	t.Run("non safe set", func(t *testing.T) {
 		should := require.New(t)
-		output, err := jsoniter.Marshal([]byte{1, 2, 3, 15})
+		output, err := jsoniter.Marshal([]byte{1, 2, 3, 23})
 		should.NoError(err)
-		should.Equal(`"\\x01\\x02\\x03\\x0f"`, string(output))
+		should.Equal(`"\\x01\\x02\\x03\\x17"`, string(output))
 		var val []byte
 		should.NoError(jsoniter.Unmarshal(output, &val))
-		should.Equal([]byte{1, 2, 3, 15}, val)
+		should.Equal([]byte{1, 2, 3, 23}, val)
 	})
 }

go.mod

@@ -6,6 +6,6 @@ require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/google/gofuzz v1.0.0
 	github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421
-	github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742
+	github.com/modern-go/reflect2 v1.0.2
-	github.com/stretchr/testify v1.3.0
+	github.com/stretchr/testify v1.8.0
 )

go.sum

@@ -5,10 +5,17 @@ github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

iter_float.go

@@ -288,6 +288,9 @@ non_decimal_loop:
 				return iter.readFloat64SlowPath()
 			}
 			value = (value << 3) + (value << 1) + uint64(ind)
+			if value > maxFloat64 {
+				return iter.readFloat64SlowPath()
+			}
 		}
 	}
 	return iter.readFloat64SlowPath()

iter_int.go

@@ -9,6 +9,7 @@ var intDigits []int8
 
 const uint32SafeToMultiply10 = uint32(0xffffffff)/10 - 1
 const uint64SafeToMultiple10 = uint64(0xffffffffffffffff)/10 - 1
+const maxFloat64 = 1<<53 - 1
 
 func init() {
 	intDigits = make([]int8, 256)
@@ -339,7 +340,7 @@ func (iter *Iterator) readUint64(c byte) (ret uint64) {
 }
 
 func (iter *Iterator) assertInteger() {
-	if iter.head < len(iter.buf) && iter.buf[iter.head] == '.' {
+	if iter.head < iter.tail && iter.buf[iter.head] == '.' {
 		iter.ReportError("assertInteger", "can not decode float as int")
 	}
 }

misc_tests/jsoniter_int_test.go

@@ -5,7 +5,9 @@ package misc_tests
 import (
 	"bytes"
 	"encoding/json"
+	"io"
 	"io/ioutil"
+	"math/rand"
 	"strconv"
 	"testing"
 
@@ -70,6 +72,95 @@ func Test_float_as_int(t *testing.T) {
 	should.NotNil(jsoniter.Unmarshal([]byte(`1.1`), &i))
 }
 
+// chunkedData is io.Reader which returns random amount of data in range [1, chunkedData.chunkSize].
+// It simulates chunked data on from HTTP server, which is commonly used by net/http package.
+type chunkedData struct {
+	chunkSize int
+	data      []byte
+	head      int
+}
+
+// Read is implementation of the io.Reader which returns random amount of data in range [1, chunkedData.chunkSize].
+func (c *chunkedData) Read(p []byte) (n int, err error) {
+	to := c.head + int(rand.Int31n(int32(c.chunkSize))+1)
+
+	// copy does not copy more data then p can consume
+	n = copy(p, c.data[c.head:to])
+	c.head = c.head + n
+	if c.head >= len(c.data) {
+		err = io.EOF
+	}
+	return n, err
+}
+
+// TestIterator_ReadInt_chunkedInput validates the behaviour of Iterator.ReadInt() method in where:
+// - it reads data from io.Reader,
+// - expected value is 0 (zero)
+// - Iterator.tail == Iterator.head
+// - Iterator.tail < len(Iterator.buf)
+// - value in buffer after Iterator.tail is presented from previous read and has '.' character.
+func TestIterator_ReadInt_chunkedInput(t *testing.T) {
+	should := require.New(t)
+
+	data := &chunkedData{
+		data: jsonFloatIntArray(t, 10),
+	}
+
+	// because this test is rely on randomness of chunkedData, we are doing multiple iterations to
+	// be sure, that we can hit a required case.
+	for data.chunkSize = 3; data.chunkSize <= len(data.data); data.chunkSize++ {
+		data.head = 0
+
+		iter := jsoniter.Parse(jsoniter.ConfigDefault, data, data.chunkSize)
+		i := 0
+		for iter.ReadArray() {
+			// every even item is float, let's just skip it.
+			if i%2 == 0 {
+				iter.Skip()
+				i++
+				continue
+			}
+
+			should.Zero(iter.ReadInt())
+			should.NoError(iter.Error)
+
+			i++
+		}
+	}
+}
+
+// jsonFloatIntArray generates JSON array where every
+//  - even item is float 0.1
+//  - odd item is integer 0
+//
+//  [0.1, 0, 0.1, 0]
+func jsonFloatIntArray(t *testing.T, numberOfItems int) []byte {
+	t.Helper()
+	numbers := make([]jsoniter.Any, numberOfItems)
+	for i := range numbers {
+		switch i % 2 {
+		case 0:
+			numbers[i] = jsoniter.WrapFloat64(0.1)
+		default:
+			numbers[i] = jsoniter.WrapInt64(0)
+		}
+	}
+
+	fixture, err := jsoniter.ConfigFastest.Marshal(numbers)
+	if err != nil {
+		panic(err)
+	}
+
+	b := &bytes.Buffer{}
+
+	require.NoError(
+		t,
+		json.Compact(b, fixture),
+		"json should be compactable",
+	)
+	return b.Bytes()
+}
+
 func Benchmark_jsoniter_encode_int(b *testing.B) {
 	stream := jsoniter.NewStream(jsoniter.ConfigDefault, ioutil.Discard, 64)
 	for n := 0; n < b.N; n++ {

misc_tests/jsoniter_raw_message_test.go

@@ -42,6 +42,25 @@ func Test_marshal_invalid_json_raw_message(t *testing.T) {
 	should.Nil(aouterr)
 }
 
+func Test_marshal_nil_json_raw_message(t *testing.T) {
+	type A struct {
+		Nil1 jsoniter.RawMessage `json:"raw1"`
+		Nil2 json.RawMessage     `json:"raw2"`
+	}
+
+	a := A{}
+	should := require.New(t)
+	aout, aouterr := jsoniter.Marshal(&a)
+	should.Equal(`{"raw1":null,"raw2":null}`, string(aout))
+	should.Nil(aouterr)
+
+	a.Nil1 = []byte(`Any`)
+	a.Nil2 = []byte(`Any`)
+	should.Nil(jsoniter.Unmarshal(aout, &a))
+	should.Nil(a.Nil1)
+	should.Nil(a.Nil2)
+}
+
 func Test_raw_message_memory_not_copied_issue(t *testing.T) {
 	jsonStream := `{"name":"xxxxx","bundle_id":"com.zonst.majiang","app_platform":"ios","app_category":"100103", "budget_day":1000,"bidding_min":1,"bidding_max":2,"bidding_type":"CPM", "freq":{"open":true,"type":"day","num":100},"speed":1, "targeting":{"vendor":{"open":true,"list":["zonst"]}, "geo_code":{"open":true,"list":["156110100"]},"app_category":{"open":true,"list":["100101"]}, "day_parting":{"open":true,"list":["100409","100410"]},"device_type":{"open":true,"list":["ipad"]}, "os_version":{"open":true,"list":[10]},"carrier":{"open":true,"list":["mobile"]}, "network":{"open":true,"list":["4G"]}},"url":{"tracking_imp_url":"http://www.baidu.com", "tracking_clk_url":"http://www.baidu.com","jump_url":"http://www.baidu.com","deep_link_url":"http://www.baidu.com"}}`
 	type IteratorObject struct {

reflect.go

@@ -65,7 +65,7 @@ func (iter *Iterator) ReadVal(obj interface{}) {
 	decoder := iter.cfg.getDecoderFromCache(cacheKey)
 	if decoder == nil {
 		typ := reflect2.TypeOf(obj)
-		if typ.Kind() != reflect.Ptr {
+		if typ == nil || typ.Kind() != reflect.Ptr {
 			iter.ReportError("ReadVal", "can only unmarshal into pointer")
 			return
 		}

reflect_json_raw_message.go

@@ -33,11 +33,19 @@ type jsonRawMessageCodec struct {
 }
 
 func (codec *jsonRawMessageCodec) Decode(ptr unsafe.Pointer, iter *Iterator) {
-	*((*json.RawMessage)(ptr)) = json.RawMessage(iter.SkipAndReturnBytes())
+	if iter.ReadNil() {
+		*((*json.RawMessage)(ptr)) = nil
+	} else {
+		*((*json.RawMessage)(ptr)) = iter.SkipAndReturnBytes()
+	}
 }
 
 func (codec *jsonRawMessageCodec) Encode(ptr unsafe.Pointer, stream *Stream) {
-	stream.WriteRaw(string(*((*json.RawMessage)(ptr))))
+	if *((*json.RawMessage)(ptr)) == nil {
+		stream.WriteNil()
+	} else {
+		stream.WriteRaw(string(*((*json.RawMessage)(ptr))))
+	}
 }
 
 func (codec *jsonRawMessageCodec) IsEmpty(ptr unsafe.Pointer) bool {
@@ -48,11 +56,19 @@ type jsoniterRawMessageCodec struct {
 }
 
 func (codec *jsoniterRawMessageCodec) Decode(ptr unsafe.Pointer, iter *Iterator) {
-	*((*RawMessage)(ptr)) = RawMessage(iter.SkipAndReturnBytes())
+	if iter.ReadNil() {
+		*((*RawMessage)(ptr)) = nil
+	} else {
+		*((*RawMessage)(ptr)) = iter.SkipAndReturnBytes()
+	}
 }
 
 func (codec *jsoniterRawMessageCodec) Encode(ptr unsafe.Pointer, stream *Stream) {
-	stream.WriteRaw(string(*((*RawMessage)(ptr))))
+	if *((*RawMessage)(ptr)) == nil {
+		stream.WriteNil()
+	} else {
+		stream.WriteRaw(string(*((*RawMessage)(ptr))))
+	}
 }
 
 func (codec *jsoniterRawMessageCodec) IsEmpty(ptr unsafe.Pointer) bool {

reflect_map.go

@@ -2,11 +2,12 @@ package jsoniter
 
 import (
 	"fmt"
-	"github.com/modern-go/reflect2"
 	"io"
 	"reflect"
 	"sort"
 	"unsafe"
+
+	"github.com/modern-go/reflect2"
 )
 
 func decoderOfMap(ctx *ctx, typ reflect2.Type) ValDecoder {
@@ -106,15 +107,17 @@ func encoderOfMapKey(ctx *ctx, typ reflect2.Type) ValEncoder {
 		}
 	}
 
-	if typ == textMarshalerType {
+	if typ.Kind() != reflect.String {
-		return &directTextMarshalerEncoder{
+		if typ == textMarshalerType {
-			stringEncoder: ctx.EncoderOf(reflect2.TypeOf("")),
+			return &directTextMarshalerEncoder{
+				stringEncoder: ctx.EncoderOf(reflect2.TypeOf("")),
+			}
 		}
-	}
+		if typ.Implements(textMarshalerType) {
-	if typ.Implements(textMarshalerType) {
+			return &textMarshalerEncoder{
-		return &textMarshalerEncoder{
+				valType:       typ,
-			valType:       typ,
+				stringEncoder: ctx.EncoderOf(reflect2.TypeOf("")),
-			stringEncoder: ctx.EncoderOf(reflect2.TypeOf("")),
+			}
 		}
 	}
 

reflect_struct_decoder.go

@@ -1075,6 +1075,11 @@ type stringModeNumberDecoder struct {
 }
 
 func (decoder *stringModeNumberDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) {
+	if iter.WhatIsNext() == NilValue {
+		decoder.elemDecoder.Decode(ptr, iter)
+		return
+	}
+
 	c := iter.nextToken()
 	if c != '"' {
 		iter.ReportError("stringModeNumberDecoder", `expect ", but found `+string([]byte{c}))

stream_float.go

@@ -27,6 +27,14 @@ func (stream *Stream) WriteFloat32(val float32) {
 		}
 	}
 	stream.buf = strconv.AppendFloat(stream.buf, float64(val), fmt, -1, 32)
+	if fmt == 'e' {
+		// clean up e-09 to e-9
+		n := len(stream.buf)
+		if n >= 4 && stream.buf[n-4] == 'e' && stream.buf[n-3] == '-' && stream.buf[n-2] == '0' {
+			stream.buf[n-2] = stream.buf[n-1]
+			stream.buf = stream.buf[:n-1]
+		}
+	}
 }
 
 // WriteFloat32Lossy write float32 to stream with ONLY 6 digits precision although much much faster
@@ -76,6 +84,14 @@ func (stream *Stream) WriteFloat64(val float64) {
 		}
 	}
 	stream.buf = strconv.AppendFloat(stream.buf, float64(val), fmt, -1, 64)
+	if fmt == 'e' {
+		// clean up e-09 to e-9
+		n := len(stream.buf)
+		if n >= 4 && stream.buf[n-4] == 'e' && stream.buf[n-3] == '-' && stream.buf[n-2] == '0' {
+			stream.buf[n-2] = stream.buf[n-1]
+			stream.buf = stream.buf[:n-1]
+		}
+	}
 }
 
 // WriteFloat64Lossy write float64 to stream with ONLY 6 digits precision although much much faster

value_tests/float_test.go

@@ -4,10 +4,11 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"github.com/json-iterator/go"
-	"github.com/stretchr/testify/require"
 	"strconv"
 	"testing"
+
+	jsoniter "github.com/json-iterator/go"
+	"github.com/stretchr/testify/require"
 )
 
 func Test_read_float(t *testing.T) {
@@ -88,7 +89,7 @@ func Test_write_float32(t *testing.T) {
 
 	stream = jsoniter.NewStream(jsoniter.ConfigDefault, nil, 0)
 	stream.WriteFloat32(float32(0.0000001))
-	should.Equal("1e-07", string(stream.Buffer()))
+	should.Equal("1e-7", string(stream.Buffer()))
 }
 
 func Test_write_float64(t *testing.T) {
@@ -125,5 +126,5 @@ func Test_write_float64(t *testing.T) {
 
 	stream = jsoniter.NewStream(jsoniter.ConfigDefault, nil, 0)
 	stream.WriteFloat64(float64(0.0000001))
-	should.Equal("1e-07", string(stream.Buffer()))
+	should.Equal("1e-7", string(stream.Buffer()))
 }

value_tests/struct_test.go

@@ -26,6 +26,11 @@ func init() {
 			Field int `json:"field"`
 		})(nil),
 		input: `{"field": null}`,
+	}, unmarshalCase{
+		ptr: (*struct {
+			Field int `json:"field,string"`
+		})(nil),
+		input: `{"field": null}`,
 	}, unmarshalCase{
 		ptr: (*struct {
 			ID      int                    `json:"id"`