fix go 1.18 compatibility

fix: invalid unicode while binary to string

README.md

@@ -8,8 +8,6 @@
 
 A high-performance 100% compatible drop-in replacement of "encoding/json"
 
-You can also use thrift like JSON using [thrift-iterator](https://github.com/thrift-iterator/go)
-
 # Benchmark
 
 ![benchmark](http://jsoniter.com/benchmarks/go-benchmark.png)

extra/binary_as_string_codec.go

@@ -180,9 +180,9 @@ func readHex(iter *jsoniter.Iterator, b1, b2 byte) byte {
 	}
 	ret *= 16
 	if b2 >= '0' && b2 <= '9' {
-		ret = b2 - '0'
+		ret += b2 - '0'
 	} else if b2 >= 'a' && b2 <= 'f' {
-		ret = b2 - 'a' + 10
+		ret += b2 - 'a' + 10
 	} else {
 		iter.ReportError("read hex", "expects 0~9 or a~f, but found "+string([]byte{b2}))
 		return 0

extra/binary_as_string_codec_test.go

@@ -22,11 +22,11 @@ func TestBinaryAsStringCodec(t *testing.T) {
 	})
 	t.Run("non safe set", func(t *testing.T) {
 		should := require.New(t)
-		output, err := jsoniter.Marshal([]byte{1, 2, 3, 15})
+		output, err := jsoniter.Marshal([]byte{1, 2, 3, 23})
 		should.NoError(err)
-		should.Equal(`"\\x01\\x02\\x03\\x0f"`, string(output))
+		should.Equal(`"\\x01\\x02\\x03\\x17"`, string(output))
 		var val []byte
 		should.NoError(jsoniter.Unmarshal(output, &val))
-		should.Equal([]byte{1, 2, 3, 15}, val)
+		should.Equal([]byte{1, 2, 3, 23}, val)
 	})
 }

go.mod

@@ -6,6 +6,6 @@ require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/google/gofuzz v1.0.0
 	github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421
-	github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742
+	github.com/modern-go/reflect2 v1.0.2
 	github.com/stretchr/testify v1.3.0
 )

go.sum

@@ -5,8 +5,8 @@ github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

iter_float.go

@@ -288,6 +288,9 @@ non_decimal_loop:
 				return iter.readFloat64SlowPath()
 			}
 			value = (value << 3) + (value << 1) + uint64(ind)
+			if value > maxFloat64 {
+				return iter.readFloat64SlowPath()
+			}
 		}
 	}
 	return iter.readFloat64SlowPath()

iter_int.go

@@ -9,6 +9,7 @@ var intDigits []int8
 
 const uint32SafeToMultiply10 = uint32(0xffffffff)/10 - 1
 const uint64SafeToMultiple10 = uint64(0xffffffffffffffff)/10 - 1
+const maxFloat64 = 1<<53 - 1
 
 func init() {
 	intDigits = make([]int8, 256)
@@ -339,7 +340,7 @@ func (iter *Iterator) readUint64(c byte) (ret uint64) {
 }
 
 func (iter *Iterator) assertInteger() {
-	if iter.head < len(iter.buf) && iter.buf[iter.head] == '.' {
+	if iter.head < iter.tail && iter.buf[iter.head] == '.' {
 		iter.ReportError("assertInteger", "can not decode float as int")
 	}
 }

misc_tests/jsoniter_int_test.go

@@ -5,7 +5,9 @@ package misc_tests
 import (
 	"bytes"
 	"encoding/json"
+	"io"
 	"io/ioutil"
+	"math/rand"
 	"strconv"
 	"testing"
 
@@ -70,6 +72,95 @@ func Test_float_as_int(t *testing.T) {
 	should.NotNil(jsoniter.Unmarshal([]byte(`1.1`), &i))
 }
 
+// chunkedData is io.Reader which returns random amount of data in range [1, chunkedData.chunkSize].
+// It simulates chunked data on from HTTP server, which is commonly used by net/http package.
+type chunkedData struct {
+	chunkSize int
+	data      []byte
+	head      int
+}
+
+// Read is implementation of the io.Reader which returns random amount of data in range [1, chunkedData.chunkSize].
+func (c *chunkedData) Read(p []byte) (n int, err error) {
+	to := c.head + int(rand.Int31n(int32(c.chunkSize))+1)
+
+	// copy does not copy more data then p can consume
+	n = copy(p, c.data[c.head:to])
+	c.head = c.head + n
+	if c.head >= len(c.data) {
+		err = io.EOF
+	}
+	return n, err
+}
+
+// TestIterator_ReadInt_chunkedInput validates the behaviour of Iterator.ReadInt() method in where:
+// - it reads data from io.Reader,
+// - expected value is 0 (zero)
+// - Iterator.tail == Iterator.head
+// - Iterator.tail < len(Iterator.buf)
+// - value in buffer after Iterator.tail is presented from previous read and has '.' character.
+func TestIterator_ReadInt_chunkedInput(t *testing.T) {
+	should := require.New(t)
+
+	data := &chunkedData{
+		data: jsonFloatIntArray(t, 10),
+	}
+
+	// because this test is rely on randomness of chunkedData, we are doing multiple iterations to
+	// be sure, that we can hit a required case.
+	for data.chunkSize = 3; data.chunkSize <= len(data.data); data.chunkSize++ {
+		data.head = 0
+
+		iter := jsoniter.Parse(jsoniter.ConfigDefault, data, data.chunkSize)
+		i := 0
+		for iter.ReadArray() {
+			// every even item is float, let's just skip it.
+			if i%2 == 0 {
+				iter.Skip()
+				i++
+				continue
+			}
+
+			should.Zero(iter.ReadInt())
+			should.NoError(iter.Error)
+
+			i++
+		}
+	}
+}
+
+// jsonFloatIntArray generates JSON array where every
+//  - even item is float 0.1
+//  - odd item is integer 0
+//
+//  [0.1, 0, 0.1, 0]
+func jsonFloatIntArray(t *testing.T, numberOfItems int) []byte {
+	t.Helper()
+	numbers := make([]jsoniter.Any, numberOfItems)
+	for i := range numbers {
+		switch i % 2 {
+		case 0:
+			numbers[i] = jsoniter.WrapFloat64(0.1)
+		default:
+			numbers[i] = jsoniter.WrapInt64(0)
+		}
+	}
+
+	fixture, err := jsoniter.ConfigFastest.Marshal(numbers)
+	if err != nil {
+		panic(err)
+	}
+
+	b := &bytes.Buffer{}
+
+	require.NoError(
+		t,
+		json.Compact(b, fixture),
+		"json should be compactable",
+	)
+	return b.Bytes()
+}
+
 func Benchmark_jsoniter_encode_int(b *testing.B) {
 	stream := jsoniter.NewStream(jsoniter.ConfigDefault, ioutil.Discard, 64)
 	for n := 0; n < b.N; n++ {

misc_tests/jsoniter_raw_message_test.go

@@ -42,6 +42,25 @@ func Test_marshal_invalid_json_raw_message(t *testing.T) {
 	should.Nil(aouterr)
 }
 
+func Test_marshal_nil_json_raw_message(t *testing.T) {
+	type A struct {
+		Nil1 jsoniter.RawMessage `json:"raw1"`
+		Nil2 json.RawMessage     `json:"raw2"`
+	}
+
+	a := A{}
+	should := require.New(t)
+	aout, aouterr := jsoniter.Marshal(&a)
+	should.Equal(`{"raw1":null,"raw2":null}`, string(aout))
+	should.Nil(aouterr)
+
+	a.Nil1 = []byte(`Any`)
+	a.Nil2 = []byte(`Any`)
+	should.Nil(jsoniter.Unmarshal(aout, &a))
+	should.Nil(a.Nil1)
+	should.Nil(a.Nil2)
+}
+
 func Test_raw_message_memory_not_copied_issue(t *testing.T) {
 	jsonStream := `{"name":"xxxxx","bundle_id":"com.zonst.majiang","app_platform":"ios","app_category":"100103", "budget_day":1000,"bidding_min":1,"bidding_max":2,"bidding_type":"CPM", "freq":{"open":true,"type":"day","num":100},"speed":1, "targeting":{"vendor":{"open":true,"list":["zonst"]}, "geo_code":{"open":true,"list":["156110100"]},"app_category":{"open":true,"list":["100101"]}, "day_parting":{"open":true,"list":["100409","100410"]},"device_type":{"open":true,"list":["ipad"]}, "os_version":{"open":true,"list":[10]},"carrier":{"open":true,"list":["mobile"]}, "network":{"open":true,"list":["4G"]}},"url":{"tracking_imp_url":"http://www.baidu.com", "tracking_clk_url":"http://www.baidu.com","jump_url":"http://www.baidu.com","deep_link_url":"http://www.baidu.com"}}`
 	type IteratorObject struct {

reflect.go

@@ -65,7 +65,7 @@ func (iter *Iterator) ReadVal(obj interface{}) {
 	decoder := iter.cfg.getDecoderFromCache(cacheKey)
 	if decoder == nil {
 		typ := reflect2.TypeOf(obj)
-		if typ.Kind() != reflect.Ptr {
+		if typ == nil || typ.Kind() != reflect.Ptr {
 			iter.ReportError("ReadVal", "can only unmarshal into pointer")
 			return
 		}

reflect_json_raw_message.go

@@ -33,11 +33,19 @@ type jsonRawMessageCodec struct {
 }
 
 func (codec *jsonRawMessageCodec) Decode(ptr unsafe.Pointer, iter *Iterator) {
-	*((*json.RawMessage)(ptr)) = json.RawMessage(iter.SkipAndReturnBytes())
+	if iter.ReadNil() {
+		*((*json.RawMessage)(ptr)) = nil
+	} else {
+		*((*json.RawMessage)(ptr)) = iter.SkipAndReturnBytes()
+	}
 }
 
 func (codec *jsonRawMessageCodec) Encode(ptr unsafe.Pointer, stream *Stream) {
-	stream.WriteRaw(string(*((*json.RawMessage)(ptr))))
+	if *((*json.RawMessage)(ptr)) == nil {
+		stream.WriteNil()
+	} else {
+		stream.WriteRaw(string(*((*json.RawMessage)(ptr))))
+	}
 }
 
 func (codec *jsonRawMessageCodec) IsEmpty(ptr unsafe.Pointer) bool {
@@ -48,11 +56,19 @@ type jsoniterRawMessageCodec struct {
 }
 
 func (codec *jsoniterRawMessageCodec) Decode(ptr unsafe.Pointer, iter *Iterator) {
-	*((*RawMessage)(ptr)) = RawMessage(iter.SkipAndReturnBytes())
+	if iter.ReadNil() {
+		*((*RawMessage)(ptr)) = nil
+	} else {
+		*((*RawMessage)(ptr)) = iter.SkipAndReturnBytes()
+	}
 }
 
 func (codec *jsoniterRawMessageCodec) Encode(ptr unsafe.Pointer, stream *Stream) {
-	stream.WriteRaw(string(*((*RawMessage)(ptr))))
+	if *((*RawMessage)(ptr)) == nil {
+		stream.WriteNil()
+	} else {
+		stream.WriteRaw(string(*((*RawMessage)(ptr))))
+	}
 }
 
 func (codec *jsoniterRawMessageCodec) IsEmpty(ptr unsafe.Pointer) bool {

reflect_struct_decoder.go

@@ -1075,6 +1075,11 @@ type stringModeNumberDecoder struct {
 }
 
 func (decoder *stringModeNumberDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) {
+	if iter.WhatIsNext() == NilValue {
+		decoder.elemDecoder.Decode(ptr, iter)
+		return
+	}
+
 	c := iter.nextToken()
 	if c != '"' {
 		iter.ReportError("stringModeNumberDecoder", `expect ", but found `+string([]byte{c}))

value_tests/struct_test.go

@@ -26,6 +26,11 @@ func init() {
 			Field int `json:"field"`
 		})(nil),
 		input: `{"field": null}`,
+	}, unmarshalCase{
+		ptr: (*struct {
+			Field int `json:"field,string"`
+		})(nil),
+		input: `{"field": null}`,
 	}, unmarshalCase{
 		ptr: (*struct {
 			ID      int                    `json:"id"`