Merge pull request #373 from ernado/append-skip

fix #372: add AppendSkip iterator method

adapter.go

@@ -16,7 +16,7 @@ func Unmarshal(data []byte, v interface{}) error {
 	return ConfigDefault.Unmarshal(data, v)
 }
 
-// UnmarshalFromString convenient method to read from string instead of []byte
+// UnmarshalFromString is a convenient method to read from string instead of []byte
 func UnmarshalFromString(str string, v interface{}) error {
 	return ConfigDefault.UnmarshalFromString(str, v)
 }

go.mod

@@ -0,0 +1,11 @@
+module github.com/json-iterator/go
+
+go 1.12
+
+require (
+	github.com/davecgh/go-spew v1.1.1
+	github.com/google/gofuzz v1.0.0
+	github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421
+	github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742
+	github.com/stretchr/testify v1.3.0
+)

go.sum

@@ -0,0 +1,14 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=

iter_skip.go

@@ -37,17 +37,24 @@ func (iter *Iterator) SkipAndReturnBytes() []byte {
 	return iter.stopCapture()
 }
 
-type captureBuffer struct {
-	startedAt int
-	captured  []byte
+// SkipAndAppendBytes skips next JSON element and appends its content to
+// buffer, returning the result.
+func (iter *Iterator) SkipAndAppendBytes(buf []byte) []byte {
+	iter.startCaptureTo(buf, iter.head)
+	iter.Skip()
+	return iter.stopCapture()
 }
 
-func (iter *Iterator) startCapture(captureStartedAt int) {
+func (iter *Iterator) startCaptureTo(buf []byte, captureStartedAt int) {
 	if iter.captured != nil {
 		panic("already in capture mode")
 	}
 	iter.captureStartedAt = captureStartedAt
-	iter.captured = make([]byte, 0, 32)
+	iter.captured = buf
+}
+
+func (iter *Iterator) startCapture(captureStartedAt int) {
+	iter.startCaptureTo(make([]byte, 0, 32), captureStartedAt)
 }
 
 func (iter *Iterator) stopCapture() []byte {
@@ -58,13 +65,7 @@ func (iter *Iterator) stopCapture() []byte {
 	remaining := iter.buf[iter.captureStartedAt:iter.head]
 	iter.captureStartedAt = -1
 	iter.captured = nil
-	if len(captured) == 0 {
-		copied := make([]byte, len(remaining))
-		copy(copied, remaining)
-		return copied
-	}
-	captured = append(captured, remaining...)
-	return captured
+	return append(captured, remaining...)
 }
 
 // Skip skips a json object and positions to relatively the next json object

misc_tests/jsoniter_array_test.go

@@ -158,6 +158,27 @@ func Test_encode_byte_array(t *testing.T) {
 	should.Equal(`"AQID"`, string(bytes))
 }
 
+func Test_encode_empty_byte_array(t *testing.T) {
+	should := require.New(t)
+	bytes, err := json.Marshal([]byte{})
+	should.Nil(err)
+	should.Equal(`""`, string(bytes))
+	bytes, err = jsoniter.Marshal([]byte{})
+	should.Nil(err)
+	should.Equal(`""`, string(bytes))
+}
+
+func Test_encode_nil_byte_array(t *testing.T) {
+	should := require.New(t)
+	var nilSlice []byte
+	bytes, err := json.Marshal(nilSlice)
+	should.Nil(err)
+	should.Equal(`null`, string(bytes))
+	bytes, err = jsoniter.Marshal(nilSlice)
+	should.Nil(err)
+	should.Equal(`null`, string(bytes))
+}
+
 func Test_decode_byte_array_from_base64(t *testing.T) {
 	should := require.New(t)
 	data := []byte{}

misc_tests/jsoniter_float_test.go

@@ -2,6 +2,7 @@ package misc_tests
 
 import (
 	"encoding/json"
+	"math"
 	"testing"
 
 	"github.com/json-iterator/go"
@@ -77,6 +78,26 @@ func Test_read_number(t *testing.T) {
 	should.Equal(`92233720368547758079223372036854775807`, string(val))
 }
 
+func Test_encode_inf(t *testing.T) {
+	should := require.New(t)
+	_, err := json.Marshal(math.Inf(1))
+	should.Error(err)
+	_, err = jsoniter.Marshal(float32(math.Inf(1)))
+	should.Error(err)
+	_, err = jsoniter.Marshal(math.Inf(-1))
+	should.Error(err)
+}
+
+func Test_encode_nan(t *testing.T) {
+	should := require.New(t)
+	_, err := json.Marshal(math.NaN())
+	should.Error(err)
+	_, err = jsoniter.Marshal(float32(math.NaN()))
+	should.Error(err)
+	_, err = jsoniter.Marshal(math.NaN())
+	should.Error(err)
+}
+
 func Benchmark_jsoniter_float(b *testing.B) {
 	b.ReportAllocs()
 	input := []byte(`1.1123,`)

reflect_native.go

@@ -432,17 +432,19 @@ func (codec *base64Codec) Decode(ptr unsafe.Pointer, iter *Iterator) {
 }
 
 func (codec *base64Codec) Encode(ptr unsafe.Pointer, stream *Stream) {
-	src := *((*[]byte)(ptr))
-	if len(src) == 0 {
+	if codec.sliceType.UnsafeIsNil(ptr) {
 		stream.WriteNil()
 		return
 	}
+	src := *((*[]byte)(ptr))
 	encoding := base64.StdEncoding
 	stream.writeByte('"')
+	if len(src) != 0 {
 		size := encoding.EncodedLen(len(src))
 		buf := make([]byte, size)
 		encoding.Encode(buf, src)
 		stream.buf = append(stream.buf, buf...)
+	}
 	stream.writeByte('"')
 }
 

reflect_struct_decoder.go

@@ -530,8 +530,8 @@ func (decoder *generalStructDecoder) decodeOneField(ptr unsafe.Pointer, iter *It
 		}
 	}
 	if fieldDecoder == nil {
-		msg := "found unknown field: " + field
 		if decoder.disallowUnknownFields {
+			msg := "found unknown field: " + field
 			iter.ReportError("ReadObject", msg)
 		}
 		c := iter.nextToken()

skip_tests/jsoniter_skip_test.go

@@ -105,6 +105,15 @@ func Test_skip_and_return_bytes_with_reader(t *testing.T) {
 	should.Equal(`{"a" : [{"stream": "c"}], "d": 102 }`, string(skipped))
 }
 
+func Test_append_skip_and_return_bytes_with_reader(t *testing.T) {
+	should := require.New(t)
+	iter := jsoniter.Parse(jsoniter.ConfigDefault, bytes.NewBufferString(`[ {"a" : [{"stream": "c"}], "d": 102 }, "stream"]`), 4)
+	iter.ReadArray()
+	buf := make([]byte, 0, 1024)
+	buf = iter.SkipAndAppendBytes(buf)
+	should.Equal(`{"a" : [{"stream": "c"}], "d": 102 }`, string(buf))
+}
+
 func Test_skip_empty(t *testing.T) {
 	should := require.New(t)
 	should.NotNil(jsoniter.Get([]byte("")).LastError())

stream_float.go

@@ -1,6 +1,7 @@
 package jsoniter
 
 import (
+	"fmt"
 	"math"
 	"strconv"
 )
@@ -13,6 +14,10 @@ func init() {
 
 // WriteFloat32 write float32 to stream
 func (stream *Stream) WriteFloat32(val float32) {
+	if math.IsInf(float64(val), 0) || math.IsNaN(float64(val)) {
+		stream.Error = fmt.Errorf("unsupported value: %f", val)
+		return
+	}
 	abs := math.Abs(float64(val))
 	fmt := byte('f')
 	// Note: Must use float32 comparisons for underlying float32 value to get precise cutoffs right.
@@ -26,6 +31,10 @@ func (stream *Stream) WriteFloat32(val float32) {
 
 // WriteFloat32Lossy write float32 to stream with ONLY 6 digits precision although much much faster
 func (stream *Stream) WriteFloat32Lossy(val float32) {
+	if math.IsInf(float64(val), 0) || math.IsNaN(float64(val)) {
+		stream.Error = fmt.Errorf("unsupported value: %f", val)
+		return
+	}
 	if val < 0 {
 		stream.writeByte('-')
 		val = -val
@@ -54,6 +63,10 @@ func (stream *Stream) WriteFloat32Lossy(val float32) {
 
 // WriteFloat64 write float64 to stream
 func (stream *Stream) WriteFloat64(val float64) {
+	if math.IsInf(val, 0) || math.IsNaN(val) {
+		stream.Error = fmt.Errorf("unsupported value: %f", val)
+		return
+	}
 	abs := math.Abs(val)
 	fmt := byte('f')
 	// Note: Must use float32 comparisons for underlying float32 value to get precise cutoffs right.
@@ -67,6 +80,10 @@ func (stream *Stream) WriteFloat64(val float64) {
 
 // WriteFloat64Lossy write float64 to stream with ONLY 6 digits precision although much much faster
 func (stream *Stream) WriteFloat64Lossy(val float64) {
+	if math.IsInf(val, 0) || math.IsNaN(val) {
+		stream.Error = fmt.Errorf("unsupported value: %f", val)
+		return
+	}
 	if val < 0 {
 		stream.writeByte('-')
 		val = -val