diff --git a/feature_adapter.go b/feature_adapter.go index cde2660..04c7b16 100644 --- a/feature_adapter.go +++ b/feature_adapter.go @@ -3,6 +3,7 @@ package jsoniter import ( "io" "unsafe" + "bytes" ) // Unmarshal adapts to json/encoding APIs @@ -15,7 +16,8 @@ func Unmarshal(data []byte, v interface{}) error { return iter.Error } -func UnmarshalString(str string, v interface{}) error { +func UnmarshalFromString(str string, v interface{}) error { + // safe to do the unsafe cast here, as str is always referenced in this scope data := *(*[]byte)(unsafe.Pointer(&str)) iter := ParseBytes(data) iter.Read(v) @@ -24,3 +26,21 @@ func UnmarshalString(str string, v interface{}) error { } return iter.Error } + +func Marshal(v interface{}) ([]byte, error) { + buf := &bytes.Buffer{} + stream := NewStream(buf, 4096) + stream.WriteVal(v) + if stream.Error != nil { + return nil, stream.Error + } + return buf.Bytes(), nil +} + +func MarshalToString(v interface{}) (string, error) { + buf, err := Marshal(v) + if err != nil { + return "", err + } + return string(buf), nil +} \ No newline at end of file diff --git a/jsoniter.go b/iterator.go similarity index 98% rename from jsoniter.go rename to iterator.go index d1c316d..b400626 100644 --- a/jsoniter.go +++ b/iterator.go @@ -21,22 +21,22 @@ const ( Object ) -var digits []byte +var atoiDigits []byte var valueTypes []ValueType func init() { - digits = make([]byte, 256) - for i := 0; i < len(digits); i++ { - digits[i] = 255 + atoiDigits = make([]byte, 256) + for i := 0; i < len(atoiDigits); i++ { + atoiDigits[i] = 255 } for i := '0'; i <= '9'; i++ { - digits[i] = byte(i - '0') + atoiDigits[i] = byte(i - '0') } for i := 'a'; i <= 'f'; i++ { - digits[i] = byte((i - 'a') + 10) + atoiDigits[i] = byte((i - 'a') + 10) } for i := 'A'; i <= 'F'; i++ { - digits[i] = byte((i - 'A') + 10) + atoiDigits[i] = byte((i - 'A') + 10) } valueTypes = make([]ValueType, 256) for i := 0; i < len(valueTypes); i++ { @@ -278,7 +278,7 @@ func (iter *Iterator) ReadUint32() (ret uint32) { // ReadUint64 reads a json object as Uint64 func (iter *Iterator) ReadUint64() (ret uint64) { c := iter.nextToken() - v := digits[c] + v := atoiDigits[c] if v == 0 { return 0 // single zero } @@ -293,7 +293,7 @@ func (iter *Iterator) ReadUint64() (ret uint64) { } ret = ret*10 + uint64(v) c = iter.readByte() - v = digits[c] + v = atoiDigits[c] if v == 255 { iter.unreadByte() break diff --git a/jsoniter_int_test.go b/jsoniter_int_test.go index 6695a3d..520e77b 100644 --- a/jsoniter_int_test.go +++ b/jsoniter_int_test.go @@ -4,9 +4,12 @@ import ( "bytes" "encoding/json" "testing" + "github.com/json-iterator/go/require" + "fmt" + "strconv" ) -func Test_uint64_0(t *testing.T) { +func Test_decode_decode_uint64_0(t *testing.T) { iter := Parse(bytes.NewBufferString("0"), 4096) val := iter.ReadUint64() if iter.Error != nil { @@ -17,7 +20,7 @@ func Test_uint64_0(t *testing.T) { } } -func Test_uint64_1(t *testing.T) { +func Test_decode_uint64_1(t *testing.T) { iter := Parse(bytes.NewBufferString("1"), 4096) val := iter.ReadUint64() if val != 1 { @@ -25,7 +28,7 @@ func Test_uint64_1(t *testing.T) { } } -func Test_uint64_100(t *testing.T) { +func Test_decode_uint64_100(t *testing.T) { iter := Parse(bytes.NewBufferString("100"), 4096) val := iter.ReadUint64() if val != 100 { @@ -33,7 +36,7 @@ func Test_uint64_100(t *testing.T) { } } -func Test_uint64_100_comma(t *testing.T) { +func Test_decode_uint64_100_comma(t *testing.T) { iter := Parse(bytes.NewBufferString("100,"), 4096) val := iter.ReadUint64() if iter.Error != nil { @@ -44,7 +47,7 @@ func Test_uint64_100_comma(t *testing.T) { } } -func Test_uint64_invalid(t *testing.T) { +func Test_decode_uint64_invalid(t *testing.T) { iter := Parse(bytes.NewBufferString(","), 4096) iter.ReadUint64() if iter.Error == nil { @@ -52,7 +55,7 @@ func Test_uint64_invalid(t *testing.T) { } } -func Test_int64_100(t *testing.T) { +func Test_decode_int64_100(t *testing.T) { iter := Parse(bytes.NewBufferString("100"), 4096) val := iter.ReadInt64() if val != 100 { @@ -60,7 +63,7 @@ func Test_int64_100(t *testing.T) { } } -func Test_int64_minus_100(t *testing.T) { +func Test_decode_int64_minus_100(t *testing.T) { iter := Parse(bytes.NewBufferString("-100"), 4096) val := iter.ReadInt64() if val != -100 { @@ -68,6 +71,75 @@ func Test_int64_minus_100(t *testing.T) { } } +func Test_write_uint8(t *testing.T) { + vals := []uint8{0, 1, 11, 111, 255} + for _, val := range vals { + t.Run(fmt.Sprintf("%v", val), func(t *testing.T) { + should := require.New(t) + buf := &bytes.Buffer{} + stream := NewStream(buf, 4096) + stream.WriteUint8(val) + stream.Flush() + should.Nil(stream.Error) + should.Equal(strconv.Itoa(int(val)), buf.String()) + }) + } + should := require.New(t) + buf := &bytes.Buffer{} + stream := NewStream(buf, 3) + stream.WriteString("a") + stream.WriteUint8(100) // should clear buffer + stream.Flush() + should.Nil(stream.Error) + should.Equal("a100", buf.String()) +} + +func Test_write_int8(t *testing.T) { + vals := []int8{0, 1, -1, 99, 0x7f, -0x7f} + for _, val := range vals { + t.Run(fmt.Sprintf("%v", val), func(t *testing.T) { + should := require.New(t) + buf := &bytes.Buffer{} + stream := NewStream(buf, 4096) + stream.WriteInt8(val) + stream.Flush() + should.Nil(stream.Error) + should.Equal(strconv.Itoa(int(val)), buf.String()) + }) + } + should := require.New(t) + buf := &bytes.Buffer{} + stream := NewStream(buf, 4) + stream.WriteString("a") + stream.WriteInt8(-100) // should clear buffer + stream.Flush() + should.Nil(stream.Error) + should.Equal("a-100", buf.String()) +} + +func Test_write_uint16(t *testing.T) { + vals := []uint16{0, 1, 11, 111, 255, 0xfff, 0xffff} + for _, val := range vals { + t.Run(fmt.Sprintf("%v", val), func(t *testing.T) { + should := require.New(t) + buf := &bytes.Buffer{} + stream := NewStream(buf, 4096) + stream.WriteUint16(val) + stream.Flush() + should.Nil(stream.Error) + should.Equal(strconv.Itoa(int(val)), buf.String()) + }) + } + should := require.New(t) + buf := &bytes.Buffer{} + stream := NewStream(buf, 5) + stream.WriteString("a") + stream.WriteUint16(10000) // should clear buffer + stream.Flush() + should.Nil(stream.Error) + should.Equal("a10000", buf.String()) +} + func Benchmark_jsoniter_int(b *testing.B) { for n := 0; n < b.N; n++ { iter := ParseString(`-100`) diff --git a/jsoniter_null_test.go b/jsoniter_null_test.go index 5e15831..56fac2e 100644 --- a/jsoniter_null_test.go +++ b/jsoniter_null_test.go @@ -2,16 +2,35 @@ package jsoniter import ( "testing" + "github.com/json-iterator/go/require" + "bytes" ) -func Test_null(t *testing.T) { +func Test_decode_null(t *testing.T) { iter := ParseString(`null`) if iter.ReadNil() != true { t.FailNow() } } -func Test_null_object(t *testing.T) { +func Test_write_null(t *testing.T) { + should := require.New(t) + buf := &bytes.Buffer{} + stream := NewStream(buf, 4096) + stream.WriteNull() + stream.Flush() + should.Nil(stream.Error) + should.Equal("null", buf.String()) +} + +func Test_encode_null(t *testing.T) { + should := require.New(t) + str, err := MarshalToString(nil) + should.Nil(err) + should.Equal("null", str) +} + +func Test_decode_null_object(t *testing.T) { iter := ParseString(`[null,"a"]`) iter.ReadArray() if iter.ReadObject() != "" { @@ -23,7 +42,7 @@ func Test_null_object(t *testing.T) { } } -func Test_null_array(t *testing.T) { +func Test_decode_null_array(t *testing.T) { iter := ParseString(`[null,"a"]`) iter.ReadArray() if iter.ReadArray() != false { @@ -35,7 +54,7 @@ func Test_null_array(t *testing.T) { } } -func Test_null_string(t *testing.T) { +func Test_decode_null_string(t *testing.T) { iter := ParseString(`[null,"a"]`) iter.ReadArray() if iter.ReadString() != "" { @@ -47,7 +66,7 @@ func Test_null_string(t *testing.T) { } } -func Test_null_skip(t *testing.T) { +func Test_decode_null_skip(t *testing.T) { iter := ParseString(`[null,"a"]`) iter.ReadArray() iter.Skip() diff --git a/jsoniter_reflect_struct_test.go b/jsoniter_reflect_struct_test.go index 6e3be9f..3130b70 100644 --- a/jsoniter_reflect_struct_test.go +++ b/jsoniter_reflect_struct_test.go @@ -11,9 +11,9 @@ func Test_decode_one_field_struct(t *testing.T) { field1 string } obj := TestObject{} - should.Nil(UnmarshalString(`{}`, &obj)) + should.Nil(UnmarshalFromString(`{}`, &obj)) should.Equal("", obj.field1) - should.Nil(UnmarshalString(`{"field1": "hello"}`, &obj)) + should.Nil(UnmarshalFromString(`{"field1": "hello"}`, &obj)) should.Equal("hello", obj.field1) } @@ -24,9 +24,9 @@ func Test_decode_two_fields_struct(t *testing.T) { field2 string } obj := TestObject{} - should.Nil(UnmarshalString(`{}`, &obj)) + should.Nil(UnmarshalFromString(`{}`, &obj)) should.Equal("", obj.field1) - should.Nil(UnmarshalString(`{"field1": "a", "field2": "b"}`, &obj)) + should.Nil(UnmarshalFromString(`{"field1": "a", "field2": "b"}`, &obj)) should.Equal("a", obj.field1) should.Equal("b", obj.field2) } @@ -39,9 +39,9 @@ func Test_decode_three_fields_struct(t *testing.T) { field3 string } obj := TestObject{} - should.Nil(UnmarshalString(`{}`, &obj)) + should.Nil(UnmarshalFromString(`{}`, &obj)) should.Equal("", obj.field1) - should.Nil(UnmarshalString(`{"field1": "a", "field2": "b", "field3": "c"}`, &obj)) + should.Nil(UnmarshalFromString(`{"field1": "a", "field2": "b", "field3": "c"}`, &obj)) should.Equal("a", obj.field1) should.Equal("b", obj.field2) should.Equal("c", obj.field3) @@ -56,9 +56,9 @@ func Test_decode_four_fields_struct(t *testing.T) { field4 string } obj := TestObject{} - should.Nil(UnmarshalString(`{}`, &obj)) + should.Nil(UnmarshalFromString(`{}`, &obj)) should.Equal("", obj.field1) - should.Nil(UnmarshalString(`{"field1": "a", "field2": "b", "field3": "c", "field4": "d"}`, &obj)) + should.Nil(UnmarshalFromString(`{"field1": "a", "field2": "b", "field3": "c", "field4": "d"}`, &obj)) should.Equal("a", obj.field1) should.Equal("b", obj.field2) should.Equal("c", obj.field3) @@ -75,9 +75,9 @@ func Test_decode_five_fields_struct(t *testing.T) { field5 string } obj := TestObject{} - should.Nil(UnmarshalString(`{}`, &obj)) + should.Nil(UnmarshalFromString(`{}`, &obj)) should.Equal("", obj.field1) - should.Nil(UnmarshalString(`{"field1": "a", "field2": "b", "field3": "c", "field4": "d", "field5": "e"}`, &obj)) + should.Nil(UnmarshalFromString(`{"field1": "a", "field2": "b", "field3": "c", "field4": "d", "field5": "e"}`, &obj)) should.Equal("a", obj.field1) should.Equal("b", obj.field2) should.Equal("c", obj.field3) @@ -92,7 +92,7 @@ func Test_decode_struct_with_optional_field(t *testing.T) { field2 *string } obj := TestObject{} - UnmarshalString(`{"field1": null, "field2": "world"}`, &obj) + UnmarshalFromString(`{"field1": null, "field2": "world"}`, &obj) should.Nil(obj.field1) should.Equal("world", *obj.field2) } @@ -105,7 +105,7 @@ func Test_decode_struct_field_with_tag(t *testing.T) { Field3 int `json:",string"` } obj := TestObject{Field2: "world"} - UnmarshalString(`{"field-1": "hello", "field2": "", "Field3": "100"}`, &obj) + UnmarshalFromString(`{"field-1": "hello", "field2": "", "Field3": "100"}`, &obj) should.Equal("hello", obj.Field1) should.Equal("world", obj.Field2) should.Equal(100, obj.Field3) diff --git a/jsoniter_reflect_test.go b/jsoniter_reflect_test.go index 28dc99c..06048c7 100644 --- a/jsoniter_reflect_test.go +++ b/jsoniter_reflect_test.go @@ -11,14 +11,14 @@ import ( func Test_decode_slice(t *testing.T) { should := require.New(t) slice := make([]string, 0, 5) - UnmarshalString(`["hello", "world"]`, &slice) + UnmarshalFromString(`["hello", "world"]`, &slice) should.Equal([]string{"hello", "world"}, slice) } func Test_decode_large_slice(t *testing.T) { should := require.New(t) slice := make([]int, 0, 1) - UnmarshalString(`[1,2,3,4,5,6,7,8,9]`, &slice) + UnmarshalFromString(`[1,2,3,4,5,6,7,8,9]`, &slice) should.Equal([]int{1, 2, 3, 4, 5, 6, 7, 8, 9}, slice) } diff --git a/jsoniter_string_test.go b/jsoniter_string_test.go index 5fa042e..2660eff 100644 --- a/jsoniter_string_test.go +++ b/jsoniter_string_test.go @@ -4,9 +4,10 @@ import ( "bytes" "encoding/json" "testing" + "github.com/json-iterator/go/require" ) -func Test_string_empty(t *testing.T) { +func Test_decode_string_empty(t *testing.T) { iter := Parse(bytes.NewBufferString(`""`), 4096) val := iter.ReadString() if iter.Error != nil { @@ -17,7 +18,7 @@ func Test_string_empty(t *testing.T) { } } -func Test_string_hello(t *testing.T) { +func Test_decode_string_hello(t *testing.T) { iter := Parse(bytes.NewBufferString(`"hello"`), 4096) val := iter.ReadString() if iter.Error != nil { @@ -28,7 +29,7 @@ func Test_string_hello(t *testing.T) { } } -func Test_string_escape_quote(t *testing.T) { +func Test_decode_string_escape_quote(t *testing.T) { iter := Parse(bytes.NewBufferString(`"hel\"lo"`), 4096) val := iter.ReadString() if iter.Error != nil { @@ -39,7 +40,7 @@ func Test_string_escape_quote(t *testing.T) { } } -func Test_string_escape_newline(t *testing.T) { +func Test_decode_string_escape_newline(t *testing.T) { iter := Parse(bytes.NewBufferString(`"hel\nlo"`), 4096) val := iter.ReadString() if iter.Error != nil { @@ -50,7 +51,7 @@ func Test_string_escape_newline(t *testing.T) { } } -func Test_string_escape_unicode(t *testing.T) { +func Test_decode_string_escape_unicode(t *testing.T) { iter := Parse(bytes.NewBufferString(`"\u4e2d\u6587"`), 4096) val := iter.ReadString() if iter.Error != nil { @@ -61,7 +62,7 @@ func Test_string_escape_unicode(t *testing.T) { } } -func Test_string_escape_unicode_with_surrogate(t *testing.T) { +func Test_decode_string_escape_unicode_with_surrogate(t *testing.T) { iter := Parse(bytes.NewBufferString(`"\ud83d\udc4a"`), 4096) val := iter.ReadString() if iter.Error != nil { @@ -72,7 +73,7 @@ func Test_string_escape_unicode_with_surrogate(t *testing.T) { } } -func Test_string_as_bytes(t *testing.T) { +func Test_decode_string_as_bytes(t *testing.T) { iter := Parse(bytes.NewBufferString(`"hello""world"`), 4096) val := string(iter.readStringAsBytes()) if val != "hello" { @@ -84,6 +85,16 @@ func Test_string_as_bytes(t *testing.T) { } } +func Test_write_string(t *testing.T) { + should := require.New(t) + buf := &bytes.Buffer{} + stream := NewStream(buf, 4096) + stream.WriteString("hello") + stream.Flush() + should.Nil(stream.Error) + should.Equal("hello", buf.String()) +} + func Benchmark_jsoniter_unicode(b *testing.B) { for n := 0; n < b.N; n++ { iter := ParseString(`"\ud83d\udc4a"`) diff --git a/stream.go b/stream.go new file mode 100644 index 0000000..96ac48d --- /dev/null +++ b/stream.go @@ -0,0 +1,229 @@ +package jsoniter + +import ( + "io" +) + +var bytesNull []byte +var digits []uint8; + +func init() { + bytesNull = []byte("null") + digits = []uint8{ + '0', '1', '2', '3', '4', '5', + '6', '7', '8', '9', 'a', 'b', + 'c', 'd', 'e', 'f', 'g', 'h', + 'i', 'j', 'k', 'l', 'm', 'n', + 'o', 'p', 'q', 'r', 's', 't', + 'u', 'v', 'w', 'x', 'y', 'z', + } +} + +type Stream struct { + out io.Writer + buf []byte + n int + Error error +} + +func NewStream(out io.Writer, bufSize int) *Stream { + return &Stream{out, make([]byte, bufSize), 0, nil} +} + + +// Available returns how many bytes are unused in the buffer. +func (b *Stream) Available() int { + return len(b.buf) - b.n +} + +// Buffered returns the number of bytes that have been written into the current buffer. +func (b *Stream) Buffered() int { + return b.n +} + +// Write writes the contents of p into the buffer. +// It returns the number of bytes written. +// If nn < len(p), it also returns an error explaining +// why the write is short. +func (b *Stream) Write(p []byte) (nn int, err error) { + for len(p) > b.Available() && b.Error == nil { + var n int + if b.Buffered() == 0 { + // Large write, empty buffer. + // Write directly from p to avoid copy. + n, b.Error = b.out.Write(p) + } else { + n = copy(b.buf[b.n:], p) + b.n += n + b.Flush() + } + nn += n + p = p[n:] + } + if b.Error != nil { + return nn, b.Error + } + n := copy(b.buf[b.n:], p) + b.n += n + nn += n + return nn, nil +} + + +// WriteByte writes a single byte. +func (b *Stream) WriteByte(c byte) error { + if b.Error != nil { + return b.Error + } + if b.Available() <= 0 && b.Flush() != nil { + return b.Error + } + b.buf[b.n] = c + b.n++ + return nil +} + +// Flush writes any buffered data to the underlying io.Writer. +func (b *Stream) Flush() error { + if b.Error != nil { + return b.Error + } + if b.n == 0 { + return nil + } + n, err := b.out.Write(b.buf[0:b.n]) + if n < b.n && err == nil { + err = io.ErrShortWrite + } + if err != nil { + if n > 0 && n < b.n { + copy(b.buf[0:b.n - n], b.buf[n:b.n]) + } + b.n -= n + b.Error = err + return err + } + b.n = 0 + return nil +} + +func (b *Stream) WriteString(s string) { + for len(s) > b.Available() && b.Error == nil { + n := copy(b.buf[b.n:], s) + b.n += n + s = s[n:] + b.Flush() + } + if b.Error != nil { + return + } + n := copy(b.buf[b.n:], s) + b.n += n +} + +func (stream *Stream) WriteNull() { + stream.Write(bytesNull) +} + +func (stream *Stream) WriteUint8(val uint8) { + if stream.Available() < 3 { + stream.Flush() + } + charPos := stream.n + if val <= 9 { + charPos += 1; + } else { + if val <= 99 { + charPos += 2; + } else { + charPos += 3; + } + } + stream.n = charPos + var q uint8 + var r uint8 + for { + q = val / 10 + r = val - ((q << 3) + (q << 1)) // r = i-(q*10) ... + charPos-- + stream.buf[charPos] = digits[r] + val = q; + if val == 0 { + break + } + } +} + +func (stream *Stream) WriteInt8(val int8) { + if stream.Available() < 4 { + stream.Flush() + } + charPos := stream.n + if (val < 0) { + charPos += 1 + val = -val + stream.buf[stream.n] = '-' + } + if val <= 9 { + charPos += 1; + } else { + if val <= 99 { + charPos += 2; + } else { + charPos += 3; + } + } + stream.n = charPos + var q int8 + var r int8 + for { + q = val / 10 + r = val - ((q << 3) + (q << 1)) // r = i-(q*10) ... + charPos-- + stream.buf[charPos] = digits[r] + val = q; + if val == 0 { + break + } + } +} + +func (stream *Stream) WriteUint16(val uint16) { + if stream.Available() < 5 { + stream.Flush() + } + charPos := stream.n + if val <= 99 { + if val <= 9 { + charPos += 1; + } else { + charPos += 2; + } + } else { + if val <= 999 { + charPos += 3; + } else { + if val <= 9999 { + charPos += 4; + } else { + charPos += 5; + } + } + } + stream.n = charPos + var q uint16 + var r uint16 + for { + q = val / 10 + r = val - ((q << 3) + (q << 1)) // r = i-(q*10) ... + charPos-- + stream.buf[charPos] = digits[r] + val = q; + if val == 0 { + break + } + } +} + +func (stream *Stream) WriteVal(val interface{}) { +} \ No newline at end of file