go/json-iterator
1
0
Fork 0
mirror of https://github.com/json-iterator/go.git synced 2024-11-24 08:22:14 +02:00
Code Issues Releases Activity

add stream

Browse Source
This commit is contained in:
Tao Wen 2017-01-07 12:28:16 +08:00
parent 5af8cc4b09
commit 6f57d41461
8 changed files with 394 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
feature_adapter.go
View File

@ -3,6 +3,7 @@ package jsoniter
import (
"io"
"unsafe"
"bytes"
)
// Unmarshal adapts to json/encoding APIs
@ -15,7 +16,8 @@ func Unmarshal(data []byte, v interface{}) error {
return iter.Error
}
func UnmarshalString(str string, v interface{}) error {
func UnmarshalFromString(str string, v interface{}) error {
// safe to do the unsafe cast here, as str is always referenced in this scope
data := *(*[]byte)(unsafe.Pointer(&str))
iter := ParseBytes(data)
iter.Read(v)
@ -24,3 +26,21 @@ func UnmarshalString(str string, v interface{}) error {
}
return iter.Error
}
func Marshal(v interface{}) ([]byte, error) {
buf := &bytes.Buffer{}
stream := NewStream(buf, 4096)
stream.WriteVal(v)
if stream.Error != nil {
return nil, stream.Error
}
return buf.Bytes(), nil
}
func MarshalToString(v interface{}) (string, error) {
buf, err := Marshal(v)
if err != nil {
return "", err
}
return string(buf), nil
}

18
jsoniter.go → iterator.go
View File

@ -21,22 +21,22 @@ const (
Object
)
var digits []byte
var atoiDigits []byte
var valueTypes []ValueType
func init() {
digits = make([]byte, 256)
for i := 0; i < len(digits); i++ {
digits[i] = 255
atoiDigits = make([]byte, 256)
for i := 0; i < len(atoiDigits); i++ {
atoiDigits[i] = 255
}
for i := '0'; i <= '9'; i++ {
digits[i] = byte(i - '0')
atoiDigits[i] = byte(i - '0')
}
for i := 'a'; i <= 'f'; i++ {
digits[i] = byte((i - 'a') + 10)
atoiDigits[i] = byte((i - 'a') + 10)
}
for i := 'A'; i <= 'F'; i++ {
digits[i] = byte((i - 'A') + 10)
atoiDigits[i] = byte((i - 'A') + 10)
}
valueTypes = make([]ValueType, 256)
for i := 0; i < len(valueTypes); i++ {
@ -278,7 +278,7 @@ func (iter *Iterator) ReadUint32() (ret uint32) {
// ReadUint64 reads a json object as Uint64
func (iter *Iterator) ReadUint64() (ret uint64) {
c := iter.nextToken()
v := digits[c]
v := atoiDigits[c]
if v == 0 {
return 0 // single zero
}
@ -293,7 +293,7 @@ func (iter *Iterator) ReadUint64() (ret uint64) {
}
ret = ret*10 + uint64(v)
c = iter.readByte()
v = digits[c]
v = atoiDigits[c]
if v == 255 {
iter.unreadByte()
break

86
jsoniter_int_test.go
View File

@ -4,9 +4,12 @@ import (
"bytes"
"encoding/json"
"testing"
"github.com/json-iterator/go/require"
"fmt"
"strconv"
)
func Test_uint64_0(t *testing.T) {
func Test_decode_decode_uint64_0(t *testing.T) {
iter := Parse(bytes.NewBufferString("0"), 4096)
val := iter.ReadUint64()
if iter.Error != nil {
@ -17,7 +20,7 @@ func Test_uint64_0(t *testing.T) {
}
}
func Test_uint64_1(t *testing.T) {
func Test_decode_uint64_1(t *testing.T) {
iter := Parse(bytes.NewBufferString("1"), 4096)
val := iter.ReadUint64()
if val != 1 {
@ -25,7 +28,7 @@ func Test_uint64_1(t *testing.T) {
}
}
func Test_uint64_100(t *testing.T) {
func Test_decode_uint64_100(t *testing.T) {
iter := Parse(bytes.NewBufferString("100"), 4096)
val := iter.ReadUint64()
if val != 100 {
@ -33,7 +36,7 @@ func Test_uint64_100(t *testing.T) {
}
}
func Test_uint64_100_comma(t *testing.T) {
func Test_decode_uint64_100_comma(t *testing.T) {
iter := Parse(bytes.NewBufferString("100,"), 4096)
val := iter.ReadUint64()
if iter.Error != nil {
@ -44,7 +47,7 @@ func Test_uint64_100_comma(t *testing.T) {
}
}
func Test_uint64_invalid(t *testing.T) {
func Test_decode_uint64_invalid(t *testing.T) {
iter := Parse(bytes.NewBufferString(","), 4096)
iter.ReadUint64()
if iter.Error == nil {
@ -52,7 +55,7 @@ func Test_uint64_invalid(t *testing.T) {
}
}
func Test_int64_100(t *testing.T) {
func Test_decode_int64_100(t *testing.T) {
iter := Parse(bytes.NewBufferString("100"), 4096)
val := iter.ReadInt64()
if val != 100 {
@ -60,7 +63,7 @@ func Test_int64_100(t *testing.T) {
}
}
func Test_int64_minus_100(t *testing.T) {
func Test_decode_int64_minus_100(t *testing.T) {
iter := Parse(bytes.NewBufferString("-100"), 4096)
val := iter.ReadInt64()
if val != -100 {
@ -68,6 +71,75 @@ func Test_int64_minus_100(t *testing.T) {
}
}
func Test_write_uint8(t *testing.T) {
vals := []uint8{0, 1, 11, 111, 255}
for _, val := range vals {
t.Run(fmt.Sprintf("%v", val), func(t *testing.T) {
should := require.New(t)
buf := &bytes.Buffer{}
stream := NewStream(buf, 4096)
stream.WriteUint8(val)
stream.Flush()
should.Nil(stream.Error)
should.Equal(strconv.Itoa(int(val)), buf.String())
})
}
should := require.New(t)
buf := &bytes.Buffer{}
stream := NewStream(buf, 3)
stream.WriteString("a")
stream.WriteUint8(100) // should clear buffer
stream.Flush()
should.Nil(stream.Error)
should.Equal("a100", buf.String())
}
func Test_write_int8(t *testing.T) {
vals := []int8{0, 1, -1, 99, 0x7f, -0x7f}
for _, val := range vals {
t.Run(fmt.Sprintf("%v", val), func(t *testing.T) {
should := require.New(t)
buf := &bytes.Buffer{}
stream := NewStream(buf, 4096)
stream.WriteInt8(val)
stream.Flush()
should.Nil(stream.Error)
should.Equal(strconv.Itoa(int(val)), buf.String())
})
}
should := require.New(t)
buf := &bytes.Buffer{}
stream := NewStream(buf, 4)
stream.WriteString("a")
stream.WriteInt8(-100) // should clear buffer
stream.Flush()
should.Nil(stream.Error)
should.Equal("a-100", buf.String())
}
func Test_write_uint16(t *testing.T) {
vals := []uint16{0, 1, 11, 111, 255, 0xfff, 0xffff}
for _, val := range vals {
t.Run(fmt.Sprintf("%v", val), func(t *testing.T) {
should := require.New(t)
buf := &bytes.Buffer{}
stream := NewStream(buf, 4096)
stream.WriteUint16(val)
stream.Flush()
should.Nil(stream.Error)
should.Equal(strconv.Itoa(int(val)), buf.String())
})
}
should := require.New(t)
buf := &bytes.Buffer{}
stream := NewStream(buf, 5)
stream.WriteString("a")
stream.WriteUint16(10000) // should clear buffer
stream.Flush()
should.Nil(stream.Error)
should.Equal("a10000", buf.String())
}
func Benchmark_jsoniter_int(b *testing.B) {
for n := 0; n < b.N; n++ {
iter := ParseString(`-100`)

29
jsoniter_null_test.go
View File

@ -2,16 +2,35 @@ package jsoniter
import (
"testing"
"github.com/json-iterator/go/require"
"bytes"
)
func Test_null(t *testing.T) {
func Test_decode_null(t *testing.T) {
iter := ParseString(`null`)
if iter.ReadNil() != true {
t.FailNow()
}
}
func Test_null_object(t *testing.T) {
func Test_write_null(t *testing.T) {
should := require.New(t)
buf := &bytes.Buffer{}
stream := NewStream(buf, 4096)
stream.WriteNull()
stream.Flush()
should.Nil(stream.Error)
should.Equal("null", buf.String())
}
func Test_encode_null(t *testing.T) {
should := require.New(t)
str, err := MarshalToString(nil)
should.Nil(err)
should.Equal("null", str)
}
func Test_decode_null_object(t *testing.T) {
iter := ParseString(`[null,"a"]`)
iter.ReadArray()
if iter.ReadObject() != "" {
@ -23,7 +42,7 @@ func Test_null_object(t *testing.T) {
}
}
func Test_null_array(t *testing.T) {
func Test_decode_null_array(t *testing.T) {
iter := ParseString(`[null,"a"]`)
iter.ReadArray()
if iter.ReadArray() != false {
@ -35,7 +54,7 @@ func Test_null_array(t *testing.T) {
}
}
func Test_null_string(t *testing.T) {
func Test_decode_null_string(t *testing.T) {
iter := ParseString(`[null,"a"]`)
iter.ReadArray()
if iter.ReadString() != "" {
@ -47,7 +66,7 @@ func Test_null_string(t *testing.T) {
}
}
func Test_null_skip(t *testing.T) {
func Test_decode_null_skip(t *testing.T) {
iter := ParseString(`[null,"a"]`)
iter.ReadArray()
iter.Skip()

24
jsoniter_reflect_struct_test.go
View File

@ -11,9 +11,9 @@ func Test_decode_one_field_struct(t *testing.T) {
field1 string
}
obj := TestObject{}
should.Nil(UnmarshalString(`{}`, &obj))
should.Nil(UnmarshalFromString(`{}`, &obj))
should.Equal("", obj.field1)
should.Nil(UnmarshalString(`{"field1": "hello"}`, &obj))
should.Nil(UnmarshalFromString(`{"field1": "hello"}`, &obj))
should.Equal("hello", obj.field1)
}
@ -24,9 +24,9 @@ func Test_decode_two_fields_struct(t *testing.T) {
field2 string
}
obj := TestObject{}
should.Nil(UnmarshalString(`{}`, &obj))
should.Nil(UnmarshalFromString(`{}`, &obj))
should.Equal("", obj.field1)
should.Nil(UnmarshalString(`{"field1": "a", "field2": "b"}`, &obj))
should.Nil(UnmarshalFromString(`{"field1": "a", "field2": "b"}`, &obj))
should.Equal("a", obj.field1)
should.Equal("b", obj.field2)
}
@ -39,9 +39,9 @@ func Test_decode_three_fields_struct(t *testing.T) {
field3 string
}
obj := TestObject{}
should.Nil(UnmarshalString(`{}`, &obj))
should.Nil(UnmarshalFromString(`{}`, &obj))
should.Equal("", obj.field1)
should.Nil(UnmarshalString(`{"field1": "a", "field2": "b", "field3": "c"}`, &obj))
should.Nil(UnmarshalFromString(`{"field1": "a", "field2": "b", "field3": "c"}`, &obj))
should.Equal("a", obj.field1)
should.Equal("b", obj.field2)
should.Equal("c", obj.field3)
@ -56,9 +56,9 @@ func Test_decode_four_fields_struct(t *testing.T) {
field4 string
}
obj := TestObject{}
should.Nil(UnmarshalString(`{}`, &obj))
should.Nil(UnmarshalFromString(`{}`, &obj))
should.Equal("", obj.field1)
should.Nil(UnmarshalString(`{"field1": "a", "field2": "b", "field3": "c", "field4": "d"}`, &obj))
should.Nil(UnmarshalFromString(`{"field1": "a", "field2": "b", "field3": "c", "field4": "d"}`, &obj))
should.Equal("a", obj.field1)
should.Equal("b", obj.field2)
should.Equal("c", obj.field3)
@ -75,9 +75,9 @@ func Test_decode_five_fields_struct(t *testing.T) {
field5 string
}
obj := TestObject{}
should.Nil(UnmarshalString(`{}`, &obj))
should.Nil(UnmarshalFromString(`{}`, &obj))
should.Equal("", obj.field1)
should.Nil(UnmarshalString(`{"field1": "a", "field2": "b", "field3": "c", "field4": "d", "field5": "e"}`, &obj))
should.Nil(UnmarshalFromString(`{"field1": "a", "field2": "b", "field3": "c", "field4": "d", "field5": "e"}`, &obj))
should.Equal("a", obj.field1)
should.Equal("b", obj.field2)
should.Equal("c", obj.field3)
@ -92,7 +92,7 @@ func Test_decode_struct_with_optional_field(t *testing.T) {
field2 *string
}
obj := TestObject{}
UnmarshalString(`{"field1": null, "field2": "world"}`, &obj)
UnmarshalFromString(`{"field1": null, "field2": "world"}`, &obj)
should.Nil(obj.field1)
should.Equal("world", *obj.field2)
}
@ -105,7 +105,7 @@ func Test_decode_struct_field_with_tag(t *testing.T) {
Field3 int `json:",string"`
}
obj := TestObject{Field2: "world"}
UnmarshalString(`{"field-1": "hello", "field2": "", "Field3": "100"}`, &obj)
UnmarshalFromString(`{"field-1": "hello", "field2": "", "Field3": "100"}`, &obj)
should.Equal("hello", obj.Field1)
should.Equal("world", obj.Field2)
should.Equal(100, obj.Field3)

4
jsoniter_reflect_test.go
View File

@ -11,14 +11,14 @@ import (
func Test_decode_slice(t *testing.T) {
should := require.New(t)
slice := make([]string, 0, 5)
UnmarshalString(`["hello", "world"]`, &slice)
UnmarshalFromString(`["hello", "world"]`, &slice)
should.Equal([]string{"hello", "world"}, slice)
}
func Test_decode_large_slice(t *testing.T) {
should := require.New(t)
slice := make([]int, 0, 1)
UnmarshalString(`[1,2,3,4,5,6,7,8,9]`, &slice)
UnmarshalFromString(`[1,2,3,4,5,6,7,8,9]`, &slice)
should.Equal([]int{1, 2, 3, 4, 5, 6, 7, 8, 9}, slice)
}

25
jsoniter_string_test.go
View File

@ -4,9 +4,10 @@ import (
"bytes"
"encoding/json"
"testing"
"github.com/json-iterator/go/require"
)
func Test_string_empty(t *testing.T) {
func Test_decode_string_empty(t *testing.T) {
iter := Parse(bytes.NewBufferString(`""`), 4096)
val := iter.ReadString()
if iter.Error != nil {
@ -17,7 +18,7 @@ func Test_string_empty(t *testing.T) {
}
}
func Test_string_hello(t *testing.T) {
func Test_decode_string_hello(t *testing.T) {
iter := Parse(bytes.NewBufferString(`"hello"`), 4096)
val := iter.ReadString()
if iter.Error != nil {
@ -28,7 +29,7 @@ func Test_string_hello(t *testing.T) {
}
}
func Test_string_escape_quote(t *testing.T) {
func Test_decode_string_escape_quote(t *testing.T) {
iter := Parse(bytes.NewBufferString(`"hel\"lo"`), 4096)
val := iter.ReadString()
if iter.Error != nil {
@ -39,7 +40,7 @@ func Test_string_escape_quote(t *testing.T) {
}
}
func Test_string_escape_newline(t *testing.T) {
func Test_decode_string_escape_newline(t *testing.T) {
iter := Parse(bytes.NewBufferString(`"hel\nlo"`), 4096)
val := iter.ReadString()
if iter.Error != nil {
@ -50,7 +51,7 @@ func Test_string_escape_newline(t *testing.T) {
}
}
func Test_string_escape_unicode(t *testing.T) {
func Test_decode_string_escape_unicode(t *testing.T) {
iter := Parse(bytes.NewBufferString(`"\u4e2d\u6587"`), 4096)
val := iter.ReadString()
if iter.Error != nil {
@ -61,7 +62,7 @@ func Test_string_escape_unicode(t *testing.T) {
}
}
func Test_string_escape_unicode_with_surrogate(t *testing.T) {
func Test_decode_string_escape_unicode_with_surrogate(t *testing.T) {
iter := Parse(bytes.NewBufferString(`"\ud83d\udc4a"`), 4096)
val := iter.ReadString()
if iter.Error != nil {
@ -72,7 +73,7 @@ func Test_string_escape_unicode_with_surrogate(t *testing.T) {
}
}
func Test_string_as_bytes(t *testing.T) {
func Test_decode_string_as_bytes(t *testing.T) {
iter := Parse(bytes.NewBufferString(`"hello""world"`), 4096)
val := string(iter.readStringAsBytes())
if val != "hello" {
@ -84,6 +85,16 @@ func Test_string_as_bytes(t *testing.T) {
}
}
func Test_write_string(t *testing.T) {
should := require.New(t)
buf := &bytes.Buffer{}
stream := NewStream(buf, 4096)
stream.WriteString("hello")
stream.Flush()
should.Nil(stream.Error)
should.Equal("hello", buf.String())
}
func Benchmark_jsoniter_unicode(b *testing.B) {
for n := 0; n < b.N; n++ {
iter := ParseString(`"\ud83d\udc4a"`)

229
stream.go Normal file
View File

@ -0,0 +1,229 @@
package jsoniter
import (
"io"
)
var bytesNull []byte
var digits []uint8;
func init() {
bytesNull = []byte("null")
digits = []uint8{
'0', '1', '2', '3', '4', '5',
'6', '7', '8', '9', 'a', 'b',
'c', 'd', 'e', 'f', 'g', 'h',
'i', 'j', 'k', 'l', 'm', 'n',
'o', 'p', 'q', 'r', 's', 't',
'u', 'v', 'w', 'x', 'y', 'z',
}
}
type Stream struct {
out io.Writer
buf []byte
n int
Error error
}
func NewStream(out io.Writer, bufSize int) *Stream {
return &Stream{out, make([]byte, bufSize), 0, nil}
}
// Available returns how many bytes are unused in the buffer.
func (b *Stream) Available() int {
return len(b.buf) - b.n
}
// Buffered returns the number of bytes that have been written into the current buffer.
func (b *Stream) Buffered() int {
return b.n
}
// Write writes the contents of p into the buffer.
// It returns the number of bytes written.
// If nn < len(p), it also returns an error explaining
// why the write is short.
func (b *Stream) Write(p []byte) (nn int, err error) {
for len(p) > b.Available() && b.Error == nil {
var n int
if b.Buffered() == 0 {
// Large write, empty buffer.
// Write directly from p to avoid copy.
n, b.Error = b.out.Write(p)
} else {
n = copy(b.buf[b.n:], p)
b.n += n
b.Flush()
}
nn += n
p = p[n:]
}
if b.Error != nil {
return nn, b.Error
}
n := copy(b.buf[b.n:], p)
b.n += n
nn += n
return nn, nil
}
// WriteByte writes a single byte.
func (b *Stream) WriteByte(c byte) error {
if b.Error != nil {
return b.Error
}
if b.Available() <= 0 && b.Flush() != nil {
return b.Error
}
b.buf[b.n] = c
b.n++
return nil
}
// Flush writes any buffered data to the underlying io.Writer.
func (b *Stream) Flush() error {
if b.Error != nil {
return b.Error
}
if b.n == 0 {
return nil
}
n, err := b.out.Write(b.buf[0:b.n])
if n < b.n && err == nil {
err = io.ErrShortWrite
}
if err != nil {
if n > 0 && n < b.n {
copy(b.buf[0:b.n - n], b.buf[n:b.n])
}
b.n -= n
b.Error = err
return err
}
b.n = 0
return nil
}
func (b *Stream) WriteString(s string) {
for len(s) > b.Available() && b.Error == nil {
n := copy(b.buf[b.n:], s)
b.n += n
s = s[n:]
b.Flush()
}
if b.Error != nil {
return
}
n := copy(b.buf[b.n:], s)
b.n += n
}
func (stream *Stream) WriteNull() {
stream.Write(bytesNull)
}
func (stream *Stream) WriteUint8(val uint8) {
if stream.Available() < 3 {
stream.Flush()
}
charPos := stream.n
if val <= 9 {
charPos += 1;
} else {
if val <= 99 {
charPos += 2;
} else {
charPos += 3;
}
}
stream.n = charPos
var q uint8
var r uint8
for {
q = val / 10
r = val - ((q << 3) + (q << 1)) // r = i-(q*10) ...
charPos--
stream.buf[charPos] = digits[r]
val = q;
if val == 0 {
break
}
}
}
func (stream *Stream) WriteInt8(val int8) {
if stream.Available() < 4 {
stream.Flush()
}
charPos := stream.n
if (val < 0) {
charPos += 1
val = -val
stream.buf[stream.n] = '-'
}
if val <= 9 {
charPos += 1;
} else {
if val <= 99 {
charPos += 2;
} else {
charPos += 3;
}
}
stream.n = charPos
var q int8
var r int8
for {
q = val / 10
r = val - ((q << 3) + (q << 1)) // r = i-(q*10) ...
charPos--
stream.buf[charPos] = digits[r]
val = q;
if val == 0 {
break
}
}
}
func (stream *Stream) WriteUint16(val uint16) {
if stream.Available() < 5 {
stream.Flush()
}
charPos := stream.n
if val <= 99 {
if val <= 9 {
charPos += 1;
} else {
charPos += 2;
}
} else {
if val <= 999 {
charPos += 3;
} else {
if val <= 9999 {
charPos += 4;
} else {
charPos += 5;
}
}
}
stream.n = charPos
var q uint16
var r uint16
for {
q = val / 10
r = val - ((q << 3) + (q << 1)) // r = i-(q*10) ...
charPos--
stream.buf[charPos] = digits[r]
val = q;
if val == 0 {
break
}
}
}
func (stream *Stream) WriteVal(val interface{}) {
}