go/json-iterator
1
0
Fork 0
mirror of https://github.com/json-iterator/go.git synced 2025-01-08 13:06:29 +02:00
Code Issues Releases Activity

support []byte; marshal without copy

Browse Source
This commit is contained in:
Tao Wen 2017-06-06 09:44:56 +08:00
parent 3979955e69
commit a4e5abf492
9 changed files with 225 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
feature_adapter.go
View File

@ -89,14 +89,12 @@ func UnmarshalAnyFromString(str string) (Any, error) {
// Marshal returns the JSON encoding of v, adapts to json/encoding Marshal API
// Refer to https://godoc.org/encoding/json#Marshal for more information
func Marshal(v interface{}) ([]byte, error) {
buf := &bytes.Buffer{}
stream := NewStream(buf, 512)
stream := NewStream(nil, 256)
stream.WriteVal(v)
stream.Flush()
if stream.Error != nil {
return nil, stream.Error
}
return buf.Bytes(), nil
return stream.Buffer(), nil
}
func MarshalToString(v interface{}) (string, error) {

11
feature_reflect.go
View File

@ -159,15 +159,18 @@ func RegisterExtension(extension ExtensionFunc) {
extensions = append(extensions, extension)
}
// CleanDecoders cleans decoders registered
// CleanDecoders cleans decoders registered or cached
func CleanDecoders() {
typeDecoders = map[string]Decoder{}
fieldDecoders = map[string]Decoder{}
atomic.StorePointer(&DECODERS, unsafe.Pointer(&map[string]Decoder{}))
}
// CleanEncoders cleans decoders registered or cached
func CleanEncoders() {
typeEncoders = map[string]Encoder{}
fieldEncoders = map[string]Encoder{}
atomic.StorePointer(&ENCODERS, unsafe.Pointer(&map[string]Encoder{}))
}
type optionalDecoder struct {
@ -339,6 +342,9 @@ func decoderOfType(typ reflect.Type) (Decoder, error) {
}
func createDecoderOfType(typ reflect.Type) (Decoder, error) {
if typ.String() == "[]uint8" {
return &base64Codec{}, nil
}
if typ.AssignableTo(jsonRawMessageType) {
return &jsonRawMessageCodec{}, nil
}
@ -426,6 +432,9 @@ func encoderOfType(typ reflect.Type) (Encoder, error) {
}
func createEncoderOfType(typ reflect.Type) (Encoder, error) {
if typ.String() == "[]uint8" {
return &base64Codec{}, nil
}
if typ.AssignableTo(jsonRawMessageType) {
return &jsonRawMessageCodec{}, nil
}

46
feature_reflect_native.go
View File

@ -3,6 +3,7 @@ package jsoniter
import (
"unsafe"
"encoding/json"
"encoding/base64"
)
type stringCodec struct {
@ -379,6 +380,49 @@ func (encoder *jsonRawMessageCodec) isEmpty(ptr unsafe.Pointer) bool {
return len(*((*json.RawMessage)(ptr))) == 0
}
type base64Codec struct {
}
func (codec *base64Codec) decode(ptr unsafe.Pointer, iter *Iterator) {
encoding := base64.StdEncoding
src := iter.SkipAndReturnBytes()
src = src[1:len(src)-1]
decodedLen := encoding.DecodedLen(len(src))
dst := make([]byte, decodedLen)
_, err := encoding.Decode(dst, src)
if err != nil {
iter.reportError("decode base64", err.Error())
} else {
*((*[]byte)(ptr)) = dst
}
}
func (codec *base64Codec) encode(ptr unsafe.Pointer, stream *Stream) {
encoding := base64.StdEncoding
stream.writeByte('"')
src := *((*[]byte)(ptr))
toGrow := encoding.EncodedLen(len(src))
stream.ensure(toGrow)
encoding.Encode(stream.buf[stream.n:], src)
stream.n += toGrow
stream.writeByte('"')
}
func (encoder *base64Codec) encodeInterface(val interface{}, stream *Stream) {
encoding := base64.StdEncoding
stream.writeByte('"')
src := val.([]byte)
toGrow := encoding.EncodedLen(len(src))
stream.ensure(toGrow)
encoding.Encode(stream.buf[stream.n:], src)
stream.n += toGrow
stream.writeByte('"')
}
func (encoder *base64Codec) isEmpty(ptr unsafe.Pointer) bool {
return len(*((*[]byte)(ptr))) == 0
}
type stringNumberDecoder struct {
elemDecoder Decoder
}
@ -447,4 +491,4 @@ func (decoder *unmarshalerDecoder) decode(ptr unsafe.Pointer, iter *Iterator) {
if err != nil {
iter.reportError("unmarshaler", err.Error())
}
}
}

137
feature_stream.go
View File

@ -32,24 +32,32 @@ func (b *Stream) Buffered() int {
return b.n
}
func (b *Stream) Buffer() []byte {
return b.buf[:b.n]
}
// Write writes the contents of p into the buffer.
// It returns the number of bytes written.
// If nn < len(p), it also returns an error explaining
// why the write is short.
func (b *Stream) Write(p []byte) (nn int, err error) {
for len(p) > b.Available() && b.Error == nil {
var n int
if b.Buffered() == 0 {
// Large write, empty buffer.
// Write directly from p to avoid copy.
n, b.Error = b.out.Write(p)
if b.out == nil {
b.growAtLeast(len(p))
} else {
n = copy(b.buf[b.n:], p)
b.n += n
b.Flush()
var n int
if b.Buffered() == 0 {
// Large write, empty buffer.
// Write directly from p to avoid copy.
n, b.Error = b.out.Write(p)
} else {
n = copy(b.buf[b.n:], p)
b.n += n
b.Flush()
}
nn += n
p = p[n:]
}
nn += n
p = p[n:]
}
if b.Error != nil {
return nn, b.Error
@ -60,14 +68,13 @@ func (b *Stream) Write(p []byte) (nn int, err error) {
return nn, nil
}
// WriteByte writes a single byte.
func (b *Stream) writeByte(c byte) {
if b.Error != nil {
return
}
if b.Available() <= 0 && b.Flush() != nil {
return
if b.Available() < 1 {
b.growAtLeast(1)
}
b.buf[b.n] = c
b.n++
@ -77,11 +84,11 @@ func (b *Stream) writeTwoBytes(c1 byte, c2 byte) {
if b.Error != nil {
return
}
if b.Available() <= 1 && b.Flush() != nil {
return
if b.Available() < 2 {
b.growAtLeast(2)
}
b.buf[b.n] = c1
b.buf[b.n + 1] = c2
b.buf[b.n+1] = c2
b.n += 2
}
@ -89,12 +96,12 @@ func (b *Stream) writeThreeBytes(c1 byte, c2 byte, c3 byte) {
if b.Error != nil {
return
}
if b.Available() <= 2 && b.Flush() != nil {
return
if b.Available() < 3 {
b.growAtLeast(3)
}
b.buf[b.n] = c1
b.buf[b.n + 1] = c2
b.buf[b.n + 2] = c3
b.buf[b.n+1] = c2
b.buf[b.n+2] = c3
b.n += 3
}
@ -102,13 +109,13 @@ func (b *Stream) writeFourBytes(c1 byte, c2 byte, c3 byte, c4 byte) {
if b.Error != nil {
return
}
if b.Available() <= 3 && b.Flush() != nil {
return
if b.Available() < 4 {
b.growAtLeast(4)
}
b.buf[b.n] = c1
b.buf[b.n + 1] = c2
b.buf[b.n + 2] = c3
b.buf[b.n + 3] = c4
b.buf[b.n+1] = c2
b.buf[b.n+2] = c3
b.buf[b.n+3] = c4
b.n += 4
}
@ -116,19 +123,22 @@ func (b *Stream) writeFiveBytes(c1 byte, c2 byte, c3 byte, c4 byte, c5 byte) {
if b.Error != nil {
return
}
if b.Available() <= 3 && b.Flush() != nil {
return
if b.Available() < 5 {
b.growAtLeast(5)
}
b.buf[b.n] = c1
b.buf[b.n + 1] = c2
b.buf[b.n + 2] = c3
b.buf[b.n + 3] = c4
b.buf[b.n + 4] = c5
b.buf[b.n+1] = c2
b.buf[b.n+2] = c3
b.buf[b.n+3] = c4
b.buf[b.n+4] = c5
b.n += 5
}
// Flush writes any buffered data to the underlying io.Writer.
func (b *Stream) Flush() error {
if b.out == nil {
return nil
}
if b.Error != nil {
return b.Error
}
@ -141,7 +151,7 @@ func (b *Stream) Flush() error {
}
if err != nil {
if n > 0 && n < b.n {
copy(b.buf[0:b.n - n], b.buf[n:b.n])
copy(b.buf[0:b.n-n], b.buf[n:b.n])
}
b.n -= n
b.Error = err
@ -151,13 +161,28 @@ func (b *Stream) Flush() error {
return nil
}
func (b *Stream) WriteRaw(s string) {
for len(s) > b.Available() && b.Error == nil {
n := copy(b.buf[b.n:], s)
b.n += n
s = s[n:]
b.Flush()
func (b *Stream) ensure(minimal int) {
available := b.Available()
if available < minimal {
if available < 128 {
b.Flush()
}
b.growAtLeast(minimal)
}
}
func (b *Stream) growAtLeast(minimal int) {
toGrow := len(b.buf)
if toGrow < minimal {
toGrow = minimal
}
newBuf := make([]byte, len(b.buf)+toGrow)
copy(newBuf, b.Buffer())
b.buf = newBuf
}
func (b *Stream) WriteRaw(s string) {
b.ensure(len(s))
if b.Error != nil {
return
}
@ -166,18 +191,13 @@ func (b *Stream) WriteRaw(s string) {
}
func (stream *Stream) WriteString(s string) {
stream.ensure(32)
valLen := len(s)
toWriteLen := valLen
bufLengthMinusTwo := len(stream.buf) - 2 // make room for the quotes
if stream.n + toWriteLen > bufLengthMinusTwo {
if stream.n+toWriteLen > bufLengthMinusTwo {
toWriteLen = bufLengthMinusTwo - stream.n
}
if toWriteLen < 0 {
stream.Flush()
if stream.n + toWriteLen > bufLengthMinusTwo {
toWriteLen = bufLengthMinusTwo - stream.n
}
}
n := stream.n
stream.buf[n] = '"'
n++
@ -189,7 +209,7 @@ func (stream *Stream) WriteString(s string) {
stream.buf[n] = c
n++
} else {
break;
break
}
}
if i == valLen {
@ -200,14 +220,14 @@ func (stream *Stream) WriteString(s string) {
}
stream.n = n
// for the remaining parts, we process them char by char
stream.writeStringSlowPath(s, i, valLen);
stream.writeStringSlowPath(s, i, valLen)
stream.writeByte('"')
}
func (stream *Stream) writeStringSlowPath(s string, i int, valLen int) {
for ; i < valLen; i++ {
c := s[i]
switch (c) {
switch c {
case '"':
stream.writeTwoBytes('\\', '"')
case '\\':
@ -223,7 +243,7 @@ func (stream *Stream) writeStringSlowPath(s string, i int, valLen int) {
case '\t':
stream.writeTwoBytes('\\', 't')
default:
stream.writeByte(c);
stream.writeByte(c)
}
}
}
@ -293,21 +313,14 @@ func (stream *Stream) WriteArrayEnd() {
}
func (stream *Stream) writeIndention(delta int) {
if (stream.indention == 0) {
if stream.indention == 0 {
return
}
stream.writeByte('\n')
toWrite := stream.indention - delta
i := 0
for {
for ; i < toWrite && stream.n < len(stream.buf); i++ {
stream.buf[stream.n] = ' '
stream.n ++
}
if i == toWrite {
break;
} else {
stream.Flush()
}
stream.ensure(toWrite)
for i := 0 ; i < toWrite && stream.n < len(stream.buf); i++ {
stream.buf[stream.n] = ' '
stream.n ++
}
}
}

8
feature_stream_float.go
View File

@ -33,9 +33,7 @@ func (stream *Stream) WriteFloat32Lossy(val float32) {
return
}
stream.writeByte('.')
if stream.Available() < 10 {
stream.Flush()
}
stream.ensure(10)
for p := precision - 1; p > 0 && fval < POW10[p]; p-- {
stream.writeByte('0')
}
@ -67,9 +65,7 @@ func (stream *Stream) WriteFloat64Lossy(val float64) {
return
}
stream.writeByte('.')
if stream.Available() < 10 {
stream.Flush()
}
stream.ensure(10)
for p := precision - 1; p > 0 && fval < POW10[p]; p-- {
stream.writeByte('0')
}

81
feature_stream_int.go
View File

@ -1,43 +1,8 @@
package jsoniter
var digits []uint8
var digitTens []uint8
var digitOnes []uint8
var DIGITS []uint32
func init() {
digits = []uint8{
'0', '1', '2', '3', '4', '5',
'6', '7', '8', '9', 'a', 'b',
'c', 'd', 'e', 'f', 'g', 'h',
'i', 'j', 'k', 'l', 'm', 'n',
'o', 'p', 'q', 'r', 's', 't',
'u', 'v', 'w', 'x', 'y', 'z',
}
digitTens = []uint8{
'0', '0', '0', '0', '0', '0', '0', '0', '0', '0',
'1', '1', '1', '1', '1', '1', '1', '1', '1', '1',
'2', '2', '2', '2', '2', '2', '2', '2', '2', '2',
'3', '3', '3', '3', '3', '3', '3', '3', '3', '3',
'4', '4', '4', '4', '4', '4', '4', '4', '4', '4',
'5', '5', '5', '5', '5', '5', '5', '5', '5', '5',
'6', '6', '6', '6', '6', '6', '6', '6', '6', '6',
'7', '7', '7', '7', '7', '7', '7', '7', '7', '7',
'8', '8', '8', '8', '8', '8', '8', '8', '8', '8',
'9', '9', '9', '9', '9', '9', '9', '9', '9', '9',
}
digitOnes = []uint8{
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
}
DIGITS = make([]uint32, 1000)
for i := uint32(0); i < 1000; i++ {
DIGITS[i] = (((i / 100) + '0') << 16) + ((((i / 10) % 10) + '0') << 8) + i % 10 + '0';
@ -72,19 +37,15 @@ func writeBuf(buf []byte, v uint32, n int) {
}
func (stream *Stream) WriteUint8(val uint8) {
if stream.Available() < 3 {
stream.Flush()
}
stream.ensure(3)
stream.n = writeFirstBuf(stream.buf, DIGITS[val], stream.n)
}
func (stream *Stream) WriteInt8(nval int8) {
if stream.Available() < 4 {
stream.Flush()
}
stream.ensure(4)
n := stream.n
var val uint8
if (nval < 0) {
if nval < 0 {
val = uint8(-nval)
stream.buf[n] = '-'
n++
@ -95,15 +56,13 @@ func (stream *Stream) WriteInt8(nval int8) {
}
func (stream *Stream) WriteUint16(val uint16) {
if stream.Available() < 5 {
stream.Flush()
}
stream.ensure(5)
q1 := val / 1000
if q1 == 0 {
stream.n = writeFirstBuf(stream.buf, DIGITS[val], stream.n)
return
}
r1 := val - q1 * 1000;
r1 := val - q1 * 1000
n := writeFirstBuf(stream.buf, DIGITS[q1], stream.n)
writeBuf(stream.buf, DIGITS[r1], n)
stream.n = n + 3
@ -111,12 +70,10 @@ func (stream *Stream) WriteUint16(val uint16) {
}
func (stream *Stream) WriteInt16(nval int16) {
if stream.Available() < 6 {
stream.Flush()
}
stream.ensure(6)
n := stream.n
var val uint16
if (nval < 0) {
if nval < 0 {
val = uint16(-nval)
stream.buf[n] = '-'
n++
@ -128,7 +85,7 @@ func (stream *Stream) WriteInt16(nval int16) {
stream.n = writeFirstBuf(stream.buf, DIGITS[val], n)
return
}
r1 := val - q1 * 1000;
r1 := val - q1 * 1000
n = writeFirstBuf(stream.buf, DIGITS[q1], n)
writeBuf(stream.buf, DIGITS[r1], n)
stream.n = n + 3
@ -136,16 +93,14 @@ func (stream *Stream) WriteInt16(nval int16) {
}
func (stream *Stream) WriteUint32(val uint32) {
if stream.Available() < 10 {
stream.Flush()
}
stream.ensure(10)
n := stream.n
q1 := val / 1000
if q1 == 0 {
stream.n = writeFirstBuf(stream.buf, DIGITS[val], n)
return
}
r1 := val - q1 * 1000;
r1 := val - q1 * 1000
q2 := q1 / 1000
if q2 == 0 {
n := writeFirstBuf(stream.buf, DIGITS[q1], n)
@ -170,9 +125,7 @@ func (stream *Stream) WriteUint32(val uint32) {
}
func (stream *Stream) WriteInt32(nval int32) {
if stream.Available() < 11 {
stream.Flush()
}
stream.ensure(11)
n := stream.n
var val uint32
if (nval < 0) {
@ -187,7 +140,7 @@ func (stream *Stream) WriteInt32(nval int32) {
stream.n = writeFirstBuf(stream.buf, DIGITS[val], n)
return
}
r1 := val - q1 * 1000;
r1 := val - q1 * 1000
q2 := q1 / 1000
if q2 == 0 {
n := writeFirstBuf(stream.buf, DIGITS[q1], n)
@ -212,16 +165,14 @@ func (stream *Stream) WriteInt32(nval int32) {
}
func (stream *Stream) WriteUint64(val uint64) {
if stream.Available() < 20 {
stream.Flush()
}
stream.ensure(20)
n := stream.n
q1 := val / 1000
if q1 == 0 {
stream.n = writeFirstBuf(stream.buf, DIGITS[val], n)
return
}
r1 := val - q1 * 1000;
r1 := val - q1 * 1000
q2 := q1 / 1000
if q2 == 0 {
n := writeFirstBuf(stream.buf, DIGITS[q1], n)
@ -278,9 +229,7 @@ func (stream *Stream) WriteUint64(val uint64) {
}
func (stream *Stream) WriteInt64(nval int64) {
if stream.Available() < 20 {
stream.Flush()
}
stream.ensure(20)
n := stream.n
var val uint64
if (nval < 0) {

21
jsoniter_array_test.go
View File

@ -264,6 +264,27 @@ func Test_json_RawMessage(t *testing.T) {
should.Equal(`[1,2,3]`, str)
}
func Test_encode_byte_array(t *testing.T) {
should := require.New(t)
bytes, err := json.Marshal([]byte{1,2,3})
should.Nil(err)
should.Equal(`"AQID"`, string(bytes))
bytes, err = Marshal([]byte{1,2,3})
should.Nil(err)
should.Equal(`"AQID"`, string(bytes))
}
func Test_decode_byte_array(t *testing.T) {
should := require.New(t)
data := []byte{}
err := json.Unmarshal([]byte(`"AQID"`), &data)
should.Nil(err)
should.Equal([]byte{1,2,3}, data)
err = Unmarshal([]byte(`"AQID"`), &data)
should.Nil(err)
should.Equal([]byte{1,2,3}, data)
}
func Benchmark_jsoniter_array(b *testing.B) {
b.ReportAllocs()
input := []byte(`[1,2,3,4,5,6,7,8,9]`)

1
jsoniter_customize_test.go
View File

@ -45,6 +45,7 @@ func Test_customize_type_encoder(t *testing.T) {
}
func Test_customize_byte_array_encoder(t *testing.T) {
CleanEncoders()
should := require.New(t)
RegisterTypeEncoder("[]uint8", func(ptr unsafe.Pointer, stream *Stream) {
t := *((*[]byte)(ptr))

54
jsoniter_stream_test.go Normal file
View File

@ -0,0 +1,54 @@
package jsoniter
import (
"testing"
"github.com/json-iterator/go/require"
)
func Test_writeByte_should_grow_buffer(t *testing.T) {
should := require.New(t)
stream := NewStream(nil, 1)
stream.writeByte('1')
should.Equal("1", string(stream.Buffer()))
should.Equal(1, len(stream.buf))
stream.writeByte('2')
should.Equal("12", string(stream.Buffer()))
should.Equal(2, len(stream.buf))
stream.writeThreeBytes('3', '4', '5')
should.Equal("12345", string(stream.Buffer()))
}
func Test_writeBytes_should_grow_buffer(t *testing.T) {
should := require.New(t)
stream := NewStream(nil, 1)
stream.Write([]byte{'1', '2'})
should.Equal("12", string(stream.Buffer()))
should.Equal(3, len(stream.buf))
stream.Write([]byte{'3', '4', '5', '6', '7'})
should.Equal("1234567", string(stream.Buffer()))
should.Equal(8, len(stream.buf))
}
func Test_writeIndention_should_grow_buffer(t *testing.T) {
should := require.New(t)
stream := NewStream(nil, 1)
stream.IndentionStep = 2
stream.WriteVal([]int{1,2,3})
should.Equal("[\n 1,\n 2,\n 3\n]", string(stream.Buffer()))
}
func Test_writeRaw_should_grow_buffer(t *testing.T) {
should := require.New(t)
stream := NewStream(nil, 1)
stream.WriteRaw("123")
should.Nil(stream.Error)
should.Equal("123", string(stream.Buffer()))
}
func Test_writeString_should_grow_buffer(t *testing.T) {
should := require.New(t)
stream := NewStream(nil, 0)
stream.WriteString("123")
should.Nil(stream.Error)
should.Equal(`"123"`, string(stream.Buffer()))
}