1
0
mirror of https://github.com/json-iterator/go.git synced 2025-06-15 22:50:24 +02:00

33 Commits

Author SHA1 Message Date
1624edc445 fix #295 decoder more was not compatible with standard library 2018-08-06 14:07:27 +08:00
5d789e5e02 fix #291 omit empty was not handled properly for json raw message 2018-08-06 13:58:33 +08:00
0260c89b54 fix #286 calcHash should use byte not rune to calc hash 2018-08-06 13:23:06 +08:00
10a568c511 fix #293 copy extensions 2018-07-22 11:51:51 +08:00
ab8a2e0c74 fix #276 allow rename when set naming strategy 2018-07-01 15:16:28 +08:00
2fbdfbb595 merge 2018-07-01 13:06:34 +08:00
720ab8dc7f add tests for #283 2018-07-01 13:05:25 +08:00
f2b4162afb Merge pull request #285 from nikhita/fix-case-sensitivity
Fix case sensitivity
2018-06-12 13:28:35 -07:00
3830516ed0 Fix case sensitivity for nested fields 2018-06-12 11:27:24 +05:30
7cceb6c2e3 Merge pull request #282 from caesarxuchao/optional-case-sensitivity
Make case sensitivity optional
2018-06-10 17:13:47 +08:00
b92cf78708 Make case sensitivity optional. Fix
https://github.com/kubernetes/kubernetes/issues/64612
2018-06-07 21:01:05 -07:00
8744d7c5c7 \n should not be ignored in base64 decode 2018-05-26 09:43:29 +08:00
37cc313d18 fix #274, unescape before base64 decode 2018-05-26 09:38:52 +08:00
2ddf6d7582 Merge pull request #266 from ceshihao/fix_base64_with_whitespace
fix base64 contains newline case
2018-04-24 08:46:23 +08:00
6a6742f0a2 fix base64 contains newline characters \r or \n 2018-04-23 23:10:55 +08:00
6c702ce12a fix #264 check io.EOF when test decoder.More 2018-04-20 16:10:56 +08:00
f88871b601 fix #263, support empty string as 0 in fuzz mode 2018-04-18 16:34:54 +08:00
f246f80f14 fix #260, support rename for extra.SupportPrivateFields 2018-04-18 16:28:55 +08:00
51dd70305b add more test for #252 2018-04-18 16:22:47 +08:00
a949c42748 fix #261 should load from reader 2018-04-18 16:11:14 +08:00
f89479f5c0 Merge pull request #257 from ash2k/release-writer
Release writer to enable GC
2018-04-08 08:25:46 +08:00
b858ec296c Release writer to enable GC 2018-04-07 21:40:08 +10:00
885a41a0a6 Merge branch 'master' of https://github.com/json-iterator/go 2018-04-03 13:41:12 +08:00
9e9a97040e always benchmark yourself 2018-04-03 13:41:01 +08:00
fb4d53e4cc Merge pull request #255 from bboreham/error-test
Add a test for input errors, and fix one bug that it finds
2018-04-02 13:50:44 +08:00
b53656d459 Check that a struct ends with closing brace 2018-04-01 22:02:44 +00:00
8f27a81d90 Add a test for input errors
Send various malformed JSON strings into the decoder for each type,
and check we get an error each time.
2018-04-01 22:01:21 +00:00
4930b053b8 explit test case sensitive for #252 2018-03-24 22:38:32 +08:00
06e0f9391e fix #250 case insensitive field match 2018-03-20 21:43:30 +08:00
ca39e5af3e suport encode map[interface{}]interface{} 2018-03-15 21:28:16 +08:00
39acec93e0 expose DecoderOf and EncoderOf 2018-03-14 23:18:20 +08:00
25fa392355 fix #245, always reuse existing value even UseNumber 2018-03-01 19:23:20 +08:00
d51e841de0 fix build on 1.8 2018-02-28 20:29:23 +08:00
32 changed files with 595 additions and 81 deletions

6
Gopkg.lock generated
View File

@ -10,12 +10,12 @@
[[projects]]
name = "github.com/modern-go/reflect2"
packages = ["."]
revision = "1df9eeb2bb81f327b96228865c5687bc2194af3f"
version = "1.0.0"
revision = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd"
version = "1.0.1"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "ac7003b5a981716353a43055ab7d4c5357403cb30a60de2dbdeb446c1544beaa"
inputs-digest = "ea54a775e5a354cb015502d2e7aa4b74230fc77e894f34a838b268c25ec8eeb8"
solver-name = "gps-cdcl"
solver-version = 1

View File

@ -23,4 +23,4 @@ ignored = ["github.com/davecgh/go-spew*","github.com/google/gofuzz*","github.com
[[constraint]]
name = "github.com/modern-go/reflect2"
version = "1.0.0"
version = "1.0.1"

View File

@ -31,6 +31,9 @@ Raw Result (easyjson requires static code generation)
| easyjson encode | 883 ns/op | 576 B/op | 3 allocs/op |
| jsoniter encode | 837 ns/op | 384 B/op | 4 allocs/op |
Always benchmark with your own workload.
The result depends heavily on the data input.
# Usage
100% compatibility with standard lib

View File

@ -77,7 +77,16 @@ func (adapter *Decoder) Decode(obj interface{}) error {
// More is there more?
func (adapter *Decoder) More() bool {
return adapter.iter.head != adapter.iter.tail
iter := adapter.iter
if iter.Error != nil {
return false
}
c := iter.nextToken()
if c == 0 {
return false
}
iter.unreadByte()
return c != ']' && c != '}'
}
// Buffered remaining buffer
@ -91,7 +100,7 @@ func (adapter *Decoder) Buffered() io.Reader {
func (adapter *Decoder) UseNumber() {
cfg := adapter.iter.cfg.configBeforeFrozen
cfg.UseNumber = true
adapter.iter.cfg = cfg.frozeWithCacheReuse()
adapter.iter.cfg = cfg.frozeWithCacheReuse(adapter.iter.cfg.extraExtensions)
}
// DisallowUnknownFields causes the Decoder to return an error when the destination
@ -100,7 +109,7 @@ func (adapter *Decoder) UseNumber() {
func (adapter *Decoder) DisallowUnknownFields() {
cfg := adapter.iter.cfg.configBeforeFrozen
cfg.DisallowUnknownFields = true
adapter.iter.cfg = cfg.frozeWithCacheReuse()
adapter.iter.cfg = cfg.frozeWithCacheReuse(adapter.iter.cfg.extraExtensions)
}
// NewEncoder same as json.NewEncoder
@ -125,14 +134,14 @@ func (adapter *Encoder) Encode(val interface{}) error {
func (adapter *Encoder) SetIndent(prefix, indent string) {
config := adapter.stream.cfg.configBeforeFrozen
config.IndentionStep = len(indent)
adapter.stream.cfg = config.frozeWithCacheReuse()
adapter.stream.cfg = config.frozeWithCacheReuse(adapter.stream.cfg.extraExtensions)
}
// SetEscapeHTML escape html by default, set to false to disable
func (adapter *Encoder) SetEscapeHTML(escapeHTML bool) {
config := adapter.stream.cfg.configBeforeFrozen
config.EscapeHTML = escapeHTML
adapter.stream.cfg = config.frozeWithCacheReuse()
adapter.stream.cfg = config.frozeWithCacheReuse(adapter.stream.cfg.extraExtensions)
}
// Valid reports whether data is a valid JSON encoding.

View File

@ -2,9 +2,10 @@ package test
import (
"encoding/json"
"testing"
"github.com/json-iterator/go"
"github.com/stretchr/testify/require"
"testing"
)
func Test_use_number_for_unmarshal(t *testing.T) {
@ -45,3 +46,129 @@ func Test_read_large_number_as_interface(t *testing.T) {
should.Nil(err)
should.Equal(`123456789123456789123456789`, output)
}
type caseSensitiveStruct struct {
A string `json:"a"`
B string `json:"b,omitempty"`
C *C `json:"C,omitempty"`
}
type C struct {
D int64 `json:"D,omitempty"`
E *E `json:"e,omitempty"`
}
type E struct {
F string `json:"F,omitempty"`
}
func Test_CaseSensitive(t *testing.T) {
should := require.New(t)
testCases := []struct {
input string
expectedOutput string
caseSensitive bool
}{
{
input: `{"A":"foo","B":"bar"}`,
expectedOutput: `{"a":"foo","b":"bar"}`,
caseSensitive: false,
},
{
input: `{"a":"foo","b":"bar"}`,
expectedOutput: `{"a":"foo","b":"bar"}`,
caseSensitive: true,
},
{
input: `{"a":"foo","b":"bar","C":{"D":10}}`,
expectedOutput: `{"a":"foo","b":"bar","C":{"D":10}}`,
caseSensitive: true,
},
{
input: `{"a":"foo","B":"bar","c":{"d":10}}`,
expectedOutput: `{"a":"foo"}`,
caseSensitive: true,
},
{
input: `{"a":"foo","C":{"d":10}}`,
expectedOutput: `{"a":"foo","C":{}}`,
caseSensitive: true,
},
{
input: `{"a":"foo","C":{"D":10,"e":{"f":"baz"}}}`,
expectedOutput: `{"a":"foo","C":{"D":10,"e":{}}}`,
caseSensitive: true,
},
{
input: `{"a":"foo","C":{"D":10,"e":{"F":"baz"}}}`,
expectedOutput: `{"a":"foo","C":{"D":10,"e":{"F":"baz"}}}`,
caseSensitive: true,
},
{
input: `{"A":"foo","c":{"d":10,"E":{"f":"baz"}}}`,
expectedOutput: `{"a":"foo","C":{"D":10,"e":{"F":"baz"}}}`,
caseSensitive: false,
},
}
for _, tc := range testCases {
val := caseSensitiveStruct{}
err := jsoniter.Config{CaseSensitive: tc.caseSensitive}.Froze().UnmarshalFromString(tc.input, &val)
should.Nil(err)
output, err := jsoniter.MarshalToString(val)
should.Nil(err)
should.Equal(tc.expectedOutput, output)
}
}
type structWithElevenFields struct {
A string `json:"A,omitempty"`
B string `json:"B,omitempty"`
C string `json:"C,omitempty"`
D string `json:"d,omitempty"`
E string `json:"e,omitempty"`
F string `json:"f,omitempty"`
G string `json:"g,omitempty"`
H string `json:"h,omitempty"`
I string `json:"i,omitempty"`
J string `json:"j,omitempty"`
K string `json:"k,omitempty"`
}
func Test_CaseSensitive_MoreThanTenFields(t *testing.T) {
should := require.New(t)
testCases := []struct {
input string
expectedOutput string
caseSensitive bool
}{
{
input: `{"A":"1","B":"2","C":"3","d":"4","e":"5","f":"6","g":"7","h":"8","i":"9","j":"10","k":"11"}`,
expectedOutput: `{"A":"1","B":"2","C":"3","d":"4","e":"5","f":"6","g":"7","h":"8","i":"9","j":"10","k":"11"}`,
caseSensitive: true,
},
{
input: `{"a":"1","b":"2","c":"3","D":"4","E":"5","F":"6"}`,
expectedOutput: `{"A":"1","B":"2","C":"3","d":"4","e":"5","f":"6"}`,
caseSensitive: false,
},
{
input: `{"A":"1","b":"2","d":"4","E":"5"}`,
expectedOutput: `{"A":"1","d":"4"}`,
caseSensitive: true,
},
}
for _, tc := range testCases {
val := structWithElevenFields{}
err := jsoniter.Config{CaseSensitive: tc.caseSensitive}.Froze().UnmarshalFromString(tc.input, &val)
should.Nil(err)
output, err := jsoniter.MarshalToString(val)
should.Nil(err)
should.Equal(tc.expectedOutput, output)
}
}

View File

@ -56,3 +56,9 @@ func Test_use_number(t *testing.T) {
should.Nil(decoder2.Decode(&obj2))
should.Equal(json.Number("123"), obj2)
}
func Test_decoder_more(t *testing.T) {
should := require.New(t)
decoder := jsoniter.NewDecoder(bytes.NewBufferString("abcde"))
should.True(decoder.More())
}

View File

@ -2,11 +2,13 @@ package jsoniter
import (
"encoding/json"
"github.com/modern-go/reflect2"
"io"
"reflect"
"sync"
"unsafe"
"github.com/modern-go/concurrent"
"github.com/modern-go/reflect2"
)
// Config customize how the API should behave.
@ -22,6 +24,7 @@ type Config struct {
OnlyTaggedField bool
ValidateJsonRawMessage bool
ObjectFieldMustBeSimpleString bool
CaseSensitive bool
}
// API the public interface of this package.
@ -39,6 +42,8 @@ type API interface {
NewDecoder(reader io.Reader) *Decoder
Valid(data []byte) bool
RegisterExtension(extension Extension)
DecoderOf(typ reflect2.Type) ValDecoder
EncoderOf(typ reflect2.Type) ValEncoder
}
// ConfigDefault the default API
@ -60,7 +65,6 @@ var ConfigFastest = Config{
ObjectFieldMustBeSimpleString: true, // do not unescape object field
}.Froze()
type frozenConfig struct {
configBeforeFrozen Config
sortMapKeys bool
@ -70,9 +74,12 @@ type frozenConfig struct {
disallowUnknownFields bool
decoderCache *concurrent.Map
encoderCache *concurrent.Map
extensions []Extension
encoderExtension Extension
decoderExtension Extension
extraExtensions []Extension
streamPool *sync.Pool
iteratorPool *sync.Pool
caseSensitive bool
}
func (cfg *frozenConfig) initCache() {
@ -104,7 +111,7 @@ func (cfg *frozenConfig) getEncoderFromCache(cacheKey uintptr) ValEncoder {
return nil
}
var cfgCache = &sync.Map{}
var cfgCache = concurrent.NewMap()
func getFrozenConfigFromCache(cfg Config) *frozenConfig {
obj, found := cfgCache.Load(cfg)
@ -126,6 +133,7 @@ func (cfg Config) Froze() API {
objectFieldMustBeSimpleString: cfg.ObjectFieldMustBeSimpleString,
onlyTaggedField: cfg.OnlyTaggedField,
disallowUnknownFields: cfg.DisallowUnknownFields,
caseSensitive: cfg.CaseSensitive,
}
api.streamPool = &sync.Pool{
New: func() interface{} {
@ -152,22 +160,21 @@ func (cfg Config) Froze() API {
if cfg.ValidateJsonRawMessage {
api.validateJsonRawMessage(encoderExtension)
}
if len(encoderExtension) > 0 {
api.extensions = append(api.extensions, encoderExtension)
}
if len(decoderExtension) > 0 {
api.extensions = append(api.extensions, decoderExtension)
}
api.encoderExtension = encoderExtension
api.decoderExtension = decoderExtension
api.configBeforeFrozen = cfg
return api
}
func (cfg Config) frozeWithCacheReuse() *frozenConfig {
func (cfg Config) frozeWithCacheReuse(extraExtensions []Extension) *frozenConfig {
api := getFrozenConfigFromCache(cfg)
if api != nil {
return api
}
api = cfg.Froze().(*frozenConfig)
for _, extension := range extraExtensions {
api.RegisterExtension(extension)
}
addFrozenConfigToCache(cfg, api)
return api
}
@ -184,7 +191,7 @@ func (cfg *frozenConfig) validateJsonRawMessage(extension EncoderExtension) {
stream.WriteRaw(string(rawMessage))
}
}, func(ptr unsafe.Pointer) bool {
return false
return len(*((*json.RawMessage)(ptr))) == 0
}}
extension[reflect2.TypeOfPtr((*json.RawMessage)(nil)).Elem()] = encoder
extension[reflect2.TypeOfPtr((*RawMessage)(nil)).Elem()] = encoder
@ -192,6 +199,11 @@ func (cfg *frozenConfig) validateJsonRawMessage(extension EncoderExtension) {
func (cfg *frozenConfig) useNumber(extension DecoderExtension) {
extension[reflect2.TypeOfPtr((*interface{})(nil)).Elem()] = &funcDecoder{func(ptr unsafe.Pointer, iter *Iterator) {
exitingValue := *((*interface{})(ptr))
if exitingValue != nil && reflect.TypeOf(exitingValue).Kind() == reflect.Ptr {
iter.ReadVal(exitingValue)
return
}
if iter.WhatIsNext() == NumberValue {
*((*interface{})(ptr)) = json.Number(iter.readNumberAsString())
} else {
@ -208,7 +220,9 @@ func (cfg *frozenConfig) getTagKey() string {
}
func (cfg *frozenConfig) RegisterExtension(extension Extension) {
cfg.extensions = append(cfg.extensions, extension)
cfg.extraExtensions = append(cfg.extraExtensions, extension)
copied := cfg.configBeforeFrozen
cfg.configBeforeFrozen = copied
}
type lossyFloat32Encoder struct {
@ -303,7 +317,7 @@ func (cfg *frozenConfig) MarshalIndent(v interface{}, prefix, indent string) ([]
}
newCfg := cfg.configBeforeFrozen
newCfg.IndentionStep = len(indent)
return newCfg.frozeWithCacheReuse().Marshal(v)
return newCfg.frozeWithCacheReuse(cfg.extraExtensions).Marshal(v)
}
func (cfg *frozenConfig) UnmarshalFromString(str string, v interface{}) error {

View File

@ -7,6 +7,7 @@ import (
"testing"
"time"
"unsafe"
"bytes"
)
func Test_customize_type_decoder(t *testing.T) {
@ -98,3 +99,92 @@ func Test_read_custom_interface(t *testing.T) {
should.Nil(err)
should.Equal("hello", val.Hello())
}
const flow1 = `
{"A":"hello"}
{"A":"hello"}
{"A":"hello"}
{"A":"hello"}
{"A":"hello"}`
const flow2 = `
{"A":"hello"}
{"A":"hello"}
{"A":"hello"}
{"A":"hello"}
{"A":"hello"}
`
type (
Type1 struct {
A string
}
Type2 struct {
A string
}
)
func (t *Type2) UnmarshalJSON(data []byte) error {
return nil
}
func (t *Type2) MarshalJSON() ([]byte, error) {
return nil, nil
}
func TestType1NoFinalLF(t *testing.T) {
reader := bytes.NewReader([]byte(flow1))
dec := jsoniter.NewDecoder(reader)
i := 0
for dec.More() {
data := &Type1{}
if err := dec.Decode(data); err != nil {
t.Errorf("at %v got %v", i, err)
}
i++
}
}
func TestType1FinalLF(t *testing.T) {
reader := bytes.NewReader([]byte(flow2))
dec := jsoniter.NewDecoder(reader)
i := 0
for dec.More() {
data := &Type1{}
if err := dec.Decode(data); err != nil {
t.Errorf("at %v got %v", i, err)
}
i++
}
}
func TestType2NoFinalLF(t *testing.T) {
reader := bytes.NewReader([]byte(flow1))
dec := jsoniter.NewDecoder(reader)
i := 0
for dec.More() {
data := &Type2{}
if err := dec.Decode(data); err != nil {
t.Errorf("at %v got %v", i, err)
}
i++
}
}
func TestType2FinalLF(t *testing.T) {
reader := bytes.NewReader([]byte(flow2))
dec := jsoniter.NewDecoder(reader)
i := 0
for dec.More() {
data := &Type2{}
if err := dec.Decode(data); err != nil {
t.Errorf("at %v got %v", i, err)
}
i++
}
}

View File

@ -2,8 +2,8 @@ package test
import (
"github.com/json-iterator/go"
"github.com/stretchr/testify/require"
"github.com/modern-go/reflect2"
"github.com/stretchr/testify/require"
"reflect"
"strconv"
"testing"

View File

@ -2,9 +2,9 @@ package extra
import (
"github.com/json-iterator/go"
"unsafe"
"unicode/utf8"
"github.com/modern-go/reflect2"
"unicode/utf8"
"unsafe"
)
// safeSet holds the value true if the ASCII character with the given array
@ -171,18 +171,18 @@ func (codec *binaryAsStringCodec) Encode(ptr unsafe.Pointer, stream *jsoniter.St
func readHex(iter *jsoniter.Iterator, b1, b2 byte) byte {
var ret byte
if b1 >= '0' && b1 <= '9' {
ret = b1-'0'
ret = b1 - '0'
} else if b1 >= 'a' && b1 <= 'f' {
ret = b1-'a'+10
ret = b1 - 'a' + 10
} else {
iter.ReportError("read hex", "expects 0~9 or a~f, but found "+string([]byte{b1}))
return 0
}
ret = ret * 16
if b2 >= '0' && b2 <= '9' {
ret = b2-'0'
ret = b2 - '0'
} else if b2 >= 'a' && b2 <= 'f' {
ret = b2-'a'+10
ret = b2 - 'a' + 10
} else {
iter.ReportError("read hex", "expects 0~9 or a~f, but found "+string([]byte{b2}))
return 0

View File

@ -1,9 +1,9 @@
package extra
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/json-iterator/go"
"github.com/stretchr/testify/require"
"testing"
)
func init() {

View File

@ -217,6 +217,9 @@ func (decoder *fuzzyIntegerDecoder) Decode(ptr unsafe.Pointer, iter *jsoniter.It
default:
iter.ReportError("fuzzyIntegerDecoder", "not number or string")
}
if len(str) == 0 {
str = "0"
}
newIter := iter.Pool().BorrowIterator([]byte(str))
defer iter.Pool().ReturnIterator(newIter)
isFloat := strings.IndexByte(str, '.') != -1

View File

@ -37,6 +37,8 @@ func Test_any_to_int64(t *testing.T) {
should.Equal(int64(10), val)
should.Nil(jsoniter.UnmarshalFromString(`10`, &val))
should.Equal(int64(10), val)
should.Nil(jsoniter.UnmarshalFromString(`""`, &val))
should.Equal(int64(0), val)
// bool part
should.Nil(jsoniter.UnmarshalFromString(`false`, &val))

View File

@ -2,6 +2,7 @@ package extra
import (
"github.com/json-iterator/go"
"strings"
"unicode"
)
@ -17,6 +18,16 @@ type namingStrategyExtension struct {
func (extension *namingStrategyExtension) UpdateStructDescriptor(structDescriptor *jsoniter.StructDescriptor) {
for _, binding := range structDescriptor.Fields {
tag, hastag := binding.Field.Tag().Lookup("json")
if hastag {
tagParts := strings.Split(tag, ",")
if tagParts[0] == "-" {
continue // hidden field
}
if tagParts[0] != "" {
continue // field explicitly named
}
}
binding.ToNames = []string{extension.translate(binding.Field.Name())}
binding.FromNames = []string{extension.translate(binding.Field.Name())}
}

View File

@ -21,3 +21,30 @@ func Test_lower_case_with_underscores(t *testing.T) {
should.Nil(err)
should.Equal(`{"user_name":"taowen","first_language":"Chinese"}`, string(output))
}
func Test_set_naming_strategy_with_overrides(t *testing.T) {
should := require.New(t)
SetNamingStrategy(LowerCaseWithUnderscores)
output, err := jsoniter.Marshal(struct {
UserName string `json:"UserName"`
FirstLanguage string
}{
UserName: "taowen",
FirstLanguage: "Chinese",
})
should.Nil(err)
should.Equal(`{"UserName":"taowen","first_language":"Chinese"}`, string(output))
}
func Test_set_naming_strategy_with_omitempty(t *testing.T) {
should := require.New(t)
SetNamingStrategy(LowerCaseWithUnderscores)
output, err := jsoniter.Marshal(struct {
UserName string
FirstLanguage string `json:",omitempty"`
}{
UserName: "taowen",
})
should.Nil(err)
should.Equal(`{"user_name":"taowen"}`, string(output))
}

View File

@ -2,6 +2,7 @@ package extra
import (
"github.com/json-iterator/go"
"strings"
"unicode"
)
@ -18,8 +19,36 @@ func (extension *privateFieldsExtension) UpdateStructDescriptor(structDescriptor
for _, binding := range structDescriptor.Fields {
isPrivate := unicode.IsLower(rune(binding.Field.Name()[0]))
if isPrivate {
tag, hastag := binding.Field.Tag().Lookup("json")
if !hastag {
binding.FromNames = []string{binding.Field.Name()}
binding.ToNames = []string{binding.Field.Name()}
continue
}
tagParts := strings.Split(tag, ",")
names := calcFieldNames(binding.Field.Name(), tagParts[0], tag)
binding.FromNames = names
binding.ToNames = names
}
}
}
func calcFieldNames(originalFieldName string, tagProvidedFieldName string, wholeTag string) []string {
// ignore?
if wholeTag == "-" {
return []string{}
}
// rename?
var fieldNames []string
if tagProvidedFieldName == "" {
fieldNames = []string{originalFieldName}
} else {
fieldNames = []string{tagProvidedFieldName}
}
// private?
isNotExported := unicode.IsLower(rune(originalFieldName[0]))
if isNotExported {
fieldNames = []string{}
}
return fieldNames
}

View File

@ -2,7 +2,7 @@ package jsoniter
import (
"fmt"
"unicode"
"strings"
)
// ReadObject read one field from object.
@ -60,7 +60,7 @@ func (iter *Iterator) readFieldHash() int64 {
if b == '\\' {
iter.head = i
for _, b := range iter.readStringSlowPath() {
if 'A' <= b && b <= 'Z' {
if 'A' <= b && b <= 'Z' && !iter.cfg.caseSensitive {
b += 'a' - 'A'
}
hash ^= int64(b)
@ -82,7 +82,7 @@ func (iter *Iterator) readFieldHash() int64 {
}
return hash
}
if 'A' <= b && b <= 'Z' {
if 'A' <= b && b <= 'Z' && !iter.cfg.caseSensitive {
b += 'a' - 'A'
}
hash ^= int64(b)
@ -95,10 +95,13 @@ func (iter *Iterator) readFieldHash() int64 {
}
}
func calcHash(str string) int64 {
func calcHash(str string, caseSensitive bool) int64 {
if !caseSensitive {
str = strings.ToLower(str)
}
hash := int64(0x811c9dc5)
for _, b := range str {
hash ^= int64(unicode.ToLower(b))
for _, b := range []byte(str) {
hash ^= int64(b)
hash *= 0x1000193
}
return int64(hash)

View File

@ -3,9 +3,10 @@ package misc_tests
import (
"bytes"
"encoding/json"
"testing"
"github.com/json-iterator/go"
"github.com/stretchr/testify/require"
"testing"
)
func Test_empty_array(t *testing.T) {
@ -168,6 +169,17 @@ func Test_decode_byte_array_from_base64(t *testing.T) {
should.Equal([]byte{1, 2, 3}, data)
}
func Test_decode_byte_array_from_base64_with_newlines(t *testing.T) {
should := require.New(t)
data := []byte{}
err := json.Unmarshal([]byte(`"A\rQ\nID"`), &data)
should.Nil(err)
should.Equal([]byte{1, 2, 3}, data)
err = jsoniter.Unmarshal([]byte(`"A\rQ\nID"`), &data)
should.Nil(err)
should.Equal([]byte{1, 2, 3}, data)
}
func Test_decode_byte_array_from_array(t *testing.T) {
should := require.New(t)
data := []byte{}

View File

@ -31,3 +31,14 @@ func Test_read_map_with_reader(t *testing.T) {
should.Equal(m2, m1)
should.Equal("1.0.76", m1["note"].(map[string]interface{})["CoreServices"].(map[string]interface{})["version_name"])
}
func Test_map_eface_of_eface(t *testing.T) {
should := require.New(t)
json := jsoniter.ConfigCompatibleWithStandardLibrary
output, err := json.MarshalToString(map[interface{}]interface{}{
"1": 2,
3: "4",
})
should.NoError(err)
should.Equal(`{"1":2,"3":"4"}`, output)
}

View File

@ -130,3 +130,20 @@ func Test_reader_and_load_more(t *testing.T) {
obj := TestObject{}
should.Nil(decoder.Decode(&obj))
}
func Test_unmarshal_into_existing_value(t *testing.T) {
should := require.New(t)
type TestObject struct {
Field1 int
Field2 interface{}
}
var obj TestObject
m := map[string]interface{}{}
obj.Field2 = &m
cfg := jsoniter.Config{UseNumber: true}.Froze()
err := cfg.Unmarshal([]byte(`{"Field1":1,"Field2":{"k":"v"}}`), &obj)
should.NoError(err)
should.Equal(map[string]interface{}{
"k": "v",
}, m)
}

View File

@ -23,6 +23,7 @@ func (cfg *frozenConfig) BorrowStream(writer io.Writer) *Stream {
}
func (cfg *frozenConfig) ReturnStream(stream *Stream) {
stream.out = nil
stream.Error = nil
stream.Attachment = nil
cfg.streamPool.Put(stream)

View File

@ -2,9 +2,10 @@ package jsoniter
import (
"fmt"
"github.com/modern-go/reflect2"
"reflect"
"unsafe"
"github.com/modern-go/reflect2"
)
// ValDecoder is an internal type registered to cache as needed.
@ -40,6 +41,14 @@ type ctx struct {
decoders map[reflect2.Type]ValDecoder
}
func (b *ctx) caseSensitive() bool {
if b.frozenConfig == nil {
// default is case-insensitive
return false
}
return b.frozenConfig.caseSensitive
}
func (b *ctx) append(prefix string) *ctx {
return &ctx{
frozenConfig: b.frozenConfig,
@ -111,7 +120,8 @@ func decoderOfType(ctx *ctx, typ reflect2.Type) ValDecoder {
for _, extension := range extensions {
decoder = extension.DecorateDecoder(typ, decoder)
}
for _, extension := range ctx.extensions {
decoder = ctx.decoderExtension.DecorateDecoder(typ, decoder)
for _, extension := range ctx.extraExtensions {
decoder = extension.DecorateDecoder(typ, decoder)
}
return decoder
@ -213,7 +223,8 @@ func encoderOfType(ctx *ctx, typ reflect2.Type) ValEncoder {
for _, extension := range extensions {
encoder = extension.DecorateEncoder(typ, encoder)
}
for _, extension := range ctx.extensions {
encoder = ctx.encoderExtension.DecorateEncoder(typ, encoder)
for _, extension := range ctx.extraExtensions {
encoder = extension.DecorateEncoder(typ, encoder)
}
return encoder

View File

@ -246,7 +246,8 @@ func getTypeDecoderFromExtension(ctx *ctx, typ reflect2.Type) ValDecoder {
for _, extension := range extensions {
decoder = extension.DecorateDecoder(typ, decoder)
}
for _, extension := range ctx.extensions {
decoder = ctx.decoderExtension.DecorateDecoder(typ, decoder)
for _, extension := range ctx.extraExtensions {
decoder = extension.DecorateDecoder(typ, decoder)
}
}
@ -259,14 +260,18 @@ func _getTypeDecoderFromExtension(ctx *ctx, typ reflect2.Type) ValDecoder {
return decoder
}
}
for _, extension := range ctx.extensions {
decoder := ctx.decoderExtension.CreateDecoder(typ)
if decoder != nil {
return decoder
}
for _, extension := range ctx.extraExtensions {
decoder := extension.CreateDecoder(typ)
if decoder != nil {
return decoder
}
}
typeName := typ.String()
decoder := typeDecoders[typeName]
decoder = typeDecoders[typeName]
if decoder != nil {
return decoder
}
@ -286,7 +291,8 @@ func getTypeEncoderFromExtension(ctx *ctx, typ reflect2.Type) ValEncoder {
for _, extension := range extensions {
encoder = extension.DecorateEncoder(typ, encoder)
}
for _, extension := range ctx.extensions {
encoder = ctx.encoderExtension.DecorateEncoder(typ, encoder)
for _, extension := range ctx.extraExtensions {
encoder = extension.DecorateEncoder(typ, encoder)
}
}
@ -300,14 +306,18 @@ func _getTypeEncoderFromExtension(ctx *ctx, typ reflect2.Type) ValEncoder {
return encoder
}
}
for _, extension := range ctx.extensions {
encoder := ctx.encoderExtension.CreateEncoder(typ)
if encoder != nil {
return encoder
}
for _, extension := range ctx.extraExtensions {
encoder := extension.CreateEncoder(typ)
if encoder != nil {
return encoder
}
}
typeName := typ.String()
encoder := typeEncoders[typeName]
encoder = typeEncoders[typeName]
if encoder != nil {
return encoder
}
@ -393,7 +403,9 @@ func createStructDescriptor(ctx *ctx, typ reflect2.Type, bindings []*Binding, em
for _, extension := range extensions {
extension.UpdateStructDescriptor(structDescriptor)
}
for _, extension := range ctx.extensions {
ctx.encoderExtension.UpdateStructDescriptor(structDescriptor)
ctx.decoderExtension.UpdateStructDescriptor(structDescriptor)
for _, extension := range ctx.extraExtensions {
extension.UpdateStructDescriptor(structDescriptor)
}
processTags(structDescriptor, ctx.frozenConfig)

View File

@ -3,6 +3,7 @@ package jsoniter
import (
"fmt"
"github.com/modern-go/reflect2"
"io"
"reflect"
"sort"
"unsafe"
@ -38,7 +39,11 @@ func encoderOfMap(ctx *ctx, typ reflect2.Type) ValEncoder {
}
func decoderOfMapKey(ctx *ctx, typ reflect2.Type) ValDecoder {
for _, extension := range ctx.extensions {
decoder := ctx.decoderExtension.CreateMapKeyDecoder(typ)
if decoder != nil {
return decoder
}
for _, extension := range ctx.extraExtensions {
decoder := extension.CreateMapKeyDecoder(typ)
if decoder != nil {
return decoder
@ -76,7 +81,11 @@ func decoderOfMapKey(ctx *ctx, typ reflect2.Type) ValDecoder {
}
func encoderOfMapKey(ctx *ctx, typ reflect2.Type) ValEncoder {
for _, extension := range ctx.extensions {
encoder := ctx.encoderExtension.CreateMapKeyEncoder(typ)
if encoder != nil {
return encoder
}
for _, extension := range ctx.extraExtensions {
encoder := extension.CreateMapKeyEncoder(typ)
if encoder != nil {
return encoder
@ -107,6 +116,9 @@ func encoderOfMapKey(ctx *ctx, typ reflect2.Type) ValEncoder {
stringEncoder: ctx.EncoderOf(reflect2.TypeOf("")),
}
}
if typ.Kind() == reflect.Interface {
return &dynamicMapKeyEncoder{ctx, typ}
}
return &lazyErrorEncoder{err: fmt.Errorf("unsupported map key type: %v", typ)}
}
}
@ -203,6 +215,21 @@ func (encoder *numericMapKeyEncoder) IsEmpty(ptr unsafe.Pointer) bool {
return false
}
type dynamicMapKeyEncoder struct {
ctx *ctx
valType reflect2.Type
}
func (encoder *dynamicMapKeyEncoder) Encode(ptr unsafe.Pointer, stream *Stream) {
obj := encoder.valType.UnsafeIndirect(ptr)
encoderOfMapKey(encoder.ctx, reflect2.TypeOf(obj)).Encode(reflect2.PtrOf(obj), stream)
}
func (encoder *dynamicMapKeyEncoder) IsEmpty(ptr unsafe.Pointer) bool {
obj := encoder.valType.UnsafeIndirect(ptr)
return encoderOfMapKey(encoder.ctx, reflect2.TypeOf(obj)).IsEmpty(reflect2.PtrOf(obj))
}
type mapEncoder struct {
mapType *reflect2.UnsafeMapType
keyEncoder ValEncoder
@ -253,6 +280,9 @@ func (encoder *sortKeysMapEncoder) Encode(ptr unsafe.Pointer, stream *Stream) {
subStream.buf = make([]byte, 0, 64)
key, elem := mapIter.UnsafeNext()
encoder.keyEncoder.Encode(key, subStream)
if subStream.Error != nil && subStream.Error != io.EOF && stream.Error == nil {
stream.Error = subStream.Error
}
encodedKey := subStream.Buffer()
subIter.ResetBytes(encodedKey)
decodedKey := subIter.ReadString()

View File

@ -2,10 +2,11 @@ package jsoniter
import (
"encoding/base64"
"github.com/modern-go/reflect2"
"reflect"
"strconv"
"unsafe"
"github.com/modern-go/reflect2"
)
const ptrSize = 32 << uintptr(^uintptr(0)>>63)
@ -416,16 +417,11 @@ func (codec *base64Codec) Decode(ptr unsafe.Pointer, iter *Iterator) {
}
switch iter.WhatIsNext() {
case StringValue:
encoding := base64.StdEncoding
src := iter.SkipAndReturnBytes()
src = src[1 : len(src)-1]
decodedLen := encoding.DecodedLen(len(src))
dst := make([]byte, decodedLen)
len, err := encoding.Decode(dst, src)
src := iter.ReadString()
dst, err := base64.StdEncoding.DecodeString(src)
if err != nil {
iter.ReportError("decode base64", err.Error())
} else {
dst = dst[:len]
codec.sliceType.UnsafeSet(ptr, unsafe.Pointer(&dst))
}
case ArrayValue:

View File

@ -2,10 +2,11 @@ package jsoniter
import (
"fmt"
"github.com/modern-go/reflect2"
"io"
"strings"
"unsafe"
"github.com/modern-go/reflect2"
)
func decoderOfStruct(ctx *ctx, typ reflect2.Type) ValDecoder {
@ -31,6 +32,15 @@ func decoderOfStruct(ctx *ctx, typ reflect2.Type) ValDecoder {
for k, binding := range bindings {
fields[k] = binding.Decoder.(*structFieldDecoder)
}
if !ctx.caseSensitive() {
for k, binding := range bindings {
if _, found := fields[strings.ToLower(k)]; !found {
fields[strings.ToLower(k)] = binding.Decoder.(*structFieldDecoder)
}
}
}
return createStructDecoder(ctx, typ, fields)
}
@ -41,12 +51,13 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF
knownHash := map[int64]struct{}{
0: {},
}
switch len(fields) {
case 0:
return &skipObjectDecoder{typ}
case 1:
for fieldName, fieldDecoder := range fields {
fieldHash := calcHash(fieldName)
fieldHash := calcHash(fieldName, ctx.caseSensitive())
_, known := knownHash[fieldHash]
if known {
return &generalStructDecoder{typ, fields, false}
@ -60,7 +71,7 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF
var fieldDecoder1 *structFieldDecoder
var fieldDecoder2 *structFieldDecoder
for fieldName, fieldDecoder := range fields {
fieldHash := calcHash(fieldName)
fieldHash := calcHash(fieldName, ctx.caseSensitive())
_, known := knownHash[fieldHash]
if known {
return &generalStructDecoder{typ, fields, false}
@ -83,7 +94,7 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF
var fieldDecoder2 *structFieldDecoder
var fieldDecoder3 *structFieldDecoder
for fieldName, fieldDecoder := range fields {
fieldHash := calcHash(fieldName)
fieldHash := calcHash(fieldName, ctx.caseSensitive())
_, known := knownHash[fieldHash]
if known {
return &generalStructDecoder{typ, fields, false}
@ -114,7 +125,7 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF
var fieldDecoder3 *structFieldDecoder
var fieldDecoder4 *structFieldDecoder
for fieldName, fieldDecoder := range fields {
fieldHash := calcHash(fieldName)
fieldHash := calcHash(fieldName, ctx.caseSensitive())
_, known := knownHash[fieldHash]
if known {
return &generalStructDecoder{typ, fields, false}
@ -151,7 +162,7 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF
var fieldDecoder4 *structFieldDecoder
var fieldDecoder5 *structFieldDecoder
for fieldName, fieldDecoder := range fields {
fieldHash := calcHash(fieldName)
fieldHash := calcHash(fieldName, ctx.caseSensitive())
_, known := knownHash[fieldHash]
if known {
return &generalStructDecoder{typ, fields, false}
@ -194,7 +205,7 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF
var fieldDecoder5 *structFieldDecoder
var fieldDecoder6 *structFieldDecoder
for fieldName, fieldDecoder := range fields {
fieldHash := calcHash(fieldName)
fieldHash := calcHash(fieldName, ctx.caseSensitive())
_, known := knownHash[fieldHash]
if known {
return &generalStructDecoder{typ, fields, false}
@ -243,7 +254,7 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF
var fieldDecoder6 *structFieldDecoder
var fieldDecoder7 *structFieldDecoder
for fieldName, fieldDecoder := range fields {
fieldHash := calcHash(fieldName)
fieldHash := calcHash(fieldName, ctx.caseSensitive())
_, known := knownHash[fieldHash]
if known {
return &generalStructDecoder{typ, fields, false}
@ -298,7 +309,7 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF
var fieldDecoder7 *structFieldDecoder
var fieldDecoder8 *structFieldDecoder
for fieldName, fieldDecoder := range fields {
fieldHash := calcHash(fieldName)
fieldHash := calcHash(fieldName, ctx.caseSensitive())
_, known := knownHash[fieldHash]
if known {
return &generalStructDecoder{typ, fields, false}
@ -359,7 +370,7 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF
var fieldDecoder8 *structFieldDecoder
var fieldDecoder9 *structFieldDecoder
for fieldName, fieldDecoder := range fields {
fieldHash := calcHash(fieldName)
fieldHash := calcHash(fieldName, ctx.caseSensitive())
_, known := knownHash[fieldHash]
if known {
return &generalStructDecoder{typ, fields, false}
@ -426,7 +437,7 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF
var fieldDecoder9 *structFieldDecoder
var fieldDecoder10 *structFieldDecoder
for fieldName, fieldDecoder := range fields {
fieldHash := calcHash(fieldName)
fieldHash := calcHash(fieldName, ctx.caseSensitive())
_, known := knownHash[fieldHash]
if known {
return &generalStructDecoder{typ, fields, false}
@ -489,13 +500,16 @@ func (decoder *generalStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator)
if !iter.readObjectStart() {
return
}
decoder.decodeOneField(ptr, iter)
for iter.nextToken() == ',' {
var c byte
for c = ','; c == ','; c = iter.nextToken() {
decoder.decodeOneField(ptr, iter)
}
if iter.Error != nil && iter.Error != io.EOF {
iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error())
}
if c != '}' {
iter.ReportError("struct Decode", `expect }, but found `+string([]byte{c}))
}
}
func (decoder *generalStructDecoder) decodeOneField(ptr unsafe.Pointer, iter *Iterator) {
@ -505,13 +519,13 @@ func (decoder *generalStructDecoder) decodeOneField(ptr unsafe.Pointer, iter *It
fieldBytes := iter.ReadStringAsSlice()
field = *(*string)(unsafe.Pointer(&fieldBytes))
fieldDecoder = decoder.fields[field]
if fieldDecoder == nil {
if fieldDecoder == nil && !iter.cfg.caseSensitive {
fieldDecoder = decoder.fields[strings.ToLower(field)]
}
} else {
field = iter.ReadString()
fieldDecoder = decoder.fields[field]
if fieldDecoder == nil {
if fieldDecoder == nil && !iter.cfg.caseSensitive {
fieldDecoder = decoder.fields[strings.ToLower(field)]
}
}

View File

@ -145,6 +145,9 @@ func init() {
(*struct {
Field bool `json:",omitempty,string"`
})(nil),
(*struct {
Field bool `json:"中文"`
})(nil),
)
}

36
value_tests/error_test.go Normal file
View File

@ -0,0 +1,36 @@
package test
import (
"github.com/json-iterator/go"
"github.com/stretchr/testify/require"
"reflect"
"testing"
)
func Test_errorInput(t *testing.T) {
for _, testCase := range unmarshalCases {
if testCase.obj != nil {
continue
}
valType := reflect.TypeOf(testCase.ptr).Elem()
t.Run(valType.String(), func(t *testing.T) {
for _, data := range []string{
`x`,
`n`,
`nul`,
`{x}`,
`{"x"}`,
`{"x": "y"x}`,
`{"x": "y"`,
`{"x": "y", "a"}`,
`[`,
`[{"x": "y"}`,
} {
ptrVal := reflect.New(valType)
ptr := ptrVal.Interface()
err := jsoniter.ConfigCompatibleWithStandardLibrary.Unmarshal([]byte(data), ptr)
require.Error(t, err, "on input %q", data)
}
})
}
}

View File

@ -1,10 +1,18 @@
package test
import "encoding/json"
import (
"encoding/json"
)
func init() {
marshalCases = append(marshalCases,
json.RawMessage("{}"),
selectedMarshalCase{struct {
Env string `json:"env"`
Extra json.RawMessage `json:"extra,omitempty"`
}{
Env: "jfdk",
}},
)
unmarshalCases = append(unmarshalCases, unmarshalCase{
ptr: (*json.RawMessage)(nil),

View File

@ -6,7 +6,7 @@ func init() {
[]interface{}{"hello"},
nilSlice,
&nilSlice,
selectedMarshalCase{[]byte{1,2,3}},
[]byte{1, 2, 3},
)
unmarshalCases = append(unmarshalCases, unmarshalCase{
ptr: (*[]string)(nil),
@ -20,6 +20,8 @@ func init() {
}, unmarshalCase{
ptr: (*[]byte)(nil),
input: `"aGVsbG8="`,
selected: true,
}, unmarshalCase{
ptr: (*[]byte)(nil),
input: `"c3ViamVjdHM\/X2Q9MQ=="`,
})
}

View File

@ -63,6 +63,43 @@ func init() {
d *time.Timer
})(nil),
input: `{"a": 444, "b":"bad", "C":256, "d":{"not":"a timer"}}`,
}, unmarshalCase{
ptr: (*struct {
A string
B string
C string
D string
E string
F string
G string
H string
I string
J string
K string
})(nil),
input: `{"a":"1","b":"2","c":"3","d":"4","e":"5","f":"6","g":"7","h":"8","i":"9","j":"10","k":"11"}`,
}, unmarshalCase{
ptr: (*struct {
T float64 `json:"T"`
})(nil),
input: `{"t":10.0}`,
}, unmarshalCase{
ptr: (*struct {
T float64 `json:"T"`
})(nil),
input: `{"T":10.0}`,
}, unmarshalCase{
ptr: (*struct {
T float64 `json:"t"`
})(nil),
input: `{"T":10.0}`,
}, unmarshalCase{
ptr: (*struct {
KeyString string `json:"key_string"`
Type string `json:"type"`
Asks [][2]float64 `json:"asks"`
})(nil),
input: `{"key_string": "KEYSTRING","type": "TYPE","asks": [[1e+66,1]]}`,
})
marshalCases = append(marshalCases,
struct {

View File

@ -4,8 +4,8 @@ import (
"encoding/json"
"fmt"
"github.com/json-iterator/go"
"github.com/stretchr/testify/require"
"github.com/modern-go/reflect2"
"github.com/stretchr/testify/require"
"testing"
)