From b92cf78708ec8f0f12daad8579188cbd24d7dfeb Mon Sep 17 00:00:00 2001 From: Chao Xu Date: Thu, 7 Jun 2018 18:36:06 -0700 Subject: [PATCH 1/2] Make case sensitivity optional. Fix https://github.com/kubernetes/kubernetes/issues/64612 --- Gopkg.lock | 8 +++++++- config.go | 8 ++++++-- iter_object.go | 12 ++++++++---- reflect.go | 11 ++++++++++- reflect_struct_decoder.go | 23 ++++++++++++----------- 5 files changed, 43 insertions(+), 19 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 338f1c6..3719afe 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1,6 +1,12 @@ # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. +[[projects]] + name = "github.com/json-iterator/go" + packages = ["."] + revision = "ca39e5af3ece67bbcda3d0f4f56a8e24d9f2dad4" + version = "1.1.3" + [[projects]] name = "github.com/modern-go/concurrent" packages = ["."] @@ -16,6 +22,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "ac7003b5a981716353a43055ab7d4c5357403cb30a60de2dbdeb446c1544beaa" + inputs-digest = "56a0b9e9e61d2bc8af5e1b68537401b7f4d60805eda3d107058f3171aa5cf793" solver-name = "gps-cdcl" solver-version = 1 diff --git a/config.go b/config.go index bd66947..8358191 100644 --- a/config.go +++ b/config.go @@ -2,12 +2,13 @@ package jsoniter import ( "encoding/json" - "github.com/modern-go/concurrent" - "github.com/modern-go/reflect2" "io" "reflect" "sync" "unsafe" + + "github.com/modern-go/concurrent" + "github.com/modern-go/reflect2" ) // Config customize how the API should behave. @@ -23,6 +24,7 @@ type Config struct { OnlyTaggedField bool ValidateJsonRawMessage bool ObjectFieldMustBeSimpleString bool + CaseSensitive bool } // API the public interface of this package. @@ -75,6 +77,7 @@ type frozenConfig struct { extensions []Extension streamPool *sync.Pool iteratorPool *sync.Pool + caseSensitive bool } func (cfg *frozenConfig) initCache() { @@ -128,6 +131,7 @@ func (cfg Config) Froze() API { objectFieldMustBeSimpleString: cfg.ObjectFieldMustBeSimpleString, onlyTaggedField: cfg.OnlyTaggedField, disallowUnknownFields: cfg.DisallowUnknownFields, + caseSensitive: cfg.CaseSensitive, } api.streamPool = &sync.Pool{ New: func() interface{} { diff --git a/iter_object.go b/iter_object.go index ebd3da8..6e7c370 100644 --- a/iter_object.go +++ b/iter_object.go @@ -60,7 +60,7 @@ func (iter *Iterator) readFieldHash() int64 { if b == '\\' { iter.head = i for _, b := range iter.readStringSlowPath() { - if 'A' <= b && b <= 'Z' { + if 'A' <= b && b <= 'Z' && !iter.cfg.caseSensitive { b += 'a' - 'A' } hash ^= int64(b) @@ -82,7 +82,7 @@ func (iter *Iterator) readFieldHash() int64 { } return hash } - if 'A' <= b && b <= 'Z' { + if 'A' <= b && b <= 'Z' && !iter.cfg.caseSensitive { b += 'a' - 'A' } hash ^= int64(b) @@ -95,10 +95,14 @@ func (iter *Iterator) readFieldHash() int64 { } } -func calcHash(str string) int64 { +func calcHash(str string, caseSensitive bool) int64 { hash := int64(0x811c9dc5) for _, b := range str { - hash ^= int64(unicode.ToLower(b)) + if caseSensitive { + hash ^= int64(b) + } else { + hash ^= int64(unicode.ToLower(b)) + } hash *= 0x1000193 } return int64(hash) diff --git a/reflect.go b/reflect.go index 5c7d3a8..be7a0e2 100644 --- a/reflect.go +++ b/reflect.go @@ -2,9 +2,10 @@ package jsoniter import ( "fmt" - "github.com/modern-go/reflect2" "reflect" "unsafe" + + "github.com/modern-go/reflect2" ) // ValDecoder is an internal type registered to cache as needed. @@ -40,6 +41,14 @@ type ctx struct { decoders map[reflect2.Type]ValDecoder } +func (b *ctx) caseSensitive() bool { + if b.frozenConfig == nil { + // default is case-insensitive + return false + } + return b.frozenConfig.caseSensitive +} + func (b *ctx) append(prefix string) *ctx { return &ctx{ frozenConfig: b.frozenConfig, diff --git a/reflect_struct_decoder.go b/reflect_struct_decoder.go index fd76647..c76cecd 100644 --- a/reflect_struct_decoder.go +++ b/reflect_struct_decoder.go @@ -2,10 +2,11 @@ package jsoniter import ( "fmt" - "github.com/modern-go/reflect2" "io" "strings" "unsafe" + + "github.com/modern-go/reflect2" ) func decoderOfStruct(ctx *ctx, typ reflect2.Type) ValDecoder { @@ -51,7 +52,7 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF return &skipObjectDecoder{typ} case 1: for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName) + fieldHash := calcHash(fieldName, ctx.caseSensitive()) _, known := knownHash[fieldHash] if known { return &generalStructDecoder{typ, fields, false} @@ -65,7 +66,7 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF var fieldDecoder1 *structFieldDecoder var fieldDecoder2 *structFieldDecoder for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName) + fieldHash := calcHash(fieldName, ctx.caseSensitive()) _, known := knownHash[fieldHash] if known { return &generalStructDecoder{typ, fields, false} @@ -88,7 +89,7 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF var fieldDecoder2 *structFieldDecoder var fieldDecoder3 *structFieldDecoder for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName) + fieldHash := calcHash(fieldName, ctx.caseSensitive()) _, known := knownHash[fieldHash] if known { return &generalStructDecoder{typ, fields, false} @@ -119,7 +120,7 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF var fieldDecoder3 *structFieldDecoder var fieldDecoder4 *structFieldDecoder for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName) + fieldHash := calcHash(fieldName, ctx.caseSensitive()) _, known := knownHash[fieldHash] if known { return &generalStructDecoder{typ, fields, false} @@ -156,7 +157,7 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF var fieldDecoder4 *structFieldDecoder var fieldDecoder5 *structFieldDecoder for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName) + fieldHash := calcHash(fieldName, ctx.caseSensitive()) _, known := knownHash[fieldHash] if known { return &generalStructDecoder{typ, fields, false} @@ -199,7 +200,7 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF var fieldDecoder5 *structFieldDecoder var fieldDecoder6 *structFieldDecoder for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName) + fieldHash := calcHash(fieldName, ctx.caseSensitive()) _, known := knownHash[fieldHash] if known { return &generalStructDecoder{typ, fields, false} @@ -248,7 +249,7 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF var fieldDecoder6 *structFieldDecoder var fieldDecoder7 *structFieldDecoder for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName) + fieldHash := calcHash(fieldName, ctx.caseSensitive()) _, known := knownHash[fieldHash] if known { return &generalStructDecoder{typ, fields, false} @@ -303,7 +304,7 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF var fieldDecoder7 *structFieldDecoder var fieldDecoder8 *structFieldDecoder for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName) + fieldHash := calcHash(fieldName, ctx.caseSensitive()) _, known := knownHash[fieldHash] if known { return &generalStructDecoder{typ, fields, false} @@ -364,7 +365,7 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF var fieldDecoder8 *structFieldDecoder var fieldDecoder9 *structFieldDecoder for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName) + fieldHash := calcHash(fieldName, ctx.caseSensitive()) _, known := knownHash[fieldHash] if known { return &generalStructDecoder{typ, fields, false} @@ -431,7 +432,7 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF var fieldDecoder9 *structFieldDecoder var fieldDecoder10 *structFieldDecoder for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName) + fieldHash := calcHash(fieldName, ctx.caseSensitive()) _, known := knownHash[fieldHash] if known { return &generalStructDecoder{typ, fields, false} From 3830516ed0efb2a55f64eb22cf7894fecd8aea4c Mon Sep 17 00:00:00 2001 From: Nikhita Raghunath Date: Mon, 11 Jun 2018 15:35:33 +0530 Subject: [PATCH 2/2] Fix case sensitivity for nested fields --- api_tests/config_test.go | 129 +++++++++++++++++++++++++++++++++++++- reflect_struct_decoder.go | 15 +++-- 2 files changed, 138 insertions(+), 6 deletions(-) diff --git a/api_tests/config_test.go b/api_tests/config_test.go index d3e93e5..7aa349b 100644 --- a/api_tests/config_test.go +++ b/api_tests/config_test.go @@ -2,9 +2,10 @@ package test import ( "encoding/json" + "testing" + "github.com/json-iterator/go" "github.com/stretchr/testify/require" - "testing" ) func Test_use_number_for_unmarshal(t *testing.T) { @@ -45,3 +46,129 @@ func Test_read_large_number_as_interface(t *testing.T) { should.Nil(err) should.Equal(`123456789123456789123456789`, output) } + +type caseSensitiveStruct struct { + A string `json:"a"` + B string `json:"b,omitempty"` + C *C `json:"C,omitempty"` +} + +type C struct { + D int64 `json:"D,omitempty"` + E *E `json:"e,omitempty"` +} + +type E struct { + F string `json:"F,omitempty"` +} + +func Test_CaseSensitive(t *testing.T) { + should := require.New(t) + + testCases := []struct { + input string + expectedOutput string + caseSensitive bool + }{ + { + input: `{"A":"foo","B":"bar"}`, + expectedOutput: `{"a":"foo","b":"bar"}`, + caseSensitive: false, + }, + { + input: `{"a":"foo","b":"bar"}`, + expectedOutput: `{"a":"foo","b":"bar"}`, + caseSensitive: true, + }, + { + input: `{"a":"foo","b":"bar","C":{"D":10}}`, + expectedOutput: `{"a":"foo","b":"bar","C":{"D":10}}`, + caseSensitive: true, + }, + { + input: `{"a":"foo","B":"bar","c":{"d":10}}`, + expectedOutput: `{"a":"foo"}`, + caseSensitive: true, + }, + { + input: `{"a":"foo","C":{"d":10}}`, + expectedOutput: `{"a":"foo","C":{}}`, + caseSensitive: true, + }, + { + input: `{"a":"foo","C":{"D":10,"e":{"f":"baz"}}}`, + expectedOutput: `{"a":"foo","C":{"D":10,"e":{}}}`, + caseSensitive: true, + }, + { + input: `{"a":"foo","C":{"D":10,"e":{"F":"baz"}}}`, + expectedOutput: `{"a":"foo","C":{"D":10,"e":{"F":"baz"}}}`, + caseSensitive: true, + }, + { + input: `{"A":"foo","c":{"d":10,"E":{"f":"baz"}}}`, + expectedOutput: `{"a":"foo","C":{"D":10,"e":{"F":"baz"}}}`, + caseSensitive: false, + }, + } + + for _, tc := range testCases { + val := caseSensitiveStruct{} + err := jsoniter.Config{CaseSensitive: tc.caseSensitive}.Froze().UnmarshalFromString(tc.input, &val) + should.Nil(err) + + output, err := jsoniter.MarshalToString(val) + should.Nil(err) + should.Equal(tc.expectedOutput, output) + } +} + +type structWithElevenFields struct { + A string `json:"A,omitempty"` + B string `json:"B,omitempty"` + C string `json:"C,omitempty"` + D string `json:"d,omitempty"` + E string `json:"e,omitempty"` + F string `json:"f,omitempty"` + G string `json:"g,omitempty"` + H string `json:"h,omitempty"` + I string `json:"i,omitempty"` + J string `json:"j,omitempty"` + K string `json:"k,omitempty"` +} + +func Test_CaseSensitive_MoreThanTenFields(t *testing.T) { + should := require.New(t) + + testCases := []struct { + input string + expectedOutput string + caseSensitive bool + }{ + { + input: `{"A":"1","B":"2","C":"3","d":"4","e":"5","f":"6","g":"7","h":"8","i":"9","j":"10","k":"11"}`, + expectedOutput: `{"A":"1","B":"2","C":"3","d":"4","e":"5","f":"6","g":"7","h":"8","i":"9","j":"10","k":"11"}`, + caseSensitive: true, + }, + { + input: `{"a":"1","b":"2","c":"3","D":"4","E":"5","F":"6"}`, + expectedOutput: `{"A":"1","B":"2","C":"3","d":"4","e":"5","f":"6"}`, + caseSensitive: false, + }, + { + input: `{"A":"1","b":"2","d":"4","E":"5"}`, + expectedOutput: `{"A":"1","d":"4"}`, + caseSensitive: true, + }, + } + + for _, tc := range testCases { + val := structWithElevenFields{} + err := jsoniter.Config{CaseSensitive: tc.caseSensitive}.Froze().UnmarshalFromString(tc.input, &val) + should.Nil(err) + + output, err := jsoniter.MarshalToString(val) + should.Nil(err) + should.Equal(tc.expectedOutput, output) + } +} diff --git a/reflect_struct_decoder.go b/reflect_struct_decoder.go index c76cecd..355d2d1 100644 --- a/reflect_struct_decoder.go +++ b/reflect_struct_decoder.go @@ -32,11 +32,15 @@ func decoderOfStruct(ctx *ctx, typ reflect2.Type) ValDecoder { for k, binding := range bindings { fields[k] = binding.Decoder.(*structFieldDecoder) } - for k, binding := range bindings { - if _, found := fields[strings.ToLower(k)]; !found { - fields[strings.ToLower(k)] = binding.Decoder.(*structFieldDecoder) + + if !ctx.caseSensitive() { + for k, binding := range bindings { + if _, found := fields[strings.ToLower(k)]; !found { + fields[strings.ToLower(k)] = binding.Decoder.(*structFieldDecoder) + } } } + return createStructDecoder(ctx, typ, fields) } @@ -47,6 +51,7 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF knownHash := map[int64]struct{}{ 0: {}, } + switch len(fields) { case 0: return &skipObjectDecoder{typ} @@ -514,13 +519,13 @@ func (decoder *generalStructDecoder) decodeOneField(ptr unsafe.Pointer, iter *It fieldBytes := iter.ReadStringAsSlice() field = *(*string)(unsafe.Pointer(&fieldBytes)) fieldDecoder = decoder.fields[field] - if fieldDecoder == nil { + if fieldDecoder == nil && !iter.cfg.caseSensitive { fieldDecoder = decoder.fields[strings.ToLower(field)] } } else { field = iter.ReadString() fieldDecoder = decoder.fields[field] - if fieldDecoder == nil { + if fieldDecoder == nil && !iter.cfg.caseSensitive { fieldDecoder = decoder.fields[strings.ToLower(field)] } }