fix #295 decoder more was not compatible with standard library

adapter.go

@@ -81,10 +81,12 @@ func (adapter *Decoder) More() bool {
 	if iter.Error != nil {
 		return false
 	}
-	if iter.head != iter.tail {
+	c := iter.nextToken()
-		return true
+	if c == 0 {
+		return false
 	}
-	return iter.loadMore()
+	iter.unreadByte()
+	return c != ']' && c != '}'
 }
 
 // Buffered remaining buffer
@@ -98,7 +100,7 @@ func (adapter *Decoder) Buffered() io.Reader {
 func (adapter *Decoder) UseNumber() {
 	cfg := adapter.iter.cfg.configBeforeFrozen
 	cfg.UseNumber = true
-	adapter.iter.cfg = cfg.frozeWithCacheReuse()
+	adapter.iter.cfg = cfg.frozeWithCacheReuse(adapter.iter.cfg.extraExtensions)
 }
 
 // DisallowUnknownFields causes the Decoder to return an error when the destination
@@ -107,7 +109,7 @@ func (adapter *Decoder) UseNumber() {
 func (adapter *Decoder) DisallowUnknownFields() {
 	cfg := adapter.iter.cfg.configBeforeFrozen
 	cfg.DisallowUnknownFields = true
-	adapter.iter.cfg = cfg.frozeWithCacheReuse()
+	adapter.iter.cfg = cfg.frozeWithCacheReuse(adapter.iter.cfg.extraExtensions)
 }
 
 // NewEncoder same as json.NewEncoder
@@ -132,14 +134,14 @@ func (adapter *Encoder) Encode(val interface{}) error {
 func (adapter *Encoder) SetIndent(prefix, indent string) {
 	config := adapter.stream.cfg.configBeforeFrozen
 	config.IndentionStep = len(indent)
-	adapter.stream.cfg = config.frozeWithCacheReuse()
+	adapter.stream.cfg = config.frozeWithCacheReuse(adapter.stream.cfg.extraExtensions)
 }
 
 // SetEscapeHTML escape html by default, set to false to disable
 func (adapter *Encoder) SetEscapeHTML(escapeHTML bool) {
 	config := adapter.stream.cfg.configBeforeFrozen
 	config.EscapeHTML = escapeHTML
-	adapter.stream.cfg = config.frozeWithCacheReuse()
+	adapter.stream.cfg = config.frozeWithCacheReuse(adapter.stream.cfg.extraExtensions)
 }
 
 // Valid reports whether data is a valid JSON encoding.

config.go

@@ -74,7 +74,9 @@ type frozenConfig struct {
 	disallowUnknownFields         bool
 	decoderCache                  *concurrent.Map
 	encoderCache                  *concurrent.Map
-	extensions                    []Extension
+	encoderExtension              Extension
+	decoderExtension              Extension
+	extraExtensions               []Extension
 	streamPool                    *sync.Pool
 	iteratorPool                  *sync.Pool
 	caseSensitive                 bool
@@ -158,22 +160,21 @@ func (cfg Config) Froze() API {
 	if cfg.ValidateJsonRawMessage {
 		api.validateJsonRawMessage(encoderExtension)
 	}
-	if len(encoderExtension) > 0 {
+	api.encoderExtension = encoderExtension
-		api.extensions = append(api.extensions, encoderExtension)
+	api.decoderExtension = decoderExtension
-	}
-	if len(decoderExtension) > 0 {
-		api.extensions = append(api.extensions, decoderExtension)
-	}
 	api.configBeforeFrozen = cfg
 	return api
 }
 
-func (cfg Config) frozeWithCacheReuse() *frozenConfig {
+func (cfg Config) frozeWithCacheReuse(extraExtensions []Extension) *frozenConfig {
 	api := getFrozenConfigFromCache(cfg)
 	if api != nil {
 		return api
 	}
 	api = cfg.Froze().(*frozenConfig)
+	for _, extension := range extraExtensions {
+		api.RegisterExtension(extension)
+	}
 	addFrozenConfigToCache(cfg, api)
 	return api
 }
@@ -190,7 +191,7 @@ func (cfg *frozenConfig) validateJsonRawMessage(extension EncoderExtension) {
 			stream.WriteRaw(string(rawMessage))
 		}
 	}, func(ptr unsafe.Pointer) bool {
-		return false
+		return len(*((*json.RawMessage)(ptr))) == 0
 	}}
 	extension[reflect2.TypeOfPtr((*json.RawMessage)(nil)).Elem()] = encoder
 	extension[reflect2.TypeOfPtr((*RawMessage)(nil)).Elem()] = encoder
@@ -219,7 +220,9 @@ func (cfg *frozenConfig) getTagKey() string {
 }
 
 func (cfg *frozenConfig) RegisterExtension(extension Extension) {
-	cfg.extensions = append(cfg.extensions, extension)
+	cfg.extraExtensions = append(cfg.extraExtensions, extension)
+	copied := cfg.configBeforeFrozen
+	cfg.configBeforeFrozen = copied
 }
 
 type lossyFloat32Encoder struct {
@@ -314,7 +317,7 @@ func (cfg *frozenConfig) MarshalIndent(v interface{}, prefix, indent string) ([]
 	}
 	newCfg := cfg.configBeforeFrozen
 	newCfg.IndentionStep = len(indent)
-	return newCfg.frozeWithCacheReuse().Marshal(v)
+	return newCfg.frozeWithCacheReuse(cfg.extraExtensions).Marshal(v)
 }
 
 func (cfg *frozenConfig) UnmarshalFromString(str string, v interface{}) error {

extension_tests/decoder_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 	"unsafe"
+	"bytes"
 )
 
 func Test_customize_type_decoder(t *testing.T) {
@@ -98,3 +99,92 @@ func Test_read_custom_interface(t *testing.T) {
 	should.Nil(err)
 	should.Equal("hello", val.Hello())
 }
+
+const flow1 = `
+{"A":"hello"}
+{"A":"hello"}
+{"A":"hello"}
+{"A":"hello"}
+{"A":"hello"}`
+
+const flow2 = `
+{"A":"hello"}
+{"A":"hello"}
+{"A":"hello"}
+{"A":"hello"}
+{"A":"hello"}
+`
+
+type (
+	Type1 struct {
+		A string
+	}
+
+	Type2 struct {
+		A string
+	}
+)
+
+func (t *Type2) UnmarshalJSON(data []byte) error {
+	return nil
+}
+
+func (t *Type2) MarshalJSON() ([]byte, error) {
+	return nil, nil
+}
+
+func TestType1NoFinalLF(t *testing.T) {
+	reader := bytes.NewReader([]byte(flow1))
+	dec := jsoniter.NewDecoder(reader)
+
+	i := 0
+	for dec.More() {
+		data := &Type1{}
+		if err := dec.Decode(data); err != nil {
+			t.Errorf("at %v got %v", i, err)
+		}
+		i++
+	}
+}
+
+func TestType1FinalLF(t *testing.T) {
+	reader := bytes.NewReader([]byte(flow2))
+	dec := jsoniter.NewDecoder(reader)
+
+	i := 0
+	for dec.More() {
+		data := &Type1{}
+		if err := dec.Decode(data); err != nil {
+			t.Errorf("at %v got %v", i, err)
+		}
+		i++
+	}
+}
+
+func TestType2NoFinalLF(t *testing.T) {
+	reader := bytes.NewReader([]byte(flow1))
+	dec := jsoniter.NewDecoder(reader)
+
+	i := 0
+	for dec.More() {
+		data := &Type2{}
+		if err := dec.Decode(data); err != nil {
+			t.Errorf("at %v got %v", i, err)
+		}
+		i++
+	}
+}
+
+func TestType2FinalLF(t *testing.T) {
+	reader := bytes.NewReader([]byte(flow2))
+	dec := jsoniter.NewDecoder(reader)
+
+	i := 0
+	for dec.More() {
+		data := &Type2{}
+		if err := dec.Decode(data); err != nil {
+			t.Errorf("at %v got %v", i, err)
+		}
+		i++
+	}
+}

iter_object.go

@@ -2,7 +2,7 @@ package jsoniter
 
 import (
 	"fmt"
-	"unicode"
+	"strings"
 )
 
 // ReadObject read one field from object.
@@ -96,13 +96,12 @@ func (iter *Iterator) readFieldHash() int64 {
 }
 
 func calcHash(str string, caseSensitive bool) int64 {
+	if !caseSensitive {
+		str = strings.ToLower(str)
+	}
 	hash := int64(0x811c9dc5)
-	for _, b := range str {
+	for _, b := range []byte(str) {
-		if caseSensitive {
+		hash ^= int64(b)
-			hash ^= int64(b)
-		} else {
-			hash ^= int64(unicode.ToLower(b))
-		}
 		hash *= 0x1000193
 	}
 	return int64(hash)

reflect.go

@@ -120,7 +120,8 @@ func decoderOfType(ctx *ctx, typ reflect2.Type) ValDecoder {
 	for _, extension := range extensions {
 		decoder = extension.DecorateDecoder(typ, decoder)
 	}
-	for _, extension := range ctx.extensions {
+	decoder = ctx.decoderExtension.DecorateDecoder(typ, decoder)
+	for _, extension := range ctx.extraExtensions {
 		decoder = extension.DecorateDecoder(typ, decoder)
 	}
 	return decoder
@@ -222,7 +223,8 @@ func encoderOfType(ctx *ctx, typ reflect2.Type) ValEncoder {
 	for _, extension := range extensions {
 		encoder = extension.DecorateEncoder(typ, encoder)
 	}
-	for _, extension := range ctx.extensions {
+	encoder = ctx.encoderExtension.DecorateEncoder(typ, encoder)
+	for _, extension := range ctx.extraExtensions {
 		encoder = extension.DecorateEncoder(typ, encoder)
 	}
 	return encoder

reflect_extension.go

@@ -246,7 +246,8 @@ func getTypeDecoderFromExtension(ctx *ctx, typ reflect2.Type) ValDecoder {
 		for _, extension := range extensions {
 			decoder = extension.DecorateDecoder(typ, decoder)
 		}
-		for _, extension := range ctx.extensions {
+		decoder = ctx.decoderExtension.DecorateDecoder(typ, decoder)
+		for _, extension := range ctx.extraExtensions {
 			decoder = extension.DecorateDecoder(typ, decoder)
 		}
 	}
@@ -259,14 +260,18 @@ func _getTypeDecoderFromExtension(ctx *ctx, typ reflect2.Type) ValDecoder {
 			return decoder
 		}
 	}
-	for _, extension := range ctx.extensions {
+	decoder := ctx.decoderExtension.CreateDecoder(typ)
+	if decoder != nil {
+		return decoder
+	}
+	for _, extension := range ctx.extraExtensions {
 		decoder := extension.CreateDecoder(typ)
 		if decoder != nil {
 			return decoder
 		}
 	}
 	typeName := typ.String()
-	decoder := typeDecoders[typeName]
+	decoder = typeDecoders[typeName]
 	if decoder != nil {
 		return decoder
 	}
@@ -286,7 +291,8 @@ func getTypeEncoderFromExtension(ctx *ctx, typ reflect2.Type) ValEncoder {
 		for _, extension := range extensions {
 			encoder = extension.DecorateEncoder(typ, encoder)
 		}
-		for _, extension := range ctx.extensions {
+		encoder = ctx.encoderExtension.DecorateEncoder(typ, encoder)
+		for _, extension := range ctx.extraExtensions {
 			encoder = extension.DecorateEncoder(typ, encoder)
 		}
 	}
@@ -300,14 +306,18 @@ func _getTypeEncoderFromExtension(ctx *ctx, typ reflect2.Type) ValEncoder {
 			return encoder
 		}
 	}
-	for _, extension := range ctx.extensions {
+	encoder := ctx.encoderExtension.CreateEncoder(typ)
+	if encoder != nil {
+		return encoder
+	}
+	for _, extension := range ctx.extraExtensions {
 		encoder := extension.CreateEncoder(typ)
 		if encoder != nil {
 			return encoder
 		}
 	}
 	typeName := typ.String()
-	encoder := typeEncoders[typeName]
+	encoder = typeEncoders[typeName]
 	if encoder != nil {
 		return encoder
 	}
@@ -393,7 +403,9 @@ func createStructDescriptor(ctx *ctx, typ reflect2.Type, bindings []*Binding, em
 	for _, extension := range extensions {
 		extension.UpdateStructDescriptor(structDescriptor)
 	}
-	for _, extension := range ctx.extensions {
+	ctx.encoderExtension.UpdateStructDescriptor(structDescriptor)
+	ctx.decoderExtension.UpdateStructDescriptor(structDescriptor)
+	for _, extension := range ctx.extraExtensions {
 		extension.UpdateStructDescriptor(structDescriptor)
 	}
 	processTags(structDescriptor, ctx.frozenConfig)

reflect_map.go

@@ -39,7 +39,11 @@ func encoderOfMap(ctx *ctx, typ reflect2.Type) ValEncoder {
 }
 
 func decoderOfMapKey(ctx *ctx, typ reflect2.Type) ValDecoder {
-	for _, extension := range ctx.extensions {
+	decoder := ctx.decoderExtension.CreateMapKeyDecoder(typ)
+	if decoder != nil {
+		return decoder
+	}
+	for _, extension := range ctx.extraExtensions {
 		decoder := extension.CreateMapKeyDecoder(typ)
 		if decoder != nil {
 			return decoder
@@ -77,7 +81,11 @@ func decoderOfMapKey(ctx *ctx, typ reflect2.Type) ValDecoder {
 }
 
 func encoderOfMapKey(ctx *ctx, typ reflect2.Type) ValEncoder {
-	for _, extension := range ctx.extensions {
+	encoder := ctx.encoderExtension.CreateMapKeyEncoder(typ)
+	if encoder != nil {
+		return encoder
+	}
+	for _, extension := range ctx.extraExtensions {
 		encoder := extension.CreateMapKeyEncoder(typ)
 		if encoder != nil {
 			return encoder

type_tests/struct_tags_test.go

@@ -145,6 +145,9 @@ func init() {
 		(*struct {
 			Field bool `json:",omitempty,string"`
 		})(nil),
+		(*struct {
+			Field bool `json:"中文"`
+		})(nil),
 	)
 }
 

value_tests/raw_message_test.go

@@ -1,10 +1,18 @@
 package test
 
-import "encoding/json"
+import (
+	"encoding/json"
+)
 
 func init() {
 	marshalCases = append(marshalCases,
 		json.RawMessage("{}"),
+		selectedMarshalCase{struct {
+			Env   string          `json:"env"`
+			Extra json.RawMessage `json:"extra,omitempty"`
+		}{
+			Env: "jfdk",
+		}},
 	)
 	unmarshalCases = append(unmarshalCases, unmarshalCase{
 		ptr:   (*json.RawMessage)(nil),