From da14cebb76ca18d368033effb51891195562d805 Mon Sep 17 00:00:00 2001 From: Unknwon Date: Wed, 21 Dec 2016 05:59:58 -0500 Subject: [PATCH] Hanlde BOM for UTF-16 --- ini.go | 2 +- parser.go | 23 ++++++++++++++++---- parser_test.go | 42 +++++++++++++++++++++++++++++++++++++ testdata/UTF-16-BE-BOM.ini | Bin 0 -> 56 bytes testdata/UTF-16-LE-BOM.ini | Bin 0 -> 56 bytes testdata/UTF-8-BOM.ini | 2 ++ testdata/conf.ini | 2 +- 7 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 parser_test.go create mode 100644 testdata/UTF-16-BE-BOM.ini create mode 100644 testdata/UTF-16-LE-BOM.ini create mode 100644 testdata/UTF-8-BOM.ini diff --git a/ini.go b/ini.go index cd065e7..e812819 100644 --- a/ini.go +++ b/ini.go @@ -36,7 +36,7 @@ const ( // Maximum allowed depth when recursively substituing variable names. _DEPTH_VALUES = 99 - _VERSION = "1.21.1" + _VERSION = "1.21.2" ) // Version returns current package version literal. diff --git a/parser.go b/parser.go index dc6df87..8ddf4e2 100644 --- a/parser.go +++ b/parser.go @@ -48,16 +48,31 @@ func newParser(r io.Reader) *parser { } } -// BOM handles header of BOM-UTF8 format. +// BOM handles header of UTF-8, UTF-16 LE and UTF-16 BE's BOM format. // http://en.wikipedia.org/wiki/Byte_order_mark#Representations_of_byte_order_marks_by_encoding func (p *parser) BOM() error { - mask, err := p.buf.Peek(3) + mask, err := p.buf.Peek(2) if err != nil && err != io.EOF { return err - } else if len(mask) < 3 { + } else if len(mask) < 2 { return nil - } else if mask[0] == 239 && mask[1] == 187 && mask[2] == 191 { + } + + switch { + case mask[0] == 254 && mask[1] == 255: + fallthrough + case mask[0] == 255 && mask[1] == 254: p.buf.Read(mask) + case mask[0] == 239 && mask[1] == 187: + mask, err := p.buf.Peek(3) + if err != nil && err != io.EOF { + return err + } else if len(mask) < 3 { + return nil + } + if mask[2] == 191 { + p.buf.Read(mask) + } } return nil } diff --git a/parser_test.go b/parser_test.go new file mode 100644 index 0000000..0525819 --- /dev/null +++ b/parser_test.go @@ -0,0 +1,42 @@ +// Copyright 2016 Unknwon +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package ini + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func Test_BOM(t *testing.T) { + Convey("Test handling BOM", t, func() { + Convey("UTF-8-BOM", func() { + cfg, err := Load("testdata/UTF-8-BOM.ini") + So(err, ShouldBeNil) + So(cfg, ShouldNotBeNil) + + So(cfg.Section("author").Key("E-MAIL").String(), ShouldEqual, "u@gogs.io") + }) + + Convey("UTF-16-LE-BOM", func() { + cfg, err := Load("testdata/UTF-16-LE-BOM.ini") + So(err, ShouldBeNil) + So(cfg, ShouldNotBeNil) + }) + + Convey("UTF-16-BE-BOM", func() { + }) + }) +} diff --git a/testdata/UTF-16-BE-BOM.ini b/testdata/UTF-16-BE-BOM.ini new file mode 100644 index 0000000000000000000000000000000000000000..c8bf82c8ffa2416cb3b462457a6d4c4126f9c410 GIT binary patch literal 56 zcmezOpCOtdk)f2Kgdu|=pP`5$mVt}Gl|h%mm%)+2lfj2Ufx#AtOBoy((t+~n48;t3 I44Dl10AFkh&;S4c literal 0 HcmV?d00001 diff --git a/testdata/UTF-16-LE-BOM.ini b/testdata/UTF-16-LE-BOM.ini new file mode 100644 index 0000000000000000000000000000000000000000..27f62186e20bc90b1d302415baf68ffb00cc69e7 GIT binary patch literal 56 zcmezWFPb5dp_HM7A%h{Gp@<=tfs4VFL6^ao!I8m}!G}SC!4`;185|hWf%54L#SD53 HnLri*Xg>