diff --git a/ch/chschema/block.go b/ch/chschema/block.go index a53eaa5..ca1e9a8 100644 --- a/ch/chschema/block.go +++ b/ch/chschema/block.go @@ -37,13 +37,13 @@ func (b *Block) Column(colName, colType string) *Column { var col *Column if b.Table != nil { - col = b.Table.NewColumn(colName, colType, b.NumRow) + col = b.Table.NewColumn(colName, colType) } if col == nil { col = &Column{ Name: colName, Type: colType, - Columnar: NewColumnFromCHType(colType, b.NumRow), + Columnar: NewColumn(colType, nil), } } diff --git a/ch/chschema/column.go b/ch/chschema/column.go index 75fd4d4..2464c93 100644 --- a/ch/chschema/column.go +++ b/ch/chschema/column.go @@ -32,8 +32,9 @@ func (c *Column) String() string { } type Columnar interface { - ReadFrom(rd *chproto.Reader, numRow int) error - WriteTo(wr *chproto.Writer) error + Init(chType string) error + AllocForReading(numRow int) + ResetForWriting(numRow int) Type() reflect.Type Set(v any) @@ -44,15 +45,14 @@ type Columnar interface { Index(idx int) any Slice(s, e int) any ConvertAssign(idx int, dest reflect.Value) error + + ReadFrom(rd *chproto.Reader, numRow int) error + WriteTo(wr *chproto.Writer) error } -func NewColumn(typ reflect.Type, chType string, numRow int) Columnar { - return ColumnFactory(typ, chType)(typ, chType, numRow) -} - -func NewColumnFromCHType(chType string, numRow int) Columnar { - typ := goType(chType) - return NewColumn(typ, chType, numRow) +type ArrayColumnar interface { + WriteOffset(wr *chproto.Writer, offset int) int + WriteData(wr *chproto.Writer) error } //------------------------------------------------------------------------------ @@ -61,13 +61,11 @@ type ColumnOf[T any] struct { Column []T } -func NewColumnOf[T any](numRow int) ColumnOf[T] { - return ColumnOf[T]{ - Column: make([]T, 0, numRow), - } +func (c *ColumnOf[T]) Init(chType string) error { + return nil } -func (c *ColumnOf[T]) Alloc(numRow int) { +func (c *ColumnOf[T]) AllocForReading(numRow int) { if cap(c.Column) >= numRow { c.Column = c.Column[:numRow] } else { @@ -75,7 +73,7 @@ func (c *ColumnOf[T]) Alloc(numRow int) { } } -func (c *ColumnOf[T]) Reset(numRow int) { +func (c *ColumnOf[T]) ResetForWriting(numRow int) { if cap(c.Column) >= numRow { c.Column = c.Column[:0] } else { @@ -128,12 +126,6 @@ type NumericColumnOf[T constraints.Integer | constraints.Float] struct { ColumnOf[T] } -func NewNumericColumnOf[T constraints.Integer | constraints.Float](numRow int) NumericColumnOf[T] { - col := NumericColumnOf[T]{} - col.Column = make([]T, 0, numRow) - return col -} - func (c NumericColumnOf[T]) ConvertAssign(idx int, v reflect.Value) error { switch v.Kind() { case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: @@ -148,24 +140,6 @@ func (c NumericColumnOf[T]) ConvertAssign(idx int, v reflect.Value) error { return nil } -//------------------------------------------------------------------------------ - -type BoolColumn struct { - ColumnOf[bool] -} - -var _ Columnar = (*BoolColumn)(nil) - -func NewBoolColumn(typ reflect.Type, chType string, numRow int) Columnar { - return &BoolColumn{ - ColumnOf: NewColumnOf[bool](numRow), - } -} - -func (c BoolColumn) Type() reflect.Type { - return boolType -} - func (c BoolColumn) ConvertAssign(idx int, v reflect.Value) error { switch v.Kind() { case reflect.Bool: @@ -176,269 +150,6 @@ func (c BoolColumn) ConvertAssign(idx int, v reflect.Value) error { return nil } -func (c *BoolColumn) AppendValue(v reflect.Value) { - c.Column = append(c.Column, v.Bool()) -} - -func (c *BoolColumn) ReadFrom(rd *chproto.Reader, numRow int) error { - c.Alloc(numRow) - - for i := range c.Column { - flag, err := rd.Bool() - if err != nil { - return err - } - c.Column[i] = flag - } - - return nil -} - -func (c BoolColumn) WriteTo(wr *chproto.Writer) error { - for _, flag := range c.Column { - wr.Bool(flag) - } - return nil -} - -//------------------------------------------------------------------------------ - -type Int8Column struct { - NumericColumnOf[int8] -} - -var _ Columnar = (*Int8Column)(nil) - -func NewInt8Column(typ reflect.Type, chType string, numRow int) Columnar { - return &Int8Column{ - NumericColumnOf: NewNumericColumnOf[int8](numRow), - } -} - -func (c Int8Column) Type() reflect.Type { - return int8Type -} - -func (c *Int8Column) AppendValue(v reflect.Value) { - c.Column = append(c.Column, int8(v.Int())) -} - -//------------------------------------------------------------------------------ - -type Int16Column struct { - NumericColumnOf[int16] -} - -var _ Columnar = (*Int16Column)(nil) - -func NewInt16Column(typ reflect.Type, chType string, numRow int) Columnar { - return &Int16Column{ - NumericColumnOf: NewNumericColumnOf[int16](numRow), - } -} - -func (c Int16Column) Type() reflect.Type { - return int16Type -} - -func (c *Int16Column) AppendValue(v reflect.Value) { - c.Column = append(c.Column, int16(v.Int())) -} - -//------------------------------------------------------------------------------ - -type Int32Column struct { - NumericColumnOf[int32] -} - -var _ Columnar = (*Int32Column)(nil) - -func NewInt32Column(typ reflect.Type, chType string, numRow int) Columnar { - return &Int32Column{ - NumericColumnOf: NewNumericColumnOf[int32](numRow), - } -} - -func (c Int32Column) Type() reflect.Type { - return int32Type -} - -func (c *Int32Column) AppendValue(v reflect.Value) { - c.Column = append(c.Column, int32(v.Int())) -} - -//------------------------------------------------------------------------------ - -type Int64Column struct { - NumericColumnOf[int64] -} - -var _ Columnar = (*Int64Column)(nil) - -func NewInt64Column(typ reflect.Type, chType string, numRow int) Columnar { - return &Int64Column{ - NumericColumnOf: NewNumericColumnOf[int64](numRow), - } -} - -func (c Int64Column) Type() reflect.Type { - return int64Type -} - -func (c *Int64Column) AppendValue(v reflect.Value) { - c.Column = append(c.Column, v.Int()) -} - -//------------------------------------------------------------------------------ - -type UInt8Column struct { - NumericColumnOf[uint8] -} - -var _ Columnar = (*UInt8Column)(nil) - -func NewUInt8Column(typ reflect.Type, chType string, numRow int) Columnar { - return &UInt8Column{ - NumericColumnOf: NewNumericColumnOf[uint8](numRow), - } -} - -func (c UInt8Column) Type() reflect.Type { - return uint8Type -} - -func (c *UInt8Column) AppendValue(v reflect.Value) { - c.Column = append(c.Column, uint8(v.Uint())) -} - -//------------------------------------------------------------------------------ - -type UInt16Column struct { - NumericColumnOf[uint16] -} - -var _ Columnar = (*UInt16Column)(nil) - -func NewUInt16Column(typ reflect.Type, chType string, numRow int) Columnar { - return &UInt16Column{ - NumericColumnOf: NewNumericColumnOf[uint16](numRow), - } -} - -func (c UInt16Column) Type() reflect.Type { - return uint16Type -} - -func (c *UInt16Column) AppendValue(v reflect.Value) { - c.Column = append(c.Column, uint16(v.Uint())) -} - -//------------------------------------------------------------------------------ - -type UInt32Column struct { - NumericColumnOf[uint32] -} - -var _ Columnar = (*UInt32Column)(nil) - -func NewUInt32Column(typ reflect.Type, chType string, numRow int) Columnar { - return &UInt32Column{ - NumericColumnOf: NewNumericColumnOf[uint32](numRow), - } -} - -func (c UInt32Column) Type() reflect.Type { - return uint32Type -} - -func (c *UInt32Column) AppendValue(v reflect.Value) { - c.Column = append(c.Column, uint32(v.Uint())) -} - -//------------------------------------------------------------------------------ - -type UInt64Column struct { - NumericColumnOf[uint64] -} - -var _ Columnar = (*UInt64Column)(nil) - -func NewUInt64Column(typ reflect.Type, chType string, numRow int) Columnar { - return &UInt64Column{ - NumericColumnOf: NewNumericColumnOf[uint64](numRow), - } -} - -func (c UInt64Column) Type() reflect.Type { - return uint64Type -} - -func (c *UInt64Column) AppendValue(v reflect.Value) { - c.Column = append(c.Column, v.Uint()) -} - -//------------------------------------------------------------------------------ - -type Float32Column struct { - NumericColumnOf[float32] -} - -var _ Columnar = (*Float32Column)(nil) - -func NewFloat32Column(typ reflect.Type, chType string, numRow int) Columnar { - return &Float32Column{ - NumericColumnOf: NewNumericColumnOf[float32](numRow), - } -} - -func (c Float32Column) Type() reflect.Type { - return float32Type -} - -func (c *Float32Column) AppendValue(v reflect.Value) { - c.Column = append(c.Column, float32(v.Float())) -} - -//------------------------------------------------------------------------------ - -type Float64Column struct { - NumericColumnOf[float64] -} - -var _ Columnar = (*Float64Column)(nil) - -func NewFloat64Column(typ reflect.Type, chType string, numRow int) Columnar { - return &Float64Column{ - NumericColumnOf: NewNumericColumnOf[float64](numRow), - } -} - -func (c Float64Column) Type() reflect.Type { - return float64Type -} - -func (c *Float64Column) AppendValue(v reflect.Value) { - c.Column = append(c.Column, v.Float()) -} - -//------------------------------------------------------------------------------ - -type StringColumn struct { - ColumnOf[string] -} - -var _ Columnar = (*StringColumn)(nil) - -func NewStringColumn(typ reflect.Type, chType string, numRow int) Columnar { - return &StringColumn{ - ColumnOf: NewColumnOf[string](numRow), - } -} - -func (c StringColumn) Type() reflect.Type { - return stringType -} - func (c StringColumn) ConvertAssign(idx int, v reflect.Value) error { switch v.Kind() { case reflect.String: @@ -460,31 +171,6 @@ func (c StringColumn) ConvertAssign(idx int, v reflect.Value) error { return fmt.Errorf("ch: can't scan %s into %s", "string", v.Type()) } -func (c *StringColumn) AppendValue(v reflect.Value) { - c.Column = append(c.Column, v.String()) -} - -func (c *StringColumn) ReadFrom(rd *chproto.Reader, numRow int) error { - c.Alloc(numRow) - - for i := range c.Column { - n, err := rd.String() - if err != nil { - return err - } - c.Column[i] = n - } - - return nil -} - -func (c StringColumn) WriteTo(wr *chproto.Writer) error { - for _, s := range c.Column { - wr.String(s) - } - return nil -} - //------------------------------------------------------------------------------ type UUID [16]byte @@ -496,10 +182,8 @@ type UUIDColumn struct { var _ Columnar = (*UUIDColumn)(nil) -func NewUUIDColumn(typ reflect.Type, chType string, numRow int) Columnar { - return &UUIDColumn{ - ColumnOf: NewColumnOf[UUID](numRow), - } +func NewUUIDColumn() Columnar { + return new(UUIDColumn) } func (c UUIDColumn) Type() reflect.Type { @@ -517,7 +201,7 @@ func (c *UUIDColumn) AppendValue(v reflect.Value) { } func (c *UUIDColumn) ReadFrom(rd *chproto.Reader, numRow int) error { - c.Alloc(numRow) + c.AllocForReading(numRow) for i := range c.Column { err := rd.UUID(c.Column[i][:]) @@ -548,10 +232,8 @@ type IPColumn struct { var _ Columnar = (*IPColumn)(nil) -func NewIPColumn(typ reflect.Type, chType string, numRow int) Columnar { - return &IPColumn{ - ColumnOf: NewColumnOf[net.IP](numRow), - } +func NewIPColumn() Columnar { + return new(IPColumn) } func (c IPColumn) Type() reflect.Type { @@ -568,7 +250,7 @@ func (c *IPColumn) AppendValue(v reflect.Value) { } func (c *IPColumn) ReadFrom(rd *chproto.Reader, numRow int) error { - c.Alloc(numRow) + c.AllocForReading(numRow) mem := make([]byte, ipSize*numRow) var idx int @@ -603,54 +285,6 @@ func (c IPColumn) WriteTo(wr *chproto.Writer) error { //------------------------------------------------------------------------------ -type DateTimeColumn struct { - ColumnOf[time.Time] -} - -var _ Columnar = (*DateTimeColumn)(nil) - -func NewDateTimeColumn(typ reflect.Type, chType string, numRow int) Columnar { - return &DateTimeColumn{ - ColumnOf: NewColumnOf[time.Time](numRow), - } -} - -func (c DateTimeColumn) Type() reflect.Type { - return timeType -} - -func (c DateTimeColumn) ConvertAssign(idx int, v reflect.Value) error { - v.Set(reflect.ValueOf(c.Column[idx])) - return nil -} - -func (c *DateTimeColumn) AppendValue(v reflect.Value) { - c.Column = append(c.Column, v.Interface().(time.Time)) -} - -func (c *DateTimeColumn) ReadFrom(rd *chproto.Reader, numRow int) error { - c.Alloc(numRow) - - for i := range c.Column { - n, err := rd.DateTime() - if err != nil { - return err - } - c.Column[i] = n - } - - return nil -} - -func (c DateTimeColumn) WriteTo(wr *chproto.Writer) error { - for i := range c.Column { - wr.DateTime(c.Column[i]) - } - return nil -} - -//------------------------------------------------------------------------------ - type DateTime64Column struct { ColumnOf[time.Time] prec int @@ -658,11 +292,13 @@ type DateTime64Column struct { var _ Columnar = (*DateTime64Column)(nil) -func NewDateTime64Column(typ reflect.Type, chType string, numRow int) Columnar { - return &DateTime64Column{ - ColumnOf: NewColumnOf[time.Time](numRow), - prec: parseDateTime64Prec(chType), - } +func NewDateTime64Column() Columnar { + return new(DateTime64Column) +} + +func (c *DateTime64Column) Init(chType string) error { + c.prec = parseDateTime64Prec(chType) + return nil } func (c *DateTime64Column) Type() reflect.Type { @@ -675,7 +311,7 @@ func (c *DateTime64Column) ConvertAssign(idx int, v reflect.Value) error { } func (c *DateTime64Column) ReadFrom(rd *chproto.Reader, numRow int) error { - c.Alloc(numRow) + c.AllocForReading(numRow) mul := int64(math.Pow10(9 - c.prec)) for i := range c.Column { @@ -699,59 +335,18 @@ func (c *DateTime64Column) WriteTo(wr *chproto.Writer) error { //------------------------------------------------------------------------------ -type Int64TimeColumn struct { - Int64Column -} - -var _ Columnar = (*Int64TimeColumn)(nil) - -func NewInt64TimeColumn(typ reflect.Type, chType string, numRow int) Columnar { - return &Int64TimeColumn{ - Int64Column: Int64Column{ - NumericColumnOf: NewNumericColumnOf[int64](numRow), - }, - } -} - -func (c *Int64TimeColumn) ReadFrom(rd *chproto.Reader, numRow int) error { - c.Alloc(numRow) - - for i := range c.Column { - n, err := rd.UInt32() - if err != nil { - return err - } - c.Column[i] = int64(n) * int64(time.Second) - } - - return nil -} - -func (c Int64TimeColumn) WriteTo(wr *chproto.Writer) error { - for i := range c.Column { - wr.UInt32(uint32(c.Column[i] / int64(time.Second))) - } - return nil -} - -//------------------------------------------------------------------------------ - type DateColumn struct { DateTimeColumn } var _ Columnar = (*DateColumn)(nil) -func NewDateColumn(typ reflect.Type, chType string, numRow int) Columnar { - return &DateColumn{ - DateTimeColumn: DateTimeColumn{ - ColumnOf: NewColumnOf[time.Time](numRow), - }, - } +func NewDateColumn() Columnar { + return new(DateColumn) } func (c *DateColumn) ReadFrom(rd *chproto.Reader, numRow int) error { - c.Alloc(numRow) + c.AllocForReading(numRow) for i := range c.Column { n, err := rd.Date() @@ -781,16 +376,12 @@ type TimeColumn struct { var _ Columnar = (*TimeColumn)(nil) -func NewTimeColumn(typ reflect.Type, chType string, numRow int) Columnar { - return &TimeColumn{ - DateTimeColumn: DateTimeColumn{ - ColumnOf: NewColumnOf[time.Time](numRow), - }, - } +func NewTimeColumn() Columnar { + return new(TimeColumn) } func (c *TimeColumn) ReadFrom(rd *chproto.Reader, numRow int) error { - c.Alloc(numRow) + c.AllocForReading(numRow) for i := range c.Column { n, err := rd.Int64() @@ -812,63 +403,43 @@ func (c TimeColumn) WriteTo(wr *chproto.Writer) error { //------------------------------------------------------------------------------ -type BytesColumn struct { - ColumnOf[[]byte] +type EnumColumn struct { + StringColumn + enum *enumInfo } -var _ Columnar = (*BytesColumn)(nil) +var _ Columnar = (*EnumColumn)(nil) -func NewBytesColumn(typ reflect.Type, chType string, numRow int) Columnar { - return &BytesColumn{ - ColumnOf: NewColumnOf[[]byte](numRow), - } +func NewEnumColumn() Columnar { + return new(EnumColumn) } -func (c *BytesColumn) Reset(numRow int) { - if cap(c.Column) >= numRow { - for i := range c.Column { - c.Column[i] = nil - } - c.Column = c.Column[:0] - } else { - c.Column = make([][]byte, 0, numRow) - } -} - -func (c BytesColumn) Type() reflect.Type { - return bytesType -} - -func (c BytesColumn) ConvertAssign(idx int, v reflect.Value) error { - v.SetBytes(c.Column[idx]) +func (c *EnumColumn) Init(chType string) error { + c.enum = parseEnum(chType) return nil } -func (c *BytesColumn) AppendValue(v reflect.Value) { - c.Column = append(c.Column, v.Bytes()) -} +func (c *EnumColumn) ReadFrom(rd *chproto.Reader, numRow int) error { + c.AllocForReading(numRow) -func (c *BytesColumn) ReadFrom(rd *chproto.Reader, numRow int) error { - if cap(c.Column) >= numRow { - c.Column = c.Column[:numRow] - } else { - c.Column = make([][]byte, numRow) - } - - for i := 0; i < len(c.Column); i++ { - b, err := rd.Bytes() + for i := range c.Column { + n, err := rd.Int8() if err != nil { return err } - c.Column[i] = b + c.Column[i] = c.enum.Decode(int16(n)) } return nil } -func (c BytesColumn) WriteTo(wr *chproto.Writer) error { - for _, b := range c.Column { - wr.Bytes(b) +func (c *EnumColumn) WriteTo(wr *chproto.Writer) error { + for _, s := range c.Column { + n, ok := c.enum.Encode(s) + if !ok { + log.Printf("unknown enum value in %s: %s", c.enum.chType, s) + } + wr.Int8(int8(n)) } return nil } @@ -882,13 +453,13 @@ type JSONColumn struct { var _ Columnar = (*JSONColumn)(nil) -func NewJSONColumn(typ reflect.Type, chType string, numRow int) Columnar { +func NewJSONColumn() Columnar { return new(JSONColumn) } -func (c *JSONColumn) Reset(numRow int) { +func (c *JSONColumn) ResetForWriting(numRow int) { c.Values = c.Values[:0] - c.BytesColumn.Reset(numRow) + c.BytesColumn.ResetForWriting(numRow) } func (c *JSONColumn) Len() int { @@ -926,258 +497,14 @@ func (c *JSONColumn) WriteTo(wr *chproto.Writer) error { //------------------------------------------------------------------------------ -type EnumColumn struct { - StringColumn - enum *enumInfo -} - -var _ Columnar = (*EnumColumn)(nil) - -func NewEnumColumn(typ reflect.Type, chType string, numRow int) Columnar { - return &EnumColumn{ - StringColumn: StringColumn{ - ColumnOf: NewColumnOf[string](numRow), - }, - enum: parseEnum(chType), - } -} - -func (c *EnumColumn) ReadFrom(rd *chproto.Reader, numRow int) error { - if cap(c.Column) >= numRow { - c.Column = c.Column[:numRow] - } else { - c.Column = make([]string, numRow) - } - - for i := 0; i < len(c.Column); i++ { - n, err := rd.Int8() - if err != nil { - return err - } - c.Column[i] = c.enum.Decode(int16(n)) - } - - return nil -} - -func (c *EnumColumn) WriteTo(wr *chproto.Writer) error { - for _, s := range c.Column { - n, ok := c.enum.Encode(s) - if !ok { - log.Printf("unknown enum value in %s: %s", c.enum.chType, s) - } - wr.Int8(int8(n)) - } - return nil -} - -//------------------------------------------------------------------------------ - -type LCStringColumn struct { - StringColumn -} - -var _ Columnar = (*LCStringColumn)(nil) - -func NewLCStringColumn(typ reflect.Type, chType string, numRow int) Columnar { - col := new(LCStringColumn) - col.Column = make([]string, 0, numRow) - return col -} - -func (c *LCStringColumn) ReadFrom(rd *chproto.Reader, numRow int) error { - if numRow == 0 { - return nil - } - if err := c.readPrefix(rd, numRow); err != nil { - return err - } - return c.readData(rd, numRow) -} - -func (c *LCStringColumn) readPrefix(rd *chproto.Reader, numRow int) error { - version, err := rd.Int64() - if err != nil { - return err - } - if version != 1 { - return fmt.Errorf("ch: got version=%d, wanted 1", version) - } - return nil -} - -func (c *LCStringColumn) readData(rd *chproto.Reader, numRow int) error { - flags, err := rd.Int64() - if err != nil { - return err - } - lcKey := newLCKeyType(flags & 0xf) - - dictSize, err := rd.UInt64() - if err != nil { - return err - } - - dict := make([]string, dictSize) - - for i := range dict { - s, err := rd.String() - if err != nil { - return err - } - dict[i] = s - } - - numKey, err := rd.UInt64() - if err != nil { - return err - } - if int(numKey) != numRow { - return fmt.Errorf("%d != %d", numKey, numRow) - } - - if cap(c.Column) >= int(numKey) { - c.Column = c.Column[:numKey] - } else { - c.Column = make([]string, numKey) - } - - for i := 0; i < int(numKey); i++ { - key, err := lcKey.read(rd) - if err != nil { - return err - } - c.Column[i] = dict[key] - } - - return nil -} - -func (c *LCStringColumn) WriteTo(wr *chproto.Writer) error { - c.writePrefix(wr) - c.writeData(wr) - return nil -} - -func (c *LCStringColumn) writePrefix(wr *chproto.Writer) { - wr.Int64(1) -} - -func (c *LCStringColumn) writeData(wr *chproto.Writer) { - if len(c.Column) == 0 { - return - } - - keys := make([]int, len(c.Column)) - var lc lowCard - - for i, s := range c.Column { - keys[i] = lc.Add(s) - } - - const hasAdditionalKeys = 1 << 9 - const needUpdateDict = 1 << 10 - - dict := lc.Dict() - lcKey := newLCKey(int64(len(dict))) - - wr.Int64(int64(lcKey.typ) | hasAdditionalKeys | needUpdateDict) - - wr.Int64(int64(len(dict))) - for _, s := range dict { - wr.String(s) - } - - wr.Int64(int64(len(keys))) - for _, key := range keys { - lcKey.write(wr, key) - } -} - -//------------------------------------------------------------------------------ - -type lcKey struct { - typ int8 - read func(*chproto.Reader) (int, error) - write func(*chproto.Writer, int) -} - -func newLCKey(numKey int64) lcKey { - if numKey <= math.MaxUint8 { - return newLCKeyType(0) - } - if numKey <= math.MaxUint16 { - return newLCKeyType(1) - } - if numKey <= math.MaxUint32 { - return newLCKeyType(2) - } - return newLCKeyType(3) -} - -func newLCKeyType(typ int64) lcKey { - switch typ { - case 0: - return lcKey{ - typ: 0, - read: func(rd *chproto.Reader) (int, error) { - n, err := rd.UInt8() - return int(n), err - }, - write: func(wr *chproto.Writer, n int) { - wr.UInt8(uint8(n)) - }, - } - case 1: - return lcKey{ - typ: int8(1), - read: func(rd *chproto.Reader) (int, error) { - n, err := rd.UInt16() - return int(n), err - }, - write: func(wr *chproto.Writer, n int) { - wr.UInt16(uint16(n)) - }, - } - case 2: - return lcKey{ - typ: 2, - read: func(rd *chproto.Reader) (int, error) { - n, err := rd.UInt32() - return int(n), err - }, - write: func(wr *chproto.Writer, n int) { - wr.UInt32(uint32(n)) - }, - } - case 3: - return lcKey{ - typ: 3, - read: func(rd *chproto.Reader) (int, error) { - n, err := rd.UInt64() - return int(n), err - }, - write: func(wr *chproto.Writer, n int) { - wr.UInt64(uint64(n)) - }, - } - default: - panic("not reached") - } -} - -//------------------------------------------------------------------------------ - type BFloat16HistColumn struct { ColumnOf[bfloat16.Map] } var _ Columnar = (*BFloat16HistColumn)(nil) -func NewBFloat16HistColumn(typ reflect.Type, chType string, numRow int) Columnar { - return &BFloat16HistColumn{ - ColumnOf: NewColumnOf[bfloat16.Map](numRow), - } +func NewBFloat16HistColumn() Columnar { + return new(BFloat16HistColumn) } func (c BFloat16HistColumn) Type() reflect.Type { @@ -1189,7 +516,7 @@ func (c *BFloat16HistColumn) ReadFrom(rd *chproto.Reader, numRow int) error { return nil } - c.Alloc(numRow) + c.AllocForReading(numRow) for i := range c.Column { n, err := rd.Uvarint() diff --git a/ch/chschema/column_array.go b/ch/chschema/column_array.go deleted file mode 100644 index b134535..0000000 --- a/ch/chschema/column_array.go +++ /dev/null @@ -1,539 +0,0 @@ -package chschema - -import ( - "fmt" - "reflect" - "unsafe" - - "github.com/uptrace/go-clickhouse/ch/chproto" -) - -type ArrayColumnar interface { - WriteOffset(wr *chproto.Writer, offset int) int - WriteData(wr *chproto.Writer) error -} - -//------------------------------------------------------------------------------ - -type ArrayColumnOf[T any] struct { - Column [][]T - elem Columnar -} - -func (c *ArrayColumnOf[T]) Reset(numRow int) { - if cap(c.Column) >= numRow { - c.Column = c.Column[:0] - } else { - c.Column = make([][]T, 0, numRow) - } -} - -func (c *ArrayColumnOf[T]) Set(v any) { - c.Column = v.([][]T) -} - -func (c *ArrayColumnOf[T]) Value() any { - return c.Column -} - -func (c *ArrayColumnOf[T]) Nullable(nulls UInt8Column) any { - panic("not implemented") -} - -func (c *ArrayColumnOf[T]) Len() int { - return len(c.Column) -} - -func (c *ArrayColumnOf[T]) Index(idx int) any { - return c.Column[idx] -} - -func (c *ArrayColumnOf[T]) Slice(s, e int) any { - return c.Column[s:e] -} - -func (c *ArrayColumnOf[T]) ConvertAssign(idx int, v reflect.Value) error { - v.Set(reflect.ValueOf(c.Column[idx])) - return nil -} - -func (c *ArrayColumnOf[T]) AppendValue(v reflect.Value) { - ptr := unsafe.Pointer(v.UnsafeAddr()) - c.AppendPointer(v.Type(), ptr) -} - -func (c *ArrayColumnOf[T]) AppendPointer(typ reflect.Type, ptr unsafe.Pointer) { - c.Column = append(c.Column, *(*[]T)(ptr)) -} - -func (c *ArrayColumnOf[T]) ReadFrom(rd *chproto.Reader, numRow int) error { - if cap(c.Column) >= numRow { - c.Column = c.Column[:numRow] - } else { - c.Column = make([][]T, numRow) - } - - if numRow == 0 { - return nil - } - - offsets := make([]int, numRow) - for i := 0; i < numRow; i++ { - offset, err := rd.UInt64() - if err != nil { - return err - } - offsets[i] = int(offset) - } - - if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { - return err - } - - var prev int - for i, offset := range offsets { - c.Column[i] = c.elem.Slice(prev, offset).([]T) - prev = offset - } - - return nil -} - -func (c *ArrayColumnOf[T]) WriteTo(wr *chproto.Writer) error { - _ = c.WriteOffset(wr, 0) - return c.WriteData(wr) -} - -var _ ArrayColumnar = (*Int64ArrayColumn)(nil) - -func (c *ArrayColumnOf[T]) WriteOffset(wr *chproto.Writer, offset int) int { - for _, el := range c.Column { - offset += len(el) - wr.UInt64(uint64(offset)) - } - return offset -} - -func (c *ArrayColumnOf[T]) WriteData(wr *chproto.Writer) error { - for _, ss := range c.Column { - c.elem.Set(ss) - if err := c.elem.WriteTo(wr); err != nil { - return err - } - } - return nil -} - -//------------------------------------------------------------------------------ - -type Int64ArrayColumn struct { - ArrayColumnOf[int64] -} - -var _ Columnar = (*Int64ArrayColumn)(nil) - -func NewInt64ArrayColumn(typ reflect.Type, chType string, numRow int) Columnar { - return &Int64ArrayColumn{ - ArrayColumnOf: ArrayColumnOf[int64]{ - Column: make([][]int64, 0, numRow), - elem: NewInt64Column(typ.Elem(), "", 0), - }, - } -} - -func (c *Int64ArrayColumn) Type() reflect.Type { - return int64SliceType -} - -//------------------------------------------------------------------------------ - -type Uint64ArrayColumn struct { - ArrayColumnOf[uint64] -} - -var _ Columnar = (*Uint64ArrayColumn)(nil) - -func NewUint64ArrayColumn(typ reflect.Type, chType string, numRow int) Columnar { - return &Uint64ArrayColumn{ - ArrayColumnOf: ArrayColumnOf[uint64]{ - Column: make([][]uint64, 0, numRow), - elem: NewUInt64Column(typ.Elem(), "", 0), - }, - } -} - -func (c *Uint64ArrayColumn) Type() reflect.Type { - return uint64SliceType -} - -//------------------------------------------------------------------------------ - -type Float64ArrayColumn struct { - ArrayColumnOf[float64] -} - -var _ Columnar = (*Float64ArrayColumn)(nil) - -func NewFloat64ArrayColumn(typ reflect.Type, chType string, numRow int) Columnar { - return &Float64ArrayColumn{ - ArrayColumnOf: ArrayColumnOf[float64]{ - Column: make([][]float64, 0, numRow), - elem: NewFloat64Column(typ.Elem(), "", 0), - }, - } -} - -func (c *Float64ArrayColumn) Type() reflect.Type { - return float64SliceType -} - -//------------------------------------------------------------------------------ - -type StringArrayColumn struct { - Column [][]string - elem Columnar - stringElem *StringColumn - lcElem *LCStringColumn -} - -var _ Columnar = (*StringArrayColumn)(nil) - -func NewStringArrayColumn(typ reflect.Type, chType string, numRow int) Columnar { - if _, funcType := aggFuncNameAndType(chType); funcType != "" { - chType = funcType - } - elemType := chArrayElemType(chType) - if elemType == "" { - panic(fmt.Errorf("invalid array type: %q (Go type is %s)", - chType, typ.String())) - } - - columnar := NewColumn(typ.Elem(), elemType, 0) - var stringElem *StringColumn - var lcElem *LCStringColumn - - switch v := columnar.(type) { - case *StringColumn: - stringElem = v - case *LCStringColumn: - stringElem = &v.StringColumn - lcElem = v - columnar = &ArrayLCStringColumn{v} - case *EnumColumn: - stringElem = &v.StringColumn - default: - panic(fmt.Errorf("unsupported column: %T", v)) - } - - return &StringArrayColumn{ - Column: make([][]string, 0, numRow), - elem: columnar, - stringElem: stringElem, - lcElem: lcElem, - } -} - -func (c *StringArrayColumn) Reset(numRow int) { - if cap(c.Column) >= numRow { - c.Column = c.Column[:0] - } else { - c.Column = make([][]string, 0, numRow) - } -} - -func (c *StringArrayColumn) Type() reflect.Type { - return stringSliceType -} - -func (c *StringArrayColumn) Set(v any) { - c.Column = v.([][]string) -} - -func (c *StringArrayColumn) Value() any { - return c.Column -} - -func (c *StringArrayColumn) Nullable(nulls UInt8Column) any { - panic("not implemented") -} - -func (c *StringArrayColumn) Len() int { - return len(c.Column) -} - -func (c *StringArrayColumn) Index(idx int) any { - return c.Column[idx] -} - -func (c StringArrayColumn) Slice(s, e int) any { - return c.Column[s:e] -} - -func (c *StringArrayColumn) ConvertAssign(idx int, v reflect.Value) error { - v.Set(reflect.ValueOf(c.Column[idx])) - return nil -} - -func (c *StringArrayColumn) AppendValue(v reflect.Value) { - c.Column = append(c.Column, v.Interface().([]string)) -} - -func (c *StringArrayColumn) AppendPointer(typ reflect.Type, ptr unsafe.Pointer) { - c.Column = append(c.Column, *(*[]string)(ptr)) -} - -func (c *StringArrayColumn) ReadFrom(rd *chproto.Reader, numRow int) error { - if numRow == 0 { - return nil - } - - if cap(c.Column) >= numRow { - c.Column = c.Column[:numRow] - } else { - c.Column = make([][]string, numRow) - } - - if c.lcElem != nil { - if err := c.lcElem.readPrefix(rd, numRow); err != nil { - return err - } - } - - offsets := make([]int, numRow) - - for i := 0; i < len(offsets); i++ { - offset, err := rd.UInt64() - if err != nil { - return err - } - offsets[i] = int(offset) - } - - if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { - return err - } - - var prev int - for i, offset := range offsets { - c.Column[i] = c.stringElem.Column[prev:offset] - prev = offset - } - - return nil -} - -func (c *StringArrayColumn) WriteTo(wr *chproto.Writer) error { - if c.lcElem != nil { - c.lcElem.writePrefix(wr) - } - - _ = c.WriteOffset(wr, 0) - return c.WriteData(wr) -} - -var _ ArrayColumnar = (*StringArrayColumn)(nil) - -func (c *StringArrayColumn) WriteOffset(wr *chproto.Writer, offset int) int { - for _, el := range c.Column { - offset += len(el) - wr.UInt64(uint64(offset)) - } - return offset -} - -func (c *StringArrayColumn) WriteData(wr *chproto.Writer) error { - for _, ss := range c.Column { - c.stringElem.Column = ss - if err := c.elem.WriteTo(wr); err != nil { - return err - } - } - return nil -} - -//------------------------------------------------------------------------------ - -type ArrayLCStringColumn struct { - *LCStringColumn -} - -func (c ArrayLCStringColumn) Type() reflect.Type { - return stringSliceType -} - -func (c *ArrayLCStringColumn) WriteTo(wr *chproto.Writer) error { - c.writeData(wr) - return nil -} - -func (c *ArrayLCStringColumn) ReadFrom(rd *chproto.Reader, numRow int) error { - if numRow == 0 { - return nil - } - return c.readData(rd, numRow) -} - -//------------------------------------------------------------------------------ - -type GenericArrayColumn struct { - Column reflect.Value - - typ reflect.Type - elem Columnar - arrayElem ArrayColumnar -} - -var _ Columnar = (*GenericArrayColumn)(nil) - -func NewGenericArrayColumn(typ reflect.Type, chType string, numRow int) Columnar { - elemType := chArrayElemType(chType) - if elemType == "" { - panic(fmt.Errorf("invalid array type: %q (Go type is %s)", - chType, typ.String())) - } - - elem := NewColumn(typ.Elem(), elemType, 0) - var arrayElem ArrayColumnar - - if _, ok := elem.(*LCStringColumn); ok { - panic("not reached") - } - arrayElem, _ = elem.(ArrayColumnar) - - c := &GenericArrayColumn{ - typ: reflect.SliceOf(typ), - elem: elem, - arrayElem: arrayElem, - } - - c.Column = reflect.MakeSlice(c.typ, 0, numRow) - - return c -} - -func (c GenericArrayColumn) Type() reflect.Type { - return c.typ.Elem() -} - -func (c *GenericArrayColumn) Reset(numRow int) { - if c.Column.Cap() >= numRow { - c.Column = c.Column.Slice(0, 0) - } else { - c.Column = reflect.MakeSlice(c.typ, 0, numRow) - } -} - -func (c *GenericArrayColumn) Set(v any) { - c.Column = reflect.ValueOf(v) -} - -func (c *GenericArrayColumn) Value() any { - return c.Column.Interface() -} - -func (c *GenericArrayColumn) Nullable(nulls UInt8Column) any { - panic("not implemented") -} - -func (c *GenericArrayColumn) Len() int { - return c.Column.Len() -} - -func (c *GenericArrayColumn) Index(idx int) any { - return c.Column.Index(idx).Interface() -} - -func (c GenericArrayColumn) Slice(s, e int) any { - return c.Column.Slice(s, e).Interface() -} - -func (c *GenericArrayColumn) ConvertAssign(idx int, v reflect.Value) error { - v.Set(c.Column.Index(idx)) - return nil -} - -func (c *GenericArrayColumn) AppendValue(v reflect.Value) { - c.Column = reflect.Append(c.Column, v) -} - -func (c *GenericArrayColumn) AppendPointer(typ reflect.Type, ptr unsafe.Pointer) { - c.AppendValue(reflect.NewAt(typ.Elem(), ptr).Elem()) -} - -func (c *GenericArrayColumn) ReadFrom(rd *chproto.Reader, numRow int) error { - if c.Column.Cap() >= numRow { - c.Column = c.Column.Slice(0, numRow) - } else { - c.Column = reflect.MakeSlice(c.typ, numRow, numRow) - } - - if numRow == 0 { - return nil - } - - offsets := make([]int, numRow) - for i := 0; i < len(offsets); i++ { - offset, err := rd.UInt64() - if err != nil { - return err - } - offsets[i] = int(offset) - } - - if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { - return err - } - - var prev int - for i, offset := range offsets { - c.Column.Index(i).Set(reflect.ValueOf(c.elem.Slice(prev, offset))) - prev = offset - } - - return nil -} - -func (c *GenericArrayColumn) WriteTo(wr *chproto.Writer) error { - _ = c.WriteOffset(wr, 0) - - colLen := c.Column.Len() - for i := 0; i < colLen; i++ { - // TODO: add SetValue or SetPointer - c.elem.Set(c.Column.Index(i).Interface()) - - var err error - if c.arrayElem != nil { - err = c.arrayElem.WriteData(wr) - } else { - err = c.elem.WriteTo(wr) - } - if err != nil { - return err - } - } - - return nil -} - -func (c *GenericArrayColumn) WriteOffset(wr *chproto.Writer, offset int) int { - colLen := c.Column.Len() - - for i := 0; i < colLen; i++ { - el := c.Column.Index(i) - offset += el.Len() - wr.UInt64(uint64(offset)) - } - - if c.arrayElem == nil { - return offset - } - - offset = 0 - for i := 0; i < colLen; i++ { - el := c.Column.Index(i) - c.elem.Set(el.Interface()) // Use SetValue or SetPointer - offset = c.arrayElem.WriteOffset(wr, offset) - } - - return offset -} diff --git a/ch/chschema/column_gen.go b/ch/chschema/column_gen.go new file mode 100644 index 0000000..5180b68 --- /dev/null +++ b/ch/chschema/column_gen.go @@ -0,0 +1,3008 @@ +package chschema + +import ( + "reflect" + "time" + + "github.com/uptrace/go-clickhouse/ch/chproto" +) + +type Int8Column struct { + NumericColumnOf[int8] +} + +var _ Columnar = (*Int8Column)(nil) + +func NewInt8Column() Columnar { + return new(Int8Column) +} + +var _Int8Type = reflect.TypeOf((*int8)(nil)).Elem() + +func (c *Int8Column) Type() reflect.Type { + return _Int8Type +} + +func (c *Int8Column) AppendValue(v reflect.Value) { + c.Column = append(c.Column, int8(v.Int())) +} + +//------------------------------------------------------------------------------ + +type ArrayInt8Column struct { + ColumnOf[[]int8] + elem Int8Column +} + +var ( + _ Columnar = (*ArrayInt8Column)(nil) + _ ArrayColumnar = (*ArrayInt8Column)(nil) +) + +func NewArrayInt8Column() Columnar { + return new(ArrayInt8Column) +} + +func (c *ArrayInt8Column) Init(chType string) error { + return c.elem.Init(chArrayElemType(chType)) +} + +func (c *ArrayInt8Column) Type() reflect.Type { + return reflect.TypeOf((*[]int8)(nil)).Elem() +} + +func (c *ArrayInt8Column) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayInt8Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayInt8Column) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayInt8Column) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + return offset +} + +func (c *ArrayInt8Column) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteTo(wr); err != nil { + return err + } + } + return nil +} + +//------------------------------------------------------------------------------ + +type ArrayArrayInt8Column struct { + ColumnOf[[][]int8] + elem ArrayInt8Column +} + +var ( + _ Columnar = (*ArrayArrayInt8Column)(nil) + _ ArrayColumnar = (*ArrayArrayInt8Column)(nil) +) + +func NewArrayArrayInt8Column() Columnar { + return new(ArrayArrayInt8Column) +} + +func (c *ArrayArrayInt8Column) Init(chType string) error { + return c.elem.Init(chArrayElemType(chArrayElemType(chType))) +} + +func (c *ArrayArrayInt8Column) Type() reflect.Type { + return reflect.TypeOf((*[][]int8)(nil)).Elem() +} + +func (c *ArrayArrayInt8Column) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayArrayInt8Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayArrayInt8Column) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayArrayInt8Column) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + offset = 0 + for _, elem := range c.Column { + c.elem.Column = elem + offset = c.elem.WriteOffset(wr, offset) + } + + return offset +} + +func (c *ArrayArrayInt8Column) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteData(wr); err != nil { + return err + } + } + return nil +} + +type UInt8Column struct { + NumericColumnOf[uint8] +} + +var _ Columnar = (*UInt8Column)(nil) + +func NewUInt8Column() Columnar { + return new(UInt8Column) +} + +var _UInt8Type = reflect.TypeOf((*uint8)(nil)).Elem() + +func (c *UInt8Column) Type() reflect.Type { + return _UInt8Type +} + +func (c *UInt8Column) AppendValue(v reflect.Value) { + c.Column = append(c.Column, uint8(v.Uint())) +} + +//------------------------------------------------------------------------------ + +type ArrayUInt8Column struct { + ColumnOf[[]uint8] + elem UInt8Column +} + +var ( + _ Columnar = (*ArrayUInt8Column)(nil) + _ ArrayColumnar = (*ArrayUInt8Column)(nil) +) + +func NewArrayUInt8Column() Columnar { + return new(ArrayUInt8Column) +} + +func (c *ArrayUInt8Column) Init(chType string) error { + return c.elem.Init(chArrayElemType(chType)) +} + +func (c *ArrayUInt8Column) Type() reflect.Type { + return reflect.TypeOf((*[]uint8)(nil)).Elem() +} + +func (c *ArrayUInt8Column) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayUInt8Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayUInt8Column) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayUInt8Column) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + return offset +} + +func (c *ArrayUInt8Column) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteTo(wr); err != nil { + return err + } + } + return nil +} + +//------------------------------------------------------------------------------ + +type ArrayArrayUInt8Column struct { + ColumnOf[[][]uint8] + elem ArrayUInt8Column +} + +var ( + _ Columnar = (*ArrayArrayUInt8Column)(nil) + _ ArrayColumnar = (*ArrayArrayUInt8Column)(nil) +) + +func NewArrayArrayUInt8Column() Columnar { + return new(ArrayArrayUInt8Column) +} + +func (c *ArrayArrayUInt8Column) Init(chType string) error { + return c.elem.Init(chArrayElemType(chArrayElemType(chType))) +} + +func (c *ArrayArrayUInt8Column) Type() reflect.Type { + return reflect.TypeOf((*[][]uint8)(nil)).Elem() +} + +func (c *ArrayArrayUInt8Column) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayArrayUInt8Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayArrayUInt8Column) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayArrayUInt8Column) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + offset = 0 + for _, elem := range c.Column { + c.elem.Column = elem + offset = c.elem.WriteOffset(wr, offset) + } + + return offset +} + +func (c *ArrayArrayUInt8Column) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteData(wr); err != nil { + return err + } + } + return nil +} + +type Int16Column struct { + NumericColumnOf[int16] +} + +var _ Columnar = (*Int16Column)(nil) + +func NewInt16Column() Columnar { + return new(Int16Column) +} + +var _Int16Type = reflect.TypeOf((*int16)(nil)).Elem() + +func (c *Int16Column) Type() reflect.Type { + return _Int16Type +} + +func (c *Int16Column) AppendValue(v reflect.Value) { + c.Column = append(c.Column, int16(v.Int())) +} + +//------------------------------------------------------------------------------ + +type ArrayInt16Column struct { + ColumnOf[[]int16] + elem Int16Column +} + +var ( + _ Columnar = (*ArrayInt16Column)(nil) + _ ArrayColumnar = (*ArrayInt16Column)(nil) +) + +func NewArrayInt16Column() Columnar { + return new(ArrayInt16Column) +} + +func (c *ArrayInt16Column) Init(chType string) error { + return c.elem.Init(chArrayElemType(chType)) +} + +func (c *ArrayInt16Column) Type() reflect.Type { + return reflect.TypeOf((*[]int16)(nil)).Elem() +} + +func (c *ArrayInt16Column) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayInt16Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayInt16Column) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayInt16Column) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + return offset +} + +func (c *ArrayInt16Column) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteTo(wr); err != nil { + return err + } + } + return nil +} + +//------------------------------------------------------------------------------ + +type ArrayArrayInt16Column struct { + ColumnOf[[][]int16] + elem ArrayInt16Column +} + +var ( + _ Columnar = (*ArrayArrayInt16Column)(nil) + _ ArrayColumnar = (*ArrayArrayInt16Column)(nil) +) + +func NewArrayArrayInt16Column() Columnar { + return new(ArrayArrayInt16Column) +} + +func (c *ArrayArrayInt16Column) Init(chType string) error { + return c.elem.Init(chArrayElemType(chArrayElemType(chType))) +} + +func (c *ArrayArrayInt16Column) Type() reflect.Type { + return reflect.TypeOf((*[][]int16)(nil)).Elem() +} + +func (c *ArrayArrayInt16Column) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayArrayInt16Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayArrayInt16Column) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayArrayInt16Column) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + offset = 0 + for _, elem := range c.Column { + c.elem.Column = elem + offset = c.elem.WriteOffset(wr, offset) + } + + return offset +} + +func (c *ArrayArrayInt16Column) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteData(wr); err != nil { + return err + } + } + return nil +} + +type UInt16Column struct { + NumericColumnOf[uint16] +} + +var _ Columnar = (*UInt16Column)(nil) + +func NewUInt16Column() Columnar { + return new(UInt16Column) +} + +var _UInt16Type = reflect.TypeOf((*uint16)(nil)).Elem() + +func (c *UInt16Column) Type() reflect.Type { + return _UInt16Type +} + +func (c *UInt16Column) AppendValue(v reflect.Value) { + c.Column = append(c.Column, uint16(v.Uint())) +} + +//------------------------------------------------------------------------------ + +type ArrayUInt16Column struct { + ColumnOf[[]uint16] + elem UInt16Column +} + +var ( + _ Columnar = (*ArrayUInt16Column)(nil) + _ ArrayColumnar = (*ArrayUInt16Column)(nil) +) + +func NewArrayUInt16Column() Columnar { + return new(ArrayUInt16Column) +} + +func (c *ArrayUInt16Column) Init(chType string) error { + return c.elem.Init(chArrayElemType(chType)) +} + +func (c *ArrayUInt16Column) Type() reflect.Type { + return reflect.TypeOf((*[]uint16)(nil)).Elem() +} + +func (c *ArrayUInt16Column) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayUInt16Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayUInt16Column) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayUInt16Column) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + return offset +} + +func (c *ArrayUInt16Column) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteTo(wr); err != nil { + return err + } + } + return nil +} + +//------------------------------------------------------------------------------ + +type ArrayArrayUInt16Column struct { + ColumnOf[[][]uint16] + elem ArrayUInt16Column +} + +var ( + _ Columnar = (*ArrayArrayUInt16Column)(nil) + _ ArrayColumnar = (*ArrayArrayUInt16Column)(nil) +) + +func NewArrayArrayUInt16Column() Columnar { + return new(ArrayArrayUInt16Column) +} + +func (c *ArrayArrayUInt16Column) Init(chType string) error { + return c.elem.Init(chArrayElemType(chArrayElemType(chType))) +} + +func (c *ArrayArrayUInt16Column) Type() reflect.Type { + return reflect.TypeOf((*[][]uint16)(nil)).Elem() +} + +func (c *ArrayArrayUInt16Column) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayArrayUInt16Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayArrayUInt16Column) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayArrayUInt16Column) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + offset = 0 + for _, elem := range c.Column { + c.elem.Column = elem + offset = c.elem.WriteOffset(wr, offset) + } + + return offset +} + +func (c *ArrayArrayUInt16Column) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteData(wr); err != nil { + return err + } + } + return nil +} + +type Int32Column struct { + NumericColumnOf[int32] +} + +var _ Columnar = (*Int32Column)(nil) + +func NewInt32Column() Columnar { + return new(Int32Column) +} + +var _Int32Type = reflect.TypeOf((*int32)(nil)).Elem() + +func (c *Int32Column) Type() reflect.Type { + return _Int32Type +} + +func (c *Int32Column) AppendValue(v reflect.Value) { + c.Column = append(c.Column, int32(v.Int())) +} + +//------------------------------------------------------------------------------ + +type ArrayInt32Column struct { + ColumnOf[[]int32] + elem Int32Column +} + +var ( + _ Columnar = (*ArrayInt32Column)(nil) + _ ArrayColumnar = (*ArrayInt32Column)(nil) +) + +func NewArrayInt32Column() Columnar { + return new(ArrayInt32Column) +} + +func (c *ArrayInt32Column) Init(chType string) error { + return c.elem.Init(chArrayElemType(chType)) +} + +func (c *ArrayInt32Column) Type() reflect.Type { + return reflect.TypeOf((*[]int32)(nil)).Elem() +} + +func (c *ArrayInt32Column) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayInt32Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayInt32Column) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayInt32Column) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + return offset +} + +func (c *ArrayInt32Column) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteTo(wr); err != nil { + return err + } + } + return nil +} + +//------------------------------------------------------------------------------ + +type ArrayArrayInt32Column struct { + ColumnOf[[][]int32] + elem ArrayInt32Column +} + +var ( + _ Columnar = (*ArrayArrayInt32Column)(nil) + _ ArrayColumnar = (*ArrayArrayInt32Column)(nil) +) + +func NewArrayArrayInt32Column() Columnar { + return new(ArrayArrayInt32Column) +} + +func (c *ArrayArrayInt32Column) Init(chType string) error { + return c.elem.Init(chArrayElemType(chArrayElemType(chType))) +} + +func (c *ArrayArrayInt32Column) Type() reflect.Type { + return reflect.TypeOf((*[][]int32)(nil)).Elem() +} + +func (c *ArrayArrayInt32Column) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayArrayInt32Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayArrayInt32Column) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayArrayInt32Column) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + offset = 0 + for _, elem := range c.Column { + c.elem.Column = elem + offset = c.elem.WriteOffset(wr, offset) + } + + return offset +} + +func (c *ArrayArrayInt32Column) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteData(wr); err != nil { + return err + } + } + return nil +} + +type UInt32Column struct { + NumericColumnOf[uint32] +} + +var _ Columnar = (*UInt32Column)(nil) + +func NewUInt32Column() Columnar { + return new(UInt32Column) +} + +var _UInt32Type = reflect.TypeOf((*uint32)(nil)).Elem() + +func (c *UInt32Column) Type() reflect.Type { + return _UInt32Type +} + +func (c *UInt32Column) AppendValue(v reflect.Value) { + c.Column = append(c.Column, uint32(v.Uint())) +} + +//------------------------------------------------------------------------------ + +type ArrayUInt32Column struct { + ColumnOf[[]uint32] + elem UInt32Column +} + +var ( + _ Columnar = (*ArrayUInt32Column)(nil) + _ ArrayColumnar = (*ArrayUInt32Column)(nil) +) + +func NewArrayUInt32Column() Columnar { + return new(ArrayUInt32Column) +} + +func (c *ArrayUInt32Column) Init(chType string) error { + return c.elem.Init(chArrayElemType(chType)) +} + +func (c *ArrayUInt32Column) Type() reflect.Type { + return reflect.TypeOf((*[]uint32)(nil)).Elem() +} + +func (c *ArrayUInt32Column) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayUInt32Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayUInt32Column) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayUInt32Column) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + return offset +} + +func (c *ArrayUInt32Column) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteTo(wr); err != nil { + return err + } + } + return nil +} + +//------------------------------------------------------------------------------ + +type ArrayArrayUInt32Column struct { + ColumnOf[[][]uint32] + elem ArrayUInt32Column +} + +var ( + _ Columnar = (*ArrayArrayUInt32Column)(nil) + _ ArrayColumnar = (*ArrayArrayUInt32Column)(nil) +) + +func NewArrayArrayUInt32Column() Columnar { + return new(ArrayArrayUInt32Column) +} + +func (c *ArrayArrayUInt32Column) Init(chType string) error { + return c.elem.Init(chArrayElemType(chArrayElemType(chType))) +} + +func (c *ArrayArrayUInt32Column) Type() reflect.Type { + return reflect.TypeOf((*[][]uint32)(nil)).Elem() +} + +func (c *ArrayArrayUInt32Column) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayArrayUInt32Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayArrayUInt32Column) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayArrayUInt32Column) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + offset = 0 + for _, elem := range c.Column { + c.elem.Column = elem + offset = c.elem.WriteOffset(wr, offset) + } + + return offset +} + +func (c *ArrayArrayUInt32Column) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteData(wr); err != nil { + return err + } + } + return nil +} + +type Int64Column struct { + NumericColumnOf[int64] +} + +var _ Columnar = (*Int64Column)(nil) + +func NewInt64Column() Columnar { + return new(Int64Column) +} + +var _Int64Type = reflect.TypeOf((*int64)(nil)).Elem() + +func (c *Int64Column) Type() reflect.Type { + return _Int64Type +} + +func (c *Int64Column) AppendValue(v reflect.Value) { + c.Column = append(c.Column, int64(v.Int())) +} + +//------------------------------------------------------------------------------ + +type ArrayInt64Column struct { + ColumnOf[[]int64] + elem Int64Column +} + +var ( + _ Columnar = (*ArrayInt64Column)(nil) + _ ArrayColumnar = (*ArrayInt64Column)(nil) +) + +func NewArrayInt64Column() Columnar { + return new(ArrayInt64Column) +} + +func (c *ArrayInt64Column) Init(chType string) error { + return c.elem.Init(chArrayElemType(chType)) +} + +func (c *ArrayInt64Column) Type() reflect.Type { + return reflect.TypeOf((*[]int64)(nil)).Elem() +} + +func (c *ArrayInt64Column) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayInt64Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayInt64Column) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayInt64Column) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + return offset +} + +func (c *ArrayInt64Column) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteTo(wr); err != nil { + return err + } + } + return nil +} + +//------------------------------------------------------------------------------ + +type ArrayArrayInt64Column struct { + ColumnOf[[][]int64] + elem ArrayInt64Column +} + +var ( + _ Columnar = (*ArrayArrayInt64Column)(nil) + _ ArrayColumnar = (*ArrayArrayInt64Column)(nil) +) + +func NewArrayArrayInt64Column() Columnar { + return new(ArrayArrayInt64Column) +} + +func (c *ArrayArrayInt64Column) Init(chType string) error { + return c.elem.Init(chArrayElemType(chArrayElemType(chType))) +} + +func (c *ArrayArrayInt64Column) Type() reflect.Type { + return reflect.TypeOf((*[][]int64)(nil)).Elem() +} + +func (c *ArrayArrayInt64Column) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayArrayInt64Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayArrayInt64Column) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayArrayInt64Column) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + offset = 0 + for _, elem := range c.Column { + c.elem.Column = elem + offset = c.elem.WriteOffset(wr, offset) + } + + return offset +} + +func (c *ArrayArrayInt64Column) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteData(wr); err != nil { + return err + } + } + return nil +} + +type UInt64Column struct { + NumericColumnOf[uint64] +} + +var _ Columnar = (*UInt64Column)(nil) + +func NewUInt64Column() Columnar { + return new(UInt64Column) +} + +var _UInt64Type = reflect.TypeOf((*uint64)(nil)).Elem() + +func (c *UInt64Column) Type() reflect.Type { + return _UInt64Type +} + +func (c *UInt64Column) AppendValue(v reflect.Value) { + c.Column = append(c.Column, uint64(v.Uint())) +} + +//------------------------------------------------------------------------------ + +type ArrayUInt64Column struct { + ColumnOf[[]uint64] + elem UInt64Column +} + +var ( + _ Columnar = (*ArrayUInt64Column)(nil) + _ ArrayColumnar = (*ArrayUInt64Column)(nil) +) + +func NewArrayUInt64Column() Columnar { + return new(ArrayUInt64Column) +} + +func (c *ArrayUInt64Column) Init(chType string) error { + return c.elem.Init(chArrayElemType(chType)) +} + +func (c *ArrayUInt64Column) Type() reflect.Type { + return reflect.TypeOf((*[]uint64)(nil)).Elem() +} + +func (c *ArrayUInt64Column) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayUInt64Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayUInt64Column) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayUInt64Column) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + return offset +} + +func (c *ArrayUInt64Column) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteTo(wr); err != nil { + return err + } + } + return nil +} + +//------------------------------------------------------------------------------ + +type ArrayArrayUInt64Column struct { + ColumnOf[[][]uint64] + elem ArrayUInt64Column +} + +var ( + _ Columnar = (*ArrayArrayUInt64Column)(nil) + _ ArrayColumnar = (*ArrayArrayUInt64Column)(nil) +) + +func NewArrayArrayUInt64Column() Columnar { + return new(ArrayArrayUInt64Column) +} + +func (c *ArrayArrayUInt64Column) Init(chType string) error { + return c.elem.Init(chArrayElemType(chArrayElemType(chType))) +} + +func (c *ArrayArrayUInt64Column) Type() reflect.Type { + return reflect.TypeOf((*[][]uint64)(nil)).Elem() +} + +func (c *ArrayArrayUInt64Column) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayArrayUInt64Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayArrayUInt64Column) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayArrayUInt64Column) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + offset = 0 + for _, elem := range c.Column { + c.elem.Column = elem + offset = c.elem.WriteOffset(wr, offset) + } + + return offset +} + +func (c *ArrayArrayUInt64Column) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteData(wr); err != nil { + return err + } + } + return nil +} + +type Float32Column struct { + NumericColumnOf[float32] +} + +var _ Columnar = (*Float32Column)(nil) + +func NewFloat32Column() Columnar { + return new(Float32Column) +} + +var _Float32Type = reflect.TypeOf((*float32)(nil)).Elem() + +func (c *Float32Column) Type() reflect.Type { + return _Float32Type +} + +func (c *Float32Column) AppendValue(v reflect.Value) { + c.Column = append(c.Column, float32(v.Float())) +} + +//------------------------------------------------------------------------------ + +type ArrayFloat32Column struct { + ColumnOf[[]float32] + elem Float32Column +} + +var ( + _ Columnar = (*ArrayFloat32Column)(nil) + _ ArrayColumnar = (*ArrayFloat32Column)(nil) +) + +func NewArrayFloat32Column() Columnar { + return new(ArrayFloat32Column) +} + +func (c *ArrayFloat32Column) Init(chType string) error { + return c.elem.Init(chArrayElemType(chType)) +} + +func (c *ArrayFloat32Column) Type() reflect.Type { + return reflect.TypeOf((*[]float32)(nil)).Elem() +} + +func (c *ArrayFloat32Column) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayFloat32Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayFloat32Column) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayFloat32Column) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + return offset +} + +func (c *ArrayFloat32Column) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteTo(wr); err != nil { + return err + } + } + return nil +} + +//------------------------------------------------------------------------------ + +type ArrayArrayFloat32Column struct { + ColumnOf[[][]float32] + elem ArrayFloat32Column +} + +var ( + _ Columnar = (*ArrayArrayFloat32Column)(nil) + _ ArrayColumnar = (*ArrayArrayFloat32Column)(nil) +) + +func NewArrayArrayFloat32Column() Columnar { + return new(ArrayArrayFloat32Column) +} + +func (c *ArrayArrayFloat32Column) Init(chType string) error { + return c.elem.Init(chArrayElemType(chArrayElemType(chType))) +} + +func (c *ArrayArrayFloat32Column) Type() reflect.Type { + return reflect.TypeOf((*[][]float32)(nil)).Elem() +} + +func (c *ArrayArrayFloat32Column) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayArrayFloat32Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayArrayFloat32Column) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayArrayFloat32Column) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + offset = 0 + for _, elem := range c.Column { + c.elem.Column = elem + offset = c.elem.WriteOffset(wr, offset) + } + + return offset +} + +func (c *ArrayArrayFloat32Column) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteData(wr); err != nil { + return err + } + } + return nil +} + +type Float64Column struct { + NumericColumnOf[float64] +} + +var _ Columnar = (*Float64Column)(nil) + +func NewFloat64Column() Columnar { + return new(Float64Column) +} + +var _Float64Type = reflect.TypeOf((*float64)(nil)).Elem() + +func (c *Float64Column) Type() reflect.Type { + return _Float64Type +} + +func (c *Float64Column) AppendValue(v reflect.Value) { + c.Column = append(c.Column, float64(v.Float())) +} + +//------------------------------------------------------------------------------ + +type ArrayFloat64Column struct { + ColumnOf[[]float64] + elem Float64Column +} + +var ( + _ Columnar = (*ArrayFloat64Column)(nil) + _ ArrayColumnar = (*ArrayFloat64Column)(nil) +) + +func NewArrayFloat64Column() Columnar { + return new(ArrayFloat64Column) +} + +func (c *ArrayFloat64Column) Init(chType string) error { + return c.elem.Init(chArrayElemType(chType)) +} + +func (c *ArrayFloat64Column) Type() reflect.Type { + return reflect.TypeOf((*[]float64)(nil)).Elem() +} + +func (c *ArrayFloat64Column) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayFloat64Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayFloat64Column) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayFloat64Column) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + return offset +} + +func (c *ArrayFloat64Column) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteTo(wr); err != nil { + return err + } + } + return nil +} + +//------------------------------------------------------------------------------ + +type ArrayArrayFloat64Column struct { + ColumnOf[[][]float64] + elem ArrayFloat64Column +} + +var ( + _ Columnar = (*ArrayArrayFloat64Column)(nil) + _ ArrayColumnar = (*ArrayArrayFloat64Column)(nil) +) + +func NewArrayArrayFloat64Column() Columnar { + return new(ArrayArrayFloat64Column) +} + +func (c *ArrayArrayFloat64Column) Init(chType string) error { + return c.elem.Init(chArrayElemType(chArrayElemType(chType))) +} + +func (c *ArrayArrayFloat64Column) Type() reflect.Type { + return reflect.TypeOf((*[][]float64)(nil)).Elem() +} + +func (c *ArrayArrayFloat64Column) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayArrayFloat64Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayArrayFloat64Column) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayArrayFloat64Column) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + offset = 0 + for _, elem := range c.Column { + c.elem.Column = elem + offset = c.elem.WriteOffset(wr, offset) + } + + return offset +} + +func (c *ArrayArrayFloat64Column) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteData(wr); err != nil { + return err + } + } + return nil +} + +type BoolColumn struct { + ColumnOf[bool] +} + +var _ Columnar = (*BoolColumn)(nil) + +func NewBoolColumn() Columnar { + return new(BoolColumn) +} + +var _BoolType = reflect.TypeOf((*bool)(nil)).Elem() + +func (c *BoolColumn) Type() reflect.Type { + return _BoolType +} + +func (c *BoolColumn) AppendValue(v reflect.Value) { + c.Column = append(c.Column, bool(v.Bool())) +} + +func (c *BoolColumn) ReadFrom(rd *chproto.Reader, numRow int) error { + c.AllocForReading(numRow) + + for i := range c.Column { + n, err := rd.Bool() + if err != nil { + return err + } + c.Column[i] = n + } + + return nil +} + +func (c *BoolColumn) WriteTo(wr *chproto.Writer) error { + for _, n := range c.Column { + wr.Bool(n) + } + return nil +} + +//------------------------------------------------------------------------------ + +type ArrayBoolColumn struct { + ColumnOf[[]bool] + elem BoolColumn +} + +var ( + _ Columnar = (*ArrayBoolColumn)(nil) + _ ArrayColumnar = (*ArrayBoolColumn)(nil) +) + +func NewArrayBoolColumn() Columnar { + return new(ArrayBoolColumn) +} + +func (c *ArrayBoolColumn) Init(chType string) error { + return c.elem.Init(chArrayElemType(chType)) +} + +func (c *ArrayBoolColumn) Type() reflect.Type { + return reflect.TypeOf((*[]bool)(nil)).Elem() +} + +func (c *ArrayBoolColumn) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayBoolColumn) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayBoolColumn) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayBoolColumn) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + return offset +} + +func (c *ArrayBoolColumn) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteTo(wr); err != nil { + return err + } + } + return nil +} + +//------------------------------------------------------------------------------ + +type ArrayArrayBoolColumn struct { + ColumnOf[[][]bool] + elem ArrayBoolColumn +} + +var ( + _ Columnar = (*ArrayArrayBoolColumn)(nil) + _ ArrayColumnar = (*ArrayArrayBoolColumn)(nil) +) + +func NewArrayArrayBoolColumn() Columnar { + return new(ArrayArrayBoolColumn) +} + +func (c *ArrayArrayBoolColumn) Init(chType string) error { + return c.elem.Init(chArrayElemType(chArrayElemType(chType))) +} + +func (c *ArrayArrayBoolColumn) Type() reflect.Type { + return reflect.TypeOf((*[][]bool)(nil)).Elem() +} + +func (c *ArrayArrayBoolColumn) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayArrayBoolColumn) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayArrayBoolColumn) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayArrayBoolColumn) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + offset = 0 + for _, elem := range c.Column { + c.elem.Column = elem + offset = c.elem.WriteOffset(wr, offset) + } + + return offset +} + +func (c *ArrayArrayBoolColumn) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteData(wr); err != nil { + return err + } + } + return nil +} + +type StringColumn struct { + ColumnOf[string] +} + +var _ Columnar = (*StringColumn)(nil) + +func NewStringColumn() Columnar { + return new(StringColumn) +} + +var _StringType = reflect.TypeOf((*string)(nil)).Elem() + +func (c *StringColumn) Type() reflect.Type { + return _StringType +} + +func (c *StringColumn) AppendValue(v reflect.Value) { + c.Column = append(c.Column, string(v.String())) +} + +func (c *StringColumn) ReadFrom(rd *chproto.Reader, numRow int) error { + c.AllocForReading(numRow) + + for i := range c.Column { + n, err := rd.String() + if err != nil { + return err + } + c.Column[i] = n + } + + return nil +} + +func (c *StringColumn) WriteTo(wr *chproto.Writer) error { + for _, n := range c.Column { + wr.String(n) + } + return nil +} + +//------------------------------------------------------------------------------ + +type ArrayStringColumn struct { + ColumnOf[[]string] + elem StringColumn +} + +var ( + _ Columnar = (*ArrayStringColumn)(nil) + _ ArrayColumnar = (*ArrayStringColumn)(nil) +) + +func NewArrayStringColumn() Columnar { + return new(ArrayStringColumn) +} + +func (c *ArrayStringColumn) Init(chType string) error { + return c.elem.Init(chArrayElemType(chType)) +} + +func (c *ArrayStringColumn) Type() reflect.Type { + return reflect.TypeOf((*[]string)(nil)).Elem() +} + +func (c *ArrayStringColumn) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayStringColumn) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayStringColumn) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayStringColumn) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + return offset +} + +func (c *ArrayStringColumn) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteTo(wr); err != nil { + return err + } + } + return nil +} + +//------------------------------------------------------------------------------ + +type ArrayArrayStringColumn struct { + ColumnOf[[][]string] + elem ArrayStringColumn +} + +var ( + _ Columnar = (*ArrayArrayStringColumn)(nil) + _ ArrayColumnar = (*ArrayArrayStringColumn)(nil) +) + +func NewArrayArrayStringColumn() Columnar { + return new(ArrayArrayStringColumn) +} + +func (c *ArrayArrayStringColumn) Init(chType string) error { + return c.elem.Init(chArrayElemType(chArrayElemType(chType))) +} + +func (c *ArrayArrayStringColumn) Type() reflect.Type { + return reflect.TypeOf((*[][]string)(nil)).Elem() +} + +func (c *ArrayArrayStringColumn) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayArrayStringColumn) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayArrayStringColumn) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayArrayStringColumn) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + offset = 0 + for _, elem := range c.Column { + c.elem.Column = elem + offset = c.elem.WriteOffset(wr, offset) + } + + return offset +} + +func (c *ArrayArrayStringColumn) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteData(wr); err != nil { + return err + } + } + return nil +} + +type BytesColumn struct { + ColumnOf[[]byte] +} + +var _ Columnar = (*BytesColumn)(nil) + +func NewBytesColumn() Columnar { + return new(BytesColumn) +} + +var _BytesType = reflect.TypeOf((*[]byte)(nil)).Elem() + +func (c *BytesColumn) Type() reflect.Type { + return _BytesType +} + +func (c *BytesColumn) AppendValue(v reflect.Value) { + c.Column = append(c.Column, []byte(v.Bytes())) +} + +func (c *BytesColumn) ReadFrom(rd *chproto.Reader, numRow int) error { + c.AllocForReading(numRow) + + for i := range c.Column { + n, err := rd.Bytes() + if err != nil { + return err + } + c.Column[i] = n + } + + return nil +} + +func (c *BytesColumn) WriteTo(wr *chproto.Writer) error { + for _, n := range c.Column { + wr.Bytes(n) + } + return nil +} + +//------------------------------------------------------------------------------ + +type ArrayBytesColumn struct { + ColumnOf[[][]byte] + elem BytesColumn +} + +var ( + _ Columnar = (*ArrayBytesColumn)(nil) + _ ArrayColumnar = (*ArrayBytesColumn)(nil) +) + +func NewArrayBytesColumn() Columnar { + return new(ArrayBytesColumn) +} + +func (c *ArrayBytesColumn) Init(chType string) error { + return c.elem.Init(chArrayElemType(chType)) +} + +func (c *ArrayBytesColumn) Type() reflect.Type { + return reflect.TypeOf((*[][]byte)(nil)).Elem() +} + +func (c *ArrayBytesColumn) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayBytesColumn) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayBytesColumn) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayBytesColumn) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + return offset +} + +func (c *ArrayBytesColumn) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteTo(wr); err != nil { + return err + } + } + return nil +} + +//------------------------------------------------------------------------------ + +type ArrayArrayBytesColumn struct { + ColumnOf[[][][]byte] + elem ArrayBytesColumn +} + +var ( + _ Columnar = (*ArrayArrayBytesColumn)(nil) + _ ArrayColumnar = (*ArrayArrayBytesColumn)(nil) +) + +func NewArrayArrayBytesColumn() Columnar { + return new(ArrayArrayBytesColumn) +} + +func (c *ArrayArrayBytesColumn) Init(chType string) error { + return c.elem.Init(chArrayElemType(chArrayElemType(chType))) +} + +func (c *ArrayArrayBytesColumn) Type() reflect.Type { + return reflect.TypeOf((*[][][]byte)(nil)).Elem() +} + +func (c *ArrayArrayBytesColumn) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayArrayBytesColumn) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayArrayBytesColumn) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayArrayBytesColumn) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + offset = 0 + for _, elem := range c.Column { + c.elem.Column = elem + offset = c.elem.WriteOffset(wr, offset) + } + + return offset +} + +func (c *ArrayArrayBytesColumn) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteData(wr); err != nil { + return err + } + } + return nil +} + +//------------------------------------------------------------------------------ + +type ArrayEnumColumn struct { + ColumnOf[[]string] + elem EnumColumn +} + +var ( + _ Columnar = (*ArrayEnumColumn)(nil) + _ ArrayColumnar = (*ArrayEnumColumn)(nil) +) + +func NewArrayEnumColumn() Columnar { + return new(ArrayEnumColumn) +} + +func (c *ArrayEnumColumn) Init(chType string) error { + return c.elem.Init(chArrayElemType(chType)) +} + +func (c *ArrayEnumColumn) Type() reflect.Type { + return reflect.TypeOf((*[]string)(nil)).Elem() +} + +func (c *ArrayEnumColumn) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayEnumColumn) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayEnumColumn) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayEnumColumn) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + return offset +} + +func (c *ArrayEnumColumn) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteTo(wr); err != nil { + return err + } + } + return nil +} + +//------------------------------------------------------------------------------ + +type ArrayArrayEnumColumn struct { + ColumnOf[[][]string] + elem ArrayEnumColumn +} + +var ( + _ Columnar = (*ArrayArrayEnumColumn)(nil) + _ ArrayColumnar = (*ArrayArrayEnumColumn)(nil) +) + +func NewArrayArrayEnumColumn() Columnar { + return new(ArrayArrayEnumColumn) +} + +func (c *ArrayArrayEnumColumn) Init(chType string) error { + return c.elem.Init(chArrayElemType(chArrayElemType(chType))) +} + +func (c *ArrayArrayEnumColumn) Type() reflect.Type { + return reflect.TypeOf((*[][]string)(nil)).Elem() +} + +func (c *ArrayArrayEnumColumn) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayArrayEnumColumn) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayArrayEnumColumn) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayArrayEnumColumn) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + offset = 0 + for _, elem := range c.Column { + c.elem.Column = elem + offset = c.elem.WriteOffset(wr, offset) + } + + return offset +} + +func (c *ArrayArrayEnumColumn) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteData(wr); err != nil { + return err + } + } + return nil +} + +type DateTimeColumn struct { + ColumnOf[time.Time] +} + +var _ Columnar = (*DateTimeColumn)(nil) + +func NewDateTimeColumn() Columnar { + return new(DateTimeColumn) +} + +var _DateTimeType = reflect.TypeOf((*time.Time)(nil)).Elem() + +func (c *DateTimeColumn) Type() reflect.Type { + return _DateTimeType +} + +func (c *DateTimeColumn) ReadFrom(rd *chproto.Reader, numRow int) error { + c.AllocForReading(numRow) + + for i := range c.Column { + n, err := rd.DateTime() + if err != nil { + return err + } + c.Column[i] = n + } + + return nil +} + +func (c *DateTimeColumn) WriteTo(wr *chproto.Writer) error { + for _, n := range c.Column { + wr.DateTime(n) + } + return nil +} + +//------------------------------------------------------------------------------ + +type ArrayDateTimeColumn struct { + ColumnOf[[]time.Time] + elem DateTimeColumn +} + +var ( + _ Columnar = (*ArrayDateTimeColumn)(nil) + _ ArrayColumnar = (*ArrayDateTimeColumn)(nil) +) + +func NewArrayDateTimeColumn() Columnar { + return new(ArrayDateTimeColumn) +} + +func (c *ArrayDateTimeColumn) Init(chType string) error { + return c.elem.Init(chArrayElemType(chType)) +} + +func (c *ArrayDateTimeColumn) Type() reflect.Type { + return reflect.TypeOf((*[]time.Time)(nil)).Elem() +} + +func (c *ArrayDateTimeColumn) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayDateTimeColumn) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayDateTimeColumn) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayDateTimeColumn) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + return offset +} + +func (c *ArrayDateTimeColumn) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteTo(wr); err != nil { + return err + } + } + return nil +} + +//------------------------------------------------------------------------------ + +type ArrayArrayDateTimeColumn struct { + ColumnOf[[][]time.Time] + elem ArrayDateTimeColumn +} + +var ( + _ Columnar = (*ArrayArrayDateTimeColumn)(nil) + _ ArrayColumnar = (*ArrayArrayDateTimeColumn)(nil) +) + +func NewArrayArrayDateTimeColumn() Columnar { + return new(ArrayArrayDateTimeColumn) +} + +func (c *ArrayArrayDateTimeColumn) Init(chType string) error { + return c.elem.Init(chArrayElemType(chArrayElemType(chType))) +} + +func (c *ArrayArrayDateTimeColumn) Type() reflect.Type { + return reflect.TypeOf((*[][]time.Time)(nil)).Elem() +} + +func (c *ArrayArrayDateTimeColumn) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayArrayDateTimeColumn) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayArrayDateTimeColumn) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayArrayDateTimeColumn) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + offset = 0 + for _, elem := range c.Column { + c.elem.Column = elem + offset = c.elem.WriteOffset(wr, offset) + } + + return offset +} + +func (c *ArrayArrayDateTimeColumn) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteData(wr); err != nil { + return err + } + } + return nil +} diff --git a/ch/chschema/column_lowcard.go b/ch/chschema/column_lowcard.go new file mode 100644 index 0000000..c090a5c --- /dev/null +++ b/ch/chschema/column_lowcard.go @@ -0,0 +1,319 @@ +package chschema + +import ( + "fmt" + "math" + "reflect" + + "github.com/uptrace/go-clickhouse/ch/chproto" +) + +type LCStringColumn struct { + StringColumn +} + +var _ Columnar = (*LCStringColumn)(nil) + +func NewLCStringColumn() Columnar { + return new(LCStringColumn) +} + +func (c *LCStringColumn) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + if err := c.readPrefix(rd, numRow); err != nil { + return err + } + return c.readData(rd, numRow) +} + +func (c *LCStringColumn) readPrefix(rd *chproto.Reader, numRow int) error { + version, err := rd.Int64() + if err != nil { + return err + } + if version != 1 { + return fmt.Errorf("ch: got version=%d, wanted 1", version) + } + return nil +} + +func (c *LCStringColumn) readData(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + flags, err := rd.Int64() + if err != nil { + return err + } + lcKey := newLCKeyType(flags & 0xf) + + dictSize, err := rd.UInt64() + if err != nil { + return err + } + + dict := make([]string, dictSize) + + for i := range dict { + s, err := rd.String() + if err != nil { + return err + } + dict[i] = s + } + + numKey, err := rd.UInt64() + if err != nil { + return err + } + if int(numKey) != numRow { + return fmt.Errorf("%d != %d", numKey, numRow) + } + + if cap(c.Column) >= int(numKey) { + c.Column = c.Column[:numKey] + } else { + c.Column = make([]string, numKey) + } + + for i := 0; i < int(numKey); i++ { + key, err := lcKey.read(rd) + if err != nil { + return err + } + c.Column[i] = dict[key] + } + + return nil +} + +func (c *LCStringColumn) WriteTo(wr *chproto.Writer) error { + c.writePrefix(wr) + c.writeData(wr) + return nil +} + +func (c *LCStringColumn) writePrefix(wr *chproto.Writer) { + wr.Int64(1) +} + +func (c *LCStringColumn) writeData(wr *chproto.Writer) { + if len(c.Column) == 0 { + return + } + + keys := make([]int, len(c.Column)) + var lc lowCard + + for i, s := range c.Column { + keys[i] = lc.Add(s) + } + + const hasAdditionalKeys = 1 << 9 + const needUpdateDict = 1 << 10 + + dict := lc.Dict() + lcKey := newLCKey(int64(len(dict))) + + wr.Int64(int64(lcKey.typ) | hasAdditionalKeys | needUpdateDict) + + wr.Int64(int64(len(dict))) + for _, s := range dict { + wr.String(s) + } + + wr.Int64(int64(len(keys))) + for _, key := range keys { + lcKey.write(wr, key) + } +} + +//------------------------------------------------------------------------------ + +type ArrayLCStringColumn struct { + ArrayStringColumn + lc LCStringColumn +} + +var _ Columnar = (*ArrayLCStringColumn)(nil) + +func NewArrayLCStringColumn() Columnar { + return new(ArrayLCStringColumn) +} + +func (c *ArrayLCStringColumn) ConvertAssign(idx int, dest reflect.Value) error { + dest.Set(reflect.ValueOf(c.Column[idx])) + return nil +} + +func (c *ArrayLCStringColumn) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + if err := c.lc.readPrefix(rd, numRow); err != nil { + return err + } + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.lc.readData(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.lc.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayLCStringColumn) WriteTo(wr *chproto.Writer) error { + c.lc.writePrefix(wr) + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayLCStringColumn) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.lc.Column = ss + c.lc.writeData(wr) + } + return nil +} + +//------------------------------------------------------------------------------ + +type lcKey struct { + typ int8 + read func(*chproto.Reader) (int, error) + write func(*chproto.Writer, int) +} + +func newLCKey(numKey int64) lcKey { + if numKey <= math.MaxUint8 { + return newLCKeyType(0) + } + if numKey <= math.MaxUint16 { + return newLCKeyType(1) + } + if numKey <= math.MaxUint32 { + return newLCKeyType(2) + } + return newLCKeyType(3) +} + +func newLCKeyType(typ int64) lcKey { + switch typ { + case 0: + return lcKey{ + typ: 0, + read: func(rd *chproto.Reader) (int, error) { + n, err := rd.UInt8() + return int(n), err + }, + write: func(wr *chproto.Writer, n int) { + wr.UInt8(uint8(n)) + }, + } + case 1: + return lcKey{ + typ: int8(1), + read: func(rd *chproto.Reader) (int, error) { + n, err := rd.UInt16() + return int(n), err + }, + write: func(wr *chproto.Writer, n int) { + wr.UInt16(uint16(n)) + }, + } + case 2: + return lcKey{ + typ: 2, + read: func(rd *chproto.Reader) (int, error) { + n, err := rd.UInt32() + return int(n), err + }, + write: func(wr *chproto.Writer, n int) { + wr.UInt32(uint32(n)) + }, + } + case 3: + return lcKey{ + typ: 3, + read: func(rd *chproto.Reader) (int, error) { + n, err := rd.UInt64() + return int(n), err + }, + write: func(wr *chproto.Writer, n int) { + wr.UInt64(uint64(n)) + }, + } + default: + panic("not reached") + } +} + +//------------------------------------------------------------------------------ + +type lowCard struct { + slice sliceMap + dict map[string]int +} + +func (lc *lowCard) Add(word string) int { + if i, ok := lc.dict[word]; ok { + return i + } + + if lc.dict == nil { + lc.dict = make(map[string]int) + } + + i := lc.slice.Add(word) + lc.dict[word] = i + + return i +} + +func (lc *lowCard) Dict() []string { + return lc.slice.Slice() +} + +//------------------------------------------------------------------------------ + +type sliceMap struct { + ss []string +} + +func (m sliceMap) Len() int { + return len(m.ss) +} + +func (m sliceMap) Get(word string) (int, bool) { + for i, s := range m.ss { + if s == word { + return i, true + } + } + return 0, false +} + +func (m *sliceMap) Add(word string) int { + m.ss = append(m.ss, word) + return len(m.ss) - 1 +} + +func (m sliceMap) Slice() []string { + return m.ss +} diff --git a/ch/chschema/column_nullable.go b/ch/chschema/column_nullable.go index cc97fb5..773b6c7 100644 --- a/ch/chschema/column_nullable.go +++ b/ch/chschema/column_nullable.go @@ -12,16 +12,30 @@ type NullableColumn struct { nullable reflect.Value // reflect.Slice } -func NullableNewColumnFunc(fn NewColumnFunc) NewColumnFunc { - return func(typ reflect.Type, chType string, numRow int) Columnar { +func NewNullableColumnFunc(fn NewColumnFunc) NewColumnFunc { + return func() Columnar { return &NullableColumn{ - Values: fn(typ, chType, numRow), + Values: fn(), } } } var _ Columnar = (*NullableColumn)(nil) +func (c *NullableColumn) Init(chType string) error { + return nil +} + +func (c *NullableColumn) AllocForReading(numRow int) { + c.Nulls.AllocForReading(numRow) + c.Values.AllocForReading(numRow) +} + +func (c *NullableColumn) ResetForWriting(numRow int) { + c.Nulls.ResetForWriting(numRow) + c.Values.ResetForWriting(numRow) +} + func (c *NullableColumn) Type() reflect.Type { return reflect.PtrTo(c.Values.Type()) } diff --git a/ch/chschema/column_safe_gen.go b/ch/chschema/column_safe_gen.go index 33a8696..06c7d58 100644 --- a/ch/chschema/column_safe_gen.go +++ b/ch/chschema/column_safe_gen.go @@ -7,7 +7,7 @@ import ( ) func (c *Int8Column) ReadFrom(rd *chproto.Reader, numRow int) error { - c.Alloc(numRow) + c.AllocForReading(numRow) for i := range c.Column { n, err := rd.Int8() @@ -28,7 +28,7 @@ func (c *Int8Column) WriteTo(wr *chproto.Writer) error { } func (c *UInt8Column) ReadFrom(rd *chproto.Reader, numRow int) error { - c.Alloc(numRow) + c.AllocForReading(numRow) for i := range c.Column { n, err := rd.UInt8() @@ -49,7 +49,7 @@ func (c *UInt8Column) WriteTo(wr *chproto.Writer) error { } func (c *Int16Column) ReadFrom(rd *chproto.Reader, numRow int) error { - c.Alloc(numRow) + c.AllocForReading(numRow) for i := range c.Column { n, err := rd.Int16() @@ -70,7 +70,7 @@ func (c *Int16Column) WriteTo(wr *chproto.Writer) error { } func (c *UInt16Column) ReadFrom(rd *chproto.Reader, numRow int) error { - c.Alloc(numRow) + c.AllocForReading(numRow) for i := range c.Column { n, err := rd.UInt16() @@ -91,7 +91,7 @@ func (c *UInt16Column) WriteTo(wr *chproto.Writer) error { } func (c *Int32Column) ReadFrom(rd *chproto.Reader, numRow int) error { - c.Alloc(numRow) + c.AllocForReading(numRow) for i := range c.Column { n, err := rd.Int32() @@ -112,7 +112,7 @@ func (c *Int32Column) WriteTo(wr *chproto.Writer) error { } func (c *UInt32Column) ReadFrom(rd *chproto.Reader, numRow int) error { - c.Alloc(numRow) + c.AllocForReading(numRow) for i := range c.Column { n, err := rd.UInt32() @@ -133,7 +133,7 @@ func (c *UInt32Column) WriteTo(wr *chproto.Writer) error { } func (c *Int64Column) ReadFrom(rd *chproto.Reader, numRow int) error { - c.Alloc(numRow) + c.AllocForReading(numRow) for i := range c.Column { n, err := rd.Int64() @@ -154,7 +154,7 @@ func (c *Int64Column) WriteTo(wr *chproto.Writer) error { } func (c *UInt64Column) ReadFrom(rd *chproto.Reader, numRow int) error { - c.Alloc(numRow) + c.AllocForReading(numRow) for i := range c.Column { n, err := rd.UInt64() @@ -175,7 +175,7 @@ func (c *UInt64Column) WriteTo(wr *chproto.Writer) error { } func (c *Float32Column) ReadFrom(rd *chproto.Reader, numRow int) error { - c.Alloc(numRow) + c.AllocForReading(numRow) for i := range c.Column { n, err := rd.Float32() @@ -196,7 +196,7 @@ func (c *Float32Column) WriteTo(wr *chproto.Writer) error { } func (c *Float64Column) ReadFrom(rd *chproto.Reader, numRow int) error { - c.Alloc(numRow) + c.AllocForReading(numRow) for i := range c.Column { n, err := rd.Float64() diff --git a/ch/chschema/column_unsafe_gen.go b/ch/chschema/column_unsafe_gen.go index d4f2bd5..d2e69b8 100644 --- a/ch/chschema/column_unsafe_gen.go +++ b/ch/chschema/column_unsafe_gen.go @@ -17,7 +17,7 @@ func (c *Int8Column) ReadFrom(rd *chproto.Reader, numRow int) error { return nil } - c.Alloc(numRow) + c.AllocForReading(numRow) slice := *(*reflect.SliceHeader)(unsafe.Pointer(&c.Column)) slice.Len *= size @@ -51,7 +51,7 @@ func (c *UInt8Column) ReadFrom(rd *chproto.Reader, numRow int) error { return nil } - c.Alloc(numRow) + c.AllocForReading(numRow) slice := *(*reflect.SliceHeader)(unsafe.Pointer(&c.Column)) slice.Len *= size @@ -85,7 +85,7 @@ func (c *Int16Column) ReadFrom(rd *chproto.Reader, numRow int) error { return nil } - c.Alloc(numRow) + c.AllocForReading(numRow) slice := *(*reflect.SliceHeader)(unsafe.Pointer(&c.Column)) slice.Len *= size @@ -119,7 +119,7 @@ func (c *UInt16Column) ReadFrom(rd *chproto.Reader, numRow int) error { return nil } - c.Alloc(numRow) + c.AllocForReading(numRow) slice := *(*reflect.SliceHeader)(unsafe.Pointer(&c.Column)) slice.Len *= size @@ -153,7 +153,7 @@ func (c *Int32Column) ReadFrom(rd *chproto.Reader, numRow int) error { return nil } - c.Alloc(numRow) + c.AllocForReading(numRow) slice := *(*reflect.SliceHeader)(unsafe.Pointer(&c.Column)) slice.Len *= size @@ -187,7 +187,7 @@ func (c *UInt32Column) ReadFrom(rd *chproto.Reader, numRow int) error { return nil } - c.Alloc(numRow) + c.AllocForReading(numRow) slice := *(*reflect.SliceHeader)(unsafe.Pointer(&c.Column)) slice.Len *= size @@ -221,7 +221,7 @@ func (c *Int64Column) ReadFrom(rd *chproto.Reader, numRow int) error { return nil } - c.Alloc(numRow) + c.AllocForReading(numRow) slice := *(*reflect.SliceHeader)(unsafe.Pointer(&c.Column)) slice.Len *= size @@ -255,7 +255,7 @@ func (c *UInt64Column) ReadFrom(rd *chproto.Reader, numRow int) error { return nil } - c.Alloc(numRow) + c.AllocForReading(numRow) slice := *(*reflect.SliceHeader)(unsafe.Pointer(&c.Column)) slice.Len *= size @@ -289,7 +289,7 @@ func (c *Float32Column) ReadFrom(rd *chproto.Reader, numRow int) error { return nil } - c.Alloc(numRow) + c.AllocForReading(numRow) slice := *(*reflect.SliceHeader)(unsafe.Pointer(&c.Column)) slice.Len *= size @@ -323,7 +323,7 @@ func (c *Float64Column) ReadFrom(rd *chproto.Reader, numRow int) error { return nil } - c.Alloc(numRow) + c.AllocForReading(numRow) slice := *(*reflect.SliceHeader)(unsafe.Pointer(&c.Column)) slice.Len *= size diff --git a/ch/chschema/enum.go b/ch/chschema/enum.go index ed31e46..efe65be 100644 --- a/ch/chschema/enum.go +++ b/ch/chschema/enum.go @@ -40,7 +40,7 @@ func parseEnum(s string) *enumInfo { } func _parseEnum(chType string) (*enumInfo, error) { - s := enumType(chType) + s := chEnumType(chType) if s == "" { return nil, fmt.Errorf("can't parse enum type: %q", chType) } diff --git a/ch/chschema/lowcard.go b/ch/chschema/lowcard.go deleted file mode 100644 index bd2a8b7..0000000 --- a/ch/chschema/lowcard.go +++ /dev/null @@ -1,53 +0,0 @@ -package chschema - -type lowCard struct { - slice sliceMap - dict map[string]int -} - -func (lc *lowCard) Add(word string) int { - if i, ok := lc.dict[word]; ok { - return i - } - - if lc.dict == nil { - lc.dict = make(map[string]int) - } - - i := lc.slice.Add(word) - lc.dict[word] = i - - return i -} - -func (lc *lowCard) Dict() []string { - return lc.slice.Slice() -} - -//------------------------------------------------------------------------------ - -type sliceMap struct { - ss []string -} - -func (m sliceMap) Len() int { - return len(m.ss) -} - -func (m sliceMap) Get(word string) (int, bool) { - for i, s := range m.ss { - if s == word { - return i, true - } - } - return 0, false -} - -func (m *sliceMap) Add(word string) int { - m.ss = append(m.ss, word) - return len(m.ss) - 1 -} - -func (m sliceMap) Slice() []string { - return m.ss -} diff --git a/ch/chschema/table.go b/ch/chschema/table.go index d8eea16..4570992 100644 --- a/ch/chschema/table.go +++ b/ch/chschema/table.go @@ -154,7 +154,7 @@ func (t *Table) addFields(typ reflect.Type, baseIndex []int) { } } if f.NewColumn == nil { - f.NewColumn = ColumnFactory(f.Type, f.CHType) + f.NewColumn = ColumnFactory(f.CHType, f.Type) } } } @@ -211,7 +211,7 @@ func (t *Table) newField(f reflect.StructField, index []int, tag tagparser.Tag) field.CHType = s field.setFlag(customTypeFlag) } else { - field.CHType = clickhouseType(f.Type) + field.CHType = chType(f.Type) } if tag.HasOption("lc") { @@ -251,7 +251,7 @@ func (t *Table) addField(field *Field) { t.FieldMap[field.CHName] = field } -func (t *Table) NewColumn(colName, colType string, numRow int) *Column { +func (t *Table) NewColumn(colName, colType string) *Column { field, ok := t.FieldMap[colName] if !ok { internal.Logger.Printf("ch: %s has no column=%q", t, colName) @@ -259,22 +259,20 @@ func (t *Table) NewColumn(colName, colType string, numRow int) *Column { } if colType != field.CHType { - if field.CHType != chtype.Any && false { - internal.Logger.Printf("got column type %q, but %s.%s has type %q", - colType, t.Type.Name(), field.GoName, field.CHType) - } - return &Column{ Name: colName, Type: colType, - Columnar: ColumnFactory(field.Type, colType)(field.Type, colType, numRow), + Columnar: NewColumn(colType, field.Type), } } + col := field.NewColumn() + col.Init(field.CHType) + return &Column{ Name: colName, Type: field.CHType, - Columnar: field.NewColumn(field.Type, field.CHType, numRow), + Columnar: col, } } diff --git a/ch/chschema/types.go b/ch/chschema/types.go index f8aa5b9..4d07da6 100644 --- a/ch/chschema/types.go +++ b/ch/chschema/types.go @@ -13,7 +13,32 @@ import ( "github.com/uptrace/go-clickhouse/ch/internal" ) -var chType = [...]string{ +var ( + boolType = reflect.TypeOf(false) + int8Type = reflect.TypeOf(int8(0)) + int16Type = reflect.TypeOf(int16(0)) + int32Type = reflect.TypeOf(int32(0)) + int64Type = reflect.TypeOf(int64(0)) + uint8Type = reflect.TypeOf(uint8(0)) + uint16Type = reflect.TypeOf(uint16(0)) + uint32Type = reflect.TypeOf(uint32(0)) + uint64Type = reflect.TypeOf(uint64(0)) + float32Type = reflect.TypeOf(float32(0)) + float64Type = reflect.TypeOf(float64(0)) + + stringType = reflect.TypeOf("") + bytesType = reflect.TypeOf((*[]byte)(nil)).Elem() + uuidType = reflect.TypeOf((*UUID)(nil)).Elem() + timeType = reflect.TypeOf((*time.Time)(nil)).Elem() + ipType = reflect.TypeOf((*net.IP)(nil)).Elem() + ipNetType = reflect.TypeOf((*net.IPNet)(nil)).Elem() + bfloat16MapType = reflect.TypeOf((*bfloat16.Map)(nil)).Elem() + + sliceUint64Type = reflect.TypeOf((*[]uint64)(nil)).Elem() + sliceFloat32Type = reflect.TypeOf((*[]float32)(nil)).Elem() +) + +var chTypes = [...]string{ reflect.Bool: chtype.UInt8, reflect.Int: chtype.Int64, reflect.Int8: chtype.Int8, @@ -42,152 +67,16 @@ var chType = [...]string{ reflect.UnsafePointer: "", } -type NewColumnFunc func(typ reflect.Type, chType string, numRow int) Columnar +type NewColumnFunc func() Columnar -var kindToColumn = [...]NewColumnFunc{ - reflect.Bool: NewBoolColumn, - reflect.Int: NewInt64Column, - reflect.Int8: NewInt8Column, - reflect.Int16: NewInt16Column, - reflect.Int32: NewInt32Column, - reflect.Int64: NewInt64Column, - reflect.Uint: NewUInt64Column, - reflect.Uint8: NewUInt8Column, - reflect.Uint16: NewUInt16Column, - reflect.Uint32: NewUInt32Column, - reflect.Uint64: NewUInt64Column, - reflect.Uintptr: nil, - reflect.Float32: NewFloat32Column, - reflect.Float64: NewFloat64Column, - reflect.Complex64: nil, - reflect.Complex128: nil, - reflect.Array: nil, - reflect.Chan: nil, - reflect.Func: nil, - reflect.Interface: nil, - reflect.Map: NewJSONColumn, - reflect.Ptr: nil, - reflect.Slice: nil, - reflect.String: NewStringColumn, - reflect.Struct: NewJSONColumn, - reflect.UnsafePointer: nil, +func NewColumn(chType string, typ reflect.Type) Columnar { + col := ColumnFactory(chType, typ)() + col.Init(chType) + return col } -// keep in sync with clickhouseType -func ColumnFactory(typ reflect.Type, chType string) NewColumnFunc { - if chType == chtype.Any { - return nil - } - - if s := lowCardinalityType(chType); s != "" { - switch s { - case chtype.String: - return NewLCStringColumn - } - panic(fmt.Errorf("got %s, wanted LowCardinality(String)", chType)) - } - - if s := enumType(chType); s != "" { - return NewEnumColumn - } - if isDateTime64Type(chType) { - return NewDateTime64Column - } - - if strings.HasPrefix(chType, "SimpleAggregateFunction(") { - chType = chSubType(chType, "SimpleAggregateFunction(") - } else if s := dateTimeType(chType); s != "" { - chType = s - } else if funcName, _ := aggFuncNameAndType(chType); funcName != "" { - switch funcName { - case "quantileBFloat16", "quantilesBFloat16": - return NewBFloat16HistColumn - default: - panic(fmt.Errorf("unsupported ClickHouse type: %s", chType)) - } - } - - switch typ { - case timeType: - switch chType { - case chtype.DateTime: - return NewDateTimeColumn - case chtype.Date: - return NewDateColumn - case chtype.Int64: - return NewTimeColumn - } - case ipType: - return NewIPColumn - } - - kind := typ.Kind() - - switch kind { - case reflect.Ptr: - if typ.Elem().Kind() == reflect.Struct { - return NewJSONColumn - } - return NullableNewColumnFunc(ColumnFactory(typ.Elem(), nullableType(chType))) - case reflect.Slice: - switch elem := typ.Elem(); elem.Kind() { - case reflect.Ptr: - if elem.Elem().Kind() == reflect.Struct { - return NewJSONColumn - } - case reflect.Int64: - return NewInt64ArrayColumn - case reflect.Uint64: - return NewUint64ArrayColumn - case reflect.Float64: - return NewFloat64ArrayColumn - case reflect.Uint8: - if chType == chtype.String { - return NewBytesColumn - } - case reflect.String: - return NewStringArrayColumn - case reflect.Struct: - if elem != timeType { - return NewJSONColumn - } - } - - return NewGenericArrayColumn - case reflect.Array: - if isUUID(typ) { - return NewUUIDColumn - } - case reflect.Interface: - return columnFromCHType(chType) - } - +func ColumnFactory(chType string, typ reflect.Type) NewColumnFunc { switch chType { - case chtype.DateTime: - switch typ { - case uint32Type: - return NewUInt32Column - case int64Type: - return NewInt64TimeColumn - default: - return NewDateTimeColumn - } - } - - fn := kindToColumn[kind] - if fn != nil { - return fn - } - - panic(fmt.Errorf("unsupported go_type=%q ch_type=%q", typ.String(), chType)) -} - -func columnFromCHType(chType string) NewColumnFunc { - switch chType { - case chtype.String: - return NewStringColumn - case chtype.UUID: - return NewUUIDColumn case chtype.Int8: return NewInt8Column case chtype.Int16: @@ -208,108 +97,125 @@ func columnFromCHType(chType string) NewColumnFunc { return NewFloat32Column case chtype.Float64: return NewFloat64Column + + case chtype.String: + if typ == bytesType { + return NewBytesColumn + } + return NewStringColumn + case "LowCardinality(String)": + return NewLCStringColumn + case chtype.Bool: + return NewBoolColumn + case chtype.UUID: + return NewUUIDColumn + case chtype.IPv6: + return NewIPColumn + case chtype.DateTime: return NewDateTimeColumn case chtype.DateTime64: return NewDateTime64Column case chtype.Date: return NewDateColumn - case chtype.IPv6: - return NewIPColumn - default: + + case "Array(Int8)": + return NewArrayInt8Column + case "Array(UInt8)": + return NewArrayUInt8Column + case "Array(Int16)": + return NewArrayInt16Column + case "Array(UInt16)": + return NewArrayUInt16Column + case "Array(Int32)": + return NewArrayInt32Column + case "Array(UInt32)": + return NewArrayUInt32Column + case "Array(Int64)": + return NewArrayInt64Column + case "Array(UInt64)": + return NewArrayUInt64Column + case "Array(Float32)": + return NewArrayFloat32Column + case "Array(Float64)": + return NewArrayFloat64Column + + case "Array(String)": + return NewArrayStringColumn + case "Array(LowCardinality(String))": + return NewArrayLCStringColumn + case "Array(DateTime)": + return NewArrayDateTimeColumn + + case "Array(Array(String))": + return NewArrayArrayStringColumn + case chtype.Any: return nil } -} -var ( - boolType = reflect.TypeOf(false) - int8Type = reflect.TypeOf(int8(0)) - int16Type = reflect.TypeOf(int16(0)) - int32Type = reflect.TypeOf(int32(0)) - int64Type = reflect.TypeOf(int64(0)) - uint8Type = reflect.TypeOf(uint8(0)) - uint16Type = reflect.TypeOf(uint16(0)) - uint32Type = reflect.TypeOf(uint32(0)) - uint64Type = reflect.TypeOf(uint64(0)) - float32Type = reflect.TypeOf(float32(0)) - float64Type = reflect.TypeOf(float64(0)) - - stringType = reflect.TypeOf("") - bytesType = reflect.TypeOf((*[]byte)(nil)).Elem() - uuidType = reflect.TypeOf((*UUID)(nil)).Elem() - timeType = reflect.TypeOf((*time.Time)(nil)).Elem() - ipType = reflect.TypeOf((*net.IP)(nil)).Elem() - ipNetType = reflect.TypeOf((*net.IPNet)(nil)).Elem() - bfloat16MapType = reflect.TypeOf((*bfloat16.Map)(nil)).Elem() - - int64SliceType = reflect.TypeOf((*[]int64)(nil)).Elem() - uint64SliceType = reflect.TypeOf((*[]uint64)(nil)).Elem() - float32SliceType = reflect.TypeOf((*[]float32)(nil)).Elem() - float64SliceType = reflect.TypeOf((*[]float64)(nil)).Elem() - stringSliceType = reflect.TypeOf((*[]string)(nil)).Elem() -) - -func goType(chType string) reflect.Type { - switch chType { - case chtype.Int8: - return int8Type - case chtype.Int32: - return int32Type - case chtype.Int64: - return int64Type - case chtype.UInt8: - return uint8Type - case chtype.UInt16: - return uint16Type - case chtype.UInt32: - return uint32Type - case chtype.UInt64: - return uint64Type - case chtype.Float32: - return float32Type - case chtype.Float64: - return float64Type - case chtype.String: - return stringType - case chtype.UUID: - return uuidType - case chtype.DateTime: - return timeType - case chtype.Date: - return timeType - case chtype.IPv6: - return ipType - default: + if chType := chEnumType(chType); chType != "" { + return NewEnumColumn } - - if s := chArrayElemType(chType); s != "" { - return reflect.SliceOf(goType(s)) - } - if s := lowCardinalityType(chType); s != "" { - return goType(s) - } - if s := enumType(chType); s != "" { - return stringType - } - if s := dateTimeType(chType); s != "" { - return timeType + if chType := chArrayElemType(chType); chType != "" { + if chType := chEnumType(chType); chType != "" { + return NewArrayEnumColumn + } } if isDateTime64Type(chType) { - return timeType + return NewDateTime64Column } - if s := nullableType(chType); s != "" { - return reflect.PtrTo(goType(s)) + if chType := chDateTimeType(chType); chType != "" { + return ColumnFactory(chType, typ) } - if _, funcType := aggFuncNameAndType(chType); funcType != "" { - return goType(funcType) + if chType := chNullableType(chType); chType != "" { + if typ != nil { + typ = typ.Elem() + } + return NewNullableColumnFunc(ColumnFactory(chType, typ)) } - panic(fmt.Errorf("unsupported ClickHouse type=%q", chType)) + if chType := chSimpleAggFunc(chType); chType != "" { + return ColumnFactory(chType, typ) + } + + if funcName, _ := aggFuncNameAndType(chType); funcName != "" { + switch funcName { + case "quantileBFloat16", "quantilesBFloat16": + return NewBFloat16HistColumn + default: + panic(fmt.Errorf("unsupported ClickHouse type: %s", chType)) + } + } + + if typ == nil { + panic(fmt.Errorf("unsupported ClickHouse column: %s", chType)) + } + + kind := typ.Kind() + + switch kind { + case reflect.Ptr: + if typ.Elem().Kind() == reflect.Struct { + return NewJSONColumn + } + return NewNullableColumnFunc(ColumnFactory(chNullableType(chType), typ.Elem())) + case reflect.Slice: + switch elem := typ.Elem(); elem.Kind() { + case reflect.Ptr: + if elem.Elem().Kind() == reflect.Struct { + return NewJSONColumn + } + case reflect.Struct: + if elem != timeType { + return NewJSONColumn + } + } + } + + panic(fmt.Errorf("unsupported ClickHouse column: %s", chType)) } -// clickhouseType returns ClickHouse type for the given Go type. -// Keep in sync with ColumnFactory. -func clickhouseType(typ reflect.Type) string { +func chType(typ reflect.Type) string { switch typ { case timeType: return chtype.DateTime @@ -323,7 +229,7 @@ func clickhouseType(typ reflect.Type) string { if typ.Elem().Kind() == reflect.Struct { return chtype.String } - return fmt.Sprintf("Nullable(%s)", clickhouseType(typ.Elem())) + return fmt.Sprintf("Nullable(%s)", chType(typ.Elem())) case reflect.Slice: switch elem := typ.Elem(); elem.Kind() { case reflect.Ptr: @@ -338,14 +244,14 @@ func clickhouseType(typ reflect.Type) string { return chtype.String // []byte } - return "Array(" + clickhouseType(typ.Elem()) + ")" + return "Array(" + chType(typ.Elem()) + ")" case reflect.Array: if isUUID(typ) { return chtype.UUID } } - if s := chType[kind]; s != "" { + if s := chTypes[kind]; s != "" { return s } @@ -353,6 +259,13 @@ func clickhouseType(typ reflect.Type) string { } func chArrayElemType(s string) string { + if s := chSubType(s, "SimpleAggregateFunction("); s != "" { + if i := strings.Index(s, ", "); i >= 0 { + s = s[i+2:] + } + return chSubType(s, "Array(") + } + s = chSubType(s, "Array(") if s == "" { return "" @@ -371,15 +284,23 @@ func chArrayElemType(s string) string { return s } -func lowCardinalityType(s string) string { - return chSubType(s, "LowCardinality(") -} - -func enumType(s string) string { +func chEnumType(s string) string { return chSubType(s, "Enum8(") } -func dateTimeType(s string) string { +func chSimpleAggFunc(s string) string { + s = chSubType(s, "SimpleAggregateFunction(") + if s == "" { + return "" + } + i := strings.Index(s, ", ") + if i == -1 { + return "" + } + return s[i+2:] +} + +func chDateTimeType(s string) string { s = chSubType(s, "DateTime(") if s == "" { return "" @@ -406,7 +327,7 @@ func parseDateTime64Prec(s string) int { return prec } -func nullableType(s string) string { +func chNullableType(s string) string { return chSubType(s, "Nullable(") } diff --git a/ch/chtype/chtype.go b/ch/chtype/chtype.go index c9e03b6..8d60ca5 100644 --- a/ch/chtype/chtype.go +++ b/ch/chtype/chtype.go @@ -2,6 +2,7 @@ package chtype const ( Any = "_" // for decoding into interface{} + Bool = "Bool" String = "String" UUID = "UUID" Int8 = "Int8" diff --git a/ch/db_test.go b/ch/db_test.go index 01a2c83..5f248e8 100644 --- a/ch/db_test.go +++ b/ch/db_test.go @@ -314,30 +314,6 @@ func TestInvalidType(t *testing.T) { require.Equal(t, []float64{}, dest.Numbers) } -type Event struct { - ch.CHModel `ch:"goch_events,partition:toYYYYMM(created_at)"` - - ID uint64 - Name string `ch:",lc"` - Count uint32 - Keys []string `ch:",lc"` - Values [][]string - Kind string `ch:"type:Enum8('invalid' = 0, 'hello' = 1, 'world' = 2)"` - CreatedAt time.Time `ch:",pk"` -} - -type EventColumnar struct { - ch.CHModel `ch:"goch_events,columnar"` - - ID []uint64 - Name []string `ch:",lc"` - Count []uint32 - Keys [][]string `ch:"type:Array(LowCardinality(String))"` - Values [][][]string - Kind []string `ch:"type:Enum8('invalid' = 0, 'hello' = 1, 'world' = 2)"` - CreatedAt []time.Time -} - func TestClickhouse(t *testing.T) { ctx := context.Background() @@ -376,6 +352,30 @@ func testWhereBytes(ctx context.Context, t *testing.T, db *ch.DB) { require.Equal(t, data.Bytes, got.Bytes) } +type Event struct { + ch.CHModel `ch:"goch_events,partition:toYYYYMM(created_at)"` + + ID uint64 + Name string `ch:",lc"` + Count uint32 + Keys []string `ch:",lc"` + Values [][]string + Kind string `ch:"type:Enum8('invalid' = 0, 'hello' = 1, 'world' = 2)"` + CreatedAt time.Time `ch:",pk"` +} + +type EventColumnar struct { + ch.CHModel `ch:"goch_events,columnar"` + + ID []uint64 + Name []string `ch:",lc"` + Count []uint32 + Keys [][]string `ch:"type:Array(LowCardinality(String))"` + Values [][][]string + Kind []string `ch:"type:Enum8('invalid' = 0, 'hello' = 1, 'world' = 2)"` + CreatedAt []time.Time +} + func TestORM(t *testing.T) { ctx := context.Background() diff --git a/ch/internal/codegen/column.tpl b/ch/internal/codegen/column.tpl new file mode 100644 index 0000000..8e59c98 --- /dev/null +++ b/ch/internal/codegen/column.tpl @@ -0,0 +1,238 @@ +package chschema + +import ( + "time" + "reflect" + + "github.com/uptrace/go-clickhouse/ch/chproto" +) + +{{- range . }} + +{{ if not .IsCustom }} + +type {{ .Name }}Column struct { + {{ if gt .Size 0 }}Numeric{{ end }}ColumnOf[{{ .GoType }}] +} + +var _ Columnar = (*{{ .Name }}Column)(nil) + +func New{{ .Name }}Column() Columnar { + return new({{ .Name }}Column) +} + +var _{{ .Name }}Type = reflect.TypeOf((*{{ .GoType }})(nil)).Elem() + +func (c *{{ .Name }}Column) Type() reflect.Type { + return _{{ .Name }}Type +} + +{{ if .GoReflect }} +func (c *{{ .Name }}Column) AppendValue(v reflect.Value) { + c.Column = append(c.Column, {{ .GoType }}(v.{{ .GoReflect }}())) +} +{{ end }} + +{{ if eq .Size 0 }} + +func (c *{{ .Name }}Column) ReadFrom(rd *chproto.Reader, numRow int) error { + c.AllocForReading(numRow) + + for i := range c.Column { + n, err := rd.{{ .Name }}() + if err != nil { + return err + } + c.Column[i] = n + } + + return nil +} + +func (c *{{ .Name }}Column) WriteTo(wr *chproto.Writer) error { + for _, n := range c.Column { + wr.{{ .Name }}(n) + } + return nil +} + +{{ end }} +{{ end }} + +//------------------------------------------------------------------------------ + +type Array{{ .Name }}Column struct { + ColumnOf[[]{{ .GoType }}] + elem {{ .Name }}Column +} + +var ( + _ Columnar = (*Array{{ .Name }}Column)(nil) + _ ArrayColumnar = (*Array{{ .Name }}Column)(nil) +) + +func NewArray{{ .Name }}Column() Columnar { + return new(Array{{ .Name }}Column) +} + +func (c *Array{{ .Name }}Column) Init(chType string) error { + return c.elem.Init(chArrayElemType(chType)) +} + +func (c *Array{{ .Name }}Column) Type() reflect.Type { + return reflect.TypeOf((*[]{{ .GoType }})(nil)).Elem() +} + +func (c *Array{{ .Name }}Column) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *Array{{ .Name }}Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *Array{{ .Name }}Column) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *Array{{ .Name }}Column) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + return offset +} + +func (c *Array{{ .Name }}Column) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteTo(wr); err != nil { + return err + } + } + return nil +} + +//------------------------------------------------------------------------------ + +type ArrayArray{{ .Name }}Column struct { + ColumnOf[[][]{{ .GoType }}] + elem Array{{ .Name }}Column +} + +var ( + _ Columnar = (*ArrayArray{{ .Name }}Column)(nil) + _ ArrayColumnar = (*ArrayArray{{ .Name }}Column)(nil) +) + +func NewArrayArray{{ .Name }}Column() Columnar { + return new(ArrayArray{{ .Name }}Column) +} + +func (c *ArrayArray{{ .Name }}Column) Init(chType string) error { + return c.elem.Init(chArrayElemType(chArrayElemType(chType))) +} + +func (c *ArrayArray{{ .Name }}Column) Type() reflect.Type { + return reflect.TypeOf((*[][]{{ .GoType }})(nil)).Elem() +} + +func (c *ArrayArray{{ .Name }}Column) ReadFrom(rd *chproto.Reader, numRow int) error { + if numRow == 0 { + return nil + } + + c.AllocForReading(numRow) + + offsets, err := c.readOffsets(rd, numRow) + if err != nil { + return err + } + + if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil { + return err + } + + var prev int + for i, offset := range offsets { + c.Column[i] = c.elem.Column[prev:offset] + prev = offset + } + + return nil +} + +func (c *ArrayArray{{ .Name }}Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) { + offsets := make([]int, numRow) + for i := range offsets { + offset, err := rd.UInt64() + if err != nil { + return nil, err + } + offsets[i] = int(offset) + } + return offsets, nil +} + +func (c *ArrayArray{{ .Name }}Column) WriteTo(wr *chproto.Writer) error { + _ = c.WriteOffset(wr, 0) + return c.WriteData(wr) +} + +func (c *ArrayArray{{ .Name }}Column) WriteOffset(wr *chproto.Writer, offset int) int { + for _, el := range c.Column { + offset += len(el) + wr.UInt64(uint64(offset)) + } + + offset = 0 + for _, elem := range c.Column { + c.elem.Column = elem + offset = c.elem.WriteOffset(wr, offset) + } + + return offset +} + +func (c *ArrayArray{{ .Name }}Column) WriteData(wr *chproto.Writer) error { + for _, ss := range c.Column { + c.elem.Column = ss + if err := c.elem.WriteData(wr); err != nil { + return err + } + } + return nil +} + +{{- end }} diff --git a/ch/internal/codegen/column_safe.tpl b/ch/internal/codegen/column_safe.tpl index d00d426..0d0f8a4 100644 --- a/ch/internal/codegen/column_safe.tpl +++ b/ch/internal/codegen/column_safe.tpl @@ -8,11 +8,13 @@ import ( {{- range . }} -func (c *{{ .CHType }}Column) ReadFrom(rd *chproto.Reader, numRow int) error { - c.Alloc(numRow) +{{ if eq .Size 0 }} {{ continue }} {{ end }} + +func (c *{{ .Name }}Column) ReadFrom(rd *chproto.Reader, numRow int) error { + c.AllocForReading(numRow) for i := range c.Column { - n, err := rd.{{ .CHType }}() + n, err := rd.{{ .Name }}() if err != nil { return err } @@ -22,9 +24,9 @@ func (c *{{ .CHType }}Column) ReadFrom(rd *chproto.Reader, numRow int) error { return nil } -func (c *{{ .CHType }}Column) WriteTo(wr *chproto.Writer) error { +func (c *{{ .Name }}Column) WriteTo(wr *chproto.Writer) error { for _, n := range c.Column { - wr.{{ .CHType }}(n) + wr.{{ .Name }}(n) } return nil } diff --git a/ch/internal/codegen/column_unsafe.tpl b/ch/internal/codegen/column_unsafe.tpl index 7cc1ea0..1dd14ce 100644 --- a/ch/internal/codegen/column_unsafe.tpl +++ b/ch/internal/codegen/column_unsafe.tpl @@ -12,14 +12,16 @@ import ( {{- range . }} -func (c *{{ .CHType }}Column) ReadFrom(rd *chproto.Reader, numRow int) error { +{{ if eq .Size 0 }} {{ continue }} {{ end }} + +func (c *{{ .Name }}Column) ReadFrom(rd *chproto.Reader, numRow int) error { const size = {{ .Size }} / 8 if numRow == 0 { return nil } - c.Alloc(numRow) + c.AllocForReading(numRow) slice := *(*reflect.SliceHeader)(unsafe.Pointer(&c.Column)) slice.Len *= size @@ -30,7 +32,7 @@ func (c *{{ .CHType }}Column) ReadFrom(rd *chproto.Reader, numRow int) error { return err } -func (c *{{ .CHType }}Column) WriteTo(wr *chproto.Writer) error { +func (c *{{ .Name }}Column) WriteTo(wr *chproto.Writer) error { const size = {{ .Size }} / 8 if len(c.Column) == 0 {