go/go-clickhouse
1
0
Fork 0
mirror of https://github.com/uptrace/go-clickhouse.git synced 2025-06-14 23:44:59 +02:00
Code Issues Releases Activity

chore: improve code gen

Browse Source
This commit is contained in:
Vladimir Mihailenco 2023-01-21 12:14:00 +02:00
parent b5016bdc7a
commit 44465cd278
17 changed files with 3867 additions and 1629 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ch/chschema/block.go
View File

@ -37,13 +37,13 @@ func (b *Block) Column(colName, colType string) *Column {
var col *Column
if b.Table != nil {
col = b.Table.NewColumn(colName, colType, b.NumRow)
col = b.Table.NewColumn(colName, colType)
}
if col == nil {
col = &Column{
Name: colName,
Type: colType,
Columnar: NewColumnFromCHType(colType, b.NumRow),
Columnar: NewColumn(colType, nil),
}
}

791
ch/chschema/column.go
View File

@ -32,8 +32,9 @@ func (c *Column) String() string {
}
type Columnar interface {
ReadFrom(rd *chproto.Reader, numRow int) error
WriteTo(wr *chproto.Writer) error
Init(chType string) error
AllocForReading(numRow int)
ResetForWriting(numRow int)
Type() reflect.Type
Set(v any)
@ -44,15 +45,14 @@ type Columnar interface {
Index(idx int) any
Slice(s, e int) any
ConvertAssign(idx int, dest reflect.Value) error
ReadFrom(rd *chproto.Reader, numRow int) error
WriteTo(wr *chproto.Writer) error
}
func NewColumn(typ reflect.Type, chType string, numRow int) Columnar {
return ColumnFactory(typ, chType)(typ, chType, numRow)
}
func NewColumnFromCHType(chType string, numRow int) Columnar {
typ := goType(chType)
return NewColumn(typ, chType, numRow)
type ArrayColumnar interface {
WriteOffset(wr *chproto.Writer, offset int) int
WriteData(wr *chproto.Writer) error
}
//------------------------------------------------------------------------------
@ -61,13 +61,11 @@ type ColumnOf[T any] struct {
Column []T
}
func NewColumnOf[T any](numRow int) ColumnOf[T] {
return ColumnOf[T]{
Column: make([]T, 0, numRow),
}
func (c *ColumnOf[T]) Init(chType string) error {
return nil
}
func (c *ColumnOf[T]) Alloc(numRow int) {
func (c *ColumnOf[T]) AllocForReading(numRow int) {
if cap(c.Column) >= numRow {
c.Column = c.Column[:numRow]
} else {
@ -75,7 +73,7 @@ func (c *ColumnOf[T]) Alloc(numRow int) {
}
}
func (c *ColumnOf[T]) Reset(numRow int) {
func (c *ColumnOf[T]) ResetForWriting(numRow int) {
if cap(c.Column) >= numRow {
c.Column = c.Column[:0]
} else {
@ -128,12 +126,6 @@ type NumericColumnOf[T constraints.Integer | constraints.Float] struct {
ColumnOf[T]
}
func NewNumericColumnOf[T constraints.Integer | constraints.Float](numRow int) NumericColumnOf[T] {
col := NumericColumnOf[T]{}
col.Column = make([]T, 0, numRow)
return col
}
func (c NumericColumnOf[T]) ConvertAssign(idx int, v reflect.Value) error {
switch v.Kind() {
case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int:
@ -148,24 +140,6 @@ func (c NumericColumnOf[T]) ConvertAssign(idx int, v reflect.Value) error {
return nil
}
//------------------------------------------------------------------------------
type BoolColumn struct {
ColumnOf[bool]
}
var _ Columnar = (*BoolColumn)(nil)
func NewBoolColumn(typ reflect.Type, chType string, numRow int) Columnar {
return &BoolColumn{
ColumnOf: NewColumnOf[bool](numRow),
}
}
func (c BoolColumn) Type() reflect.Type {
return boolType
}
func (c BoolColumn) ConvertAssign(idx int, v reflect.Value) error {
switch v.Kind() {
case reflect.Bool:
@ -176,269 +150,6 @@ func (c BoolColumn) ConvertAssign(idx int, v reflect.Value) error {
return nil
}
func (c *BoolColumn) AppendValue(v reflect.Value) {
c.Column = append(c.Column, v.Bool())
}
func (c *BoolColumn) ReadFrom(rd *chproto.Reader, numRow int) error {
c.Alloc(numRow)
for i := range c.Column {
flag, err := rd.Bool()
if err != nil {
return err
}
c.Column[i] = flag
}
return nil
}
func (c BoolColumn) WriteTo(wr *chproto.Writer) error {
for _, flag := range c.Column {
wr.Bool(flag)
}
return nil
}
//------------------------------------------------------------------------------
type Int8Column struct {
NumericColumnOf[int8]
}
var _ Columnar = (*Int8Column)(nil)
func NewInt8Column(typ reflect.Type, chType string, numRow int) Columnar {
return &Int8Column{
NumericColumnOf: NewNumericColumnOf[int8](numRow),
}
}
func (c Int8Column) Type() reflect.Type {
return int8Type
}
func (c *Int8Column) AppendValue(v reflect.Value) {
c.Column = append(c.Column, int8(v.Int()))
}
//------------------------------------------------------------------------------
type Int16Column struct {
NumericColumnOf[int16]
}
var _ Columnar = (*Int16Column)(nil)
func NewInt16Column(typ reflect.Type, chType string, numRow int) Columnar {
return &Int16Column{
NumericColumnOf: NewNumericColumnOf[int16](numRow),
}
}
func (c Int16Column) Type() reflect.Type {
return int16Type
}
func (c *Int16Column) AppendValue(v reflect.Value) {
c.Column = append(c.Column, int16(v.Int()))
}
//------------------------------------------------------------------------------
type Int32Column struct {
NumericColumnOf[int32]
}
var _ Columnar = (*Int32Column)(nil)
func NewInt32Column(typ reflect.Type, chType string, numRow int) Columnar {
return &Int32Column{
NumericColumnOf: NewNumericColumnOf[int32](numRow),
}
}
func (c Int32Column) Type() reflect.Type {
return int32Type
}
func (c *Int32Column) AppendValue(v reflect.Value) {
c.Column = append(c.Column, int32(v.Int()))
}
//------------------------------------------------------------------------------
type Int64Column struct {
NumericColumnOf[int64]
}
var _ Columnar = (*Int64Column)(nil)
func NewInt64Column(typ reflect.Type, chType string, numRow int) Columnar {
return &Int64Column{
NumericColumnOf: NewNumericColumnOf[int64](numRow),
}
}
func (c Int64Column) Type() reflect.Type {
return int64Type
}
func (c *Int64Column) AppendValue(v reflect.Value) {
c.Column = append(c.Column, v.Int())
}
//------------------------------------------------------------------------------
type UInt8Column struct {
NumericColumnOf[uint8]
}
var _ Columnar = (*UInt8Column)(nil)
func NewUInt8Column(typ reflect.Type, chType string, numRow int) Columnar {
return &UInt8Column{
NumericColumnOf: NewNumericColumnOf[uint8](numRow),
}
}
func (c UInt8Column) Type() reflect.Type {
return uint8Type
}
func (c *UInt8Column) AppendValue(v reflect.Value) {
c.Column = append(c.Column, uint8(v.Uint()))
}
//------------------------------------------------------------------------------
type UInt16Column struct {
NumericColumnOf[uint16]
}
var _ Columnar = (*UInt16Column)(nil)
func NewUInt16Column(typ reflect.Type, chType string, numRow int) Columnar {
return &UInt16Column{
NumericColumnOf: NewNumericColumnOf[uint16](numRow),
}
}
func (c UInt16Column) Type() reflect.Type {
return uint16Type
}
func (c *UInt16Column) AppendValue(v reflect.Value) {
c.Column = append(c.Column, uint16(v.Uint()))
}
//------------------------------------------------------------------------------
type UInt32Column struct {
NumericColumnOf[uint32]
}
var _ Columnar = (*UInt32Column)(nil)
func NewUInt32Column(typ reflect.Type, chType string, numRow int) Columnar {
return &UInt32Column{
NumericColumnOf: NewNumericColumnOf[uint32](numRow),
}
}
func (c UInt32Column) Type() reflect.Type {
return uint32Type
}
func (c *UInt32Column) AppendValue(v reflect.Value) {
c.Column = append(c.Column, uint32(v.Uint()))
}
//------------------------------------------------------------------------------
type UInt64Column struct {
NumericColumnOf[uint64]
}
var _ Columnar = (*UInt64Column)(nil)
func NewUInt64Column(typ reflect.Type, chType string, numRow int) Columnar {
return &UInt64Column{
NumericColumnOf: NewNumericColumnOf[uint64](numRow),
}
}
func (c UInt64Column) Type() reflect.Type {
return uint64Type
}
func (c *UInt64Column) AppendValue(v reflect.Value) {
c.Column = append(c.Column, v.Uint())
}
//------------------------------------------------------------------------------
type Float32Column struct {
NumericColumnOf[float32]
}
var _ Columnar = (*Float32Column)(nil)
func NewFloat32Column(typ reflect.Type, chType string, numRow int) Columnar {
return &Float32Column{
NumericColumnOf: NewNumericColumnOf[float32](numRow),
}
}
func (c Float32Column) Type() reflect.Type {
return float32Type
}
func (c *Float32Column) AppendValue(v reflect.Value) {
c.Column = append(c.Column, float32(v.Float()))
}
//------------------------------------------------------------------------------
type Float64Column struct {
NumericColumnOf[float64]
}
var _ Columnar = (*Float64Column)(nil)
func NewFloat64Column(typ reflect.Type, chType string, numRow int) Columnar {
return &Float64Column{
NumericColumnOf: NewNumericColumnOf[float64](numRow),
}
}
func (c Float64Column) Type() reflect.Type {
return float64Type
}
func (c *Float64Column) AppendValue(v reflect.Value) {
c.Column = append(c.Column, v.Float())
}
//------------------------------------------------------------------------------
type StringColumn struct {
ColumnOf[string]
}
var _ Columnar = (*StringColumn)(nil)
func NewStringColumn(typ reflect.Type, chType string, numRow int) Columnar {
return &StringColumn{
ColumnOf: NewColumnOf[string](numRow),
}
}
func (c StringColumn) Type() reflect.Type {
return stringType
}
func (c StringColumn) ConvertAssign(idx int, v reflect.Value) error {
switch v.Kind() {
case reflect.String:
@ -460,31 +171,6 @@ func (c StringColumn) ConvertAssign(idx int, v reflect.Value) error {
return fmt.Errorf("ch: can't scan %s into %s", "string", v.Type())
}
func (c *StringColumn) AppendValue(v reflect.Value) {
c.Column = append(c.Column, v.String())
}
func (c *StringColumn) ReadFrom(rd *chproto.Reader, numRow int) error {
c.Alloc(numRow)
for i := range c.Column {
n, err := rd.String()
if err != nil {
return err
}
c.Column[i] = n
}
return nil
}
func (c StringColumn) WriteTo(wr *chproto.Writer) error {
for _, s := range c.Column {
wr.String(s)
}
return nil
}
//------------------------------------------------------------------------------
type UUID [16]byte
@ -496,10 +182,8 @@ type UUIDColumn struct {
var _ Columnar = (*UUIDColumn)(nil)
func NewUUIDColumn(typ reflect.Type, chType string, numRow int) Columnar {
return &UUIDColumn{
ColumnOf: NewColumnOf[UUID](numRow),
}
func NewUUIDColumn() Columnar {
return new(UUIDColumn)
}
func (c UUIDColumn) Type() reflect.Type {
@ -517,7 +201,7 @@ func (c *UUIDColumn) AppendValue(v reflect.Value) {
}
func (c *UUIDColumn) ReadFrom(rd *chproto.Reader, numRow int) error {
c.Alloc(numRow)
c.AllocForReading(numRow)
for i := range c.Column {
err := rd.UUID(c.Column[i][:])
@ -548,10 +232,8 @@ type IPColumn struct {
var _ Columnar = (*IPColumn)(nil)
func NewIPColumn(typ reflect.Type, chType string, numRow int) Columnar {
return &IPColumn{
ColumnOf: NewColumnOf[net.IP](numRow),
}
func NewIPColumn() Columnar {
return new(IPColumn)
}
func (c IPColumn) Type() reflect.Type {
@ -568,7 +250,7 @@ func (c *IPColumn) AppendValue(v reflect.Value) {
}
func (c *IPColumn) ReadFrom(rd *chproto.Reader, numRow int) error {
c.Alloc(numRow)
c.AllocForReading(numRow)
mem := make([]byte, ipSize*numRow)
var idx int
@ -603,54 +285,6 @@ func (c IPColumn) WriteTo(wr *chproto.Writer) error {
//------------------------------------------------------------------------------
type DateTimeColumn struct {
ColumnOf[time.Time]
}
var _ Columnar = (*DateTimeColumn)(nil)
func NewDateTimeColumn(typ reflect.Type, chType string, numRow int) Columnar {
return &DateTimeColumn{
ColumnOf: NewColumnOf[time.Time](numRow),
}
}
func (c DateTimeColumn) Type() reflect.Type {
return timeType
}
func (c DateTimeColumn) ConvertAssign(idx int, v reflect.Value) error {
v.Set(reflect.ValueOf(c.Column[idx]))
return nil
}
func (c *DateTimeColumn) AppendValue(v reflect.Value) {
c.Column = append(c.Column, v.Interface().(time.Time))
}
func (c *DateTimeColumn) ReadFrom(rd *chproto.Reader, numRow int) error {
c.Alloc(numRow)
for i := range c.Column {
n, err := rd.DateTime()
if err != nil {
return err
}
c.Column[i] = n
}
return nil
}
func (c DateTimeColumn) WriteTo(wr *chproto.Writer) error {
for i := range c.Column {
wr.DateTime(c.Column[i])
}
return nil
}
//------------------------------------------------------------------------------
type DateTime64Column struct {
ColumnOf[time.Time]
prec int
@ -658,11 +292,13 @@ type DateTime64Column struct {
var _ Columnar = (*DateTime64Column)(nil)
func NewDateTime64Column(typ reflect.Type, chType string, numRow int) Columnar {
return &DateTime64Column{
ColumnOf: NewColumnOf[time.Time](numRow),
prec: parseDateTime64Prec(chType),
}
func NewDateTime64Column() Columnar {
return new(DateTime64Column)
}
func (c *DateTime64Column) Init(chType string) error {
c.prec = parseDateTime64Prec(chType)
return nil
}
func (c *DateTime64Column) Type() reflect.Type {
@ -675,7 +311,7 @@ func (c *DateTime64Column) ConvertAssign(idx int, v reflect.Value) error {
}
func (c *DateTime64Column) ReadFrom(rd *chproto.Reader, numRow int) error {
c.Alloc(numRow)
c.AllocForReading(numRow)
mul := int64(math.Pow10(9 - c.prec))
for i := range c.Column {
@ -699,59 +335,18 @@ func (c *DateTime64Column) WriteTo(wr *chproto.Writer) error {
//------------------------------------------------------------------------------
type Int64TimeColumn struct {
Int64Column
}
var _ Columnar = (*Int64TimeColumn)(nil)
func NewInt64TimeColumn(typ reflect.Type, chType string, numRow int) Columnar {
return &Int64TimeColumn{
Int64Column: Int64Column{
NumericColumnOf: NewNumericColumnOf[int64](numRow),
},
}
}
func (c *Int64TimeColumn) ReadFrom(rd *chproto.Reader, numRow int) error {
c.Alloc(numRow)
for i := range c.Column {
n, err := rd.UInt32()
if err != nil {
return err
}
c.Column[i] = int64(n) * int64(time.Second)
}
return nil
}
func (c Int64TimeColumn) WriteTo(wr *chproto.Writer) error {
for i := range c.Column {
wr.UInt32(uint32(c.Column[i] / int64(time.Second)))
}
return nil
}
//------------------------------------------------------------------------------
type DateColumn struct {
DateTimeColumn
}
var _ Columnar = (*DateColumn)(nil)
func NewDateColumn(typ reflect.Type, chType string, numRow int) Columnar {
return &DateColumn{
DateTimeColumn: DateTimeColumn{
ColumnOf: NewColumnOf[time.Time](numRow),
},
}
func NewDateColumn() Columnar {
return new(DateColumn)
}
func (c *DateColumn) ReadFrom(rd *chproto.Reader, numRow int) error {
c.Alloc(numRow)
c.AllocForReading(numRow)
for i := range c.Column {
n, err := rd.Date()
@ -781,16 +376,12 @@ type TimeColumn struct {
var _ Columnar = (*TimeColumn)(nil)
func NewTimeColumn(typ reflect.Type, chType string, numRow int) Columnar {
return &TimeColumn{
DateTimeColumn: DateTimeColumn{
ColumnOf: NewColumnOf[time.Time](numRow),
},
}
func NewTimeColumn() Columnar {
return new(TimeColumn)
}
func (c *TimeColumn) ReadFrom(rd *chproto.Reader, numRow int) error {
c.Alloc(numRow)
c.AllocForReading(numRow)
for i := range c.Column {
n, err := rd.Int64()
@ -812,63 +403,43 @@ func (c TimeColumn) WriteTo(wr *chproto.Writer) error {
//------------------------------------------------------------------------------
type BytesColumn struct {
ColumnOf[[]byte]
type EnumColumn struct {
StringColumn
enum *enumInfo
}
var _ Columnar = (*BytesColumn)(nil)
var _ Columnar = (*EnumColumn)(nil)
func NewBytesColumn(typ reflect.Type, chType string, numRow int) Columnar {
return &BytesColumn{
ColumnOf: NewColumnOf[[]byte](numRow),
}
func NewEnumColumn() Columnar {
return new(EnumColumn)
}
func (c *BytesColumn) Reset(numRow int) {
if cap(c.Column) >= numRow {
for i := range c.Column {
c.Column[i] = nil
}
c.Column = c.Column[:0]
} else {
c.Column = make([][]byte, 0, numRow)
}
}
func (c BytesColumn) Type() reflect.Type {
return bytesType
}
func (c BytesColumn) ConvertAssign(idx int, v reflect.Value) error {
v.SetBytes(c.Column[idx])
func (c *EnumColumn) Init(chType string) error {
c.enum = parseEnum(chType)
return nil
}
func (c *BytesColumn) AppendValue(v reflect.Value) {
c.Column = append(c.Column, v.Bytes())
}
func (c *EnumColumn) ReadFrom(rd *chproto.Reader, numRow int) error {
c.AllocForReading(numRow)
func (c *BytesColumn) ReadFrom(rd *chproto.Reader, numRow int) error {
if cap(c.Column) >= numRow {
c.Column = c.Column[:numRow]
} else {
c.Column = make([][]byte, numRow)
}
for i := 0; i < len(c.Column); i++ {
b, err := rd.Bytes()
for i := range c.Column {
n, err := rd.Int8()
if err != nil {
return err
}
c.Column[i] = b
c.Column[i] = c.enum.Decode(int16(n))
}
return nil
}
func (c BytesColumn) WriteTo(wr *chproto.Writer) error {
for _, b := range c.Column {
wr.Bytes(b)
func (c *EnumColumn) WriteTo(wr *chproto.Writer) error {
for _, s := range c.Column {
n, ok := c.enum.Encode(s)
if !ok {
log.Printf("unknown enum value in %s: %s", c.enum.chType, s)
}
wr.Int8(int8(n))
}
return nil
}
@ -882,13 +453,13 @@ type JSONColumn struct {
var _ Columnar = (*JSONColumn)(nil)
func NewJSONColumn(typ reflect.Type, chType string, numRow int) Columnar {
func NewJSONColumn() Columnar {
return new(JSONColumn)
}
func (c *JSONColumn) Reset(numRow int) {
func (c *JSONColumn) ResetForWriting(numRow int) {
c.Values = c.Values[:0]
c.BytesColumn.Reset(numRow)
c.BytesColumn.ResetForWriting(numRow)
}
func (c *JSONColumn) Len() int {
@ -926,258 +497,14 @@ func (c *JSONColumn) WriteTo(wr *chproto.Writer) error {
//------------------------------------------------------------------------------
type EnumColumn struct {
StringColumn
enum *enumInfo
}
var _ Columnar = (*EnumColumn)(nil)
func NewEnumColumn(typ reflect.Type, chType string, numRow int) Columnar {
return &EnumColumn{
StringColumn: StringColumn{
ColumnOf: NewColumnOf[string](numRow),
},
enum: parseEnum(chType),
}
}
func (c *EnumColumn) ReadFrom(rd *chproto.Reader, numRow int) error {
if cap(c.Column) >= numRow {
c.Column = c.Column[:numRow]
} else {
c.Column = make([]string, numRow)
}
for i := 0; i < len(c.Column); i++ {
n, err := rd.Int8()
if err != nil {
return err
}
c.Column[i] = c.enum.Decode(int16(n))
}
return nil
}
func (c *EnumColumn) WriteTo(wr *chproto.Writer) error {
for _, s := range c.Column {
n, ok := c.enum.Encode(s)
if !ok {
log.Printf("unknown enum value in %s: %s", c.enum.chType, s)
}
wr.Int8(int8(n))
}
return nil
}
//------------------------------------------------------------------------------
type LCStringColumn struct {
StringColumn
}
var _ Columnar = (*LCStringColumn)(nil)
func NewLCStringColumn(typ reflect.Type, chType string, numRow int) Columnar {
col := new(LCStringColumn)
col.Column = make([]string, 0, numRow)
return col
}
func (c *LCStringColumn) ReadFrom(rd *chproto.Reader, numRow int) error {
if numRow == 0 {
return nil
}
if err := c.readPrefix(rd, numRow); err != nil {
return err
}
return c.readData(rd, numRow)
}
func (c *LCStringColumn) readPrefix(rd *chproto.Reader, numRow int) error {
version, err := rd.Int64()
if err != nil {
return err
}
if version != 1 {
return fmt.Errorf("ch: got version=%d, wanted 1", version)
}
return nil
}
func (c *LCStringColumn) readData(rd *chproto.Reader, numRow int) error {
flags, err := rd.Int64()
if err != nil {
return err
}
lcKey := newLCKeyType(flags & 0xf)
dictSize, err := rd.UInt64()
if err != nil {
return err
}
dict := make([]string, dictSize)
for i := range dict {
s, err := rd.String()
if err != nil {
return err
}
dict[i] = s
}
numKey, err := rd.UInt64()
if err != nil {
return err
}
if int(numKey) != numRow {
return fmt.Errorf("%d != %d", numKey, numRow)
}
if cap(c.Column) >= int(numKey) {
c.Column = c.Column[:numKey]
} else {
c.Column = make([]string, numKey)
}
for i := 0; i < int(numKey); i++ {
key, err := lcKey.read(rd)
if err != nil {
return err
}
c.Column[i] = dict[key]
}
return nil
}
func (c *LCStringColumn) WriteTo(wr *chproto.Writer) error {
c.writePrefix(wr)
c.writeData(wr)
return nil
}
func (c *LCStringColumn) writePrefix(wr *chproto.Writer) {
wr.Int64(1)
}
func (c *LCStringColumn) writeData(wr *chproto.Writer) {
if len(c.Column) == 0 {
return
}
keys := make([]int, len(c.Column))
var lc lowCard
for i, s := range c.Column {
keys[i] = lc.Add(s)
}
const hasAdditionalKeys = 1 << 9
const needUpdateDict = 1 << 10
dict := lc.Dict()
lcKey := newLCKey(int64(len(dict)))
wr.Int64(int64(lcKey.typ) | hasAdditionalKeys | needUpdateDict)
wr.Int64(int64(len(dict)))
for _, s := range dict {
wr.String(s)
}
wr.Int64(int64(len(keys)))
for _, key := range keys {
lcKey.write(wr, key)
}
}
//------------------------------------------------------------------------------
type lcKey struct {
typ int8
read func(*chproto.Reader) (int, error)
write func(*chproto.Writer, int)
}
func newLCKey(numKey int64) lcKey {
if numKey <= math.MaxUint8 {
return newLCKeyType(0)
}
if numKey <= math.MaxUint16 {
return newLCKeyType(1)
}
if numKey <= math.MaxUint32 {
return newLCKeyType(2)
}
return newLCKeyType(3)
}
func newLCKeyType(typ int64) lcKey {
switch typ {
case 0:
return lcKey{
typ: 0,
read: func(rd *chproto.Reader) (int, error) {
n, err := rd.UInt8()
return int(n), err
},
write: func(wr *chproto.Writer, n int) {
wr.UInt8(uint8(n))
},
}
case 1:
return lcKey{
typ: int8(1),
read: func(rd *chproto.Reader) (int, error) {
n, err := rd.UInt16()
return int(n), err
},
write: func(wr *chproto.Writer, n int) {
wr.UInt16(uint16(n))
},
}
case 2:
return lcKey{
typ: 2,
read: func(rd *chproto.Reader) (int, error) {
n, err := rd.UInt32()
return int(n), err
},
write: func(wr *chproto.Writer, n int) {
wr.UInt32(uint32(n))
},
}
case 3:
return lcKey{
typ: 3,
read: func(rd *chproto.Reader) (int, error) {
n, err := rd.UInt64()
return int(n), err
},
write: func(wr *chproto.Writer, n int) {
wr.UInt64(uint64(n))
},
}
default:
panic("not reached")
}
}
//------------------------------------------------------------------------------
type BFloat16HistColumn struct {
ColumnOf[bfloat16.Map]
}
var _ Columnar = (*BFloat16HistColumn)(nil)
func NewBFloat16HistColumn(typ reflect.Type, chType string, numRow int) Columnar {
return &BFloat16HistColumn{
ColumnOf: NewColumnOf[bfloat16.Map](numRow),
}
func NewBFloat16HistColumn() Columnar {
return new(BFloat16HistColumn)
}
func (c BFloat16HistColumn) Type() reflect.Type {
@ -1189,7 +516,7 @@ func (c *BFloat16HistColumn) ReadFrom(rd *chproto.Reader, numRow int) error {
return nil
}
c.Alloc(numRow)
c.AllocForReading(numRow)
for i := range c.Column {
n, err := rd.Uvarint()

539
ch/chschema/column_array.go
View File

@ -1,539 +0,0 @@
package chschema
import (
"fmt"
"reflect"
"unsafe"
"github.com/uptrace/go-clickhouse/ch/chproto"
)
type ArrayColumnar interface {
WriteOffset(wr *chproto.Writer, offset int) int
WriteData(wr *chproto.Writer) error
}
//------------------------------------------------------------------------------
type ArrayColumnOf[T any] struct {
Column [][]T
elem Columnar
}
func (c *ArrayColumnOf[T]) Reset(numRow int) {
if cap(c.Column) >= numRow {
c.Column = c.Column[:0]
} else {
c.Column = make([][]T, 0, numRow)
}
}
func (c *ArrayColumnOf[T]) Set(v any) {
c.Column = v.([][]T)
}
func (c *ArrayColumnOf[T]) Value() any {
return c.Column
}
func (c *ArrayColumnOf[T]) Nullable(nulls UInt8Column) any {
panic("not implemented")
}
func (c *ArrayColumnOf[T]) Len() int {
return len(c.Column)
}
func (c *ArrayColumnOf[T]) Index(idx int) any {
return c.Column[idx]
}
func (c *ArrayColumnOf[T]) Slice(s, e int) any {
return c.Column[s:e]
}
func (c *ArrayColumnOf[T]) ConvertAssign(idx int, v reflect.Value) error {
v.Set(reflect.ValueOf(c.Column[idx]))
return nil
}
func (c *ArrayColumnOf[T]) AppendValue(v reflect.Value) {
ptr := unsafe.Pointer(v.UnsafeAddr())
c.AppendPointer(v.Type(), ptr)
}
func (c *ArrayColumnOf[T]) AppendPointer(typ reflect.Type, ptr unsafe.Pointer) {
c.Column = append(c.Column, *(*[]T)(ptr))
}
func (c *ArrayColumnOf[T]) ReadFrom(rd *chproto.Reader, numRow int) error {
if cap(c.Column) >= numRow {
c.Column = c.Column[:numRow]
} else {
c.Column = make([][]T, numRow)
}
if numRow == 0 {
return nil
}
offsets := make([]int, numRow)
for i := 0; i < numRow; i++ {
offset, err := rd.UInt64()
if err != nil {
return err
}
offsets[i] = int(offset)
}
if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil {
return err
}
var prev int
for i, offset := range offsets {
c.Column[i] = c.elem.Slice(prev, offset).([]T)
prev = offset
}
return nil
}
func (c *ArrayColumnOf[T]) WriteTo(wr *chproto.Writer) error {
_ = c.WriteOffset(wr, 0)
return c.WriteData(wr)
}
var _ ArrayColumnar = (*Int64ArrayColumn)(nil)
func (c *ArrayColumnOf[T]) WriteOffset(wr *chproto.Writer, offset int) int {
for _, el := range c.Column {
offset += len(el)
wr.UInt64(uint64(offset))
}
return offset
}
func (c *ArrayColumnOf[T]) WriteData(wr *chproto.Writer) error {
for _, ss := range c.Column {
c.elem.Set(ss)
if err := c.elem.WriteTo(wr); err != nil {
return err
}
}
return nil
}
//------------------------------------------------------------------------------
type Int64ArrayColumn struct {
ArrayColumnOf[int64]
}
var _ Columnar = (*Int64ArrayColumn)(nil)
func NewInt64ArrayColumn(typ reflect.Type, chType string, numRow int) Columnar {
return &Int64ArrayColumn{
ArrayColumnOf: ArrayColumnOf[int64]{
Column: make([][]int64, 0, numRow),
elem: NewInt64Column(typ.Elem(), "", 0),
},
}
}
func (c *Int64ArrayColumn) Type() reflect.Type {
return int64SliceType
}
//------------------------------------------------------------------------------
type Uint64ArrayColumn struct {
ArrayColumnOf[uint64]
}
var _ Columnar = (*Uint64ArrayColumn)(nil)
func NewUint64ArrayColumn(typ reflect.Type, chType string, numRow int) Columnar {
return &Uint64ArrayColumn{
ArrayColumnOf: ArrayColumnOf[uint64]{
Column: make([][]uint64, 0, numRow),
elem: NewUInt64Column(typ.Elem(), "", 0),
},
}
}
func (c *Uint64ArrayColumn) Type() reflect.Type {
return uint64SliceType
}
//------------------------------------------------------------------------------
type Float64ArrayColumn struct {
ArrayColumnOf[float64]
}
var _ Columnar = (*Float64ArrayColumn)(nil)
func NewFloat64ArrayColumn(typ reflect.Type, chType string, numRow int) Columnar {
return &Float64ArrayColumn{
ArrayColumnOf: ArrayColumnOf[float64]{
Column: make([][]float64, 0, numRow),
elem: NewFloat64Column(typ.Elem(), "", 0),
},
}
}
func (c *Float64ArrayColumn) Type() reflect.Type {
return float64SliceType
}
//------------------------------------------------------------------------------
type StringArrayColumn struct {
Column [][]string
elem Columnar
stringElem *StringColumn
lcElem *LCStringColumn
}
var _ Columnar = (*StringArrayColumn)(nil)
func NewStringArrayColumn(typ reflect.Type, chType string, numRow int) Columnar {
if _, funcType := aggFuncNameAndType(chType); funcType != "" {
chType = funcType
}
elemType := chArrayElemType(chType)
if elemType == "" {
panic(fmt.Errorf("invalid array type: %q (Go type is %s)",
chType, typ.String()))
}
columnar := NewColumn(typ.Elem(), elemType, 0)
var stringElem *StringColumn
var lcElem *LCStringColumn
switch v := columnar.(type) {
case *StringColumn:
stringElem = v
case *LCStringColumn:
stringElem = &v.StringColumn
lcElem = v
columnar = &ArrayLCStringColumn{v}
case *EnumColumn:
stringElem = &v.StringColumn
default:
panic(fmt.Errorf("unsupported column: %T", v))
}
return &StringArrayColumn{
Column: make([][]string, 0, numRow),
elem: columnar,
stringElem: stringElem,
lcElem: lcElem,
}
}
func (c *StringArrayColumn) Reset(numRow int) {
if cap(c.Column) >= numRow {
c.Column = c.Column[:0]
} else {
c.Column = make([][]string, 0, numRow)
}
}
func (c *StringArrayColumn) Type() reflect.Type {
return stringSliceType
}
func (c *StringArrayColumn) Set(v any) {
c.Column = v.([][]string)
}
func (c *StringArrayColumn) Value() any {
return c.Column
}
func (c *StringArrayColumn) Nullable(nulls UInt8Column) any {
panic("not implemented")
}
func (c *StringArrayColumn) Len() int {
return len(c.Column)
}
func (c *StringArrayColumn) Index(idx int) any {
return c.Column[idx]
}
func (c StringArrayColumn) Slice(s, e int) any {
return c.Column[s:e]
}
func (c *StringArrayColumn) ConvertAssign(idx int, v reflect.Value) error {
v.Set(reflect.ValueOf(c.Column[idx]))
return nil
}
func (c *StringArrayColumn) AppendValue(v reflect.Value) {
c.Column = append(c.Column, v.Interface().([]string))
}
func (c *StringArrayColumn) AppendPointer(typ reflect.Type, ptr unsafe.Pointer) {
c.Column = append(c.Column, *(*[]string)(ptr))
}
func (c *StringArrayColumn) ReadFrom(rd *chproto.Reader, numRow int) error {
if numRow == 0 {
return nil
}
if cap(c.Column) >= numRow {
c.Column = c.Column[:numRow]
} else {
c.Column = make([][]string, numRow)
}
if c.lcElem != nil {
if err := c.lcElem.readPrefix(rd, numRow); err != nil {
return err
}
}
offsets := make([]int, numRow)
for i := 0; i < len(offsets); i++ {
offset, err := rd.UInt64()
if err != nil {
return err
}
offsets[i] = int(offset)
}
if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil {
return err
}
var prev int
for i, offset := range offsets {
c.Column[i] = c.stringElem.Column[prev:offset]
prev = offset
}
return nil
}
func (c *StringArrayColumn) WriteTo(wr *chproto.Writer) error {
if c.lcElem != nil {
c.lcElem.writePrefix(wr)
}
_ = c.WriteOffset(wr, 0)
return c.WriteData(wr)
}
var _ ArrayColumnar = (*StringArrayColumn)(nil)
func (c *StringArrayColumn) WriteOffset(wr *chproto.Writer, offset int) int {
for _, el := range c.Column {
offset += len(el)
wr.UInt64(uint64(offset))
}
return offset
}
func (c *StringArrayColumn) WriteData(wr *chproto.Writer) error {
for _, ss := range c.Column {
c.stringElem.Column = ss
if err := c.elem.WriteTo(wr); err != nil {
return err
}
}
return nil
}
//------------------------------------------------------------------------------
type ArrayLCStringColumn struct {
*LCStringColumn
}
func (c ArrayLCStringColumn) Type() reflect.Type {
return stringSliceType
}
func (c *ArrayLCStringColumn) WriteTo(wr *chproto.Writer) error {
c.writeData(wr)
return nil
}
func (c *ArrayLCStringColumn) ReadFrom(rd *chproto.Reader, numRow int) error {
if numRow == 0 {
return nil
}
return c.readData(rd, numRow)
}
//------------------------------------------------------------------------------
type GenericArrayColumn struct {
Column reflect.Value
typ reflect.Type
elem Columnar
arrayElem ArrayColumnar
}
var _ Columnar = (*GenericArrayColumn)(nil)
func NewGenericArrayColumn(typ reflect.Type, chType string, numRow int) Columnar {
elemType := chArrayElemType(chType)
if elemType == "" {
panic(fmt.Errorf("invalid array type: %q (Go type is %s)",
chType, typ.String()))
}
elem := NewColumn(typ.Elem(), elemType, 0)
var arrayElem ArrayColumnar
if _, ok := elem.(*LCStringColumn); ok {
panic("not reached")
}
arrayElem, _ = elem.(ArrayColumnar)
c := &GenericArrayColumn{
typ: reflect.SliceOf(typ),
elem: elem,
arrayElem: arrayElem,
}
c.Column = reflect.MakeSlice(c.typ, 0, numRow)
return c
}
func (c GenericArrayColumn) Type() reflect.Type {
return c.typ.Elem()
}
func (c *GenericArrayColumn) Reset(numRow int) {
if c.Column.Cap() >= numRow {
c.Column = c.Column.Slice(0, 0)
} else {
c.Column = reflect.MakeSlice(c.typ, 0, numRow)
}
}
func (c *GenericArrayColumn) Set(v any) {
c.Column = reflect.ValueOf(v)
}
func (c *GenericArrayColumn) Value() any {
return c.Column.Interface()
}
func (c *GenericArrayColumn) Nullable(nulls UInt8Column) any {
panic("not implemented")
}
func (c *GenericArrayColumn) Len() int {
return c.Column.Len()
}
func (c *GenericArrayColumn) Index(idx int) any {
return c.Column.Index(idx).Interface()
}
func (c GenericArrayColumn) Slice(s, e int) any {
return c.Column.Slice(s, e).Interface()
}
func (c *GenericArrayColumn) ConvertAssign(idx int, v reflect.Value) error {
v.Set(c.Column.Index(idx))
return nil
}
func (c *GenericArrayColumn) AppendValue(v reflect.Value) {
c.Column = reflect.Append(c.Column, v)
}
func (c *GenericArrayColumn) AppendPointer(typ reflect.Type, ptr unsafe.Pointer) {
c.AppendValue(reflect.NewAt(typ.Elem(), ptr).Elem())
}
func (c *GenericArrayColumn) ReadFrom(rd *chproto.Reader, numRow int) error {
if c.Column.Cap() >= numRow {
c.Column = c.Column.Slice(0, numRow)
} else {
c.Column = reflect.MakeSlice(c.typ, numRow, numRow)
}
if numRow == 0 {
return nil
}
offsets := make([]int, numRow)
for i := 0; i < len(offsets); i++ {
offset, err := rd.UInt64()
if err != nil {
return err
}
offsets[i] = int(offset)
}
if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil {
return err
}
var prev int
for i, offset := range offsets {
c.Column.Index(i).Set(reflect.ValueOf(c.elem.Slice(prev, offset)))
prev = offset
}
return nil
}
func (c *GenericArrayColumn) WriteTo(wr *chproto.Writer) error {
_ = c.WriteOffset(wr, 0)
colLen := c.Column.Len()
for i := 0; i < colLen; i++ {
// TODO: add SetValue or SetPointer
c.elem.Set(c.Column.Index(i).Interface())
var err error
if c.arrayElem != nil {
err = c.arrayElem.WriteData(wr)
} else {
err = c.elem.WriteTo(wr)
}
if err != nil {
return err
}
}
return nil
}
func (c *GenericArrayColumn) WriteOffset(wr *chproto.Writer, offset int) int {
colLen := c.Column.Len()
for i := 0; i < colLen; i++ {
el := c.Column.Index(i)
offset += el.Len()
wr.UInt64(uint64(offset))
}
if c.arrayElem == nil {
return offset
}
offset = 0
for i := 0; i < colLen; i++ {
el := c.Column.Index(i)
c.elem.Set(el.Interface()) // Use SetValue or SetPointer
offset = c.arrayElem.WriteOffset(wr, offset)
}
return offset
}

3008
ch/chschema/column_gen.go Normal file
View File

File diff suppressed because it is too large Load Diff

319
ch/chschema/column_lowcard.go Normal file
View File

@ -0,0 +1,319 @@
package chschema
import (
"fmt"
"math"
"reflect"
"github.com/uptrace/go-clickhouse/ch/chproto"
)
type LCStringColumn struct {
StringColumn
}
var _ Columnar = (*LCStringColumn)(nil)
func NewLCStringColumn() Columnar {
return new(LCStringColumn)
}
func (c *LCStringColumn) ReadFrom(rd *chproto.Reader, numRow int) error {
if numRow == 0 {
return nil
}
if err := c.readPrefix(rd, numRow); err != nil {
return err
}
return c.readData(rd, numRow)
}
func (c *LCStringColumn) readPrefix(rd *chproto.Reader, numRow int) error {
version, err := rd.Int64()
if err != nil {
return err
}
if version != 1 {
return fmt.Errorf("ch: got version=%d, wanted 1", version)
}
return nil
}
func (c *LCStringColumn) readData(rd *chproto.Reader, numRow int) error {
if numRow == 0 {
return nil
}
flags, err := rd.Int64()
if err != nil {
return err
}
lcKey := newLCKeyType(flags & 0xf)
dictSize, err := rd.UInt64()
if err != nil {
return err
}
dict := make([]string, dictSize)
for i := range dict {
s, err := rd.String()
if err != nil {
return err
}
dict[i] = s
}
numKey, err := rd.UInt64()
if err != nil {
return err
}
if int(numKey) != numRow {
return fmt.Errorf("%d != %d", numKey, numRow)
}
if cap(c.Column) >= int(numKey) {
c.Column = c.Column[:numKey]
} else {
c.Column = make([]string, numKey)
}
for i := 0; i < int(numKey); i++ {
key, err := lcKey.read(rd)
if err != nil {
return err
}
c.Column[i] = dict[key]
}
return nil
}
func (c *LCStringColumn) WriteTo(wr *chproto.Writer) error {
c.writePrefix(wr)
c.writeData(wr)
return nil
}
func (c *LCStringColumn) writePrefix(wr *chproto.Writer) {
wr.Int64(1)
}
func (c *LCStringColumn) writeData(wr *chproto.Writer) {
if len(c.Column) == 0 {
return
}
keys := make([]int, len(c.Column))
var lc lowCard
for i, s := range c.Column {
keys[i] = lc.Add(s)
}
const hasAdditionalKeys = 1 << 9
const needUpdateDict = 1 << 10
dict := lc.Dict()
lcKey := newLCKey(int64(len(dict)))
wr.Int64(int64(lcKey.typ) | hasAdditionalKeys | needUpdateDict)
wr.Int64(int64(len(dict)))
for _, s := range dict {
wr.String(s)
}
wr.Int64(int64(len(keys)))
for _, key := range keys {
lcKey.write(wr, key)
}
}
//------------------------------------------------------------------------------
type ArrayLCStringColumn struct {
ArrayStringColumn
lc LCStringColumn
}
var _ Columnar = (*ArrayLCStringColumn)(nil)
func NewArrayLCStringColumn() Columnar {
return new(ArrayLCStringColumn)
}
func (c *ArrayLCStringColumn) ConvertAssign(idx int, dest reflect.Value) error {
dest.Set(reflect.ValueOf(c.Column[idx]))
return nil
}
func (c *ArrayLCStringColumn) ReadFrom(rd *chproto.Reader, numRow int) error {
if numRow == 0 {
return nil
}
c.AllocForReading(numRow)
if err := c.lc.readPrefix(rd, numRow); err != nil {
return err
}
offsets, err := c.readOffsets(rd, numRow)
if err != nil {
return err
}
if err := c.lc.readData(rd, offsets[len(offsets)-1]); err != nil {
return err
}
var prev int
for i, offset := range offsets {
c.Column[i] = c.lc.Column[prev:offset]
prev = offset
}
return nil
}
func (c *ArrayLCStringColumn) WriteTo(wr *chproto.Writer) error {
c.lc.writePrefix(wr)
_ = c.WriteOffset(wr, 0)
return c.WriteData(wr)
}
func (c *ArrayLCStringColumn) WriteData(wr *chproto.Writer) error {
for _, ss := range c.Column {
c.lc.Column = ss
c.lc.writeData(wr)
}
return nil
}
//------------------------------------------------------------------------------
type lcKey struct {
typ int8
read func(*chproto.Reader) (int, error)
write func(*chproto.Writer, int)
}
func newLCKey(numKey int64) lcKey {
if numKey <= math.MaxUint8 {
return newLCKeyType(0)
}
if numKey <= math.MaxUint16 {
return newLCKeyType(1)
}
if numKey <= math.MaxUint32 {
return newLCKeyType(2)
}
return newLCKeyType(3)
}
func newLCKeyType(typ int64) lcKey {
switch typ {
case 0:
return lcKey{
typ: 0,
read: func(rd *chproto.Reader) (int, error) {
n, err := rd.UInt8()
return int(n), err
},
write: func(wr *chproto.Writer, n int) {
wr.UInt8(uint8(n))
},
}
case 1:
return lcKey{
typ: int8(1),
read: func(rd *chproto.Reader) (int, error) {
n, err := rd.UInt16()
return int(n), err
},
write: func(wr *chproto.Writer, n int) {
wr.UInt16(uint16(n))
},
}
case 2:
return lcKey{
typ: 2,
read: func(rd *chproto.Reader) (int, error) {
n, err := rd.UInt32()
return int(n), err
},
write: func(wr *chproto.Writer, n int) {
wr.UInt32(uint32(n))
},
}
case 3:
return lcKey{
typ: 3,
read: func(rd *chproto.Reader) (int, error) {
n, err := rd.UInt64()
return int(n), err
},
write: func(wr *chproto.Writer, n int) {
wr.UInt64(uint64(n))
},
}
default:
panic("not reached")
}
}
//------------------------------------------------------------------------------
type lowCard struct {
slice sliceMap
dict map[string]int
}
func (lc *lowCard) Add(word string) int {
if i, ok := lc.dict[word]; ok {
return i
}
if lc.dict == nil {
lc.dict = make(map[string]int)
}
i := lc.slice.Add(word)
lc.dict[word] = i
return i
}
func (lc *lowCard) Dict() []string {
return lc.slice.Slice()
}
//------------------------------------------------------------------------------
type sliceMap struct {
ss []string
}
func (m sliceMap) Len() int {
return len(m.ss)
}
func (m sliceMap) Get(word string) (int, bool) {
for i, s := range m.ss {
if s == word {
return i, true
}
}
return 0, false
}
func (m *sliceMap) Add(word string) int {
m.ss = append(m.ss, word)
return len(m.ss) - 1
}
func (m sliceMap) Slice() []string {
return m.ss
}

20
ch/chschema/column_nullable.go
View File

@ -12,16 +12,30 @@ type NullableColumn struct {
nullable reflect.Value // reflect.Slice
}
func NullableNewColumnFunc(fn NewColumnFunc) NewColumnFunc {
return func(typ reflect.Type, chType string, numRow int) Columnar {
func NewNullableColumnFunc(fn NewColumnFunc) NewColumnFunc {
return func() Columnar {
return &NullableColumn{
Values: fn(typ, chType, numRow),
Values: fn(),
}
}
}
var _ Columnar = (*NullableColumn)(nil)
func (c *NullableColumn) Init(chType string) error {
return nil
}
func (c *NullableColumn) AllocForReading(numRow int) {
c.Nulls.AllocForReading(numRow)
c.Values.AllocForReading(numRow)
}
func (c *NullableColumn) ResetForWriting(numRow int) {
c.Nulls.ResetForWriting(numRow)
c.Values.ResetForWriting(numRow)
}
func (c *NullableColumn) Type() reflect.Type {
return reflect.PtrTo(c.Values.Type())
}

20
ch/chschema/column_safe_gen.go
View File

@ -7,7 +7,7 @@ import (
)
func (c *Int8Column) ReadFrom(rd *chproto.Reader, numRow int) error {
c.Alloc(numRow)
c.AllocForReading(numRow)
for i := range c.Column {
n, err := rd.Int8()
@ -28,7 +28,7 @@ func (c *Int8Column) WriteTo(wr *chproto.Writer) error {
}
func (c *UInt8Column) ReadFrom(rd *chproto.Reader, numRow int) error {
c.Alloc(numRow)
c.AllocForReading(numRow)
for i := range c.Column {
n, err := rd.UInt8()
@ -49,7 +49,7 @@ func (c *UInt8Column) WriteTo(wr *chproto.Writer) error {
}
func (c *Int16Column) ReadFrom(rd *chproto.Reader, numRow int) error {
c.Alloc(numRow)
c.AllocForReading(numRow)
for i := range c.Column {
n, err := rd.Int16()
@ -70,7 +70,7 @@ func (c *Int16Column) WriteTo(wr *chproto.Writer) error {
}
func (c *UInt16Column) ReadFrom(rd *chproto.Reader, numRow int) error {
c.Alloc(numRow)
c.AllocForReading(numRow)
for i := range c.Column {
n, err := rd.UInt16()
@ -91,7 +91,7 @@ func (c *UInt16Column) WriteTo(wr *chproto.Writer) error {
}
func (c *Int32Column) ReadFrom(rd *chproto.Reader, numRow int) error {
c.Alloc(numRow)
c.AllocForReading(numRow)
for i := range c.Column {
n, err := rd.Int32()
@ -112,7 +112,7 @@ func (c *Int32Column) WriteTo(wr *chproto.Writer) error {
}
func (c *UInt32Column) ReadFrom(rd *chproto.Reader, numRow int) error {
c.Alloc(numRow)
c.AllocForReading(numRow)
for i := range c.Column {
n, err := rd.UInt32()
@ -133,7 +133,7 @@ func (c *UInt32Column) WriteTo(wr *chproto.Writer) error {
}
func (c *Int64Column) ReadFrom(rd *chproto.Reader, numRow int) error {
c.Alloc(numRow)
c.AllocForReading(numRow)
for i := range c.Column {
n, err := rd.Int64()
@ -154,7 +154,7 @@ func (c *Int64Column) WriteTo(wr *chproto.Writer) error {
}
func (c *UInt64Column) ReadFrom(rd *chproto.Reader, numRow int) error {
c.Alloc(numRow)
c.AllocForReading(numRow)
for i := range c.Column {
n, err := rd.UInt64()
@ -175,7 +175,7 @@ func (c *UInt64Column) WriteTo(wr *chproto.Writer) error {
}
func (c *Float32Column) ReadFrom(rd *chproto.Reader, numRow int) error {
c.Alloc(numRow)
c.AllocForReading(numRow)
for i := range c.Column {
n, err := rd.Float32()
@ -196,7 +196,7 @@ func (c *Float32Column) WriteTo(wr *chproto.Writer) error {
}
func (c *Float64Column) ReadFrom(rd *chproto.Reader, numRow int) error {
c.Alloc(numRow)
c.AllocForReading(numRow)
for i := range c.Column {
n, err := rd.Float64()

20
ch/chschema/column_unsafe_gen.go
View File

@ -17,7 +17,7 @@ func (c *Int8Column) ReadFrom(rd *chproto.Reader, numRow int) error {
return nil
}
c.Alloc(numRow)
c.AllocForReading(numRow)
slice := *(*reflect.SliceHeader)(unsafe.Pointer(&c.Column))
slice.Len *= size
@ -51,7 +51,7 @@ func (c *UInt8Column) ReadFrom(rd *chproto.Reader, numRow int) error {
return nil
}
c.Alloc(numRow)
c.AllocForReading(numRow)
slice := *(*reflect.SliceHeader)(unsafe.Pointer(&c.Column))
slice.Len *= size
@ -85,7 +85,7 @@ func (c *Int16Column) ReadFrom(rd *chproto.Reader, numRow int) error {
return nil
}
c.Alloc(numRow)
c.AllocForReading(numRow)
slice := *(*reflect.SliceHeader)(unsafe.Pointer(&c.Column))
slice.Len *= size
@ -119,7 +119,7 @@ func (c *UInt16Column) ReadFrom(rd *chproto.Reader, numRow int) error {
return nil
}
c.Alloc(numRow)
c.AllocForReading(numRow)
slice := *(*reflect.SliceHeader)(unsafe.Pointer(&c.Column))
slice.Len *= size
@ -153,7 +153,7 @@ func (c *Int32Column) ReadFrom(rd *chproto.Reader, numRow int) error {
return nil
}
c.Alloc(numRow)
c.AllocForReading(numRow)
slice := *(*reflect.SliceHeader)(unsafe.Pointer(&c.Column))
slice.Len *= size
@ -187,7 +187,7 @@ func (c *UInt32Column) ReadFrom(rd *chproto.Reader, numRow int) error {
return nil
}
c.Alloc(numRow)
c.AllocForReading(numRow)
slice := *(*reflect.SliceHeader)(unsafe.Pointer(&c.Column))
slice.Len *= size
@ -221,7 +221,7 @@ func (c *Int64Column) ReadFrom(rd *chproto.Reader, numRow int) error {
return nil
}
c.Alloc(numRow)
c.AllocForReading(numRow)
slice := *(*reflect.SliceHeader)(unsafe.Pointer(&c.Column))
slice.Len *= size
@ -255,7 +255,7 @@ func (c *UInt64Column) ReadFrom(rd *chproto.Reader, numRow int) error {
return nil
}
c.Alloc(numRow)
c.AllocForReading(numRow)
slice := *(*reflect.SliceHeader)(unsafe.Pointer(&c.Column))
slice.Len *= size
@ -289,7 +289,7 @@ func (c *Float32Column) ReadFrom(rd *chproto.Reader, numRow int) error {
return nil
}
c.Alloc(numRow)
c.AllocForReading(numRow)
slice := *(*reflect.SliceHeader)(unsafe.Pointer(&c.Column))
slice.Len *= size
@ -323,7 +323,7 @@ func (c *Float64Column) ReadFrom(rd *chproto.Reader, numRow int) error {
return nil
}
c.Alloc(numRow)
c.AllocForReading(numRow)
slice := *(*reflect.SliceHeader)(unsafe.Pointer(&c.Column))
slice.Len *= size

2
ch/chschema/enum.go
View File

@ -40,7 +40,7 @@ func parseEnum(s string) *enumInfo {
}
func _parseEnum(chType string) (*enumInfo, error) {
s := enumType(chType)
s := chEnumType(chType)
if s == "" {
return nil, fmt.Errorf("can't parse enum type: %q", chType)
}

53
ch/chschema/lowcard.go
View File

@ -1,53 +0,0 @@
package chschema
type lowCard struct {
slice sliceMap
dict map[string]int
}
func (lc *lowCard) Add(word string) int {
if i, ok := lc.dict[word]; ok {
return i
}
if lc.dict == nil {
lc.dict = make(map[string]int)
}
i := lc.slice.Add(word)
lc.dict[word] = i
return i
}
func (lc *lowCard) Dict() []string {
return lc.slice.Slice()
}
//------------------------------------------------------------------------------
type sliceMap struct {
ss []string
}
func (m sliceMap) Len() int {
return len(m.ss)
}
func (m sliceMap) Get(word string) (int, bool) {
for i, s := range m.ss {
if s == word {
return i, true
}
}
return 0, false
}
func (m *sliceMap) Add(word string) int {
m.ss = append(m.ss, word)
return len(m.ss) - 1
}
func (m sliceMap) Slice() []string {
return m.ss
}

18
ch/chschema/table.go
View File

@ -154,7 +154,7 @@ func (t *Table) addFields(typ reflect.Type, baseIndex []int) {
}
}
if f.NewColumn == nil {
f.NewColumn = ColumnFactory(f.Type, f.CHType)
f.NewColumn = ColumnFactory(f.CHType, f.Type)
}
}
}
@ -211,7 +211,7 @@ func (t *Table) newField(f reflect.StructField, index []int, tag tagparser.Tag)
field.CHType = s
field.setFlag(customTypeFlag)
} else {
field.CHType = clickhouseType(f.Type)
field.CHType = chType(f.Type)
}
if tag.HasOption("lc") {
@ -251,7 +251,7 @@ func (t *Table) addField(field *Field) {
t.FieldMap[field.CHName] = field
}
func (t *Table) NewColumn(colName, colType string, numRow int) *Column {
func (t *Table) NewColumn(colName, colType string) *Column {
field, ok := t.FieldMap[colName]
if !ok {
internal.Logger.Printf("ch: %s has no column=%q", t, colName)
@ -259,22 +259,20 @@ func (t *Table) NewColumn(colName, colType string, numRow int) *Column {
}
if colType != field.CHType {
if field.CHType != chtype.Any && false {
internal.Logger.Printf("got column type %q, but %s.%s has type %q",
colType, t.Type.Name(), field.GoName, field.CHType)
}
return &Column{
Name: colName,
Type: colType,
Columnar: ColumnFactory(field.Type, colType)(field.Type, colType, numRow),
Columnar: NewColumn(colType, field.Type),
}
}
col := field.NewColumn()
col.Init(field.CHType)
return &Column{
Name: colName,
Type: field.CHType,
Columnar: field.NewColumn(field.Type, field.CHType, numRow),
Columnar: col,
}
}

395
ch/chschema/types.go
View File

@ -13,7 +13,32 @@ import (
"github.com/uptrace/go-clickhouse/ch/internal"
)
var chType = [...]string{
var (
boolType = reflect.TypeOf(false)
int8Type = reflect.TypeOf(int8(0))
int16Type = reflect.TypeOf(int16(0))
int32Type = reflect.TypeOf(int32(0))
int64Type = reflect.TypeOf(int64(0))
uint8Type = reflect.TypeOf(uint8(0))
uint16Type = reflect.TypeOf(uint16(0))
uint32Type = reflect.TypeOf(uint32(0))
uint64Type = reflect.TypeOf(uint64(0))
float32Type = reflect.TypeOf(float32(0))
float64Type = reflect.TypeOf(float64(0))
stringType = reflect.TypeOf("")
bytesType = reflect.TypeOf((*[]byte)(nil)).Elem()
uuidType = reflect.TypeOf((*UUID)(nil)).Elem()
timeType = reflect.TypeOf((*time.Time)(nil)).Elem()
ipType = reflect.TypeOf((*net.IP)(nil)).Elem()
ipNetType = reflect.TypeOf((*net.IPNet)(nil)).Elem()
bfloat16MapType = reflect.TypeOf((*bfloat16.Map)(nil)).Elem()
sliceUint64Type = reflect.TypeOf((*[]uint64)(nil)).Elem()
sliceFloat32Type = reflect.TypeOf((*[]float32)(nil)).Elem()
)
var chTypes = [...]string{
reflect.Bool: chtype.UInt8,
reflect.Int: chtype.Int64,
reflect.Int8: chtype.Int8,
@ -42,152 +67,16 @@ var chType = [...]string{
reflect.UnsafePointer: "",
}
type NewColumnFunc func(typ reflect.Type, chType string, numRow int) Columnar
type NewColumnFunc func() Columnar
var kindToColumn = [...]NewColumnFunc{
reflect.Bool: NewBoolColumn,
reflect.Int: NewInt64Column,
reflect.Int8: NewInt8Column,
reflect.Int16: NewInt16Column,
reflect.Int32: NewInt32Column,
reflect.Int64: NewInt64Column,
reflect.Uint: NewUInt64Column,
reflect.Uint8: NewUInt8Column,
reflect.Uint16: NewUInt16Column,
reflect.Uint32: NewUInt32Column,
reflect.Uint64: NewUInt64Column,
reflect.Uintptr: nil,
reflect.Float32: NewFloat32Column,
reflect.Float64: NewFloat64Column,
reflect.Complex64: nil,
reflect.Complex128: nil,
reflect.Array: nil,
reflect.Chan: nil,
reflect.Func: nil,
reflect.Interface: nil,
reflect.Map: NewJSONColumn,
reflect.Ptr: nil,
reflect.Slice: nil,
reflect.String: NewStringColumn,
reflect.Struct: NewJSONColumn,
reflect.UnsafePointer: nil,
func NewColumn(chType string, typ reflect.Type) Columnar {
col := ColumnFactory(chType, typ)()
col.Init(chType)
return col
}
// keep in sync with clickhouseType
func ColumnFactory(typ reflect.Type, chType string) NewColumnFunc {
if chType == chtype.Any {
return nil
}
if s := lowCardinalityType(chType); s != "" {
switch s {
case chtype.String:
return NewLCStringColumn
}
panic(fmt.Errorf("got %s, wanted LowCardinality(String)", chType))
}
if s := enumType(chType); s != "" {
return NewEnumColumn
}
if isDateTime64Type(chType) {
return NewDateTime64Column
}
if strings.HasPrefix(chType, "SimpleAggregateFunction(") {
chType = chSubType(chType, "SimpleAggregateFunction(")
} else if s := dateTimeType(chType); s != "" {
chType = s
} else if funcName, _ := aggFuncNameAndType(chType); funcName != "" {
switch funcName {
case "quantileBFloat16", "quantilesBFloat16":
return NewBFloat16HistColumn
default:
panic(fmt.Errorf("unsupported ClickHouse type: %s", chType))
}
}
switch typ {
case timeType:
switch chType {
case chtype.DateTime:
return NewDateTimeColumn
case chtype.Date:
return NewDateColumn
case chtype.Int64:
return NewTimeColumn
}
case ipType:
return NewIPColumn
}
kind := typ.Kind()
switch kind {
case reflect.Ptr:
if typ.Elem().Kind() == reflect.Struct {
return NewJSONColumn
}
return NullableNewColumnFunc(ColumnFactory(typ.Elem(), nullableType(chType)))
case reflect.Slice:
switch elem := typ.Elem(); elem.Kind() {
case reflect.Ptr:
if elem.Elem().Kind() == reflect.Struct {
return NewJSONColumn
}
case reflect.Int64:
return NewInt64ArrayColumn
case reflect.Uint64:
return NewUint64ArrayColumn
case reflect.Float64:
return NewFloat64ArrayColumn
case reflect.Uint8:
if chType == chtype.String {
return NewBytesColumn
}
case reflect.String:
return NewStringArrayColumn
case reflect.Struct:
if elem != timeType {
return NewJSONColumn
}
}
return NewGenericArrayColumn
case reflect.Array:
if isUUID(typ) {
return NewUUIDColumn
}
case reflect.Interface:
return columnFromCHType(chType)
}
func ColumnFactory(chType string, typ reflect.Type) NewColumnFunc {
switch chType {
case chtype.DateTime:
switch typ {
case uint32Type:
return NewUInt32Column
case int64Type:
return NewInt64TimeColumn
default:
return NewDateTimeColumn
}
}
fn := kindToColumn[kind]
if fn != nil {
return fn
}
panic(fmt.Errorf("unsupported go_type=%q ch_type=%q", typ.String(), chType))
}
func columnFromCHType(chType string) NewColumnFunc {
switch chType {
case chtype.String:
return NewStringColumn
case chtype.UUID:
return NewUUIDColumn
case chtype.Int8:
return NewInt8Column
case chtype.Int16:
@ -208,108 +97,125 @@ func columnFromCHType(chType string) NewColumnFunc {
return NewFloat32Column
case chtype.Float64:
return NewFloat64Column
case chtype.String:
if typ == bytesType {
return NewBytesColumn
}
return NewStringColumn
case "LowCardinality(String)":
return NewLCStringColumn
case chtype.Bool:
return NewBoolColumn
case chtype.UUID:
return NewUUIDColumn
case chtype.IPv6:
return NewIPColumn
case chtype.DateTime:
return NewDateTimeColumn
case chtype.DateTime64:
return NewDateTime64Column
case chtype.Date:
return NewDateColumn
case chtype.IPv6:
return NewIPColumn
default:
case "Array(Int8)":
return NewArrayInt8Column
case "Array(UInt8)":
return NewArrayUInt8Column
case "Array(Int16)":
return NewArrayInt16Column
case "Array(UInt16)":
return NewArrayUInt16Column
case "Array(Int32)":
return NewArrayInt32Column
case "Array(UInt32)":
return NewArrayUInt32Column
case "Array(Int64)":
return NewArrayInt64Column
case "Array(UInt64)":
return NewArrayUInt64Column
case "Array(Float32)":
return NewArrayFloat32Column
case "Array(Float64)":
return NewArrayFloat64Column
case "Array(String)":
return NewArrayStringColumn
case "Array(LowCardinality(String))":
return NewArrayLCStringColumn
case "Array(DateTime)":
return NewArrayDateTimeColumn
case "Array(Array(String))":
return NewArrayArrayStringColumn
case chtype.Any:
return nil
}
}
var (
boolType = reflect.TypeOf(false)
int8Type = reflect.TypeOf(int8(0))
int16Type = reflect.TypeOf(int16(0))
int32Type = reflect.TypeOf(int32(0))
int64Type = reflect.TypeOf(int64(0))
uint8Type = reflect.TypeOf(uint8(0))
uint16Type = reflect.TypeOf(uint16(0))
uint32Type = reflect.TypeOf(uint32(0))
uint64Type = reflect.TypeOf(uint64(0))
float32Type = reflect.TypeOf(float32(0))
float64Type = reflect.TypeOf(float64(0))
stringType = reflect.TypeOf("")
bytesType = reflect.TypeOf((*[]byte)(nil)).Elem()
uuidType = reflect.TypeOf((*UUID)(nil)).Elem()
timeType = reflect.TypeOf((*time.Time)(nil)).Elem()
ipType = reflect.TypeOf((*net.IP)(nil)).Elem()
ipNetType = reflect.TypeOf((*net.IPNet)(nil)).Elem()
bfloat16MapType = reflect.TypeOf((*bfloat16.Map)(nil)).Elem()
int64SliceType = reflect.TypeOf((*[]int64)(nil)).Elem()
uint64SliceType = reflect.TypeOf((*[]uint64)(nil)).Elem()
float32SliceType = reflect.TypeOf((*[]float32)(nil)).Elem()
float64SliceType = reflect.TypeOf((*[]float64)(nil)).Elem()
stringSliceType = reflect.TypeOf((*[]string)(nil)).Elem()
)
func goType(chType string) reflect.Type {
switch chType {
case chtype.Int8:
return int8Type
case chtype.Int32:
return int32Type
case chtype.Int64:
return int64Type
case chtype.UInt8:
return uint8Type
case chtype.UInt16:
return uint16Type
case chtype.UInt32:
return uint32Type
case chtype.UInt64:
return uint64Type
case chtype.Float32:
return float32Type
case chtype.Float64:
return float64Type
case chtype.String:
return stringType
case chtype.UUID:
return uuidType
case chtype.DateTime:
return timeType
case chtype.Date:
return timeType
case chtype.IPv6:
return ipType
default:
if chType := chEnumType(chType); chType != "" {
return NewEnumColumn
}
if s := chArrayElemType(chType); s != "" {
return reflect.SliceOf(goType(s))
}
if s := lowCardinalityType(chType); s != "" {
return goType(s)
}
if s := enumType(chType); s != "" {
return stringType
}
if s := dateTimeType(chType); s != "" {
return timeType
if chType := chArrayElemType(chType); chType != "" {
if chType := chEnumType(chType); chType != "" {
return NewArrayEnumColumn
}
}
if isDateTime64Type(chType) {
return timeType
return NewDateTime64Column
}
if s := nullableType(chType); s != "" {
return reflect.PtrTo(goType(s))
if chType := chDateTimeType(chType); chType != "" {
return ColumnFactory(chType, typ)
}
if _, funcType := aggFuncNameAndType(chType); funcType != "" {
return goType(funcType)
if chType := chNullableType(chType); chType != "" {
if typ != nil {
typ = typ.Elem()
}
return NewNullableColumnFunc(ColumnFactory(chType, typ))
}
panic(fmt.Errorf("unsupported ClickHouse type=%q", chType))
if chType := chSimpleAggFunc(chType); chType != "" {
return ColumnFactory(chType, typ)
}
if funcName, _ := aggFuncNameAndType(chType); funcName != "" {
switch funcName {
case "quantileBFloat16", "quantilesBFloat16":
return NewBFloat16HistColumn
default:
panic(fmt.Errorf("unsupported ClickHouse type: %s", chType))
}
}
if typ == nil {
panic(fmt.Errorf("unsupported ClickHouse column: %s", chType))
}
kind := typ.Kind()
switch kind {
case reflect.Ptr:
if typ.Elem().Kind() == reflect.Struct {
return NewJSONColumn
}
return NewNullableColumnFunc(ColumnFactory(chNullableType(chType), typ.Elem()))
case reflect.Slice:
switch elem := typ.Elem(); elem.Kind() {
case reflect.Ptr:
if elem.Elem().Kind() == reflect.Struct {
return NewJSONColumn
}
case reflect.Struct:
if elem != timeType {
return NewJSONColumn
}
}
}
panic(fmt.Errorf("unsupported ClickHouse column: %s", chType))
}
// clickhouseType returns ClickHouse type for the given Go type.
// Keep in sync with ColumnFactory.
func clickhouseType(typ reflect.Type) string {
func chType(typ reflect.Type) string {
switch typ {
case timeType:
return chtype.DateTime
@ -323,7 +229,7 @@ func clickhouseType(typ reflect.Type) string {
if typ.Elem().Kind() == reflect.Struct {
return chtype.String
}
return fmt.Sprintf("Nullable(%s)", clickhouseType(typ.Elem()))
return fmt.Sprintf("Nullable(%s)", chType(typ.Elem()))
case reflect.Slice:
switch elem := typ.Elem(); elem.Kind() {
case reflect.Ptr:
@ -338,14 +244,14 @@ func clickhouseType(typ reflect.Type) string {
return chtype.String // []byte
}
return "Array(" + clickhouseType(typ.Elem()) + ")"
return "Array(" + chType(typ.Elem()) + ")"
case reflect.Array:
if isUUID(typ) {
return chtype.UUID
}
}
if s := chType[kind]; s != "" {
if s := chTypes[kind]; s != "" {
return s
}
@ -353,6 +259,13 @@ func clickhouseType(typ reflect.Type) string {
}
func chArrayElemType(s string) string {
if s := chSubType(s, "SimpleAggregateFunction("); s != "" {
if i := strings.Index(s, ", "); i >= 0 {
s = s[i+2:]
}
return chSubType(s, "Array(")
}
s = chSubType(s, "Array(")
if s == "" {
return ""
@ -371,15 +284,23 @@ func chArrayElemType(s string) string {
return s
}
func lowCardinalityType(s string) string {
return chSubType(s, "LowCardinality(")
}
func enumType(s string) string {
func chEnumType(s string) string {
return chSubType(s, "Enum8(")
}
func dateTimeType(s string) string {
func chSimpleAggFunc(s string) string {
s = chSubType(s, "SimpleAggregateFunction(")
if s == "" {
return ""
}
i := strings.Index(s, ", ")
if i == -1 {
return ""
}
return s[i+2:]
}
func chDateTimeType(s string) string {
s = chSubType(s, "DateTime(")
if s == "" {
return ""
@ -406,7 +327,7 @@ func parseDateTime64Prec(s string) int {
return prec
}
func nullableType(s string) string {
func chNullableType(s string) string {
return chSubType(s, "Nullable(")
}

1
ch/chtype/chtype.go
View File

@ -2,6 +2,7 @@ package chtype
const (
Any = "_" // for decoding into interface{}
Bool = "Bool"
String = "String"
UUID = "UUID"
Int8 = "Int8"

48
ch/db_test.go
View File

@ -314,30 +314,6 @@ func TestInvalidType(t *testing.T) {
require.Equal(t, []float64{}, dest.Numbers)
}
type Event struct {
ch.CHModel `ch:"goch_events,partition:toYYYYMM(created_at)"`
ID uint64
Name string `ch:",lc"`
Count uint32
Keys []string `ch:",lc"`
Values [][]string
Kind string `ch:"type:Enum8('invalid' = 0, 'hello' = 1, 'world' = 2)"`
CreatedAt time.Time `ch:",pk"`
}
type EventColumnar struct {
ch.CHModel `ch:"goch_events,columnar"`
ID []uint64
Name []string `ch:",lc"`
Count []uint32
Keys [][]string `ch:"type:Array(LowCardinality(String))"`
Values [][][]string
Kind []string `ch:"type:Enum8('invalid' = 0, 'hello' = 1, 'world' = 2)"`
CreatedAt []time.Time
}
func TestClickhouse(t *testing.T) {
ctx := context.Background()
@ -376,6 +352,30 @@ func testWhereBytes(ctx context.Context, t *testing.T, db *ch.DB) {
require.Equal(t, data.Bytes, got.Bytes)
}
type Event struct {
ch.CHModel `ch:"goch_events,partition:toYYYYMM(created_at)"`
ID uint64
Name string `ch:",lc"`
Count uint32
Keys []string `ch:",lc"`
Values [][]string
Kind string `ch:"type:Enum8('invalid' = 0, 'hello' = 1, 'world' = 2)"`
CreatedAt time.Time `ch:",pk"`
}
type EventColumnar struct {
ch.CHModel `ch:"goch_events,columnar"`
ID []uint64
Name []string `ch:",lc"`
Count []uint32
Keys [][]string `ch:"type:Array(LowCardinality(String))"`
Values [][][]string
Kind []string `ch:"type:Enum8('invalid' = 0, 'hello' = 1, 'world' = 2)"`
CreatedAt []time.Time
}
func TestORM(t *testing.T) {
ctx := context.Background()

238
ch/internal/codegen/column.tpl Normal file
View File

@ -0,0 +1,238 @@
package chschema
import (
"time"
"reflect"
"github.com/uptrace/go-clickhouse/ch/chproto"
)
{{- range . }}
{{ if not .IsCustom }}
type {{ .Name }}Column struct {
{{ if gt .Size 0 }}Numeric{{ end }}ColumnOf[{{ .GoType }}]
}
var _ Columnar = (*{{ .Name }}Column)(nil)
func New{{ .Name }}Column() Columnar {
return new({{ .Name }}Column)
}
var _{{ .Name }}Type = reflect.TypeOf((*{{ .GoType }})(nil)).Elem()
func (c *{{ .Name }}Column) Type() reflect.Type {
return _{{ .Name }}Type
}
{{ if .GoReflect }}
func (c *{{ .Name }}Column) AppendValue(v reflect.Value) {
c.Column = append(c.Column, {{ .GoType }}(v.{{ .GoReflect }}()))
}
{{ end }}
{{ if eq .Size 0 }}
func (c *{{ .Name }}Column) ReadFrom(rd *chproto.Reader, numRow int) error {
c.AllocForReading(numRow)
for i := range c.Column {
n, err := rd.{{ .Name }}()
if err != nil {
return err
}
c.Column[i] = n
}
return nil
}
func (c *{{ .Name }}Column) WriteTo(wr *chproto.Writer) error {
for _, n := range c.Column {
wr.{{ .Name }}(n)
}
return nil
}
{{ end }}
{{ end }}
//------------------------------------------------------------------------------
type Array{{ .Name }}Column struct {
ColumnOf[[]{{ .GoType }}]
elem {{ .Name }}Column
}
var (
_ Columnar = (*Array{{ .Name }}Column)(nil)
_ ArrayColumnar = (*Array{{ .Name }}Column)(nil)
)
func NewArray{{ .Name }}Column() Columnar {
return new(Array{{ .Name }}Column)
}
func (c *Array{{ .Name }}Column) Init(chType string) error {
return c.elem.Init(chArrayElemType(chType))
}
func (c *Array{{ .Name }}Column) Type() reflect.Type {
return reflect.TypeOf((*[]{{ .GoType }})(nil)).Elem()
}
func (c *Array{{ .Name }}Column) ReadFrom(rd *chproto.Reader, numRow int) error {
if numRow == 0 {
return nil
}
c.AllocForReading(numRow)
offsets, err := c.readOffsets(rd, numRow)
if err != nil {
return err
}
if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil {
return err
}
var prev int
for i, offset := range offsets {
c.Column[i] = c.elem.Column[prev:offset]
prev = offset
}
return nil
}
func (c *Array{{ .Name }}Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) {
offsets := make([]int, numRow)
for i := range offsets {
offset, err := rd.UInt64()
if err != nil {
return nil, err
}
offsets[i] = int(offset)
}
return offsets, nil
}
func (c *Array{{ .Name }}Column) WriteTo(wr *chproto.Writer) error {
_ = c.WriteOffset(wr, 0)
return c.WriteData(wr)
}
func (c *Array{{ .Name }}Column) WriteOffset(wr *chproto.Writer, offset int) int {
for _, el := range c.Column {
offset += len(el)
wr.UInt64(uint64(offset))
}
return offset
}
func (c *Array{{ .Name }}Column) WriteData(wr *chproto.Writer) error {
for _, ss := range c.Column {
c.elem.Column = ss
if err := c.elem.WriteTo(wr); err != nil {
return err
}
}
return nil
}
//------------------------------------------------------------------------------
type ArrayArray{{ .Name }}Column struct {
ColumnOf[[][]{{ .GoType }}]
elem Array{{ .Name }}Column
}
var (
_ Columnar = (*ArrayArray{{ .Name }}Column)(nil)
_ ArrayColumnar = (*ArrayArray{{ .Name }}Column)(nil)
)
func NewArrayArray{{ .Name }}Column() Columnar {
return new(ArrayArray{{ .Name }}Column)
}
func (c *ArrayArray{{ .Name }}Column) Init(chType string) error {
return c.elem.Init(chArrayElemType(chArrayElemType(chType)))
}
func (c *ArrayArray{{ .Name }}Column) Type() reflect.Type {
return reflect.TypeOf((*[][]{{ .GoType }})(nil)).Elem()
}
func (c *ArrayArray{{ .Name }}Column) ReadFrom(rd *chproto.Reader, numRow int) error {
if numRow == 0 {
return nil
}
c.AllocForReading(numRow)
offsets, err := c.readOffsets(rd, numRow)
if err != nil {
return err
}
if err := c.elem.ReadFrom(rd, offsets[len(offsets)-1]); err != nil {
return err
}
var prev int
for i, offset := range offsets {
c.Column[i] = c.elem.Column[prev:offset]
prev = offset
}
return nil
}
func (c *ArrayArray{{ .Name }}Column) readOffsets(rd *chproto.Reader, numRow int) ([]int, error) {
offsets := make([]int, numRow)
for i := range offsets {
offset, err := rd.UInt64()
if err != nil {
return nil, err
}
offsets[i] = int(offset)
}
return offsets, nil
}
func (c *ArrayArray{{ .Name }}Column) WriteTo(wr *chproto.Writer) error {
_ = c.WriteOffset(wr, 0)
return c.WriteData(wr)
}
func (c *ArrayArray{{ .Name }}Column) WriteOffset(wr *chproto.Writer, offset int) int {
for _, el := range c.Column {
offset += len(el)
wr.UInt64(uint64(offset))
}
offset = 0
for _, elem := range c.Column {
c.elem.Column = elem
offset = c.elem.WriteOffset(wr, offset)
}
return offset
}
func (c *ArrayArray{{ .Name }}Column) WriteData(wr *chproto.Writer) error {
for _, ss := range c.Column {
c.elem.Column = ss
if err := c.elem.WriteData(wr); err != nil {
return err
}
}
return nil
}
{{- end }}

12
ch/internal/codegen/column_safe.tpl
View File

@ -8,11 +8,13 @@ import (
{{- range . }}
func (c *{{ .CHType }}Column) ReadFrom(rd *chproto.Reader, numRow int) error {
c.Alloc(numRow)
{{ if eq .Size 0 }} {{ continue }} {{ end }}
func (c *{{ .Name }}Column) ReadFrom(rd *chproto.Reader, numRow int) error {
c.AllocForReading(numRow)
for i := range c.Column {
n, err := rd.{{ .CHType }}()
n, err := rd.{{ .Name }}()
if err != nil {
return err
}
@ -22,9 +24,9 @@ func (c *{{ .CHType }}Column) ReadFrom(rd *chproto.Reader, numRow int) error {
return nil
}
func (c *{{ .CHType }}Column) WriteTo(wr *chproto.Writer) error {
func (c *{{ .Name }}Column) WriteTo(wr *chproto.Writer) error {
for _, n := range c.Column {
wr.{{ .CHType }}(n)
wr.{{ .Name }}(n)
}
return nil
}

8
ch/internal/codegen/column_unsafe.tpl
View File

@ -12,14 +12,16 @@ import (
{{- range . }}
func (c *{{ .CHType }}Column) ReadFrom(rd *chproto.Reader, numRow int) error {
{{ if eq .Size 0 }} {{ continue }} {{ end }}
func (c *{{ .Name }}Column) ReadFrom(rd *chproto.Reader, numRow int) error {
const size = {{ .Size }} / 8
if numRow == 0 {
return nil
}
c.Alloc(numRow)
c.AllocForReading(numRow)
slice := *(*reflect.SliceHeader)(unsafe.Pointer(&c.Column))
slice.Len *= size
@ -30,7 +32,7 @@ func (c *{{ .CHType }}Column) ReadFrom(rd *chproto.Reader, numRow int) error {
return err
}
func (c *{{ .CHType }}Column) WriteTo(wr *chproto.Writer) error {
func (c *{{ .Name }}Column) WriteTo(wr *chproto.Writer) error {
const size = {{ .Size }} / 8
if len(c.Column) == 0 {