1
0
mirror of https://github.com/ManyakRus/crud_generator.git synced 2024-11-27 08:40:53 +02:00
crud_generator/pkg/dbmeta/meta.go
Nikitin Aleksandr a6ea64ee3e новый
2023-10-24 18:03:04 +03:00

877 lines
21 KiB
Go

package dbmeta
import (
"bytes"
"database/sql"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"reflect"
"runtime"
"strings"
"time"
"github.com/bxcodec/faker/v3"
"github.com/iancoleman/strcase"
dynamicstruct "github.com/ompluscator/dynamic-struct"
)
type metaDataLoader func(db *sql.DB, sqlType, sqlDatabase, tableName string) (DbTableMeta, error)
var metaDataFuncs = make(map[string]metaDataLoader)
var sqlMappings = make(map[string]*SQLMapping)
func init() {
metaDataFuncs["sqlite3"] = LoadSqliteMeta
metaDataFuncs["sqlite"] = LoadSqliteMeta
metaDataFuncs["mssql"] = LoadMsSQLMeta
metaDataFuncs["postgres"] = LoadPostgresMeta
metaDataFuncs["mysql"] = LoadMysqlMeta
}
// SQLMappings mappings for sql types to json, go etc
type SQLMappings struct {
SQLMappings []*SQLMapping `json:"mappings"`
}
// SQLMapping mapping
type SQLMapping struct {
// SQLType sql type reported from db
SQLType string `json:"sql_type"`
// GoType mapped go type
GoType string `json:"go_type"`
// JSONType mapped json type
JSONType string `json:"json_type"`
// ProtobufType mapped protobuf type
ProtobufType string `json:"protobuf_type"`
// GureguType mapped go type using Guregu
GureguType string `json:"guregu_type"`
// GoNullableType mapped go type using nullable
GoNullableType string `json:"go_nullable_type"`
// SwaggerType mapped type
SwaggerType string `json:"swagger_type"`
}
func (m *SQLMapping) String() interface{} {
return fmt.Sprintf("SQLType: %-15s GoType: %-15s GureguType: %-15s GoNullableType: %-15s JSONType: %-15s ProtobufType: %-15s",
m.SQLType,
m.GoType, m.GureguType, m.GoNullableType,
m.JSONType, m.ProtobufType)
}
// IsAutoIncrement return is column is a primary key column
func (ci *columnMeta) IsPrimaryKey() bool {
return ci.isPrimaryKey
}
// IsArray return is column is an array type
func (ci *columnMeta) IsArray() bool {
return ci.isArray
}
// IsAutoIncrement return is column is an auto increment column
func (ci *columnMeta) IsAutoIncrement() bool {
return ci.isAutoIncrement
}
type columnMeta struct {
index int
// ct *sql.ColumnType
nullable bool
isPrimaryKey bool
isAutoIncrement bool
isArray bool
colDDL string
columnType string
columnLen int64
defaultVal string
notes string
comment string
databaseTypeName string
name string
}
// ColumnType column type
func (ci *columnMeta) ColumnType() string {
return ci.columnType
}
// Notes notes on column generation
func (ci *columnMeta) Notes() string {
return ci.notes
}
// Comment column comment
func (ci *columnMeta) Comment() string {
return ci.comment
}
// ColumnLength column length for text or varhar
func (ci *columnMeta) ColumnLength() int64 {
return ci.columnLen
}
// DefaultValue default value of column
func (ci *columnMeta) DefaultValue() string {
return ci.defaultVal
}
// Name name of column
func (ci *columnMeta) Name() string {
return ci.name
}
// Index index of column in db
func (ci *columnMeta) Index() int {
return ci.index
}
// String friendly string for columnMeta
func (ci *columnMeta) String() string {
return fmt.Sprintf("[%2d] %-45s %-20s null: %-6t primary: %-6t isArray: %-6t auto: %-6t col: %-15s len: %-7d default: [%s]",
ci.index, ci.name, ci.DatabaseTypePretty(),
ci.nullable, ci.isPrimaryKey, ci.isArray,
ci.isAutoIncrement, ci.columnType, ci.columnLen, ci.defaultVal)
}
// Nullable reports whether the column may be null.
// If a driver does not support this property ok will be false.
func (ci *columnMeta) Nullable() bool {
return ci.nullable
}
// ColDDL string of the ddl for the column
func (ci *columnMeta) ColDDL() string {
return ci.colDDL
}
// DatabaseTypeName returns the database system name of the column type. If an empty
// string is returned the driver type name is not supported.
// Consult your driver documentation for a list of driver data types. Length specifiers
// are not included.
// Common type include "VARCHAR", "TEXT", "NVARCHAR", "DECIMAL", "BOOL", "INT", "BIGINT".
func (ci *columnMeta) DatabaseTypeName() string {
return ci.databaseTypeName
}
// DatabaseTypePretty string of the db type
func (ci *columnMeta) DatabaseTypePretty() string {
if ci.columnLen > 0 {
return fmt.Sprintf("%s(%d)", ci.columnType, ci.columnLen)
}
return ci.columnType
}
// DbTableMeta table meta data
type DbTableMeta interface {
Columns() []ColumnMeta
SQLType() string
SQLDatabase() string
TableName() string
DDL() string
}
// ColumnMeta meta data for a column
type ColumnMeta interface {
Name() string
String() string
Nullable() bool
DatabaseTypeName() string
DatabaseTypePretty() string
Index() int
IsPrimaryKey() bool
IsAutoIncrement() bool
IsArray() bool
ColumnType() string
Notes() string
Comment() string
ColumnLength() int64
DefaultValue() string
}
type dbTableMeta struct {
sqlType string
sqlDatabase string
tableName string
columns []*columnMeta
ddl string
primaryKeyPos int
}
// PrimaryKeyPos ordinal pos of primary key
func (m *dbTableMeta) PrimaryKeyPos() int {
return m.primaryKeyPos
}
// SQLType sql db type
func (m *dbTableMeta) SQLType() string {
return m.sqlType
}
// SQLDatabase sql database name
func (m *dbTableMeta) SQLDatabase() string {
return m.sqlDatabase
}
// TableName sql table name
func (m *dbTableMeta) TableName() string {
return m.tableName
}
// Columns ColumnMeta for columns in a sql table
func (m *dbTableMeta) Columns() []ColumnMeta {
cols := make([]ColumnMeta, len(m.columns))
for i, v := range m.columns {
cols[i] = ColumnMeta(v)
}
return cols
}
// DDL string for a sql table
func (m *dbTableMeta) DDL() string {
return m.ddl
}
// ModelInfo info for a sql table
type ModelInfo struct {
Index int
IndexPlus1 int
PackageName string
StructName string
ShortStructName string
TableName string
Fields []string
DBMeta DbTableMeta
Instance interface{}
CodeFields []*FieldInfo
}
// Notes notes on table generation
func (m *ModelInfo) Notes() string {
buf := bytes.Buffer{}
for i, j := range m.DBMeta.Columns() {
if j.Notes() != "" {
buf.WriteString(fmt.Sprintf("[%2d] %s\n", i, j.Notes()))
}
}
for i, j := range m.CodeFields {
if j.Notes != "" {
buf.WriteString(fmt.Sprintf("[%2d] %s\n", i, j.Notes))
}
}
return buf.String()
}
// FieldInfo codegen info for each column in sql table
type FieldInfo struct {
Index int
GoFieldName string
GoFieldType string
GoAnnotations []string
JSONFieldName string
ProtobufFieldName string
ProtobufType string
ProtobufPos int
Comment string
Notes string
Code string
FakeData interface{}
ColumnMeta ColumnMeta
PrimaryKeyFieldParser string
PrimaryKeyArgName string
SQLMapping *SQLMapping
GormAnnotation string
JSONAnnotation string
XMLAnnotation string
DBAnnotation string
GoGoMoreTags string
}
// GetFunctionName get function name
func GetFunctionName(i interface{}) string {
return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()
}
// LoadMeta loads the DbTableMeta data from the db connection for the table
func LoadMeta(sqlType string, db *sql.DB, sqlDatabase, tableName string) (DbTableMeta, error) {
dbMetaFunc, haveMeta := metaDataFuncs[sqlType]
if !haveMeta {
dbMetaFunc = LoadUnknownMeta
}
dbMeta, err := dbMetaFunc(db, sqlType, sqlDatabase, tableName)
//if err != nil {
// fmt.Printf("Error calling func: %s error: %v\n", GetFunctionName(dbMetaFunc), err)
//}
return dbMeta, err
}
// GenerateFieldsTypes FieldInfo slice from DbTableMeta
func (c *Config) GenerateFieldsTypes(dbMeta DbTableMeta) ([]*FieldInfo, error) {
var fields []*FieldInfo
field := ""
for i, col := range dbMeta.Columns() {
fieldName := col.Name()
fi := &FieldInfo{
Index: i,
}
valueType, err := SQLTypeToGoType(strings.ToLower(col.DatabaseTypeName()), col.Nullable(), c.UseGureguTypes)
if err != nil { // unknown type
fmt.Printf("table: %s unable to generate struct field: %s type: %s error: %v\n", dbMeta.TableName(), fieldName, col.DatabaseTypeName(), err)
continue
}
fieldName = Replace(c.FieldNamingTemplate, fieldName)
fieldName = checkDupeFieldName(fields, fieldName)
fi.GormAnnotation = createGormAnnotation(col)
fi.JSONAnnotation = createJSONAnnotation(c.JSONNameFormat, col)
fi.XMLAnnotation = createXMLAnnotation(c.XMLNameFormat, col)
fi.DBAnnotation = createDBAnnotation(col)
var annotations []string
if c.AddGormAnnotation {
annotations = append(annotations, fi.GormAnnotation)
}
if c.AddJSONAnnotation {
annotations = append(annotations, fi.JSONAnnotation)
}
if c.AddXMLAnnotation {
annotations = append(annotations, fi.XMLAnnotation)
}
if c.AddDBAnnotation {
annotations = append(annotations, fi.DBAnnotation)
}
gogoTags := []string{fi.GormAnnotation, fi.JSONAnnotation, fi.XMLAnnotation, fi.DBAnnotation}
GoGoMoreTags := strings.Join(gogoTags, " ")
if c.AddProtobufAnnotation {
annotation, err := createProtobufAnnotation(c.ProtobufNameFormat, col)
if err == nil {
annotations = append(annotations, annotation)
}
}
if len(annotations) > 0 {
field = fmt.Sprintf("%s %s `%s`",
fieldName,
valueType,
strings.Join(annotations, " "))
} else {
field = fmt.Sprintf("%s %s", fieldName, valueType)
}
field = fmt.Sprintf("//%s\n %s", col.String(), field)
if col.Comment() != "" {
field = fmt.Sprintf("%s // %s", field, col.Comment())
}
sqlMapping, _ := SQLTypeToMapping(strings.ToLower(col.DatabaseTypeName()))
goType, _ := SQLTypeToGoType(strings.ToLower(col.DatabaseTypeName()), false, false)
protobufType, _ := SQLTypeToProtobufType(col.DatabaseTypeName())
// fmt.Printf("protobufType: %v DatabaseTypeName: %v\n", protobufType, col.DatabaseTypeName())
fakeData := createFakeData(goType, fieldName)
//if c.Verbose {
// fmt.Printf("table: %-10s type: %-10s fieldname: %-20s val: %v\n", c.DatabaseTypeName(), goType, fieldName, fakeData)
// spew.Dump(fakeData)
//}
//fmt.Printf("%+v", fakeData)
primaryKeyFieldParser := ""
if col.IsPrimaryKey() {
var ok bool
primaryKeyFieldParser, ok = parsePrimaryKeys[goType]
if !ok {
primaryKeyFieldParser = "unsupported"
}
}
fi.Code = field
fi.GoFieldName = fieldName
fi.GoFieldType = valueType
fi.GoAnnotations = annotations
fi.FakeData = fakeData
fi.Comment = col.String()
fi.JSONFieldName = formatFieldName(c.JSONNameFormat, col.Name())
fi.ProtobufFieldName = formatFieldName(c.ProtobufNameFormat, col.Name())
fi.ProtobufType = protobufType
fi.ProtobufPos = i + 1
fi.ColumnMeta = col
fi.PrimaryKeyFieldParser = primaryKeyFieldParser
fi.SQLMapping = sqlMapping
fi.GoGoMoreTags = GoGoMoreTags
fi.JSONFieldName = checkDupeJSONFieldName(fields, fi.JSONFieldName)
fi.ProtobufFieldName = checkDupeProtoBufFieldName(fields, fi.ProtobufFieldName)
fields = append(fields, fi)
}
return fields, nil
}
func formatFieldName(nameFormat string, name string) string {
var jsonName string
switch nameFormat {
case "snake":
jsonName = strcase.ToSnake(name)
case "camel":
jsonName = strcase.ToCamel(name)
case "lower_camel":
jsonName = strcase.ToLowerCamel(name)
case "none":
jsonName = name
default:
jsonName = name
}
return jsonName
}
func createJSONAnnotation(nameFormat string, c ColumnMeta) string {
name := formatFieldName(nameFormat, c.Name())
return fmt.Sprintf("json:\"%s\"", name)
}
func createXMLAnnotation(nameFormat string, c ColumnMeta) string {
name := formatFieldName(nameFormat, c.Name())
return fmt.Sprintf("xml:\"%s\"", name)
}
func createDBAnnotation(c ColumnMeta) string {
return fmt.Sprintf("db:\"%s\"", c.Name())
}
func createProtobufAnnotation(nameFormat string, c ColumnMeta) (string, error) {
protoBufType, err := SQLTypeToProtobufType(c.DatabaseTypeName())
if err != nil {
return "", err
}
if protoBufType != "" {
name := formatFieldName(nameFormat, c.Name())
return fmt.Sprintf("protobuf:\"%s,%d,opt,name=%s\"", protoBufType, c.Index(), name), nil
}
return "", fmt.Errorf("unknown sql name: %s", c.Name())
}
func createGormAnnotation(c ColumnMeta) string {
buf := bytes.Buffer{}
key := c.Name()
buf.WriteString("gorm:\"")
if c.IsPrimaryKey() {
buf.WriteString("primary_key;")
}
if c.IsAutoIncrement() {
buf.WriteString("AUTO_INCREMENT;")
}
buf.WriteString("column:")
buf.WriteString(key)
buf.WriteString(";")
if c.DatabaseTypeName() != "" {
buf.WriteString("type:")
buf.WriteString(c.DatabaseTypeName())
buf.WriteString(";")
if c.ColumnLength() > 0 {
buf.WriteString(fmt.Sprintf("size:%d;", c.ColumnLength()))
}
if c.DefaultValue() != "" {
value := c.DefaultValue()
value = strings.Replace(value, "\"", "'", -1)
if value == "NULL" || value == "null" {
value = ""
}
if value != "" && !strings.Contains(value, "()") {
buf.WriteString(fmt.Sprintf("default:%s;", value))
}
}
}
buf.WriteString("\"")
return buf.String()
}
// BuildDefaultTableDDL create a ddl mock using the ColumnMeta data
func BuildDefaultTableDDL(tableName string, cols []*columnMeta) string {
buf := bytes.Buffer{}
buf.WriteString(fmt.Sprintf("Table: %s\n", tableName))
for _, ct := range cols {
buf.WriteString(fmt.Sprintf("%s\n", ct.String()))
}
return buf.String()
}
// ProcessMappings process the json for mappings to load sql mappings
func ProcessMappings(source string, mappingJsonstring []byte, verbose bool) error {
var mappings = &SQLMappings{}
err := json.Unmarshal(mappingJsonstring, mappings)
if err != nil {
fmt.Printf("Error unmarshalling json error: %v\n", err)
return err
}
if verbose {
fmt.Printf("Loaded %d mappings from: %s\n", len(mappings.SQLMappings), source)
}
for i, value := range mappings.SQLMappings {
if verbose {
fmt.Printf(" Mapping:[%2d] -> %s\n", i, value.SQLType)
}
sqlMappings[value.SQLType] = value
}
return nil
}
// LoadMappings load sql mappings to load mapping json file
func LoadMappings(mappingFileName string, verbose bool) error {
mappingFile, err := os.Open(mappingFileName)
if err != nil {
fmt.Printf("Error loading mapping file %s error: %v\n", mappingFileName, err)
return err
}
defer func() {
_ = mappingFile.Close()
}()
byteValue, err := ioutil.ReadAll(mappingFile)
if err != nil {
fmt.Printf("Error loading mapping file %s error: %v\n", mappingFileName, err)
return err
}
absPath, err := filepath.Abs(mappingFileName)
if err != nil {
absPath = mappingFileName
}
return ProcessMappings(absPath, byteValue, verbose)
}
// SQLTypeToGoType map a sql type to a go type
func SQLTypeToGoType(sqlType string, nullable bool, gureguTypes bool) (string, error) {
mapping, err := SQLTypeToMapping(sqlType)
if err != nil {
return "", err
}
if nullable && gureguTypes {
return mapping.GureguType, nil
} else if nullable {
return mapping.GoNullableType, nil
} else {
return mapping.GoType, nil
}
}
// SQLTypeToProtobufType map a sql type to a protobuf type
func SQLTypeToProtobufType(sqlType string) (string, error) {
mapping, err := SQLTypeToMapping(sqlType)
if err != nil {
return "", err
}
return mapping.ProtobufType, nil
}
// SQLTypeToMapping retrieve a SQLMapping based on a sql type
func SQLTypeToMapping(sqlType string) (*SQLMapping, error) {
sqlType = cleanupSQLType(sqlType)
mapping, ok := sqlMappings[sqlType]
if !ok {
return nil, fmt.Errorf("unknown sql type: %s", sqlType)
}
return mapping, nil
}
func cleanupSQLType(sqlType string) string {
sqlType = strings.ToLower(sqlType)
sqlType = strings.Trim(sqlType, " \t")
sqlType = strings.ToLower(sqlType)
idx := strings.Index(sqlType, "(")
if idx > -1 {
sqlType = sqlType[0:idx]
}
return sqlType
}
// GetMappings get all mappings
func GetMappings() map[string]*SQLMapping {
return sqlMappings
}
func createFakeData(valueType string, name string) interface{} {
switch valueType {
case "[]byte":
return []byte("hello world")
case "bool":
return true
case "float32":
return float32(1.0)
case "float64":
return float64(1.0)
case "int":
return int(1)
case "int64":
return int64(1)
case "string":
return "hello world"
case "time.Time":
return time.Now()
case "interface{}":
return 1
default:
return 1
}
}
// FindInSlice takes a slice and looks for an element in it. If found it will
// return it's key, otherwise it will return -1 and a bool of false.
func FindInSlice(slice []string, val string) (int, bool) {
for i, item := range slice {
if item == val {
return i, true
}
}
return -1, false
}
// LoadTableInfo load table info from db connection, and list of tables
func LoadTableInfo(db *sql.DB, dbTables []string, excludeDbTables []string, conf *Config) map[string]*ModelInfo {
tableInfos := make(map[string]*ModelInfo)
// generate go files for each table
var tableIdx = 0
for i, tableName := range dbTables {
_, ok := FindInSlice(excludeDbTables, tableName)
if ok {
fmt.Printf("Skipping excluded table %s\n", tableName)
continue
}
if strings.HasPrefix(tableName, "[") && strings.HasSuffix(tableName, "]") {
tableName = tableName[1 : len(tableName)-1]
}
dbMeta, err := LoadMeta(conf.SQLType, db, conf.SQLDatabase, tableName)
if err != nil {
msg := fmt.Sprintf("Warning - LoadMeta skipping table info for %s error: %v\n", tableName, err)
if au != nil {
fmt.Print(au.Yellow(msg))
} else {
fmt.Printf(msg)
}
continue
}
modelInfo, err := GenerateModelInfo(tableInfos, dbMeta, tableName, conf)
if err != nil {
msg := fmt.Sprintf("Error - %v\n", err)
if au != nil {
fmt.Print(au.Red(msg))
} else {
fmt.Printf(msg)
}
continue
}
if len(modelInfo.Fields) == 0 {
if conf.Verbose {
fmt.Printf("[%d] Table: %s - No Fields Available\n", i, tableName)
}
continue
}
modelInfo.Index = tableIdx
modelInfo.IndexPlus1 = tableIdx + 1
tableIdx++
tableInfos[tableName] = modelInfo
}
return tableInfos
}
// GenerateModelInfo generates a struct for the given table.
func GenerateModelInfo(tables map[string]*ModelInfo, dbMeta DbTableMeta,
tableName string,
conf *Config) (*ModelInfo, error) {
structName := Replace(conf.ModelNamingTemplate, tableName)
structName = CheckForDupeTable(tables, structName)
fields, err := conf.GenerateFieldsTypes(dbMeta)
if err != nil {
return nil, err
}
if conf.Verbose {
fmt.Printf("\ntableName: %s\n", tableName)
for _, c := range dbMeta.Columns() {
fmt.Printf(" %s\n", c.String())
}
fmt.Print("\n")
}
generator := dynamicstruct.NewStruct()
noOfPrimaryKeys := 0
for i, c := range fields {
meta := dbMeta.Columns()[i]
jsonName := formatFieldName(conf.JSONNameFormat, meta.Name())
tag := fmt.Sprintf(`json:"%s"`, jsonName)
fakeData := c.FakeData
generator = generator.AddField(c.GoFieldName, fakeData, tag)
if meta.IsPrimaryKey() {
//c.PrimaryKeyArgName = RenameReservedName(strcase.ToLowerCamel(c.GoFieldName))
c.PrimaryKeyArgName = fmt.Sprintf("arg%s", FmtFieldName(c.GoFieldName))
noOfPrimaryKeys++
}
}
instance := generator.Build().New()
err = faker.FakeData(&instance)
if err != nil {
fmt.Println(err)
}
// fmt.Printf("%+v", instance)
var code []string
for _, f := range fields {
if f.PrimaryKeyFieldParser == "unsupported" {
return nil, fmt.Errorf("unable to generate code for table: %s, primary key column: [%d] %s has unsupported type: %s / %s",
dbMeta.TableName(), f.ColumnMeta.Index(), f.ColumnMeta.Name(), f.ColumnMeta.DatabaseTypeName(), f.GoFieldType)
}
code = append(code, f.Code)
}
var modelInfo = &ModelInfo{
PackageName: conf.ModelPackageName,
StructName: structName,
TableName: tableName,
ShortStructName: strings.ToLower(string(structName[0])),
Fields: code,
CodeFields: fields,
DBMeta: dbMeta,
Instance: instance,
}
return modelInfo, nil
}
// CheckForDupeTable check for duplicate table name, returns available name
func CheckForDupeTable(tables map[string]*ModelInfo, name string) string {
found := false
for _, model := range tables {
if model.StructName == name {
found = true
}
}
if found {
name = CheckForDupeTable(tables, name+"_")
}
if name == "Result" {
name = "DBTableResult"
}
return name
}
func checkDupeFieldName(fields []*FieldInfo, fieldName string) string {
var match bool
for _, field := range fields {
if fieldName == field.GoFieldName {
match = true
break
}
}
if match {
fieldName = checkDupeFieldName(fields, generateAlternativeName(fieldName))
}
return fieldName
}
func checkDupeJSONFieldName(fields []*FieldInfo, fieldName string) string {
var match bool
for _, field := range fields {
if fieldName == field.JSONFieldName {
match = true
break
}
}
if match {
fieldName = checkDupeJSONFieldName(fields, generateAlternativeName(fieldName))
}
return fieldName
}
func checkDupeProtoBufFieldName(fields []*FieldInfo, fieldName string) string {
var match bool
for _, field := range fields {
if fieldName == field.ProtobufFieldName {
match = true
break
}
}
if match {
fieldName = checkDupeProtoBufFieldName(fields, generateAlternativeName(fieldName))
}
return fieldName
}
// @TODO In progress - need more elegant renaming
func generateAlternativeName(name string) string {
name = name + "alt1"
return name
}