From 05ff93555c73388fb485611548d9e9e660a693a1 Mon Sep 17 00:00:00 2001 From: Lee Brown Date: Mon, 12 Aug 2019 10:26:00 -0800 Subject: [PATCH] Reduce countries loaded for geonames to US only by default. Schema loads in around 1min now. --- internal/geonames/geonames.go | 14 +++++++++----- internal/schema/init_schema.go | 3 ++- internal/schema/migrations.go | 10 +++++----- internal/schema/schema.go | 7 ++++--- tools/devops/cmd/cicd/service_deploy.go | 9 ++++----- tools/schema/main.go | 13 ++++++++++++- 6 files changed, 36 insertions(+), 20 deletions(-) diff --git a/internal/geonames/geonames.go b/internal/geonames/geonames.go index e74547d..47a4e48 100644 --- a/internal/geonames/geonames.go +++ b/internal/geonames/geonames.go @@ -11,6 +11,7 @@ import ( "strconv" "strings" + "geeks-accelerator/oss/saas-starter-kit/internal/platform/web/webcontext" "github.com/huandu/go-sqlbuilder" "github.com/jmoiron/sqlx" "github.com/pkg/errors" @@ -24,9 +25,12 @@ const ( geonamesTableName = "geonames" ) -var ( - // List of country codes that will geonames will be downloaded for. - ValidGeonameCountries = []string{ +// List of country codes that will geonames will be downloaded for. +func ValidGeonameCountries(ctx context.Context) []string { + if webcontext.ContextEnv(ctx) == webcontext.Env_Dev { + return []string{"US"} + } + return []string{ "AD", "AR", "AS", "AT", "AU", "AX", "BD", "BE", "BG", "BM", "BR", "BY", "CA", "CH", "CO", "CR", "CZ", "DE", "DK", "DO", "DZ", "ES", "FI", "FO", "FR", "GB", "GF", "GG", "GL", "GP", @@ -36,7 +40,7 @@ var ( "PK", "PL", "PM", "PR", "PT", "RE", "RO", "RU", "SE", "SI", "SJ", "SK", "SM", "TH", "TR", "UA", "US", "UY", "VA", "VI", "WF", "YT", "ZA"} -) +} // FindGeonames .... func FindGeonames(ctx context.Context, dbConn *sqlx.DB, orderBy, where string, args ...interface{}) ([]*Geoname, error) { @@ -194,7 +198,7 @@ func LoadGeonames(ctx context.Context, rr chan<- interface{}, countries ...strin defer close(rr) if len(countries) == 0 { - countries = ValidGeonameCountries + countries = ValidGeonameCountries(ctx) } for _, country := range countries { diff --git a/internal/schema/init_schema.go b/internal/schema/init_schema.go index 6326de6..19102c5 100644 --- a/internal/schema/init_schema.go +++ b/internal/schema/init_schema.go @@ -1,6 +1,7 @@ package schema import ( + "context" "log" "github.com/jmoiron/sqlx" @@ -8,7 +9,7 @@ import ( // initSchema runs before any migrations are executed. This happens when no other migrations // have previously been executed. -func initSchema(db *sqlx.DB, log *log.Logger, isUnittest bool) func(*sqlx.DB) error { +func initSchema(ctx context.Context, db *sqlx.DB, log *log.Logger, isUnittest bool) func(*sqlx.DB) error { f := func(db *sqlx.DB) error { return nil } diff --git a/internal/schema/migrations.go b/internal/schema/migrations.go index 730c403..fe6a3c9 100644 --- a/internal/schema/migrations.go +++ b/internal/schema/migrations.go @@ -18,7 +18,7 @@ import ( // migrationList returns a list of migrations to be executed. If the id of the // migration already exists in the migrations table it will be skipped. -func migrationList(db *sqlx.DB, log *log.Logger, isUnittest bool) []*sqlxmigrate.Migration { +func migrationList(ctx context.Context, db *sqlx.DB, log *log.Logger, isUnittest bool) []*sqlxmigrate.Migration { return []*sqlxmigrate.Migration{ // Create table users. { @@ -213,7 +213,7 @@ func migrationList(db *sqlx.DB, log *log.Logger, isUnittest bool) []*sqlxmigrate }, // Load new geonames table. { - ID: "20190731-02b", + ID: "20190731-02h", Migrate: func(tx *sql.Tx) error { schemas := []string{ @@ -253,7 +253,7 @@ func migrationList(db *sqlx.DB, log *log.Logger, isUnittest bool) []*sqlxmigrate } else { resChan := make(chan interface{}) - go geonames.LoadGeonames(context.Background(), resChan) + go geonames.LoadGeonames(ctx, resChan) for r := range resChan { switch v := r.(type) { @@ -288,7 +288,7 @@ func migrationList(db *sqlx.DB, log *log.Logger, isUnittest bool) []*sqlxmigrate }, // Load new countries table. { - ID: "20190731-02d", + ID: "20190731-02f", Migrate: func(tx *sql.Tx) error { schemas := []string{ @@ -489,7 +489,7 @@ func migrationList(db *sqlx.DB, log *log.Logger, isUnittest bool) []*sqlxmigrate }, // Load new country_timezones table. { - ID: "20190731-03d", + ID: "20190731-03e", Migrate: func(tx *sql.Tx) error { queries := []string{ diff --git a/internal/schema/schema.go b/internal/schema/schema.go index d4121d4..044c769 100644 --- a/internal/schema/schema.go +++ b/internal/schema/schema.go @@ -1,21 +1,22 @@ package schema import ( + "context" "log" "github.com/geeks-accelerator/sqlxmigrate" "github.com/jmoiron/sqlx" ) -func Migrate(masterDb *sqlx.DB, log *log.Logger, isUnittest bool) error { +func Migrate(ctx context.Context, masterDb *sqlx.DB, log *log.Logger, isUnittest bool) error { // Load list of Schema migrations and init new sqlxmigrate client - migrations := migrationList(masterDb, log, isUnittest) + migrations := migrationList(ctx, masterDb, log, isUnittest) m := sqlxmigrate.New(masterDb, sqlxmigrate.DefaultOptions, migrations) m.SetLogger(log) // Append any schema that need to be applied if this is a fresh migration // ie. the migrations database table does not exist. - m.InitSchema(initSchema(masterDb, log, isUnittest)) + m.InitSchema(initSchema(ctx, masterDb, log, isUnittest)) // Execute the migrations return m.Migrate() diff --git a/tools/devops/cmd/cicd/service_deploy.go b/tools/devops/cmd/cicd/service_deploy.go index 38dd2e5..9f5ee45 100644 --- a/tools/devops/cmd/cicd/service_deploy.go +++ b/tools/devops/cmd/cicd/service_deploy.go @@ -66,7 +66,7 @@ type ServiceDeployFlags struct { DockerFile string `validate:"omitempty" example:"./cmd/web-api/Dockerfile"` EnableLambdaVPC bool `validate:"omitempty" example:"false"` EnableEcsElb bool `validate:"omitempty" example:"false"` - IsLambda bool `validate:"omitempty" example:"false"` + IsLambda bool `validate:"omitempty" example:"false"` StaticFilesS3Enable bool `validate:"omitempty" example:"false"` StaticFilesImgResizeEnable bool `validate:"omitempty" example:"false"` @@ -137,7 +137,7 @@ type serviceDeployRequest struct { VpcPublicSubnets []*ec2.CreateSubnetInput EnableLambdaVPC bool `validate:"omitempty"` - IsLambda bool `validate:"omitempty"` + IsLambda bool `validate:"omitempty"` RecreateService bool `validate:"omitempty"` SDNamepsace *servicediscovery.CreatePrivateDnsNamespaceInput @@ -194,7 +194,7 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic S3BucketPrivateName: flags.S3BucketPrivateName, S3BucketPublicName: flags.S3BucketPublicName, - IsLambda: flags.IsLambda, + IsLambda: flags.IsLambda, EnableLambdaVPC: flags.EnableLambdaVPC, EnableEcsElb: flags.EnableEcsElb, RecreateService: flags.RecreateService, @@ -441,7 +441,7 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic if req.IsLambda { - } else { + } else { } @@ -861,7 +861,6 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic log.Printf("\t%s\tDefaults set.", tests.Success) } - r, err := regexp.Compile(`^(\d+)`) if err != nil { return nil, errors.WithStack(err) diff --git a/tools/schema/main.go b/tools/schema/main.go index bdf2588..0b7f0ab 100644 --- a/tools/schema/main.go +++ b/tools/schema/main.go @@ -1,11 +1,14 @@ package main import ( + "context" "encoding/json" "expvar" + "geeks-accelerator/oss/saas-starter-kit/internal/platform/web/webcontext" "log" "net/url" "os" + "time" "geeks-accelerator/oss/saas-starter-kit/internal/platform/flag" "geeks-accelerator/oss/saas-starter-kit/internal/schema" @@ -125,8 +128,16 @@ func main() { // ========================================================================= // Start Migrations + // Set the context with the required values to + // process the request. + v := webcontext.Values{ + Now: time.Now(), + Env: cfg.Env, + } + ctx := context.WithValue(context.Background(), webcontext.KeyValues, &v) + // Execute the migrations - if err = schema.Migrate(masterDb, log, false); err != nil { + if err = schema.Migrate(ctx, masterDb, log, false); err != nil { log.Fatalf("main : Migrate : %v", err) } log.Printf("main : Migrate : Completed")