diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2ba1b97..40b98c2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,28 +50,35 @@ db:migrate:dev: - master - dev - /^dev-.*$/ - variables: TARGET_ENV: 'dev' + AWS_USE_ROLE: 'true' webapi:build:dev: <<: *build_tmpl stage: build:dev tags: - dev - variables: - TARGET_ENV: 'dev' - SERVICE: 'web-api' - AWS_USE_ROLE: 'true' only: - master - dev - dev-web-api + variables: + TARGET_ENV: 'dev' + SERVICE: 'web-api' + AWS_USE_ROLE: 'true' webapi:deploy:dev: <<: *deploy_tmpl stage: deploy:dev tags: - dev + only: + - master + - dev + - dev-web-api + dependencies: + - 'webapi:build:dev' + # - 'db:migrate:dev' variables: TARGET_ENV: 'dev' SERVICE: 'web-api' @@ -82,13 +89,8 @@ webapi:deploy:dev: S3_BUCKET_PRIVATE: 'saas-starter-kit-private' S3_BUCKET_PUBLIC: 'saas-starter-kit-public' AWS_USE_ROLE: 'true' - dependencies: - - 'webapi:build:dev' -# - 'db:migrate:dev' - only: - - master - - dev - - dev-web-api + + #ddlogscollector:deploy:stage: diff --git a/tools/devops/cmd/cicd/migrate.go b/tools/devops/cmd/cicd/migrate.go new file mode 100644 index 0000000..79b1fb6 --- /dev/null +++ b/tools/devops/cmd/cicd/migrate.go @@ -0,0 +1,228 @@ +package cicd + +import ( + "encoding/json" + "log" + "net/url" + "path/filepath" + + "geeks-accelerator/oss/saas-starter-kit/internal/platform/tests" + "geeks-accelerator/oss/saas-starter-kit/internal/schema" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/secretsmanager" + "github.com/lib/pq" + _ "github.com/lib/pq" + "github.com/pkg/errors" + sqltrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/database/sql" + sqlxtrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/jmoiron/sqlx" + "gopkg.in/go-playground/validator.v9" +) + +// MigrateFlags defines the flags used for executing schema migration. +type MigrateFlags struct { + // Required flags. + Env string `validate:"oneof=dev stage prod" example:"dev"` + + // Optional flags. + ProjectRoot string `validate:"omitempty" example:"."` + ProjectName string ` validate:"omitempty" example:"example-project"` +} + +// migrateRequest defines the details needed to execute a service build. +type migrateRequest struct { + Env string `validate:"oneof=dev stage prod"` + ProjectRoot string `validate:"required"` + ProjectName string `validate:"required"` + GoModFile string `validate:"required"` + GoModName string `validate:"required"` + + AwsCreds awsCredentials `validate:"required,dive,required"` + _awsSession *session.Session + + flags MigrateFlags +} + +// awsSession returns the current AWS session for the serviceDeployRequest. +func (r *migrateRequest) awsSession() *session.Session { + if r._awsSession == nil { + r._awsSession = r.AwsCreds.Session() + } + + return r._awsSession +} + +// NewMigrateRequest generates a new request for executing schema migration for a given set of CLI flags. +func NewMigrateRequest(log *log.Logger, flags MigrateFlags) (*migrateRequest, error) { + + // Validates specified CLI flags map to struct successfully. + log.Println("Validate flags.") + { + errs := validator.New().Struct(flags) + if errs != nil { + return nil, errs + } + log.Printf("\t%s\tFlags ok.", tests.Success) + } + + // Generate a migrate request using CLI flags and AWS credentials. + log.Println("Generate migrate request.") + var req migrateRequest + { + + // Define new migrate request. + req = migrateRequest{ + Env: flags.Env, + ProjectRoot: flags.ProjectRoot, + ProjectName: flags.ProjectName, + + flags: flags, + } + + // When project root directory is empty or set to current working path, then search for the project root by locating + // the go.mod file. + log.Println("\tDetermining the project root directory.") + { + if req.ProjectRoot == "" || req.ProjectRoot == "." { + log.Println("\tAttempting to location project root directory from current working directory.") + + var err error + req.GoModFile, err = findProjectGoModFile() + if err != nil { + return nil, err + } + req.ProjectRoot = filepath.Dir(req.GoModFile) + } else { + log.Printf("\t\tUsing supplied project root directory '%s'.\n", req.ProjectRoot) + req.GoModFile = filepath.Join(req.ProjectRoot, "go.mod") + } + log.Printf("\t\t\tproject root: %s", req.ProjectRoot) + log.Printf("\t\t\tgo.mod: %s", req.GoModFile) + } + + log.Println("\tExtracting go module name from go.mod.") + { + var err error + req.GoModName, err = loadGoModName(req.GoModFile) + if err != nil { + return nil, err + } + log.Printf("\t\t\tmodule name: %s", req.GoModName) + } + + log.Println("\tDetermining the project name.") + { + if req.ProjectName != "" { + log.Printf("\t\tUse provided value.") + } else { + req.ProjectName = filepath.Base(req.GoModName) + log.Printf("\t\tSet from go module.") + } + log.Printf("\t\t\tproject name: %s", req.ProjectName) + } + + // Verifies AWS credentials specified as environment variables. + log.Println("\tVerify AWS credentials.") + { + var err error + req.AwsCreds, err = GetAwsCredentials(req.Env) + if err != nil { + return nil, err + } + if req.AwsCreds.UseRole { + log.Printf("\t\t\tUsing role") + } else { + log.Printf("\t\t\tAccessKeyID: '%s'", req.AwsCreds.AccessKeyID) + } + + log.Printf("\t\t\tRegion: '%s'", req.AwsCreds.Region) + log.Printf("\t%s\tAWS credentials valid.", tests.Success) + } + } + + return &req, nil +} + +// Run is the main entrypoint for migration of database schema for a given target environment. +func Migrate(log *log.Logger, req *migrateRequest) error { + + // Load the database details. + var db DB + { + log.Println("Get Database Details from AWS Secret Manager") + + dbId := dBInstanceIdentifier(req.ProjectName, req.Env) + + // Secret ID used to store the DB username and password across deploys. + dbSecretId := secretID(req.ProjectName, req.Env, dbId) + + // Retrieve the current secret value if something is stored. + { + sm := secretsmanager.New(req.awsSession()) + res, err := sm.GetSecretValue(&secretsmanager.GetSecretValueInput{ + SecretId: aws.String(dbSecretId), + }) + if err != nil { + if aerr, ok := err.(awserr.Error); !ok || aerr.Code() != secretsmanager.ErrCodeResourceNotFoundException { + return errors.Wrapf(err, "Failed to get value for secret id %s", dbSecretId) + } + } else { + err = json.Unmarshal([]byte(*res.SecretString), &db) + if err != nil { + return errors.Wrap(err, "Failed to json decode db credentials") + } + } + + log.Printf("\t%s\tDatabase credentials found.", tests.Success) + } + } + + // Start Database and run the migration. + { + log.Println("Proceed with schema migration") + + var dbUrl url.URL + { + // Query parameters. + var q url.Values = make(map[string][]string) + + // Handle SSL Mode + if db.DisableTLS { + q.Set("sslmode", "disable") + } else { + q.Set("sslmode", "require") + } + + // Construct url. + dbUrl = url.URL{ + Scheme: db.Driver, + User: url.UserPassword(db.User, db.Pass), + Host: db.Host, + Path: db.Database, + RawQuery: q.Encode(), + } + } + + log.Printf("\t\tOpen database connection") + // Register informs the sqlxtrace package of the driver that we will be using in our program. + // It uses a default service name, in the below case "postgres.db". To use a custom service + // name use RegisterWithServiceName. + sqltrace.Register(db.Driver, &pq.Driver{}, sqltrace.WithServiceName("devops:migrate")) + masterDb, err := sqlxtrace.Open(db.Driver, dbUrl.String()) + if err != nil { + return errors.WithStack(err) + } + defer masterDb.Close() + + // Start Migrations + log.Printf("\t\tStart migrations.") + if err = schema.Migrate(masterDb, log); err != nil { + return errors.WithStack(err) + } + + log.Printf("\t%s\tMigrate complete.", tests.Success) + } + + return nil +} diff --git a/tools/devops/cmd/cicd/service.go b/tools/devops/cmd/cicd/service.go index 10401a3..272087c 100644 --- a/tools/devops/cmd/cicd/service.go +++ b/tools/devops/cmd/cicd/service.go @@ -157,6 +157,16 @@ func releaseImage(env, serviceName, repositoryUri string) string { return releaseImage } +// dBInstanceIdentifier returns the database name. +func dBInstanceIdentifier(projectName, env string) string { + return projectName + "-" + env +} + +// secretID returns the secret name with a standard prefix. +func secretID(projectName, env, secretName string) string { + return filepath.Join(projectName, env, secretName) +} + // findProjectGoModFile finds the project root directory from the current working directory. func findProjectGoModFile() (string, error) { var err error diff --git a/tools/devops/cmd/cicd/service_build.go b/tools/devops/cmd/cicd/service_build.go index a8db9c5..2f56b47 100644 --- a/tools/devops/cmd/cicd/service_build.go +++ b/tools/devops/cmd/cicd/service_build.go @@ -55,22 +55,22 @@ func NewServiceBuildRequest(log *log.Logger, flags ServiceBuildFlags) (*serviceB log.Printf("\t%s\tFlags ok.", tests.Success) } - // Define new service request. - sr := &serviceRequest{ - ServiceName: flags.ServiceName, - Env: flags.Env, - ProjectRoot: flags.ProjectRoot, - ProjectName: flags.ProjectName, - DockerFile: flags.DockerFile, - } - if err := sr.init(log); err != nil { - return nil, err - } - // Generate a deploy request using CLI flags and AWS credentials. log.Println("Generate deploy request.") var req serviceBuildRequest { + // Define new service request. + sr := &serviceRequest{ + ServiceName: flags.ServiceName, + Env: flags.Env, + ProjectRoot: flags.ProjectRoot, + ProjectName: flags.ProjectName, + DockerFile: flags.DockerFile, + } + if err := sr.init(log); err != nil { + return nil, err + } + req = serviceBuildRequest{ serviceRequest: sr, diff --git a/tools/devops/cmd/cicd/service_deploy.go b/tools/devops/cmd/cicd/service_deploy.go index f7c6ebf..8af7bf3 100644 --- a/tools/devops/cmd/cicd/service_deploy.go +++ b/tools/devops/cmd/cicd/service_deploy.go @@ -144,22 +144,22 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic log.Printf("\t%s\tFlags ok.", tests.Success) } - // Define new service request. - sr := &serviceRequest{ - ServiceName: flags.ServiceName, - Env: flags.Env, - ProjectRoot: flags.ProjectRoot, - ProjectName: flags.ProjectName, - DockerFile: flags.DockerFile, - } - if err := sr.init(log); err != nil { - return nil, err - } - // Generate a deploy request using CLI flags and AWS credentials. log.Println("Generate deploy request.") var req serviceDeployRequest { + // Define new service request. + sr := &serviceRequest{ + ServiceName: flags.ServiceName, + Env: flags.Env, + ProjectRoot: flags.ProjectRoot, + ProjectName: flags.ProjectName, + DockerFile: flags.DockerFile, + } + if err := sr.init(log); err != nil { + return nil, err + } + req = serviceDeployRequest{ serviceRequest: sr, @@ -718,7 +718,7 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic // RDS settings for a Postgres database Instance. Could defined different settings by env. req.DBInstance = &rds.CreateDBInstanceInput{ - DBInstanceIdentifier: aws.String(req.ProjectName + "-" + req.Env), + DBInstanceIdentifier: aws.String(dBInstanceIdentifier(req.ProjectName, req.Env)), DBName: aws.String("shared"), Engine: aws.String("postgres"), MasterUsername: aws.String("god"), @@ -799,7 +799,7 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error { // 2. Check AWS Secrets Manager for datadog entry prefixed with target environment. if datadogApiKey == "" { - prefixedSecretId := strings.ToUpper(req.Env) + "/DATADOG" + prefixedSecretId := secretID(req.ProjectName, req.Env, "datadog") var err error datadogApiKey, err = GetAwsSecretValue(req.AwsCreds, prefixedSecretId) if err != nil { @@ -1164,7 +1164,7 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error { log.Println("RDS - Get or Create Database Instance") // Secret ID used to store the DB username and password across deploys. - dbSecretId := filepath.Join(req.ProjectName, req.Env, *req.DBInstance.DBInstanceIdentifier) + dbSecretId := secretID(req.ProjectName, req.Env, *req.DBInstance.DBInstanceIdentifier) // Retrieve the current secret value if something is stored. { diff --git a/tools/devops/main.go b/tools/devops/main.go index 866b2dc..2021bad 100644 --- a/tools/devops/main.go +++ b/tools/devops/main.go @@ -39,8 +39,9 @@ func main() { // Start Truss var ( - buildFlags cicd.ServiceBuildFlags - deployFlags cicd.ServiceDeployFlags + buildFlags cicd.ServiceBuildFlags + deployFlags cicd.ServiceDeployFlags + migrateFlags cicd.MigrateFlags ) app := cli.NewApp() @@ -110,10 +111,17 @@ func main() { { Name: "migrate", Usage: "-env=dev", - Flags: []cli.Flag{}, + Flags: []cli.Flag{ + cli.StringFlag{Name: "env", Usage: "dev, stage, or prod", Destination: &migrateFlags.Env}, + cli.StringFlag{Name: "root", Usage: "project root directory", Destination: &migrateFlags.ProjectRoot}, + cli.StringFlag{Name: "project", Usage: "name of project", Destination: &migrateFlags.ProjectName}, + }, Action: func(c *cli.Context) error { - - return nil + req, err := cicd.NewMigrateRequest(log, migrateFlags) + if err != nil { + return err + } + return cicd.Migrate(log, req) }, }, } diff --git a/tools/schema/main.go b/tools/schema/main.go index 2c3d821..20a43ca 100644 --- a/tools/schema/main.go +++ b/tools/schema/main.go @@ -53,19 +53,6 @@ func main() { Timezone string `default:"utc" envconfig:"TIMEZONE"` DisableTLS bool `default:"true" envconfig:"DISABLE_TLS"` } - Project struct { - Name string `default:"saas-starter-kit" envconfig:"Name"` - } - Aws struct { - AccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"` // WEB_API_AWS_AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY_ID - SecretAccessKey string `envconfig:"AWS_SECRET_ACCESS_KEY" json:"-"` // don't print - Region string `default:"us-east-1" envconfig:"AWS_REGION"` - - // Get an AWS session from an implicit source if no explicit - // configuration is provided. This is useful for taking advantage of - // EC2/ECS instance roles. - UseRole bool `envconfig:"AWS_USE_ROLE"` - } } // For additional details refer to https://github.com/kelseyhightower/envconfig @@ -80,16 +67,6 @@ func main() { return // We displayed help. } - /* - // - DBInstanceIdentifier: aws.String(req.ProjectName + "-" + req.Env), - - - Secret ID used to store the DB username and password across deploys. - dbSecretId := filepath.Join(req.ProjectName, req.Env, *req.DBInstance.DBInstanceIdentifier) - - */ - // ========================================================================= // Log App Info