added entry point for schema migration to devops that loads db creds

from aws secrets manager

.gitlab-ci.yml

@@ -50,28 +50,35 @@ db:migrate:dev:
     - master
     - dev
     - /^dev-.*$/
-
   variables:
     TARGET_ENV: 'dev'
+    AWS_USE_ROLE: 'true'
 
 webapi:build:dev:
   <<: *build_tmpl
   stage: build:dev
   tags:
     - dev
-  variables:
-    TARGET_ENV: 'dev'
-    SERVICE: 'web-api'
-    AWS_USE_ROLE: 'true'
   only:
     - master
     - dev
     - dev-web-api
+  variables:
+    TARGET_ENV: 'dev'
+    SERVICE: 'web-api'
+    AWS_USE_ROLE: 'true'
 webapi:deploy:dev:
   <<: *deploy_tmpl
   stage: deploy:dev
   tags:
     - dev
+  only:
+    - master
+    - dev
+    - dev-web-api
+  dependencies:
+    - 'webapi:build:dev'
+  #    - 'db:migrate:dev'
   variables:
     TARGET_ENV: 'dev'
     SERVICE: 'web-api'
@@ -82,13 +89,8 @@ webapi:deploy:dev:
     S3_BUCKET_PRIVATE: 'saas-starter-kit-private'
     S3_BUCKET_PUBLIC: 'saas-starter-kit-public'
     AWS_USE_ROLE: 'true'
-  dependencies:
+
-    - 'webapi:build:dev'
+
-#    - 'db:migrate:dev'
-  only:
-    - master
-    - dev
-    - dev-web-api
 
 
 #ddlogscollector:deploy:stage:

tools/devops/cmd/cicd/migrate.go

@@ -0,0 +1,228 @@
+package cicd
+
+import (
+	"encoding/json"
+	"log"
+	"net/url"
+	"path/filepath"
+
+	"geeks-accelerator/oss/saas-starter-kit/internal/platform/tests"
+	"geeks-accelerator/oss/saas-starter-kit/internal/schema"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/secretsmanager"
+	"github.com/lib/pq"
+	_ "github.com/lib/pq"
+	"github.com/pkg/errors"
+	sqltrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/database/sql"
+	sqlxtrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/jmoiron/sqlx"
+	"gopkg.in/go-playground/validator.v9"
+)
+
+// MigrateFlags defines the flags used for executing schema migration.
+type MigrateFlags struct {
+	// Required flags.
+	Env string `validate:"oneof=dev stage prod" example:"dev"`
+
+	// Optional flags.
+	ProjectRoot string `validate:"omitempty" example:"."`
+	ProjectName string ` validate:"omitempty" example:"example-project"`
+}
+
+// migrateRequest defines the details needed to execute a service build.
+type migrateRequest struct {
+	Env         string `validate:"oneof=dev stage prod"`
+	ProjectRoot string `validate:"required"`
+	ProjectName string `validate:"required"`
+	GoModFile   string `validate:"required"`
+	GoModName   string `validate:"required"`
+
+	AwsCreds    awsCredentials `validate:"required,dive,required"`
+	_awsSession *session.Session
+
+	flags MigrateFlags
+}
+
+// awsSession returns the current AWS session for the serviceDeployRequest.
+func (r *migrateRequest) awsSession() *session.Session {
+	if r._awsSession == nil {
+		r._awsSession = r.AwsCreds.Session()
+	}
+
+	return r._awsSession
+}
+
+// NewMigrateRequest generates a new request for executing schema migration for a given set of CLI flags.
+func NewMigrateRequest(log *log.Logger, flags MigrateFlags) (*migrateRequest, error) {
+
+	// Validates specified CLI flags map to struct successfully.
+	log.Println("Validate flags.")
+	{
+		errs := validator.New().Struct(flags)
+		if errs != nil {
+			return nil, errs
+		}
+		log.Printf("\t%s\tFlags ok.", tests.Success)
+	}
+
+	// Generate a migrate request using CLI flags and AWS credentials.
+	log.Println("Generate migrate request.")
+	var req migrateRequest
+	{
+
+		// Define new migrate request.
+		req = migrateRequest{
+			Env:         flags.Env,
+			ProjectRoot: flags.ProjectRoot,
+			ProjectName: flags.ProjectName,
+
+			flags: flags,
+		}
+
+		// When project root directory is empty or set to current working path, then search for the project root by locating
+		// the go.mod file.
+		log.Println("\tDetermining the project root directory.")
+		{
+			if req.ProjectRoot == "" || req.ProjectRoot == "." {
+				log.Println("\tAttempting to location project root directory from current working directory.")
+
+				var err error
+				req.GoModFile, err = findProjectGoModFile()
+				if err != nil {
+					return nil, err
+				}
+				req.ProjectRoot = filepath.Dir(req.GoModFile)
+			} else {
+				log.Printf("\t\tUsing supplied project root directory '%s'.\n", req.ProjectRoot)
+				req.GoModFile = filepath.Join(req.ProjectRoot, "go.mod")
+			}
+			log.Printf("\t\t\tproject root: %s", req.ProjectRoot)
+			log.Printf("\t\t\tgo.mod: %s", req.GoModFile)
+		}
+
+		log.Println("\tExtracting go module name from go.mod.")
+		{
+			var err error
+			req.GoModName, err = loadGoModName(req.GoModFile)
+			if err != nil {
+				return nil, err
+			}
+			log.Printf("\t\t\tmodule name: %s", req.GoModName)
+		}
+
+		log.Println("\tDetermining the project name.")
+		{
+			if req.ProjectName != "" {
+				log.Printf("\t\tUse provided value.")
+			} else {
+				req.ProjectName = filepath.Base(req.GoModName)
+				log.Printf("\t\tSet from go module.")
+			}
+			log.Printf("\t\t\tproject name: %s", req.ProjectName)
+		}
+
+		// Verifies AWS credentials specified as environment variables.
+		log.Println("\tVerify AWS credentials.")
+		{
+			var err error
+			req.AwsCreds, err = GetAwsCredentials(req.Env)
+			if err != nil {
+				return nil, err
+			}
+			if req.AwsCreds.UseRole {
+				log.Printf("\t\t\tUsing role")
+			} else {
+				log.Printf("\t\t\tAccessKeyID: '%s'", req.AwsCreds.AccessKeyID)
+			}
+
+			log.Printf("\t\t\tRegion: '%s'", req.AwsCreds.Region)
+			log.Printf("\t%s\tAWS credentials valid.", tests.Success)
+		}
+	}
+
+	return &req, nil
+}
+
+// Run is the main entrypoint for migration of database schema for a given target environment.
+func Migrate(log *log.Logger, req *migrateRequest) error {
+
+	// Load the database details.
+	var db DB
+	{
+		log.Println("Get Database Details from AWS Secret Manager")
+
+		dbId := dBInstanceIdentifier(req.ProjectName, req.Env)
+
+		// Secret ID used to store the DB username and password across deploys.
+		dbSecretId := secretID(req.ProjectName, req.Env, dbId)
+
+		// Retrieve the current secret value if something is stored.
+		{
+			sm := secretsmanager.New(req.awsSession())
+			res, err := sm.GetSecretValue(&secretsmanager.GetSecretValueInput{
+				SecretId: aws.String(dbSecretId),
+			})
+			if err != nil {
+				if aerr, ok := err.(awserr.Error); !ok || aerr.Code() != secretsmanager.ErrCodeResourceNotFoundException {
+					return errors.Wrapf(err, "Failed to get value for secret id %s", dbSecretId)
+				}
+			} else {
+				err = json.Unmarshal([]byte(*res.SecretString), &db)
+				if err != nil {
+					return errors.Wrap(err, "Failed to json decode db credentials")
+				}
+			}
+
+			log.Printf("\t%s\tDatabase credentials found.", tests.Success)
+		}
+	}
+
+	// Start Database and run the migration.
+	{
+		log.Println("Proceed with schema migration")
+
+		var dbUrl url.URL
+		{
+			// Query parameters.
+			var q url.Values = make(map[string][]string)
+
+			// Handle SSL Mode
+			if db.DisableTLS {
+				q.Set("sslmode", "disable")
+			} else {
+				q.Set("sslmode", "require")
+			}
+
+			// Construct url.
+			dbUrl = url.URL{
+				Scheme:   db.Driver,
+				User:     url.UserPassword(db.User, db.Pass),
+				Host:     db.Host,
+				Path:     db.Database,
+				RawQuery: q.Encode(),
+			}
+		}
+
+		log.Printf("\t\tOpen database connection")
+		// Register informs the sqlxtrace package of the driver that we will be using in our program.
+		// It uses a default service name, in the below case "postgres.db". To use a custom service
+		// name use RegisterWithServiceName.
+		sqltrace.Register(db.Driver, &pq.Driver{}, sqltrace.WithServiceName("devops:migrate"))
+		masterDb, err := sqlxtrace.Open(db.Driver, dbUrl.String())
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		defer masterDb.Close()
+
+		// Start Migrations
+		log.Printf("\t\tStart migrations.")
+		if err = schema.Migrate(masterDb, log); err != nil {
+			return errors.WithStack(err)
+		}
+
+		log.Printf("\t%s\tMigrate complete.", tests.Success)
+	}
+
+	return nil
+}

tools/devops/cmd/cicd/service.go

@@ -157,6 +157,16 @@ func releaseImage(env, serviceName, repositoryUri string) string {
 	return releaseImage
 }
 
+// dBInstanceIdentifier returns the database name.
+func dBInstanceIdentifier(projectName, env string) string {
+	return projectName + "-" + env
+}
+
+// secretID returns the secret name with a standard prefix.
+func secretID(projectName, env, secretName string) string {
+	return filepath.Join(projectName, env, secretName)
+}
+
 // findProjectGoModFile finds the project root directory from the current working directory.
 func findProjectGoModFile() (string, error) {
 	var err error

tools/devops/cmd/cicd/service_build.go

@@ -55,22 +55,22 @@ func NewServiceBuildRequest(log *log.Logger, flags ServiceBuildFlags) (*serviceB
 		log.Printf("\t%s\tFlags ok.", tests.Success)
 	}
 
-	// Define new service request.
-	sr := &serviceRequest{
-		ServiceName: flags.ServiceName,
-		Env:         flags.Env,
-		ProjectRoot: flags.ProjectRoot,
-		ProjectName: flags.ProjectName,
-		DockerFile:  flags.DockerFile,
-	}
-	if err := sr.init(log); err != nil {
-		return nil, err
-	}
-
 	// Generate a deploy request using CLI flags and AWS credentials.
 	log.Println("Generate deploy request.")
 	var req serviceBuildRequest
 	{
+		// Define new service request.
+		sr := &serviceRequest{
+			ServiceName: flags.ServiceName,
+			Env:         flags.Env,
+			ProjectRoot: flags.ProjectRoot,
+			ProjectName: flags.ProjectName,
+			DockerFile:  flags.DockerFile,
+		}
+		if err := sr.init(log); err != nil {
+			return nil, err
+		}
+
 		req = serviceBuildRequest{
 			serviceRequest: sr,
 

tools/devops/cmd/cicd/service_deploy.go

@@ -144,22 +144,22 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
 		log.Printf("\t%s\tFlags ok.", tests.Success)
 	}
 
-	// Define new service request.
-	sr := &serviceRequest{
-		ServiceName: flags.ServiceName,
-		Env:         flags.Env,
-		ProjectRoot: flags.ProjectRoot,
-		ProjectName: flags.ProjectName,
-		DockerFile:  flags.DockerFile,
-	}
-	if err := sr.init(log); err != nil {
-		return nil, err
-	}
-
 	// Generate a deploy request using CLI flags and AWS credentials.
 	log.Println("Generate deploy request.")
 	var req serviceDeployRequest
 	{
+		// Define new service request.
+		sr := &serviceRequest{
+			ServiceName: flags.ServiceName,
+			Env:         flags.Env,
+			ProjectRoot: flags.ProjectRoot,
+			ProjectName: flags.ProjectName,
+			DockerFile:  flags.DockerFile,
+		}
+		if err := sr.init(log); err != nil {
+			return nil, err
+		}
+
 		req = serviceDeployRequest{
 			serviceRequest: sr,
 
@@ -718,7 +718,7 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
 
 			// RDS settings for a Postgres database Instance. Could defined different settings by env.
 			req.DBInstance = &rds.CreateDBInstanceInput{
-				DBInstanceIdentifier:      aws.String(req.ProjectName + "-" + req.Env),
+				DBInstanceIdentifier:      aws.String(dBInstanceIdentifier(req.ProjectName, req.Env)),
 				DBName:                    aws.String("shared"),
 				Engine:                    aws.String("postgres"),
 				MasterUsername:            aws.String("god"),
@@ -799,7 +799,7 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 
 		// 2. Check AWS Secrets Manager for datadog entry prefixed with target environment.
 		if datadogApiKey == "" {
-			prefixedSecretId := strings.ToUpper(req.Env) + "/DATADOG"
+			prefixedSecretId := secretID(req.ProjectName, req.Env, "datadog")
 			var err error
 			datadogApiKey, err = GetAwsSecretValue(req.AwsCreds, prefixedSecretId)
 			if err != nil {
@@ -1164,7 +1164,7 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 		log.Println("RDS - Get or Create Database Instance")
 
 		// Secret ID used to store the DB username and password across deploys.
-		dbSecretId := filepath.Join(req.ProjectName, req.Env, *req.DBInstance.DBInstanceIdentifier)
+		dbSecretId := secretID(req.ProjectName, req.Env, *req.DBInstance.DBInstanceIdentifier)
 
 		// Retrieve the current secret value if something is stored.
 		{

tools/devops/main.go

@@ -39,8 +39,9 @@ func main() {
 	// Start Truss
 
 	var (
-		buildFlags  cicd.ServiceBuildFlags
+		buildFlags   cicd.ServiceBuildFlags
-		deployFlags cicd.ServiceDeployFlags
+		deployFlags  cicd.ServiceDeployFlags
+		migrateFlags cicd.MigrateFlags
 	)
 
 	app := cli.NewApp()
@@ -110,10 +111,17 @@ func main() {
 		{
 			Name:  "migrate",
 			Usage: "-env=dev",
-			Flags: []cli.Flag{},
+			Flags: []cli.Flag{
+				cli.StringFlag{Name: "env", Usage: "dev, stage, or prod", Destination: &migrateFlags.Env},
+				cli.StringFlag{Name: "root", Usage: "project root directory", Destination: &migrateFlags.ProjectRoot},
+				cli.StringFlag{Name: "project", Usage: "name of project", Destination: &migrateFlags.ProjectName},
+			},
 			Action: func(c *cli.Context) error {
-
+				req, err := cicd.NewMigrateRequest(log, migrateFlags)
-				return nil
+				if err != nil {
+					return err
+				}
+				return cicd.Migrate(log, req)
 			},
 		},
 	}

tools/schema/main.go

@@ -53,19 +53,6 @@ func main() {
 			Timezone   string `default:"utc" envconfig:"TIMEZONE"`
 			DisableTLS bool   `default:"true" envconfig:"DISABLE_TLS"`
 		}
-		Project struct {
-			Name string `default:"saas-starter-kit" envconfig:"Name"`
-		}
-		Aws struct {
-			AccessKeyID     string `envconfig:"AWS_ACCESS_KEY_ID"`              // WEB_API_AWS_AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY_ID
-			SecretAccessKey string `envconfig:"AWS_SECRET_ACCESS_KEY" json:"-"` // don't print
-			Region          string `default:"us-east-1" envconfig:"AWS_REGION"`
-
-			// Get an AWS session from an implicit source if no explicit
-			// configuration is provided. This is useful for taking advantage of
-			// EC2/ECS instance roles.
-			UseRole bool `envconfig:"AWS_USE_ROLE"`
-		}
 	}
 
 	// For additional details refer to https://github.com/kelseyhightower/envconfig
@@ -80,16 +67,6 @@ func main() {
 		return // We displayed help.
 	}
 
-	/*
-		//
-		DBInstanceIdentifier:      aws.String(req.ProjectName + "-" + req.Env),
-
-
-		Secret ID used to store the DB username and password across deploys.
-				dbSecretId := filepath.Join(req.ProjectName, req.Env, *req.DBInstance.DBInstanceIdentifier)
-
-	*/
-
 	// =========================================================================
 	// Log App Info