go/golang-saas-starter-kit
1
0
Fork 0
mirror of https://github.com/raseels-repos/golang-saas-starter-kit.git synced 2025-06-10 23:57:45 +02:00
Code Issues Releases Activity

added entry point for schema migration to devops that loads db creds

Browse Source 
from aws secrets manager
This commit is contained in:
Lee Brown 2019-07-14 15:33:23 -08:00
parent 1a36b52113
commit e1b3377e88
7 changed files with 292 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
.gitlab-ci.yml
View File

@ -50,28 +50,35 @@ db:migrate:dev:
- master
- dev
- /^dev-.*$/
variables:
TARGET_ENV: 'dev'
AWS_USE_ROLE: 'true'
webapi:build:dev:
<<: *build_tmpl
stage: build:dev
tags:
- dev
variables:
TARGET_ENV: 'dev'
SERVICE: 'web-api'
AWS_USE_ROLE: 'true'
only:
- master
- dev
- dev-web-api
variables:
TARGET_ENV: 'dev'
SERVICE: 'web-api'
AWS_USE_ROLE: 'true'
webapi:deploy:dev:
<<: *deploy_tmpl
stage: deploy:dev
tags:
- dev
only:
- master
- dev
- dev-web-api
dependencies:
- 'webapi:build:dev'
# - 'db:migrate:dev'
variables:
TARGET_ENV: 'dev'
SERVICE: 'web-api'
@ -82,13 +89,8 @@ webapi:deploy:dev:
S3_BUCKET_PRIVATE: 'saas-starter-kit-private'
S3_BUCKET_PUBLIC: 'saas-starter-kit-public'
AWS_USE_ROLE: 'true'
dependencies:
- 'webapi:build:dev'
# - 'db:migrate:dev'
only:
- master
- dev
- dev-web-api
#ddlogscollector:deploy:stage:

228
tools/devops/cmd/cicd/migrate.go Normal file
View File

@ -0,0 +1,228 @@
package cicd
import (
"encoding/json"
"log"
"net/url"
"path/filepath"
"geeks-accelerator/oss/saas-starter-kit/internal/platform/tests"
"geeks-accelerator/oss/saas-starter-kit/internal/schema"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/secretsmanager"
"github.com/lib/pq"
_ "github.com/lib/pq"
"github.com/pkg/errors"
sqltrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/database/sql"
sqlxtrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/jmoiron/sqlx"
"gopkg.in/go-playground/validator.v9"
)
// MigrateFlags defines the flags used for executing schema migration.
type MigrateFlags struct {
// Required flags.
Env string `validate:"oneof=dev stage prod" example:"dev"`
// Optional flags.
ProjectRoot string `validate:"omitempty" example:"."`
ProjectName string ` validate:"omitempty" example:"example-project"`
}
// migrateRequest defines the details needed to execute a service build.
type migrateRequest struct {
Env string `validate:"oneof=dev stage prod"`
ProjectRoot string `validate:"required"`
ProjectName string `validate:"required"`
GoModFile string `validate:"required"`
GoModName string `validate:"required"`
AwsCreds awsCredentials `validate:"required,dive,required"`
_awsSession *session.Session
flags MigrateFlags
}
// awsSession returns the current AWS session for the serviceDeployRequest.
func (r *migrateRequest) awsSession() *session.Session {
if r._awsSession == nil {
r._awsSession = r.AwsCreds.Session()
}
return r._awsSession
}
// NewMigrateRequest generates a new request for executing schema migration for a given set of CLI flags.
func NewMigrateRequest(log *log.Logger, flags MigrateFlags) (*migrateRequest, error) {
// Validates specified CLI flags map to struct successfully.
log.Println("Validate flags.")
{
errs := validator.New().Struct(flags)
if errs != nil {
return nil, errs
}
log.Printf("\t%s\tFlags ok.", tests.Success)
}
// Generate a migrate request using CLI flags and AWS credentials.
log.Println("Generate migrate request.")
var req migrateRequest
{
// Define new migrate request.
req = migrateRequest{
Env: flags.Env,
ProjectRoot: flags.ProjectRoot,
ProjectName: flags.ProjectName,
flags: flags,
}
// When project root directory is empty or set to current working path, then search for the project root by locating
// the go.mod file.
log.Println("\tDetermining the project root directory.")
{
if req.ProjectRoot == "" || req.ProjectRoot == "." {
log.Println("\tAttempting to location project root directory from current working directory.")
var err error
req.GoModFile, err = findProjectGoModFile()
if err != nil {
return nil, err
}
req.ProjectRoot = filepath.Dir(req.GoModFile)
} else {
log.Printf("\t\tUsing supplied project root directory '%s'.\n", req.ProjectRoot)
req.GoModFile = filepath.Join(req.ProjectRoot, "go.mod")
}
log.Printf("\t\t\tproject root: %s", req.ProjectRoot)
log.Printf("\t\t\tgo.mod: %s", req.GoModFile)
}
log.Println("\tExtracting go module name from go.mod.")
{
var err error
req.GoModName, err = loadGoModName(req.GoModFile)
if err != nil {
return nil, err
}
log.Printf("\t\t\tmodule name: %s", req.GoModName)
}
log.Println("\tDetermining the project name.")
{
if req.ProjectName != "" {
log.Printf("\t\tUse provided value.")
} else {
req.ProjectName = filepath.Base(req.GoModName)
log.Printf("\t\tSet from go module.")
}
log.Printf("\t\t\tproject name: %s", req.ProjectName)
}
// Verifies AWS credentials specified as environment variables.
log.Println("\tVerify AWS credentials.")
{
var err error
req.AwsCreds, err = GetAwsCredentials(req.Env)
if err != nil {
return nil, err
}
if req.AwsCreds.UseRole {
log.Printf("\t\t\tUsing role")
} else {
log.Printf("\t\t\tAccessKeyID: '%s'", req.AwsCreds.AccessKeyID)
}
log.Printf("\t\t\tRegion: '%s'", req.AwsCreds.Region)
log.Printf("\t%s\tAWS credentials valid.", tests.Success)
}
}
return &req, nil
}
// Run is the main entrypoint for migration of database schema for a given target environment.
func Migrate(log *log.Logger, req *migrateRequest) error {
// Load the database details.
var db DB
{
log.Println("Get Database Details from AWS Secret Manager")
dbId := dBInstanceIdentifier(req.ProjectName, req.Env)
// Secret ID used to store the DB username and password across deploys.
dbSecretId := secretID(req.ProjectName, req.Env, dbId)
// Retrieve the current secret value if something is stored.
{
sm := secretsmanager.New(req.awsSession())
res, err := sm.GetSecretValue(&secretsmanager.GetSecretValueInput{
SecretId: aws.String(dbSecretId),
})
if err != nil {
if aerr, ok := err.(awserr.Error); !ok || aerr.Code() != secretsmanager.ErrCodeResourceNotFoundException {
return errors.Wrapf(err, "Failed to get value for secret id %s", dbSecretId)
}
} else {
err = json.Unmarshal([]byte(*res.SecretString), &db)
if err != nil {
return errors.Wrap(err, "Failed to json decode db credentials")
}
}
log.Printf("\t%s\tDatabase credentials found.", tests.Success)
}
}
// Start Database and run the migration.
{
log.Println("Proceed with schema migration")
var dbUrl url.URL
{
// Query parameters.
var q url.Values = make(map[string][]string)
// Handle SSL Mode
if db.DisableTLS {
q.Set("sslmode", "disable")
} else {
q.Set("sslmode", "require")
}
// Construct url.
dbUrl = url.URL{
Scheme: db.Driver,
User: url.UserPassword(db.User, db.Pass),
Host: db.Host,
Path: db.Database,
RawQuery: q.Encode(),
}
}
log.Printf("\t\tOpen database connection")
// Register informs the sqlxtrace package of the driver that we will be using in our program.
// It uses a default service name, in the below case "postgres.db". To use a custom service
// name use RegisterWithServiceName.
sqltrace.Register(db.Driver, &pq.Driver{}, sqltrace.WithServiceName("devops:migrate"))
masterDb, err := sqlxtrace.Open(db.Driver, dbUrl.String())
if err != nil {
return errors.WithStack(err)
}
defer masterDb.Close()
// Start Migrations
log.Printf("\t\tStart migrations.")
if err = schema.Migrate(masterDb, log); err != nil {
return errors.WithStack(err)
}
log.Printf("\t%s\tMigrate complete.", tests.Success)
}
return nil
}

10
tools/devops/cmd/cicd/service.go
View File

@ -157,6 +157,16 @@ func releaseImage(env, serviceName, repositoryUri string) string {
return releaseImage
}
// dBInstanceIdentifier returns the database name.
func dBInstanceIdentifier(projectName, env string) string {
return projectName + "-" + env
}
// secretID returns the secret name with a standard prefix.
func secretID(projectName, env, secretName string) string {
return filepath.Join(projectName, env, secretName)
}
// findProjectGoModFile finds the project root directory from the current working directory.
func findProjectGoModFile() (string, error) {
var err error

24
tools/devops/cmd/cicd/service_build.go
View File

@ -55,22 +55,22 @@ func NewServiceBuildRequest(log *log.Logger, flags ServiceBuildFlags) (*serviceB
log.Printf("\t%s\tFlags ok.", tests.Success)
}
// Define new service request.
sr := &serviceRequest{
ServiceName: flags.ServiceName,
Env: flags.Env,
ProjectRoot: flags.ProjectRoot,
ProjectName: flags.ProjectName,
DockerFile: flags.DockerFile,
}
if err := sr.init(log); err != nil {
return nil, err
}
// Generate a deploy request using CLI flags and AWS credentials.
log.Println("Generate deploy request.")
var req serviceBuildRequest
{
// Define new service request.
sr := &serviceRequest{
ServiceName: flags.ServiceName,
Env: flags.Env,
ProjectRoot: flags.ProjectRoot,
ProjectName: flags.ProjectName,
DockerFile: flags.DockerFile,
}
if err := sr.init(log); err != nil {
return nil, err
}
req = serviceBuildRequest{
serviceRequest: sr,

30
tools/devops/cmd/cicd/service_deploy.go
View File

@ -144,22 +144,22 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
log.Printf("\t%s\tFlags ok.", tests.Success)
}
// Define new service request.
sr := &serviceRequest{
ServiceName: flags.ServiceName,
Env: flags.Env,
ProjectRoot: flags.ProjectRoot,
ProjectName: flags.ProjectName,
DockerFile: flags.DockerFile,
}
if err := sr.init(log); err != nil {
return nil, err
}
// Generate a deploy request using CLI flags and AWS credentials.
log.Println("Generate deploy request.")
var req serviceDeployRequest
{
// Define new service request.
sr := &serviceRequest{
ServiceName: flags.ServiceName,
Env: flags.Env,
ProjectRoot: flags.ProjectRoot,
ProjectName: flags.ProjectName,
DockerFile: flags.DockerFile,
}
if err := sr.init(log); err != nil {
return nil, err
}
req = serviceDeployRequest{
serviceRequest: sr,
@ -718,7 +718,7 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
// RDS settings for a Postgres database Instance. Could defined different settings by env.
req.DBInstance = &rds.CreateDBInstanceInput{
DBInstanceIdentifier: aws.String(req.ProjectName + "-" + req.Env),
DBInstanceIdentifier: aws.String(dBInstanceIdentifier(req.ProjectName, req.Env)),
DBName: aws.String("shared"),
Engine: aws.String("postgres"),
MasterUsername: aws.String("god"),
@ -799,7 +799,7 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
// 2. Check AWS Secrets Manager for datadog entry prefixed with target environment.
if datadogApiKey == "" {
prefixedSecretId := strings.ToUpper(req.Env) + "/DATADOG"
prefixedSecretId := secretID(req.ProjectName, req.Env, "datadog")
var err error
datadogApiKey, err = GetAwsSecretValue(req.AwsCreds, prefixedSecretId)
if err != nil {
@ -1164,7 +1164,7 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
log.Println("RDS - Get or Create Database Instance")
// Secret ID used to store the DB username and password across deploys.
dbSecretId := filepath.Join(req.ProjectName, req.Env, *req.DBInstance.DBInstanceIdentifier)
dbSecretId := secretID(req.ProjectName, req.Env, *req.DBInstance.DBInstanceIdentifier)
// Retrieve the current secret value if something is stored.
{

18
tools/devops/main.go
View File

@ -39,8 +39,9 @@ func main() {
// Start Truss
var (
buildFlags cicd.ServiceBuildFlags
deployFlags cicd.ServiceDeployFlags
buildFlags cicd.ServiceBuildFlags
deployFlags cicd.ServiceDeployFlags
migrateFlags cicd.MigrateFlags
)
app := cli.NewApp()
@ -110,10 +111,17 @@ func main() {
{
Name: "migrate",
Usage: "-env=dev",
Flags: []cli.Flag{},
Flags: []cli.Flag{
cli.StringFlag{Name: "env", Usage: "dev, stage, or prod", Destination: &migrateFlags.Env},
cli.StringFlag{Name: "root", Usage: "project root directory", Destination: &migrateFlags.ProjectRoot},
cli.StringFlag{Name: "project", Usage: "name of project", Destination: &migrateFlags.ProjectName},
},
Action: func(c *cli.Context) error {
return nil
req, err := cicd.NewMigrateRequest(log, migrateFlags)
if err != nil {
return err
}
return cicd.Migrate(log, req)
},
},
}

23
tools/schema/main.go
View File

@ -53,19 +53,6 @@ func main() {
Timezone string `default:"utc" envconfig:"TIMEZONE"`
DisableTLS bool `default:"true" envconfig:"DISABLE_TLS"`
}
Project struct {
Name string `default:"saas-starter-kit" envconfig:"Name"`
}
Aws struct {
AccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"` // WEB_API_AWS_AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY_ID
SecretAccessKey string `envconfig:"AWS_SECRET_ACCESS_KEY" json:"-"` // don't print
Region string `default:"us-east-1" envconfig:"AWS_REGION"`
// Get an AWS session from an implicit source if no explicit
// configuration is provided. This is useful for taking advantage of
// EC2/ECS instance roles.
UseRole bool `envconfig:"AWS_USE_ROLE"`
}
}
// For additional details refer to https://github.com/kelseyhightower/envconfig
@ -80,16 +67,6 @@ func main() {
return // We displayed help.
}
/*
//
DBInstanceIdentifier: aws.String(req.ProjectName + "-" + req.Env),
Secret ID used to store the DB username and password across deploys.
dbSecretId := filepath.Join(req.ProjectName, req.Env, *req.DBInstance.DBInstanceIdentifier)
*/
// =========================================================================
// Log App Info