1
0
mirror of https://github.com/raseels-repos/golang-saas-starter-kit.git synced 2025-06-08 23:56:37 +02:00

checkpoint

This commit is contained in:
Lee Brown 2019-07-10 04:52:38 -08:00
parent 7e98e9d839
commit acd1bca501

View File

@ -381,11 +381,12 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
startTime := time.Now() startTime := time.Now()
// Load the ECR repository. // Load the AWS ECR repository. Try to find by name else create new one.
log.Println("ECR - Get or create repository.")
var docker *dockerClient.Client var docker *dockerClient.Client
var registryAuth string var registryAuth string
{ {
log.Println("ECR - Get or create repository.")
svc := ecr.New(req.awsSession()) svc := ecr.New(req.awsSession())
var awsRepo *ecr.Repository var awsRepo *ecr.Repository
@ -575,9 +576,11 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
return nil return nil
} }
log.Println("Datadog - Get API Key") // Try to find the datadog API Key, this value is optional.
var datadogApiKey string var datadogApiKey string
{ {
log.Println("Datadog - Get API Key")
// Load Datadog API Key which can be either stored in an env var or in AWS Secrets Manager. // Load Datadog API Key which can be either stored in an env var or in AWS Secrets Manager.
// 1. Check env vars for [DEV|STAGE|PROD]_DD_API_KEY and DD_API_KEY // 1. Check env vars for [DEV|STAGE|PROD]_DD_API_KEY and DD_API_KEY
datadogApiKey = getTargetEnv(req.Env, "DD_API_KEY") datadogApiKey = getTargetEnv(req.Env, "DD_API_KEY")
@ -613,8 +616,10 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
} }
} }
log.Println("CloudWatch Logs - Get or Create Log Group") // Try to find the AWS Cloudwatch Log Group by name or create new one.
{ {
log.Println("CloudWatch Logs - Get or Create Log Group")
svc := cloudwatchlogs.New(req.awsSession()) svc := cloudwatchlogs.New(req.awsSession())
// If no log group was found, create one. // If no log group was found, create one.
@ -639,8 +644,10 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
log.Printf("\t%s\tUsing Log Group '%s'.\n", tests.Success, req.CloudWatchLogGroupName) log.Printf("\t%s\tUsing Log Group '%s'.\n", tests.Success, req.CloudWatchLogGroupName)
} }
log.Println("S3 - Setup Buckets") // Try to find the AWS S3 Buckets by names or create new ones.
{ {
log.Println("S3 - Setup Buckets")
svc := s3.New(req.awsSession()) svc := s3.New(req.awsSession())
var bucketNames []string var bucketNames []string
@ -840,10 +847,12 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
log.Printf("\t%s\tBuckets setup.\n", tests.Success) log.Printf("\t%s\tBuckets setup.\n", tests.Success)
} }
log.Println("EC2 - Find Subnets") // Find the default VPC and associated subnets. Custom subnets outside of the default VPC are not currently supported.
var subnetsIDs []string var subnetsIDs []string
var vpcId string var vpcId string
{ {
log.Println("EC2 - Find Subnets")
svc := ec2.New(req.awsSession()) svc := ec2.New(req.awsSession())
var subnets []*ec2.Subnet var subnets []*ec2.Subnet
@ -900,9 +909,11 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
log.Printf("\t\tFound %d subnets.\n", len(subnets)) log.Printf("\t\tFound %d subnets.\n", len(subnets))
} }
log.Println("EC2 - Find Security Group") // Try to find the AWS Security Group by name or create a new one.
var securityGroupId string var securityGroupId string
{ {
log.Println("EC2 - Find Security Group")
svc := ec2.New(req.awsSession()) svc := ec2.New(req.awsSession())
log.Printf("\t\tFind security group '%s'.\n", req.Ec2SecurityGroupName) log.Printf("\t\tFind security group '%s'.\n", req.Ec2SecurityGroupName)
@ -995,6 +1006,7 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
log.Printf("\t%s\tUsing Security Group '%s'.\n", tests.Success, req.Ec2SecurityGroupName) log.Printf("\t%s\tUsing Security Group '%s'.\n", tests.Success, req.Ec2SecurityGroupName)
} }
// If a database cluster is defined, ensure it exists else create a new one.
var dbCluster *rds.DBCluster var dbCluster *rds.DBCluster
if req.DBCluster != nil { if req.DBCluster != nil {
log.Println("RDS - Get or Create Database Cluster") log.Println("RDS - Get or Create Database Cluster")
@ -1031,6 +1043,7 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
log.Printf("\t%s\tUsing DB Cluster '%s'.\n", tests.Success, *dbCluster.DatabaseName) log.Printf("\t%s\tUsing DB Cluster '%s'.\n", tests.Success, *dbCluster.DatabaseName)
} }
// If a database instance is defined, ensure it exists else create a new one.
var db *DB var db *DB
if req.DBInstance != nil { if req.DBInstance != nil {
log.Println("RDS - Get or Create Database Instance") log.Println("RDS - Get or Create Database Instance")
@ -1041,7 +1054,6 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
// Retrieve the current secret value if something is stored. // Retrieve the current secret value if something is stored.
{ {
sm := secretsmanager.New(req.awsSession()) sm := secretsmanager.New(req.awsSession())
res, err := sm.GetSecretValue(&secretsmanager.GetSecretValueInput{ res, err := sm.GetSecretValue(&secretsmanager.GetSecretValueInput{
SecretId: aws.String(dbSecretId), SecretId: aws.String(dbSecretId),
}) })
@ -1057,16 +1069,21 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
} }
} }
// Init a new RDS client.
svc := rds.New(req.awsSession()) svc := rds.New(req.awsSession())
// Always set the VPC Security Group ID dynamically with the one created/found previously.
req.DBInstance.VpcSecurityGroupIds = aws.StringSlice([]string{securityGroupId}) req.DBInstance.VpcSecurityGroupIds = aws.StringSlice([]string{securityGroupId})
// When a DB cluster exists, add the identifier to the instance input. This is for creating databases with
// the storage engine of AWS Aurora.
if dbCluster != nil { if dbCluster != nil {
req.DBInstance.DBClusterIdentifier = dbCluster.DBClusterIdentifier req.DBInstance.DBClusterIdentifier = dbCluster.DBClusterIdentifier
} else { } else {
req.DBInstance.DBClusterIdentifier = nil req.DBInstance.DBClusterIdentifier = nil
} }
// Try to find an existing DB instance with the same identifier.
var dbInstance *rds.DBInstance var dbInstance *rds.DBInstance
descRes, err := svc.DescribeDBInstances(&rds.DescribeDBInstancesInput{ descRes, err := svc.DescribeDBInstances(&rds.DescribeDBInstancesInput{
DBInstanceIdentifier: req.DBInstance.DBInstanceIdentifier, DBInstanceIdentifier: req.DBInstance.DBInstanceIdentifier,
@ -1079,31 +1096,39 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
dbInstance = descRes.DBInstances[0] dbInstance = descRes.DBInstances[0]
} }
// No DB instance was found, so create a new one.
if dbInstance == nil { if dbInstance == nil {
if db == nil { if db == nil {
// If master password is not set, pull from cluster or generate random.
if req.DBInstance.MasterUserPassword == nil {
if req.DBCluster.MasterUserPassword != nil && *req.DBCluster.MasterUserPassword != "" {
req.DBInstance.MasterUserPassword = req.DBCluster.MasterUserPassword
} else {
req.DBInstance.MasterUserPassword = aws.String(uuid.NewRandom().String())
}
}
// Only set the password right now, all other details will be set after the database instance is created.
db = &DB{ db = &DB{
Host: fmt.Sprintf("%s:%d", *dbInstance.Endpoint.Address, *dbInstance.Endpoint.Port),
User: *dbInstance.MasterUsername,
Pass: *req.DBInstance.MasterUserPassword, Pass: *req.DBInstance.MasterUserPassword,
Database : *dbInstance.DBName,
Driver: *dbInstance.Engine,
DisableTLS: false,
} }
// Store the secret first in the event that create fails. // Store the secret first in the event that create fails.
{ {
sm := secretsmanager.New(req.awsSession()) // Json encode the db details to be stored as secret text.
dat, err := json.Marshal(db) dat, err := json.Marshal(db)
if err != nil { if err != nil {
return errors.Wrap(err, "failed to marshal db credentials") return errors.Wrap(err, "failed to marshal db credentials")
} }
// Create the new entry in AWS Secret Manager with the database password.
sm := secretsmanager.New(req.awsSession())
_, err = sm.CreateSecret(&secretsmanager.CreateSecretInput{ _, err = sm.CreateSecret(&secretsmanager.CreateSecretInput{
Name: aws.String(dbSecretId), Name: aws.String(dbSecretId),
SecretString: aws.String(string(dat)), SecretString: aws.String(string(dat)),
}) })
if err != nil { if err != nil {
/*
if aerr, ok := err.(awserr.Error); !ok || aerr.Code() != secretsmanager.ErrCodeResourceExistsException { if aerr, ok := err.(awserr.Error); !ok || aerr.Code() != secretsmanager.ErrCodeResourceExistsException {
return errors.Wrap(err, "failed to create new secret with db credentials") return errors.Wrap(err, "failed to create new secret with db credentials")
} }
@ -1113,19 +1138,13 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "failed to update secret with db credentials") return errors.Wrap(err, "failed to update secret with db credentials")
} }*/
return errors.Wrap(err, "failed to create new secret with db credentials")
} }
log.Printf("\t\tStored Secret\n") log.Printf("\t\tStored Secret\n")
} }
}
// If master password is not set, pull from cluster or generate random.
if req.DBInstance.MasterUserPassword == nil {
if req.DBCluster.MasterUserPassword != nil && *req.DBCluster.MasterUserPassword != "" {
req.DBInstance.MasterUserPassword = req.DBCluster.MasterUserPassword
} else { } else {
req.DBInstance.MasterUserPassword = aws.String(uuid.NewRandom().String()) req.DBInstance.MasterUserPassword = aws.String(db.Pass)
}
} }
// If no cluster was found, create one. // If no cluster was found, create one.
@ -1154,15 +1173,37 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
} }
} }
// Update the secret with the db instance details. This happens after DB create to help address when the
// db instance was successfully created, but the secret failed to save. The db details host should be empty or
// match the current instance endpoint.
curHost := fmt.Sprintf("%s:%d", *dbInstance.Endpoint.Address, *dbInstance.Endpoint.Port)
if curHost != db.Host {
// Copy the instance details to the db struct.
db.Host = curHost
db.User = *dbInstance.MasterUsername
db.Database = *dbInstance.DBName
db.Driver = *dbInstance.Engine
db.DisableTLS = false
// Json encode the db details to be stored as text via AWS Secrets Manager.
dat, err := json.Marshal(db)
if err != nil {
return errors.Wrap(err, "failed to marshal db credentials")
}
// Update the current AWS Secret.
sm := secretsmanager.New(req.awsSession())
_, err = sm.UpdateSecret(&secretsmanager.UpdateSecretInput{
SecretId: aws.String(dbSecretId),
SecretString: aws.String(string(dat)),
})
if err != nil {
return errors.Wrap(err, "failed to update secret with db credentials")
}
log.Printf("\t\tUpdate Secret\n")
}
return nil
log.Printf("\t%s\tUsing DB Instance '%s'.\n", tests.Success, *dbInstance.DBInstanceIdentifier) log.Printf("\t%s\tUsing DB Instance '%s'.\n", tests.Success, *dbInstance.DBInstanceIdentifier)
} }
@ -1289,9 +1330,11 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
log.Printf("\t%s\tUsing Cache Cluster '%s'.\n", tests.Success, *cacheCluster.CacheClusterId) log.Printf("\t%s\tUsing Cache Cluster '%s'.\n", tests.Success, *cacheCluster.CacheClusterId)
} }
log.Println("ECS - Get or Create Cluster") // Try to find AWS ECS Cluster by name or create new one.
var ecsCluster *ecs.Cluster var ecsCluster *ecs.Cluster
{ {
log.Println("ECS - Get or Create Cluster")
svc := ecs.New(req.awsSession()) svc := ecs.New(req.awsSession())
descRes, err := svc.DescribeClusters(&ecs.DescribeClustersInput{ descRes, err := svc.DescribeClusters(&ecs.DescribeClustersInput{
@ -1343,9 +1386,11 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
log.Printf("\t%s\tUsing ECS Cluster '%s'.\n", tests.Success, *ecsCluster.ClusterName) log.Printf("\t%s\tUsing ECS Cluster '%s'.\n", tests.Success, *ecsCluster.ClusterName)
} }
log.Println("ECS - Register task definition") // Register a new ECS task.
var taskDef *ecs.TaskDefinition var taskDef *ecs.TaskDefinition
{ {
log.Println("ECS - Register task definition")
// List of placeholders that can be used in task definition and replaced on deployment. // List of placeholders that can be used in task definition and replaced on deployment.
placeholders := map[string]string{ placeholders := map[string]string{
"{SERVICE}": req.ServiceName, "{SERVICE}": req.ServiceName,
@ -1908,9 +1953,12 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
} }
} }
log.Println("ECS - Find Service") // Try to find AWS ECS Service by name. This does not error on not found, but results are used to determine if
// the full creation process of a service needs to be executed.
var ecsService *ecs.Service var ecsService *ecs.Service
{ {
log.Println("ECS - Find Service")
svc := ecs.New(req.awsSession()) svc := ecs.New(req.awsSession())
res, err := svc.DescribeServices(&ecs.DescribeServicesInput{ res, err := svc.DescribeServices(&ecs.DescribeServicesInput{
@ -2581,8 +2629,9 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
} }
} }
log.Println("\tWaiting for service to enter stable state.") // Wait for the updated or created service to enter a stable state.
{ {
log.Println("\tWaiting for service to enter stable state.")
// Helper method to get the logs from cloudwatch for a specific task ID. // Helper method to get the logs from cloudwatch for a specific task ID.
getTaskLogs := func(taskId string) ([]string, error) { getTaskLogs := func(taskId string) ([]string, error) {
@ -3111,8 +3160,5 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
} }
} }
// If Elastic cache is enabled, need to add ingress to security group
return nil return nil
} }