reworked tools/devops to better support gitlab cicd

.gitlab-ci.yml

@@ -5,7 +5,6 @@ services:
 
 before_script:
   - 'cd ./tools/devops && make install && cd ../../'
-  - 'cd ./tools/schema && make install && cd ../../'
 
 stages:
   - build:dev
@@ -27,39 +26,22 @@ cache:
   only:
     - master
 
+.build_tmpl: &build_tmpl
+  <<: *job_tmpl
+  script:
+    - 'devops build -service=${SERVICE} -project=${PROJECT_NAME} -env=${TARGET_ENV}'
+
 .deploy_tmpl: &deploy_tmpl
   <<: *job_tmpl
   script:
-    - 'devops deploy -service=${SERVICE} -project=${PROJECT_NAME} -env=${TARGET_ENV} -enable_https=${ENABLE_HTTPS} -enable_elb=${ENABLE_ELB} -primary_host=${PRIMARY_HOST} -host_names=${HOST_NAMES} -private_bucket=${S3_BUCKET_PRIVATE} -public_bucket=${S3_BUCKET_PUBLIC} -no_build=${NO_BUILD} -no_deploy=${NO_DEPLOY}'
+    - 'devops deploy -service=${SERVICE} -project=${PROJECT_NAME} -env=${TARGET_ENV} -enable_https=${ENABLE_HTTPS} -enable_elb=${ENABLE_ELB} -primary_host=${PRIMARY_HOST} -host_names=${HOST_NAMES} -private_bucket=${S3_BUCKET_PRIVATE} -public_bucket=${S3_BUCKET_PUBLIC}'
 
 .migrate_tmpl: &migrate_tmpl
   <<: *job_tmpl
   script:
-    - './schema ???'
+    - 'devops migrate -project=${PROJECT_NAME} -env=${TARGET_ENV}'
 
-.deploy_dev_tmpl: &deploy_dev_tmpl
+db:migrate:dev:
-  <<: *deploy_tmpl
-  tags:
-    - dev
-  environment:
-    name: 'dev/${SERVICE}-dev'
-
-.deploy_stage_tmpl: &deploy_stage_tmpl
-  <<: *deploy_tmpl
-  tags:
-    - stage
-  environment:
-    name: 'stage/${SERVICE}-stage'
-
-.deploy_prod_tmpl: &deploy_prod_tmpl
-  <<: *deploy_tmpl
-  tags:
-    - prod
-  environment:
-    name: 'production/${SERVICE}'
-  when: manual
-
-.migrate_dev_tmpl: &migrate_dev_tmpl
   <<: *migrate_tmpl
   stage: migrate:dev
   tags:
@@ -69,33 +51,25 @@ cache:
     - dev
     - /^dev-.*$/
 
-.migrate_stage_tmpl: &migrate_stage_tmpl
+  variables:
-  <<: *migrate_tmpl
+    TARGET_ENV: 'dev'
-  stage: migrate:stage
+
+webapi:build:dev:
+  <<: *build_tmpl
+  stage: build:dev
   tags:
-    - stage
+    - dev
+  variables:
+    TARGET_ENV: 'dev'
+    SERVICE: 'web-api'
+    AWS_USE_ROLE: 'true'
   only:
-    - stage
+    - master
-    - /^stage-.*$/
+    - dev
-
+    - dev-web-api
-.migrate_prod_tmpl: &migrate_prod_tmpl
+webapi:deploy:dev:
-  <<: *migrate_tmpl
+  <<: *deploy_tmpl
-  stage: migrate:prod
+  stage: deploy:dev
-  tags:
-    - prod
-  when: manual
-  only:
-    - prod
-    - /^prod-.*$/
-
-#db:migrate:dev:
-#  <<: *migrate_dev_tmpl
-#  variables:
-#    TARGET_ENV: 'stage'
-#    SERVICE: 'schema'
-
-.deploy_dev_webapi_tmpl: &deploy_dev_webapi_tmpl
-  <<: *deploy_dev_tmpl
   tags:
     - dev
   variables:
@@ -108,24 +82,6 @@ cache:
     S3_BUCKET_PRIVATE: 'saas-starter-kit-private'
     S3_BUCKET_PUBLIC: 'saas-starter-kit-public'
     AWS_USE_ROLE: 'true'
-    NO_BUILD: 'true'
-    NO_DEPLOY: 'true'
-webapi:build:dev:
-  <<: *deploy_dev_webapi_tmpl
-  stage: build:dev
-  environment:
-    name: '${TARGET_ENV}/${SERVICE}-${TARGET_ENV}'
-    NO_BUILD: 'false'
-  only:
-    - master
-    - dev
-    - dev-web-api
-webapi:deploy:dev:
-  <<: *deploy_dev_webapi_tmpl
-  stage: deploy:dev
-  environment:
-    name: '${TARGET_ENV}/${SERVICE}-${TARGET_ENV}'
-    NO_DEPLOY: 'false'
   dependencies:
     - 'webapi:build:dev'
 #    - 'db:migrate:dev'

cmd/web-api/main.go

@@ -18,16 +18,16 @@ import (
 	"syscall"
 	"time"
 
-	"geeks-accelerator/oss/saas-starter-kit/internal/platform/web"
-	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"geeks-accelerator/oss/saas-starter-kit/cmd/web-api/docs"
 	"geeks-accelerator/oss/saas-starter-kit/cmd/web-api/handlers"
 	"geeks-accelerator/oss/saas-starter-kit/internal/mid"
 	"geeks-accelerator/oss/saas-starter-kit/internal/platform/auth"
 	"geeks-accelerator/oss/saas-starter-kit/internal/platform/devops"
 	"geeks-accelerator/oss/saas-starter-kit/internal/platform/flag"
+	"geeks-accelerator/oss/saas-starter-kit/internal/platform/web"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/go-redis/redis"
 	"github.com/kelseyhightower/envconfig"

cmd/web-app/main.go

@@ -20,12 +20,8 @@ import (
 	"syscall"
 	"time"
 
-	"geeks-accelerator/oss/saas-starter-kit/internal/mid"
-	"github.com/aws/aws-sdk-go/aws/ec2metadata"
-	"golang.org/x/crypto/acme"
-	"golang.org/x/crypto/acme/autocert"
-	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
 	"geeks-accelerator/oss/saas-starter-kit/cmd/web-app/handlers"
+	"geeks-accelerator/oss/saas-starter-kit/internal/mid"
 	"geeks-accelerator/oss/saas-starter-kit/internal/platform/devops"
 	"geeks-accelerator/oss/saas-starter-kit/internal/platform/flag"
 	img_resize "geeks-accelerator/oss/saas-starter-kit/internal/platform/img-resize"
@@ -33,14 +29,18 @@ import (
 	template_renderer "geeks-accelerator/oss/saas-starter-kit/internal/platform/web/template-renderer"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/go-redis/redis"
 	"github.com/kelseyhightower/envconfig"
 	"github.com/lib/pq"
+	"golang.org/x/crypto/acme"
+	"golang.org/x/crypto/acme/autocert"
 	awstrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/aws/aws-sdk-go/aws"
 	sqltrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/database/sql"
 	redistrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/go-redis/redis"
 	sqlxtrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/jmoiron/sqlx"
+	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
 )
 
 // build is the git version of this program. It is set using build flags in the makefile.

tools/devops/cmd/cicd/aws.go

@@ -1,8 +1,10 @@
-package deploy
+package cicd
 
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/service/s3"
 	"io/ioutil"
 	"path/filepath"
 	"sort"
@@ -26,6 +28,72 @@ const (
 	awsTagNameName              = "Name"
 )
 
+// AwsCredentials defines AWS credentials used for deployment. Unable to use roles when deploying
+// using gitlab CI/CD pipeline.
+type awsCredentials struct {
+	AccessKeyID     string `validate:"required_without=UseRole"`
+	SecretAccessKey string `validate:"required_without=UseRole"`
+	Region          string `validate:"required_without=UseRole"`
+	UseRole         bool
+}
+
+// Session returns a new AWS Session used to access AWS services.
+func (creds awsCredentials) Session() *session.Session {
+
+	if creds.UseRole {
+		// Get an AWS session from an implicit source if no explicit
+		// configuration is provided. This is useful for taking advantage of
+		// EC2/ECS instance roles.
+		sess := session.Must(session.NewSession())
+		if creds.Region != "" {
+			sess.Config.WithRegion(creds.Region)
+		}
+
+		return sess
+	}
+
+	return session.New(
+		&aws.Config{
+			Region:      aws.String(creds.Region),
+			Credentials: credentials.NewStaticCredentials(creds.AccessKeyID, creds.SecretAccessKey, ""),
+		})
+}
+
+// IamPolicyDocument defines an AWS IAM policy used for defining access for IAM roles, users, and groups.
+type IamPolicyDocument struct {
+	Version   string              `json:"Version"`
+	Statement []IamStatementEntry `json:"Statement"`
+}
+
+// IamStatementEntry defines a single statement for an IAM policy.
+type IamStatementEntry struct {
+	Sid      string      `json:"Sid"`
+	Effect   string      `json:"Effect"`
+	Action   []string    `json:"Action"`
+	Resource interface{} `json:"Resource"`
+}
+
+// S3Bucket defines the details need to create a bucket that includes additional configuration.
+type S3Bucket struct {
+	Name              string `validate:"omitempty"`
+	Input             *s3.CreateBucketInput
+	LifecycleRules    []*s3.LifecycleRule
+	CORSRules         []*s3.CORSRule
+	PublicAccessBlock *s3.PublicAccessBlockConfiguration
+	Policy            string
+}
+
+// DB mimics the general info needed for services used to define placeholders.
+type DB struct {
+	Host       string
+	User       string
+	Pass       string
+	Database   string
+	Driver     string
+	DisableTLS bool
+}
+
+// GetAwsCredentials loads the AWS Access Keys from env variables unless a role is used.
 func GetAwsCredentials(targetEnv string) (awsCredentials, error) {
 	var creds awsCredentials
 
@@ -65,6 +133,7 @@ func GetAwsCredentials(targetEnv string) (awsCredentials, error) {
 	return creds, nil
 }
 
+// GetAwsSecretValue returns the string value for a secret stored in AWS Secrets Manager.
 func GetAwsSecretValue(creds awsCredentials, secretId string) (string, error) {
 	svc := secretsmanager.New(creds.Session())
 
@@ -79,7 +148,7 @@ func GetAwsSecretValue(creds awsCredentials, secretId string) (string, error) {
 }
 
 // EcrPurgeImages ensures pipeline does not generate images for max of 10000 and prevent manual deletion of images.
-func EcrPurgeImages(req *serviceDeployRequest) ([]*ecr.ImageIdentifier, error) {
+func EcrPurgeImages(req *serviceBuildRequest) ([]*ecr.ImageIdentifier, error) {
 
 	svc := ecr.New(req.awsSession())
 

tools/devops/cmd/cicd/service.go

@@ -0,0 +1,289 @@
+package cicd
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"geeks-accelerator/oss/saas-starter-kit/internal/platform/tests"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/iancoleman/strcase"
+	"github.com/pkg/errors"
+)
+
+// serviceDeployRequest defines the details needed to execute a service deployment.
+type serviceRequest struct {
+	ServiceName string `validate:"required"`
+	ServiceDir  string `validate:"required"`
+	Env         string `validate:"oneof=dev stage prod"`
+	ProjectRoot string `validate:"required"`
+	ProjectName string `validate:"required"`
+	DockerFile  string `validate:"required"`
+	GoModFile   string `validate:"required"`
+	GoModName   string `validate:"required"`
+
+	AwsCreds    awsCredentials `validate:"required,dive,required"`
+	_awsSession *session.Session
+
+	ReleaseImage string
+}
+
+// projectNameCamel takes a project name and returns the camel cased version.
+func (r *serviceRequest) ProjectNameCamel() string {
+	s := strings.Replace(r.ProjectName, "_", " ", -1)
+	s = strings.Replace(s, "-", " ", -1)
+	s = strcase.ToCamel(s)
+	return s
+}
+
+// awsSession returns the current AWS session for the serviceDeployRequest.
+func (r *serviceRequest) awsSession() *session.Session {
+	if r._awsSession == nil {
+		r._awsSession = r.AwsCreds.Session()
+	}
+
+	return r._awsSession
+}
+
+// init sets the basic details needed for both build and deploy for serviceRequest.
+func (req *serviceRequest) init(log *log.Logger) error {
+	// When project root directory is empty or set to current working path, then search for the project root by locating
+	// the go.mod file.
+	log.Println("\tDetermining the project root directory.")
+	{
+		if req.ProjectRoot == "" || req.ProjectRoot == "." {
+			log.Println("\tAttempting to location project root directory from current working directory.")
+
+			var err error
+			req.GoModFile, err = findProjectGoModFile()
+			if err != nil {
+				return err
+			}
+			req.ProjectRoot = filepath.Dir(req.GoModFile)
+		} else {
+			log.Printf("\t\tUsing supplied project root directory '%s'.\n", req.ProjectRoot)
+			req.GoModFile = filepath.Join(req.ProjectRoot, "go.mod")
+		}
+		log.Printf("\t\t\tproject root: %s", req.ProjectRoot)
+		log.Printf("\t\t\tgo.mod: %s", req.GoModFile)
+	}
+
+	log.Println("\tExtracting go module name from go.mod.")
+	{
+		var err error
+		req.GoModName, err = loadGoModName(req.GoModFile)
+		if err != nil {
+			return err
+		}
+		log.Printf("\t\t\tmodule name: %s", req.GoModName)
+	}
+
+	log.Println("\tDetermining the project name.")
+	{
+		if req.ProjectName != "" {
+			log.Printf("\t\tUse provided value.")
+		} else {
+			req.ProjectName = filepath.Base(req.GoModName)
+			log.Printf("\t\tSet from go module.")
+		}
+		log.Printf("\t\t\tproject name: %s", req.ProjectName)
+	}
+
+	log.Println("\tAttempting to locate service directory from project root directory.")
+	{
+		if req.DockerFile != "" {
+			req.DockerFile = req.DockerFile
+			log.Printf("\t\tUse provided value.")
+
+		} else {
+			log.Printf("\t\tFind from project root looking for Dockerfile.")
+			var err error
+			req.DockerFile, err = findServiceDockerFile(req.ProjectRoot, req.ServiceName)
+			if err != nil {
+				return err
+			}
+		}
+
+		req.ServiceDir = filepath.Dir(req.DockerFile)
+
+		log.Printf("\t\t\tservice directory: %s", req.ServiceDir)
+		log.Printf("\t\t\tdockerfile: %s", req.DockerFile)
+	}
+
+	// Verifies AWS credentials specified as environment variables.
+	log.Println("\tVerify AWS credentials.")
+	{
+		var err error
+		req.AwsCreds, err = GetAwsCredentials(req.Env)
+		if err != nil {
+			return err
+		}
+		if req.AwsCreds.UseRole {
+			log.Printf("\t\t\tUsing role")
+		} else {
+			log.Printf("\t\t\tAccessKeyID: '%s'", req.AwsCreds.AccessKeyID)
+		}
+
+		log.Printf("\t\t\tRegion: '%s'", req.AwsCreds.Region)
+		log.Printf("\t%s\tAWS credentials valid.", tests.Success)
+	}
+
+	return nil
+}
+
+// ecrRepositoryName returns the name used for the AWS ECR Repository.
+func ecrRepositoryName(projectName string) string {
+	return projectName
+}
+
+// releaseImage returns the name used for tagging a release image will always include one with environment and
+// service name. If the env var CI_COMMIT_REF_NAME is set, it will be appended.
+func releaseImage(env, serviceName, repositoryUri string) string {
+
+	tag1 := env + "-" + serviceName
+
+	// Generate tags for the release image.
+	var releaseImage string
+	if v := os.Getenv("CI_COMMIT_REF_NAME"); v != "" {
+		tag2 := tag1 + "-" + v
+		releaseImage = repositoryUri + ":" + tag2
+	} else {
+		releaseImage = repositoryUri + ":" + tag1
+	}
+	return releaseImage
+}
+
+// findProjectGoModFile finds the project root directory from the current working directory.
+func findProjectGoModFile() (string, error) {
+	var err error
+	projectRoot, err := os.Getwd()
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to get current working directory")
+	}
+
+	// Try to find the project root for looking for the go.mod file in a parent directory.
+	var goModFile string
+	testDir := projectRoot
+	for i := 0; i < 3; i++ {
+		if goModFile != "" {
+			testDir = filepath.Join(testDir, "../")
+		}
+		goModFile = filepath.Join(testDir, "go.mod")
+		ok, _ := exists(goModFile)
+		if ok {
+			projectRoot = testDir
+			break
+		}
+	}
+
+	// Verify the go.mod file was found.
+	ok, err := exists(goModFile)
+	if err != nil {
+		return "", errors.WithMessagef(err, "failed to load go.mod for project using project root %s")
+	} else if !ok {
+		return "", errors.Errorf("failed to locate project go.mod in project root %s", projectRoot)
+	}
+
+	return goModFile, nil
+}
+
+// findServiceDockerFile finds the service directory.
+func findServiceDockerFile(projectRoot, targetService string) (string, error) {
+	checkDirs := []string{
+		filepath.Join(projectRoot, "cmd", targetService),
+		filepath.Join(projectRoot, "tools", targetService),
+	}
+
+	var dockerFile string
+	for _, cd := range checkDirs {
+		// Check to see if directory contains Dockerfile.
+		tf := filepath.Join(cd, "Dockerfile")
+
+		ok, _ := exists(tf)
+		if ok {
+			dockerFile = tf
+			break
+		}
+	}
+
+	if dockerFile == "" {
+		return "", errors.Errorf("failed to locate Dockerfile for service %s", targetService)
+	}
+
+	return dockerFile, nil
+}
+
+// getTargetEnv checks for an env var that is prefixed with the current target env.
+func getTargetEnv(targetEnv, envName string) string {
+	k := fmt.Sprintf("%s_%s", strings.ToUpper(targetEnv), envName)
+
+	if v := os.Getenv(k); v != "" {
+		// Set the non prefixed env var with the prefixed value.
+		os.Setenv(envName, v)
+		return v
+	}
+
+	return os.Getenv(envName)
+}
+
+// loadGoModName parses out the module name from go.mod.
+func loadGoModName(goModFile string) (string, error) {
+	ok, err := exists(goModFile)
+	if err != nil {
+		return "", errors.WithMessage(err, "Failed to load go.mod for project")
+	} else if !ok {
+		return "", errors.Errorf("Failed to locate project go.mod at %s", goModFile)
+	}
+
+	b, err := ioutil.ReadFile(goModFile)
+	if err != nil {
+		return "", errors.WithMessagef(err, "Failed to read go.mod at %s", goModFile)
+	}
+
+	var name string
+	lines := strings.Split(string(b), "\n")
+	for _, l := range lines {
+		if strings.HasPrefix(l, "module ") {
+			name = strings.TrimSpace(strings.Split(l, " ")[1])
+			break
+		}
+	}
+
+	return name, nil
+}
+
+// exists returns a bool as to whether a file path exists.
+func exists(path string) (bool, error) {
+	_, err := os.Stat(path)
+	if err == nil {
+		return true, nil
+	}
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	return true, err
+}
+
+// execCmds executes a set of commands using the current env variables.
+func execCmds(log *log.Logger, workDir string, cmds ...[]string) error {
+	for _, cmdVals := range cmds {
+		cmd := exec.Command(cmdVals[0], cmdVals[1:]...)
+		cmd.Dir = workDir
+		cmd.Env = os.Environ()
+
+		cmd.Stderr = log.Writer()
+		cmd.Stdout = log.Writer()
+
+		err := cmd.Run()
+
+		if err != nil {
+			return errors.WithMessagef(err, "failed to execute %s", strings.Join(cmdVals, " "))
+		}
+	}
+
+	return nil
+}

tools/devops/cmd/cicd/service_build.go

@@ -0,0 +1,241 @@
+package cicd
+
+import (
+	"encoding/base64"
+	"log"
+	"path/filepath"
+	"strings"
+
+	"geeks-accelerator/oss/saas-starter-kit/internal/platform/tests"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/service/ecr"
+	"github.com/pkg/errors"
+	"gopkg.in/go-playground/validator.v9"
+)
+
+// ServiceBuildFlags defines the flags used for executing a service build.
+type ServiceBuildFlags struct {
+	// Required flags.
+	ServiceName string `validate:"required" example:"web-api"`
+	Env         string `validate:"oneof=dev stage prod" example:"dev"`
+
+	// Optional flags.
+	ProjectRoot string `validate:"omitempty" example:"."`
+	ProjectName string ` validate:"omitempty" example:"example-project"`
+	DockerFile  string `validate:"omitempty" example:"./cmd/web-api/Dockerfile"`
+	NoCache     bool   `validate:"omitempty" example:"false"`
+	NoPush      bool   `validate:"omitempty" example:"false"`
+}
+
+// serviceBuildRequest defines the details needed to execute a service build.
+type serviceBuildRequest struct {
+	*serviceRequest
+
+	EcrRepositoryName      string `validate:"required"`
+	EcrRepository          *ecr.CreateRepositoryInput
+	EcrRepositoryMaxImages int `validate:"omitempty"`
+
+	NoCache bool `validate:"omitempty"`
+	NoPush  bool `validate:"omitempty"`
+
+	flags ServiceBuildFlags
+}
+
+// NewServiceBuildRequest generates a new request for executing build of a single service for a given set of CLI flags.
+func NewServiceBuildRequest(log *log.Logger, flags ServiceBuildFlags) (*serviceBuildRequest, error) {
+
+	// Validates specified CLI flags map to struct successfully.
+	log.Println("Validate flags.")
+	{
+		errs := validator.New().Struct(flags)
+		if errs != nil {
+			return nil, errs
+		}
+		log.Printf("\t%s\tFlags ok.", tests.Success)
+	}
+
+	// Define new service request.
+	sr := &serviceRequest{
+		ServiceName: flags.ServiceName,
+		Env:         flags.Env,
+		ProjectRoot: flags.ProjectRoot,
+		ProjectName: flags.ProjectName,
+		DockerFile:  flags.DockerFile,
+	}
+	if err := sr.init(log); err != nil {
+		return nil, err
+	}
+
+	// Generate a deploy request using CLI flags and AWS credentials.
+	log.Println("Generate deploy request.")
+	var req serviceBuildRequest
+	{
+		req = serviceBuildRequest{
+			serviceRequest: sr,
+
+			NoCache: flags.NoCache,
+			NoPush:  flags.NoPush,
+
+			flags: flags,
+		}
+
+		// Set default AWS ECR Repository Name.
+		req.EcrRepositoryName = ecrRepositoryName(req.ProjectName)
+		req.EcrRepository = &ecr.CreateRepositoryInput{
+			RepositoryName: aws.String(req.EcrRepositoryName),
+			Tags: []*ecr.Tag{
+				&ecr.Tag{Key: aws.String(awsTagNameProject), Value: aws.String(req.ProjectName)},
+				&ecr.Tag{Key: aws.String(awsTagNameEnv), Value: aws.String(req.Env)},
+			},
+		}
+		log.Printf("\t\t\tSet ECR Repository Name to '%s'.", req.EcrRepositoryName)
+
+		// Set default AWS ECR Regsistry Max Images.
+		req.EcrRepositoryMaxImages = defaultAwsRegistryMaxImages
+		log.Printf("\t\t\tSet ECR Regsistry Max Images to '%d'.", req.EcrRepositoryMaxImages)
+
+	}
+
+	return &req, nil
+}
+
+// Run is the main entrypoint for building a service for a given target environment.
+func ServiceBuild(log *log.Logger, req *serviceBuildRequest) error {
+
+	// Load the AWS ECR repository. Try to find by name else create new one.
+	var dockerLoginCmd []string
+	{
+		log.Println("ECR - Get or create repository.")
+
+		svc := ecr.New(req.awsSession())
+
+		// First try to find ECR repository by name.
+		var awsRepo *ecr.Repository
+		descRes, err := svc.DescribeRepositories(&ecr.DescribeRepositoriesInput{
+			RepositoryNames: []*string{aws.String(req.EcrRepositoryName)},
+		})
+		if err != nil {
+			if aerr, ok := err.(awserr.Error); !ok || aerr.Code() != ecr.ErrCodeRepositoryNotFoundException {
+				return errors.Wrapf(err, "failed to describe repository '%s'", req.EcrRepositoryName)
+			}
+		} else if len(descRes.Repositories) > 0 {
+			awsRepo = descRes.Repositories[0]
+		}
+
+		if awsRepo == nil {
+			// If no repository was found, create one.
+			createRes, err := svc.CreateRepository(req.EcrRepository)
+			if err != nil {
+				return errors.Wrapf(err, "failed to create repository '%s'", req.EcrRepositoryName)
+			}
+			awsRepo = createRes.Repository
+			log.Printf("\t\tCreated: %s.", *awsRepo.RepositoryArn)
+		} else {
+			log.Printf("\t\tFound: %s.", *awsRepo.RepositoryArn)
+
+			log.Println("\t\tChecking old ECR images.")
+			delIds, err := EcrPurgeImages(req)
+			if err != nil {
+				return err
+			}
+
+			// Since ECR has max number of repository images, need to delete old ones so can stay under limit.
+			// If there are image IDs to delete, delete them.
+			if len(delIds) > 0 {
+				log.Printf("\t\tDeleted %d images that exceeded limit of %d", len(delIds), req.EcrRepositoryMaxImages)
+				for _, imgId := range delIds {
+					log.Printf("\t\t\t%s", *imgId.ImageTag)
+				}
+			}
+		}
+
+		req.ReleaseImage = releaseImage(req.Env, req.ServiceName, *awsRepo.RepositoryUri)
+		if err != nil {
+			return err
+		}
+
+		log.Printf("\t\trelease image: %s", req.ReleaseImage)
+		log.Printf("\t%s\tRelease image valid.", tests.Success)
+
+		log.Println("ECR - Retrieve authorization token used for docker login.")
+
+		// Get the credentials necessary for logging into the AWS Elastic Container Registry
+		// made available with the AWS access key and AWS secret access keys.
+		res, err := svc.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
+		if err != nil {
+			return errors.Wrap(err, "failed to get ecr authorization token")
+		}
+
+		authToken, err := base64.StdEncoding.DecodeString(*res.AuthorizationData[0].AuthorizationToken)
+		if err != nil {
+			return errors.Wrap(err, "failed to base64 decode ecr authorization token")
+		}
+		pts := strings.Split(string(authToken), ":")
+		user := pts[0]
+		pass := pts[1]
+
+		dockerLoginCmd = []string{
+			"docker",
+			"login",
+			"-u", user,
+			"-p", pass,
+			*res.AuthorizationData[0].ProxyEndpoint,
+		}
+
+		log.Printf("\t%s\tdocker login ok.", tests.Success)
+	}
+
+	// Once we can access the repository in ECR, do the docker build.
+	{
+		log.Println("Starting docker build")
+
+		dockerFile, err := filepath.Rel(req.ProjectRoot, req.DockerFile)
+		if err != nil {
+			return errors.Wrapf(err, "Failed parse relative path for %s from %s", req.DockerFile, req.ProjectRoot)
+		}
+
+		// The initial build command slice.
+		buildCmd := []string{
+			"docker", "build",
+			"--file=" + dockerFile,
+			"--build-arg", "service=" + req.ServiceName,
+			"--build-arg", "env=" + req.Env,
+			"-t", req.ReleaseImage,
+		}
+
+		// Append additional build flags.
+		if req.NoCache {
+			buildCmd = append(buildCmd, "--no-cache")
+		}
+
+		// Finally append the build context as the current directory since os.Exec will use the project root as
+		// the working directory.
+		buildCmd = append(buildCmd, ".")
+
+		err = execCmds(log, req.ProjectRoot, buildCmd)
+		if err != nil {
+			return errors.Wrap(err, "Failed to build docker image")
+		}
+
+		// Push the newly built image of the Docker container to the registry.
+		if req.NoPush == false {
+
+			log.Printf("\t\tDocker Login")
+			err = execCmds(log, req.ProjectRoot, dockerLoginCmd)
+			if err != nil {
+				return errors.Wrapf(err, "Failed to login to AWS ECR")
+			}
+
+			log.Printf("\t\tPush release image %s", req.ReleaseImage)
+			err = execCmds(log, req.ProjectRoot, []string{"docker", "push", req.ReleaseImage})
+			if err != nil {
+				return errors.Wrapf(err, "Failed to push docker image %s", req.ReleaseImage)
+			}
+		}
+
+		log.Printf("\t%s\tbuild complete.\n", tests.Success)
+	}
+
+	return nil
+}

tools/devops/cmd/cicd/service_deploy.go

@@ -1,4 +1,4 @@
-package deploy
+package cicd
 
 import (
 	"compress/gzip"
@@ -38,11 +38,100 @@ import (
 	"github.com/iancoleman/strcase"
 	"github.com/pborman/uuid"
 	"github.com/pkg/errors"
+	"github.com/urfave/cli"
 	"gopkg.in/go-playground/validator.v9"
 )
 
-// NewServiceDeployRequest generates a new request for executing deployment of a single service
+// ServiceDeployFlags defines the flags used for executing a service deployment.
-// for a given set of CLI flags.
+type ServiceDeployFlags struct {
+	// Required flags.
+	ServiceName string `validate:"required" example:"web-api"`
+	Env         string `validate:"oneof=dev stage prod" example:"dev"`
+
+	// Optional flags.
+	EnableHTTPS         bool            `validate:"omitempty" example:"false"`
+	ServiceHostPrimary  string          `validate:"omitempty" example:"example-project.com"`
+	ServiceHostNames    cli.StringSlice `validate:"omitempty" example:"subdomain.example-project.com"`
+	S3BucketPrivateName string          `validate:"omitempty" example:"saas-example-project-private"`
+	S3BucketPublicName  string          `validate:"omitempty" example:"saas-example-project-public"`
+
+	ProjectRoot     string `validate:"omitempty" example:"."`
+	ProjectName     string ` validate:"omitempty" example:"example-project"`
+	DockerFile      string `validate:"omitempty" example:"./cmd/web-api/Dockerfile"`
+	EnableLambdaVPC bool   `validate:"omitempty" example:"false"`
+	EnableEcsElb    bool   `validate:"omitempty" example:"false"`
+	RecreateService bool   `validate:"omitempty" example:"false"`
+}
+
+// serviceDeployRequest defines the details needed to execute a service deployment.
+type serviceDeployRequest struct {
+	*serviceRequest
+
+	EnableHTTPS        bool     `validate:"omitempty"`
+	ServiceHostPrimary string   `validate:"omitempty,required_with=EnableHTTPS,fqdn"`
+	ServiceHostNames   []string `validate:"omitempty,dive,fqdn"`
+
+	EcrRepositoryName string `validate:"required"`
+
+	EcsClusterName string `validate:"required"`
+	EcsCluster     *ecs.CreateClusterInput
+
+	EcsServiceName                          string `validate:"required"`
+	EcsServiceDesiredCount                  int64  `validate:"required"`
+	EcsServiceMinimumHealthyPercent         *int64 `validate:"omitempty"`
+	EcsServiceMaximumPercent                *int64 `validate:"omitempty"`
+	EscServiceHealthCheckGracePeriodSeconds *int64 `validate:"omitempty"`
+
+	EcsExecutionRoleName       string `validate:"required"`
+	EcsExecutionRole           *iam.CreateRoleInput
+	EcsExecutionRolePolicyArns []string `validate:"required"`
+
+	EcsTaskRoleName string `validate:"required"`
+	EcsTaskRole     *iam.CreateRoleInput
+
+	EcsTaskPolicyName     string `validate:"required"`
+	EcsTaskPolicy         *iam.CreatePolicyInput
+	EcsTaskPolicyDocument IamPolicyDocument
+
+	Ec2SecurityGroupName string `validate:"required"`
+	Ec2SecurityGroup     *ec2.CreateSecurityGroupInput
+
+	CloudWatchLogGroupName string `validate:"required"`
+	CloudWatchLogGroup     *cloudwatchlogs.CreateLogGroupInput
+
+	S3BucketTempPrefix  string `validate:"required_with=S3BucketPrivateName S3BucketPublicName"`
+	S3BucketPrivateName string `validate:"omitempty"`
+	S3BucketPublicName  string `validate:"omitempty"`
+	S3Buckets           []S3Bucket
+
+	EnableEcsElb           bool   `validate:"omitempty"`
+	ElbLoadBalancerName    string `validate:"omitempty"`
+	ElbDeregistrationDelay *int   `validate:"omitempty"`
+	ElbLoadBalancer        *elbv2.CreateLoadBalancerInput
+
+	ElbTargetGroupName string `validate:"omitempty"`
+	ElbTargetGroup     *elbv2.CreateTargetGroupInput
+
+	VpcPublicName    string `validate:"omitempty"`
+	VpcPublic        *ec2.CreateVpcInput
+	VpcPublicSubnets []*ec2.CreateSubnetInput
+
+	EnableLambdaVPC bool `validate:"omitempty"`
+	RecreateService bool `validate:"omitempty"`
+
+	SDNamepsace *servicediscovery.CreatePrivateDnsNamespaceInput
+	SDService   *servicediscovery.CreateServiceInput
+
+	CacheCluster          *elasticache.CreateCacheClusterInput
+	CacheClusterParameter []*elasticache.ParameterNameValue
+
+	DBCluster  *rds.CreateDBClusterInput
+	DBInstance *rds.CreateDBInstanceInput
+
+	flags ServiceDeployFlags
+}
+
+// NewServiceDeployRequest generates a new request for executing deployment of a single service for a given set of CLI flags.
 func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*serviceDeployRequest, error) {
 
 	// Validates specified CLI flags map to struct successfully.
@@ -55,39 +144,25 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
 		log.Printf("\t%s\tFlags ok.", tests.Success)
 	}
 
-	// Verifies AWS credentials specified as environment variables.
+	// Define new service request.
-	log.Println("\tVerify AWS credentials.")
+	sr := &serviceRequest{
-	var awsCreds awsCredentials
+		ServiceName: flags.ServiceName,
-	{
+		Env:         flags.Env,
-		var err error
+		ProjectRoot: flags.ProjectRoot,
-		awsCreds, err = GetAwsCredentials(flags.Env)
+		ProjectName: flags.ProjectName,
-		if err != nil {
+		DockerFile:  flags.DockerFile,
+	}
+	if err := sr.init(log); err != nil {
 		return nil, err
 	}
-		if awsCreds.UseRole {
-			log.Printf("\t\t\tUsing role")
-		} else {
-			log.Printf("\t\t\tAccessKeyID: '%s'", awsCreds.AccessKeyID)
-		}
-
-		log.Printf("\t\t\tRegion: '%s'", awsCreds.Region)
-		log.Printf("\t%s\tAWS credentials valid.", tests.Success)
-	}
 
 	// Generate a deploy request using CLI flags and AWS credentials.
 	log.Println("Generate deploy request.")
 	var req serviceDeployRequest
 	{
 		req = serviceDeployRequest{
-			// Required flags.
+			serviceRequest: sr,
-			ServiceName: flags.ServiceName,
-			Env:         flags.Env,
-			AwsCreds:    awsCreds,
 
-			// Optional flags.
-			ProjectRoot:         flags.ProjectRoot,
-			ProjectName:         flags.ProjectName,
-			DockerFile:          flags.DockerFile,
 			EnableHTTPS:         flags.EnableHTTPS,
 			ServiceHostPrimary:  flags.ServiceHostPrimary,
 			ServiceHostNames:    flags.ServiceHostNames,
@@ -95,79 +170,11 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
 			S3BucketPublicName:  flags.S3BucketPublicName,
 			EnableLambdaVPC:     flags.EnableLambdaVPC,
 			EnableEcsElb:        flags.EnableEcsElb,
-			NoBuild:             flags.NoBuild,
-			NoDeploy:            flags.NoDeploy,
-			NoCache:             flags.NoCache,
-			NoPush:              flags.NoPush,
 			RecreateService:     flags.RecreateService,
 
 			flags: flags,
 		}
 
-		// When project root directory is empty or set to current working path, then search for the project root by locating
-		// the go.mod file.
-		log.Println("\tDetermining the project root directory.")
-		{
-			if flags.ProjectRoot == "" || flags.ProjectRoot == "." {
-				log.Println("\tAttempting to location project root directory from current working directory.")
-
-				var err error
-				req.GoModFile, err = findProjectGoModFile()
-				if err != nil {
-					return nil, err
-				}
-				req.ProjectRoot = filepath.Dir(req.GoModFile)
-			} else {
-				log.Println("\t\tUsing supplied project root directory.")
-				req.GoModFile = filepath.Join(flags.ProjectRoot, "go.mod")
-			}
-			log.Printf("\t\t\tproject root: %s", req.ProjectRoot)
-			log.Printf("\t\t\tgo.mod: %s", req.GoModFile)
-		}
-
-		log.Println("\tExtracting go module name from go.mod.")
-		{
-			var err error
-			req.GoModName, err = loadGoModName(req.GoModFile)
-			if err != nil {
-				return nil, err
-			}
-			log.Printf("\t\t\tmodule name: %s", req.GoModName)
-		}
-
-		log.Println("\tDetermining the project name.")
-		{
-			if flags.ProjectName != "" {
-				req.ProjectName = flags.ProjectName
-				log.Printf("\t\tUse provided value.")
-			} else {
-				req.ProjectName = filepath.Base(req.GoModName)
-				log.Printf("\t\tSet from go module.")
-			}
-			log.Printf("\t\t\tproject name: %s", req.ProjectName)
-		}
-
-		log.Println("\tAttempting to locate service directory from project root directory.")
-		{
-			if flags.DockerFile != "" {
-				req.DockerFile = flags.DockerFile
-				log.Printf("\t\tUse provided value.")
-
-			} else {
-				log.Printf("\t\tFind from project root looking for Dockerfile.")
-				var err error
-				req.DockerFile, err = findServiceDockerFile(req.ProjectRoot, req.ServiceName)
-				if err != nil {
-					return nil, err
-				}
-			}
-
-			req.ServiceDir = filepath.Dir(req.DockerFile)
-
-			log.Printf("\t\t\tservice directory: %s", req.ServiceDir)
-			log.Printf("\t\t\tdockerfile: %s", req.DockerFile)
-		}
-
 		// Set default configuration values. Primarily setting default values for all the AWS services:
 		// - AWS S3 bucket settings
 		// - AWS ECR repository settings
@@ -325,20 +332,9 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
 			}
 
 			// Set default AWS ECR Repository Name.
-			req.EcrRepositoryName = req.ProjectName
+			req.EcrRepositoryName = ecrRepositoryName(req.ProjectName)
-			req.EcrRepository = &ecr.CreateRepositoryInput{
-				RepositoryName: aws.String(req.EcrRepositoryName),
-				Tags: []*ecr.Tag{
-					&ecr.Tag{Key: aws.String(awsTagNameProject), Value: aws.String(req.ProjectName)},
-					&ecr.Tag{Key: aws.String(awsTagNameEnv), Value: aws.String(req.Env)},
-				},
-			}
 			log.Printf("\t\t\tSet ECR Repository Name to '%s'.", req.EcrRepositoryName)
 
-			// Set default AWS ECR Regsistry Max Images.
-			req.EcrRepositoryMaxImages = defaultAwsRegistryMaxImages
-			log.Printf("\t\t\tSet ECR Regsistry Max Images to '%d'.", req.EcrRepositoryMaxImages)
-
 			// Set default AWS ECS Cluster Name.
 			req.EcsClusterName = req.ProjectName + "-" + req.Env
 			req.EcsCluster = &ecs.CreateClusterInput{
@@ -764,7 +760,6 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 	startTime := time.Now()
 
 	// Load the AWS ECR repository. Try to find by name else create new one.
-	var dockerLoginCmd []string
 	{
 		log.Println("ECR - Get or create repository.")
 
@@ -776,128 +771,20 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 			RepositoryNames: []*string{aws.String(req.EcrRepositoryName)},
 		})
 		if err != nil {
-			if aerr, ok := err.(awserr.Error); !ok || aerr.Code() != ecr.ErrCodeRepositoryNotFoundException {
+			// The repository should have been created by build or manually created and should exist at this point.
-				return errors.Wrapf(err, "failed to describe repository '%s'", req.EcrRepositoryName)
+			return errors.Wrapf(err, "Failed to describe repository '%s'.", req.EcrRepositoryName)
-			}
 		} else if len(descRes.Repositories) > 0 {
 			awsRepo = descRes.Repositories[0]
 		}
-
-		if awsRepo == nil {
-			// If no repository was found, create one.
-			createRes, err := svc.CreateRepository(req.EcrRepository)
-			if err != nil {
-				return errors.Wrapf(err, "failed to create repository '%s'", req.EcrRepositoryName)
-			}
-			awsRepo = createRes.Repository
-			log.Printf("\t\tCreated: %s.", *awsRepo.RepositoryArn)
-		} else {
 		log.Printf("\t\tFound: %s.", *awsRepo.RepositoryArn)
 
-			log.Println("\t\tChecking old ECR images.")
+		req.ReleaseImage = releaseImage(req.Env, req.ServiceName, *awsRepo.RepositoryUri)
-			delIds, err := EcrPurgeImages(req)
 		if err != nil {
 			return err
 		}
 
-			// Since ECR has max number of repository images, need to delete old ones so can stay under limit.
-			// If there are image IDs to delete, delete them.
-			if len(delIds) > 0 {
-				log.Printf("\t\tDeleted %d images that exceeded limit of %d", len(delIds), req.EcrRepositoryMaxImages)
-				for _, imgId := range delIds {
-					log.Printf("\t\t\t%s", *imgId.ImageTag)
-				}
-			}
-		}
-
-		// Tag of release image will always include one with environment and service name.
-		tag1 := req.Env + "-" + req.ServiceName
-
-		// Generate tags for the release image.
-		if v := os.Getenv("CI_COMMIT_REF_NAME"); v != "" {
-			tag2 := tag1 + "-" + v
-			req.ReleaseImage = *awsRepo.RepositoryUri + ":" + tag2
-		} else {
-			req.ReleaseImage = *awsRepo.RepositoryUri + ":" + tag1
-		}
-
 		log.Printf("\t\trelease image: %s", req.ReleaseImage)
 		log.Printf("\t%s\tRelease image valid.", tests.Success)
-
-		log.Println("ECR - Retrieve authorization token used for docker login.")
-
-		// Get the credentials necessary for logging into the AWS Elastic Container Registry
-		// made available with the AWS access key and AWS secret access keys.
-		res, err := svc.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
-		if err != nil {
-			return errors.Wrap(err, "failed to get ecr authorization token")
-		}
-
-		authToken, err := base64.StdEncoding.DecodeString(*res.AuthorizationData[0].AuthorizationToken)
-		if err != nil {
-			return errors.Wrap(err, "failed to base64 decode ecr authorization token")
-		}
-		pts := strings.Split(string(authToken), ":")
-		user := pts[0]
-		pass := pts[1]
-
-		dockerLoginCmd = []string{
-			"docker",
-			"login",
-			"-u", user,
-			"-p", pass,
-			*res.AuthorizationData[0].ProxyEndpoint,
-		}
-
-		log.Printf("\t%s\tdocker login ok.", tests.Success)
-	}
-
-	// Once we can access the repository in ECR, do the docker build.
-	if req.NoBuild == false {
-		dockerFile, err := filepath.Rel(req.ProjectRoot, req.DockerFile)
-		if err != nil {
-			return errors.Wrapf(err, "Failed parse relative path for %s from %s", req.DockerFile, req.ProjectRoot)
-		}
-
-		// The initial build command slice.
-		buildCmd := []string{
-			"docker", "build",
-			"--file="+dockerFile,
-			"--build-arg", "service="+req.ServiceName,
-			"--build-arg", "env="+req.Env,
-			"-t", req.ReleaseImage,
-			".",
-		}
-
-		log.Println("Starting docker build")
-		err = execCmds(log, req.ProjectRoot, buildCmd)
-		if err != nil {
-			return errors.Wrap(err, "Failed to build docker image")
-		}
-
-		// Push the newly built image of the Docker container to the registry.
-		if req.NoPush == false {
-
-			log.Printf("\t\tDocker Login")
-			err = execCmds(log, req.ProjectRoot, dockerLoginCmd)
-			if err != nil {
-				return errors.Wrapf(err, "Failed to login to AWS ECR")
-			}
-
-			log.Printf("\t\tPush release image %s", req.ReleaseImage)
-			err = execCmds(log, req.ProjectRoot, []string{"docker", "push", req.ReleaseImage})
-			if err != nil {
-				return errors.Wrapf(err, "Failed to push docker image %s", req.ReleaseImage)
-			}
-		}
-
-		log.Printf("\t%s\tbuild complete.\n", tests.Success)
-	}
-
-	// If skip deploy is set, exit and don't continue.
-	// Used to test build for a service without actually pushing image to specified environment.
-	if req.NoDeploy == true {
-		return nil
 	}
 
 	// Try to find the Datadog API key, this value is optional.

tools/devops/cmd/deploy/devops.go

@@ -1,144 +0,0 @@
-package deploy
-
-import (
-	"fmt"
-	"io/ioutil"
-	"log"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-
-	"github.com/pkg/errors"
-)
-
-// findProjectGoModFile finds the project root directory from the current working directory.
-func findProjectGoModFile() (string, error) {
-	var err error
-	projectRoot, err := os.Getwd()
-	if err != nil {
-		return "", errors.WithMessage(err, "failed to get current working directory")
-	}
-
-	// Try to find the project root for looking for the go.mod file in a parent directory.
-	var goModFile string
-	testDir := projectRoot
-	for i := 0; i < 3; i++ {
-		if goModFile != "" {
-			testDir = filepath.Join(testDir, "../")
-		}
-		goModFile = filepath.Join(testDir, "go.mod")
-		ok, _ := exists(goModFile)
-		if ok {
-			projectRoot = testDir
-			break
-		}
-	}
-
-	// Verify the go.mod file was found.
-	ok, err := exists(goModFile)
-	if err != nil {
-		return "", errors.WithMessagef(err, "failed to load go.mod for project using project root %s")
-	} else if !ok {
-		return "", errors.Errorf("failed to locate project go.mod in project root %s", projectRoot)
-	}
-
-	return goModFile, nil
-}
-
-// findServiceDockerFile finds the service directory.
-func findServiceDockerFile(projectRoot, targetService string) (string, error) {
-	checkDirs := []string{
-		filepath.Join(projectRoot, "cmd", targetService),
-		filepath.Join(projectRoot, "tools", targetService),
-	}
-
-	var dockerFile string
-	for _, cd := range checkDirs {
-		// Check to see if directory contains Dockerfile.
-		tf := filepath.Join(cd, "Dockerfile")
-
-		ok, _ := exists(tf)
-		if ok {
-			dockerFile = tf
-			break
-		}
-	}
-
-	if dockerFile == "" {
-		return "", errors.Errorf("failed to locate Dockerfile for service %s", targetService)
-	}
-
-	return dockerFile, nil
-}
-
-// getTargetEnv checks for an env var that is prefixed with the current target env.
-func getTargetEnv(targetEnv, envName string) string {
-	k := fmt.Sprintf("%s_%s", strings.ToUpper(targetEnv), envName)
-
-	if v := os.Getenv(k); v != "" {
-		// Set the non prefixed env var with the prefixed value.
-		os.Setenv(envName, v)
-		return v
-	}
-
-	return os.Getenv(envName)
-}
-
-// loadGoModName parses out the module name from go.mod.
-func loadGoModName(goModFile string) (string, error) {
-	ok, err := exists(goModFile)
-	if err != nil {
-		return "", errors.WithMessage(err, "Failed to load go.mod for project")
-	} else if !ok {
-		return "", errors.Errorf("Failed to locate project go.mod at %s", goModFile)
-	}
-
-	b, err := ioutil.ReadFile(goModFile)
-	if err != nil {
-		return "", errors.WithMessagef(err, "Failed to read go.mod at %s", goModFile)
-	}
-
-	var name string
-	lines := strings.Split(string(b), "\n")
-	for _, l := range lines {
-		if strings.HasPrefix(l, "module ") {
-			name = strings.TrimSpace(strings.Split(l, " ")[1])
-			break
-		}
-	}
-
-	return name, nil
-}
-
-// exists returns a bool as to whether a file path exists.
-func exists(path string) (bool, error) {
-	_, err := os.Stat(path)
-	if err == nil {
-		return true, nil
-	}
-	if os.IsNotExist(err) {
-		return false, nil
-	}
-	return true, err
-}
-
-// execCmds executes a set of commands using the current env variables.
-func execCmds(log *log.Logger, workDir string, cmds ...[]string) (error) {
-	for _, cmdVals := range cmds {
-		cmd := exec.Command(cmdVals[0], cmdVals[1:]...)
-		cmd.Dir = workDir
-		cmd.Env = os.Environ()
-
-		cmd.Stderr = log.Writer()
-		cmd.Stdout = log.Writer()
-
-		err := cmd.Run()
-
-		if err != nil {
-			return  errors.WithMessagef(err, "failed to execute %s",  strings.Join(cmdVals, " "))
-		}
-	}
-
-	return  nil
-}

tools/devops/cmd/deploy/models.go

@@ -1,212 +0,0 @@
-package deploy
-
-import (
-	"strings"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/aws/aws-sdk-go/service/ecr"
-	"github.com/aws/aws-sdk-go/service/ecs"
-	"github.com/aws/aws-sdk-go/service/elasticache"
-	"github.com/aws/aws-sdk-go/service/elbv2"
-	"github.com/aws/aws-sdk-go/service/iam"
-	"github.com/aws/aws-sdk-go/service/rds"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/aws/aws-sdk-go/service/servicediscovery"
-	"github.com/iancoleman/strcase"
-	"github.com/urfave/cli"
-)
-
-// ServiceDeployFlags defines the flags used for executing a service deployment.
-type ServiceDeployFlags struct {
-	// Required flags.
-	ServiceName string `validate:"required" example:"web-api"`
-	Env         string `validate:"oneof=dev stage prod" example:"dev"`
-
-	// Optional flags.
-	EnableHTTPS         bool            `validate:"omitempty" example:"false"`
-	ServiceHostPrimary  string          `validate:"omitempty" example:"example-project.com"`
-	ServiceHostNames    cli.StringSlice `validate:"omitempty" example:"subdomain.example-project.com"`
-	S3BucketPrivateName string          `validate:"omitempty" example:"saas-example-project-private"`
-	S3BucketPublicName  string          `validate:"omitempty" example:"saas-example-project-public"`
-
-	ProjectRoot     string `validate:"omitempty" example:"."`
-	ProjectName     string ` validate:"omitempty" example:"example-project"`
-	DockerFile      string `validate:"omitempty" example:"./cmd/web-api/Dockerfile"`
-	EnableLambdaVPC bool   `validate:"omitempty" example:"false"`
-	EnableEcsElb    bool   `validate:"omitempty" example:"false"`
-	NoBuild         bool   `validate:"omitempty" example:"false"`
-	NoDeploy        bool   `validate:"omitempty" example:"false"`
-	NoCache         bool   `validate:"omitempty" example:"false"`
-	NoPush          bool   `validate:"omitempty" example:"false"`
-	RecreateService bool   `validate:"omitempty" example:"false"`
-}
-
-// serviceDeployRequest defines the details needed to execute a service deployment.
-type serviceDeployRequest struct {
-	ServiceName string `validate:"required"`
-	ServiceDir  string `validate:"required"`
-	Env         string `validate:"oneof=dev stage prod"`
-	ProjectRoot string `validate:"required"`
-	ProjectName string `validate:"required"`
-	DockerFile  string `validate:"required"`
-	GoModFile   string `validate:"required"`
-	GoModName   string `validate:"required"`
-
-	EnableHTTPS        bool     `validate:"omitempty"`
-	ServiceHostPrimary string   `validate:"omitempty,required_with=EnableHTTPS,fqdn"`
-	ServiceHostNames   []string `validate:"omitempty,dive,fqdn"`
-
-	AwsCreds awsCredentials `validate:"required,dive,required"`
-
-	EcrRepositoryName      string `validate:"required"`
-	EcrRepository          *ecr.CreateRepositoryInput
-	EcrRepositoryMaxImages int `validate:"omitempty"`
-
-	EcsClusterName string `validate:"required"`
-	EcsCluster     *ecs.CreateClusterInput
-
-	EcsServiceName                          string `validate:"required"`
-	EcsServiceDesiredCount                  int64  `validate:"required"`
-	EcsServiceMinimumHealthyPercent         *int64 `validate:"omitempty"`
-	EcsServiceMaximumPercent                *int64 `validate:"omitempty"`
-	EscServiceHealthCheckGracePeriodSeconds *int64 `validate:"omitempty"`
-
-	EcsExecutionRoleName       string `validate:"required"`
-	EcsExecutionRole           *iam.CreateRoleInput
-	EcsExecutionRolePolicyArns []string `validate:"required"`
-
-	EcsTaskRoleName string `validate:"required"`
-	EcsTaskRole     *iam.CreateRoleInput
-
-	EcsTaskPolicyName     string `validate:"required"`
-	EcsTaskPolicy         *iam.CreatePolicyInput
-	EcsTaskPolicyDocument IamPolicyDocument
-
-	Ec2SecurityGroupName string `validate:"required"`
-	Ec2SecurityGroup     *ec2.CreateSecurityGroupInput
-
-	CloudWatchLogGroupName string `validate:"required"`
-	CloudWatchLogGroup     *cloudwatchlogs.CreateLogGroupInput
-
-	S3BucketTempPrefix  string `validate:"required_with=S3BucketPrivateName S3BucketPublicName"`
-	S3BucketPrivateName string `validate:"omitempty"`
-	S3BucketPublicName  string `validate:"omitempty"`
-	S3Buckets           []S3Bucket
-
-	EnableEcsElb           bool   `validate:"omitempty"`
-	ElbLoadBalancerName    string `validate:"omitempty"`
-	ElbDeregistrationDelay *int   `validate:"omitempty"`
-	ElbLoadBalancer        *elbv2.CreateLoadBalancerInput
-
-	ElbTargetGroupName string `validate:"omitempty"`
-	ElbTargetGroup     *elbv2.CreateTargetGroupInput
-
-	VpcPublicName    string `validate:"omitempty"`
-	VpcPublic        *ec2.CreateVpcInput
-	VpcPublicSubnets []*ec2.CreateSubnetInput
-
-	EnableLambdaVPC bool `validate:"omitempty"`
-	NoBuild         bool `validate:"omitempty"`
-	NoDeploy        bool `validate:"omitempty"`
-	NoCache         bool `validate:"omitempty"`
-	NoPush          bool `validate:"omitempty"`
-	RecreateService bool `validate:"omitempty"`
-
-	SDNamepsace *servicediscovery.CreatePrivateDnsNamespaceInput
-	SDService   *servicediscovery.CreateServiceInput
-
-	CacheCluster          *elasticache.CreateCacheClusterInput
-	CacheClusterParameter []*elasticache.ParameterNameValue
-
-	DBCluster  *rds.CreateDBClusterInput
-	DBInstance *rds.CreateDBInstanceInput
-
-	ReleaseImage string
-	flags        ServiceDeployFlags
-	_awsSession  *session.Session
-}
-
-type S3Bucket struct {
-	Name              string `validate:"omitempty"`
-	Input             *s3.CreateBucketInput
-	LifecycleRules    []*s3.LifecycleRule
-	CORSRules         []*s3.CORSRule
-	PublicAccessBlock *s3.PublicAccessBlockConfiguration
-	Policy            string
-}
-
-// DB mimics the general info needed for services used to define placeholders.
-type DB struct {
-	Host       string
-	User       string
-	Pass       string
-	Database   string
-	Driver     string
-	DisableTLS bool
-}
-
-// projectNameCamel takes a project name and returns the camel cased version.
-func (r *serviceDeployRequest) ProjectNameCamel() string {
-	s := strings.Replace(r.ProjectName, "_", " ", -1)
-	s = strings.Replace(s, "-", " ", -1)
-	s = strcase.ToCamel(s)
-	return s
-}
-
-// awsSession returns the current AWS session for the serviceDeployRequest.
-func (r *serviceDeployRequest) awsSession() *session.Session {
-	if r._awsSession == nil {
-		r._awsSession = r.AwsCreds.Session()
-	}
-
-	return r._awsSession
-}
-
-// AwsCredentials defines AWS credentials used for deployment. Unable to use roles when deploying
-// using gitlab CI/CD pipeline.
-type awsCredentials struct {
-	AccessKeyID     string `validate:"required_without=UseRole"`
-	SecretAccessKey string `validate:"required_without=UseRole"`
-	Region          string `validate:"required_without=UseRole"`
-	UseRole         bool
-}
-
-// Session returns a new AWS Session used to access AWS services.
-func (creds awsCredentials) Session() *session.Session {
-
-	if creds.UseRole {
-		// Get an AWS session from an implicit source if no explicit
-		// configuration is provided. This is useful for taking advantage of
-		// EC2/ECS instance roles.
-		sess := session.Must(session.NewSession())
-		if creds.Region != "" {
-			sess.Config.WithRegion(creds.Region)
-		}
-
-		return sess
-	}
-
-	return session.New(
-		&aws.Config{
-			Region:      aws.String(creds.Region),
-			Credentials: credentials.NewStaticCredentials(creds.AccessKeyID, creds.SecretAccessKey, ""),
-		})
-}
-
-// IamPolicyDocument defines an AWS IAM policy used for defining access for IAM roles, users, and groups.
-type IamPolicyDocument struct {
-	Version   string              `json:"Version"`
-	Statement []IamStatementEntry `json:"Statement"`
-}
-
-// IamStatementEntry defines a single statement for an IAM policy.
-type IamStatementEntry struct {
-	Sid      string      `json:"Sid"`
-	Effect   string      `json:"Effect"`
-	Action   []string    `json:"Action"`
-	Resource interface{} `json:"Resource"`
-}

tools/devops/main.go

@@ -6,7 +6,7 @@ import (
 	"os"
 	"strings"
 
-	"geeks-accelerator/oss/saas-starter-kit/tools/devops/cmd/deploy"
+	"geeks-accelerator/oss/saas-starter-kit/tools/devops/cmd/cicd"
 	_ "github.com/lib/pq"
 	"github.com/urfave/cli"
 )
@@ -38,13 +38,35 @@ func main() {
 	// =========================================================================
 	// Start Truss
 
-	var deployFlags deploy.ServiceDeployFlags
+	var (
+		buildFlags  cicd.ServiceBuildFlags
+		deployFlags cicd.ServiceDeployFlags
+	)
 
 	app := cli.NewApp()
 	app.Commands = []cli.Command{
+		{
+			Name:  "build",
+			Usage: "-service=web-api -env=dev",
+			Flags: []cli.Flag{
+				cli.StringFlag{Name: "service", Usage: "name of cmd", Destination: &buildFlags.ServiceName},
+				cli.StringFlag{Name: "env", Usage: "dev, stage, or prod", Destination: &buildFlags.Env},
+				cli.StringFlag{Name: "dockerfile", Usage: "DockerFile for service", Destination: &buildFlags.DockerFile},
+				cli.StringFlag{Name: "root", Usage: "project root directory", Destination: &buildFlags.ProjectRoot},
+				cli.StringFlag{Name: "project", Usage: "name of project", Destination: &buildFlags.ProjectName},
+				cli.BoolFlag{Name: "no_cache", Usage: "skip docker cache", Destination: &buildFlags.NoCache},
+				cli.BoolFlag{Name: "no_push", Usage: "skip docker push after build", Destination: &buildFlags.NoPush},
+			},
+			Action: func(c *cli.Context) error {
+				req, err := cicd.NewServiceBuildRequest(log, buildFlags)
+				if err != nil {
+					return err
+				}
+				return cicd.ServiceBuild(log, req)
+			},
+		},
 		{
 			Name:  "deploy",
-			Aliases: []string{"serviceDeploy"},
 			Usage: "-service=web-api -env=dev",
 			Flags: []cli.Flag{
 				cli.StringFlag{Name: "service", Usage: "name of cmd", Destination: &deployFlags.ServiceName},
@@ -59,10 +81,6 @@ func main() {
 				cli.StringFlag{Name: "project", Usage: "name of project", Destination: &deployFlags.ProjectName},
 				cli.BoolFlag{Name: "enable_elb", Usage: "enable deployed to use Elastic Load Balancer", Destination: &deployFlags.EnableEcsElb},
 				cli.BoolTFlag{Name: "lambda_vpc", Usage: "deploy lambda behind VPC", Destination: &deployFlags.EnableLambdaVPC},
-				cli.BoolFlag{Name: "no_build", Usage: "skip build and continue directly to deploy", Destination: &deployFlags.NoBuild},
-				cli.BoolFlag{Name: "no_deploy", Usage: "skip deploy after build", Destination: &deployFlags.NoDeploy},
-				cli.BoolFlag{Name: "no_cache", Usage: "skip docker cache", Destination: &deployFlags.NoCache},
-				cli.BoolFlag{Name: "no_push", Usage: "skip docker push after build", Destination: &deployFlags.NoPush},
 				cli.BoolFlag{Name: "recreate_service", Usage: "skip docker push after build", Destination: &deployFlags.RecreateService},
 			},
 			Action: func(c *cli.Context) error {
@@ -82,11 +100,20 @@ func main() {
 					deployFlags.ServiceHostNames = hostNames
 				}
 
-				req, err := deploy.NewServiceDeployRequest(log, deployFlags)
+				req, err := cicd.NewServiceDeployRequest(log, deployFlags)
 				if err != nil {
 					return err
 				}
-				return deploy.ServiceDeploy(log, req)
+				return cicd.ServiceDeploy(log, req)
+			},
+		},
+		{
+			Name:  "migrate",
+			Usage: "-env=dev",
+			Flags: []cli.Flag{},
+			Action: func(c *cli.Context) error {
+
+				return nil
 			},
 		},
 	}

tools/schema/main.go

@@ -34,7 +34,6 @@ type DB struct {
 	DisableTLS bool
 }
 
-
 func main() {
 	// =========================================================================
 	// Logging
@@ -81,7 +80,6 @@ func main() {
 		return // We displayed help.
 	}
 
-
 	/*
 		//
 		DBInstanceIdentifier:      aws.String(req.ProjectName + "-" + req.Env),
@@ -92,7 +90,6 @@ func main() {
 
 	*/
 
-
 	// =========================================================================
 	// Log App Info