go/golang-saas-starter-kit
1
0
Fork 0
mirror of https://github.com/raseels-repos/golang-saas-starter-kit.git synced 2025-06-17 00:17:59 +02:00
Code Issues Releases Activity

reworked tools/devops to better support gitlab cicd

Browse Source
This commit is contained in:
Lee Brown
2019-07-14 14:55:34 -08:00
parent e31991efc8
commit 1a36b52113
11 changed files with 790 additions and 680 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
.gitlab-ci.yml
View File

@ -5,7 +5,6 @@ services:
before_script:
- 'cd ./tools/devops && make install && cd ../../'
- 'cd ./tools/schema && make install && cd ../../'
stages:
- build:dev
@ -27,39 +26,22 @@ cache:
only:
- master
.build_tmpl: &build_tmpl
<<: *job_tmpl
script:
- 'devops build -service=${SERVICE} -project=${PROJECT_NAME} -env=${TARGET_ENV}'
.deploy_tmpl: &deploy_tmpl
<<: *job_tmpl
script:
- 'devops deploy -service=${SERVICE} -project=${PROJECT_NAME} -env=${TARGET_ENV} -enable_https=${ENABLE_HTTPS} -enable_elb=${ENABLE_ELB} -primary_host=${PRIMARY_HOST} -host_names=${HOST_NAMES} -private_bucket=${S3_BUCKET_PRIVATE} -public_bucket=${S3_BUCKET_PUBLIC} -no_build=${NO_BUILD} -no_deploy=${NO_DEPLOY}'
- 'devops deploy -service=${SERVICE} -project=${PROJECT_NAME} -env=${TARGET_ENV} -enable_https=${ENABLE_HTTPS} -enable_elb=${ENABLE_ELB} -primary_host=${PRIMARY_HOST} -host_names=${HOST_NAMES} -private_bucket=${S3_BUCKET_PRIVATE} -public_bucket=${S3_BUCKET_PUBLIC}'
.migrate_tmpl: &migrate_tmpl
<<: *job_tmpl
script:
- './schema ???'
- 'devops migrate -project=${PROJECT_NAME} -env=${TARGET_ENV}'
.deploy_dev_tmpl: &deploy_dev_tmpl
<<: *deploy_tmpl
tags:
- dev
environment:
name: 'dev/${SERVICE}-dev'
.deploy_stage_tmpl: &deploy_stage_tmpl
<<: *deploy_tmpl
tags:
- stage
environment:
name: 'stage/${SERVICE}-stage'
.deploy_prod_tmpl: &deploy_prod_tmpl
<<: *deploy_tmpl
tags:
- prod
environment:
name: 'production/${SERVICE}'
when: manual
.migrate_dev_tmpl: &migrate_dev_tmpl
db:migrate:dev:
<<: *migrate_tmpl
stage: migrate:dev
tags:
@ -69,33 +51,25 @@ cache:
- dev
- /^dev-.*$/
.migrate_stage_tmpl: &migrate_stage_tmpl
<<: *migrate_tmpl
stage: migrate:stage
variables:
TARGET_ENV: 'dev'
webapi:build:dev:
<<: *build_tmpl
stage: build:dev
tags:
- stage
- dev
variables:
TARGET_ENV: 'dev'
SERVICE: 'web-api'
AWS_USE_ROLE: 'true'
only:
- stage
- /^stage-.*$/
.migrate_prod_tmpl: &migrate_prod_tmpl
<<: *migrate_tmpl
stage: migrate:prod
tags:
- prod
when: manual
only:
- prod
- /^prod-.*$/
#db:migrate:dev:
# <<: *migrate_dev_tmpl
# variables:
# TARGET_ENV: 'stage'
# SERVICE: 'schema'
.deploy_dev_webapi_tmpl: &deploy_dev_webapi_tmpl
<<: *deploy_dev_tmpl
- master
- dev
- dev-web-api
webapi:deploy:dev:
<<: *deploy_tmpl
stage: deploy:dev
tags:
- dev
variables:
@ -108,24 +82,6 @@ cache:
S3_BUCKET_PRIVATE: 'saas-starter-kit-private'
S3_BUCKET_PUBLIC: 'saas-starter-kit-public'
AWS_USE_ROLE: 'true'
NO_BUILD: 'true'
NO_DEPLOY: 'true'
webapi:build:dev:
<<: *deploy_dev_webapi_tmpl
stage: build:dev
environment:
name: '${TARGET_ENV}/${SERVICE}-${TARGET_ENV}'
NO_BUILD: 'false'
only:
- master
- dev
- dev-web-api
webapi:deploy:dev:
<<: *deploy_dev_webapi_tmpl
stage: deploy:dev
environment:
name: '${TARGET_ENV}/${SERVICE}-${TARGET_ENV}'
NO_DEPLOY: 'false'
dependencies:
- 'webapi:build:dev'
# - 'db:migrate:dev'

4
cmd/web-api/main.go
View File

@ -18,16 +18,16 @@ import (
"syscall"
"time"
"geeks-accelerator/oss/saas-starter-kit/internal/platform/web"
"github.com/aws/aws-sdk-go/aws/ec2metadata"
"geeks-accelerator/oss/saas-starter-kit/cmd/web-api/docs"
"geeks-accelerator/oss/saas-starter-kit/cmd/web-api/handlers"
"geeks-accelerator/oss/saas-starter-kit/internal/mid"
"geeks-accelerator/oss/saas-starter-kit/internal/platform/auth"
"geeks-accelerator/oss/saas-starter-kit/internal/platform/devops"
"geeks-accelerator/oss/saas-starter-kit/internal/platform/flag"
"geeks-accelerator/oss/saas-starter-kit/internal/platform/web"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/ec2metadata"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/go-redis/redis"
"github.com/kelseyhightower/envconfig"

10
cmd/web-app/main.go
View File

@ -20,12 +20,8 @@ import (
"syscall"
"time"
"geeks-accelerator/oss/saas-starter-kit/internal/mid"
"github.com/aws/aws-sdk-go/aws/ec2metadata"
"golang.org/x/crypto/acme"
"golang.org/x/crypto/acme/autocert"
"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
"geeks-accelerator/oss/saas-starter-kit/cmd/web-app/handlers"
"geeks-accelerator/oss/saas-starter-kit/internal/mid"
"geeks-accelerator/oss/saas-starter-kit/internal/platform/devops"
"geeks-accelerator/oss/saas-starter-kit/internal/platform/flag"
img_resize "geeks-accelerator/oss/saas-starter-kit/internal/platform/img-resize"
@ -33,14 +29,18 @@ import (
template_renderer "geeks-accelerator/oss/saas-starter-kit/internal/platform/web/template-renderer"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/ec2metadata"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/go-redis/redis"
"github.com/kelseyhightower/envconfig"
"github.com/lib/pq"
"golang.org/x/crypto/acme"
"golang.org/x/crypto/acme/autocert"
awstrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/aws/aws-sdk-go/aws"
sqltrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/database/sql"
redistrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/go-redis/redis"
sqlxtrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/jmoiron/sqlx"
"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
)
// build is the git version of this program. It is set using build flags in the makefile.

73
tools/devops/cmd/deploy/aws.go → tools/devops/cmd/cicd/aws.go
View File

@ -1,8 +1,10 @@
package deploy
package cicd
import (
"encoding/json"
"fmt"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/service/s3"
"io/ioutil"
"path/filepath"
"sort"
@ -26,6 +28,72 @@ const (
awsTagNameName = "Name"
)
// AwsCredentials defines AWS credentials used for deployment. Unable to use roles when deploying
// using gitlab CI/CD pipeline.
type awsCredentials struct {
AccessKeyID string `validate:"required_without=UseRole"`
SecretAccessKey string `validate:"required_without=UseRole"`
Region string `validate:"required_without=UseRole"`
UseRole bool
}
// Session returns a new AWS Session used to access AWS services.
func (creds awsCredentials) Session() *session.Session {
if creds.UseRole {
// Get an AWS session from an implicit source if no explicit
// configuration is provided. This is useful for taking advantage of
// EC2/ECS instance roles.
sess := session.Must(session.NewSession())
if creds.Region != "" {
sess.Config.WithRegion(creds.Region)
}
return sess
}
return session.New(
&aws.Config{
Region: aws.String(creds.Region),
Credentials: credentials.NewStaticCredentials(creds.AccessKeyID, creds.SecretAccessKey, ""),
})
}
// IamPolicyDocument defines an AWS IAM policy used for defining access for IAM roles, users, and groups.
type IamPolicyDocument struct {
Version string `json:"Version"`
Statement []IamStatementEntry `json:"Statement"`
}
// IamStatementEntry defines a single statement for an IAM policy.
type IamStatementEntry struct {
Sid string `json:"Sid"`
Effect string `json:"Effect"`
Action []string `json:"Action"`
Resource interface{} `json:"Resource"`
}
// S3Bucket defines the details need to create a bucket that includes additional configuration.
type S3Bucket struct {
Name string `validate:"omitempty"`
Input *s3.CreateBucketInput
LifecycleRules []*s3.LifecycleRule
CORSRules []*s3.CORSRule
PublicAccessBlock *s3.PublicAccessBlockConfiguration
Policy string
}
// DB mimics the general info needed for services used to define placeholders.
type DB struct {
Host string
User string
Pass string
Database string
Driver string
DisableTLS bool
}
// GetAwsCredentials loads the AWS Access Keys from env variables unless a role is used.
func GetAwsCredentials(targetEnv string) (awsCredentials, error) {
var creds awsCredentials
@ -65,6 +133,7 @@ func GetAwsCredentials(targetEnv string) (awsCredentials, error) {
return creds, nil
}
// GetAwsSecretValue returns the string value for a secret stored in AWS Secrets Manager.
func GetAwsSecretValue(creds awsCredentials, secretId string) (string, error) {
svc := secretsmanager.New(creds.Session())
@ -79,7 +148,7 @@ func GetAwsSecretValue(creds awsCredentials, secretId string) (string, error) {
}
// EcrPurgeImages ensures pipeline does not generate images for max of 10000 and prevent manual deletion of images.
func EcrPurgeImages(req *serviceDeployRequest) ([]*ecr.ImageIdentifier, error) {
func EcrPurgeImages(req *serviceBuildRequest) ([]*ecr.ImageIdentifier, error) {
svc := ecr.New(req.awsSession())

289
tools/devops/cmd/cicd/service.go Normal file
View File

@ -0,0 +1,289 @@
package cicd
import (
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"geeks-accelerator/oss/saas-starter-kit/internal/platform/tests"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/iancoleman/strcase"
"github.com/pkg/errors"
)
// serviceDeployRequest defines the details needed to execute a service deployment.
type serviceRequest struct {
ServiceName string `validate:"required"`
ServiceDir string `validate:"required"`
Env string `validate:"oneof=dev stage prod"`
ProjectRoot string `validate:"required"`
ProjectName string `validate:"required"`
DockerFile string `validate:"required"`
GoModFile string `validate:"required"`
GoModName string `validate:"required"`
AwsCreds awsCredentials `validate:"required,dive,required"`
_awsSession *session.Session
ReleaseImage string
}
// projectNameCamel takes a project name and returns the camel cased version.
func (r *serviceRequest) ProjectNameCamel() string {
s := strings.Replace(r.ProjectName, "_", " ", -1)
s = strings.Replace(s, "-", " ", -1)
s = strcase.ToCamel(s)
return s
}
// awsSession returns the current AWS session for the serviceDeployRequest.
func (r *serviceRequest) awsSession() *session.Session {
if r._awsSession == nil {
r._awsSession = r.AwsCreds.Session()
}
return r._awsSession
}
// init sets the basic details needed for both build and deploy for serviceRequest.
func (req *serviceRequest) init(log *log.Logger) error {
// When project root directory is empty or set to current working path, then search for the project root by locating
// the go.mod file.
log.Println("\tDetermining the project root directory.")
{
if req.ProjectRoot == "" || req.ProjectRoot == "." {
log.Println("\tAttempting to location project root directory from current working directory.")
var err error
req.GoModFile, err = findProjectGoModFile()
if err != nil {
return err
}
req.ProjectRoot = filepath.Dir(req.GoModFile)
} else {
log.Printf("\t\tUsing supplied project root directory '%s'.\n", req.ProjectRoot)
req.GoModFile = filepath.Join(req.ProjectRoot, "go.mod")
}
log.Printf("\t\t\tproject root: %s", req.ProjectRoot)
log.Printf("\t\t\tgo.mod: %s", req.GoModFile)
}
log.Println("\tExtracting go module name from go.mod.")
{
var err error
req.GoModName, err = loadGoModName(req.GoModFile)
if err != nil {
return err
}
log.Printf("\t\t\tmodule name: %s", req.GoModName)
}
log.Println("\tDetermining the project name.")
{
if req.ProjectName != "" {
log.Printf("\t\tUse provided value.")
} else {
req.ProjectName = filepath.Base(req.GoModName)
log.Printf("\t\tSet from go module.")
}
log.Printf("\t\t\tproject name: %s", req.ProjectName)
}
log.Println("\tAttempting to locate service directory from project root directory.")
{
if req.DockerFile != "" {
req.DockerFile = req.DockerFile
log.Printf("\t\tUse provided value.")
} else {
log.Printf("\t\tFind from project root looking for Dockerfile.")
var err error
req.DockerFile, err = findServiceDockerFile(req.ProjectRoot, req.ServiceName)
if err != nil {
return err
}
}
req.ServiceDir = filepath.Dir(req.DockerFile)
log.Printf("\t\t\tservice directory: %s", req.ServiceDir)
log.Printf("\t\t\tdockerfile: %s", req.DockerFile)
}
// Verifies AWS credentials specified as environment variables.
log.Println("\tVerify AWS credentials.")
{
var err error
req.AwsCreds, err = GetAwsCredentials(req.Env)
if err != nil {
return err
}
if req.AwsCreds.UseRole {
log.Printf("\t\t\tUsing role")
} else {
log.Printf("\t\t\tAccessKeyID: '%s'", req.AwsCreds.AccessKeyID)
}
log.Printf("\t\t\tRegion: '%s'", req.AwsCreds.Region)
log.Printf("\t%s\tAWS credentials valid.", tests.Success)
}
return nil
}
// ecrRepositoryName returns the name used for the AWS ECR Repository.
func ecrRepositoryName(projectName string) string {
return projectName
}
// releaseImage returns the name used for tagging a release image will always include one with environment and
// service name. If the env var CI_COMMIT_REF_NAME is set, it will be appended.
func releaseImage(env, serviceName, repositoryUri string) string {
tag1 := env + "-" + serviceName
// Generate tags for the release image.
var releaseImage string
if v := os.Getenv("CI_COMMIT_REF_NAME"); v != "" {
tag2 := tag1 + "-" + v
releaseImage = repositoryUri + ":" + tag2
} else {
releaseImage = repositoryUri + ":" + tag1
}
return releaseImage
}
// findProjectGoModFile finds the project root directory from the current working directory.
func findProjectGoModFile() (string, error) {
var err error
projectRoot, err := os.Getwd()
if err != nil {
return "", errors.WithMessage(err, "failed to get current working directory")
}
// Try to find the project root for looking for the go.mod file in a parent directory.
var goModFile string
testDir := projectRoot
for i := 0; i < 3; i++ {
if goModFile != "" {
testDir = filepath.Join(testDir, "../")
}
goModFile = filepath.Join(testDir, "go.mod")
ok, _ := exists(goModFile)
if ok {
projectRoot = testDir
break
}
}
// Verify the go.mod file was found.
ok, err := exists(goModFile)
if err != nil {
return "", errors.WithMessagef(err, "failed to load go.mod for project using project root %s")
} else if !ok {
return "", errors.Errorf("failed to locate project go.mod in project root %s", projectRoot)
}
return goModFile, nil
}
// findServiceDockerFile finds the service directory.
func findServiceDockerFile(projectRoot, targetService string) (string, error) {
checkDirs := []string{
filepath.Join(projectRoot, "cmd", targetService),
filepath.Join(projectRoot, "tools", targetService),
}
var dockerFile string
for _, cd := range checkDirs {
// Check to see if directory contains Dockerfile.
tf := filepath.Join(cd, "Dockerfile")
ok, _ := exists(tf)
if ok {
dockerFile = tf
break
}
}
if dockerFile == "" {
return "", errors.Errorf("failed to locate Dockerfile for service %s", targetService)
}
return dockerFile, nil
}
// getTargetEnv checks for an env var that is prefixed with the current target env.
func getTargetEnv(targetEnv, envName string) string {
k := fmt.Sprintf("%s_%s", strings.ToUpper(targetEnv), envName)
if v := os.Getenv(k); v != "" {
// Set the non prefixed env var with the prefixed value.
os.Setenv(envName, v)
return v
}
return os.Getenv(envName)
}
// loadGoModName parses out the module name from go.mod.
func loadGoModName(goModFile string) (string, error) {
ok, err := exists(goModFile)
if err != nil {
return "", errors.WithMessage(err, "Failed to load go.mod for project")
} else if !ok {
return "", errors.Errorf("Failed to locate project go.mod at %s", goModFile)
}
b, err := ioutil.ReadFile(goModFile)
if err != nil {
return "", errors.WithMessagef(err, "Failed to read go.mod at %s", goModFile)
}
var name string
lines := strings.Split(string(b), "\n")
for _, l := range lines {
if strings.HasPrefix(l, "module ") {
name = strings.TrimSpace(strings.Split(l, " ")[1])
break
}
}
return name, nil
}
// exists returns a bool as to whether a file path exists.
func exists(path string) (bool, error) {
_, err := os.Stat(path)
if err == nil {
return true, nil
}
if os.IsNotExist(err) {
return false, nil
}
return true, err
}
// execCmds executes a set of commands using the current env variables.
func execCmds(log *log.Logger, workDir string, cmds ...[]string) error {
for _, cmdVals := range cmds {
cmd := exec.Command(cmdVals[0], cmdVals[1:]...)
cmd.Dir = workDir
cmd.Env = os.Environ()
cmd.Stderr = log.Writer()
cmd.Stdout = log.Writer()
err := cmd.Run()
if err != nil {
return errors.WithMessagef(err, "failed to execute %s", strings.Join(cmdVals, " "))
}
}
return nil
}

241
tools/devops/cmd/cicd/service_build.go Normal file
View File

@ -0,0 +1,241 @@
package cicd
import (
"encoding/base64"
"log"
"path/filepath"
"strings"
"geeks-accelerator/oss/saas-starter-kit/internal/platform/tests"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/ecr"
"github.com/pkg/errors"
"gopkg.in/go-playground/validator.v9"
)
// ServiceBuildFlags defines the flags used for executing a service build.
type ServiceBuildFlags struct {
// Required flags.
ServiceName string `validate:"required" example:"web-api"`
Env string `validate:"oneof=dev stage prod" example:"dev"`
// Optional flags.
ProjectRoot string `validate:"omitempty" example:"."`
ProjectName string ` validate:"omitempty" example:"example-project"`
DockerFile string `validate:"omitempty" example:"./cmd/web-api/Dockerfile"`
NoCache bool `validate:"omitempty" example:"false"`
NoPush bool `validate:"omitempty" example:"false"`
}
// serviceBuildRequest defines the details needed to execute a service build.
type serviceBuildRequest struct {
*serviceRequest
EcrRepositoryName string `validate:"required"`
EcrRepository *ecr.CreateRepositoryInput
EcrRepositoryMaxImages int `validate:"omitempty"`
NoCache bool `validate:"omitempty"`
NoPush bool `validate:"omitempty"`
flags ServiceBuildFlags
}
// NewServiceBuildRequest generates a new request for executing build of a single service for a given set of CLI flags.
func NewServiceBuildRequest(log *log.Logger, flags ServiceBuildFlags) (*serviceBuildRequest, error) {
// Validates specified CLI flags map to struct successfully.
log.Println("Validate flags.")
{
errs := validator.New().Struct(flags)
if errs != nil {
return nil, errs
}
log.Printf("\t%s\tFlags ok.", tests.Success)
}
// Define new service request.
sr := &serviceRequest{
ServiceName: flags.ServiceName,
Env: flags.Env,
ProjectRoot: flags.ProjectRoot,
ProjectName: flags.ProjectName,
DockerFile: flags.DockerFile,
}
if err := sr.init(log); err != nil {
return nil, err
}
// Generate a deploy request using CLI flags and AWS credentials.
log.Println("Generate deploy request.")
var req serviceBuildRequest
{
req = serviceBuildRequest{
serviceRequest: sr,
NoCache: flags.NoCache,
NoPush: flags.NoPush,
flags: flags,
}
// Set default AWS ECR Repository Name.
req.EcrRepositoryName = ecrRepositoryName(req.ProjectName)
req.EcrRepository = &ecr.CreateRepositoryInput{
RepositoryName: aws.String(req.EcrRepositoryName),
Tags: []*ecr.Tag{
&ecr.Tag{Key: aws.String(awsTagNameProject), Value: aws.String(req.ProjectName)},
&ecr.Tag{Key: aws.String(awsTagNameEnv), Value: aws.String(req.Env)},
},
}
log.Printf("\t\t\tSet ECR Repository Name to '%s'.", req.EcrRepositoryName)
// Set default AWS ECR Regsistry Max Images.
req.EcrRepositoryMaxImages = defaultAwsRegistryMaxImages
log.Printf("\t\t\tSet ECR Regsistry Max Images to '%d'.", req.EcrRepositoryMaxImages)
}
return &req, nil
}
// Run is the main entrypoint for building a service for a given target environment.
func ServiceBuild(log *log.Logger, req *serviceBuildRequest) error {
// Load the AWS ECR repository. Try to find by name else create new one.
var dockerLoginCmd []string
{
log.Println("ECR - Get or create repository.")
svc := ecr.New(req.awsSession())
// First try to find ECR repository by name.
var awsRepo *ecr.Repository
descRes, err := svc.DescribeRepositories(&ecr.DescribeRepositoriesInput{
RepositoryNames: []*string{aws.String(req.EcrRepositoryName)},
})
if err != nil {
if aerr, ok := err.(awserr.Error); !ok || aerr.Code() != ecr.ErrCodeRepositoryNotFoundException {
return errors.Wrapf(err, "failed to describe repository '%s'", req.EcrRepositoryName)
}
} else if len(descRes.Repositories) > 0 {
awsRepo = descRes.Repositories[0]
}
if awsRepo == nil {
// If no repository was found, create one.
createRes, err := svc.CreateRepository(req.EcrRepository)
if err != nil {
return errors.Wrapf(err, "failed to create repository '%s'", req.EcrRepositoryName)
}
awsRepo = createRes.Repository
log.Printf("\t\tCreated: %s.", *awsRepo.RepositoryArn)
} else {
log.Printf("\t\tFound: %s.", *awsRepo.RepositoryArn)
log.Println("\t\tChecking old ECR images.")
delIds, err := EcrPurgeImages(req)
if err != nil {
return err
}
// Since ECR has max number of repository images, need to delete old ones so can stay under limit.
// If there are image IDs to delete, delete them.
if len(delIds) > 0 {
log.Printf("\t\tDeleted %d images that exceeded limit of %d", len(delIds), req.EcrRepositoryMaxImages)
for _, imgId := range delIds {
log.Printf("\t\t\t%s", *imgId.ImageTag)
}
}
}
req.ReleaseImage = releaseImage(req.Env, req.ServiceName, *awsRepo.RepositoryUri)
if err != nil {
return err
}
log.Printf("\t\trelease image: %s", req.ReleaseImage)
log.Printf("\t%s\tRelease image valid.", tests.Success)
log.Println("ECR - Retrieve authorization token used for docker login.")
// Get the credentials necessary for logging into the AWS Elastic Container Registry
// made available with the AWS access key and AWS secret access keys.
res, err := svc.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
if err != nil {
return errors.Wrap(err, "failed to get ecr authorization token")
}
authToken, err := base64.StdEncoding.DecodeString(*res.AuthorizationData[0].AuthorizationToken)
if err != nil {
return errors.Wrap(err, "failed to base64 decode ecr authorization token")
}
pts := strings.Split(string(authToken), ":")
user := pts[0]
pass := pts[1]
dockerLoginCmd = []string{
"docker",
"login",
"-u", user,
"-p", pass,
*res.AuthorizationData[0].ProxyEndpoint,
}
log.Printf("\t%s\tdocker login ok.", tests.Success)
}
// Once we can access the repository in ECR, do the docker build.
{
log.Println("Starting docker build")
dockerFile, err := filepath.Rel(req.ProjectRoot, req.DockerFile)
if err != nil {
return errors.Wrapf(err, "Failed parse relative path for %s from %s", req.DockerFile, req.ProjectRoot)
}
// The initial build command slice.
buildCmd := []string{
"docker", "build",
"--file=" + dockerFile,
"--build-arg", "service=" + req.ServiceName,
"--build-arg", "env=" + req.Env,
"-t", req.ReleaseImage,
}
// Append additional build flags.
if req.NoCache {
buildCmd = append(buildCmd, "--no-cache")
}
// Finally append the build context as the current directory since os.Exec will use the project root as
// the working directory.
buildCmd = append(buildCmd, ".")
err = execCmds(log, req.ProjectRoot, buildCmd)
if err != nil {
return errors.Wrap(err, "Failed to build docker image")
}
// Push the newly built image of the Docker container to the registry.
if req.NoPush == false {
log.Printf("\t\tDocker Login")
err = execCmds(log, req.ProjectRoot, dockerLoginCmd)
if err != nil {
return errors.Wrapf(err, "Failed to login to AWS ECR")
}
log.Printf("\t\tPush release image %s", req.ReleaseImage)
err = execCmds(log, req.ProjectRoot, []string{"docker", "push", req.ReleaseImage})
if err != nil {
return errors.Wrapf(err, "Failed to push docker image %s", req.ReleaseImage)
}
}
log.Printf("\t%s\tbuild complete.\n", tests.Success)
}
return nil
}

325
tools/devops/cmd/deploy/service_deploy.go → tools/devops/cmd/cicd/service_deploy.go
View File

@ -1,4 +1,4 @@
package deploy
package cicd
import (
"compress/gzip"
@ -38,11 +38,100 @@ import (
"github.com/iancoleman/strcase"
"github.com/pborman/uuid"
"github.com/pkg/errors"
"github.com/urfave/cli"
"gopkg.in/go-playground/validator.v9"
)
// NewServiceDeployRequest generates a new request for executing deployment of a single service
// for a given set of CLI flags.
// ServiceDeployFlags defines the flags used for executing a service deployment.
type ServiceDeployFlags struct {
// Required flags.
ServiceName string `validate:"required" example:"web-api"`
Env string `validate:"oneof=dev stage prod" example:"dev"`
// Optional flags.
EnableHTTPS bool `validate:"omitempty" example:"false"`
ServiceHostPrimary string `validate:"omitempty" example:"example-project.com"`
ServiceHostNames cli.StringSlice `validate:"omitempty" example:"subdomain.example-project.com"`
S3BucketPrivateName string `validate:"omitempty" example:"saas-example-project-private"`
S3BucketPublicName string `validate:"omitempty" example:"saas-example-project-public"`
ProjectRoot string `validate:"omitempty" example:"."`
ProjectName string ` validate:"omitempty" example:"example-project"`
DockerFile string `validate:"omitempty" example:"./cmd/web-api/Dockerfile"`
EnableLambdaVPC bool `validate:"omitempty" example:"false"`
EnableEcsElb bool `validate:"omitempty" example:"false"`
RecreateService bool `validate:"omitempty" example:"false"`
}
// serviceDeployRequest defines the details needed to execute a service deployment.
type serviceDeployRequest struct {
*serviceRequest
EnableHTTPS bool `validate:"omitempty"`
ServiceHostPrimary string `validate:"omitempty,required_with=EnableHTTPS,fqdn"`
ServiceHostNames []string `validate:"omitempty,dive,fqdn"`
EcrRepositoryName string `validate:"required"`
EcsClusterName string `validate:"required"`
EcsCluster *ecs.CreateClusterInput
EcsServiceName string `validate:"required"`
EcsServiceDesiredCount int64 `validate:"required"`
EcsServiceMinimumHealthyPercent *int64 `validate:"omitempty"`
EcsServiceMaximumPercent *int64 `validate:"omitempty"`
EscServiceHealthCheckGracePeriodSeconds *int64 `validate:"omitempty"`
EcsExecutionRoleName string `validate:"required"`
EcsExecutionRole *iam.CreateRoleInput
EcsExecutionRolePolicyArns []string `validate:"required"`
EcsTaskRoleName string `validate:"required"`
EcsTaskRole *iam.CreateRoleInput
EcsTaskPolicyName string `validate:"required"`
EcsTaskPolicy *iam.CreatePolicyInput
EcsTaskPolicyDocument IamPolicyDocument
Ec2SecurityGroupName string `validate:"required"`
Ec2SecurityGroup *ec2.CreateSecurityGroupInput
CloudWatchLogGroupName string `validate:"required"`
CloudWatchLogGroup *cloudwatchlogs.CreateLogGroupInput
S3BucketTempPrefix string `validate:"required_with=S3BucketPrivateName S3BucketPublicName"`
S3BucketPrivateName string `validate:"omitempty"`
S3BucketPublicName string `validate:"omitempty"`
S3Buckets []S3Bucket
EnableEcsElb bool `validate:"omitempty"`
ElbLoadBalancerName string `validate:"omitempty"`
ElbDeregistrationDelay *int `validate:"omitempty"`
ElbLoadBalancer *elbv2.CreateLoadBalancerInput
ElbTargetGroupName string `validate:"omitempty"`
ElbTargetGroup *elbv2.CreateTargetGroupInput
VpcPublicName string `validate:"omitempty"`
VpcPublic *ec2.CreateVpcInput
VpcPublicSubnets []*ec2.CreateSubnetInput
EnableLambdaVPC bool `validate:"omitempty"`
RecreateService bool `validate:"omitempty"`
SDNamepsace *servicediscovery.CreatePrivateDnsNamespaceInput
SDService *servicediscovery.CreateServiceInput
CacheCluster *elasticache.CreateCacheClusterInput
CacheClusterParameter []*elasticache.ParameterNameValue
DBCluster *rds.CreateDBClusterInput
DBInstance *rds.CreateDBInstanceInput
flags ServiceDeployFlags
}
// NewServiceDeployRequest generates a new request for executing deployment of a single service for a given set of CLI flags.
func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*serviceDeployRequest, error) {
// Validates specified CLI flags map to struct successfully.
@ -55,39 +144,25 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
log.Printf("\t%s\tFlags ok.", tests.Success)
}
// Verifies AWS credentials specified as environment variables.
log.Println("\tVerify AWS credentials.")
var awsCreds awsCredentials
{
var err error
awsCreds, err = GetAwsCredentials(flags.Env)
if err != nil {
// Define new service request.
sr := &serviceRequest{
ServiceName: flags.ServiceName,
Env: flags.Env,
ProjectRoot: flags.ProjectRoot,
ProjectName: flags.ProjectName,
DockerFile: flags.DockerFile,
}
if err := sr.init(log); err != nil {
return nil, err
}
if awsCreds.UseRole {
log.Printf("\t\t\tUsing role")
} else {
log.Printf("\t\t\tAccessKeyID: '%s'", awsCreds.AccessKeyID)
}
log.Printf("\t\t\tRegion: '%s'", awsCreds.Region)
log.Printf("\t%s\tAWS credentials valid.", tests.Success)
}
// Generate a deploy request using CLI flags and AWS credentials.
log.Println("Generate deploy request.")
var req serviceDeployRequest
{
req = serviceDeployRequest{
// Required flags.
ServiceName: flags.ServiceName,
Env: flags.Env,
AwsCreds: awsCreds,
serviceRequest: sr,
// Optional flags.
ProjectRoot: flags.ProjectRoot,
ProjectName: flags.ProjectName,
DockerFile: flags.DockerFile,
EnableHTTPS: flags.EnableHTTPS,
ServiceHostPrimary: flags.ServiceHostPrimary,
ServiceHostNames: flags.ServiceHostNames,
@ -95,79 +170,11 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
S3BucketPublicName: flags.S3BucketPublicName,
EnableLambdaVPC: flags.EnableLambdaVPC,
EnableEcsElb: flags.EnableEcsElb,
NoBuild: flags.NoBuild,
NoDeploy: flags.NoDeploy,
NoCache: flags.NoCache,
NoPush: flags.NoPush,
RecreateService: flags.RecreateService,
flags: flags,
}
// When project root directory is empty or set to current working path, then search for the project root by locating
// the go.mod file.
log.Println("\tDetermining the project root directory.")
{
if flags.ProjectRoot == "" || flags.ProjectRoot == "." {
log.Println("\tAttempting to location project root directory from current working directory.")
var err error
req.GoModFile, err = findProjectGoModFile()
if err != nil {
return nil, err
}
req.ProjectRoot = filepath.Dir(req.GoModFile)
} else {
log.Println("\t\tUsing supplied project root directory.")
req.GoModFile = filepath.Join(flags.ProjectRoot, "go.mod")
}
log.Printf("\t\t\tproject root: %s", req.ProjectRoot)
log.Printf("\t\t\tgo.mod: %s", req.GoModFile)
}
log.Println("\tExtracting go module name from go.mod.")
{
var err error
req.GoModName, err = loadGoModName(req.GoModFile)
if err != nil {
return nil, err
}
log.Printf("\t\t\tmodule name: %s", req.GoModName)
}
log.Println("\tDetermining the project name.")
{
if flags.ProjectName != "" {
req.ProjectName = flags.ProjectName
log.Printf("\t\tUse provided value.")
} else {
req.ProjectName = filepath.Base(req.GoModName)
log.Printf("\t\tSet from go module.")
}
log.Printf("\t\t\tproject name: %s", req.ProjectName)
}
log.Println("\tAttempting to locate service directory from project root directory.")
{
if flags.DockerFile != "" {
req.DockerFile = flags.DockerFile
log.Printf("\t\tUse provided value.")
} else {
log.Printf("\t\tFind from project root looking for Dockerfile.")
var err error
req.DockerFile, err = findServiceDockerFile(req.ProjectRoot, req.ServiceName)
if err != nil {
return nil, err
}
}
req.ServiceDir = filepath.Dir(req.DockerFile)
log.Printf("\t\t\tservice directory: %s", req.ServiceDir)
log.Printf("\t\t\tdockerfile: %s", req.DockerFile)
}
// Set default configuration values. Primarily setting default values for all the AWS services:
// - AWS S3 bucket settings
// - AWS ECR repository settings
@ -325,20 +332,9 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
}
// Set default AWS ECR Repository Name.
req.EcrRepositoryName = req.ProjectName
req.EcrRepository = &ecr.CreateRepositoryInput{
RepositoryName: aws.String(req.EcrRepositoryName),
Tags: []*ecr.Tag{
&ecr.Tag{Key: aws.String(awsTagNameProject), Value: aws.String(req.ProjectName)},
&ecr.Tag{Key: aws.String(awsTagNameEnv), Value: aws.String(req.Env)},
},
}
req.EcrRepositoryName = ecrRepositoryName(req.ProjectName)
log.Printf("\t\t\tSet ECR Repository Name to '%s'.", req.EcrRepositoryName)
// Set default AWS ECR Regsistry Max Images.
req.EcrRepositoryMaxImages = defaultAwsRegistryMaxImages
log.Printf("\t\t\tSet ECR Regsistry Max Images to '%d'.", req.EcrRepositoryMaxImages)
// Set default AWS ECS Cluster Name.
req.EcsClusterName = req.ProjectName + "-" + req.Env
req.EcsCluster = &ecs.CreateClusterInput{
@ -764,7 +760,6 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
startTime := time.Now()
// Load the AWS ECR repository. Try to find by name else create new one.
var dockerLoginCmd []string
{
log.Println("ECR - Get or create repository.")
@ -776,128 +771,20 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
RepositoryNames: []*string{aws.String(req.EcrRepositoryName)},
})
if err != nil {
if aerr, ok := err.(awserr.Error); !ok || aerr.Code() != ecr.ErrCodeRepositoryNotFoundException {
return errors.Wrapf(err, "failed to describe repository '%s'", req.EcrRepositoryName)
}
// The repository should have been created by build or manually created and should exist at this point.
return errors.Wrapf(err, "Failed to describe repository '%s'.", req.EcrRepositoryName)
} else if len(descRes.Repositories) > 0 {
awsRepo = descRes.Repositories[0]
}
if awsRepo == nil {
// If no repository was found, create one.
createRes, err := svc.CreateRepository(req.EcrRepository)
if err != nil {
return errors.Wrapf(err, "failed to create repository '%s'", req.EcrRepositoryName)
}
awsRepo = createRes.Repository
log.Printf("\t\tCreated: %s.", *awsRepo.RepositoryArn)
} else {
log.Printf("\t\tFound: %s.", *awsRepo.RepositoryArn)
log.Println("\t\tChecking old ECR images.")
delIds, err := EcrPurgeImages(req)
req.ReleaseImage = releaseImage(req.Env, req.ServiceName, *awsRepo.RepositoryUri)
if err != nil {
return err
}
// Since ECR has max number of repository images, need to delete old ones so can stay under limit.
// If there are image IDs to delete, delete them.
if len(delIds) > 0 {
log.Printf("\t\tDeleted %d images that exceeded limit of %d", len(delIds), req.EcrRepositoryMaxImages)
for _, imgId := range delIds {
log.Printf("\t\t\t%s", *imgId.ImageTag)
}
}
}
// Tag of release image will always include one with environment and service name.
tag1 := req.Env + "-" + req.ServiceName
// Generate tags for the release image.
if v := os.Getenv("CI_COMMIT_REF_NAME"); v != "" {
tag2 := tag1 + "-" + v
req.ReleaseImage = *awsRepo.RepositoryUri + ":" + tag2
} else {
req.ReleaseImage = *awsRepo.RepositoryUri + ":" + tag1
}
log.Printf("\t\trelease image: %s", req.ReleaseImage)
log.Printf("\t%s\tRelease image valid.", tests.Success)
log.Println("ECR - Retrieve authorization token used for docker login.")
// Get the credentials necessary for logging into the AWS Elastic Container Registry
// made available with the AWS access key and AWS secret access keys.
res, err := svc.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
if err != nil {
return errors.Wrap(err, "failed to get ecr authorization token")
}
authToken, err := base64.StdEncoding.DecodeString(*res.AuthorizationData[0].AuthorizationToken)
if err != nil {
return errors.Wrap(err, "failed to base64 decode ecr authorization token")
}
pts := strings.Split(string(authToken), ":")
user := pts[0]
pass := pts[1]
dockerLoginCmd = []string{
"docker",
"login",
"-u", user,
"-p", pass,
*res.AuthorizationData[0].ProxyEndpoint,
}
log.Printf("\t%s\tdocker login ok.", tests.Success)
}
// Once we can access the repository in ECR, do the docker build.
if req.NoBuild == false {
dockerFile, err := filepath.Rel(req.ProjectRoot, req.DockerFile)
if err != nil {
return errors.Wrapf(err, "Failed parse relative path for %s from %s", req.DockerFile, req.ProjectRoot)
}
// The initial build command slice.
buildCmd := []string{
"docker", "build",
"--file="+dockerFile,
"--build-arg", "service="+req.ServiceName,
"--build-arg", "env="+req.Env,
"-t", req.ReleaseImage,
".",
}
log.Println("Starting docker build")
err = execCmds(log, req.ProjectRoot, buildCmd)
if err != nil {
return errors.Wrap(err, "Failed to build docker image")
}
// Push the newly built image of the Docker container to the registry.
if req.NoPush == false {
log.Printf("\t\tDocker Login")
err = execCmds(log, req.ProjectRoot, dockerLoginCmd)
if err != nil {
return errors.Wrapf(err, "Failed to login to AWS ECR")
}
log.Printf("\t\tPush release image %s", req.ReleaseImage)
err = execCmds(log, req.ProjectRoot, []string{"docker", "push", req.ReleaseImage})
if err != nil {
return errors.Wrapf(err, "Failed to push docker image %s", req.ReleaseImage)
}
}
log.Printf("\t%s\tbuild complete.\n", tests.Success)
}
// If skip deploy is set, exit and don't continue.
// Used to test build for a service without actually pushing image to specified environment.
if req.NoDeploy == true {
return nil
}
// Try to find the Datadog API key, this value is optional.

144
tools/devops/cmd/deploy/devops.go
View File

@ -1,144 +0,0 @@
package deploy
import (
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/pkg/errors"
)
// findProjectGoModFile finds the project root directory from the current working directory.
func findProjectGoModFile() (string, error) {
var err error
projectRoot, err := os.Getwd()
if err != nil {
return "", errors.WithMessage(err, "failed to get current working directory")
}
// Try to find the project root for looking for the go.mod file in a parent directory.
var goModFile string
testDir := projectRoot
for i := 0; i < 3; i++ {
if goModFile != "" {
testDir = filepath.Join(testDir, "../")
}
goModFile = filepath.Join(testDir, "go.mod")
ok, _ := exists(goModFile)
if ok {
projectRoot = testDir
break
}
}
// Verify the go.mod file was found.
ok, err := exists(goModFile)
if err != nil {
return "", errors.WithMessagef(err, "failed to load go.mod for project using project root %s")
} else if !ok {
return "", errors.Errorf("failed to locate project go.mod in project root %s", projectRoot)
}
return goModFile, nil
}
// findServiceDockerFile finds the service directory.
func findServiceDockerFile(projectRoot, targetService string) (string, error) {
checkDirs := []string{
filepath.Join(projectRoot, "cmd", targetService),
filepath.Join(projectRoot, "tools", targetService),
}
var dockerFile string
for _, cd := range checkDirs {
// Check to see if directory contains Dockerfile.
tf := filepath.Join(cd, "Dockerfile")
ok, _ := exists(tf)
if ok {
dockerFile = tf
break
}
}
if dockerFile == "" {
return "", errors.Errorf("failed to locate Dockerfile for service %s", targetService)
}
return dockerFile, nil
}
// getTargetEnv checks for an env var that is prefixed with the current target env.
func getTargetEnv(targetEnv, envName string) string {
k := fmt.Sprintf("%s_%s", strings.ToUpper(targetEnv), envName)
if v := os.Getenv(k); v != "" {
// Set the non prefixed env var with the prefixed value.
os.Setenv(envName, v)
return v
}
return os.Getenv(envName)
}
// loadGoModName parses out the module name from go.mod.
func loadGoModName(goModFile string) (string, error) {
ok, err := exists(goModFile)
if err != nil {
return "", errors.WithMessage(err, "Failed to load go.mod for project")
} else if !ok {
return "", errors.Errorf("Failed to locate project go.mod at %s", goModFile)
}
b, err := ioutil.ReadFile(goModFile)
if err != nil {
return "", errors.WithMessagef(err, "Failed to read go.mod at %s", goModFile)
}
var name string
lines := strings.Split(string(b), "\n")
for _, l := range lines {
if strings.HasPrefix(l, "module ") {
name = strings.TrimSpace(strings.Split(l, " ")[1])
break
}
}
return name, nil
}
// exists returns a bool as to whether a file path exists.
func exists(path string) (bool, error) {
_, err := os.Stat(path)
if err == nil {
return true, nil
}
if os.IsNotExist(err) {
return false, nil
}
return true, err
}
// execCmds executes a set of commands using the current env variables.
func execCmds(log *log.Logger, workDir string, cmds ...[]string) (error) {
for _, cmdVals := range cmds {
cmd := exec.Command(cmdVals[0], cmdVals[1:]...)
cmd.Dir = workDir
cmd.Env = os.Environ()
cmd.Stderr = log.Writer()
cmd.Stdout = log.Writer()
err := cmd.Run()
if err != nil {
return errors.WithMessagef(err, "failed to execute %s", strings.Join(cmdVals, " "))
}
}
return nil
}

212
tools/devops/cmd/deploy/models.go
View File

@ -1,212 +0,0 @@
package deploy
import (
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/ecr"
"github.com/aws/aws-sdk-go/service/ecs"
"github.com/aws/aws-sdk-go/service/elasticache"
"github.com/aws/aws-sdk-go/service/elbv2"
"github.com/aws/aws-sdk-go/service/iam"
"github.com/aws/aws-sdk-go/service/rds"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/servicediscovery"
"github.com/iancoleman/strcase"
"github.com/urfave/cli"
)
// ServiceDeployFlags defines the flags used for executing a service deployment.
type ServiceDeployFlags struct {
// Required flags.
ServiceName string `validate:"required" example:"web-api"`
Env string `validate:"oneof=dev stage prod" example:"dev"`
// Optional flags.
EnableHTTPS bool `validate:"omitempty" example:"false"`
ServiceHostPrimary string `validate:"omitempty" example:"example-project.com"`
ServiceHostNames cli.StringSlice `validate:"omitempty" example:"subdomain.example-project.com"`
S3BucketPrivateName string `validate:"omitempty" example:"saas-example-project-private"`
S3BucketPublicName string `validate:"omitempty" example:"saas-example-project-public"`
ProjectRoot string `validate:"omitempty" example:"."`
ProjectName string ` validate:"omitempty" example:"example-project"`
DockerFile string `validate:"omitempty" example:"./cmd/web-api/Dockerfile"`
EnableLambdaVPC bool `validate:"omitempty" example:"false"`
EnableEcsElb bool `validate:"omitempty" example:"false"`
NoBuild bool `validate:"omitempty" example:"false"`
NoDeploy bool `validate:"omitempty" example:"false"`
NoCache bool `validate:"omitempty" example:"false"`
NoPush bool `validate:"omitempty" example:"false"`
RecreateService bool `validate:"omitempty" example:"false"`
}
// serviceDeployRequest defines the details needed to execute a service deployment.
type serviceDeployRequest struct {
ServiceName string `validate:"required"`
ServiceDir string `validate:"required"`
Env string `validate:"oneof=dev stage prod"`
ProjectRoot string `validate:"required"`
ProjectName string `validate:"required"`
DockerFile string `validate:"required"`
GoModFile string `validate:"required"`
GoModName string `validate:"required"`
EnableHTTPS bool `validate:"omitempty"`
ServiceHostPrimary string `validate:"omitempty,required_with=EnableHTTPS,fqdn"`
ServiceHostNames []string `validate:"omitempty,dive,fqdn"`
AwsCreds awsCredentials `validate:"required,dive,required"`
EcrRepositoryName string `validate:"required"`
EcrRepository *ecr.CreateRepositoryInput
EcrRepositoryMaxImages int `validate:"omitempty"`
EcsClusterName string `validate:"required"`
EcsCluster *ecs.CreateClusterInput
EcsServiceName string `validate:"required"`
EcsServiceDesiredCount int64 `validate:"required"`
EcsServiceMinimumHealthyPercent *int64 `validate:"omitempty"`
EcsServiceMaximumPercent *int64 `validate:"omitempty"`
EscServiceHealthCheckGracePeriodSeconds *int64 `validate:"omitempty"`
EcsExecutionRoleName string `validate:"required"`
EcsExecutionRole *iam.CreateRoleInput
EcsExecutionRolePolicyArns []string `validate:"required"`
EcsTaskRoleName string `validate:"required"`
EcsTaskRole *iam.CreateRoleInput
EcsTaskPolicyName string `validate:"required"`
EcsTaskPolicy *iam.CreatePolicyInput
EcsTaskPolicyDocument IamPolicyDocument
Ec2SecurityGroupName string `validate:"required"`
Ec2SecurityGroup *ec2.CreateSecurityGroupInput
CloudWatchLogGroupName string `validate:"required"`
CloudWatchLogGroup *cloudwatchlogs.CreateLogGroupInput
S3BucketTempPrefix string `validate:"required_with=S3BucketPrivateName S3BucketPublicName"`
S3BucketPrivateName string `validate:"omitempty"`
S3BucketPublicName string `validate:"omitempty"`
S3Buckets []S3Bucket
EnableEcsElb bool `validate:"omitempty"`
ElbLoadBalancerName string `validate:"omitempty"`
ElbDeregistrationDelay *int `validate:"omitempty"`
ElbLoadBalancer *elbv2.CreateLoadBalancerInput
ElbTargetGroupName string `validate:"omitempty"`
ElbTargetGroup *elbv2.CreateTargetGroupInput
VpcPublicName string `validate:"omitempty"`
VpcPublic *ec2.CreateVpcInput
VpcPublicSubnets []*ec2.CreateSubnetInput
EnableLambdaVPC bool `validate:"omitempty"`
NoBuild bool `validate:"omitempty"`
NoDeploy bool `validate:"omitempty"`
NoCache bool `validate:"omitempty"`
NoPush bool `validate:"omitempty"`
RecreateService bool `validate:"omitempty"`
SDNamepsace *servicediscovery.CreatePrivateDnsNamespaceInput
SDService *servicediscovery.CreateServiceInput
CacheCluster *elasticache.CreateCacheClusterInput
CacheClusterParameter []*elasticache.ParameterNameValue
DBCluster *rds.CreateDBClusterInput
DBInstance *rds.CreateDBInstanceInput
ReleaseImage string
flags ServiceDeployFlags
_awsSession *session.Session
}
type S3Bucket struct {
Name string `validate:"omitempty"`
Input *s3.CreateBucketInput
LifecycleRules []*s3.LifecycleRule
CORSRules []*s3.CORSRule
PublicAccessBlock *s3.PublicAccessBlockConfiguration
Policy string
}
// DB mimics the general info needed for services used to define placeholders.
type DB struct {
Host string
User string
Pass string
Database string
Driver string
DisableTLS bool
}
// projectNameCamel takes a project name and returns the camel cased version.
func (r *serviceDeployRequest) ProjectNameCamel() string {
s := strings.Replace(r.ProjectName, "_", " ", -1)
s = strings.Replace(s, "-", " ", -1)
s = strcase.ToCamel(s)
return s
}
// awsSession returns the current AWS session for the serviceDeployRequest.
func (r *serviceDeployRequest) awsSession() *session.Session {
if r._awsSession == nil {
r._awsSession = r.AwsCreds.Session()
}
return r._awsSession
}
// AwsCredentials defines AWS credentials used for deployment. Unable to use roles when deploying
// using gitlab CI/CD pipeline.
type awsCredentials struct {
AccessKeyID string `validate:"required_without=UseRole"`
SecretAccessKey string `validate:"required_without=UseRole"`
Region string `validate:"required_without=UseRole"`
UseRole bool
}
// Session returns a new AWS Session used to access AWS services.
func (creds awsCredentials) Session() *session.Session {
if creds.UseRole {
// Get an AWS session from an implicit source if no explicit
// configuration is provided. This is useful for taking advantage of
// EC2/ECS instance roles.
sess := session.Must(session.NewSession())
if creds.Region != "" {
sess.Config.WithRegion(creds.Region)
}
return sess
}
return session.New(
&aws.Config{
Region: aws.String(creds.Region),
Credentials: credentials.NewStaticCredentials(creds.AccessKeyID, creds.SecretAccessKey, ""),
})
}
// IamPolicyDocument defines an AWS IAM policy used for defining access for IAM roles, users, and groups.
type IamPolicyDocument struct {
Version string `json:"Version"`
Statement []IamStatementEntry `json:"Statement"`
}
// IamStatementEntry defines a single statement for an IAM policy.
type IamStatementEntry struct {
Sid string `json:"Sid"`
Effect string `json:"Effect"`
Action []string `json:"Action"`
Resource interface{} `json:"Resource"`
}

45
tools/devops/main.go
View File

@ -6,7 +6,7 @@ import (
"os"
"strings"
"geeks-accelerator/oss/saas-starter-kit/tools/devops/cmd/deploy"
"geeks-accelerator/oss/saas-starter-kit/tools/devops/cmd/cicd"
_ "github.com/lib/pq"
"github.com/urfave/cli"
)
@ -38,13 +38,35 @@ func main() {
// =========================================================================
// Start Truss
var deployFlags deploy.ServiceDeployFlags
var (
buildFlags cicd.ServiceBuildFlags
deployFlags cicd.ServiceDeployFlags
)
app := cli.NewApp()
app.Commands = []cli.Command{
{
Name: "build",
Usage: "-service=web-api -env=dev",
Flags: []cli.Flag{
cli.StringFlag{Name: "service", Usage: "name of cmd", Destination: &buildFlags.ServiceName},
cli.StringFlag{Name: "env", Usage: "dev, stage, or prod", Destination: &buildFlags.Env},
cli.StringFlag{Name: "dockerfile", Usage: "DockerFile for service", Destination: &buildFlags.DockerFile},
cli.StringFlag{Name: "root", Usage: "project root directory", Destination: &buildFlags.ProjectRoot},
cli.StringFlag{Name: "project", Usage: "name of project", Destination: &buildFlags.ProjectName},
cli.BoolFlag{Name: "no_cache", Usage: "skip docker cache", Destination: &buildFlags.NoCache},
cli.BoolFlag{Name: "no_push", Usage: "skip docker push after build", Destination: &buildFlags.NoPush},
},
Action: func(c *cli.Context) error {
req, err := cicd.NewServiceBuildRequest(log, buildFlags)
if err != nil {
return err
}
return cicd.ServiceBuild(log, req)
},
},
{
Name: "deploy",
Aliases: []string{"serviceDeploy"},
Usage: "-service=web-api -env=dev",
Flags: []cli.Flag{
cli.StringFlag{Name: "service", Usage: "name of cmd", Destination: &deployFlags.ServiceName},
@ -59,10 +81,6 @@ func main() {
cli.StringFlag{Name: "project", Usage: "name of project", Destination: &deployFlags.ProjectName},
cli.BoolFlag{Name: "enable_elb", Usage: "enable deployed to use Elastic Load Balancer", Destination: &deployFlags.EnableEcsElb},
cli.BoolTFlag{Name: "lambda_vpc", Usage: "deploy lambda behind VPC", Destination: &deployFlags.EnableLambdaVPC},
cli.BoolFlag{Name: "no_build", Usage: "skip build and continue directly to deploy", Destination: &deployFlags.NoBuild},
cli.BoolFlag{Name: "no_deploy", Usage: "skip deploy after build", Destination: &deployFlags.NoDeploy},
cli.BoolFlag{Name: "no_cache", Usage: "skip docker cache", Destination: &deployFlags.NoCache},
cli.BoolFlag{Name: "no_push", Usage: "skip docker push after build", Destination: &deployFlags.NoPush},
cli.BoolFlag{Name: "recreate_service", Usage: "skip docker push after build", Destination: &deployFlags.RecreateService},
},
Action: func(c *cli.Context) error {
@ -82,11 +100,20 @@ func main() {
deployFlags.ServiceHostNames = hostNames
}
req, err := deploy.NewServiceDeployRequest(log, deployFlags)
req, err := cicd.NewServiceDeployRequest(log, deployFlags)
if err != nil {
return err
}
return deploy.ServiceDeploy(log, req)
return cicd.ServiceDeploy(log, req)
},
},
{
Name: "migrate",
Usage: "-env=dev",
Flags: []cli.Flag{},
Action: func(c *cli.Context) error {
return nil
},
},
}

3
tools/schema/main.go
View File

@ -34,7 +34,6 @@ type DB struct {
DisableTLS bool
}
func main() {
// =========================================================================
// Logging
@ -81,7 +80,6 @@ func main() {
return // We displayed help.
}
/*
//
DBInstanceIdentifier: aws.String(req.ProjectName + "-" + req.Env),
@ -92,7 +90,6 @@ func main() {
*/
// =========================================================================
// Log App Info