go/golang-saas-starter-kit
1
0
Fork 0
mirror of https://github.com/raseels-repos/golang-saas-starter-kit.git synced 2025-08-08 22:36:41 +02:00
Code Issues Releases Activity

devops migration

Browse Source
This commit is contained in:
Lee Brown
2019-08-21 14:31:28 -08:00
parent 1a49f8c4fb
commit f9273881bd
7 changed files with 2318 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

498
build/cicd/internal/config/config.go Normal file
View File

@ -0,0 +1,498 @@
package config
import (
"context"
"fmt"
"log"
"os"
"path/filepath"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/cloudfront"
"github.com/aws/aws-sdk-go/service/rds"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/secretsmanager"
"github.com/iancoleman/strcase"
"github.com/jmoiron/sqlx"
"github.com/pkg/errors"
"geeks-accelerator/oss/saas-starter-kit/internal/schema"
"gitlab.com/geeks-accelerator/oss/devops/pkg/devdeploy"
)
const (
// ProjectNamePrefix will be appending to the name of the project.
ProjectNamePrefix = ""
// GitLabProjectBaseUrl is the base url used to create links to a specific CI/CD job or pipeline by ID.
GitLabProjectBaseUrl = "https://gitlab.com/geeks-accelerator/oss/saas-starter-kit"
)
// Env defines the target deployment environment.
type Env = string
var (
EnvDev Env = "dev"
EnvStage Env = "stage"
EnvProd Env = "prod"
)
// List of env names used by main.go for help.
var EnvNames = []Function{
EnvDev,
EnvStage,
EnvProd,
}
// ConfigContext defines the flags for build env.
type ConfigContext struct {
// Env is the target environment used for the deployment.
Env string `validate:"oneof=dev stage prod"`
// AwsCredentials defines the credentials used for deployment.
AwsCredentials devdeploy.AwsCredentials `validate:"required,dive,required"`
}
// NewConfigContext returns the ConfigContext.
func NewConfigContext(targetEnv Env, awsCredentials devdeploy.AwsCredentials) (*ConfigContext, error) {
ctx := &ConfigContext{
Env: targetEnv,
AwsCredentials: awsCredentials,
}
// If AWS Credentials are not set and use role is not enabled, try to load the credentials from env vars.
if ctx.AwsCredentials.UseRole == false && ctx.AwsCredentials.AccessKeyID == "" {
var err error
ctx.AwsCredentials, err = devdeploy.GetAwsCredentialsFromEnv(ctx.Env)
if err != nil {
return nil, err
}
} else if ctx.AwsCredentials.Region == "" {
awsCreds, err := devdeploy.GetAwsCredentialsFromEnv(ctx.Env)
if err != nil {
return nil, err
}
ctx.AwsCredentials.Region = awsCreds.Region
}
return ctx, nil
}
// Config defines the details to setup the target environment for the project to build services and functions.
func (cfgCtx *ConfigContext) Config(log *log.Logger) (*devdeploy.Config, error) {
// Init a new build target environment for the project.
cfg := &devdeploy.Config{
Env: cfgCtx.Env,
AwsCredentials: cfgCtx.AwsCredentials,
}
// Get the current working directory. This should be somewhere contained within the project.
workDir, err := os.Getwd()
if err != nil {
return cfg, errors.WithMessage(err, "Failed to get current working directory.")
}
// Set the project root directory and project name. This is current set by finding the go.mod file for the project
// repo. Project name is the directory name.
modDetails, err := devdeploy.LoadModuleDetails(workDir)
if err != nil {
return cfg, err
}
// ProjectRoot should be the root directory for the project.
cfg.ProjectRoot = modDetails.ProjectRoot
// ProjectName will be used for prefixing AWS resources. This could be changed as needed or manually defined.
cfg.ProjectName = ProjectNamePrefix + modDetails.ProjectName
// Set default AWS ECR Repository Name.
cfg.AwsEcrRepository = &devdeploy.AwsEcrRepository{
RepositoryName: cfg.ProjectName,
Tags: []devdeploy.Tag{
{Key: devdeploy.AwsTagNameProject, Value: cfg.ProjectName},
{Key: devdeploy.AwsTagNameEnv, Value: cfg.Env},
},
}
// Set the deployment to use the default VPC for the region.
cfg.AwsEc2Vpc = &devdeploy.AwsEc2Vpc{
IsDefault: true,
}
// Set the security group to use for the deployed services, database and cluster. This will used the VPC ID defined
// for the deployment.
cfg.AwsEc2SecurityGroup = &devdeploy.AwsEc2SecurityGroup{
GroupName: cfg.ProjectName + "-" + cfg.Env,
Description: fmt.Sprintf("Security group for %s services running on ECS", cfg.ProjectName),
Tags: []devdeploy.Tag{
{Key: devdeploy.AwsTagNameProject, Value: cfg.ProjectName},
{Key: devdeploy.AwsTagNameEnv, Value: cfg.Env},
},
}
// Set the name of the EC2 Security Group used by the gitlab runner. This is used to ensure the security
// group defined above has access to the RDS cluster/instance and can thus handle schema migrations.
cfg.GitlabRunnerEc2SecurityGroupName = "gitlab-runner"
// Set the s3 buckets used by the deployed services.
// S3 temp prefix used by services for short term storage. A lifecycle policy will be used for expiration.
s3BucketTempPrefix := "tmp/"
// Defines a life cycle policy to expire keys for the temp directory.
bucketLifecycleTempRule := &s3.LifecycleRule{
ID: aws.String("Rule for : " + s3BucketTempPrefix),
Status: aws.String("Enabled"),
Filter: &s3.LifecycleRuleFilter{
Prefix: aws.String(s3BucketTempPrefix),
},
Expiration: &s3.LifecycleExpiration{
// Indicates the lifetime, in days, of the objects that are subject to the rule.
// The value must be a non-zero positive integer.
Days: aws.Int64(1),
},
// Specifies the days since the initiation of an incomplete multipart upload
// that Amazon S3 will wait before permanently removing all parts of the upload.
// For more information, see Aborting Incomplete Multipart Uploads Using a Bucket
// Lifecycle Policy (https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html#mpu-abort-incomplete-mpu-lifecycle-config)
// in the Amazon Simple Storage Service Developer Guide.
AbortIncompleteMultipartUpload: &s3.AbortIncompleteMultipartUpload{
DaysAfterInitiation: aws.Int64(1),
},
}
// Define the public S3 bucket used to serve static files for all the services.
cfg.AwsS3BucketPublic = &devdeploy.AwsS3Bucket{
BucketName: cfg.ProjectName + "-public",
IsPublic: true,
TempPrefix: s3BucketTempPrefix,
LocationConstraint: &cfg.AwsCredentials.Region,
LifecycleRules: []*s3.LifecycleRule{bucketLifecycleTempRule},
CORSRules: []*s3.CORSRule{
&s3.CORSRule{
// Headers that are specified in the Access-Control-Request-Headers header.
// These headers are allowed in a preflight OPTIONS request. In response to
// any preflight OPTIONS request, Amazon S3 returns any requested headers that
// are allowed.
// AllowedHeaders: aws.StringSlice([]string{}),
// An HTTP method that you allow the origin to execute. Valid values are GET,
// PUT, HEAD, POST, and DELETE.
//
// AllowedMethods is a required field
AllowedMethods: aws.StringSlice([]string{"GET", "POST"}),
// One or more origins you want customers to be able to access the bucket from.
//
// AllowedOrigins is a required field
AllowedOrigins: aws.StringSlice([]string{"*"}),
// One or more headers in the response that you want customers to be able to
// access from their applications (for example, from a JavaScript XMLHttpRequest
// object).
// ExposeHeaders: aws.StringSlice([]string{}),
// The time in seconds that your browser is to cache the preflight response
// for the specified resource.
// MaxAgeSeconds: aws.Int64(),
},
},
}
// The base s3 key prefix used to upload static files.
cfg.AwsS3BucketPublicKeyPrefix = "/public"
// For production, enable Cloudfront CDN for all static files to avoid serving them from the slower S3 option.
if cfg.Env == EnvProd {
cfg.AwsS3BucketPublic.CloudFront = &devdeploy.AwsS3BucketCloudFront{
// S3 key prefix to request your content from a directory in your Amazon S3 bucket.
OriginPath: cfg.AwsS3BucketPublicKeyPrefix,
// A complex type that controls whether CloudFront caches the response to requests.
CachedMethods: []string{"HEAD", "GET"},
// The distribution's configuration information.
DistributionConfig: &cloudfront.DistributionConfig{
Comment: aws.String(""),
Enabled: aws.Bool(true),
HttpVersion: aws.String("http2"),
IsIPV6Enabled: aws.Bool(true),
DefaultCacheBehavior: &cloudfront.DefaultCacheBehavior{
Compress: aws.Bool(true),
DefaultTTL: aws.Int64(1209600),
MinTTL: aws.Int64(604800),
MaxTTL: aws.Int64(31536000),
ForwardedValues: &cloudfront.ForwardedValues{
QueryString: aws.Bool(true),
Cookies: &cloudfront.CookiePreference{
Forward: aws.String("none"),
},
},
TrustedSigners: &cloudfront.TrustedSigners{
Enabled: aws.Bool(false),
Quantity: aws.Int64(0),
},
ViewerProtocolPolicy: aws.String("allow-all"),
},
ViewerCertificate: &cloudfront.ViewerCertificate{
CertificateSource: aws.String("cloudfront"),
MinimumProtocolVersion: aws.String("TLSv1"),
CloudFrontDefaultCertificate: aws.Bool(true),
},
PriceClass: aws.String("PriceClass_All"),
CallerReference: aws.String("devops-deploy" + cfg.AwsS3BucketPublic.BucketName),
},
}
}
// Define the private S3 bucket used for long term file storage including but not limited to: log exports,
// AWS Lambda code, application caching.
cfg.AwsS3BucketPrivate = &devdeploy.AwsS3Bucket{
BucketName: cfg.ProjectName + "-private",
IsPublic: false,
TempPrefix: s3BucketTempPrefix,
LocationConstraint: &cfg.AwsCredentials.Region,
LifecycleRules: []*s3.LifecycleRule{bucketLifecycleTempRule},
PublicAccessBlock: &s3.PublicAccessBlockConfiguration{
// Specifies whether Amazon S3 should block public access control lists (ACLs)
// for this bucket and objects in this bucket. Setting this element to TRUE
// causes the following behavior:
//
// * PUT Bucket acl and PUT Object acl calls fail if the specified ACL is
// public.
//
// * PUT Object calls fail if the request includes a public ACL.
//
// Enabling this setting doesn't affect existing policies or ACLs.
BlockPublicAcls: aws.Bool(true),
// Specifies whether Amazon S3 should block public bucket policies for this
// bucket. Setting this element to TRUE causes Amazon S3 to reject calls to
// PUT Bucket policy if the specified bucket policy allows public access.
//
// Enabling this setting doesn't affect existing bucket policies.
BlockPublicPolicy: aws.Bool(true),
// Specifies whether Amazon S3 should restrict public bucket policies for this
// bucket. Setting this element to TRUE restricts access to this bucket to only
// AWS services and authorized users within this account if the bucket has a
// public policy.
//
// Enabling this setting doesn't affect previously stored bucket policies, except
// that public and cross-account access within any public bucket policy, including
// non-public delegation to specific accounts, is blocked.
RestrictPublicBuckets: aws.Bool(true),
// Specifies whether Amazon S3 should ignore public ACLs for this bucket and
// objects in this bucket. Setting this element to TRUE causes Amazon S3 to
// ignore all public ACLs on this bucket and objects in this bucket.
//
// Enabling this setting doesn't affect the persistence of any existing ACLs
// and doesn't prevent new public ACLs from being set.
IgnorePublicAcls: aws.Bool(true),
},
}
// Add a bucket policy to enable exports from Cloudwatch Logs for the private S3 bucket.
cfg.AwsS3BucketPrivate.Policy = func() string {
policyResource := strings.Trim(filepath.Join(cfg.AwsS3BucketPrivate.BucketName, cfg.AwsS3BucketPrivate.TempPrefix), "/")
return fmt.Sprintf(`{
"Version": "2012-10-17",
"Statement": [
{
"Action": "s3:GetBucketAcl",
"Effect": "Allow",
"Resource": "arn:aws:s3:::%s",
"Principal": { "Service": "logs.%s.amazonaws.com" }
},
{
"Action": "s3:PutObject" ,
"Effect": "Allow",
"Resource": "arn:aws:s3:::%s/*",
"Condition": { "StringEquals": { "s3:x-amz-acl": "bucket-owner-full-control" } },
"Principal": { "Service": "logs.%s.amazonaws.com" }
}
]
}`, cfg.AwsS3BucketPrivate.BucketName, cfg.AwsCredentials.Region, policyResource, cfg.AwsCredentials.Region)
}()
// Define the Redis Cache cluster used for ephemeral storage.
cfg.AwsElasticCacheCluster = &devdeploy.AwsElasticCacheCluster{
CacheClusterId: cfg.ProjectName + "-" + cfg.Env,
CacheNodeType: "cache.t2.micro",
CacheSubnetGroupName: "default",
Engine: "redis",
EngineVersion: "5.0.4",
NumCacheNodes: 1,
Port: 6379,
AutoMinorVersionUpgrade: aws.Bool(true),
SnapshotRetentionLimit: aws.Int64(7),
ParameterNameValues: []devdeploy.AwsElasticCacheParameter{
devdeploy.AwsElasticCacheParameter{
ParameterName: "maxmemory-policy",
ParameterValue: "allkeys-lru",
},
},
}
// Define the RDS Database instance for transactional data. A random one will be generated for any created instance.
cfg.AwsRdsDBInstance = &devdeploy.AwsRdsDBInstance{
DBInstanceIdentifier: cfg.ProjectName + "-" + cfg.Env,
DBName: "shared",
Engine: "postgres",
MasterUsername: "god",
Port: 5432,
DBInstanceClass: "db.t2.small",
AllocatedStorage: 20,
PubliclyAccessible: false,
BackupRetentionPeriod: aws.Int64(7),
AutoMinorVersionUpgrade: true,
CopyTagsToSnapshot: aws.Bool(true),
Tags: []devdeploy.Tag{
{Key: devdeploy.AwsTagNameProject, Value: cfg.ProjectName},
{Key: devdeploy.AwsTagNameEnv, Value: cfg.Env},
},
AfterCreate: func(res *rds.DBInstance, dbInfo *devdeploy.DBConnInfo) error {
masterDb, err := sqlx.Open(dbInfo.Driver, dbInfo.URL())
if err != nil {
return errors.WithMessage(err, "Failed to connect to db for schema migration.")
}
defer masterDb.Close()
return schema.Migrate(context.Background(), masterDb, log, false)
},
}
// AwsIamPolicy defines the name and policy that will be attached to the task role. The policy document grants
// the permissions required for deployed services to access AWS services. If the policy already exists, the
// statements will be used to add new required actions, but not for removal.
cfg.AwsIamPolicy = &devdeploy.AwsIamPolicy{
PolicyName: fmt.Sprintf("%s%sServices", cfg.ProjectNameCamel(), strcase.ToCamel(cfg.Env)),
Description: fmt.Sprintf("Defines access for %s services. ", cfg.ProjectName),
PolicyDocument: devdeploy.AwsIamPolicyDocument{
Version: "2012-10-17",
Statement: []devdeploy.AwsIamStatementEntry{
{
Sid: "DefaultServiceAccess",
Effect: "Allow",
Action: []string{
"s3:HeadBucket",
"s3:ListObjects",
"s3:PutObject",
"s3:PutObjectAcl",
"cloudfront:ListDistributions",
"ec2:DescribeNetworkInterfaces",
"ec2:DeleteNetworkInterface",
"ecs:ListTasks",
"ecs:DescribeServices",
"ecs:DescribeTasks",
"ec2:DescribeNetworkInterfaces",
"route53:ListHostedZones",
"route53:ListResourceRecordSets",
"route53:ChangeResourceRecordSets",
"ecs:UpdateService",
"ses:SendEmail",
"ses:ListIdentities",
"secretsmanager:ListSecretVersionIds",
"secretsmanager:GetSecretValue",
"secretsmanager:CreateSecret",
"secretsmanager:UpdateSecret",
"secretsmanager:RestoreSecret",
"secretsmanager:DeleteSecret",
},
Resource: "*",
},
{
Sid: "ServiceInvokeLambda",
Effect: "Allow",
Action: []string{
"iam:GetRole",
"lambda:InvokeFunction",
"lambda:ListVersionsByFunction",
"lambda:GetFunction",
"lambda:InvokeAsync",
"lambda:GetFunctionConfiguration",
"iam:PassRole",
"lambda:GetAlias",
"lambda:GetPolicy",
},
Resource: []string{
"arn:aws:iam:::role/*",
"arn:aws:lambda:::function:*",
},
},
{
Sid: "datadoglambda",
Effect: "Allow",
Action: []string{
"cloudwatch:Get*",
"cloudwatch:List*",
"ec2:Describe*",
"support:*",
"tag:GetResources",
"tag:GetTagKeys",
"tag:GetTagValues",
},
Resource: "*",
},
},
},
}
log.Printf("\t\tSet Task Policy Name to '%s'.", cfg.AwsIamPolicy.PolicyName)
return cfg, nil
}
// getDatadogApiKey tries to find the datadog api key from env variable or AWS Secrets Manager.
func getDatadogApiKey(cfg *devdeploy.Config) (string, error) {
// Load Datadog API key which can be either stored in an environment variable or in AWS Secrets Manager.
// 1. Check env vars for [DEV|STAGE|PROD]_DD_API_KEY and DD_API_KEY
apiKey := devdeploy.GetTargetEnv(cfg.Env, "DD_API_KEY")
// 2. Check AWS Secrets Manager for datadog entry prefixed with target environment.
if apiKey == "" {
prefixedSecretId := cfg.SecretID("datadog")
var err error
apiKey, err = devdeploy.GetAwsSecretValue(cfg.AwsCredentials, prefixedSecretId)
if err != nil {
if aerr, ok := errors.Cause(err).(awserr.Error); !ok || aerr.Code() != secretsmanager.ErrCodeResourceNotFoundException {
return "", err
}
}
}
// 3. Check AWS Secrets Manager for Datadog entry.
if apiKey == "" {
secretId := "DATADOG"
var err error
apiKey, err = devdeploy.GetAwsSecretValue(cfg.AwsCredentials, secretId)
if err != nil {
if aerr, ok := errors.Cause(err).(awserr.Error); !ok || aerr.Code() != secretsmanager.ErrCodeResourceNotFoundException {
return "", err
}
}
}
return apiKey, nil
}
// getCommitRef returns a string that will be used by go build to replace main.go:build constant.
func getCommitRef() string {
var commitRef string
// Set the commit ref based on the GitLab CI/CD environment variables.
if ev := os.Getenv("CI_COMMIT_TAG"); ev != "" {
commitRef = "tag-" + ev
} else if ev := os.Getenv("CI_COMMIT_REF_NAME"); ev != "" {
commitRef = "branch-" + ev
}
if commitRef != "" {
if ev := os.Getenv("CI_COMMIT_SHORT_SHA"); ev != "" {
commitRef = commitRef + "@" + ev
}
}
return commitRef
}

385
build/cicd/internal/config/function.go Normal file
View File

@ -0,0 +1,385 @@
package config
import (
"log"
"path/filepath"
"encoding/json"
"github.com/aws/aws-sdk-go/aws"
"github.com/pkg/errors"
"gitlab.com/geeks-accelerator/oss/devops/pkg/devdeploy"
)
// Function define the name of a function.
type Function = string
var (
Function_Ddlogscollector = "ddlogscollector"
Function_YourNewFunction = "your-new-function"
)
// List of function names used by main.go for help.
var FunctionNames = []Function{
// Python Datadog Logs Collector
Function_Ddlogscollector,
Function_YourNewFunction,
}
// FunctionContext defines the flags for deploying a function.
type FunctionContext struct {
// Required flags.
Name string `validate:"required" example:"aws-lambda-go-func"`
AwsLambdaFunction *devdeploy.AwsLambdaFunction `validate:"required"`
AwsIamRole *devdeploy.AwsIamRole `validate:"required"`
AwsIamPolicy *devdeploy.AwsIamPolicy `validate:"required"`
// Optional flags.
FunctionDir string `validate:"omitempty"`
BuildDir string `validate:"omitempty"`
DockerBuildContext string `validate:"omitempty" example:"."`
Dockerfile string `validate:"required" example:"./cmd/web-api/Dockerfile"`
ReleaseTag string `validate:"required"`
EnableVPC bool `validate:"omitempty" example:"false"`
}
// NewFunctionContext returns the FunctionContext.
func NewFunctionContext(funcName string, cfg *devdeploy.Config) (*FunctionContext, error) {
ctx := &FunctionContext{
Name: funcName,
FunctionDir: filepath.Join(cfg.ProjectRoot, "examples", funcName),
DockerBuildContext: ".",
// Set the release tag for the image to use include env + service name + commit hash/tag.
ReleaseTag: devdeploy.GitLabCiReleaseTag(cfg.Env, funcName),
}
switch funcName {
case Function_YourNewFunction:
// No additional settings for function.
case Function_Ddlogscollector:
// Python Datadog Logs Collector is
ctx.FunctionDir = filepath.Join(cfg.ProjectRoot, "deployments/ddlogscollector")
// Change the build directory to the function directory instead of project root.
ctx.BuildDir = ctx.FunctionDir
// AwsLambdaFunction defines the details needed to create an lambda function.
ctx.AwsLambdaFunction = &devdeploy.AwsLambdaFunction{
FunctionName: ctx.Name,
Description: "Ship logs from cloudwatch to datadog",
Handler: "lambda_function.lambda_handler",
Runtime: "python2.7",
MemorySize: 512,
Timeout: aws.Int64(300),
Environment: map[string]string{
"DD_API_KEY": "",
"LAMBDA_FUNC": ctx.Name,
},
Tags: []devdeploy.Tag{
{Key: devdeploy.AwsTagNameProject, Value: cfg.ProjectName},
{Key: devdeploy.AwsTagNameEnv, Value: cfg.Env},
},
}
ctx.AwsIamRole = &devdeploy.AwsIamRole{
RoleName: "DatadogAWSIntegrationLambdaRole",
Description: "Allows Datadog to run Lambda functions to call AWS services on your behalf.",
AssumeRolePolicyDocument: "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":[\"lambda.amazonaws.com\"]},\"Action\":[\"sts:AssumeRole\"]}]}",
Tags: []devdeploy.Tag{
{Key: devdeploy.AwsTagNameProject, Value: cfg.ProjectName},
{Key: devdeploy.AwsTagNameEnv, Value: cfg.Env},
},
}
ctx.AwsIamPolicy = &devdeploy.AwsIamPolicy{
PolicyName: "DatadogAWSIntegrationPolicy",
Description: "Provides Datadog Lambda function the ability to ship AWS service related logs back to Datadog.",
PolicyDocument: devdeploy.AwsIamPolicyDocument{
Version: "2012-10-17",
Statement: []devdeploy.AwsIamStatementEntry{
{
Action: []string{
"apigateway:GET",
"autoscaling:Describe*",
"budgets:ViewBudget",
"cloudfront:GetDistributionConfig",
"cloudfront:ListDistributions",
"cloudtrail:DescribeTrails",
"cloudtrail:GetTrailStatus",
"cloudwatch:Describe*",
"cloudwatch:Get*",
"cloudwatch:List*",
"codedeploy:List*",
"codedeploy:BatchGet*",
"directconnect:Describe*",
"dynamodb:List*",
"dynamodb:Describe*",
"ec2:Describe*",
"ecs:Describe*",
"ecs:List*",
"elasticache:Describe*",
"elasticache:List*",
"elasticfilesystem:DescribeFileSystems",
"elasticfilesystem:DescribeTags",
"elasticloadbalancing:Describe*",
"elasticmapreduce:List*",
"elasticmapreduce:Describe*",
"es:ListTags",
"es:ListDomainNames",
"es:DescribeElasticsearchDomains",
"health:DescribeEvents",
"health:DescribeEventDetails",
"health:DescribeAffectedEntities",
"kinesis:List*",
"kinesis:Describe*",
"lambda:AddPermission",
"lambda:GetPolicy",
"lambda:List*",
"lambda:RemovePermission",
"logs:Get*",
"logs:Describe*",
"logs:FilterLogEvents",
"logs:TestMetricFilter",
"logs:PutSubscriptionFilter",
"logs:DeleteSubscriptionFilter",
"logs:DescribeSubscriptionFilters",
"rds:Describe*",
"rds:List*",
"redshift:DescribeClusters",
"redshift:DescribeLoggingStatus",
"route53:List*",
"s3:GetBucketLogging",
"s3:GetBucketLocation",
"s3:GetBucketNotification",
"s3:GetBucketTagging",
"s3:ListAllMyBuckets",
"s3:PutBucketNotification",
"ses:Get*",
"sns:List*",
"sns:Publish",
"sqs:ListQueues",
"support:*",
"tag:GetResources",
"tag:GetTagKeys",
"tag:GetTagValues",
"xray:BatchGetTraces",
"xray:GetTraceSummaries",
"lambda:List*",
"logs:DescribeLogGroups",
"logs:DescribeLogStreams",
"logs:FilterLogEvents",
"tag:GetResources",
"cloudfront:GetDistributionConfig",
"cloudfront:ListDistributions",
"elasticloadbalancing:DescribeLoadBalancers",
"elasticloadbalancing:DescribeLoadBalancerAttributes",
"lambda:AddPermission",
"lambda:GetPolicy",
"lambda:RemovePermission",
"redshift:DescribeClusters",
"redshift:DescribeLoggingStatus",
"s3:GetBucketLogging",
"s3:GetBucketLocation",
"s3:GetBucketNotification",
"s3:ListAllMyBuckets",
"s3:PutBucketNotification",
"logs:PutSubscriptionFilter",
"logs:DeleteSubscriptionFilter",
"logs:DescribeSubscriptionFilters",
},
Effect: "Allow",
Resource: "*",
},
},
},
}
default:
return nil, errors.Wrapf(devdeploy.ErrInvalidFunction,
"No function context defined for function '%s'",
funcName)
}
// Append the datadog api key before execution.
ctx.AwsLambdaFunction.UpdateEnvironment = func(vars map[string]string) error {
datadogApiKey, err := getDatadogApiKey(cfg)
if err != nil {
return err
}
vars["DD_API_KEY"] = datadogApiKey
return nil
}
// Set the docker file if no custom one has been defined for the service.
if ctx.Dockerfile == "" {
ctx.Dockerfile = filepath.Join(ctx.BuildDir, "Dockerfile")
}
return ctx, nil
}
// Build handles defining all the information needed to deploy a service to AWS ECS.
func (ctx *FunctionContext) Build(log *log.Logger, noCache, noPush bool) (*devdeploy.BuildLambda, error) {
log.Printf("Define build for function '%s'.", ctx.Name)
log.Printf("\tUsing release tag %s.", ctx.ReleaseTag)
srv := &devdeploy.BuildLambda{
FuncName: ctx.Name,
ReleaseTag: ctx.ReleaseTag,
BuildDir: ctx.BuildDir,
Dockerfile: ctx.Dockerfile,
DockerBuildContext: ctx.DockerBuildContext,
NoCache: noCache,
NoPush: noPush,
}
return srv, nil
}
// Deploy handles defining all the information needed to deploy a service to AWS ECS.
func (ctx *FunctionContext) Deploy(log *log.Logger) (*devdeploy.DeployLambda, error) {
log.Printf("Define build for function '%s'.", ctx.Name)
log.Printf("\tUsing release tag %s.", ctx.ReleaseTag)
srv := &devdeploy.DeployLambda{
FuncName: ctx.Name,
EnableVPC: ctx.EnableVPC,
AwsLambdaFunction: ctx.AwsLambdaFunction,
AwsIamPolicy: ctx.AwsIamPolicy,
AwsIamRole: ctx.AwsIamRole,
}
return srv, nil
}
// S3Location returns the s3 bucket and key used to upload the code to.
func (ctx *FunctionContext) S3Location(cfg *devdeploy.Config) (string, string) {
s3Bucket := cfg.AwsS3BucketPrivate.BucketName
s3Key := filepath.Join("src", "aws", "lambda", cfg.Env, ctx.Name, ctx.ReleaseTag+".zip")
return s3Bucket, s3Key
}
// BuildFunctionForTargetEnv executes the build commands for a target function.
func BuildFunctionForTargetEnv(log *log.Logger, awsCredentials devdeploy.AwsCredentials, targetEnv Env, functionName, releaseTag string, dryRun, noCache, noPush bool) error {
cfgCtx, err := NewConfigContext(targetEnv, awsCredentials)
if err != nil {
return err
}
cfg, err := cfgCtx.Config(log)
if err != nil {
return err
}
funcCtx, err := NewFunctionContext(functionName, cfg)
if err != nil {
return err
}
// Override the release tag if set.
if releaseTag != "" {
funcCtx.ReleaseTag = releaseTag
}
details, err := funcCtx.Build(log, noCache, noPush)
if err != nil {
return err
}
// Set the s3 bucket and s3 for uploading the zip file.
details.CodeS3Bucket, details.CodeS3Key = funcCtx.S3Location(cfg)
// funcPath is used to copy the service specific code in the Dockerfile.
funcPath, err := filepath.Rel(cfg.ProjectRoot, funcCtx.FunctionDir)
if err != nil {
return err
}
// commitRef is used by main.go:build constant.
commitRef := getCommitRef()
if commitRef == "" {
commitRef = funcCtx.ReleaseTag
}
details.BuildArgs = map[string]string{
"func_path": funcPath,
"commit_ref": commitRef,
}
if dryRun {
cfgJSON, err := json.MarshalIndent(cfg, "", " ")
if err != nil {
log.Fatalf("BuildFunctionForTargetEnv : Marshalling config to JSON : %+v", err)
}
log.Printf("BuildFunctionForTargetEnv : config : %v\n", string(cfgJSON))
detailsJSON, err := json.MarshalIndent(details, "", " ")
if err != nil {
log.Fatalf("BuildFunctionForTargetEnv : Marshalling details to JSON : %+v", err)
}
log.Printf("BuildFunctionForTargetEnv : details : %v\n", string(detailsJSON))
return nil
}
return devdeploy.BuildLambdaForTargetEnv(log, cfg, details)
}
// DeployFunctionForTargetEnv executes the deploy commands for a target function.
func DeployFunctionForTargetEnv(log *log.Logger, awsCredentials devdeploy.AwsCredentials, targetEnv Env, functionName, releaseTag string, dryRun bool) error {
cfgCtx, err := NewConfigContext(targetEnv, awsCredentials)
if err != nil {
return err
}
cfg, err := cfgCtx.Config(log)
if err != nil {
return err
}
funcCtx, err := NewFunctionContext(functionName, cfg)
if err != nil {
return err
}
// Override the release tag if set.
if releaseTag != "" {
funcCtx.ReleaseTag = releaseTag
}
details, err := funcCtx.Deploy(log)
if err != nil {
return err
}
// Set the s3 bucket and s3 for uploading the zip file.
details.CodeS3Bucket, details.CodeS3Key = funcCtx.S3Location(cfg)
if dryRun {
cfgJSON, err := json.MarshalIndent(cfg, "", " ")
if err != nil {
log.Fatalf("DeployFunctionForTargetEnv : Marshalling config to JSON : %+v", err)
}
log.Printf("DeployFunctionForTargetEnv : config : %v\n", string(cfgJSON))
detailsJSON, err := json.MarshalIndent(details, "", " ")
if err != nil {
log.Fatalf("DeployFunctionForTargetEnv : Marshalling details to JSON : %+v", err)
}
log.Printf("DeployFunctionForTargetEnv : details : %v\n", string(detailsJSON))
return nil
}
return devdeploy.DeployLambdaToTargetEnv(log, cfg, details)
}

38
build/cicd/internal/config/schema.go Normal file
View File

@ -0,0 +1,38 @@
package config
import (
"context"
"log"
"github.com/jmoiron/sqlx"
"github.com/pkg/errors"
"geeks-accelerator/oss/saas-starter-kit/internal/schema"
"gitlab.com/geeks-accelerator/oss/devops/pkg/devdeploy"
)
// RunSchemaMigrationsForTargetEnv executes the build commands for a target service.
func RunSchemaMigrationsForTargetEnv(log *log.Logger, awsCredentials devdeploy.AwsCredentials, targetEnv Env, isUnittest bool) error {
cfgCtx, err := NewConfigContext(targetEnv, awsCredentials)
if err != nil {
return err
}
cfg, err := cfgCtx.Config(log)
if err != nil {
return err
}
err = devdeploy.SetupDeploymentEnv(log, cfg)
if err != nil {
return err
}
masterDb, err := sqlx.Open(cfg.DBConnInfo.Driver, cfg.DBConnInfo.URL())
if err != nil {
return errors.WithMessage(err, "Failed to connect to db for schema migration.")
}
defer masterDb.Close()
return schema.Migrate(context.Background(), masterDb, log, false)
}

756
build/cicd/internal/config/service.go Normal file
View File

@ -0,0 +1,756 @@
package config
import (
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"
"strings"
"github.com/pkg/errors"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ecs"
"github.com/iancoleman/strcase"
"gitlab.com/geeks-accelerator/oss/devops/pkg/devdeploy"
"gopkg.in/go-playground/validator.v9"
)
// Service define the name of a service.
type Service = string
var (
ServiceWebApi = "web-api"
ServiceWebApp = "web-app"
)
// List of service names used by main.go for help.
var ServiceNames = []Service{
ServiceWebApi,
ServiceWebApp,
}
// ServiceConfig defines the settings for a service.
type ServiceConfig struct {
// Required flags.
Name string `validate:"required" example:"web-api"`
ServiceHostPrimary string `validate:"required" example:"example-project.com"`
DesiredCount int `validate:"required" example:"2"`
ServiceDir string `validate:"required"`
Dockerfile string `validate:"required" example:"./cmd/web-api/Dockerfile"`
ReleaseTag string `validate:"required"`
// Optional flags.
ServiceHostNames []string `validate:"omitempty" example:"subdomain.example-project.com"`
EnableHTTPS bool `validate:"omitempty" example:"false"`
EnableElb bool `validate:"omitempty" example:"false"`
StaticFilesS3Enable bool `validate:"omitempty" example:"false"`
BuildDir string `validate:"omitempty"`
DockerBuildContext string `validate:"omitempty" example:"."`
}
// ServiceContext includes the config and task definition for building and deploying a service.
type ServiceContext struct {
ServiceConfig
// AwsEcsTaskDefinition defines the ECS task definition based on the service configs.
AwsEcsTaskDefinition func(cfg *devdeploy.Config, srv *devdeploy.DeployService) (*ecs.RegisterTaskDefinitionInput, error)
}
// NewServiceConfig returns the Service for a service that is configured for the target deployment env.
func NewServiceConfig(serviceName string, cfg *devdeploy.Config) (ServiceConfig, error) {
// =========================================================================
// New service context.
srv := ServiceConfig{
Name: serviceName,
DesiredCount: 1,
DockerBuildContext: ".",
ServiceDir: filepath.Join(cfg.ProjectRoot, "examples", serviceName),
// Set the release tag for the image to use include env + service name + commit hash/tag.
ReleaseTag: devdeploy.GitLabCiReleaseTag(cfg.Env, serviceName),
}
// =========================================================================
// Context settings based on target env.
if cfg.Env == EnvStage || cfg.Env == EnvProd {
srv.EnableHTTPS = true
srv.StaticFilesS3Enable = true
} else {
srv.EnableHTTPS = false
srv.StaticFilesS3Enable = false
}
// =========================================================================
// Service dependant settings.
switch serviceName {
case ServiceWebApp:
// Set the hostnames for the service.
if cfg.Env == EnvProd {
srv.ServiceHostPrimary = "example.saasstartupkit.com"
// Any hostname listed here that doesn't match the primary hostname will be updated in Route 53 but the
// service itself will redirect any requests back to the primary hostname.
srv.ServiceHostNames = []string{
fmt.Sprintf("%s.example.saasstartupkit.com", cfg.Env),
}
} else {
srv.ServiceHostPrimary = fmt.Sprintf("%s.example.saasstartupkit.com", cfg.Env)
}
case ServiceWebApi:
// Set the hostnames for the service.
if cfg.Env == EnvProd {
srv.ServiceHostPrimary = "api.example.saasstartupkit.com"
} else {
srv.ServiceHostPrimary = fmt.Sprintf("api.%s.example.saasstartupkit.com", cfg.Env)
}
default:
return ServiceConfig{}, errors.Wrapf(devdeploy.ErrInvalidService,
"No service config defined for service '%s'",
serviceName)
}
// Set the docker file if no custom one has been defined for the service.
if srv.Dockerfile == "" {
srv.Dockerfile = filepath.Join(srv.ServiceDir, "Dockerfile")
}
// Ensure the config is valid.
errs := validator.New().Struct(cfg)
if errs != nil {
return srv, errs
}
return srv, nil
}
// BaseUrl returns the base url for a specific service.
func (c ServiceConfig) BaseUrl() string {
var schema string
if c.EnableHTTPS {
schema = "https"
} else {
schema = "http"
}
return fmt.Sprintf("%s://%s/", schema, c.ServiceHostPrimary)
}
// NewServiceContext returns the ServiceContext for a service that is configured for the target deployment env.
func NewServiceContext(serviceName Service, cfg *devdeploy.Config) (*ServiceContext, error) {
// =========================================================================
// Shared details that could be applied to all task definitions.
// Load the web-app config for the web-api can reference it's hostname.
webAppCfg, err := NewServiceConfig(ServiceWebApp, cfg)
if err != nil {
return nil, err
}
// Load the web-api config for the web-app can reference it's hostname.
webApiCfg, err := NewServiceConfig(ServiceWebApi, cfg)
if err != nil {
return nil, err
}
// Define a base set of environment variables that can be assigned to individual container definitions.
baseEnvVals := func(cfg *devdeploy.Config, srv *devdeploy.DeployService) []*ecs.KeyValuePair {
var ciJobURL string
if id := os.Getenv("CI_JOB_ID"); id != "" {
ciJobURL = strings.TrimRight(GitLabProjectBaseUrl, "/") + "/-/jobs/" + os.Getenv("CI_JOB_ID")
}
var ciPipelineURL string
if id := os.Getenv("CI_PIPELINE_ID"); id != "" {
ciPipelineURL = strings.TrimRight(GitLabProjectBaseUrl, "/") + "/pipelines/" + os.Getenv("CI_PIPELINE_ID")
}
return []*ecs.KeyValuePair{
ecsKeyValuePair(devdeploy.ENV_KEY_ECS_CLUSTER, srv.AwsEcsCluster.ClusterName),
ecsKeyValuePair(devdeploy.ENV_KEY_ECS_SERVICE, srv.AwsEcsService.ServiceName),
ecsKeyValuePair("AWS_REGION", cfg.AwsCredentials.Region),
ecsKeyValuePair("AWS_USE_ROLE", "true"),
ecsKeyValuePair("AWSLOGS_GROUP", srv.AwsCloudWatchLogGroup.LogGroupName),
ecsKeyValuePair("ECS_ENABLE_CONTAINER_METADATA", "true"),
ecsKeyValuePair("CI_COMMIT_REF_NAME", os.Getenv("CI_COMMIT_REF_NAME")),
ecsKeyValuePair("CI_COMMIT_SHORT_SHA", os.Getenv("CI_COMMIT_SHORT_SHA")),
ecsKeyValuePair("CI_COMMIT_SHA", os.Getenv("CI_COMMIT_SHA")),
ecsKeyValuePair("CI_COMMIT_TAG", os.Getenv("CI_COMMIT_TAG")),
ecsKeyValuePair("CI_JOB_ID", os.Getenv("CI_JOB_ID")),
ecsKeyValuePair("CI_PIPELINE_ID", os.Getenv("CI_PIPELINE_ID")),
ecsKeyValuePair("CI_JOB_URL", ciJobURL),
ecsKeyValuePair("CI_PIPELINE_URL", ciPipelineURL),
ecsKeyValuePair("WEB_APP_BASE_URL", webAppCfg.BaseUrl()),
ecsKeyValuePair("WEB_API_BASE_URL", webApiCfg.BaseUrl()),
}
}
// =========================================================================
// Service dependant settings.
var ctx *ServiceContext
switch serviceName {
// Define the ServiceContext for the web-app that will be used for build and deploy.
case ServiceWebApp:
ctx := &ServiceContext{
ServiceConfig: webAppCfg,
}
// Define the service task definition with a function to enable use of config and deploy details.
ctx.AwsEcsTaskDefinition = func(cfg *devdeploy.Config, srv *devdeploy.DeployService) (*ecs.RegisterTaskDefinitionInput, error) {
// Defined a container definition for the specific service.
container1 := &ecs.ContainerDefinition{
Name: aws.String(ctx.Name),
Image: aws.String(srv.ReleaseImage),
Essential: aws.Bool(true),
LogConfiguration: &ecs.LogConfiguration{
LogDriver: aws.String("awslogs"),
Options: map[string]*string{
"awslogs-group": aws.String(srv.AwsCloudWatchLogGroup.LogGroupName),
"awslogs-region": aws.String(cfg.AwsCredentials.Region),
"awslogs-stream-prefix": aws.String("ecs"),
},
},
PortMappings: []*ecs.PortMapping{
&ecs.PortMapping{
HostPort: aws.Int64(80),
Protocol: aws.String("tcp"),
ContainerPort: aws.Int64(80),
},
},
Cpu: aws.Int64(128),
MemoryReservation: aws.Int64(128),
Environment: baseEnvVals(cfg, srv),
HealthCheck: &ecs.HealthCheck{
Retries: aws.Int64(3),
Command: aws.StringSlice([]string{
"CMD-SHELL",
"curl -f http://localhost/ping || exit 1",
}),
Timeout: aws.Int64(5),
Interval: aws.Int64(60),
StartPeriod: aws.Int64(60),
},
Ulimits: []*ecs.Ulimit{
&ecs.Ulimit{
Name: aws.String("nofile"),
SoftLimit: aws.Int64(987654),
HardLimit: aws.Int64(999999),
},
},
}
// If the service has HTTPS enabled with the use of an AWS Elastic Load Balancer, then need to enable
// traffic for port 443 for SSL traffic to get terminated on the deployed tasks.
if ctx.EnableHTTPS && !ctx.EnableElb {
container1.PortMappings = append(container1.PortMappings, &ecs.PortMapping{
HostPort: aws.Int64(443),
Protocol: aws.String("tcp"),
ContainerPort: aws.Int64(443),
})
}
// Append env vars for the service task.
container1.Environment = append(container1.Environment,
ecsKeyValuePair("SERVICE_NAME", srv.ServiceName),
ecsKeyValuePair("PROJECT_NAME", cfg.ProjectName),
// Use placeholders for these environment variables that will be replaced with devdeploy.DeployServiceToTargetEnv
ecsKeyValuePair("WEB_API_SERVICE_HOST", "{HTTP_HOST}"),
ecsKeyValuePair("WEB_API_SERVICE_HTTPS_HOST", "{HTTPS_HOST}"),
ecsKeyValuePair("WEB_API_SERVICE_ENABLE_HTTPS", "{HTTPS_ENABLED}"),
ecsKeyValuePair("WEB_API_SERVICE_BASE_URL", "{APP_BASE_URL}"),
ecsKeyValuePair("WEB_API_SERVICE_HOST_NAMES", "{HOST_NAMES}"),
ecsKeyValuePair("WEB_API_SERVICE_STATICFILES_S3_ENABLED", "{STATIC_FILES_S3_ENABLED}"),
ecsKeyValuePair("WEB_API_SERVICE_STATICFILES_S3_PREFIX", "{STATIC_FILES_S3_PREFIX}"),
ecsKeyValuePair("WEB_API_SERVICE_STATICFILES_CLOUDFRONT_ENABLED", "{STATIC_FILES_CLOUDFRONT_ENABLED}"),
ecsKeyValuePair("WEB_API_REDIS_HOST", "{CACHE_HOST}"),
ecsKeyValuePair("WEB_API_DB_HOST", "{DB_HOST}"),
ecsKeyValuePair("WEB_API_DB_USERNAME", "{DB_USER}"),
ecsKeyValuePair("WEB_API_DB_PASSWORD", "{DB_PASS}"),
ecsKeyValuePair("WEB_API_DB_DATABASE", "{DB_DATABASE}"),
ecsKeyValuePair("WEB_API_DB_DRIVER", "{DB_DRIVER}"),
ecsKeyValuePair("WEB_API_DB_DISABLE_TLS", "{DB_DISABLE_TLS}"),
ecsKeyValuePair("WEB_API_AWS_S3_BUCKET_PRIVATE", "{AWS_S3_BUCKET_PRIVATE}"),
ecsKeyValuePair("WEB_API_AWS_S3_BUCKET_PUBLIC", "{AWS_S3_BUCKET_PUBLIC}"),
ecsKeyValuePair(devdeploy.ENV_KEY_ROUTE53_UPDATE_TASK_IPS, "{ROUTE53_UPDATE_TASK_IPS}"),
ecsKeyValuePair(devdeploy.ENV_KEY_ROUTE53_ZONES, "{ROUTE53_ZONES}"),
)
// Define the full task definition for the service.
def := &ecs.RegisterTaskDefinitionInput{
Family: aws.String(srv.ServiceName),
ExecutionRoleArn: aws.String(srv.AwsEcsExecutionRole.Arn()),
TaskRoleArn: aws.String(srv.AwsEcsTaskRole.Arn()),
NetworkMode: aws.String("awsvpc"),
ContainerDefinitions: []*ecs.ContainerDefinition{
// Include the single container definition for the service. Additional definitions could be added
// here like one for datadog.
container1,
},
RequiresCompatibilities: aws.StringSlice([]string{"FARGATE"}),
}
return def, nil
}
// Define the ServiceContext for the web-api that will be used for build and deploy.
case ServiceWebApi:
ctx := &ServiceContext{
ServiceConfig: webApiCfg,
}
// Define the service task definition with a function to enable use of config and deploy details.
ctx.AwsEcsTaskDefinition = func(cfg *devdeploy.Config, srv *devdeploy.DeployService) (*ecs.RegisterTaskDefinitionInput, error) {
// Defined a container definition for the specific service.
container1 := &ecs.ContainerDefinition{
Name: aws.String(ctx.Name),
Image: aws.String(srv.ReleaseImage),
Essential: aws.Bool(true),
LogConfiguration: &ecs.LogConfiguration{
LogDriver: aws.String("awslogs"),
Options: map[string]*string{
"awslogs-group": aws.String(srv.AwsCloudWatchLogGroup.LogGroupName),
"awslogs-region": aws.String(cfg.AwsCredentials.Region),
"awslogs-stream-prefix": aws.String("ecs"),
},
},
PortMappings: []*ecs.PortMapping{
&ecs.PortMapping{
HostPort: aws.Int64(80),
Protocol: aws.String("tcp"),
ContainerPort: aws.Int64(80),
},
},
Cpu: aws.Int64(128),
MemoryReservation: aws.Int64(128),
Environment: baseEnvVals(cfg, srv),
HealthCheck: &ecs.HealthCheck{
Retries: aws.Int64(3),
Command: aws.StringSlice([]string{
"CMD-SHELL",
"curl -f http://localhost/ping || exit 1",
}),
Timeout: aws.Int64(5),
Interval: aws.Int64(60),
StartPeriod: aws.Int64(60),
},
Ulimits: []*ecs.Ulimit{
&ecs.Ulimit{
Name: aws.String("nofile"),
SoftLimit: aws.Int64(987654),
HardLimit: aws.Int64(999999),
},
},
}
// If the service has HTTPS enabled with the use of an AWS Elastic Load Balancer, then need to enable
// traffic for port 443 for SSL traffic to get terminated on the deployed tasks.
if ctx.EnableHTTPS && !ctx.EnableElb {
container1.PortMappings = append(container1.PortMappings, &ecs.PortMapping{
HostPort: aws.Int64(443),
Protocol: aws.String("tcp"),
ContainerPort: aws.Int64(443),
})
}
// Append env vars for the service task.
container1.Environment = append(container1.Environment,
ecsKeyValuePair("SERVICE_NAME", srv.ServiceName),
ecsKeyValuePair("PROJECT_NAME", cfg.ProjectName),
// Use placeholders for these environment variables that will be replaced with devdeploy.DeployServiceToTargetEnv
ecsKeyValuePair("WEB_API_SERVICE_HOST", "{HTTP_HOST}"),
ecsKeyValuePair("WEB_API_SERVICE_HTTPS_HOST", "{HTTPS_HOST}"),
ecsKeyValuePair("WEB_API_SERVICE_ENABLE_HTTPS", "{HTTPS_ENABLED}"),
ecsKeyValuePair("WEB_API_SERVICE_BASE_URL", "{APP_BASE_URL}"),
ecsKeyValuePair("WEB_API_SERVICE_HOST_NAMES", "{HOST_NAMES}"),
ecsKeyValuePair("WEB_API_SERVICE_STATICFILES_S3_ENABLED", "{STATIC_FILES_S3_ENABLED}"),
ecsKeyValuePair("WEB_API_SERVICE_STATICFILES_S3_PREFIX", "{STATIC_FILES_S3_PREFIX}"),
ecsKeyValuePair("WEB_API_SERVICE_STATICFILES_CLOUDFRONT_ENABLED", "{STATIC_FILES_CLOUDFRONT_ENABLED}"),
ecsKeyValuePair("WEB_API_REDIS_HOST", "{CACHE_HOST}"),
ecsKeyValuePair("WEB_API_DB_HOST", "{DB_HOST}"),
ecsKeyValuePair("WEB_API_DB_USERNAME", "{DB_USER}"),
ecsKeyValuePair("WEB_API_DB_PASSWORD", "{DB_PASS}"),
ecsKeyValuePair("WEB_API_DB_DATABASE", "{DB_DATABASE}"),
ecsKeyValuePair("WEB_API_DB_DRIVER", "{DB_DRIVER}"),
ecsKeyValuePair("WEB_API_DB_DISABLE_TLS", "{DB_DISABLE_TLS}"),
ecsKeyValuePair("WEB_API_AWS_S3_BUCKET_PRIVATE", "{AWS_S3_BUCKET_PRIVATE}"),
ecsKeyValuePair("WEB_API_AWS_S3_BUCKET_PUBLIC", "{AWS_S3_BUCKET_PUBLIC}"),
ecsKeyValuePair(devdeploy.ENV_KEY_ROUTE53_UPDATE_TASK_IPS, "{ROUTE53_UPDATE_TASK_IPS}"),
ecsKeyValuePair(devdeploy.ENV_KEY_ROUTE53_ZONES, "{ROUTE53_ZONES}"),
)
// Define the full task definition for the service.
def := &ecs.RegisterTaskDefinitionInput{
Family: aws.String(srv.ServiceName),
ExecutionRoleArn: aws.String(srv.AwsEcsExecutionRole.Arn()),
TaskRoleArn: aws.String(srv.AwsEcsTaskRole.Arn()),
NetworkMode: aws.String("awsvpc"),
ContainerDefinitions: []*ecs.ContainerDefinition{
// Include the single container definition for the service. Additional definitions could be added
// here like one for datadog.
container1,
},
RequiresCompatibilities: aws.StringSlice([]string{"FARGATE"}),
}
return def, nil
}
default:
return nil, errors.Wrapf(devdeploy.ErrInvalidService,
"No service context defined for service '%s'",
serviceName)
}
return ctx, nil
}
// BuildService handles defining all the information needed to a service with docker and push to AWS ECR.
func (ctx *ServiceContext) Build(log *log.Logger, noCache, noPush bool) (*devdeploy.BuildService, error) {
log.Printf("Define build for service '%s'.", ctx.Name)
log.Printf("\tUsing release tag %s.", ctx.ReleaseTag)
srv := &devdeploy.BuildService{
ServiceName: ctx.Name,
ReleaseTag: ctx.ReleaseTag,
BuildDir: ctx.BuildDir,
Dockerfile: ctx.Dockerfile,
DockerBuildContext: ctx.DockerBuildContext,
NoCache: noCache,
NoPush: noPush,
}
return srv, nil
}
// DeployService handles defining all the information needed to deploy a service to AWS ECS.
func (ctx *ServiceContext) Deploy(log *log.Logger, cfg *devdeploy.Config) (*devdeploy.DeployService, error) {
log.Printf("Define deploy for service '%s'.", ctx.Name)
log.Printf("\tUsing release tag %s.", ctx.ReleaseTag)
// Start to define all the information for the service from the service context.
srv := &devdeploy.DeployService{
ServiceName: ctx.Name,
ReleaseTag: ctx.ReleaseTag,
EnableHTTPS: ctx.EnableHTTPS,
ServiceHostPrimary: ctx.ServiceHostPrimary,
ServiceHostNames: ctx.ServiceHostNames,
}
// When only service host names are set, choose the first item as the primary host.
if srv.ServiceHostPrimary == "" && len(srv.ServiceHostNames) > 0 {
srv.ServiceHostPrimary = srv.ServiceHostNames[0]
log.Printf("\t\tSet Service Primary Host to '%s'.", srv.ServiceHostPrimary)
}
// The S3 prefix used to upload static files served to public.
if ctx.StaticFilesS3Enable {
srv.StaticFilesS3Prefix = filepath.Join(cfg.AwsS3BucketPublicKeyPrefix, srv.ReleaseTag, "static")
}
// Determine the Dockerfile for the service.
if ctx.Dockerfile != "" {
srv.Dockerfile = ctx.Dockerfile
log.Printf("\t\tUsing docker file '%s'.", srv.Dockerfile)
} else {
var err error
srv.Dockerfile, err = devdeploy.FindServiceDockerFile(cfg.ProjectRoot, srv.ServiceName)
if err != nil {
return nil, err
}
log.Printf("\t\tFound service docker file '%s'.", srv.Dockerfile)
}
// Set the service directory.
if ctx.ServiceDir == "" {
ctx.ServiceDir = filepath.Dir(srv.Dockerfile)
}
srv.StaticFilesDir = filepath.Join(ctx.ServiceDir, "static")
// Define the ECS Cluster used to host the serverless fargate tasks.
srv.AwsEcsCluster = &devdeploy.AwsEcsCluster{
ClusterName: cfg.ProjectName + "-" + cfg.Env,
Tags: []devdeploy.Tag{
{Key: devdeploy.AwsTagNameProject, Value: cfg.ProjectName},
{Key: devdeploy.AwsTagNameEnv, Value: cfg.Env},
},
}
// Define the ECS task execution role. This role executes ECS actions such as pulling the image and storing the
// application logs in cloudwatch.
srv.AwsEcsExecutionRole = &devdeploy.AwsIamRole{
RoleName: fmt.Sprintf("ecsExecutionRole%s%s", cfg.ProjectNameCamel(), strcase.ToCamel(cfg.Env)),
Description: fmt.Sprintf("Provides access to other AWS service resources that are required to run Amazon ECS tasks for %s. ", cfg.ProjectName),
AssumeRolePolicyDocument: "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":[\"ecs-tasks.amazonaws.com\"]},\"Action\":[\"sts:AssumeRole\"]}]}",
Tags: []devdeploy.Tag{
{Key: devdeploy.AwsTagNameProject, Value: cfg.ProjectName},
{Key: devdeploy.AwsTagNameEnv, Value: cfg.Env},
},
AttachRolePolicyArns: []string{"arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"},
}
log.Printf("\t\tSet ECS Execution Role Name to '%s'.", srv.AwsEcsExecutionRole.RoleName)
// Define the ECS task role. This role is used by the task itself for calling other AWS services.
srv.AwsEcsTaskRole = &devdeploy.AwsIamRole{
RoleName: fmt.Sprintf("ecsTaskRole%s%s", cfg.ProjectNameCamel(), strcase.ToCamel(cfg.Env)),
Description: fmt.Sprintf("Allows ECS tasks for %s to call AWS services on your behalf.", cfg.ProjectName),
AssumeRolePolicyDocument: "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":[\"ecs-tasks.amazonaws.com\"]},\"Action\":[\"sts:AssumeRole\"]}]}",
Tags: []devdeploy.Tag{
{Key: devdeploy.AwsTagNameProject, Value: cfg.ProjectName},
{Key: devdeploy.AwsTagNameEnv, Value: cfg.Env},
},
}
log.Printf("\t\tSet ECS Task Role Name to '%s'.", srv.AwsEcsTaskRole.RoleName)
// AwsCloudWatchLogGroup defines the name of the cloudwatch log group that will be used to store logs for the ECS tasks.
srv.AwsCloudWatchLogGroup = &devdeploy.AwsCloudWatchLogGroup{
LogGroupName: fmt.Sprintf("logs/env_%s/aws/ecs/cluster_%s/service_%s", cfg.Env, srv.AwsEcsCluster.ClusterName, srv.ServiceName),
Tags: []devdeploy.Tag{
{Key: devdeploy.AwsTagNameProject, Value: cfg.ProjectName},
{Key: devdeploy.AwsTagNameEnv, Value: cfg.Env},
},
}
log.Printf("\t\tSet AWS Log Group Name to '%s'.", srv.AwsCloudWatchLogGroup.LogGroupName)
// AwsSdPrivateDnsNamespace defines the service discovery group.
srv.AwsSdPrivateDnsNamespace = &devdeploy.AwsSdPrivateDnsNamespace{
Name: srv.AwsEcsCluster.ClusterName,
Description: fmt.Sprintf("Private DNS namespace used for services running on the ECS Cluster %s", srv.AwsEcsCluster.ClusterName),
Service: &devdeploy.AwsSdService{
Name: ctx.Name,
Description: fmt.Sprintf("Service %s running on the ECS Cluster %s", ctx.Name, srv.AwsEcsCluster.ClusterName),
DnsRecordTTL: 300,
HealthCheckFailureThreshold: 3,
},
}
log.Printf("\t\tSet AWS Service Discovery Namespace to '%s'.", srv.AwsSdPrivateDnsNamespace.Name)
// If the service is requested to use an elastic load balancer then define.
if ctx.EnableElb {
// AwsElbLoadBalancer defines if the service should use an elastic load balancer.
srv.AwsElbLoadBalancer = &devdeploy.AwsElbLoadBalancer{
Name: fmt.Sprintf("%s-%s-%s", cfg.Env, srv.AwsEcsCluster.ClusterName, srv.ServiceName),
IpAddressType: "ipv4",
Scheme: "internet-facing",
Type: "application",
Tags: []devdeploy.Tag{
{Key: devdeploy.AwsTagNameProject, Value: cfg.ProjectName},
{Key: devdeploy.AwsTagNameEnv, Value: cfg.Env},
},
}
log.Printf("\t\tSet ELB Name to '%s'.", srv.AwsElbLoadBalancer.Name)
// Define the target group for service to receive HTTP traffic from the load balancer.
srv.AwsElbLoadBalancer.TargetGroup = &devdeploy.AwsElbTargetGroup{
Name: fmt.Sprintf("%s-http", srv.ServiceName),
Port: 80,
Protocol: "HTTP",
TargetType: "ip",
HealthCheckEnabled: true,
HealthCheckIntervalSeconds: 30,
HealthCheckPath: "/ping",
HealthCheckProtocol: "HTTP",
HealthCheckTimeoutSeconds: 5,
HealthyThresholdCount: 3,
UnhealthyThresholdCount: 3,
Matcher: "200",
}
log.Printf("\t\t\tSet ELB Target Group Name for %s to '%s'.",
srv.AwsElbLoadBalancer.TargetGroup.Protocol,
srv.AwsElbLoadBalancer.TargetGroup.Name)
// Set ECS configs based on specified env.
if cfg.Env == "prod" {
srv.AwsElbLoadBalancer.EcsTaskDeregistrationDelay = 300
} else {
// Force staging to deploy immediately without waiting for connections to drain
srv.AwsElbLoadBalancer.EcsTaskDeregistrationDelay = 0
}
}
// AwsEcsService defines the details for the ecs service.
srv.AwsEcsService = &devdeploy.AwsEcsService{
ServiceName: ctx.Name,
DesiredCount: int64(ctx.DesiredCount),
EnableECSManagedTags: false,
HealthCheckGracePeriodSeconds: 60,
LaunchType: "FARGATE",
}
// Ensure when deploying a new service there is always at-least one running.
if srv.AwsEcsService.DesiredCount == 0 {
srv.AwsEcsService.DesiredCount = 1
}
// Set ECS configs based on specified env.
if cfg.Env == "prod" {
srv.AwsEcsService.DeploymentMinimumHealthyPercent = 100
srv.AwsEcsService.DeploymentMaximumPercent = 200
} else {
srv.AwsEcsService.DeploymentMinimumHealthyPercent = 100
srv.AwsEcsService.DeploymentMaximumPercent = 200
}
// AwsEcsTaskDefinition defines the details for registering a new ECS task definition.
taskDef, err := ctx.AwsEcsTaskDefinition(cfg, srv)
if err != nil {
return nil, err
}
srv.AwsEcsTaskDefinition = &devdeploy.AwsEcsTaskDefinition{
RegisterInput: taskDef,
UpdatePlaceholders: func(placeholders map[string]string) error {
// Try to find the Datadog API key, this value is optional.
// If Datadog API key is not specified, then integration with Datadog for observability will not be active.
{
datadogApiKey, err := getDatadogApiKey(cfg)
if err != nil {
return err
}
if datadogApiKey != "" {
log.Println("DATADOG API Key set.")
} else {
log.Printf("DATADOG API Key NOT set.")
}
placeholders["{DATADOG_APIKEY}"] = datadogApiKey
// When the datadog API key is empty, don't force the container to be essential have have the whole task fail.
if datadogApiKey != "" {
placeholders["{DATADOG_ESSENTIAL}"] = "true"
} else {
placeholders["{DATADOG_ESSENTIAL}"] = "false"
}
}
return nil
},
}
log.Printf("\t\tDeploying task to '%s'.", ctx.ServiceHostPrimary)
return srv, nil
}
// BuildServiceForTargetEnv executes the build commands for a target service.
func BuildServiceForTargetEnv(log *log.Logger, awsCredentials devdeploy.AwsCredentials, targetEnv Env, serviceName, releaseTag string, dryRun, noCache, noPush bool) error {
cfgCtx, err := NewConfigContext(targetEnv, awsCredentials)
if err != nil {
return err
}
cfg, err := cfgCtx.Config(log)
if err != nil {
return err
}
srvCtx, err := NewServiceContext(serviceName, cfg)
if err != nil {
return err
}
// Override the release tag if set.
if releaseTag != "" {
srvCtx.ReleaseTag = releaseTag
}
details, err := srvCtx.Build(log, noCache, noPush)
if err != nil {
return err
}
// servicePath is used to copy the service specific code in the Dockerfile.
servicePath, err := filepath.Rel(cfg.ProjectRoot, srvCtx.ServiceDir)
if err != nil {
return err
}
// commitRef is used by main.go:build constant.
commitRef := getCommitRef()
if commitRef == "" {
commitRef = srvCtx.ReleaseTag
}
details.BuildArgs = map[string]string{
"service_path": servicePath,
"commit_ref": commitRef,
}
if dryRun {
cfgJSON, err := json.MarshalIndent(cfg, "", " ")
if err != nil {
log.Fatalf("BuildServiceForTargetEnv : Marshalling config to JSON : %+v", err)
}
log.Printf("BuildServiceForTargetEnv : config : %v\n", string(cfgJSON))
detailsJSON, err := json.MarshalIndent(details, "", " ")
if err != nil {
log.Fatalf("BuildServiceForTargetEnv : Marshalling details to JSON : %+v", err)
}
log.Printf("BuildServiceForTargetEnv : details : %v\n", string(detailsJSON))
return nil
}
return devdeploy.BuildServiceForTargetEnv(log, cfg, details)
}
// DeployServiceForTargetEnv executes the build commands for a target service.
func DeployServiceForTargetEnv(log *log.Logger, awsCredentials devdeploy.AwsCredentials, targetEnv Env, serviceName, releaseTag string, dryRun bool) error {
cfgCtx, err := NewConfigContext(targetEnv, awsCredentials)
if err != nil {
return err
}
cfg, err := cfgCtx.Config(log)
if err != nil {
return err
}
srvCtx, err := NewServiceContext(serviceName, cfg)
if err != nil {
return err
}
// Override the release tag if set.
if releaseTag != "" {
srvCtx.ReleaseTag = releaseTag
}
details, err := srvCtx.Deploy(log, cfg)
if err != nil {
return err
}
return devdeploy.DeployServiceToTargetEnv(log, cfg, details)
}
// ecsKeyValuePair returns an *ecs.KeyValuePair
func ecsKeyValuePair(name, value string) *ecs.KeyValuePair {
return &ecs.KeyValuePair{
Name: aws.String(name),
Value: aws.String(value),
}
}