golang-saas-starter-kit/build/cicd/internal/config/config.go

package config

import (
	"context"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/cloudfront"
	"github.com/aws/aws-sdk-go/service/rds"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/secretsmanager"
	"github.com/iancoleman/strcase"
	"github.com/jmoiron/sqlx"
	"github.com/pkg/errors"
	"geeks-accelerator/oss/saas-starter-kit/internal/schema"
	"gitlab.com/geeks-accelerator/oss/devops/pkg/devdeploy"
)

const (
	// ProjectNamePrefix will be appending to the name of the project.
	ProjectNamePrefix = ""

	// GitLabProjectBaseUrl is the base url used to create links to a specific CI/CD job or pipeline by ID.
	GitLabProjectBaseUrl = "https://gitlab.com/geeks-accelerator/oss/saas-starter-kit"
)

// Env defines the target deployment environment.
type Env = string

var (
	EnvDev   Env = "dev"
	EnvStage Env = "stage"
	EnvProd  Env = "prod"
)

// List of env names used by main.go for help.
var EnvNames = []Function{
	EnvDev,
	EnvStage,
	EnvProd,
}

// ConfigContext defines the flags for build env.
type ConfigContext struct {
	// Env is the target environment used for the deployment.
	Env string `validate:"oneof=dev stage prod"`

	// AwsCredentials defines the credentials used for deployment.
	AwsCredentials devdeploy.AwsCredentials `validate:"required,dive,required"`
}

// NewConfigContext returns the ConfigContext.
func NewConfigContext(targetEnv Env, awsCredentials devdeploy.AwsCredentials) (*ConfigContext, error) {
	ctx := &ConfigContext{
		Env:            targetEnv,
		AwsCredentials: awsCredentials,
	}

	// If AWS Credentials are not set and use role is not enabled, try to load the credentials from env vars.
	if ctx.AwsCredentials.UseRole == false && ctx.AwsCredentials.AccessKeyID == "" {
		var err error
		ctx.AwsCredentials, err = devdeploy.GetAwsCredentialsFromEnv(ctx.Env)
		if err != nil {
			return nil, err
		}
	} else if ctx.AwsCredentials.Region == "" {
		awsCreds, err := devdeploy.GetAwsCredentialsFromEnv(ctx.Env)
		if err != nil {
			return nil, err
		}
		ctx.AwsCredentials.Region = awsCreds.Region
	}

	return ctx, nil
}

// Config defines the details to setup the target environment for the project to build services and functions.
func (cfgCtx *ConfigContext) Config(log *log.Logger) (*devdeploy.Config, error) {

	// Init a new build target environment for the project.
	cfg := &devdeploy.Config{
		Env:            cfgCtx.Env,
		AwsCredentials: cfgCtx.AwsCredentials,
	}

	// Get the current working directory. This should be somewhere contained within the project.
	workDir, err := os.Getwd()
	if err != nil {
		return cfg, errors.WithMessage(err, "Failed to get current working directory.")
	}

	// Set the project root directory and project name. This is current set by finding the go.mod file for the project
	// repo. Project name is the directory name.
	modDetails, err := devdeploy.LoadModuleDetails(workDir)
	if err != nil {
		return cfg, err
	}

	// ProjectRoot should be the root directory for the project.
	cfg.ProjectRoot = modDetails.ProjectRoot

	// ProjectName will be used for prefixing AWS resources. This could be changed as needed or manually defined.
	cfg.ProjectName = ProjectNamePrefix + modDetails.ProjectName

	// Set default AWS ECR Repository Name.
	cfg.AwsEcrRepository = &devdeploy.AwsEcrRepository{
		RepositoryName: cfg.ProjectName,
		Tags: []devdeploy.Tag{
			{Key: devdeploy.AwsTagNameProject, Value: cfg.ProjectName},
			{Key: devdeploy.AwsTagNameEnv, Value: cfg.Env},
		},
	}

	// Set the deployment to use the default VPC for the region.
	cfg.AwsEc2Vpc = &devdeploy.AwsEc2Vpc{
		IsDefault: true,
	}

	// Set the security group to use for the deployed services, database and cluster. This will used the VPC ID defined
	// for the deployment.
	cfg.AwsEc2SecurityGroup = &devdeploy.AwsEc2SecurityGroup{
		GroupName:   cfg.ProjectName + "-" + cfg.Env,
		Description: fmt.Sprintf("Security group for %s services running on ECS", cfg.ProjectName),
		Tags: []devdeploy.Tag{
			{Key: devdeploy.AwsTagNameProject, Value: cfg.ProjectName},
			{Key: devdeploy.AwsTagNameEnv, Value: cfg.Env},
		},
	}

	// Set the name of the EC2 Security Group used by the gitlab runner. This is used to ensure the security
	// group defined above has access to the RDS cluster/instance and can thus handle schema migrations.
	cfg.GitlabRunnerEc2SecurityGroupName = "gitlab-runner"

	// Set the s3 buckets used by the deployed services.
	// S3 temp prefix used by services for short term storage. A lifecycle policy will be used for expiration.
	s3BucketTempPrefix := "tmp/"

	// Defines a life cycle policy to expire keys for the temp directory.
	bucketLifecycleTempRule := &s3.LifecycleRule{
		ID:     aws.String("Rule for : " + s3BucketTempPrefix),
		Status: aws.String("Enabled"),
		Filter: &s3.LifecycleRuleFilter{
			Prefix: aws.String(s3BucketTempPrefix),
		},
		Expiration: &s3.LifecycleExpiration{
			// Indicates the lifetime, in days, of the objects that are subject to the rule.
			// The value must be a non-zero positive integer.
			Days: aws.Int64(1),
		},
		// Specifies the days since the initiation of an incomplete multipart upload
		// that Amazon S3 will wait before permanently removing all parts of the upload.
		// For more information, see Aborting Incomplete Multipart Uploads Using a Bucket
		// Lifecycle Policy (https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html#mpu-abort-incomplete-mpu-lifecycle-config)
		// in the Amazon Simple Storage Service Developer Guide.
		AbortIncompleteMultipartUpload: &s3.AbortIncompleteMultipartUpload{
			DaysAfterInitiation: aws.Int64(1),
		},
	}

	// Define the public S3 bucket used to serve static files for all the services.
	cfg.AwsS3BucketPublic = &devdeploy.AwsS3Bucket{
		BucketName:         cfg.ProjectName + "-public",
		IsPublic:           true,
		TempPrefix:         s3BucketTempPrefix,
		LocationConstraint: &cfg.AwsCredentials.Region,
		LifecycleRules:     []*s3.LifecycleRule{bucketLifecycleTempRule},
		CORSRules: []*s3.CORSRule{
			&s3.CORSRule{
				// Headers that are specified in the Access-Control-Request-Headers header.
				// These headers are allowed in a preflight OPTIONS request. In response to
				// any preflight OPTIONS request, Amazon S3 returns any requested headers that
				// are allowed.
				// AllowedHeaders: aws.StringSlice([]string{}),

				// An HTTP method that you allow the origin to execute. Valid values are GET,
				// PUT, HEAD, POST, and DELETE.
				//
				// AllowedMethods is a required field
				AllowedMethods: aws.StringSlice([]string{"GET", "POST"}),

				// One or more origins you want customers to be able to access the bucket from.
				//
				// AllowedOrigins is a required field
				AllowedOrigins: aws.StringSlice([]string{"*"}),

				// One or more headers in the response that you want customers to be able to
				// access from their applications (for example, from a JavaScript XMLHttpRequest
				// object).
				// ExposeHeaders: aws.StringSlice([]string{}),

				// The time in seconds that your browser is to cache the preflight response
				// for the specified resource.
				// MaxAgeSeconds: aws.Int64(),
			},
		},
	}

	// The base s3 key prefix used to upload static files.
	cfg.AwsS3BucketPublicKeyPrefix = "/public"

	// For production, enable Cloudfront CDN for all static files to avoid serving them from the slower S3 option.
	if cfg.Env == EnvProd {
		cfg.AwsS3BucketPublic.CloudFront = &devdeploy.AwsS3BucketCloudFront{
			// S3 key prefix to request your content from a directory in your Amazon S3 bucket.
			OriginPath: cfg.AwsS3BucketPublicKeyPrefix,

			// A complex type that controls whether CloudFront caches the response to requests.
			CachedMethods: []string{"HEAD", "GET"},

			// The distribution's configuration information.
			DistributionConfig: &cloudfront.DistributionConfig{
				Comment:       aws.String(""),
				Enabled:       aws.Bool(true),
				HttpVersion:   aws.String("http2"),
				IsIPV6Enabled: aws.Bool(true),
				DefaultCacheBehavior: &cloudfront.DefaultCacheBehavior{
					Compress:   aws.Bool(true),
					DefaultTTL: aws.Int64(1209600),
					MinTTL:     aws.Int64(604800),
					MaxTTL:     aws.Int64(31536000),
					ForwardedValues: &cloudfront.ForwardedValues{
						QueryString: aws.Bool(true),
						Cookies: &cloudfront.CookiePreference{
							Forward: aws.String("none"),
						},
					},
					TrustedSigners: &cloudfront.TrustedSigners{
						Enabled:  aws.Bool(false),
						Quantity: aws.Int64(0),
					},
					ViewerProtocolPolicy: aws.String("allow-all"),
				},
				ViewerCertificate: &cloudfront.ViewerCertificate{
					CertificateSource:            aws.String("cloudfront"),
					MinimumProtocolVersion:       aws.String("TLSv1"),
					CloudFrontDefaultCertificate: aws.Bool(true),
				},
				PriceClass:      aws.String("PriceClass_All"),
				CallerReference: aws.String("devops-deploy" + cfg.AwsS3BucketPublic.BucketName),
			},
		}
	}

	// Define the private S3 bucket used for long term file storage including but not limited to: log exports,
	// AWS Lambda code, application caching.
	cfg.AwsS3BucketPrivate = &devdeploy.AwsS3Bucket{
		BucketName:         cfg.ProjectName + "-private",
		IsPublic:           false,
		TempPrefix:         s3BucketTempPrefix,
		LocationConstraint: &cfg.AwsCredentials.Region,
		LifecycleRules:     []*s3.LifecycleRule{bucketLifecycleTempRule},
		PublicAccessBlock: &s3.PublicAccessBlockConfiguration{
			// Specifies whether Amazon S3 should block public access control lists (ACLs)
			// for this bucket and objects in this bucket. Setting this element to TRUE
			// causes the following behavior:
			//
			//    * PUT Bucket acl and PUT Object acl calls fail if the specified ACL is
			//    public.
			//
			//    * PUT Object calls fail if the request includes a public ACL.
			//
			// Enabling this setting doesn't affect existing policies or ACLs.
			BlockPublicAcls: aws.Bool(true),

			// Specifies whether Amazon S3 should block public bucket policies for this
			// bucket. Setting this element to TRUE causes Amazon S3 to reject calls to
			// PUT Bucket policy if the specified bucket policy allows public access.
			//
			// Enabling this setting doesn't affect existing bucket policies.
			BlockPublicPolicy: aws.Bool(true),

			// Specifies whether Amazon S3 should restrict public bucket policies for this
			// bucket. Setting this element to TRUE restricts access to this bucket to only
			// AWS services and authorized users within this account if the bucket has a
			// public policy.
			//
			// Enabling this setting doesn't affect previously stored bucket policies, except
			// that public and cross-account access within any public bucket policy, including
			// non-public delegation to specific accounts, is blocked.
			RestrictPublicBuckets: aws.Bool(true),

			// Specifies whether Amazon S3 should ignore public ACLs for this bucket and
			// objects in this bucket. Setting this element to TRUE causes Amazon S3 to
			// ignore all public ACLs on this bucket and objects in this bucket.
			//
			// Enabling this setting doesn't affect the persistence of any existing ACLs
			// and doesn't prevent new public ACLs from being set.
			IgnorePublicAcls: aws.Bool(true),
		},
	}

	// Add a bucket policy to enable exports from Cloudwatch Logs for the private S3 bucket.
	cfg.AwsS3BucketPrivate.Policy = func() string {
		policyResource := strings.Trim(filepath.Join(cfg.AwsS3BucketPrivate.BucketName, cfg.AwsS3BucketPrivate.TempPrefix), "/")
		return fmt.Sprintf(`{
				"Version": "2012-10-17",
				"Statement": [
				  {
					  "Action": "s3:GetBucketAcl",
					  "Effect": "Allow",
					  "Resource": "arn:aws:s3:::%s",
					  "Principal": { "Service": "logs.%s.amazonaws.com" }
				  },
				  {
					  "Action": "s3:PutObject" ,
					  "Effect": "Allow",
					  "Resource": "arn:aws:s3:::%s/*",
					  "Condition": { "StringEquals": { "s3:x-amz-acl": "bucket-owner-full-control" } },
					  "Principal": { "Service": "logs.%s.amazonaws.com" }
				  }
				]
			}`, cfg.AwsS3BucketPrivate.BucketName, cfg.AwsCredentials.Region, policyResource, cfg.AwsCredentials.Region)
	}()

	// Define the Redis Cache cluster used for ephemeral storage.
	cfg.AwsElasticCacheCluster = &devdeploy.AwsElasticCacheCluster{
		CacheClusterId:          cfg.ProjectName + "-" + cfg.Env,
		CacheNodeType:           "cache.t2.micro",
		CacheSubnetGroupName:    "default",
		Engine:                  "redis",
		EngineVersion:           "5.0.4",
		NumCacheNodes:           1,
		Port:                    6379,
		AutoMinorVersionUpgrade: aws.Bool(true),
		SnapshotRetentionLimit:  aws.Int64(7),
		ParameterNameValues: []devdeploy.AwsElasticCacheParameter{
			devdeploy.AwsElasticCacheParameter{
				ParameterName:  "maxmemory-policy",
				ParameterValue: "allkeys-lru",
			},
		},
	}

	// Define the RDS Database instance for transactional data. A random one will be generated for any created instance.
	cfg.AwsRdsDBInstance = &devdeploy.AwsRdsDBInstance{
		DBInstanceIdentifier:    cfg.ProjectName + "-" + cfg.Env,
		DBName:                  "shared",
		Engine:                  "postgres",
		MasterUsername:          "god",
		Port:                    5432,
		DBInstanceClass:         "db.t2.small",
		AllocatedStorage:        20,
		PubliclyAccessible:      false,
		BackupRetentionPeriod:   aws.Int64(7),
		AutoMinorVersionUpgrade: true,
		CopyTagsToSnapshot:      aws.Bool(true),
		Tags: []devdeploy.Tag{
			{Key: devdeploy.AwsTagNameProject, Value: cfg.ProjectName},
			{Key: devdeploy.AwsTagNameEnv, Value: cfg.Env},
		},
		AfterCreate: func(res *rds.DBInstance, dbInfo *devdeploy.DBConnInfo) error {
			masterDb, err := sqlx.Open(dbInfo.Driver, dbInfo.URL())
			if err != nil {
				return errors.WithMessage(err, "Failed to connect to db for schema migration.")
			}
			defer masterDb.Close()

			return schema.Migrate(context.Background(), masterDb, log, false)
		},
	}

	// AwsIamPolicy defines the name and policy that will be attached to the task role. The policy document grants
	// the permissions required for deployed services to access AWS services. If the policy already exists, the
	// statements will be used to add new required actions, but not for removal.
	cfg.AwsIamPolicy = &devdeploy.AwsIamPolicy{
		PolicyName:  fmt.Sprintf("%s%sServices", cfg.ProjectNameCamel(), strcase.ToCamel(cfg.Env)),
		Description: fmt.Sprintf("Defines access for %s services. ", cfg.ProjectName),
		PolicyDocument: devdeploy.AwsIamPolicyDocument{
			Version: "2012-10-17",
			Statement: []devdeploy.AwsIamStatementEntry{
				{
					Sid:    "DefaultServiceAccess",
					Effect: "Allow",
					Action: []string{
						"s3:HeadBucket",
						"s3:ListObjects",
						"s3:PutObject",
						"s3:PutObjectAcl",
						"cloudfront:ListDistributions",
						"ec2:DescribeNetworkInterfaces",
						"ec2:DeleteNetworkInterface",
						"ecs:ListTasks",
						"ecs:DescribeServices",
						"ecs:DescribeTasks",
						"ec2:DescribeNetworkInterfaces",
						"route53:ListHostedZones",
						"route53:ListResourceRecordSets",
						"route53:ChangeResourceRecordSets",
						"ecs:UpdateService",
						"ses:SendEmail",
						"ses:ListIdentities",
						"secretsmanager:ListSecretVersionIds",
						"secretsmanager:GetSecretValue",
						"secretsmanager:CreateSecret",
						"secretsmanager:UpdateSecret",
						"secretsmanager:RestoreSecret",
						"secretsmanager:DeleteSecret",
					},
					Resource: "*",
				},
				{
					Sid:    "ServiceInvokeLambda",
					Effect: "Allow",
					Action: []string{
						"iam:GetRole",
						"lambda:InvokeFunction",
						"lambda:ListVersionsByFunction",
						"lambda:GetFunction",
						"lambda:InvokeAsync",
						"lambda:GetFunctionConfiguration",
						"iam:PassRole",
						"lambda:GetAlias",
						"lambda:GetPolicy",
					},
					Resource: []string{
						"arn:aws:iam:::role/*",
						"arn:aws:lambda:::function:*",
					},
				},
				{
					Sid:    "datadoglambda",
					Effect: "Allow",
					Action: []string{
						"cloudwatch:Get*",
						"cloudwatch:List*",
						"ec2:Describe*",
						"support:*",
						"tag:GetResources",
						"tag:GetTagKeys",
						"tag:GetTagValues",
					},
					Resource: "*",
				},
			},
		},
	}
	log.Printf("\t\tSet Task Policy Name to '%s'.", cfg.AwsIamPolicy.PolicyName)

	return cfg, nil
}

// getDatadogApiKey tries to find the datadog api key from env variable or AWS Secrets Manager.
func getDatadogApiKey(cfg *devdeploy.Config) (string, error) {
	// Load Datadog API key which can be either stored in an environment variable or in AWS Secrets Manager.
	// 1. Check env vars for [DEV|STAGE|PROD]_DD_API_KEY and DD_API_KEY
	apiKey := devdeploy.GetTargetEnv(cfg.Env, "DD_API_KEY")

	// 2. Check AWS Secrets Manager for datadog entry prefixed with target environment.
	if apiKey == "" {
		prefixedSecretId := cfg.SecretID("datadog")
		var err error
		apiKey, err = devdeploy.GetAwsSecretValue(cfg.AwsCredentials, prefixedSecretId)
		if err != nil {
			if aerr, ok := errors.Cause(err).(awserr.Error); !ok || aerr.Code() != secretsmanager.ErrCodeResourceNotFoundException {
				return "", err
			}
		}
	}

	// 3. Check AWS Secrets Manager for Datadog entry.
	if apiKey == "" {
		secretId := "DATADOG"
		var err error
		apiKey, err = devdeploy.GetAwsSecretValue(cfg.AwsCredentials, secretId)
		if err != nil {
			if aerr, ok := errors.Cause(err).(awserr.Error); !ok || aerr.Code() != secretsmanager.ErrCodeResourceNotFoundException {
				return "", err
			}
		}
	}

	return apiKey, nil
}

// getCommitRef returns a string that will be used by go build to replace main.go:build constant.
func getCommitRef() string {
	var commitRef string

	// Set the commit ref based on the GitLab CI/CD environment variables.
	if ev := os.Getenv("CI_COMMIT_TAG"); ev != "" {
		commitRef = "tag-" + ev
	} else if ev := os.Getenv("CI_COMMIT_REF_NAME"); ev != "" {
		commitRef = "branch-" + ev
	}

	if commitRef != "" {
		if ev := os.Getenv("CI_COMMIT_SHORT_SHA"); ev != "" {
			commitRef = commitRef + "@" + ev
		}
	}

	return commitRef
}