From 90930b72d275e92321f8c062b22cf0dae4c3cf05 Mon Sep 17 00:00:00 2001 From: Lee Brown Date: Sat, 13 Jul 2019 22:43:49 -0800 Subject: [PATCH] added missing aws.go file due to gitignore rule --- .gitignore | 2 + tools/devops/cmd/deploy/aws.go | 209 +++++++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+) create mode 100644 tools/devops/cmd/deploy/aws.go diff --git a/.gitignore b/.gitignore index de7d35f..72a68b1 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ go.mod aws.lee aws.* .env_docker_compose + +!aws.go \ No newline at end of file diff --git a/tools/devops/cmd/deploy/aws.go b/tools/devops/cmd/deploy/aws.go new file mode 100644 index 0000000..4708a7f --- /dev/null +++ b/tools/devops/cmd/deploy/aws.go @@ -0,0 +1,209 @@ +package deploy + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "path/filepath" + "sort" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ecr" + "github.com/aws/aws-sdk-go/service/ecs" + "github.com/aws/aws-sdk-go/service/secretsmanager" + "github.com/pkg/errors" + "gopkg.in/go-playground/validator.v9" +) + +const ( + defaultAwsRegistryMaxImages = 1000 + awsTagNameProject = "project" + awsTagNameEnv = "env" + awsTagNameName = "Name" +) + +func GetAwsCredentials(targetEnv string) (awsCredentials, error) { + var creds awsCredentials + + creds.AccessKeyID = strings.TrimSpace(getTargetEnv(targetEnv, "AWS_ACCESS_KEY_ID")) + creds.SecretAccessKey = strings.TrimSpace(getTargetEnv(targetEnv, "AWS_SECRET_ACCESS_KEY")) + creds.Region = strings.TrimSpace(getTargetEnv(targetEnv, "AWS_REGION")) + + errs := validator.New().Struct(creds) + if errs != nil { + return creds, errs + } + + //os.Setenv("AWS_DEFAULT_REGION", creds.Region) + + return creds, nil +} + +func GetAwsSecretValue(creds awsCredentials, secretId string) (string, error) { + svc := secretsmanager.New(creds.Session()) + + res, err := svc.GetSecretValue(&secretsmanager.GetSecretValueInput{ + SecretId: aws.String(secretId), + }) + if err != nil { + return "", errors.Wrapf(err, "failed to get value for secret id %s", secretId) + } + + return string(res.SecretBinary), nil +} + +// EcrPurgeImages ensures pipeline does not generate images for max of 10000 and prevent manual deletion of images. +func EcrPurgeImages(req *serviceDeployRequest) ([]*ecr.ImageIdentifier, error) { + + svc := ecr.New(req.awsSession()) + + // First list all the image IDs for the repository. + var imgIds []*ecr.ImageIdentifier + err := svc.ListImagesPages(&ecr.ListImagesInput{ + RepositoryName: aws.String(req.EcrRepositoryName), + }, func(res *ecr.ListImagesOutput, lastPage bool) bool { + imgIds = append(imgIds, res.ImageIds...) + return !lastPage + }) + if err != nil { + return nil, errors.Wrapf(err, "failed to list images for repository '%s'", req.EcrRepositoryName) + } + + var ( + ts []int + tsImgIds = map[int][]*ecr.ImageIdentifier{} + ) + + // Describe all the image IDs to determine oldest. + err = svc.DescribeImagesPages(&ecr.DescribeImagesInput{ + RepositoryName: aws.String(req.EcrRepositoryName), + ImageIds: imgIds, + }, func(res *ecr.DescribeImagesOutput, lastPage bool) bool { + for _, img := range res.ImageDetails { + + imgTs := int(img.ImagePushedAt.Unix()) + + if _, ok := tsImgIds[imgTs]; !ok { + tsImgIds[imgTs] = []*ecr.ImageIdentifier{} + ts = append(ts, imgTs) + } + + if img.ImageTags != nil { + tsImgIds[imgTs] = append(tsImgIds[imgTs], &ecr.ImageIdentifier{ + ImageTag: img.ImageTags[0], + }) + } else if img.ImageDigest != nil { + tsImgIds[imgTs] = append(tsImgIds[imgTs], &ecr.ImageIdentifier{ + ImageDigest: img.ImageDigest, + }) + } + } + + return !lastPage + }) + if err != nil { + return nil, errors.Wrapf(err, "failed to describe images for repository '%s'", req.EcrRepositoryName) + } + + // Sort the image timestamps in reverse order. + sort.Sort(sort.Reverse(sort.IntSlice(ts))) + + // Loop over all the timestamps, skip the newest images until count exceeds limit. + var imgCnt int + var delIds []*ecr.ImageIdentifier + for _, imgTs := range ts { + for _, imgId := range tsImgIds[imgTs] { + imgCnt = imgCnt + 1 + + if imgCnt <= req.EcrRepositoryMaxImages { + continue + } + delIds = append(delIds, imgId) + } + } + + // If there are image IDs to delete, delete them. + if len(delIds) > 0 { + //log.Printf("\t\tECR has %d images for repository '%s' which exceeds limit of %d", imgCnt, creds.EcrRepositoryName, creds.EcrRepositoryMaxImages) + //for _, imgId := range delIds { + // log.Printf("\t\t\tDelete %s", *imgId.ImageTag) + //} + + _, err = svc.BatchDeleteImage(&ecr.BatchDeleteImageInput{ + ImageIds: delIds, + RepositoryName: aws.String(req.EcrRepositoryName), + }) + if err != nil { + return nil, errors.Wrapf(err, "failed to delete %d images for repository '%s'", len(delIds), req.EcrRepositoryName) + } + } + + return delIds, nil +} + +// EcsReadTaskDefinition reads a task definition file and json decodes it. +func EcsReadTaskDefinition(serviceDir, targetEnv string) ([]byte, error) { + checkPaths := []string{ + filepath.Join(serviceDir, fmt.Sprintf("ecs-task-definition-%s.json", targetEnv)), + filepath.Join(serviceDir, "ecs-task-definition.json"), + } + + var defFile string + for _, tf := range checkPaths { + ok, _ := exists(tf) + if ok { + defFile = tf + break + } + } + + if defFile == "" { + return nil, errors.Errorf("failed to locate task definition - checked %s", strings.Join(checkPaths, ", ")) + } + + dat, err := ioutil.ReadFile(defFile) + if err != nil { + return nil, errors.WithMessagef(err, "failed to read file %s", defFile) + } + + return dat, nil +} + +// parseTaskDefinition json decodes it. +func parseTaskDefinitionInput(dat []byte) (*ecs.RegisterTaskDefinitionInput, error) { + dat = convertKeys(dat) + + var taskDef *ecs.RegisterTaskDefinitionInput + if err := json.Unmarshal(dat, &taskDef); err != nil { + return nil, errors.WithMessagef(err, "failed to json decode task definition - %s", string(dat)) + } + + return taskDef, nil +} + +// convertKeys fixes json keys to they can be unmarshaled into aws types. No AWS structs have json tags. +func convertKeys(j json.RawMessage) json.RawMessage { + m := make(map[string]json.RawMessage) + if err := json.Unmarshal([]byte(j), &m); err != nil { + // Not a JSON object + return j + } + + for k, v := range m { + fixed := fixKey(k) + delete(m, k) + m[fixed] = convertKeys(v) + } + + b, err := json.Marshal(m) + if err != nil { + return j + } + + return json.RawMessage(b) +} + +func fixKey(key string) string { + return strings.ToTitle(key) +}