golang-saas-starter-kit/example-project/internal/platform/devops/ecs_service.go

package devops

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/sethgrid/pester"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/ecs"
	"github.com/aws/aws-sdk-go/service/route53"
	"github.com/pkg/errors"
)

// EcsServiceTaskInit allows newly spun up ECS Service Tasks to register their public IP with Route 53.
func EcsServiceTaskInit(log *log.Logger, awsSession *session.Session) error {
	ecsClusterName := os.Getenv("ECS_CLUSTER")
	ecsServiceName := os.Getenv("ECS_SERVICE")

	// If both env variables are empty, this instance of the services is not running on AWS ECS.
	if ecsClusterName == "" && ecsServiceName == "" {
		return nil
	}

	res, err := pester.Get("http://169.254.170.2/v2/metadata")
	if err != nil {
		fmt.Println("http://169.254.170.2/v2/metadata failed", err.Error())
	} else {
		dat, _ := ioutil.ReadAll(res.Body)
		res.Body.Close()
		fmt.Println("http://169.254.170.2/v2/metadata, OK", string(dat))
	}

	var zoneArecNames = map[string][]string{}
	if v := os.Getenv("ROUTE53_ZONES"); v != "" {
		dat, err := base64.RawURLEncoding.DecodeString(v)
		if err != nil {
			return errors.Wrapf(err, "failed to base64 URL decode zones")
		}

		err = json.Unmarshal(dat, &zoneArecNames)
		if err != nil {
			return errors.Wrapf(err, "failed to json decode zones - %s", string(dat))
		}
	}

	var registerServiceTasks bool
	if v := os.Getenv("ROUTE53_UPDATE_TASK_IPS"); v != "" {
		var err error
		registerServiceTasks, err = strconv.ParseBool(v)
		if err != nil {
			return errors.Wrapf(err, "failed to parse ROUTE53_UPDATE_TASK_IPS value '%s' to bool", v)
		}
	}

	if registerServiceTasks {
		if err := RegisterEcsServiceTasksRoute53(log, awsSession, ecsClusterName, ecsServiceName, zoneArecNames); err != nil {
			return err
		}
	}

	return nil
}

// EcsServiceTaskTaskShutdown allows ECS Service Tasks that are spinning down to deregister their public IP with Route 53.
func EcsServiceTaskTaskShutdown(log *log.Logger, awsSession *session.Session) error {
	// TODO: Should lookup the IP for the current running task and remove that specific IP.
	// 		 For now just run the init since it removes all non-running tasks.
	return EcsServiceTaskInit(log, awsSession)
}

// RegisterEcsServiceTasksRoute53 registers the public IPs for a ECS Service Task with Route 53.
func RegisterEcsServiceTasksRoute53(log *log.Logger, awsSession *session.Session, ecsClusterName, ecsServiceName string, zoneArecNames map[string][]string) error {
	var networkInterfaceIds []string
	for a := 0; a <= 3; a++ {
		svc := ecs.New(awsSession)

		serviceRes, err := svc.DescribeServices(&ecs.DescribeServicesInput{
			Cluster:  aws.String(ecsClusterName),
			Services: []*string{aws.String(ecsServiceName)},
		})
		if err != nil {
			return errors.Wrapf(err, "failed to describe service '%s'", ecsServiceName)
		}
		service := serviceRes.Services[0]

		servceTaskRes, err := svc.ListTasks(&ecs.ListTasksInput{
			Cluster:       aws.String(ecsClusterName),
			ServiceName:   aws.String(ecsServiceName),
			DesiredStatus: aws.String("RUNNING"),
		})
		if err != nil {
			return errors.Wrapf(err, "failed to list tasks for cluster '%s' service '%s'", ecsClusterName, ecsServiceName)
		}

		taskRes, err := svc.DescribeTasks(&ecs.DescribeTasksInput{
			Cluster: aws.String(ecsClusterName),
			Tasks:   servceTaskRes.TaskArns,
		})
		if err != nil {
			return errors.Wrapf(err, "failed to describe %d tasks for cluster '%s'", len(servceTaskRes.TaskArns), ecsClusterName)
		}

		for _, t := range taskRes.Tasks {
			if *t.TaskDefinitionArn != *service.TaskDefinition || *t.DesiredStatus != "RUNNING" {
				continue
			}

			if t.Attachments == nil {
				continue
			}

			for _, c := range t.Containers {
				if *c.Name != ecsServiceName {
					continue
				}

				if c.NetworkInterfaces == nil || len(c.NetworkInterfaces) == 0 || c.NetworkInterfaces[0].AttachmentId == nil {
					continue
				}

				for _, a := range t.Attachments {
					if a.Details == nil || *a.Id != *c.NetworkInterfaces[0].AttachmentId {
						continue
					}

					for _, ad := range a.Details {
						if ad.Name != nil && *ad.Name == "networkInterfaceId" {
							networkInterfaceIds = append(networkInterfaceIds, *ad.Value)
						}
					}
				}

				break
			}
		}

		if len(networkInterfaceIds) > 0 {
			log.Printf("Found %d network interface IDs.\n", len(networkInterfaceIds))
			break
		}

		// Found no network interfaces, try again.
		log.Println("Found no network interfaces.")
		time.Sleep((time.Duration(a) * time.Second * 10) * time.Duration(a))
	}

	log.Println("Get public IPs for network interface IDs.")
	var publicIps []string
	for a := 0; a <= 3; a++ {
		svc := ec2.New(awsSession)

		log.Println("\t\tDescribe network interfaces.")
		res, err := svc.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{
			NetworkInterfaceIds: aws.StringSlice(networkInterfaceIds),
		})
		if err != nil {
			return errors.Wrap(err, "failed to describe network interfaces")
		}

		for _, ni := range res.NetworkInterfaces {
			if ni.Association == nil || ni.Association.PublicIp == nil {
				continue
			}
			publicIps = append(publicIps, *ni.Association.PublicIp)
		}

		if len(publicIps) > 0 {
			log.Printf("Found %d public IPs.\n", len(publicIps))
			break
		}

		// Found no public IPs, try again.
		log.Println("Found no public IPs.")
		time.Sleep((time.Duration(a) * time.Second * 10) * time.Duration(a))
	}

	if len(publicIps) > 0 {
		log.Println("Update public IPs for hosted zones.")

		svc := route53.New(awsSession)

		// Public IPs to be served as round robin.
		log.Printf("\tPublic IPs:\n")
		rrs := []*route53.ResourceRecord{}
		for _, ip := range publicIps {
			log.Printf("\t\t%s\n", ip)
			rrs = append(rrs, &route53.ResourceRecord{Value: aws.String(ip)})
		}

		for zoneId, aNames := range zoneArecNames {
			log.Printf("\tChange zone '%s'.\n", zoneId)

			input := &route53.ChangeResourceRecordSetsInput{
				ChangeBatch: &route53.ChangeBatch{
					Changes: []*route53.Change{},
				},
				HostedZoneId: aws.String(zoneId),
			}

			// Add all the A record names with the same set of public IPs.
			for _, aName := range aNames {
				log.Printf("\t\tAdd A record for '%s'.\n", aName)

				input.ChangeBatch.Changes = append(input.ChangeBatch.Changes, &route53.Change{
					Action: aws.String("UPSERT"),
					ResourceRecordSet: &route53.ResourceRecordSet{
						Name:            aws.String(aName),
						ResourceRecords: rrs,
						TTL:             aws.Int64(60),
						Type:            aws.String("A"),
					},
				})
			}

			_, err := svc.ChangeResourceRecordSets(input)
			if err != nil {
				return errors.Wrapf(err, "failed to update A records for zone '%s'", zoneId)
			}
		}

		log.Printf("DNS entries updated.\n")
	}

	return nil
}

/*
http://169.254.170.2/v2/metadata,

{
	"Cluster": "arn:aws:ecs:us-west-2:888955683113:cluster/example-project-dev",
	"TaskARN": "arn:aws:ecs:us-west-2:888955683113:task/700e38dd-dec5-4201-b711-c04a51feef8a",
	"Family": "web-api",
	"Revision": "113",
	"DesiredStatus": "RUNNING",
	"KnownStatus": "RUNNING",
	"Containers": [{
		"DockerId": "c786dfdf6510b20294832ccbc3d66e6f1f915a4a79ead2588aa760a6365c839a",
		"Name": "datadog-agent",
		"DockerName": "ecs-web-api-113-datadog-agent-d884dee0c79af1fb6400",
		"Image": "datadog/agent:latest",
		"ImageID": "sha256:233c75f21f71838a59d478472d021be7006e752da6a70a11f77cf185c1050737",
		"Labels": {
			"com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:888955683113:cluster/example-project-dev",
			"com.amazonaws.ecs.container-name": "datadog-agent",
			"com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:888955683113:task/700e38dd-dec5-4201-b711-c04a51feef8a",
			"com.amazonaws.ecs.task-definition-family": "web-api",
			"com.amazonaws.ecs.task-definition-version": "113"
		},
		"DesiredStatus": "RUNNING",
		"KnownStatus": "STOPPED",
		"ExitCode": 1,
		"Limits": {
			"CPU": 128,
			"Memory": 0
		},
		"CreatedAt": "2019-07-11T05:36:54.135666318Z",
		"StartedAt": "2019-07-11T05:36:54.481305866Z",
		"FinishedAt": "2019-07-11T05:36:54.863742829Z",
		"Type": "NORMAL",
		"Networks": [{
			"NetworkMode": "awsvpc",
			"IPv4Addresses": ["172.31.62.204"]
		}],
		"Volumes": [{
			"DockerName": "0960558c657c6e79d43e0e55f4ff259a97d78f58d9ad0d738e74495f4ba3cb06",
			"Source": "/var/lib/docker/volumes/0960558c657c6e79d43e0e55f4ff259a97d78f58d9ad0d738e74495f4ba3cb06/_data",
			"Destination": "/etc/datadog-agent"
		}, {
			"DockerName": "7a103f880857a1c2947e4a1bfff48efd25d24943a2d6a6e4dd86fa9dab3f10f0",
			"Source": "/var/lib/docker/volumes/7a103f880857a1c2947e4a1bfff48efd25d24943a2d6a6e4dd86fa9dab3f10f0/_data",
			"Destination": "/tmp"
		}, {
			"DockerName": "c88c03366eadb5d9da27708919e77ac5f8e0877c3dbb32c80580cb22e5811c00",
			"Source": "/var/lib/docker/volumes/c88c03366eadb5d9da27708919e77ac5f8e0877c3dbb32c80580cb22e5811c00/_data",
			"Destination": "/var/log/datadog"
		}, {
			"DockerName": "df97387f6ccc34c023055ef8a34a41e9d1edde4715c1849f1460683d31749539",
			"Source": "/var/lib/docker/volumes/df97387f6ccc34c023055ef8a34a41e9d1edde4715c1849f1460683d31749539/_data",
			"Destination": "/var/run/s6"
		}]
	}, {
		"DockerId": "ab6bd869e675f64122a33a74da9183b304bbc60b649a15d0d83ebc48eeafdd76",
		"Name": "~internal~ecs~pause",
		"DockerName": "ecs-web-api-113-internalecspause-aab99b88b9ddadb0c701",
		"Image": "fg-proxy:tinyproxy",
		"ImageID": "",
		"Labels": {
			"com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:888955683113:cluster/example-project-dev",
			"com.amazonaws.ecs.container-name": "~internal~ecs~pause",
			"com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:888955683113:task/700e38dd-dec5-4201-b711-c04a51feef8a",
			"com.amazonaws.ecs.task-definition-family": "web-api",
			"com.amazonaws.ecs.task-definition-version": "113"
		},
		"DesiredStatus": "RESOURCES_PROVISIONED",
		"KnownStatus": "RESOURCES_PROVISIONED",
		"Limits": {
			"CPU": 0,
			"Memory": 0
		},
		"CreatedAt": "2019-07-11T05:36:34.896093577Z",
		"StartedAt": "2019-07-11T05:36:35.302359045Z",
		"Type": "CNI_PAUSE",
		"Networks": [{
			"NetworkMode": "awsvpc",
			"IPv4Addresses": ["172.31.62.204"]
		}]
	}, {
		"DockerId": "07bce50839fc992393799457811e4a0ac56979b2164c7aec6e66b40162ae3119",
		"Name": "web-api-dev",
		"DockerName": "ecs-web-api-113-web-api-dev-ceefbfb4dba2a6e05900",
		"Image": "888955683113.dkr.ecr.us-west-2.amazonaws.com/example-project:dev-web-api",
		"ImageID": "sha256:cf793de01311ac4e5e32c76cb4625f6600ec8017c726e99e28ec2199d4af599b",
		"Labels": {
			"com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:888955683113:cluster/example-project-dev",
			"com.amazonaws.ecs.container-name": "web-api-dev",
			"com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:888955683113:task/700e38dd-dec5-4201-b711-c04a51feef8a",
			"com.amazonaws.ecs.task-definition-family": "web-api",
			"com.amazonaws.ecs.task-definition-version": "113",
			"com.datadoghq.ad.check_names": "[\"web-api-dev\"]",
			"com.datadoghq.ad.init_configs": "[{}]",
			"com.datadoghq.ad.instances": "[{\"host\": \"%%host%%\", \"port\": 80}]",
			"com.datadoghq.ad.logs": "[{\"source\": \"docker\", \"service\": \"web-api-dev\", \"service_name\": \"web-api\", \"cluster\": \"example-project-dev\", \"env\": \"dev\"}]"
		},
		"DesiredStatus": "RUNNING",
		"KnownStatus": "RUNNING",
		"Limits": {
			"CPU": 128,
			"Memory": 0
		},
		"CreatedAt": "2019-07-11T05:36:42.417547421Z",
		"StartedAt": "2019-07-11T05:36:53.88095717Z",
		"Type": "NORMAL",
		"Networks": [{
			"NetworkMode": "awsvpc",
			"IPv4Addresses": ["172.31.62.204"]
		}],
		"Health": {}
	}],
	"Limits": {
		"CPU": 0.5,
		"Memory": 2048
	},
	"PullStartedAt": "2019-07-11T05:36:35.407114703Z",
	"PullStoppedAt": "2019-07-11T05:36:54.128398742Z"
}
*/