From 9b3176952b43066ccd85375dc0391adfa14b3d91 Mon Sep 17 00:00:00 2001 From: Lee Brown Date: Tue, 27 Aug 2019 22:26:42 -0800 Subject: [PATCH] ECS service autoscaling policy option in cicd config --- build/cicd/internal/config/service.go | 128 ++++++++++++++++++++++++-- cmd/web-api/handlers/check.go | 1 + cmd/web-app/handlers/check.go | 1 + go.mod | 2 +- go.sum | 14 +-- 5 files changed, 126 insertions(+), 20 deletions(-) diff --git a/build/cicd/internal/config/service.go b/build/cicd/internal/config/service.go index 91d88e1..a3d91b3 100644 --- a/build/cicd/internal/config/service.go +++ b/build/cicd/internal/config/service.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/applicationautoscaling" "github.com/aws/aws-sdk-go/service/ecs" "github.com/iancoleman/strcase" "github.com/pkg/errors" @@ -16,6 +17,26 @@ import ( "gopkg.in/go-playground/validator.v9" ) +const ( + // EnableServiceElb will enable all services to be deployed with an ELB (Elastic Load Balancer). + // This will only be applied to the prod env, but the logic can be changed in the code below. + // + // When enabled each service will require it's own ELB and therefore will add $20~ month per service when + // this is enabled. The hostnames defined for the service will be updated in Route53 to resolve to the ELB. + // If HTTPS is enabled, the ELB will be created with an AWS ACM certificate that will support SSL termination on + // the ELB, all traffic will be sent to the container as HTTP. + // This can be configured on a by service basis. + // + // When not enabled, tasks will be auto assigned a public IP. As ECS tasks for the service are launched/terminated, + // the task will update the hostnames defined for the service in Route53 to either add/remove its public IP. This + // option is good for services that only need one container running. + EnableServiceElb = false + + // EnableServiceAutoscaling will enable all services to be deployed with an application scaling policy. This should + // typically be enabled for front end services that have an ELB enabled. + EnableServiceAutoscaling = false +) + // Service define the name of a service. type Service = string @@ -35,7 +56,7 @@ type ServiceContext struct { // Required flags. Name string `validate:"required" example:"web-api"` ServiceHostPrimary string `validate:"required" example:"example-project.com"` - DesiredCount int `validate:"required" example:"2"` + DesiredCount int64 `validate:"required" example:"2"` ServiceDir string `validate:"required"` Dockerfile string `validate:"required" example:"./cmd/web-api/Dockerfile"` ReleaseTag string `validate:"required"` @@ -43,7 +64,6 @@ type ServiceContext struct { // Optional flags. ServiceHostNames []string `validate:"omitempty" example:"subdomain.example-project.com"` EnableHTTPS bool `validate:"omitempty" example:"false"` - EnableElb bool `validate:"omitempty" example:"false"` StaticFilesS3Enable bool `validate:"omitempty" example:"false"` DockerBuildDir string `validate:"omitempty"` DockerBuildContext string `validate:"omitempty" example:"."` @@ -165,6 +185,16 @@ func NewService(serviceName string, cfg *devdeploy.Config) (*devdeploy.ProjectSe srv.StaticFilesS3Prefix = filepath.Join(cfg.AwsS3BucketPublicKeyPrefix, ctx.ReleaseTag, "static") } + + // ========================================================================= + // Service settings based on target env. + var enableElb bool + if cfg.Env == EnvStage || cfg.Env == EnvProd { + if cfg.Env == EnvProd && EnableServiceElb { + enableElb = true + } + } + // ========================================================================= // Shared details that could be applied to all task definitions. @@ -227,10 +257,10 @@ func NewService(serviceName string, cfg *devdeploy.Config) (*devdeploy.ProjectSe log.Printf("\t\tSet AWS Service Discovery Namespace to '%s'.", srv.AwsSdPrivateDnsNamespace.Name) // If the service is requested to use an elastic load balancer then define. - if ctx.EnableElb { + if enableElb { // AwsElbLoadBalancer defines if the service should use an elastic load balancer. srv.AwsElbLoadBalancer = &devdeploy.AwsElbLoadBalancer{ - Name: fmt.Sprintf("%s-%s-%s", cfg.Env, srv.AwsEcsCluster.ClusterName, ctx.Name), + Name: fmt.Sprintf("%s-%s", cfg.Env, ctx.Name), IpAddressType: "ipv4", Scheme: "internet-facing", Type: "application", @@ -274,7 +304,7 @@ func NewService(serviceName string, cfg *devdeploy.Config) (*devdeploy.ProjectSe // AwsEcsService defines the details for the ecs service. srv.AwsEcsService = &devdeploy.AwsEcsService{ ServiceName: ctx.Name, - DesiredCount: int64(ctx.DesiredCount), + DesiredCount: ctx.DesiredCount, EnableECSManagedTags: false, HealthCheckGracePeriodSeconds: 60, LaunchType: "FARGATE", @@ -289,6 +319,90 @@ func NewService(serviceName string, cfg *devdeploy.Config) (*devdeploy.ProjectSe srv.AwsEcsService.DeploymentMaximumPercent = 200 } + if EnableServiceAutoscaling { + srv.AwsAppAutoscalingPolicy = &devdeploy.AwsAppAutoscalingPolicy{ + // The name of the scaling policy. + PolicyName: srv.AwsEcsService.ServiceName, + + // The policy type. This parameter is required if you are creating a scaling + // policy. + // + // The following policy types are supported: + // + // TargetTrackingScaling—Not supported for Amazon EMR or AppStream + // + // StepScaling—Not supported for Amazon DynamoDB + // + // For more information, see Step Scaling Policies for Application Auto Scaling + // (https://docs.aws.amazon.com/autoscaling/application/userguide/application-auto-scaling-step-scaling-policies.html) + // and Target Tracking Scaling Policies for Application Auto Scaling (https://docs.aws.amazon.com/autoscaling/application/userguide/application-auto-scaling-target-tracking.html) + // in the Application Auto Scaling User Guide. + PolicyType: "TargetTrackingScaling", + + // The minimum value to scale to in response to a scale-in event. MinCapacity + // is required to register a scalable target. + MinCapacity: ctx.DesiredCount, + + // The maximum value to scale to in response to a scale-out event. MaxCapacity + // is required to register a scalable target. + MaxCapacity: ctx.DesiredCount * 2, + + // A target tracking scaling policy. Includes support for predefined or customized metrics. + TargetTrackingScalingPolicyConfiguration: &applicationautoscaling.TargetTrackingScalingPolicyConfiguration{ + + // A predefined metric. You can specify either a predefined metric or a customized + // metric. + PredefinedMetricSpecification: &applicationautoscaling.PredefinedMetricSpecification{ + // The metric type. The following predefined metrics are available: + // + // * ASGAverageCPUUtilization - Average CPU utilization of the Auto Scaling + // group. + // + // * ASGAverageNetworkIn - Average number of bytes received on all network + // interfaces by the Auto Scaling group. + // + // * ASGAverageNetworkOut - Average number of bytes sent out on all network + // interfaces by the Auto Scaling group. + // + // * ALBRequestCountPerTarget - Number of requests completed per target in + // an Application Load Balancer target group. ResourceLabel will be auto populated. + // + PredefinedMetricType: aws.String("ECSServiceAverageCPUUtilization"), + }, + + // The target value for the metric. The range is 8.515920e-109 to 1.174271e+108 + // (Base 10) or 2e-360 to 2e360 (Base 2). + TargetValue: aws.Float64(70.0), + + // The amount of time, in seconds, after a scale-in activity completes before + // another scale in activity can start. + // + // The cooldown period is used to block subsequent scale-in requests until it + // has expired. The intention is to scale in conservatively to protect your + // application's availability. However, if another alarm triggers a scale-out + // policy during the cooldown period after a scale-in, Application Auto Scaling + // scales out your scalable target immediately. + ScaleInCooldown: aws.Int64(300), + + // The amount of time, in seconds, after a scale-out activity completes before + // another scale-out activity can start. + // + // While the cooldown period is in effect, the capacity that has been added + // by the previous scale-out event that initiated the cooldown is calculated + // as part of the desired capacity for the next scale out. The intention is + // to continuously (but not excessively) scale out. + ScaleOutCooldown: aws.Int64(300), + + // Indicates whether scale in by the target tracking scaling policy is disabled. + // If the value is true, scale in is disabled and the target tracking scaling + // policy won't remove capacity from the scalable resource. Otherwise, scale + // in is enabled and the target tracking scaling policy can remove capacity + // from the scalable resource. The default value is false. + DisableScaleIn: aws.Bool(false), + }, + } + } + // Load the web-app config for the web-api can reference it's hostname. webAppCtx, err := NewServiceContext(ServiceWebApp, cfg) if err != nil { @@ -386,7 +500,7 @@ func NewService(serviceName string, cfg *devdeploy.Config) (*devdeploy.ProjectSe // If the service has HTTPS enabled with the use of an AWS Elastic Load Balancer, then need to enable // traffic for port 443 for SSL traffic to get terminated on the deployed tasks. - if ctx.EnableHTTPS && !ctx.EnableElb { + if ctx.EnableHTTPS && !enableElb { container1.PortMappings = append(container1.PortMappings, &ecs.PortMapping{ HostPort: aws.Int64(443), Protocol: aws.String("tcp"), @@ -512,7 +626,7 @@ func NewService(serviceName string, cfg *devdeploy.Config) (*devdeploy.ProjectSe // If the service has HTTPS enabled with the use of an AWS Elastic Load Balancer, then need to enable // traffic for port 443 for SSL traffic to get terminated on the deployed tasks. - if ctx.EnableHTTPS && !ctx.EnableElb { + if ctx.EnableHTTPS && !enableElb { container1.PortMappings = append(container1.PortMappings, &ecs.PortMapping{ HostPort: aws.Int64(443), Protocol: aws.String("tcp"), diff --git a/cmd/web-api/handlers/check.go b/cmd/web-api/handlers/check.go index d2e3833..be8afba 100644 --- a/cmd/web-api/handlers/check.go +++ b/cmd/web-api/handlers/check.go @@ -58,6 +58,7 @@ func (c *Check) Health(ctx context.Context, w http.ResponseWriter, r *http.Reque } // Ping validates the service is ready to accept requests. +// This endpoint is used for the health check when an AWS ELastic Load Balancer is enabled. func (c *Check) Ping(ctx context.Context, w http.ResponseWriter, r *http.Request, params map[string]string) error { status := "pong" diff --git a/cmd/web-app/handlers/check.go b/cmd/web-app/handlers/check.go index 5ad03e6..5af4ab2 100644 --- a/cmd/web-app/handlers/check.go +++ b/cmd/web-app/handlers/check.go @@ -58,6 +58,7 @@ func (c *Check) Health(ctx context.Context, w http.ResponseWriter, r *http.Reque } // Ping validates the service is ready to accept requests. +// This endpoint is used for the health check when an AWS ELastic Load Balancer is enabled. func (c *Check) Ping(ctx context.Context, w http.ResponseWriter, r *http.Request, params map[string]string) error { status := "pong" diff --git a/go.mod b/go.mod index ba7b54a..c2b9268 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/tinylib/msgp v1.1.0 // indirect github.com/urfave/cli v1.21.0 github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 - gitlab.com/geeks-accelerator/oss/devops v1.0.17 + gitlab.com/geeks-accelerator/oss/devops v1.0.18 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 golang.org/x/tools v0.0.0-20190807223507-b346f7fd45de // indirect diff --git a/go.sum b/go.sum index 25449ae..5df1e2d 100644 --- a/go.sum +++ b/go.sum @@ -215,18 +215,8 @@ github.com/urfave/cli v1.21.0/go.mod h1:lxDj6qX9Q6lWQxIrbrT0nwecwUtRnhVZAJjJZrVU github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 h1:zzrxE1FKn5ryBNl9eKOeqQ58Y/Qpo3Q9QNxKHX5uzzQ= github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2/go.mod h1:hzfGeIUDq/j97IG+FhNqkowIyEcD88LrW6fyU3K3WqY= -gitlab.com/geeks-accelerator/oss/devops v1.0.11 h1:ojSvv4bSOZSyGjFMvpbJyREVfdN1A9O3CrOyTkNtb9c= -gitlab.com/geeks-accelerator/oss/devops v1.0.11/go.mod h1:xr+rhNSDXrEh0A6bkBPnfMiRIou3OiPZK0oD5h9GAAM= -gitlab.com/geeks-accelerator/oss/devops v1.0.13 h1:Wnf+vXPP8Ps4tSVdbk/vgl1rHaAELIPE3OYBAzvroG8= -gitlab.com/geeks-accelerator/oss/devops v1.0.13/go.mod h1:xr+rhNSDXrEh0A6bkBPnfMiRIou3OiPZK0oD5h9GAAM= -gitlab.com/geeks-accelerator/oss/devops v1.0.14 h1:jNLi69UAH44+FkixN/rtS7qobsSFvxwQ+g8NgVOwFt0= -gitlab.com/geeks-accelerator/oss/devops v1.0.14/go.mod h1:xr+rhNSDXrEh0A6bkBPnfMiRIou3OiPZK0oD5h9GAAM= -gitlab.com/geeks-accelerator/oss/devops v1.0.15 h1:JEadFDCPVqKSNFLFBNmqm94SU0AhO0niRRViUcwMxBc= -gitlab.com/geeks-accelerator/oss/devops v1.0.15/go.mod h1:xr+rhNSDXrEh0A6bkBPnfMiRIou3OiPZK0oD5h9GAAM= -gitlab.com/geeks-accelerator/oss/devops v1.0.16 h1:/dudDP9MjctP5caHpVZfJcxlC7ZeOW+KzaQ2UOzQ2hI= -gitlab.com/geeks-accelerator/oss/devops v1.0.16/go.mod h1:xr+rhNSDXrEh0A6bkBPnfMiRIou3OiPZK0oD5h9GAAM= -gitlab.com/geeks-accelerator/oss/devops v1.0.17 h1:5m6gEH9OXoGs48dtodcHbL0u3g6q3yiTsJ75IpjcFa8= -gitlab.com/geeks-accelerator/oss/devops v1.0.17/go.mod h1:xr+rhNSDXrEh0A6bkBPnfMiRIou3OiPZK0oD5h9GAAM= +gitlab.com/geeks-accelerator/oss/devops v1.0.18 h1:Vkk7WrTIvGd+Nnb6ru3o4r1yw4h7lJBdcnGLG71d390= +gitlab.com/geeks-accelerator/oss/devops v1.0.18/go.mod h1:xr+rhNSDXrEh0A6bkBPnfMiRIou3OiPZK0oD5h9GAAM= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=