completed autocert implimentation for web-api

example-project/cmd/web-api/ecs-task-definition.json

@@ -36,8 +36,8 @@
         {"name": "WEB_API_HTTPS_HOST", "value": "{HTTPS_HOST}"},
         {"name": "WEB_API_APP_PROJECT", "value": "{APP_PROJECT}"},
         {"name": "WEB_API_APP_BASE_URL", "value": "{APP_BASE_URL}"},
-        {"name": "WEB_API_HOST_PRIMARY", "value": "{HOST_PRIMARY}"},
+        {"name": "WEB_API_APP_HOST_PRIMARY", "value": "{HOST_PRIMARY}"},
-        {"name": "WEB_API_HOST_NAMES", "value": "{HOST_NAMES}"},
+        {"name": "WEB_API_APP_HOST_NAMES", "value": "{HOST_NAMES}"},
         {"name": "WEB_API_REDIS_HOST", "value": "{CACHE_HOST}"},
         {"name": "WEB_API_DB_HOST", "value": "{DB_HOST}"},
         {"name": "WEB_API_DB_USER", "value": "{DB_USER}"},

example-project/cmd/web-api/main.go

@@ -30,6 +30,7 @@ import (
 	"github.com/go-redis/redis"
 	"github.com/kelseyhightower/envconfig"
 	"github.com/lib/pq"
+	"golang.org/x/crypto/acme"
 	"golang.org/x/crypto/acme/autocert"
 	awstrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/aws/aws-sdk-go/aws"
 	sqltrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/database/sql"
@@ -84,6 +85,7 @@ func main() {
 			Host         string        `default:"" envconfig:"HOST"`
 			ReadTimeout  time.Duration `default:"5s" envconfig:"READ_TIMEOUT"`
 			WriteTimeout time.Duration `default:"5s" envconfig:"WRITE_TIMEOUT"`
+			DisableHTTP2 bool          `default:"false" envconfig:"DISABLE_HTTP2"`
 		}
 		App struct {
 			Name            string        `default:"web-api" envconfig:"NAME"`
@@ -179,7 +181,6 @@ func main() {
 		cfg.Aws.SecretsManagerConfigPrefix = filepath.Join(pts...)
 	}
 
-
 	// If base URL is empty, set the default value from the HTTP Host
 	if cfg.App.BaseUrl == "" {
 		baseUrl := cfg.HTTP.Host
@@ -196,7 +197,6 @@ func main() {
 		cfg.App.BaseUrl = baseUrl
 	}
 
-
 	// =========================================================================
 	// Log App Info
 
@@ -239,7 +239,6 @@ func main() {
 		awsSession = awstrace.WrapSession(awsSession)
 	}
 
-
 	// =========================================================================
 	// Start Redis
 	// Ensure the eviction policy on the redis cluster is set correctly.
@@ -310,7 +309,6 @@ func main() {
 	}
 	defer masterDb.Close()
 
-
 	// =========================================================================
 	// Init new Authenticator
 	var authenticator *auth.Authenticator
@@ -324,15 +322,17 @@ func main() {
 		log.Fatalf("main : Constructing authenticator : %+v", err)
 	}
 
-
 	// =========================================================================
 	// Init redirect middleware to ensure all requests go to the primary domain.
+	primaryDomain := cfg.App.HostPrimary
+
+	// When primary host is not set, we can parse host from the base app URL.
+	if primaryDomain == "" {
 		baseSiteUrl, err := url.Parse(cfg.App.BaseUrl)
 		if err != nil {
 			log.Fatalf("main : Parse App Base URL : %s : %+v", cfg.App.BaseUrl, err)
 		}
 
-	var primaryDomain string
 		if strings.Contains(baseSiteUrl.Host, ":") {
 			primaryDomain, _, err = net.SplitHostPort(baseSiteUrl.Host)
 			if err != nil {
@@ -341,13 +341,22 @@ func main() {
 		} else {
 			primaryDomain = baseSiteUrl.Host
 		}
+	}
 
 	redirect := mid.DomainNameRedirect(mid.DomainNameRedirectConfig{
+		RedirectConfig: mid.RedirectConfig{
+			Code: http.StatusMovedPermanently,
+			Skipper: func(ctx context.Context, w http.ResponseWriter, r *http.Request, params map[string]string) bool {
+				if r.URL.Path == "/ping" {
+					return true
+				}
+				return false
+			},
+		},
 		DomainName:   primaryDomain,
 		HTTPSEnabled: (cfg.HTTPS.Host != ""),
 	})
 
-
 	// =========================================================================
 	// Start Tracing Support
 	th := fmt.Sprintf("%s:%d", cfg.Trace.Host, cfg.Trace.Port)
@@ -369,7 +378,6 @@ func main() {
 		}()
 	}
 
-
 	// =========================================================================
 	// ECS Task registration for services that don't use an AWS Elastic Load Balancer.
 	err = devops.EcsServiceTaskInit(log, awsSession)
@@ -377,7 +385,6 @@ func main() {
 		log.Fatalf("main : Ecs Service Task init : %+v", err)
 	}
 
-
 	// =========================================================================
 	// Start API Service
 
@@ -448,7 +455,10 @@ func main() {
 		// Enable autocert to store certs via Secret Manager.
 		secretPrefix := filepath.Join(cfg.Aws.SecretsManagerConfigPrefix, "autocert")
 
-		cache, err := devops.NewSecretManagerAutocertCache(log, awsSession, secretPrefix)
+		// Local file cache to reduce requests hitting Secret Manager.
+		localCache := autocert.DirCache(os.TempDir())
+
+		cache, err := devops.NewSecretManagerAutocertCache(log, awsSession, secretPrefix, localCache)
 		if err != nil {
 			log.Fatalf("main : HTTPS : %+v", err)
 		}
@@ -459,11 +469,15 @@ func main() {
 			Cache:      cache,
 		}
 		api.TLSConfig = &tls.Config{GetCertificate: m.GetCertificate}
+		api.TLSConfig.NextProtos = append(api.TLSConfig.NextProtos, acme.ALPNProto)
+		if !cfg.HTTPS.DisableHTTP2 {
+			api.TLSConfig.NextProtos = append(api.TLSConfig.NextProtos, "h2")
+		}
 
 		httpServers = append(httpServers, api)
 
 		go func() {
-			log.Printf("main : API Listening %s", cfg.HTTPS.Host)
+			log.Printf("main : API Listening %s with SSL cert for hosts %s", cfg.HTTPS.Host, strings.Join(hosts, ", "))
 			serverErrors <- api.ListenAndServeTLS("", "")
 		}()
 	}

example-project/internal/mid/redirect.go

@@ -2,9 +2,10 @@ package mid
 
 import (
 	"context"
+	"net/http"
+
 	"geeks-accelerator/oss/saas-starter-kit/example-project/internal/platform/web"
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
-	"net/http"
 )
 
 type (
@@ -56,8 +57,6 @@ func DomainNameRedirect(config DomainNameRedirectConfig) web.Middleware  {
 		// Redirects http requests to https.
 		if config.HTTPSEnabled {
 			if ok = scheme != "https"; ok {
-				url = "https://" + host + uri
-
 				scheme = "https"
 			}
 		}
@@ -65,16 +64,15 @@ func DomainNameRedirect(config DomainNameRedirectConfig) web.Middleware  {
 		// Redirects all domain name alternatives to the primary hostname.
 		if host != config.DomainName {
 			host = config.DomainName
+			ok = true
 		}
 
-
 		url = scheme + "://" + host + uri
 
 		return
 	})
 }
 
-
 // HTTPSRedirect redirects http requests to https.
 // For example, http://geeksinthewoods.com will be redirect to https://geeksinthewoods.com.
 func HTTPSRedirect() web.Middleware {
@@ -83,7 +81,7 @@ func HTTPSRedirect() web.Middleware {
 
 // HTTPSRedirectWithConfig returns an HTTPSRedirect middleware with config.
 // See `HTTPSRedirect()`.
-func HTTPSRedirectWithConfig(config RedirectConfig)web.Middleware  {
+func HTTPSRedirectWithConfig(config RedirectConfig) web.Middleware {
 	return redirect(config, func(scheme, host, uri string) (ok bool, url string) {
 		if ok = scheme != "https"; ok {
 			url = "https://" + host + uri

example-project/internal/platform/devops/autocert.go

@@ -7,10 +7,10 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/secretsmanager"
 	"github.com/pkg/errors"
 	"golang.org/x/crypto/acme/autocert"
-	"github.com/aws/aws-sdk-go/aws/session"
 )
 
 // SecretManagerAutocertCache implements the autocert.Cache interface for AWS Secrets Manager that is used by Manager
@@ -19,14 +19,16 @@ type SecretManagerAutocertCache struct  {
 	awsSession   *session.Session
 	log          *log.Logger
 	secretPrefix string
+	cache        autocert.Cache
 }
 
 // NewSecretManagerAutocertCache provides the functionality to keep config files sync'd between running tasks and across deployments.
-func NewSecretManagerAutocertCache(log *log.Logger, awsSession *session.Session, secretPrefix string ) (*SecretManagerAutocertCache, error) {
+func NewSecretManagerAutocertCache(log *log.Logger, awsSession *session.Session, secretPrefix string, cache autocert.Cache) (*SecretManagerAutocertCache, error) {
 	return &SecretManagerAutocertCache{
 		awsSession,
 		log,
 		secretPrefix,
+		cache,
 	}, nil
 }
 
@@ -34,6 +36,16 @@ func NewSecretManagerAutocertCache(log *log.Logger, awsSession *session.Session,
 // If there's no such key, Get returns ErrCacheMiss.
 func (c *SecretManagerAutocertCache) Get(ctx context.Context, key string) ([]byte, error) {
 
+	// Check short term cache.
+	if c.cache != nil {
+		v, err := c.cache.Get(ctx, key)
+		if err != nil && err != autocert.ErrCacheMiss {
+			return nil, errors.WithStack(err)
+		} else if len(v) > 0 {
+			return v, nil
+		}
+	}
+
 	svc := secretsmanager.New(c.awsSession)
 
 	secretID := filepath.Join(c.secretPrefix, key)
@@ -43,7 +55,7 @@ func (c *SecretManagerAutocertCache) Get(ctx context.Context, key string) ([]byt
 		SecretId: aws.String(secretID),
 	})
 	if err != nil {
-		if aerr, ok := err.(awserr.Error); ok && aerr.Code() == secretsmanager.ErrCodeResourceNotFoundException {
+		if aerr, ok := err.(awserr.Error); ok && (aerr.Code() == secretsmanager.ErrCodeResourceNotFoundException || aerr.Code() == secretsmanager.ErrCodeInvalidRequestException) {
 			return nil, autocert.ErrCacheMiss
 		}
 
@@ -52,7 +64,7 @@ func (c *SecretManagerAutocertCache) Get(ctx context.Context, key string) ([]byt
 
 	log.Printf("AWS Secrets Manager : Secret %s found", secretID)
 
-	return res.SecretBinary, nil
+	return []byte(*res.SecretString), nil
 }
 
 // Put stores the data in the cache under the specified key.
@@ -70,9 +82,9 @@ func (c *SecretManagerAutocertCache) Put(ctx context.Context, key string, data [
 		SecretString: aws.String(string(data)),
 	})
 	if err != nil {
-		if aerr, ok := err.(awserr.Error); !ok {
+		aerr, ok := err.(awserr.Error)
 
-			if aerr.Code() == secretsmanager.ErrCodeInvalidRequestException {
+		if ok && aerr.Code() == secretsmanager.ErrCodeInvalidRequestException {
 			// InvalidRequestException: You can't create this secret because a secret with this
 			// 							 name is already scheduled for deletion.
 
@@ -84,10 +96,9 @@ func (c *SecretManagerAutocertCache) Put(ctx context.Context, key string, data [
 				return errors.Wrapf(err, "autocert failed to restore secret %s", secretID)
 			}
 
-			} else if aerr.Code() != secretsmanager.ErrCodeResourceExistsException {
+		} else if !ok || aerr.Code() != secretsmanager.ErrCodeResourceExistsException {
 			return errors.Wrapf(err, "autocert failed to create secret %s", secretID)
 		}
-		}
 
 		// If where was a resource exists error for create, then need to update the secret instead.
 		_, err = svc.UpdateSecret(&secretsmanager.UpdateSecretInput{
@@ -103,6 +114,13 @@ func (c *SecretManagerAutocertCache) Put(ctx context.Context, key string, data [
 		log.Printf("AWS Secrets Manager : Secret %s created", secretID)
 	}
 
+	if c.cache != nil {
+		err = c.cache.Put(ctx, key, data)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+	}
+
 	return nil
 }
 
@@ -145,7 +163,14 @@ func (c *SecretManagerAutocertCache) Delete(ctx context.Context, key string) err
 		return errors.Wrapf(err, "autocert failed to delete secret %s", secretID)
 	}
 
-	log.Printf("AWS Secrets Manager : Secret %s deleted for %s", secretID)
+	log.Printf("AWS Secrets Manager : Secret %s deleted", secretID)
+
+	if c.cache != nil {
+		err = c.cache.Delete(ctx, key)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+	}
 
 	return nil
 }

example-project/internal/platform/devops/config_sync.go

@@ -41,7 +41,6 @@ func SyncCfgInit(log *log.Logger, awsSession *session.Session, secretPrefix, wat
 		// Init the watch to wait for sync local files to Secret Manager.
 		WatchCfgDir(log, awsSession, secretPrefix, watchDir, watcher, localfiles)
 
-
 		// Init ticker to sync remote files from Secret Manager locally at the defined interval.
 		if syncInterval.Seconds() > 0 {
 			ticker := time.NewTicker(syncInterval)

example-project/internal/platform/devops/ecs_service.go

@@ -3,9 +3,6 @@ package devops
 import (
 	"encoding/base64"
 	"encoding/json"
-	"fmt"
-	"github.com/sethgrid/pester"
-	"io/ioutil"
 	"log"
 	"os"
 	"strconv"
@@ -33,15 +30,6 @@ func EcsServiceTaskInit(log *log.Logger, awsSession *session.Session) error {
 		return nil
 	}
 
-	res, err := pester.Get("http://169.254.170.2/v2/metadata")
-	if err != nil {
-		fmt.Println("http://169.254.170.2/v2/metadata failed", err.Error())
-	} else {
-		dat, _ := ioutil.ReadAll(res.Body)
-		res.Body.Close()
-		fmt.Println("http://169.254.170.2/v2/metadata, OK", string(dat))
-	}
-
 	var zoneArecNames = map[string][]string{}
 	if v := os.Getenv("ROUTE53_ZONES"); v != "" {
 		dat, err := base64.RawURLEncoding.DecodeString(v)
@@ -241,6 +229,15 @@ func RegisterEcsServiceTasksRoute53(log *log.Logger, awsSession *session.Session
 }
 
 /*
+res, err := pester.Get("http://169.254.170.2/v2/metadata")
+if err != nil {
+	fmt.Println("http://169.254.170.2/v2/metadata failed", err.Error())
+} else {
+	dat, _ := ioutil.ReadAll(res.Body)
+	res.Body.Close()
+	fmt.Println("http://169.254.170.2/v2/metadata, OK", string(dat))
+}
+
 http://169.254.170.2/v2/metadata,
 
 {

example-project/tools/truss/cmd/devops/service_deploy.go

@@ -403,6 +403,8 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
 							"secretsmanager:GetSecretValue",
 							"secretsmanager:CreateSecret",
 							"secretsmanager:UpdateSecret",
+							"secretsmanager:RestoreSecret",
+							"secretsmanager:DeleteSecret",
 						},
 						Resource: "*",
 					},
@@ -2560,8 +2562,8 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 			"{ECS_SERVICE}":           req.EcsServiceName,
 			"{AWS_REGION}":            req.AwsCreds.Region,
 			"{AWS_LOGS_GROUP}":        req.CloudWatchLogGroupName,
-			"{AWS_AWS_S3_BUCKET_PRIVATE}": req.S3BucketPrivateName,
+			"{AWS_S3_BUCKET_PRIVATE}": req.S3BucketPrivateName,
-			"{S3_BUCKET_PUBLIC}": req.S3BucketPublicName,
+			"{AWS_S3_BUCKET_PUBLIC}":  req.S3BucketPublicName,
 			"{ENV}":                   req.Env,
 			"{DATADOG_APIKEY}":        datadogApiKey,
 			"{DATADOG_ESSENTIAL}":     "true",