add web-app to gitlab cicd

tools/devops/cmd/cicd/aws.go

@@ -3,6 +3,7 @@ package cicd
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"io/ioutil"
 	"net/url"
 	"path/filepath"
@@ -262,6 +263,18 @@ func EcrPurgeImages(req *serviceBuildRequest) ([]*ecr.ImageIdentifier, error) {
 	return delIds, nil
 }
 
+// SyncPublicS3Files copies the local files from the static directory to s3 with public-read enabled.
+func SyncPublicS3Files(awsSession *session.Session, staticS3Bucket, staticS3Prefix, staticDir string) error {
+	uploader := s3manager.NewUploader(awsSession)
+
+	di := NewDirectoryIterator(staticS3Bucket, staticS3Prefix, staticDir, "public-read")
+	if err := uploader.UploadWithIterator(aws.BackgroundContext(), di); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // EcsReadTaskDefinition reads a task definition file and json decodes it.
 func EcsReadTaskDefinition(serviceDir, targetEnv string) ([]byte, error) {
 	checkPaths := []string{

tools/devops/cmd/cicd/s3_batch_upload.go

@@ -0,0 +1,103 @@
+package cicd
+
+import (
+	"bytes"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/pkg/errors"
+)
+
+// DirectoryIterator represents an iterator of a specified directory
+type DirectoryIterator struct {
+	filePaths []string
+	bucket    string
+	keyPrefix string
+	acl       string
+	next      struct {
+		path string
+		f    *os.File
+	}
+	err error
+}
+
+// NewDirectoryIterator builds a new DirectoryIterator
+func NewDirectoryIterator(bucket, keyPrefix, dir, acl string) s3manager.BatchUploadIterator {
+
+	// The key prefix could end with the base directory name,
+	// If this is the case, drop the dirname from the key prefix
+	if keyPrefix != "" {
+		dirName := filepath.Base(dir)
+		keyPrefix = strings.TrimRight(keyPrefix, "/")
+		keyPrefix = strings.TrimRight(keyPrefix, dirName)
+	}
+
+	var paths []string
+	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if !info.IsDir() {
+			paths = append(paths, path)
+		}
+		return nil
+	})
+
+	return &DirectoryIterator{
+		filePaths: paths,
+		bucket:    bucket,
+		keyPrefix: keyPrefix,
+		acl:       acl,
+	}
+}
+
+// Next returns whether next file exists or not
+func (di *DirectoryIterator) Next() bool {
+	if len(di.filePaths) == 0 {
+		di.next.f = nil
+		return false
+	}
+
+	f, err := os.Open(di.filePaths[0])
+	di.err = err
+	di.next.f = f
+	di.next.path = di.filePaths[0]
+	di.filePaths = di.filePaths[1:]
+
+	return true && di.Err() == nil
+}
+
+// Err returns error of DirectoryIterator
+func (di *DirectoryIterator) Err() error {
+	return errors.WithStack(di.err)
+}
+
+// UploadObject uploads a file
+func (di *DirectoryIterator) UploadObject() s3manager.BatchUploadObject {
+	f := di.next.f
+
+	var acl *string
+	if di.acl != "" {
+		acl = aws.String(di.acl)
+	}
+
+	// Get file size and read the file content into a buffer
+	fileInfo, _ := f.Stat()
+	var size int64 = fileInfo.Size()
+	buffer := make([]byte, size)
+	f.Read(buffer)
+
+	return s3manager.BatchUploadObject{
+		Object: &s3manager.UploadInput{
+			Bucket:      aws.String(di.bucket),
+			Key:         aws.String(filepath.Join(di.keyPrefix, di.next.path)),
+			Body:        bytes.NewReader(buffer),
+			ContentType: aws.String(http.DetectContentType(buffer)),
+			ACL:         acl,
+		},
+		After: func() error {
+			return f.Close()
+		},
+	}
+}

tools/devops/cmd/cicd/service.go

@@ -142,19 +142,26 @@ func ecrRepositoryName(projectName string) string {
 
 // releaseImage returns the name used for tagging a release image will always include one with environment and
 // service name. If the env var CI_COMMIT_REF_NAME is set, it will be appended.
-func releaseImage(env, serviceName, repositoryUri string) string {
+func releaseTag(env, serviceName string) string {
 
 	tag1 := env + "-" + serviceName
 
 	// Generate tags for the release image.
-	var releaseImage string
+	var releaseTag string
 	if v := os.Getenv("CI_COMMIT_REF_NAME"); v != "" {
 		tag2 := tag1 + "-" + v
-		releaseImage = repositoryUri + ":" + tag2
+		releaseTag = tag2
 	} else {
-		releaseImage = repositoryUri + ":" + tag1
+		releaseTag = tag1
 	}
-	return releaseImage
+	return releaseTag
+}
+
+
+// releaseImage returns the name used for tagging a release image will always include one with environment and
+// service name. If the env var CI_COMMIT_REF_NAME is set, it will be appended.
+func releaseImage(env, serviceName, repositoryUri string) string {
+	return repositoryUri + ":" + releaseTag(env, serviceName)
 }
 
 // dBInstanceIdentifier returns the database name.

tools/devops/cmd/cicd/service_deploy.go

@@ -64,7 +64,12 @@ type ServiceDeployFlags struct {
 	DockerFile      string `validate:"omitempty" example:"./cmd/web-api/Dockerfile"`
 	EnableLambdaVPC bool   `validate:"omitempty" example:"false"`
 	EnableEcsElb    bool   `validate:"omitempty" example:"false"`
-	RecreateService bool   `validate:"omitempty" example:"false"`
+
+	StaticFilesS3Enable         bool `validate:"omitempty" example:"false"`
+	StaticFilesCloudfrontEnable bool `validate:"omitempty" example:"false"`
+	StaticFilesImgResizeEnable  bool `validate:"omitempty" example:"false"`
+
+	RecreateService bool `validate:"omitempty" example:"false"`
 }
 
 // serviceDeployRequest defines the details needed to execute a service deployment.
@@ -105,10 +110,16 @@ type serviceDeployRequest struct {
 	CloudWatchLogGroupName string `validate:"required"`
 	CloudWatchLogGroup     *cloudwatchlogs.CreateLogGroupInput
 
-	S3BucketTempPrefix  string `validate:"required_with=S3BucketPrivateName S3BucketPublicName"`
-	S3BucketPrivateName string `validate:"omitempty"`
-	S3BucketPublicName  string `validate:"omitempty"`
-	S3Buckets           []S3Bucket
+	S3BucketTempPrefix      string `validate:"required_with=S3BucketPrivateName S3BucketPublicName"`
+	S3BucketPrivateName     string `validate:"omitempty"`
+	S3BucketPublicName      string `validate:"omitempty"`
+	S3BucketPublicKeyPrefix string `validate:"omitempty"`
+	S3Buckets               []S3Bucket
+
+	StaticFilesS3Enable         bool   `validate:"omitempty"`
+	StaticFilesS3Prefix         string `validate:"omitempty"`
+	StaticFilesCloudfrontEnable bool   `validate:"omitempty"`
+	StaticFilesImgResizeEnable  bool   `validate:"omitempty"`
 
 	EnableEcsElb           bool   `validate:"omitempty"`
 	ElbLoadBalancerName    string `validate:"omitempty"`
@@ -169,9 +180,14 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
 		req = serviceDeployRequest{
 			serviceRequest: sr,
 
-			EnableHTTPS:         flags.EnableHTTPS,
-			ServiceHostPrimary:  flags.ServiceHostPrimary,
-			ServiceHostNames:    flags.ServiceHostNames,
+			EnableHTTPS:        flags.EnableHTTPS,
+			ServiceHostPrimary: flags.ServiceHostPrimary,
+			ServiceHostNames:   flags.ServiceHostNames,
+
+			StaticFilesS3Enable:         flags.StaticFilesS3Enable,
+			StaticFilesCloudfrontEnable: flags.StaticFilesCloudfrontEnable,
+			StaticFilesImgResizeEnable:  flags.StaticFilesImgResizeEnable,
+
 			S3BucketPrivateName: flags.S3BucketPrivateName,
 			S3BucketPublicName:  flags.S3BucketPublicName,
 			EnableLambdaVPC:     flags.EnableLambdaVPC,
@@ -337,6 +353,16 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
 					})
 			}
 
+			// The S3 key prefix used as the origin when cloud front is enabled.
+			if req.S3BucketPublicKeyPrefix == "" {
+				req.S3BucketPublicKeyPrefix = "public"
+			}
+
+			// The S3 prefix used to upload static files served to public.
+			if req.StaticFilesS3Prefix == "" {
+				req.StaticFilesS3Prefix = filepath.Join(req.S3BucketPublicKeyPrefix, releaseTag(req.Env, req.ServiceName), "static")
+			}
+
 			// Set default AWS ECR Repository Name.
 			req.EcrRepositoryName = ecrRepositoryName(req.ProjectName)
 			log.Printf("\t\t\tSet ECR Repository Name to '%s'.", req.EcrRepositoryName)
@@ -988,7 +1014,6 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 				log.Printf("\t\t\tUpdated bucket policy")
 			}
 		}
-
 		log.Printf("\t%s\tS3 buckets configured successfully.\n", tests.Success)
 	}
 
@@ -2307,6 +2332,11 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 			"{HOST_PRIMARY}": req.ServiceHostPrimary,
 			"{HOST_NAMES}":   strings.Join(req.ServiceHostNames, ","),
 
+			"{STATIC_FILES_S3_ENABLED}":         "false",
+			"{STATIC_FILES_S3_PREFIX}":          "",
+			"{STATIC_FILES_CLOUDFRONT_ENABLED}": "false",
+			"{STATIC_FILES_IMG_RESIZE_ENABLED}": "false",
+
 			"{CACHE_HOST}": "", // Not enabled by default
 
 			"{DB_HOST}":        "",
@@ -2359,6 +2389,21 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 			placeholders["{APP_BASE_URL}"] = fmt.Sprintf("%s://%s/", appSchema, req.ServiceHostPrimary)
 		}
 
+		// Static files served from S3.
+		if req.StaticFilesS3Enable {
+			placeholders["{STATIC_FILES_S3_ENABLED}"] = "true"
+		}
+
+		// Static files served from CloudFront.
+		if req.StaticFilesCloudfrontEnable {
+			placeholders["{STATIC_FILES_CLOUDFRONT_ENABLED}"] = "true"
+		}
+
+		// Support for resizing static images files to be responsive.
+		if req.StaticFilesImgResizeEnable {
+			placeholders["{STATIC_FILES_IMG_RESIZE_ENABLED}"] = "true"
+		}
+
 		// When db is set, update the placeholders.
 		if db != nil {
 			placeholders["{DB_HOST}"] = db.Host
@@ -3157,6 +3202,20 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 		}
 	}
 
+	// When static files are enabled to be to stored on S3, we need to upload all of them.
+	if req.StaticFilesS3Enable {
+		log.Println("\tSync static files to public S3 bucket")
+
+		staticDir := filepath.Join(req.ServiceDir, "static")
+
+		err := SyncPublicS3Files(req.awsSession(), req.S3BucketPublicName, req.StaticFilesS3Prefix, staticDir)
+		if err != nil {
+			return errors.Wrapf(err, "Failed to sync static files from %s to s3://%s/%s '%s'", staticDir, req.S3BucketPublicName, req.StaticFilesS3Prefix)
+		}
+
+		log.Printf("\t%s\tFiles uploaded.\n", tests.Success)
+	}
+
 	// Wait for the updated or created service to enter a stable state.
 	{
 		log.Println("\tWaiting for service to enter stable state.")