add web-app to gitlab cicd

.gitlab-ci.yml

@@ -34,7 +34,7 @@ cache:
 .deploy_tmpl: &deploy_tmpl
   <<: *job_tmpl
   script:
-    - 'devops deploy -service=${SERVICE} -project=${PROJECT_NAME} -env=${TARGET_ENV} -enable_https=${ENABLE_HTTPS} -enable_elb=${ENABLE_ELB} -primary_host=${PRIMARY_HOST} -host_names=${HOST_NAMES} -private_bucket=${S3_BUCKET_PRIVATE} -public_bucket=${S3_BUCKET_PUBLIC}'
+    - 'devops deploy -service=${SERVICE} -project=${PROJECT_NAME} -env=${TARGET_ENV} -enable_https=${ENABLE_HTTPS} -enable_elb=${ENABLE_ELB} -primary_host=${PRIMARY_HOST} -host_names=${HOST_NAMES} -private_bucket=${S3_BUCKET_PRIVATE} -public_bucket=${S3_BUCKET_PUBLIC} -static_files_s3={STATIC_FILES_S3_ENABLED} -static_files_cloudfront={STATIC_FILES_CLOUDFRONT_ENABLED} -static_files_img_resize={STATIC_FILES_IMG_RESIZE_ENABLED}'
 
 .migrate_tmpl: &migrate_tmpl
   <<: *job_tmpl
@@ -54,6 +54,45 @@ db:migrate:dev:
     TARGET_ENV: 'dev'
     AWS_USE_ROLE: 'true'
 
+webapp:build:dev:
+  <<: *build_tmpl
+  stage: build:dev
+  tags:
+    - dev
+  only:
+    - master
+    - dev
+    - dev-web-app
+  variables:
+    TARGET_ENV: 'dev'
+    SERVICE: 'web-app'
+    AWS_USE_ROLE: 'true'
+webapp:deploy:dev:
+  <<: *deploy_tmpl
+  stage: deploy:dev
+  tags:
+    - dev
+  only:
+    - master
+    - dev
+    - dev-web-app
+  dependencies:
+    - 'webapp:build:dev'
+    - 'db:migrate:dev'
+  variables:
+    TARGET_ENV: 'dev'
+    SERVICE: 'web-app'
+    ENABLE_HTTPS: 1
+    ENABLE_ELB: 0
+    PRIMARY_HOST: 'eproc.tech'
+    HOST_NAMES: 'www.eproc.tech, dev.eproc.tech'
+    S3_BUCKET_PRIVATE: 'saas-starter-kit-private'
+    S3_BUCKET_PUBLIC: 'saas-starter-kit-public'
+    STATIC_FILES_S3_ENABLED: 'true'
+    STATIC_FILES_CLOUDFRONT_ENABLED: 'false'
+    STATIC_FILES_IMG_RESIZE_ENABLED: 'true'
+    AWS_USE_ROLE: 'true'
+
 webapi:build:dev:
   <<: *build_tmpl
   stage: build:dev
@@ -84,14 +123,15 @@ webapi:deploy:dev:
     SERVICE: 'web-api'
     ENABLE_HTTPS: 1
     ENABLE_ELB: 0
-    PRIMARY_HOST: 'eproc.tech'
+    PRIMARY_HOST: 'api.eproc.tech'
-    HOST_NAMES: 'www.eproc.tech, api.eproc.tech'
+    HOST_NAMES: 'api.dev.eproc.tech'
     S3_BUCKET_PRIVATE: 'saas-starter-kit-private'
     S3_BUCKET_PUBLIC: 'saas-starter-kit-public'
+    STATIC_FILES_S3_ENABLED: 'false'
+    STATIC_FILES_CLOUDFRONT_ENABLED: 'false'
+    STATIC_FILES_IMG_RESIZE_ENABLED: 'false'
     AWS_USE_ROLE: 'true'
 
-
-
 #ddlogscollector:deploy:stage:
 #  <<: *deploy_stage_tmpl
 #  variables:

cmd/web-app/Dockerfile

@@ -38,6 +38,7 @@ COPY --from=builder /static /static
 COPY --from=builder /templates /templates
 
 ENV TEMPLATE_DIR=/templates
+ENV STATIC_DIR=/static
 
 ARG service
 ENV SERVICE_NAME $service

cmd/web-app/main.go

@@ -20,8 +20,8 @@ import (
 	"syscall"
 	"time"
 
-	"geeks-accelerator/oss/saas-starter-kit/internal/mid"
 	"geeks-accelerator/oss/saas-starter-kit/cmd/web-app/handlers"
+	"geeks-accelerator/oss/saas-starter-kit/internal/mid"
 	"geeks-accelerator/oss/saas-starter-kit/internal/platform/devops"
 	"geeks-accelerator/oss/saas-starter-kit/internal/platform/flag"
 	img_resize "geeks-accelerator/oss/saas-starter-kit/internal/platform/img-resize"
@@ -80,10 +80,10 @@ func main() {
 			HostNames   []string `envconfig:"HOST_NAMES" example:"www.eproc.tech"`
 			EnableHTTPS bool     `default:"false" envconfig:"ENABLE_HTTPS"`
 			TemplateDir string   `default:"./templates" envconfig:"TEMPLATE_DIR"`
-			StaticDir   string   `default:"./static" envconfig:"STATIC_DIR"`
+			StaticFiles struct {
-			StaticS3    struct {
+				Dir               string `default:"./static" envconfig:"STATIC_DIR"`
-				S3Enabled         bool   `envconfig:"ENABLED"`
+				S3Enabled         bool   `envconfig:"S3_ENABLED"`
-				S3KeyPrefix       string `default:"public/web_app/static" envconfig:"KEY_PREFIX"`
+				S3Prefix          string `default:"public/web_app/static" envconfig:"S3_PREFIX"`
 				CloudFrontEnabled bool   `envconfig:"CLOUDFRONT_ENABLED"`
 				ImgResizeEnabled  bool   `envconfig:"IMG_RESIZE_ENABLED"`
 			}
@@ -369,8 +369,8 @@ func main() {
 	// s3UrlFormatter is a help function used by to convert an s3 key to
 	// a publicly available image URL.
 	var staticS3UrlFormatter func(string) string
-	if cfg.Service.StaticS3.S3Enabled || cfg.Service.StaticS3.CloudFrontEnabled || cfg.Service.StaticS3.ImgResizeEnabled {
+	if cfg.Service.StaticFiles.S3Enabled || cfg.Service.StaticFiles.CloudFrontEnabled || cfg.Service.StaticFiles.ImgResizeEnabled {
-		s3UrlFormatter, err := devops.S3UrlFormatter(awsSession, cfg.Aws.S3BucketPublic, cfg.Service.StaticS3.S3KeyPrefix, cfg.Service.StaticS3.CloudFrontEnabled)
+		s3UrlFormatter, err := devops.S3UrlFormatter(awsSession, cfg.Aws.S3BucketPublic, cfg.Service.StaticFiles.S3Prefix, cfg.Service.StaticFiles.CloudFrontEnabled)
 		if err != nil {
 			log.Fatalf("main : S3UrlFormatter failed : %+v", err)
 		}
@@ -379,7 +379,7 @@ func main() {
 			// When the path starts with a forward slash its referencing a local file,
 			// make sure the static file prefix is included
 			if strings.HasPrefix(p, "/") {
-				p = filepath.Join(cfg.Service.StaticS3.S3KeyPrefix, p)
+				p = filepath.Join(cfg.Service.StaticFiles.S3Prefix, p)
 			}
 			return s3UrlFormatter(p)
 		}
@@ -400,7 +400,7 @@ func main() {
 	// templates should be updated to use a fully qualified URL for either the public file on S3
 	// on from the cloudfront distribution.
 	var staticUrlFormatter func(string) string
-	if cfg.Service.StaticS3.S3Enabled || cfg.Service.StaticS3.CloudFrontEnabled {
+	if cfg.Service.StaticFiles.S3Enabled || cfg.Service.StaticFiles.CloudFrontEnabled {
 		staticUrlFormatter = staticS3UrlFormatter
 	} else {
 		baseUrl, err := url.Parse(cfg.Service.BaseUrl)
@@ -508,12 +508,12 @@ func main() {
 
 	// Image Formatter - additional functions exposed to templates for resizing images
 	// to support response web applications.
-	imgResizeS3KeyPrefix := filepath.Join(cfg.Service.StaticS3.S3KeyPrefix, "images/responsive")
+	imgResizeS3KeyPrefix := filepath.Join(cfg.Service.StaticFiles.S3Prefix, "images/responsive")
 
 	imgSrcAttr := func(ctx context.Context, p string, sizes []int, includeOrig bool) template.HTMLAttr {
 		u := staticUrlFormatter(p)
 		var srcAttr string
-		if cfg.Service.StaticS3.ImgResizeEnabled {
+		if cfg.Service.StaticFiles.ImgResizeEnabled {
 			srcAttr, _ = img_resize.S3ImgSrc(ctx, redisClient, staticS3UrlFormatter, awsSession, cfg.Aws.S3BucketPublic, imgResizeS3KeyPrefix, u, sizes, includeOrig)
 		} else {
 			srcAttr = fmt.Sprintf("src=\"%s\"", u)
@@ -544,7 +544,7 @@ func main() {
 	}
 	tmplFuncs["S3ImgUrl"] = func(ctx context.Context, p string, size int) string {
 		imgUrl := staticUrlFormatter(p)
-		if cfg.Service.StaticS3.ImgResizeEnabled {
+		if cfg.Service.StaticFiles.ImgResizeEnabled {
 			imgUrl, _ = img_resize.S3ImgUrl(ctx, redisClient, staticS3UrlFormatter, awsSession, cfg.Aws.S3BucketPublic, imgResizeS3KeyPrefix, imgUrl, size)
 		}
 		return imgUrl
@@ -635,7 +635,7 @@ func main() {
 	if cfg.HTTP.Host != "" {
 		api := http.Server{
 			Addr:           cfg.HTTP.Host,
-			Handler:        handlers.APP(shutdown, log, cfg.Service.StaticDir, cfg.Service.TemplateDir, masterDb, redisClient, renderer, serviceMiddlewares...),
+			Handler:        handlers.APP(shutdown, log, cfg.Service.StaticFiles.Dir, cfg.Service.TemplateDir, masterDb, redisClient, renderer, serviceMiddlewares...),
 			ReadTimeout:    cfg.HTTP.ReadTimeout,
 			WriteTimeout:   cfg.HTTP.WriteTimeout,
 			MaxHeaderBytes: 1 << 20,
@@ -652,7 +652,7 @@ func main() {
 	if cfg.HTTPS.Host != "" {
 		api := http.Server{
 			Addr:           cfg.HTTPS.Host,
-			Handler:        handlers.APP(shutdown, log, cfg.Service.StaticDir, cfg.Service.TemplateDir, masterDb, redisClient, renderer, serviceMiddlewares...),
+			Handler:        handlers.APP(shutdown, log, cfg.Service.StaticFiles.Dir, cfg.Service.TemplateDir, masterDb, redisClient, renderer, serviceMiddlewares...),
 			ReadTimeout:    cfg.HTTPS.ReadTimeout,
 			WriteTimeout:   cfg.HTTPS.WriteTimeout,
 			MaxHeaderBytes: 1 << 20,

internal/platform/devops/deploy.go

@@ -1,20 +0,0 @@
-package devops
-
-import (
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
-)
-
-// SyncS3StaticFiles copies the local files from the static directory to s3
-// with public-read enabled.
-func SyncS3StaticFiles(awsSession *session.Session, staticS3Bucket, staticS3Prefix, staticDir string) error {
-	uploader := s3manager.NewUploader(awsSession)
-
-	di := NewDirectoryIterator(staticS3Bucket, staticS3Prefix, staticDir, "public-read")
-	if err := uploader.UploadWithIterator(aws.BackgroundContext(), di); err != nil {
-		return err
-	}
-
-	return nil
-}

tools/devops/cmd/cicd/aws.go

@@ -3,6 +3,7 @@ package cicd
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"io/ioutil"
 	"net/url"
 	"path/filepath"
@@ -262,6 +263,18 @@ func EcrPurgeImages(req *serviceBuildRequest) ([]*ecr.ImageIdentifier, error) {
 	return delIds, nil
 }
 
+// SyncPublicS3Files copies the local files from the static directory to s3 with public-read enabled.
+func SyncPublicS3Files(awsSession *session.Session, staticS3Bucket, staticS3Prefix, staticDir string) error {
+	uploader := s3manager.NewUploader(awsSession)
+
+	di := NewDirectoryIterator(staticS3Bucket, staticS3Prefix, staticDir, "public-read")
+	if err := uploader.UploadWithIterator(aws.BackgroundContext(), di); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // EcsReadTaskDefinition reads a task definition file and json decodes it.
 func EcsReadTaskDefinition(serviceDir, targetEnv string) ([]byte, error) {
 	checkPaths := []string{

tools/devops/cmd/cicd/s3_batch_upload.go

@@ -1,4 +1,4 @@
-package devops
+package cicd
 
 import (
 	"bytes"

tools/devops/cmd/cicd/service.go

@@ -142,19 +142,26 @@ func ecrRepositoryName(projectName string) string {
 
 // releaseImage returns the name used for tagging a release image will always include one with environment and
 // service name. If the env var CI_COMMIT_REF_NAME is set, it will be appended.
-func releaseImage(env, serviceName, repositoryUri string) string {
+func releaseTag(env, serviceName string) string {
 
 	tag1 := env + "-" + serviceName
 
 	// Generate tags for the release image.
-	var releaseImage string
+	var releaseTag string
 	if v := os.Getenv("CI_COMMIT_REF_NAME"); v != "" {
 		tag2 := tag1 + "-" + v
-		releaseImage = repositoryUri + ":" + tag2
+		releaseTag = tag2
 	} else {
-		releaseImage = repositoryUri + ":" + tag1
+		releaseTag = tag1
 	}
-	return releaseImage
+	return releaseTag
+}
+
+
+// releaseImage returns the name used for tagging a release image will always include one with environment and
+// service name. If the env var CI_COMMIT_REF_NAME is set, it will be appended.
+func releaseImage(env, serviceName, repositoryUri string) string {
+	return repositoryUri + ":" + releaseTag(env, serviceName)
 }
 
 // dBInstanceIdentifier returns the database name.

tools/devops/cmd/cicd/service_deploy.go

@@ -64,7 +64,12 @@ type ServiceDeployFlags struct {
 	DockerFile      string `validate:"omitempty" example:"./cmd/web-api/Dockerfile"`
 	EnableLambdaVPC bool   `validate:"omitempty" example:"false"`
 	EnableEcsElb    bool   `validate:"omitempty" example:"false"`
-	RecreateService bool   `validate:"omitempty" example:"false"`
+
+	StaticFilesS3Enable         bool `validate:"omitempty" example:"false"`
+	StaticFilesCloudfrontEnable bool `validate:"omitempty" example:"false"`
+	StaticFilesImgResizeEnable  bool `validate:"omitempty" example:"false"`
+
+	RecreateService bool `validate:"omitempty" example:"false"`
 }
 
 // serviceDeployRequest defines the details needed to execute a service deployment.
@@ -105,10 +110,16 @@ type serviceDeployRequest struct {
 	CloudWatchLogGroupName string `validate:"required"`
 	CloudWatchLogGroup     *cloudwatchlogs.CreateLogGroupInput
 
-	S3BucketTempPrefix  string `validate:"required_with=S3BucketPrivateName S3BucketPublicName"`
+	S3BucketTempPrefix      string `validate:"required_with=S3BucketPrivateName S3BucketPublicName"`
-	S3BucketPrivateName string `validate:"omitempty"`
+	S3BucketPrivateName     string `validate:"omitempty"`
-	S3BucketPublicName  string `validate:"omitempty"`
+	S3BucketPublicName      string `validate:"omitempty"`
-	S3Buckets           []S3Bucket
+	S3BucketPublicKeyPrefix string `validate:"omitempty"`
+	S3Buckets               []S3Bucket
+
+	StaticFilesS3Enable         bool   `validate:"omitempty"`
+	StaticFilesS3Prefix         string `validate:"omitempty"`
+	StaticFilesCloudfrontEnable bool   `validate:"omitempty"`
+	StaticFilesImgResizeEnable  bool   `validate:"omitempty"`
 
 	EnableEcsElb           bool   `validate:"omitempty"`
 	ElbLoadBalancerName    string `validate:"omitempty"`
@@ -169,9 +180,14 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
 		req = serviceDeployRequest{
 			serviceRequest: sr,
 
-			EnableHTTPS:         flags.EnableHTTPS,
+			EnableHTTPS:        flags.EnableHTTPS,
-			ServiceHostPrimary:  flags.ServiceHostPrimary,
+			ServiceHostPrimary: flags.ServiceHostPrimary,
-			ServiceHostNames:    flags.ServiceHostNames,
+			ServiceHostNames:   flags.ServiceHostNames,
+
+			StaticFilesS3Enable:         flags.StaticFilesS3Enable,
+			StaticFilesCloudfrontEnable: flags.StaticFilesCloudfrontEnable,
+			StaticFilesImgResizeEnable:  flags.StaticFilesImgResizeEnable,
+
 			S3BucketPrivateName: flags.S3BucketPrivateName,
 			S3BucketPublicName:  flags.S3BucketPublicName,
 			EnableLambdaVPC:     flags.EnableLambdaVPC,
@@ -337,6 +353,16 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
 					})
 			}
 
+			// The S3 key prefix used as the origin when cloud front is enabled.
+			if req.S3BucketPublicKeyPrefix == "" {
+				req.S3BucketPublicKeyPrefix = "public"
+			}
+
+			// The S3 prefix used to upload static files served to public.
+			if req.StaticFilesS3Prefix == "" {
+				req.StaticFilesS3Prefix = filepath.Join(req.S3BucketPublicKeyPrefix, releaseTag(req.Env, req.ServiceName), "static")
+			}
+
 			// Set default AWS ECR Repository Name.
 			req.EcrRepositoryName = ecrRepositoryName(req.ProjectName)
 			log.Printf("\t\t\tSet ECR Repository Name to '%s'.", req.EcrRepositoryName)
@@ -988,7 +1014,6 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 				log.Printf("\t\t\tUpdated bucket policy")
 			}
 		}
-
 		log.Printf("\t%s\tS3 buckets configured successfully.\n", tests.Success)
 	}
 
@@ -2307,6 +2332,11 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 			"{HOST_PRIMARY}": req.ServiceHostPrimary,
 			"{HOST_NAMES}":   strings.Join(req.ServiceHostNames, ","),
 
+			"{STATIC_FILES_S3_ENABLED}":         "false",
+			"{STATIC_FILES_S3_PREFIX}":          "",
+			"{STATIC_FILES_CLOUDFRONT_ENABLED}": "false",
+			"{STATIC_FILES_IMG_RESIZE_ENABLED}": "false",
+
 			"{CACHE_HOST}": "", // Not enabled by default
 
 			"{DB_HOST}":        "",
@@ -2359,6 +2389,21 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 			placeholders["{APP_BASE_URL}"] = fmt.Sprintf("%s://%s/", appSchema, req.ServiceHostPrimary)
 		}
 
+		// Static files served from S3.
+		if req.StaticFilesS3Enable {
+			placeholders["{STATIC_FILES_S3_ENABLED}"] = "true"
+		}
+
+		// Static files served from CloudFront.
+		if req.StaticFilesCloudfrontEnable {
+			placeholders["{STATIC_FILES_CLOUDFRONT_ENABLED}"] = "true"
+		}
+
+		// Support for resizing static images files to be responsive.
+		if req.StaticFilesImgResizeEnable {
+			placeholders["{STATIC_FILES_IMG_RESIZE_ENABLED}"] = "true"
+		}
+
 		// When db is set, update the placeholders.
 		if db != nil {
 			placeholders["{DB_HOST}"] = db.Host
@@ -3157,6 +3202,20 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 		}
 	}
 
+	// When static files are enabled to be to stored on S3, we need to upload all of them.
+	if req.StaticFilesS3Enable {
+		log.Println("\tSync static files to public S3 bucket")
+
+		staticDir := filepath.Join(req.ServiceDir, "static")
+
+		err := SyncPublicS3Files(req.awsSession(), req.S3BucketPublicName, req.StaticFilesS3Prefix, staticDir)
+		if err != nil {
+			return errors.Wrapf(err, "Failed to sync static files from %s to s3://%s/%s '%s'", staticDir, req.S3BucketPublicName, req.StaticFilesS3Prefix)
+		}
+
+		log.Printf("\t%s\tFiles uploaded.\n", tests.Success)
+	}
+
 	// Wait for the updated or created service to enter a stable state.
 	{
 		log.Println("\tWaiting for service to enter stable state.")

tools/devops/main.go

@@ -82,6 +82,9 @@ func main() {
 				cli.StringFlag{Name: "project", Usage: "name of project", Destination: &deployFlags.ProjectName},
 				cli.BoolFlag{Name: "enable_elb", Usage: "enable deployed to use Elastic Load Balancer", Destination: &deployFlags.EnableEcsElb},
 				cli.BoolTFlag{Name: "lambda_vpc", Usage: "deploy lambda behind VPC", Destination: &deployFlags.EnableLambdaVPC},
+				cli.BoolFlag{Name: "static_files_s3", Usage: "service static files from S3", Destination: &deployFlags.StaticFilesS3Enable},
+				cli.BoolFlag{Name: "static_files_cloudfront", Usage: "serve static files from Cloudfront", Destination: &deployFlags.StaticFilesCloudfrontEnable},
+				cli.BoolFlag{Name: "static_files_img_resize", Usage: "enable response images from service", Destination: &deployFlags.StaticFilesImgResizeEnable},
 				cli.BoolFlag{Name: "recreate_service", Usage: "skip docker push after build", Destination: &deployFlags.RecreateService},
 			},
 			Action: func(c *cli.Context) error {