add web-app to gitlab cicd

2025-08-08 22:36:41 +02:00 · 2019-07-14 19:13:09 -08:00
parent 2cd374a130
commit df55acf654
9 changed files with 157 additions and 54 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -34,7 +34,7 @@ cache:
 .deploy_tmpl: &deploy_tmpl
  <<: *job_tmpl
  script:
-    - 'devops deploy -service=${SERVICE} -project=${PROJECT_NAME} -env=${TARGET_ENV} -enable_https=${ENABLE_HTTPS} -enable_elb=${ENABLE_ELB} -primary_host=${PRIMARY_HOST} -host_names=${HOST_NAMES} -private_bucket=${S3_BUCKET_PRIVATE} -public_bucket=${S3_BUCKET_PUBLIC}'
+    - 'devops deploy -service=${SERVICE} -project=${PROJECT_NAME} -env=${TARGET_ENV} -enable_https=${ENABLE_HTTPS} -enable_elb=${ENABLE_ELB} -primary_host=${PRIMARY_HOST} -host_names=${HOST_NAMES} -private_bucket=${S3_BUCKET_PRIVATE} -public_bucket=${S3_BUCKET_PUBLIC} -static_files_s3={STATIC_FILES_S3_ENABLED} -static_files_cloudfront={STATIC_FILES_CLOUDFRONT_ENABLED} -static_files_img_resize={STATIC_FILES_IMG_RESIZE_ENABLED}'

 .migrate_tmpl: &migrate_tmpl
  <<: *job_tmpl
@ -54,6 +54,45 @@ db:migrate:dev:
    TARGET_ENV: 'dev'
    AWS_USE_ROLE: 'true'

+webapp:build:dev:
+  <<: *build_tmpl
+  stage: build:dev
+  tags:
+    - dev
+  only:
+    - master
+    - dev
+    - dev-web-app
+  variables:
+    TARGET_ENV: 'dev'
+    SERVICE: 'web-app'
+    AWS_USE_ROLE: 'true'
+webapp:deploy:dev:
+  <<: *deploy_tmpl
+  stage: deploy:dev
+  tags:
+    - dev
+  only:
+    - master
+    - dev
+    - dev-web-app
+  dependencies:
+    - 'webapp:build:dev'
+    - 'db:migrate:dev'
+  variables:
+    TARGET_ENV: 'dev'
+    SERVICE: 'web-app'
+    ENABLE_HTTPS: 1
+    ENABLE_ELB: 0
+    PRIMARY_HOST: 'eproc.tech'
+    HOST_NAMES: 'www.eproc.tech, dev.eproc.tech'
+    S3_BUCKET_PRIVATE: 'saas-starter-kit-private'
+    S3_BUCKET_PUBLIC: 'saas-starter-kit-public'
+    STATIC_FILES_S3_ENABLED: 'true'
+    STATIC_FILES_CLOUDFRONT_ENABLED: 'false'
+    STATIC_FILES_IMG_RESIZE_ENABLED: 'true'
+    AWS_USE_ROLE: 'true'
+
 webapi:build:dev:
  <<: *build_tmpl
  stage: build:dev
@ -84,14 +123,15 @@ webapi:deploy:dev:
    SERVICE: 'web-api'
    ENABLE_HTTPS: 1
    ENABLE_ELB: 0
-    PRIMARY_HOST: 'eproc.tech'
-    HOST_NAMES: 'www.eproc.tech, api.eproc.tech'
+    PRIMARY_HOST: 'api.eproc.tech'
+    HOST_NAMES: 'api.dev.eproc.tech'
    S3_BUCKET_PRIVATE: 'saas-starter-kit-private'
    S3_BUCKET_PUBLIC: 'saas-starter-kit-public'
+    STATIC_FILES_S3_ENABLED: 'false'
+    STATIC_FILES_CLOUDFRONT_ENABLED: 'false'
+    STATIC_FILES_IMG_RESIZE_ENABLED: 'false'
    AWS_USE_ROLE: 'true'

-
-
 #ddlogscollector:deploy:stage:
 #  <<: *deploy_stage_tmpl
 #  variables:
--- a/cmd/web-app/Dockerfile
+++ b/cmd/web-app/Dockerfile
@ -38,6 +38,7 @@ COPY --from=builder /static /static
 COPY --from=builder /templates /templates

 ENV TEMPLATE_DIR=/templates
+ENV STATIC_DIR=/static

 ARG service
 ENV SERVICE_NAME $service
--- a/cmd/web-app/main.go
+++ b/cmd/web-app/main.go
@ -20,8 +20,8 @@ import (
 	"syscall"
 	"time"

-	"geeks-accelerator/oss/saas-starter-kit/internal/mid"
 	"geeks-accelerator/oss/saas-starter-kit/cmd/web-app/handlers"
+	"geeks-accelerator/oss/saas-starter-kit/internal/mid"
 	"geeks-accelerator/oss/saas-starter-kit/internal/platform/devops"
 	"geeks-accelerator/oss/saas-starter-kit/internal/platform/flag"
 	img_resize "geeks-accelerator/oss/saas-starter-kit/internal/platform/img-resize"
@ -80,10 +80,10 @@ func main() {
 			HostNames   []string `envconfig:"HOST_NAMES" example:"www.eproc.tech"`
 			EnableHTTPS bool     `default:"false" envconfig:"ENABLE_HTTPS"`
 			TemplateDir string   `default:"./templates" envconfig:"TEMPLATE_DIR"`
-			StaticDir   string   `default:"./static" envconfig:"STATIC_DIR"`
-			StaticS3    struct {
-				S3Enabled         bool   `envconfig:"ENABLED"`
-				S3KeyPrefix       string `default:"public/web_app/static" envconfig:"KEY_PREFIX"`
+			StaticFiles struct {
+				Dir               string `default:"./static" envconfig:"STATIC_DIR"`
+				S3Enabled         bool   `envconfig:"S3_ENABLED"`
+				S3Prefix          string `default:"public/web_app/static" envconfig:"S3_PREFIX"`
 				CloudFrontEnabled bool   `envconfig:"CLOUDFRONT_ENABLED"`
 				ImgResizeEnabled  bool   `envconfig:"IMG_RESIZE_ENABLED"`
 			}
@ -369,8 +369,8 @@ func main() {
 	// s3UrlFormatter is a help function used by to convert an s3 key to
 	// a publicly available image URL.
 	var staticS3UrlFormatter func(string) string
-	if cfg.Service.StaticS3.S3Enabled || cfg.Service.StaticS3.CloudFrontEnabled || cfg.Service.StaticS3.ImgResizeEnabled {
-		s3UrlFormatter, err := devops.S3UrlFormatter(awsSession, cfg.Aws.S3BucketPublic, cfg.Service.StaticS3.S3KeyPrefix, cfg.Service.StaticS3.CloudFrontEnabled)
+	if cfg.Service.StaticFiles.S3Enabled || cfg.Service.StaticFiles.CloudFrontEnabled || cfg.Service.StaticFiles.ImgResizeEnabled {
+		s3UrlFormatter, err := devops.S3UrlFormatter(awsSession, cfg.Aws.S3BucketPublic, cfg.Service.StaticFiles.S3Prefix, cfg.Service.StaticFiles.CloudFrontEnabled)
 		if err != nil {
 			log.Fatalf("main : S3UrlFormatter failed : %+v", err)
 		}
@ -379,7 +379,7 @@ func main() {
 			// When the path starts with a forward slash its referencing a local file,
 			// make sure the static file prefix is included
 			if strings.HasPrefix(p, "/") {
-				p = filepath.Join(cfg.Service.StaticS3.S3KeyPrefix, p)
+				p = filepath.Join(cfg.Service.StaticFiles.S3Prefix, p)
 			}
 			return s3UrlFormatter(p)
 		}
@ -400,7 +400,7 @@ func main() {
 	// templates should be updated to use a fully qualified URL for either the public file on S3
 	// on from the cloudfront distribution.
 	var staticUrlFormatter func(string) string
-	if cfg.Service.StaticS3.S3Enabled || cfg.Service.StaticS3.CloudFrontEnabled {
+	if cfg.Service.StaticFiles.S3Enabled || cfg.Service.StaticFiles.CloudFrontEnabled {
 		staticUrlFormatter = staticS3UrlFormatter
 	} else {
 		baseUrl, err := url.Parse(cfg.Service.BaseUrl)
@ -508,12 +508,12 @@ func main() {

 	// Image Formatter - additional functions exposed to templates for resizing images
 	// to support response web applications.
-	imgResizeS3KeyPrefix := filepath.Join(cfg.Service.StaticS3.S3KeyPrefix, "images/responsive")
+	imgResizeS3KeyPrefix := filepath.Join(cfg.Service.StaticFiles.S3Prefix, "images/responsive")

 	imgSrcAttr := func(ctx context.Context, p string, sizes []int, includeOrig bool) template.HTMLAttr {
 		u := staticUrlFormatter(p)
 		var srcAttr string
-		if cfg.Service.StaticS3.ImgResizeEnabled {
+		if cfg.Service.StaticFiles.ImgResizeEnabled {
 			srcAttr, _ = img_resize.S3ImgSrc(ctx, redisClient, staticS3UrlFormatter, awsSession, cfg.Aws.S3BucketPublic, imgResizeS3KeyPrefix, u, sizes, includeOrig)
 		} else {
 			srcAttr = fmt.Sprintf("src=\"%s\"", u)
@ -544,7 +544,7 @@ func main() {
 	}
 	tmplFuncs["S3ImgUrl"] = func(ctx context.Context, p string, size int) string {
 		imgUrl := staticUrlFormatter(p)
-		if cfg.Service.StaticS3.ImgResizeEnabled {
+		if cfg.Service.StaticFiles.ImgResizeEnabled {
 			imgUrl, _ = img_resize.S3ImgUrl(ctx, redisClient, staticS3UrlFormatter, awsSession, cfg.Aws.S3BucketPublic, imgResizeS3KeyPrefix, imgUrl, size)
 		}
 		return imgUrl
@ -635,7 +635,7 @@ func main() {
 	if cfg.HTTP.Host != "" {
 		api := http.Server{
 			Addr:           cfg.HTTP.Host,
-			Handler:        handlers.APP(shutdown, log, cfg.Service.StaticDir, cfg.Service.TemplateDir, masterDb, redisClient, renderer, serviceMiddlewares...),
+			Handler:        handlers.APP(shutdown, log, cfg.Service.StaticFiles.Dir, cfg.Service.TemplateDir, masterDb, redisClient, renderer, serviceMiddlewares...),
 			ReadTimeout:    cfg.HTTP.ReadTimeout,
 			WriteTimeout:   cfg.HTTP.WriteTimeout,
 			MaxHeaderBytes: 1 << 20,
@ -652,7 +652,7 @@ func main() {
 	if cfg.HTTPS.Host != "" {
 		api := http.Server{
 			Addr:           cfg.HTTPS.Host,
-			Handler:        handlers.APP(shutdown, log, cfg.Service.StaticDir, cfg.Service.TemplateDir, masterDb, redisClient, renderer, serviceMiddlewares...),
+			Handler:        handlers.APP(shutdown, log, cfg.Service.StaticFiles.Dir, cfg.Service.TemplateDir, masterDb, redisClient, renderer, serviceMiddlewares...),
 			ReadTimeout:    cfg.HTTPS.ReadTimeout,
 			WriteTimeout:   cfg.HTTPS.WriteTimeout,
 			MaxHeaderBytes: 1 << 20,
--- a/internal/platform/devops/deploy.go
+++ b/internal/platform/devops/deploy.go
@ -1,20 +0,0 @@
-package devops
-
-import (
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
-)
-
-// SyncS3StaticFiles copies the local files from the static directory to s3
-// with public-read enabled.
-func SyncS3StaticFiles(awsSession *session.Session, staticS3Bucket, staticS3Prefix, staticDir string) error {
-	uploader := s3manager.NewUploader(awsSession)
-
-	di := NewDirectoryIterator(staticS3Bucket, staticS3Prefix, staticDir, "public-read")
-	if err := uploader.UploadWithIterator(aws.BackgroundContext(), di); err != nil {
-		return err
-	}
-
-	return nil
-}
--- a/tools/devops/cmd/cicd/aws.go
+++ b/tools/devops/cmd/cicd/aws.go
@ -3,6 +3,7 @@ package cicd
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"io/ioutil"
 	"net/url"
 	"path/filepath"
@ -262,6 +263,18 @@ func EcrPurgeImages(req *serviceBuildRequest) ([]*ecr.ImageIdentifier, error) {
 	return delIds, nil
 }

+// SyncPublicS3Files copies the local files from the static directory to s3 with public-read enabled.
+func SyncPublicS3Files(awsSession *session.Session, staticS3Bucket, staticS3Prefix, staticDir string) error {
+	uploader := s3manager.NewUploader(awsSession)
+
+	di := NewDirectoryIterator(staticS3Bucket, staticS3Prefix, staticDir, "public-read")
+	if err := uploader.UploadWithIterator(aws.BackgroundContext(), di); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // EcsReadTaskDefinition reads a task definition file and json decodes it.
 func EcsReadTaskDefinition(serviceDir, targetEnv string) ([]byte, error) {
 	checkPaths := []string{
--- a/internal/platform/devops/s3_batch_upload.go
+++ b/internal/platform/devops/s3_batch_upload.go
@ -1,4 +1,4 @@
-package devops
+package cicd

 import (
 	"bytes"
--- a/tools/devops/cmd/cicd/service.go
+++ b/tools/devops/cmd/cicd/service.go
@ -142,19 +142,26 @@ func ecrRepositoryName(projectName string) string {

 // releaseImage returns the name used for tagging a release image will always include one with environment and
 // service name. If the env var CI_COMMIT_REF_NAME is set, it will be appended.
-func releaseImage(env, serviceName, repositoryUri string) string {
+func releaseTag(env, serviceName string) string {

 	tag1 := env + "-" + serviceName

 	// Generate tags for the release image.
-	var releaseImage string
+	var releaseTag string
 	if v := os.Getenv("CI_COMMIT_REF_NAME"); v != "" {
 		tag2 := tag1 + "-" + v
-		releaseImage = repositoryUri + ":" + tag2
+		releaseTag = tag2
 	} else {
-		releaseImage = repositoryUri + ":" + tag1
+		releaseTag = tag1
 	}
-	return releaseImage
+	return releaseTag
+}
+
+
+// releaseImage returns the name used for tagging a release image will always include one with environment and
+// service name. If the env var CI_COMMIT_REF_NAME is set, it will be appended.
+func releaseImage(env, serviceName, repositoryUri string) string {
+	return repositoryUri + ":" + releaseTag(env, serviceName)
 }

 // dBInstanceIdentifier returns the database name.
--- a/tools/devops/cmd/cicd/service_deploy.go
+++ b/tools/devops/cmd/cicd/service_deploy.go
@ -64,6 +64,11 @@ type ServiceDeployFlags struct {
 	DockerFile      string `validate:"omitempty" example:"./cmd/web-api/Dockerfile"`
 	EnableLambdaVPC bool   `validate:"omitempty" example:"false"`
 	EnableEcsElb    bool   `validate:"omitempty" example:"false"`
+
+	StaticFilesS3Enable         bool `validate:"omitempty" example:"false"`
+	StaticFilesCloudfrontEnable bool `validate:"omitempty" example:"false"`
+	StaticFilesImgResizeEnable  bool `validate:"omitempty" example:"false"`
+
 	RecreateService bool `validate:"omitempty" example:"false"`
 }

@ -108,8 +113,14 @@ type serviceDeployRequest struct {
 	S3BucketTempPrefix      string `validate:"required_with=S3BucketPrivateName S3BucketPublicName"`
 	S3BucketPrivateName     string `validate:"omitempty"`
 	S3BucketPublicName      string `validate:"omitempty"`
+	S3BucketPublicKeyPrefix string `validate:"omitempty"`
 	S3Buckets               []S3Bucket

+	StaticFilesS3Enable         bool   `validate:"omitempty"`
+	StaticFilesS3Prefix         string `validate:"omitempty"`
+	StaticFilesCloudfrontEnable bool   `validate:"omitempty"`
+	StaticFilesImgResizeEnable  bool   `validate:"omitempty"`
+
 	EnableEcsElb           bool   `validate:"omitempty"`
 	ElbLoadBalancerName    string `validate:"omitempty"`
 	ElbDeregistrationDelay *int   `validate:"omitempty"`
@ -172,6 +183,11 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
 			EnableHTTPS:        flags.EnableHTTPS,
 			ServiceHostPrimary: flags.ServiceHostPrimary,
 			ServiceHostNames:   flags.ServiceHostNames,
+
+			StaticFilesS3Enable:         flags.StaticFilesS3Enable,
+			StaticFilesCloudfrontEnable: flags.StaticFilesCloudfrontEnable,
+			StaticFilesImgResizeEnable:  flags.StaticFilesImgResizeEnable,
+
 			S3BucketPrivateName: flags.S3BucketPrivateName,
 			S3BucketPublicName:  flags.S3BucketPublicName,
 			EnableLambdaVPC:     flags.EnableLambdaVPC,
@ -337,6 +353,16 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
 					})
 			}

+			// The S3 key prefix used as the origin when cloud front is enabled.
+			if req.S3BucketPublicKeyPrefix == "" {
+				req.S3BucketPublicKeyPrefix = "public"
+			}
+
+			// The S3 prefix used to upload static files served to public.
+			if req.StaticFilesS3Prefix == "" {
+				req.StaticFilesS3Prefix = filepath.Join(req.S3BucketPublicKeyPrefix, releaseTag(req.Env, req.ServiceName), "static")
+			}
+
 			// Set default AWS ECR Repository Name.
 			req.EcrRepositoryName = ecrRepositoryName(req.ProjectName)
 			log.Printf("\t\t\tSet ECR Repository Name to '%s'.", req.EcrRepositoryName)
@ -988,7 +1014,6 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 				log.Printf("\t\t\tUpdated bucket policy")
 			}
 		}
-
 		log.Printf("\t%s\tS3 buckets configured successfully.\n", tests.Success)
 	}

@ -2307,6 +2332,11 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 			"{HOST_PRIMARY}": req.ServiceHostPrimary,
 			"{HOST_NAMES}":   strings.Join(req.ServiceHostNames, ","),

+			"{STATIC_FILES_S3_ENABLED}":         "false",
+			"{STATIC_FILES_S3_PREFIX}":          "",
+			"{STATIC_FILES_CLOUDFRONT_ENABLED}": "false",
+			"{STATIC_FILES_IMG_RESIZE_ENABLED}": "false",
+
 			"{CACHE_HOST}": "", // Not enabled by default

 			"{DB_HOST}":        "",
@ -2359,6 +2389,21 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 			placeholders["{APP_BASE_URL}"] = fmt.Sprintf("%s://%s/", appSchema, req.ServiceHostPrimary)
 		}

+		// Static files served from S3.
+		if req.StaticFilesS3Enable {
+			placeholders["{STATIC_FILES_S3_ENABLED}"] = "true"
+		}
+
+		// Static files served from CloudFront.
+		if req.StaticFilesCloudfrontEnable {
+			placeholders["{STATIC_FILES_CLOUDFRONT_ENABLED}"] = "true"
+		}
+
+		// Support for resizing static images files to be responsive.
+		if req.StaticFilesImgResizeEnable {
+			placeholders["{STATIC_FILES_IMG_RESIZE_ENABLED}"] = "true"
+		}
+
 		// When db is set, update the placeholders.
 		if db != nil {
 			placeholders["{DB_HOST}"] = db.Host
@ -3157,6 +3202,20 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 		}
 	}

+	// When static files are enabled to be to stored on S3, we need to upload all of them.
+	if req.StaticFilesS3Enable {
+		log.Println("\tSync static files to public S3 bucket")
+
+		staticDir := filepath.Join(req.ServiceDir, "static")
+
+		err := SyncPublicS3Files(req.awsSession(), req.S3BucketPublicName, req.StaticFilesS3Prefix, staticDir)
+		if err != nil {
+			return errors.Wrapf(err, "Failed to sync static files from %s to s3://%s/%s '%s'", staticDir, req.S3BucketPublicName, req.StaticFilesS3Prefix)
+		}
+
+		log.Printf("\t%s\tFiles uploaded.\n", tests.Success)
+	}
+
 	// Wait for the updated or created service to enter a stable state.
 	{
 		log.Println("\tWaiting for service to enter stable state.")
--- a/tools/devops/main.go
+++ b/tools/devops/main.go
@ -82,6 +82,9 @@ func main() {
 				cli.StringFlag{Name: "project", Usage: "name of project", Destination: &deployFlags.ProjectName},
 				cli.BoolFlag{Name: "enable_elb", Usage: "enable deployed to use Elastic Load Balancer", Destination: &deployFlags.EnableEcsElb},
 				cli.BoolTFlag{Name: "lambda_vpc", Usage: "deploy lambda behind VPC", Destination: &deployFlags.EnableLambdaVPC},
+				cli.BoolFlag{Name: "static_files_s3", Usage: "service static files from S3", Destination: &deployFlags.StaticFilesS3Enable},
+				cli.BoolFlag{Name: "static_files_cloudfront", Usage: "serve static files from Cloudfront", Destination: &deployFlags.StaticFilesCloudfrontEnable},
+				cli.BoolFlag{Name: "static_files_img_resize", Usage: "enable response images from service", Destination: &deployFlags.StaticFilesImgResizeEnable},
 				cli.BoolFlag{Name: "recreate_service", Usage: "skip docker push after build", Destination: &deployFlags.RecreateService},
 			},
 			Action: func(c *cli.Context) error {