From df55acf6549cd1d0005c8b8d55c368f539e3d296 Mon Sep 17 00:00:00 2001 From: Lee Brown Date: Sun, 14 Jul 2019 19:13:09 -0800 Subject: [PATCH] add web-app to gitlab cicd --- .gitlab-ci.yml | 50 ++++++++++-- cmd/web-app/Dockerfile | 1 + cmd/web-app/main.go | 28 +++---- internal/platform/devops/deploy.go | 20 ----- tools/devops/cmd/cicd/aws.go | 13 ++++ .../devops/cmd/cicd}/s3_batch_upload.go | 2 +- tools/devops/cmd/cicd/service.go | 17 ++-- tools/devops/cmd/cicd/service_deploy.go | 77 ++++++++++++++++--- tools/devops/main.go | 3 + 9 files changed, 157 insertions(+), 54 deletions(-) delete mode 100644 internal/platform/devops/deploy.go rename {internal/platform/devops => tools/devops/cmd/cicd}/s3_batch_upload.go (99%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d7117bc..44a432a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -34,7 +34,7 @@ cache: .deploy_tmpl: &deploy_tmpl <<: *job_tmpl script: - - 'devops deploy -service=${SERVICE} -project=${PROJECT_NAME} -env=${TARGET_ENV} -enable_https=${ENABLE_HTTPS} -enable_elb=${ENABLE_ELB} -primary_host=${PRIMARY_HOST} -host_names=${HOST_NAMES} -private_bucket=${S3_BUCKET_PRIVATE} -public_bucket=${S3_BUCKET_PUBLIC}' + - 'devops deploy -service=${SERVICE} -project=${PROJECT_NAME} -env=${TARGET_ENV} -enable_https=${ENABLE_HTTPS} -enable_elb=${ENABLE_ELB} -primary_host=${PRIMARY_HOST} -host_names=${HOST_NAMES} -private_bucket=${S3_BUCKET_PRIVATE} -public_bucket=${S3_BUCKET_PUBLIC} -static_files_s3={STATIC_FILES_S3_ENABLED} -static_files_cloudfront={STATIC_FILES_CLOUDFRONT_ENABLED} -static_files_img_resize={STATIC_FILES_IMG_RESIZE_ENABLED}' .migrate_tmpl: &migrate_tmpl <<: *job_tmpl @@ -54,6 +54,45 @@ db:migrate:dev: TARGET_ENV: 'dev' AWS_USE_ROLE: 'true' +webapp:build:dev: + <<: *build_tmpl + stage: build:dev + tags: + - dev + only: + - master + - dev + - dev-web-app + variables: + TARGET_ENV: 'dev' + SERVICE: 'web-app' + AWS_USE_ROLE: 'true' +webapp:deploy:dev: + <<: *deploy_tmpl + stage: deploy:dev + tags: + - dev + only: + - master + - dev + - dev-web-app + dependencies: + - 'webapp:build:dev' + - 'db:migrate:dev' + variables: + TARGET_ENV: 'dev' + SERVICE: 'web-app' + ENABLE_HTTPS: 1 + ENABLE_ELB: 0 + PRIMARY_HOST: 'eproc.tech' + HOST_NAMES: 'www.eproc.tech, dev.eproc.tech' + S3_BUCKET_PRIVATE: 'saas-starter-kit-private' + S3_BUCKET_PUBLIC: 'saas-starter-kit-public' + STATIC_FILES_S3_ENABLED: 'true' + STATIC_FILES_CLOUDFRONT_ENABLED: 'false' + STATIC_FILES_IMG_RESIZE_ENABLED: 'true' + AWS_USE_ROLE: 'true' + webapi:build:dev: <<: *build_tmpl stage: build:dev @@ -84,14 +123,15 @@ webapi:deploy:dev: SERVICE: 'web-api' ENABLE_HTTPS: 1 ENABLE_ELB: 0 - PRIMARY_HOST: 'eproc.tech' - HOST_NAMES: 'www.eproc.tech, api.eproc.tech' + PRIMARY_HOST: 'api.eproc.tech' + HOST_NAMES: 'api.dev.eproc.tech' S3_BUCKET_PRIVATE: 'saas-starter-kit-private' S3_BUCKET_PUBLIC: 'saas-starter-kit-public' + STATIC_FILES_S3_ENABLED: 'false' + STATIC_FILES_CLOUDFRONT_ENABLED: 'false' + STATIC_FILES_IMG_RESIZE_ENABLED: 'false' AWS_USE_ROLE: 'true' - - #ddlogscollector:deploy:stage: # <<: *deploy_stage_tmpl # variables: diff --git a/cmd/web-app/Dockerfile b/cmd/web-app/Dockerfile index 52d5c5c..5bfdd9b 100644 --- a/cmd/web-app/Dockerfile +++ b/cmd/web-app/Dockerfile @@ -38,6 +38,7 @@ COPY --from=builder /static /static COPY --from=builder /templates /templates ENV TEMPLATE_DIR=/templates +ENV STATIC_DIR=/static ARG service ENV SERVICE_NAME $service diff --git a/cmd/web-app/main.go b/cmd/web-app/main.go index 9e9d7a3..c4d6a78 100644 --- a/cmd/web-app/main.go +++ b/cmd/web-app/main.go @@ -20,8 +20,8 @@ import ( "syscall" "time" - "geeks-accelerator/oss/saas-starter-kit/internal/mid" "geeks-accelerator/oss/saas-starter-kit/cmd/web-app/handlers" + "geeks-accelerator/oss/saas-starter-kit/internal/mid" "geeks-accelerator/oss/saas-starter-kit/internal/platform/devops" "geeks-accelerator/oss/saas-starter-kit/internal/platform/flag" img_resize "geeks-accelerator/oss/saas-starter-kit/internal/platform/img-resize" @@ -80,10 +80,10 @@ func main() { HostNames []string `envconfig:"HOST_NAMES" example:"www.eproc.tech"` EnableHTTPS bool `default:"false" envconfig:"ENABLE_HTTPS"` TemplateDir string `default:"./templates" envconfig:"TEMPLATE_DIR"` - StaticDir string `default:"./static" envconfig:"STATIC_DIR"` - StaticS3 struct { - S3Enabled bool `envconfig:"ENABLED"` - S3KeyPrefix string `default:"public/web_app/static" envconfig:"KEY_PREFIX"` + StaticFiles struct { + Dir string `default:"./static" envconfig:"STATIC_DIR"` + S3Enabled bool `envconfig:"S3_ENABLED"` + S3Prefix string `default:"public/web_app/static" envconfig:"S3_PREFIX"` CloudFrontEnabled bool `envconfig:"CLOUDFRONT_ENABLED"` ImgResizeEnabled bool `envconfig:"IMG_RESIZE_ENABLED"` } @@ -369,8 +369,8 @@ func main() { // s3UrlFormatter is a help function used by to convert an s3 key to // a publicly available image URL. var staticS3UrlFormatter func(string) string - if cfg.Service.StaticS3.S3Enabled || cfg.Service.StaticS3.CloudFrontEnabled || cfg.Service.StaticS3.ImgResizeEnabled { - s3UrlFormatter, err := devops.S3UrlFormatter(awsSession, cfg.Aws.S3BucketPublic, cfg.Service.StaticS3.S3KeyPrefix, cfg.Service.StaticS3.CloudFrontEnabled) + if cfg.Service.StaticFiles.S3Enabled || cfg.Service.StaticFiles.CloudFrontEnabled || cfg.Service.StaticFiles.ImgResizeEnabled { + s3UrlFormatter, err := devops.S3UrlFormatter(awsSession, cfg.Aws.S3BucketPublic, cfg.Service.StaticFiles.S3Prefix, cfg.Service.StaticFiles.CloudFrontEnabled) if err != nil { log.Fatalf("main : S3UrlFormatter failed : %+v", err) } @@ -379,7 +379,7 @@ func main() { // When the path starts with a forward slash its referencing a local file, // make sure the static file prefix is included if strings.HasPrefix(p, "/") { - p = filepath.Join(cfg.Service.StaticS3.S3KeyPrefix, p) + p = filepath.Join(cfg.Service.StaticFiles.S3Prefix, p) } return s3UrlFormatter(p) } @@ -400,7 +400,7 @@ func main() { // templates should be updated to use a fully qualified URL for either the public file on S3 // on from the cloudfront distribution. var staticUrlFormatter func(string) string - if cfg.Service.StaticS3.S3Enabled || cfg.Service.StaticS3.CloudFrontEnabled { + if cfg.Service.StaticFiles.S3Enabled || cfg.Service.StaticFiles.CloudFrontEnabled { staticUrlFormatter = staticS3UrlFormatter } else { baseUrl, err := url.Parse(cfg.Service.BaseUrl) @@ -508,12 +508,12 @@ func main() { // Image Formatter - additional functions exposed to templates for resizing images // to support response web applications. - imgResizeS3KeyPrefix := filepath.Join(cfg.Service.StaticS3.S3KeyPrefix, "images/responsive") + imgResizeS3KeyPrefix := filepath.Join(cfg.Service.StaticFiles.S3Prefix, "images/responsive") imgSrcAttr := func(ctx context.Context, p string, sizes []int, includeOrig bool) template.HTMLAttr { u := staticUrlFormatter(p) var srcAttr string - if cfg.Service.StaticS3.ImgResizeEnabled { + if cfg.Service.StaticFiles.ImgResizeEnabled { srcAttr, _ = img_resize.S3ImgSrc(ctx, redisClient, staticS3UrlFormatter, awsSession, cfg.Aws.S3BucketPublic, imgResizeS3KeyPrefix, u, sizes, includeOrig) } else { srcAttr = fmt.Sprintf("src=\"%s\"", u) @@ -544,7 +544,7 @@ func main() { } tmplFuncs["S3ImgUrl"] = func(ctx context.Context, p string, size int) string { imgUrl := staticUrlFormatter(p) - if cfg.Service.StaticS3.ImgResizeEnabled { + if cfg.Service.StaticFiles.ImgResizeEnabled { imgUrl, _ = img_resize.S3ImgUrl(ctx, redisClient, staticS3UrlFormatter, awsSession, cfg.Aws.S3BucketPublic, imgResizeS3KeyPrefix, imgUrl, size) } return imgUrl @@ -635,7 +635,7 @@ func main() { if cfg.HTTP.Host != "" { api := http.Server{ Addr: cfg.HTTP.Host, - Handler: handlers.APP(shutdown, log, cfg.Service.StaticDir, cfg.Service.TemplateDir, masterDb, redisClient, renderer, serviceMiddlewares...), + Handler: handlers.APP(shutdown, log, cfg.Service.StaticFiles.Dir, cfg.Service.TemplateDir, masterDb, redisClient, renderer, serviceMiddlewares...), ReadTimeout: cfg.HTTP.ReadTimeout, WriteTimeout: cfg.HTTP.WriteTimeout, MaxHeaderBytes: 1 << 20, @@ -652,7 +652,7 @@ func main() { if cfg.HTTPS.Host != "" { api := http.Server{ Addr: cfg.HTTPS.Host, - Handler: handlers.APP(shutdown, log, cfg.Service.StaticDir, cfg.Service.TemplateDir, masterDb, redisClient, renderer, serviceMiddlewares...), + Handler: handlers.APP(shutdown, log, cfg.Service.StaticFiles.Dir, cfg.Service.TemplateDir, masterDb, redisClient, renderer, serviceMiddlewares...), ReadTimeout: cfg.HTTPS.ReadTimeout, WriteTimeout: cfg.HTTPS.WriteTimeout, MaxHeaderBytes: 1 << 20, diff --git a/internal/platform/devops/deploy.go b/internal/platform/devops/deploy.go deleted file mode 100644 index 879293c..0000000 --- a/internal/platform/devops/deploy.go +++ /dev/null @@ -1,20 +0,0 @@ -package devops - -import ( - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3/s3manager" -) - -// SyncS3StaticFiles copies the local files from the static directory to s3 -// with public-read enabled. -func SyncS3StaticFiles(awsSession *session.Session, staticS3Bucket, staticS3Prefix, staticDir string) error { - uploader := s3manager.NewUploader(awsSession) - - di := NewDirectoryIterator(staticS3Bucket, staticS3Prefix, staticDir, "public-read") - if err := uploader.UploadWithIterator(aws.BackgroundContext(), di); err != nil { - return err - } - - return nil -} diff --git a/tools/devops/cmd/cicd/aws.go b/tools/devops/cmd/cicd/aws.go index 82ddc2f..4ced254 100644 --- a/tools/devops/cmd/cicd/aws.go +++ b/tools/devops/cmd/cicd/aws.go @@ -3,6 +3,7 @@ package cicd import ( "encoding/json" "fmt" + "github.com/aws/aws-sdk-go/service/s3/s3manager" "io/ioutil" "net/url" "path/filepath" @@ -262,6 +263,18 @@ func EcrPurgeImages(req *serviceBuildRequest) ([]*ecr.ImageIdentifier, error) { return delIds, nil } +// SyncPublicS3Files copies the local files from the static directory to s3 with public-read enabled. +func SyncPublicS3Files(awsSession *session.Session, staticS3Bucket, staticS3Prefix, staticDir string) error { + uploader := s3manager.NewUploader(awsSession) + + di := NewDirectoryIterator(staticS3Bucket, staticS3Prefix, staticDir, "public-read") + if err := uploader.UploadWithIterator(aws.BackgroundContext(), di); err != nil { + return err + } + + return nil +} + // EcsReadTaskDefinition reads a task definition file and json decodes it. func EcsReadTaskDefinition(serviceDir, targetEnv string) ([]byte, error) { checkPaths := []string{ diff --git a/internal/platform/devops/s3_batch_upload.go b/tools/devops/cmd/cicd/s3_batch_upload.go similarity index 99% rename from internal/platform/devops/s3_batch_upload.go rename to tools/devops/cmd/cicd/s3_batch_upload.go index 778d6ef..248efd5 100644 --- a/internal/platform/devops/s3_batch_upload.go +++ b/tools/devops/cmd/cicd/s3_batch_upload.go @@ -1,4 +1,4 @@ -package devops +package cicd import ( "bytes" diff --git a/tools/devops/cmd/cicd/service.go b/tools/devops/cmd/cicd/service.go index 272087c..1f71d10 100644 --- a/tools/devops/cmd/cicd/service.go +++ b/tools/devops/cmd/cicd/service.go @@ -142,19 +142,26 @@ func ecrRepositoryName(projectName string) string { // releaseImage returns the name used for tagging a release image will always include one with environment and // service name. If the env var CI_COMMIT_REF_NAME is set, it will be appended. -func releaseImage(env, serviceName, repositoryUri string) string { +func releaseTag(env, serviceName string) string { tag1 := env + "-" + serviceName // Generate tags for the release image. - var releaseImage string + var releaseTag string if v := os.Getenv("CI_COMMIT_REF_NAME"); v != "" { tag2 := tag1 + "-" + v - releaseImage = repositoryUri + ":" + tag2 + releaseTag = tag2 } else { - releaseImage = repositoryUri + ":" + tag1 + releaseTag = tag1 } - return releaseImage + return releaseTag +} + + +// releaseImage returns the name used for tagging a release image will always include one with environment and +// service name. If the env var CI_COMMIT_REF_NAME is set, it will be appended. +func releaseImage(env, serviceName, repositoryUri string) string { + return repositoryUri + ":" + releaseTag(env, serviceName) } // dBInstanceIdentifier returns the database name. diff --git a/tools/devops/cmd/cicd/service_deploy.go b/tools/devops/cmd/cicd/service_deploy.go index 8b062c0..b8bb8ce 100644 --- a/tools/devops/cmd/cicd/service_deploy.go +++ b/tools/devops/cmd/cicd/service_deploy.go @@ -64,7 +64,12 @@ type ServiceDeployFlags struct { DockerFile string `validate:"omitempty" example:"./cmd/web-api/Dockerfile"` EnableLambdaVPC bool `validate:"omitempty" example:"false"` EnableEcsElb bool `validate:"omitempty" example:"false"` - RecreateService bool `validate:"omitempty" example:"false"` + + StaticFilesS3Enable bool `validate:"omitempty" example:"false"` + StaticFilesCloudfrontEnable bool `validate:"omitempty" example:"false"` + StaticFilesImgResizeEnable bool `validate:"omitempty" example:"false"` + + RecreateService bool `validate:"omitempty" example:"false"` } // serviceDeployRequest defines the details needed to execute a service deployment. @@ -105,10 +110,16 @@ type serviceDeployRequest struct { CloudWatchLogGroupName string `validate:"required"` CloudWatchLogGroup *cloudwatchlogs.CreateLogGroupInput - S3BucketTempPrefix string `validate:"required_with=S3BucketPrivateName S3BucketPublicName"` - S3BucketPrivateName string `validate:"omitempty"` - S3BucketPublicName string `validate:"omitempty"` - S3Buckets []S3Bucket + S3BucketTempPrefix string `validate:"required_with=S3BucketPrivateName S3BucketPublicName"` + S3BucketPrivateName string `validate:"omitempty"` + S3BucketPublicName string `validate:"omitempty"` + S3BucketPublicKeyPrefix string `validate:"omitempty"` + S3Buckets []S3Bucket + + StaticFilesS3Enable bool `validate:"omitempty"` + StaticFilesS3Prefix string `validate:"omitempty"` + StaticFilesCloudfrontEnable bool `validate:"omitempty"` + StaticFilesImgResizeEnable bool `validate:"omitempty"` EnableEcsElb bool `validate:"omitempty"` ElbLoadBalancerName string `validate:"omitempty"` @@ -169,9 +180,14 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic req = serviceDeployRequest{ serviceRequest: sr, - EnableHTTPS: flags.EnableHTTPS, - ServiceHostPrimary: flags.ServiceHostPrimary, - ServiceHostNames: flags.ServiceHostNames, + EnableHTTPS: flags.EnableHTTPS, + ServiceHostPrimary: flags.ServiceHostPrimary, + ServiceHostNames: flags.ServiceHostNames, + + StaticFilesS3Enable: flags.StaticFilesS3Enable, + StaticFilesCloudfrontEnable: flags.StaticFilesCloudfrontEnable, + StaticFilesImgResizeEnable: flags.StaticFilesImgResizeEnable, + S3BucketPrivateName: flags.S3BucketPrivateName, S3BucketPublicName: flags.S3BucketPublicName, EnableLambdaVPC: flags.EnableLambdaVPC, @@ -337,6 +353,16 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic }) } + // The S3 key prefix used as the origin when cloud front is enabled. + if req.S3BucketPublicKeyPrefix == "" { + req.S3BucketPublicKeyPrefix = "public" + } + + // The S3 prefix used to upload static files served to public. + if req.StaticFilesS3Prefix == "" { + req.StaticFilesS3Prefix = filepath.Join(req.S3BucketPublicKeyPrefix, releaseTag(req.Env, req.ServiceName), "static") + } + // Set default AWS ECR Repository Name. req.EcrRepositoryName = ecrRepositoryName(req.ProjectName) log.Printf("\t\t\tSet ECR Repository Name to '%s'.", req.EcrRepositoryName) @@ -988,7 +1014,6 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error { log.Printf("\t\t\tUpdated bucket policy") } } - log.Printf("\t%s\tS3 buckets configured successfully.\n", tests.Success) } @@ -2307,6 +2332,11 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error { "{HOST_PRIMARY}": req.ServiceHostPrimary, "{HOST_NAMES}": strings.Join(req.ServiceHostNames, ","), + "{STATIC_FILES_S3_ENABLED}": "false", + "{STATIC_FILES_S3_PREFIX}": "", + "{STATIC_FILES_CLOUDFRONT_ENABLED}": "false", + "{STATIC_FILES_IMG_RESIZE_ENABLED}": "false", + "{CACHE_HOST}": "", // Not enabled by default "{DB_HOST}": "", @@ -2359,6 +2389,21 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error { placeholders["{APP_BASE_URL}"] = fmt.Sprintf("%s://%s/", appSchema, req.ServiceHostPrimary) } + // Static files served from S3. + if req.StaticFilesS3Enable { + placeholders["{STATIC_FILES_S3_ENABLED}"] = "true" + } + + // Static files served from CloudFront. + if req.StaticFilesCloudfrontEnable { + placeholders["{STATIC_FILES_CLOUDFRONT_ENABLED}"] = "true" + } + + // Support for resizing static images files to be responsive. + if req.StaticFilesImgResizeEnable { + placeholders["{STATIC_FILES_IMG_RESIZE_ENABLED}"] = "true" + } + // When db is set, update the placeholders. if db != nil { placeholders["{DB_HOST}"] = db.Host @@ -3157,6 +3202,20 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error { } } + // When static files are enabled to be to stored on S3, we need to upload all of them. + if req.StaticFilesS3Enable { + log.Println("\tSync static files to public S3 bucket") + + staticDir := filepath.Join(req.ServiceDir, "static") + + err := SyncPublicS3Files(req.awsSession(), req.S3BucketPublicName, req.StaticFilesS3Prefix, staticDir) + if err != nil { + return errors.Wrapf(err, "Failed to sync static files from %s to s3://%s/%s '%s'", staticDir, req.S3BucketPublicName, req.StaticFilesS3Prefix) + } + + log.Printf("\t%s\tFiles uploaded.\n", tests.Success) + } + // Wait for the updated or created service to enter a stable state. { log.Println("\tWaiting for service to enter stable state.") diff --git a/tools/devops/main.go b/tools/devops/main.go index 2021bad..2e946a3 100644 --- a/tools/devops/main.go +++ b/tools/devops/main.go @@ -82,6 +82,9 @@ func main() { cli.StringFlag{Name: "project", Usage: "name of project", Destination: &deployFlags.ProjectName}, cli.BoolFlag{Name: "enable_elb", Usage: "enable deployed to use Elastic Load Balancer", Destination: &deployFlags.EnableEcsElb}, cli.BoolTFlag{Name: "lambda_vpc", Usage: "deploy lambda behind VPC", Destination: &deployFlags.EnableLambdaVPC}, + cli.BoolFlag{Name: "static_files_s3", Usage: "service static files from S3", Destination: &deployFlags.StaticFilesS3Enable}, + cli.BoolFlag{Name: "static_files_cloudfront", Usage: "serve static files from Cloudfront", Destination: &deployFlags.StaticFilesCloudfrontEnable}, + cli.BoolFlag{Name: "static_files_img_resize", Usage: "enable response images from service", Destination: &deployFlags.StaticFilesImgResizeEnable}, cli.BoolFlag{Name: "recreate_service", Usage: "skip docker push after build", Destination: &deployFlags.RecreateService}, }, Action: func(c *cli.Context) error {