remwork tools

.gitlab-ci.yml

@@ -1,20 +1,25 @@
-image: docker:stable
+image: geeksaccelerator/docker-library:golang1.12-docker
 
 services:
   - docker:dind
 
 variables:
-  AWS_ECS_CLUSTER: example-project
-  AWS_S3_STATIC_BASE_URI: example-project-stage/public
-  CONTAINER_IMAGE: registry.gitlab.com/$CI_PROJECT_PATH
   DOCKER_HOST: tcp://docker:2375
   DOCKER_DRIVER: overlay2
+  NO_BUILD: "false"
+  NO_DEPLOY: "false"
+  # Optional:
+  # PROJECT_NAME: example-project
+
 
 before_script:
-  - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com
+  - 'cd ./tools/truss && make install'
+  - 'cd ./cmd/schema && make install'
 
 stages:
-  - build:base
+  - build:dev
+  - migrate:dev
+  - deploy:dev
   - build:stage
   - migrate:stage
   - deploy:stage
@@ -31,41 +36,23 @@ cache:
   only:
     - master
 
-.build_tmpl: &build_tmpl
-  <<: *job_tmpl
-  script:
-    - 'git clone http://gitlab+deploy-token-50199:z2WzSQKY9Crzvw98yzTy@gitlab.com/gitw/akverse.git'
-    - 'CI=1 PUSH=${PUSH} PUSH_AWS_REGISTRY=${PUSH_AWS_REGISTRY} ./akverse/scripts/build.sh $SERVICE $TARGET_ENV -'
-
 .deploy_tmpl: &deploy_tmpl
   <<: *job_tmpl
   script:
-    - 'git clone http://gitlab+deploy-token-50199:z2WzSQKY9Crzvw98yzTy@gitlab.com/gitw/akverse.git'
+    - 'truss deploy -service=${SERVICE} -project=${PROJECT_NAME} -env=${TARGET_ENV} -enable_https=${ENABLE_HTTPS} -enable_elb=${ENABLE_ELB} -primary_host=${PRIMARY_HOST} -host_names=${HOST_NAMES} -private_bucket=${S3_BUCKET_PRIVATE} -public_bucket=${S3_BUCKET_PUBLIC} -no_build=${NO_BUILD} -no_deploy=${NO_DEPLOY}'
-    - 'LB=${ENABLE_LB} SD=${ENABLE_SD} VPC=${ENABLE_VPC} S3_BUCKET=${S3_BUCKET} S3_KEY=${S3_KEY} STATIC_S3_URI=${STATIC_S3_URI} ./akverse/scripts/deploy.sh $SERVICE $TARGET_ENV - ${ECS_CLUSTER}'
 
-.build_base_tmpl: &build_base_tmpl
+.migrate_tmpl: &migrate_tmpl
-  <<: *build_tmpl
+  <<: *job_tmpl
-  stage: build:base
+  script:
-  tags:
+    - './schema ???'
-    - stage
-  only:
-    - master
-    - stage
-    - /^stage-.*$/
-    - prod
-    - /^prod-.*$/
 
-.build_stage_tmpl: &build_stage_tmpl
+.deploy_dev_tmpl: &deploy_dev_tmpl
-  <<: *build_tmpl
+  <<: *deploy_tmpl
-  stage: build:stage
+  stage: deploy:dev
   tags:
-    - stage
+    - dev
-
+  environment:
-.build_prod_tmpl: &build_prod_tmpl
+    name: 'dev/${SERVICE}-dev'
-  <<: *build_tmpl
-  stage: build:prod
-  tags:
-    - prod
 
 .deploy_stage_tmpl: &deploy_stage_tmpl
   <<: *deploy_tmpl
@@ -84,115 +71,76 @@ cache:
     name: 'production/${SERVICE}'
   when: manual
 
+.migrate_dev_tmpl: &migrate_dev_tmpl
+  <<: *migrate_tmpl
+  stage: migrate:dev
+  tags:
+    - dev
+  only:
+    - master
+    - dev
+    - /^dev-.*$/
+
 .migrate_stage_tmpl: &migrate_stage_tmpl
-  <<: *build_tmpl
+  <<: *migrate_tmpl
   stage: migrate:stage
   tags:
     - stage
   only:
-    - master
     - stage
     - /^stage-.*$/
-    - prod
-    - /^prod-.*$/
 
 .migrate_prod_tmpl: &migrate_prod_tmpl
-  <<: *build_tmpl
+  <<: *migrate_tmpl
   stage: migrate:prod
   tags:
     - prod
   when: manual
   only:
-    - master
     - prod
     - /^prod-.*$/
 
-datadog-agent:build:stage:
+#db:migrate:dev:
-  <<: *build_stage_tmpl
+#  <<: *migrate_dev_tmpl
-  variables:
+#  variables:
-    TARGET_ENV: 'stage'
+#    TARGET_ENV: 'stage'
-    SERVICE: 'datadog-agent'
+#    SERVICE: 'schema'
-    PUSH_AWS_REGISTRY: 1
 
-datadog-agent:build:prod:
+.deploy_dev_webapi_tmpl: &deploy_dev_webapi_tmpl
-  <<: *build_prod_tmpl
+  <<: *deploy_dev_tmpl
+  stage: deploy:dev
+  tags:
+    - dev
+  environment:
+    name: 'dev/${SERVICE}-dev'
+    TARGET_ENV: 'dev'
+    SERVICE: 'web-api'
+    ENABLE_HTTPS: "true"
+    ENABLE_ELB: "false"
+    PRIMARY_HOST: eproc.tech
+    HOST_NAMES: www.eproc.tech, api.eproc.tech
+    S3_BUCKET_PRIVATE: saas-starter-kit-private
+    S3_BUCKET_PUBLIC: saas-starter-kit-public
+webapi:build:dev:
+  <<: *deploy_dev_webapi_tmpl
   variables:
-    TARGET_ENV: 'prod'
+    NO_DEPLOY: "true"
-    SERVICE: 'datadog-agent'
-    PUSH_AWS_REGISTRY: 1
-  dependencies:
-    - 'datadog-agent:build:stage'
-
-db:migrate:stage:
-  <<: *migrate_stage_tmpl
-  variables:
-    TARGET_ENV: 'stage'
-    SERVICE: 'schema'
-
-db:migrate:prod:
-  <<: *migrate_prod_tmpl
-  variables:
-    TARGET_ENV: 'prod'
-    SERVICE: 'schema'
-  dependencies:
-    - 'db:migrate:stage'
-
-webapi:build:stage:
-  <<: *build_stage_tmpl
-  variables:
-    TARGET_ENV: 'stage'
-    SERVICE: 'webapi'
   only:
     - master
-    - stage
+    - dev
+    - dev-web-api
+webapi:deploy:dev:
+  <<: *deploy_dev_webapi_tmpl
+  variables:
+    NO_BUILD: "true"
+  dependencies:
+    - 'webapi:build:dev'
+    - 'db:migrate:dev'
+  only:
+    - master
+    - dv
     - stage-webapi
-    - prod
+
-    - prod-webapi
-webapi:deploy:stage:
-  <<: *deploy_stage_tmpl
-  variables:
-    TARGET_ENV: 'stage'
-    SERVICE: 'webapi'
-    ECS_CLUSTER: '${ECS_CLUSTER}'
-    STATIC_S3_URI: '${AWS_S3_STATIC_BASE_URI}/stage/webapi'
-    ENABLE_LB: 0
-  dependencies:
-    - 'webapi:build:stage'
-    - 'db:migrate:stage'
-    - 'datadog-agent:build:stage'
-  only:
-    - master
-    - stage
-    - stage-webapi
-    - prod
-    - prod-webapi
-webapi:build:prod:
-  <<: *build_prod_tmpl
-  variables:
-    TARGET_ENV: 'prod'
-    SERVICE: 'webapi'
-  dependencies:
-    - 'webapi:deploy:stage'
-  only:
-    - master
-    - prod
-    - prod-webapi
-webapi:deploy:prod:
-  <<: *deploy_prod_tmpl
-  variables:
-    TARGET_ENV: 'prod'
-    SERVICE: 'webapi'
-    ECS_CLUSTER: '${ECS_CLUSTER}'
-    STATIC_S3_URI: '${AWS_S3_STATIC_BASE_URI}/prod/webapi'
-    ENABLE_LB: 0
-  dependencies:
-    - 'webapi:build:prod'
-    - 'db:migrate:prod'
-    - 'datadog-agent:build:prod'
-  only:
-    - master
-    - prod
-    - prod-webapi
 
 #ddlogscollector:deploy:stage:
 #  <<: *deploy_stage_tmpl

README.md

@@ -405,7 +405,7 @@ instance will be a dedicated host since we need it always up and running, thus i
         * This will limit commits to the master or dev branches from triggering the pipeline to run. This includes a 
         wildcard for any branch named with the prefix `dev-`.
     * When asked the executor type, enter `docker+machine`
-    * When asked for the default Docker image, enter `golang:alpine3.9`
+    * When asked for the default Docker image, enter `geeksaccelerator/docker-library:golang1.12-docker`
         
 13. [Configuring the GitLab Runner](https://docs.gitlab.com/runner/configuration/runner_autoscale_aws/#configuring-the-gitlab-runner)   
 
@@ -418,7 +418,7 @@ instance will be a dedicated host since we need it always up and running, thus i
     ```yaml
       [runners.docker]
         tls_verify = false
-        image = "golang:alpine3.9"
+        image = "geeksaccelerator/docker-library:golang1.12-docker"
         privileged = true
         disable_entrypoint_overwrite = false
         oom_kill_disable = false

cmd/schema/makefile

@@ -1,4 +1,10 @@
 SHELL := /bin/bash
 
+install:
+	go install .
+
+build:
+	go install .
+
 run:
 	go build . && ./schema

tools/devops/cmd/deploy/devops.go

@@ -1,4 +1,4 @@
-package devops
+package deploy
 
 import (
 	"fmt"
@@ -120,36 +120,3 @@ func exists(path string) (bool, error) {
 	}
 	return true, err
 }
-
-/*
-type EnvVars []string
-
-// execCmds executes a set of commands.
-func execCmds(workDir string, envVars *EnvVars, cmds ...[]string) ([]string, error) {
-	if envVars == nil {
-		ev := EnvVars(os.Environ())
-		envVars = &ev
-	}
-
-	var results []string
-	for _, cmdVals := range cmds {
-		cmd := exec.Command(cmdVals[0], cmdVals[1:]...)
-		cmd.Dir = workDir
-		cmd.Env = *envVars
-		out, err := cmd.CombinedOutput()
-
-		fmt.Println(string(out ))
-
-		if err != nil {
-			return results, errors.WithMessagef(err, "failed to execute %s\n%s",  strings.Join(cmdVals, " "), string(out))
-		}
-		results = append(results, string(out))
-
-		// Update the current env vars after command has been executed.
-		ev := EnvVars(cmd.Env)
-		envVars = &ev
-	}
-
-	return results, nil
-}
-*/

tools/devops/cmd/deploy/models.go

@@ -1,4 +1,4 @@
-package devops
+package deploy
 
 import (
 	"strings"

tools/devops/main.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"expvar"
+	"log"
+	"os"
+	"strings"
+
+	"geeks-accelerator/oss/saas-starter-kit/tools/devops/cmd/deploy"
+	_ "github.com/lib/pq"
+	"github.com/urfave/cli"
+)
+
+// build is the git version of this program. It is set using build flags in the makefile.
+var build = "develop"
+
+// service is the name of the program used for logging, tracing and the
+// the prefix used for loading env variables
+// ie: export TRUSS_ENV=dev
+var service = "DEVOPS"
+
+func main() {
+	// =========================================================================
+	// Logging
+
+	log := log.New(os.Stdout, service+" : ", log.LstdFlags|log.Lmicroseconds|log.Lshortfile)
+
+
+	// =========================================================================
+	// Log App Info
+
+	// Print the build version for our logs. Also expose it under /debug/vars.
+	expvar.NewString("build").Set(build)
+	log.Printf("main : Started : Application Initializing version %q", build)
+	defer log.Println("main : Completed")
+
+
+	// =========================================================================
+	// Start Truss
+
+	var deployFlags deploy.ServiceDeployFlags
+
+	app := cli.NewApp()
+	app.Commands = []cli.Command{
+		{
+			Name:    "deploy",
+			Aliases: []string{"serviceDeploy"},
+			Usage:   "-service=web-api -env=dev",
+			Flags: []cli.Flag{
+				cli.StringFlag{Name: "service", Usage: "name of cmd", Destination: &deployFlags.ServiceName},
+				cli.StringFlag{Name: "env", Usage: "dev, stage, or prod", Destination: &deployFlags.Env},
+				cli.BoolFlag{Name: "enable_https", Usage: "enable HTTPS", Destination: &deployFlags.EnableHTTPS},
+				cli.StringFlag{Name: "primary_host", Usage: "dev, stage, or prod", Destination: &deployFlags.ServiceHostPrimary},
+				cli.StringSliceFlag{Name: "host_names", Usage: "dev, stage, or prod", Value: &deployFlags.ServiceHostNames},
+				cli.StringFlag{Name: "private_bucket", Usage: "dev, stage, or prod", Destination: &deployFlags.S3BucketPrivateName},
+				cli.StringFlag{Name: "public_bucket", Usage: "dev, stage, or prod", Destination: &deployFlags.S3BucketPublicName},
+				cli.StringFlag{Name: "dockerfile", Usage: "DockerFile for service", Destination: &deployFlags.DockerFile},
+				cli.StringFlag{Name: "root", Usage: "project root directory", Destination: &deployFlags.ProjectRoot},
+				cli.StringFlag{Name: "project", Usage: "name of project", Destination: &deployFlags.ProjectName},
+				cli.BoolFlag{Name: "enable_elb", Usage: "enable deployed to use Elastic Load Balancer", Destination: &deployFlags.EnableEcsElb},
+				cli.BoolTFlag{Name: "lambda_vpc", Usage: "deploy lambda behind VPC", Destination: &deployFlags.EnableLambdaVPC},
+				cli.BoolFlag{Name: "no_build", Usage: "skip build and continue directly to deploy", Destination: &deployFlags.NoBuild},
+				cli.BoolFlag{Name: "no_deploy", Usage: "skip deploy after build", Destination: &deployFlags.NoDeploy},
+				cli.BoolFlag{Name: "no_cache", Usage: "skip docker cache", Destination: &deployFlags.NoCache},
+				cli.BoolFlag{Name: "no_push", Usage: "skip docker push after build", Destination: &deployFlags.NoPush},
+				cli.BoolFlag{Name: "recreate_service", Usage: "skip docker push after build", Destination: &deployFlags.RecreateService},
+			},
+			Action: func(c *cli.Context) error {
+				if len(deployFlags.ServiceHostNames.Value()) == 1 {
+					var hostNames []string
+					for _, inpVal := range deployFlags.ServiceHostNames.Value() {
+						pts := strings.Split(inpVal, ",")
+
+						for _, h := range pts {
+							h = strings.TrimSpace(h)
+							if h != "" {
+								hostNames = append(hostNames, h)
+							}
+						}
+					}
+
+					deployFlags.ServiceHostNames = hostNames
+				}
+
+				req, err := deploy.NewServiceDeployRequest(log, deployFlags)
+				if err != nil {
+					return err
+				}
+				return deploy.ServiceDeploy(log, req)
+			},
+		},
+	}
+
+	err := app.Run(os.Args)
+	if err != nil {
+		log.Fatalf("main : Truss : %+v", err)
+	}
+
+	log.Printf("main : Truss : Completed")
+}

tools/truss/main.go

@@ -12,7 +12,6 @@ import (
 	"strings"
 
 	"geeks-accelerator/oss/saas-starter-kit/tools/truss/cmd/dbtable2crud"
-	"geeks-accelerator/oss/saas-starter-kit/tools/truss/cmd/devops"
 	"github.com/kelseyhightower/envconfig"
 	"github.com/lib/pq"
 	_ "github.com/lib/pq"
@@ -121,8 +120,6 @@ func main() {
 	// =========================================================================
 	// Start Truss
 
-	var deployFlags devops.ServiceDeployFlags
-
 	app := cli.NewApp()
 	app.Commands = []cli.Command{
 		{
@@ -208,37 +205,6 @@ func main() {
 				return dbtable2crud.Run(masterDb, log, cfg.DB.Database, dbTable, modelFile, modelName, templateDir, projectPath, c.Bool("saveChanges"))
 			},
 		},
-		{
-			Name:    "deploy",
-			Aliases: []string{"serviceDeploy"},
-			Usage:   "-service=web-api -env=dev",
-			Flags: []cli.Flag{
-				cli.StringFlag{Name: "service", Usage: "name of cmd", Destination: &deployFlags.ServiceName},
-				cli.StringFlag{Name: "env", Usage: "dev, stage, or prod", Destination: &deployFlags.Env},
-				cli.BoolFlag{Name: "enable_https", Usage: "enable HTTPS", Destination: &deployFlags.EnableHTTPS},
-				cli.StringFlag{Name: "primary_host", Usage: "dev, stage, or prod", Destination: &deployFlags.ServiceHostPrimary},
-				cli.StringSliceFlag{Name: "host_names", Usage: "dev, stage, or prod", Value: &deployFlags.ServiceHostNames},
-				cli.StringFlag{Name: "private_bucket", Usage: "dev, stage, or prod", Destination: &deployFlags.S3BucketPrivateName},
-				cli.StringFlag{Name: "public_bucket", Usage: "dev, stage, or prod", Destination: &deployFlags.S3BucketPublicName},
-				cli.StringFlag{Name: "dockerfile", Usage: "DockerFile for service", Destination: &deployFlags.DockerFile},
-				cli.StringFlag{Name: "root", Usage: "project root directory", Destination: &deployFlags.ProjectRoot},
-				cli.StringFlag{Name: "project", Usage: "name of project", Destination: &deployFlags.ProjectName},
-				cli.BoolFlag{Name: "elb", Usage: "enable deployed to use Elastic Load Balancer", Destination: &deployFlags.EnableEcsElb},
-				cli.BoolTFlag{Name: "lambda_vpc", Usage: "deploy lambda behind VPC", Destination: &deployFlags.EnableLambdaVPC},
-				cli.BoolFlag{Name: "no_build", Usage: "skip build and continue directly to deploy", Destination: &deployFlags.NoBuild},
-				cli.BoolFlag{Name: "no_deploy", Usage: "skip deploy after build", Destination: &deployFlags.NoDeploy},
-				cli.BoolFlag{Name: "no_cache", Usage: "skip docker cache", Destination: &deployFlags.NoCache},
-				cli.BoolFlag{Name: "no_push", Usage: "skip docker push after build", Destination: &deployFlags.NoPush},
-				cli.BoolFlag{Name: "recreate_service", Usage: "skip docker push after build", Destination: &deployFlags.RecreateService},
-			},
-			Action: func(c *cli.Context) error {
-				req, err := devops.NewServiceDeployRequest(log, deployFlags)
-				if err != nil {
-					return err
-				}
-				return devops.ServiceDeploy(log, req)
-			},
-		},
 	}
 
 	err = app.Run(os.Args)

tools/truss/makefile

@@ -0,0 +1,10 @@
+SHELL := /bin/bash
+
+install:
+	go install .
+
+build:
+	go install .
+
+run:
+	go build . && ./truss