diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 58df7ca..9a8e6ee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -87,8 +87,8 @@ webapp:deploy:dev: SERVICE: 'web-app' ENABLE_HTTPS: 1 ENABLE_ELB: 0 - PRIMARY_HOST: 'eproc.tech' - HOST_NAMES: 'www.eproc.tech,dev.eproc.tech' + PRIMARY_HOST: 'example.saasstartupkit.com' + HOST_NAMES: 'example.saasstartupkit.com,dev.example.saasstartupkit.com' S3_BUCKET_PRIVATE: 'saas-starter-kit-private' S3_BUCKET_PUBLIC: 'saas-starter-kit-public' S3_BUCKET_PUBLIC_CLOUDFRONT: 'true' @@ -96,7 +96,7 @@ webapp:deploy:dev: STATIC_FILES_IMG_RESIZE: 'true' AWS_USE_ROLE: 'true' EMAIL_SENDER: 'lee+saas-starter-kit@geeksinthewoods.com' - WEB_API_BASE_URL: https://api.eproc.tech + WEB_API_BASE_URL: https://api.example.saasstartupkit.com webapi:build:dev: <<: *build_tmpl @@ -128,8 +128,8 @@ webapi:deploy:dev: SERVICE: 'web-api' ENABLE_HTTPS: 1 ENABLE_ELB: 0 - PRIMARY_HOST: 'api.eproc.tech' - HOST_NAMES: 'api.dev.eproc.tech' + PRIMARY_HOST: 'api.example.saasstartupkit.com' + HOST_NAMES: 'api.dev.example.saasstartupkit.com' S3_BUCKET_PRIVATE: 'saas-starter-kit-private' S3_BUCKET_PUBLIC: 'saas-starter-kit-public' S3_BUCKET_PUBLIC_CLOUDFRONT: 'false' @@ -137,7 +137,7 @@ webapi:deploy:dev: STATIC_FILES_IMG_RESIZE: 'false' AWS_USE_ROLE: 'true' EMAIL_SENDER: 'lee+saas-starter-kit@geeksinthewoods.com' - WEB_APP_BASE_URL: https://eproc.tech + WEB_APP_BASE_URL: https://example.saasstartupkit.com #ddlogscollector:deploy:stage: # <<: *deploy_stage_tmpl diff --git a/README.md b/README.md index 2228ec0..c5fb683 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ delivered to clients. a knowledge of a completely different expertise - DevOps. This project provides a complete continuous build pipeline that will push the code to production with minimal effort using serverless deployments to AWS Fargate with GitLab CI/CD. 5. Observability - Ensure the code is running as expected in a remote environment. This project implements Datadog to -facilitate exposing metrics, logs and request tracing to obversabe and validate your services are stable and responsive +facilitate exposing metrics, logs and request tracing to obverse and validate your services are stable and responsive for your clients (hopefully paying clients). @@ -71,7 +71,7 @@ facilitate exposing metrics, logs and request tracing to obversabe and validate The example project is a complete starter kit for building SasS with GoLang. It provides two example services: * Web App - Responsive web application to provide service to clients. Includes user signup and user authentication for direct client interaction via their web browsers. -* Web API - REST API with JWT authentication that renders results as JSON. This allows clients and other third-pary companies to develop deep +* Web API - REST API with JWT authentication that renders results as JSON. This allows clients and other third-party companies to develop deep integrations with the project. The example project also provides these tools: @@ -106,7 +106,7 @@ Accordingly, the project architecture is illustrated with the following diagram. With SaaS, a client subscribes to an online service you provide them. The example project provides functionality for clients to subscribe and then once subscribed they can interact with your software service. -The initial contributors to this project are building this saas-starter-kit based on their years of experience building enterprise B2B SaaS. Particularily, this saas-starter-kit is based on their most recent experience building the +The initial contributors to this project are building this saas-starter-kit based on their years of experience building enterprise B2B SaaS. Particularly, this saas-starter-kit is based on their most recent experience building the B2B SaaS for [standard operating procedure software](https://keeni.space) (written entirely in Golang). Please refer to the Keeni.Space website, its [SOP software pricing](https://keeni.space/pricing) and its signup process. The SaaS web app is then available at [app.keeni.space](https://app.keeni.space). They plan on leveraging this experience and build it into a simplified set @@ -175,7 +175,7 @@ $ git clone git@gitlab.com:geeks-accelerator/oss/saas-starter-kit.git $ cd saas-starter-kit/ ``` -If you have Go Modules enabled, you should be able compile the project locally. If you have Go Modulels disabled, see +If you have Go Modules enabled, you should be able compile the project locally. If you have Go Modules disabled, see the next section. @@ -269,39 +269,133 @@ builds locally, update `docker-compose.yaml` to define a volume. ### Re-starting a specific Go service for development -When writing code in an iterative fashion, it is nice to be able to restart a specific service so it will run updated -Go code. This decreases the overhead of stopping all services with `docker-compose down` and then re-starting all the -services again with 'docker-compose up'. +When writing code in an iterative fashion, it is nice to have your change automatically rebuilt. This project uses +[github.com/gravityblast/fresh](https://github.com/gravityblast/fresh) to recompile your services that will include most +changes. + + Fresh is a command line tool that builds and (re)starts your web application everytime you save a Go or template file. + +The (Fresh configuration file](https://gitlab.com/geeks-accelerator/oss/saas-starter-kit/blob/master/fresh-auto-reload.conf) +is located in the project root. By default the following folders are watched by Fresh: +- handlers +- static +- templates + +Any changes to [internal/*](https://gitlab.com/geeks-accelerator/oss/saas-starter-kit/tree/master/internal) or +additional project dependencies added to [go.mod](https://gitlab.com/geeks-accelerator/oss/saas-starter-kit/blob/master/go.mod) +will require the service to be rebuilt. -To restart a specific service, first use `docker ps` to see the list of services running. ```bash -$ docker ps -CONTAINER ID IMAGE COMMAND NAMES -35043164fd0d example-project/web-api:latest "/gosrv" saas-starter-kit_web-api_1 -d34c8fc27f3b example-project/web-app:latest "/gosrv" saas-starter-kit_web-app_1 -fd844456243e postgres:11-alpine "docker-entrypoint.s…" saas-starter-kit_postgres_1 -dda16bfbb8b5 redis:latest "redis-server --appe…" saas-starter-kit_redis_1 +docker-compose up --build -d web-app ``` -Then use `docker-compose stop` for a specific service. In the command including the name of service in `docker-compose.yaml` file for the service -to shut down. In the example command, we will shut down the web-api service so we can start it again. +### Forking your own copy + +1. Checkout the project + +2. Update references. ```bash -$ docker-compose stop web-app +flist=`grep -r "geeks-accelerator/oss/saas-starter-kit" * | awk -F ':' '{print $1}' | sort | uniq` +for f in $flist; do echo $f; sed -i "" -e "s#geeks-accelerator/oss/saas-starter-kit#geeks-accelerator/oss/aurora-cam#g" $f; done + + +flist=`grep -r "saas-starter-kit" * | awk -F ':' '{print $1}' | sort | uniq` +for f in $flist; do echo $f; sed -i "" -e "s#saas-starter-kit#aurora-cam#g" $f; done + + +flist=`grep -r "example-project" * | awk -F ':' '{print $1}' | sort | uniq` +for f in $flist; do echo $f; sed -i "" -e "s#example-project#aurora-cam#g" $f; done + + ``` -If you are not in the directory for the service you want to restart then navigate to it. We will go to the directory for the -web-api. +3. Create a new AWS Policy with the following details: +``` +Name: SaasStarterKitDevServices +Description: Defines access for saas-starter-kit services. +Policy Document: { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "DefaultServiceAccess", + "Effect": "Allow", + "Action": [ + "s3:HeadBucket", + "s3:ListObjects", + "s3:PutObject", + "s3:PutObjectAcl", + "cloudfront:ListDistributions", + "ec2:DescribeNetworkInterfaces", + "ec2:DeleteNetworkInterface", + "ecs:ListTasks", + "ecs:DescribeServices", + "ecs:DescribeTasks", + "ec2:DescribeNetworkInterfaces", + "route53:ListHostedZones", + "route53:ListResourceRecordSets", + "route53:ChangeResourceRecordSets", + "ecs:UpdateService", + "ses:SendEmail", + "ses:ListIdentities", + "ses:GetAccountSendingEnabled", + "secretsmanager:ListSecretVersionIds", + "secretsmanager:GetSecretValue", + "secretsmanager:CreateSecret", + "secretsmanager:UpdateSecret", + "secretsmanager:RestoreSecret", + "secretsmanager:DeleteSecret" + ], + "Resource": "*" + }, + { + "Sid": "ServiceInvokeLambda", + "Effect": "Allow", + "Action": [ + "iam:GetRole", + "lambda:InvokeFunction", + "lambda:ListVersionsByFunction", + "lambda:GetFunction", + "lambda:InvokeAsync", + "lambda:GetFunctionConfiguration", + "iam:PassRole", + "lambda:GetAlias", + "lambda:GetPolicy" + ], + "Resource": [ + "arn:aws:iam:::role/*", + "arn:aws:lambda:::function:*" + ] + }, + { + "Sid": "datadoglambda", + "Effect": "Allow", + "Action": [ + "cloudwatch:Get*", + "cloudwatch:List*", + "ec2:Describe*", + "support:*", + "tag:GetResources", + "tag:GetTagKeys", + "tag:GetTagValues" + ], + "Resource": "*" + } + ] + } +``` +Create a new user with programmatic access and directly attach it the policy `SaasStarterKitDevServices` + +4. Create a new docker-compose config file ```bash -$ cd cmd/web-api/ + cp sample.env_docker_compose .env_docker_compose ``` -Then you can start the service again by running main.go -```bash -$ go run main.go -``` +5. Update .env_docker_compose with the Access key ID and Secret access key + +6. Update `.gitlab-ci.yml` with relevant details. ### Optional. Set AWS and Datadog Configs diff --git a/cmd/web-api/Dockerfile b/cmd/web-api/Dockerfile index bb0d0f6..51968c2 100644 --- a/cmd/web-api/Dockerfile +++ b/cmd/web-api/Dockerfile @@ -29,8 +29,9 @@ ENV GO111MODULE="on" COPY go.mod . COPY go.sum . RUN go mod download +RUN go get github.com/pilu/fresh -FROM build_base_golang AS builder +FROM build_base_golang AS dev ARG service ARG commit_ref=- @@ -40,15 +41,22 @@ ARG swagInit COPY internal ./internal # Copy cmd specific packages. -COPY cmd/${service} ./cmd/web-api +COPY cmd/${service} ./cmd/${service} COPY cmd/${service}/templates /templates #COPY cmd/${service}/static /static # Copy the global templates. ADD resources/templates/shared /templates/shared +ADD fresh-auto-reload.conf /runner.conf + +ENV TEMPLATE_DIR=/templates WORKDIR ./cmd/${service} +ENTRYPOINT ["fresh", "-c", "/runner.conf"] + +FROM dev AS builder + # Update the API documentation. # Disabled for the moment as it takes forever to run, rely on manual execution. RUN if [ "$swagInit" != "" ]; then swag init ; fi diff --git a/cmd/web-api/README.md b/cmd/web-api/README.md index 4aa4612..31eeb01 100644 --- a/cmd/web-api/README.md +++ b/cmd/web-api/README.md @@ -254,7 +254,7 @@ swag init ### Additional Swagger Annotations -Below are some additional example annotions that can be added to `main.go` +Below are some additional example annotations that can be added to `main.go` ```go // @title SaaS Example API // @description This provides a public API... diff --git a/cmd/web-api/main.go b/cmd/web-api/main.go index 0a1b2b9..e331ead 100644 --- a/cmd/web-api/main.go +++ b/cmd/web-api/main.go @@ -66,8 +66,10 @@ func main() { // ========================================================================= // Logging + log.SetFlags(log.LstdFlags|log.Lmicroseconds|log.Lshortfile) + log.SetPrefix(service+" : ") + log := log.New(os.Stdout, log.Prefix() , log.Flags()) - log := log.New(os.Stdout, service+" : ", log.LstdFlags|log.Lmicroseconds|log.Lshortfile) // ========================================================================= // Configuration @@ -87,11 +89,11 @@ func main() { Service struct { Name string `default:"web-api" envconfig:"NAME"` Project string `default:"" envconfig:"PROJECT"` - BaseUrl string `default:"" envconfig:"BASE_URL" example:"http://api.eproc.tech"` - HostNames []string `envconfig:"HOST_NAMES" example:"alternative-subdomain.eproc.tech"` + BaseUrl string `default:"" envconfig:"BASE_URL" example:"http://api.example.saasstartupkit.com"` + HostNames []string `envconfig:"HOST_NAMES" example:"alternative-subdomain.example.saasstartupkit.com"` EnableHTTPS bool `default:"false" envconfig:"ENABLE_HTTPS"` TemplateDir string `default:"./templates" envconfig:"TEMPLATE_DIR"` - WebAppBaseUrl string `default:"http://127.0.0.1:3000" envconfig:"WEB_APP_BASE_URL" example:"www.eproc.tech"` + WebAppBaseUrl string `default:"http://127.0.0.1:3000" envconfig:"WEB_APP_BASE_URL" example:"www.example.saasstartupkit.com"` DebugHost string `default:"0.0.0.0:4000" envconfig:"DEBUG_HOST"` ShutdownTimeout time.Duration `default:"5s" envconfig:"SHUTDOWN_TIMEOUT"` } diff --git a/cmd/web-api/tests/user_test.go b/cmd/web-api/tests/user_test.go index d2778e6..4b55dd4 100644 --- a/cmd/web-api/tests/user_test.go +++ b/cmd/web-api/tests/user_test.go @@ -1378,7 +1378,7 @@ func TestUserToken(t *testing.T) { // Test user token with empty credentials. { - expectedStatus := http.StatusUnauthorized + expectedStatus := http.StatusBadRequest rt := requestTest{ fmt.Sprintf("Token %d using empty request", expectedStatus), @@ -1406,8 +1406,24 @@ func TestUserToken(t *testing.T) { expected := weberror.ErrorResponse{ StatusCode: expectedStatus, - Error: http.StatusText(expectedStatus), - Details: "must provide email and password in Basic auth", + Error: "Field validation error", + Fields: []weberror.FieldError{ + { + Field: "username", + Value: "", + Tag: "required", + Error: "username is a required field", + Display: "username is a required field", + }, + { + Field: "password", + Value: "", + Tag: "required", + Error: "password is a required field", + Display: "password is a required field", + }, + }, + Details: actual.Details, StackTrace: actual.StackTrace, } diff --git a/cmd/web-app/Dockerfile b/cmd/web-app/Dockerfile index 66ebe65..11ddfbf 100644 --- a/cmd/web-app/Dockerfile +++ b/cmd/web-app/Dockerfile @@ -13,8 +13,9 @@ ENV GO111MODULE="on" COPY go.mod . COPY go.sum . RUN go mod download +RUN go get github.com/pilu/fresh -FROM build_base_golang AS builder +FROM build_base_golang AS dev ARG service ARG commit_ref=- @@ -23,15 +24,22 @@ ARG commit_ref=- COPY internal ./internal # Copy cmd specific packages. -COPY cmd/${service} ./cmd/web-app +COPY cmd/${service} ./cmd/${service} COPY cmd/${service}/templates /templates COPY cmd/${service}/static /static # Copy the global templates. ADD resources/templates/shared /templates/shared +ADD fresh-auto-reload.conf /runner.conf + +ENV TEMPLATE_DIR=/templates WORKDIR ./cmd/${service} +ENTRYPOINT ["fresh", "-c", "/runner.conf"] + +FROM dev AS builder + RUN CGO_ENABLED=0 GOOS=linux go build -ldflags "-X main.build=${commit_ref}" -a -installsuffix nocgo -o /gosrv . FROM alpine:3.9 diff --git a/cmd/web-app/README.md b/cmd/web-app/README.md index fb718d7..83f75b8 100644 --- a/cmd/web-app/README.md +++ b/cmd/web-app/README.md @@ -24,7 +24,7 @@ http://127.0.0.1:3000/ While the web-api service has significant functionality, this web-app service is still in development. Currently this web-app services only resizes -an image and displays resvised versions of it on the index page. See section below on Future Functionality. +an image and displays resized versions of it on the index page. See section below on Future Functionality. If you would like to help, please email twins@geeksinthewoods.com. diff --git a/cmd/web-app/handlers/account.go b/cmd/web-app/handlers/account.go index 03a32d7..c2e3a0c 100644 --- a/cmd/web-app/handlers/account.go +++ b/cmd/web-app/handlers/account.go @@ -249,7 +249,7 @@ func (h *Account) Update(ctx context.Context, w http.ResponseWriter, r *http.Req return false, err } - data["geonameCountries"] = geonames.ValidGeonameCountries + data["geonameCountries"] = geonames.ValidGeonameCountries(ctx) data["countries"], err = geonames.FindCountries(ctx, h.MasterDB, "name", "") if err != nil { diff --git a/cmd/web-app/handlers/routes.go b/cmd/web-app/handlers/routes.go index e8e1373..e30a2c8 100644 --- a/cmd/web-app/handlers/routes.go +++ b/cmd/web-app/handlers/routes.go @@ -45,6 +45,7 @@ func APP(shutdown chan os.Signal, log *log.Logger, env webcontext.Env, staticDir // Build a sitemap. sm := stm.NewSitemap(1) + sm.SetVerbose(false) sm.SetDefaultHost(projectRoutes.WebAppUrl("")) sm.Create() diff --git a/cmd/web-app/handlers/signup.go b/cmd/web-app/handlers/signup.go index 0282567..f24a175 100644 --- a/cmd/web-app/handlers/signup.go +++ b/cmd/web-app/handlers/signup.go @@ -105,7 +105,7 @@ func (h *Signup) Step1(ctx context.Context, w http.ResponseWriter, r *http.Reque return nil } - data["geonameCountries"] = geonames.ValidGeonameCountries + data["geonameCountries"] = geonames.ValidGeonameCountries(ctx) data["countries"], err = geonames.FindCountries(ctx, h.MasterDB, "name", "") if err != nil { diff --git a/cmd/web-app/main.go b/cmd/web-app/main.go index 4fa82a1..046b695 100644 --- a/cmd/web-app/main.go +++ b/cmd/web-app/main.go @@ -66,8 +66,10 @@ func main() { // ========================================================================= // Logging + log.SetFlags(log.LstdFlags|log.Lmicroseconds|log.Lshortfile) + log.SetPrefix(service+" : ") + log := log.New(os.Stdout, log.Prefix() , log.Flags()) - log := log.New(os.Stdout, service+" : ", log.LstdFlags|log.Lmicroseconds|log.Lshortfile) // ========================================================================= // Configuration @@ -87,8 +89,8 @@ func main() { Service struct { Name string `default:"web-app" envconfig:"NAME"` Project string `default:"" envconfig:"PROJECT"` - BaseUrl string `default:"" envconfig:"BASE_URL" example:"http://eproc.tech"` - HostNames []string `envconfig:"HOST_NAMES" example:"www.eproc.tech"` + BaseUrl string `default:"" envconfig:"BASE_URL" example:"http://example.saasstartupkit.com"` + HostNames []string `envconfig:"HOST_NAMES" example:"www.example.saasstartupkit.com"` EnableHTTPS bool `default:"false" envconfig:"ENABLE_HTTPS"` TemplateDir string `default:"./templates" envconfig:"TEMPLATE_DIR"` SharedTemplateDir string `default:"../../resources/templates/shared" envconfig:"SHARED_TEMPLATE_DIR"` @@ -99,10 +101,10 @@ func main() { CloudFrontEnabled bool `envconfig:"CLOUDFRONT_ENABLED"` ImgResizeEnabled bool `envconfig:"IMG_RESIZE_ENABLED"` } - WebApiBaseUrl string `default:"http://127.0.0.1:3001" envconfig:"WEB_API_BASE_URL" example:"http://api.eproc.tech"` + WebApiBaseUrl string `default:"http://127.0.0.1:3001" envconfig:"WEB_API_BASE_URL" example:"http://api.example.saasstartupkit.com"` SessionKey string `default:"" envconfig:"SESSION_KEY"` SessionName string `default:"" envconfig:"SESSION_NAME"` - EmailSender string `default:"test@eproc.tech" envconfig:"EMAIL_SENDER"` + EmailSender string `default:"test@example.saasstartupkit.com" envconfig:"EMAIL_SENDER"` DebugHost string `default:"0.0.0.0:4000" envconfig:"DEBUG_HOST"` ShutdownTimeout time.Duration `default:"5s" envconfig:"SHUTDOWN_TIMEOUT"` } @@ -774,6 +776,9 @@ func main() { } imgUrlFormatter = func(p string) string { + if strings.HasPrefix(p, "http") { + return p + } baseUrl.Path = p return baseUrl.String() } @@ -818,7 +823,12 @@ func main() { tmplFuncs["S3ImgUrl"] = func(ctx context.Context, p string, size int) string { imgUrl := imgUrlFormatter(p) if cfg.Service.StaticFiles.ImgResizeEnabled { - imgUrl, _ = img_resize.S3ImgUrl(ctx, redisClient, staticS3UrlFormatter, awsSession, cfg.Aws.S3BucketPublic, imgResizeS3KeyPrefix, imgUrl, size) + var rerr error + imgUrl, rerr = img_resize.S3ImgUrl(ctx, redisClient, staticS3UrlFormatter, awsSession, cfg.Aws.S3BucketPublic, imgResizeS3KeyPrefix, imgUrl, size) + if rerr != nil { + imgUrl = "error" + log.Printf("main : S3ImgUrl : %s - %s\n", p, rerr) + } } return imgUrl } @@ -843,6 +853,10 @@ func main() { } } + if web.RequestIsImage(r) { + return err + } + switch statusCode { case http.StatusUnauthorized: http.Redirect(w, r, "/user/login?redirect="+url.QueryEscape(r.RequestURI), http.StatusFound) diff --git a/cmd/web-app/templates/content/account-update.gohtml b/cmd/web-app/templates/content/account-update.gohtml index 87daf92..5f8b176 100644 --- a/cmd/web-app/templates/content/account-update.gohtml +++ b/cmd/web-app/templates/content/account-update.gohtml @@ -3,7 +3,6 @@ {{end}} {{define "content"}} -