From 998d1bf13e6cae61ee8a9097a9379c3604d79476 Mon Sep 17 00:00:00 2001 From: Lee Brown Date: Thu, 22 Aug 2019 17:33:47 -0800 Subject: [PATCH] updates from review, WIP --- build/README.md | 7 ++ build/cicd/README.md | 7 ++ .../docker}/datadog-agent/Dockerfile | 0 .../docker}/datadog-agent/custom-init.sh | 0 cmd/README.md | 12 +++ cmd/web-app/main.go | 99 +++++++++++++++---- configs/README.md | 3 + deployments/README.md | 4 + docker-compose.yaml | 2 +- go.mod | 2 +- go.sum | 9 ++ internal/README.md | 5 + .../template-renderer/template_renderer.go | 2 - .../platform/web/webcontext/transvalidate.go | 21 +++- tools/README.md | 10 ++ 15 files changed, 158 insertions(+), 25 deletions(-) create mode 100644 build/README.md rename {docker => build/docker}/datadog-agent/Dockerfile (100%) rename {docker => build/docker}/datadog-agent/custom-init.sh (100%) create mode 100644 cmd/README.md create mode 100644 configs/README.md create mode 100644 deployments/README.md create mode 100644 internal/README.md create mode 100644 tools/README.md diff --git a/build/README.md b/build/README.md new file mode 100644 index 0000000..cde8eda --- /dev/null +++ b/build/README.md @@ -0,0 +1,7 @@ +# `/build` + +Packaging and Continuous Integration. + +Put your cloud (AMI), container (Docker), OS (deb, rpm, pkg) package configurations and scripts in the /build/package +directory. + diff --git a/build/cicd/README.md b/build/cicd/README.md index 35f1d9f..4dd069a 100644 --- a/build/cicd/README.md +++ b/build/cicd/README.md @@ -200,6 +200,13 @@ If you have a lot of migrations, it can be a pain to run all them. For example, the app into a clean database. To prevent this, use the initSchema function that will run as-if no migration was run before (in a new clean database). + +**Migrations should be backwards compatible with the existing deployed code.** Refrain from `drop table`. Instead of +renaming columns, add a new column and copy the data from the old column using an `update`. + +Ideally migrations should be idempotent to avoid possible data loss since data could have been generated between +migration runs. + Another bonus with the globally defined schema is that it enables your testing package the ability to dynamically [spin up database containers](https://gitlab.com/geeks-accelerator/oss/saas-starter-kit/blob/issue8/datadog-lambda-func/internal/platform/tests/main.go#L127) on-demand and automatically include all the migrations. This allows the testing package to diff --git a/docker/datadog-agent/Dockerfile b/build/docker/datadog-agent/Dockerfile similarity index 100% rename from docker/datadog-agent/Dockerfile rename to build/docker/datadog-agent/Dockerfile diff --git a/docker/datadog-agent/custom-init.sh b/build/docker/datadog-agent/custom-init.sh similarity index 100% rename from docker/datadog-agent/custom-init.sh rename to build/docker/datadog-agent/custom-init.sh diff --git a/cmd/README.md b/cmd/README.md new file mode 100644 index 0000000..81cb194 --- /dev/null +++ b/cmd/README.md @@ -0,0 +1,12 @@ +# `/cmd` + +Main applications for this project. + +The directory name for each application should match the name of the executable you want to have (e.g., `/cmd/myapp`). + +Don't put a lot of code in the application directory. If you think the code can be imported and used in other projects, +then it should live in the `/pkg` directory. If the code is not reusable or if you don't want others to reuse it, +put that code in the `/internal` directory. You'll be surprised what others will do, so be explicit about your intentions! + +It's common to have a small `main` function that imports and invokes the code from the `/internal` and `/pkg` +directories and nothing else. diff --git a/cmd/web-app/main.go b/cmd/web-app/main.go index 852f863..997b054 100644 --- a/cmd/web-app/main.go +++ b/cmd/web-app/main.go @@ -567,11 +567,13 @@ func main() { // Need defined functions below since they require config values, able to add additional functions // here to extend functionality. tmplFuncs := template.FuncMap{ + // BuildInfo returns the specific key from BuildInfo set in the current config. "BuildInfo": func(k string) string { r := reflect.ValueOf(cfg.BuildInfo) f := reflect.Indirect(r).FieldByName(k) return f.String() }, + // SiteBaseUrl returns the absolute URL for a relative path using the base site URL set in the config. "SiteBaseUrl": func(p string) string { u, err := url.Parse(cfg.Service.BaseUrl) if err != nil { @@ -580,21 +582,10 @@ func main() { u.Path = p return u.String() }, - "AssetUrl": func(p string) string { - var u string - if staticUrlFormatter != nil { - u = staticUrlFormatter(p) - } else { - if !strings.HasPrefix(p, "/") { - p = "/" + p - } - u = p - } - - u = browserCacheBusterFunc(u) - - return u - }, + // SiteAssetUrl returns the absolute URL for a relative path that either served locally, from the public S3 + // bucket or from Cloudfront depending on the current config settings for StaticFiles. The defined static asset + // prefix to the path defined in the current config settings for StaticFiles. The response URL includes a cache + // busting query param based on the current commit. "SiteAssetUrl": func(p string) string { var u string if staticUrlFormatter != nil { @@ -610,6 +601,8 @@ func main() { return u }, + // SiteS3Url returns the URL that for an S3 Key path that has been uploaded to S3 to the specific public s3 key + // prefix that returns the absolute S3 URL or the CloudFront equivalent if enabled. "SiteS3Url": func(p string) string { var u string if staticUrlFormatter != nil { @@ -619,6 +612,8 @@ func main() { } return u }, + // S3Url returns the URL that for an S3 Key path that has been uploaded to S3 to the public s3 key + // prefix that returns the absolute S3 URL or the CloudFront equivalent if enabled. "S3Url": func(p string) string { var u string if staticUrlFormatter != nil { @@ -628,6 +623,7 @@ func main() { } return u }, + // ValidationErrorHasField checks if the error is a validation error and has an error for the specified field. "ValidationErrorHasField": func(err interface{}, fieldName string) bool { if err == nil { return false @@ -643,6 +639,7 @@ func main() { } return false }, + // ValidationFieldErrors returns the list of validation field errors. "ValidationFieldErrors": func(err interface{}, fieldName string) []weberror.FieldError { if err == nil { return []weberror.FieldError{} @@ -659,6 +656,7 @@ func main() { } return l }, + // ValidationFieldClass returns a CSS class for validation if the field exists in the validation errors. "ValidationFieldClass": func(err interface{}, fieldName string) string { if err == nil { return "" @@ -675,6 +673,7 @@ func main() { } return "is-valid" }, + // ErrorMessage returns the error message that is formatted as a response for end users to consume. "ErrorMessage": func(ctx context.Context, err error) string { werr, ok := err.(*weberror.Error) if ok { @@ -685,6 +684,7 @@ func main() { } return fmt.Sprintf("%s", err) }, + // ErrorDetails returns the full error for dev and stage envs to help with debugging. "ErrorDetails": func(ctx context.Context, err error) string { var displayFullError bool switch webcontext.ContextEnv(ctx) { @@ -707,8 +707,7 @@ func main() { return fmt.Sprintf("%+v", err) }, - // Returns the current user from the session. - // @TODO: Need to add logging for the errors. + // ContextUser returns the current user and caches the result. "ContextUser": func(ctx context.Context) *user.UserResponse { sess := webcontext.ContextSession(ctx) @@ -716,6 +715,7 @@ func main() { u := &user.UserResponse{} if err := redisClient.Get(cacheKey).Scan(u); err != nil && err != redis.Nil { + log.Printf("main : ContextUser : Redis Get failed - %+v ", err.Error()) return nil } @@ -737,13 +737,13 @@ func main() { err = redisClient.Set(cacheKey, u, time.Hour).Err() if err != nil { + log.Printf("main : ContextAccount : Redis Set failed - %+v ", err.Error()) return nil } return u }, - // Returns the current account from the session. - // @TODO: Need to add logging for the errors. + // ContextAccount returns the account for current context user with auth is using and caches the result. "ContextAccount": func(ctx context.Context) *account.AccountResponse { sess := webcontext.ContextSession(ctx) @@ -751,6 +751,7 @@ func main() { a := &account.AccountResponse{} if err := redisClient.Get(cacheKey).Scan(a); err != nil && err != redis.Nil { + log.Printf("main : ContextAccount : Redis Get failed - %+v ", err.Error()) return nil } @@ -772,11 +773,13 @@ func main() { err = redisClient.Set(cacheKey, a, time.Hour).Err() if err != nil { + log.Printf("main : ContextAccount : Redis Set failed - %+v ", err.Error()) return nil } return a }, + // ContextCanSwitchAccount returns if the current context user has multiple accounts. "ContextCanSwitchAccount": func(ctx context.Context) bool { claims, err := auth.ClaimsFromContext(ctx) if err != nil || len(claims.AccountIDs) < 2 { @@ -784,6 +787,7 @@ func main() { } return true }, + // ContextIsVirtualSession returns true if the current context user with auth is in a virtual session. "ContextIsVirtualSession": func(ctx context.Context) bool { claims, err := auth.ClaimsFromContext(ctx) if err != nil { @@ -797,6 +801,63 @@ func main() { } return false }, + // T creates the translation for the locale given the 'key' and params passed in. + "T": func(ctx context.Context, key string, params ...string) string { + trns := webcontext.ContextTranslator(ctx) + if trns == nil { + return key + } + + res, err := trns.T(key, params...) + if err != nil { + log.Printf("main : Translate.T : Failed to translate %s with '%s' - %+v", trns.Locale(), key, err.Error()) + return key + } + return res + }, + // C creates the cardinal translation for the locale given the 'key', 'num' and 'digit' arguments and param passed in + "C": func(ctx context.Context, key string, num float64, digits uint64, param string) string { + trns := webcontext.ContextTranslator(ctx) + if trns == nil { + return key + } + + res, err := trns.C(key, num, digits, param) + if err != nil { + log.Printf("main : Translate.C : Failed to translate %s with '%s' - %+v", trns.Locale(), key, err.Error()) + return key + } + return res + }, + // O creates the ordinal translation for the locale given the 'key', 'num' and 'digit' arguments and param passed in + "O": func(ctx context.Context, key string, num float64, digits uint64, param string) string { + trns := webcontext.ContextTranslator(ctx) + if trns == nil { + return key + } + + res, err := trns.O(key, num, digits, param) + if err != nil { + log.Printf("main : Translate.O : Failed to translate %s with '%s' - %+v", trns.Locale(), key, err.Error()) + return key + } + return res + }, + // R creates the range translation for the locale given the 'key', 'num1', 'digit1', 'num2' and 'digit2' + // arguments and 'param1' and 'param2' passed in + "R": func(ctx context.Context, key string, num1 float64, digits1 uint64, num2 float64, digits2 uint64, param1, param2 string) string { + trns := webcontext.ContextTranslator(ctx) + if trns == nil { + return key + } + + res, err := trns.R(key, num1, digits1, num2, digits2, param1, param2) + if err != nil { + log.Printf("main : Translate.R : Failed to translate %s with '%s' - %+v", trns.Locale(), key, err.Error()) + return key + } + return res + }, } imgUrlFormatter := staticUrlFormatter diff --git a/configs/README.md b/configs/README.md new file mode 100644 index 0000000..b21ebc2 --- /dev/null +++ b/configs/README.md @@ -0,0 +1,3 @@ +# `/configs` + +Configuration file templates or default configs. diff --git a/deployments/README.md b/deployments/README.md new file mode 100644 index 0000000..0eecc6a --- /dev/null +++ b/deployments/README.md @@ -0,0 +1,4 @@ +# `/deployments` + +IaaS, PaaS, system and container orchestration deployment configurations and templates (docker-compose, kubernetes/helm, +mesos, terraform, bosh). \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index d043df6..b11277c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -38,7 +38,7 @@ services: datadog: image: example-project/datadog:latest build: - context: docker/datadog-agent + context: build/docker/datadog-agent dockerfile: Dockerfile ports: - 8125:8125 # metrics diff --git a/go.mod b/go.mod index 6d41d7e..5e6456c 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/tinylib/msgp v1.1.0 // indirect github.com/urfave/cli v1.21.0 github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 - gitlab.com/geeks-accelerator/oss/devops v1.0.3 + gitlab.com/geeks-accelerator/oss/devops v1.0.7 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 golang.org/x/tools v0.0.0-20190807223507-b346f7fd45de // indirect diff --git a/go.sum b/go.sum index 72166fe..108cb06 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= +github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk= github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -78,10 +79,14 @@ github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8w github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/gobuffalo/envy v1.7.0 h1:GlXgaiBkmrYMHco6t4j7SacKO4XUjvh5pwXh0f4uxXU= github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= github.com/gobuffalo/logger v1.0.0/go.mod h1:2zbswyIUa45I+c+FLXuWl9zSWEiVuthsk8ze5s8JvPs= +github.com/gobuffalo/logger v1.0.1 h1:ZEgyRGgAm4ZAhAO45YXMs5Fp+bzGLESFewzAVBMKuTg= github.com/gobuffalo/logger v1.0.1/go.mod h1:2zbswyIUa45I+c+FLXuWl9zSWEiVuthsk8ze5s8JvPs= +github.com/gobuffalo/packd v0.3.0 h1:eMwymTkA1uXsqxS0Tpoop3Lc0u3kTfiMBE6nKtQU4g4= github.com/gobuffalo/packd v0.3.0/go.mod h1:zC7QkmNkYVGKPw4tHpBQ+ml7W/3tIebgeo1b36chA3Q= +github.com/gobuffalo/packr/v2 v2.5.2 h1:4EvjeIpQLZuRIljwnidYgbRXbr1yIzVRrESiLjqKj6s= github.com/gobuffalo/packr/v2 v2.5.2/go.mod h1:sgEE1xNZ6G0FNN5xn9pevVu4nywaxHvgup67xisti08= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -117,6 +122,7 @@ github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhB github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/karrick/godirwalk v1.10.12 h1:BqUm+LuJcXjGv1d2mj3gBiQyrQ57a0rYoAmhvJQ7RDU= github.com/karrick/godirwalk v1.10.12/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= @@ -170,6 +176,7 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0 h1:X9XMOYjxEfAYSy3xK1DzO5dMkkWhs9E9UCcS1IERx2k= @@ -208,6 +215,8 @@ github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 h1:zzrxE1FKn5ryB github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2/go.mod h1:hzfGeIUDq/j97IG+FhNqkowIyEcD88LrW6fyU3K3WqY= gitlab.com/geeks-accelerator/oss/devops v1.0.3 h1:SE2ZD4Csvmm3t/50RoJkVLjDcwXKHayQYawSkpOSqIw= gitlab.com/geeks-accelerator/oss/devops v1.0.3/go.mod h1:rvI71qNJyNiO99ZgGnv/PmJCVrjJjupsXBmfYFXdjGM= +gitlab.com/geeks-accelerator/oss/devops v1.0.7 h1:ZlQufuVnRN3DwJ0I5c5KA5edhQs7OstXc0uUZ9V0ixI= +gitlab.com/geeks-accelerator/oss/devops v1.0.7/go.mod h1:JEl0T87/zftowrIzY1D+rhDMhG0AxnghuZB+VzEWuqM= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/internal/README.md b/internal/README.md new file mode 100644 index 0000000..42b1813 --- /dev/null +++ b/internal/README.md @@ -0,0 +1,5 @@ +# `/internal` + +Private application and library code. This is the code you don't want others importing in their applications or libraries. + +You can optionally add a bit of extra structure to your internal packages to separate your shared and non-shared internal code. It's not required, but it's nice to have visual clues showing the intended package use. Your actual application code can go in the /internal/app directory (e.g., /internal/app/myapp) and the code shared by those apps in the /internal/pkg directory (e.g., /internal/pkg/myprivlib). diff --git a/internal/platform/web/template-renderer/template_renderer.go b/internal/platform/web/template-renderer/template_renderer.go index efe84f5..dd8a546 100644 --- a/internal/platform/web/template-renderer/template_renderer.go +++ b/internal/platform/web/template-renderer/template_renderer.go @@ -373,8 +373,6 @@ func (r *TemplateRenderer) Render(ctx context.Context, w http.ResponseWriter, re data["error"] = terr } - data["trans"] = webcontext.ContextTranslator(ctx) - // Append request data map to render data last so any previous value can be overwritten. if data != nil { for k, v := range data { diff --git a/internal/platform/web/webcontext/transvalidate.go b/internal/platform/web/webcontext/transvalidate.go index 3c174c5..59469cd 100644 --- a/internal/platform/web/webcontext/transvalidate.go +++ b/internal/platform/web/webcontext/transvalidate.go @@ -4,6 +4,7 @@ import ( "context" "log" "os" + "path/filepath" "reflect" "strings" @@ -65,8 +66,24 @@ func init() { // Provide one or more arguments for additional supported locales. uniTrans = ut.New(en, en, fr, id, ja, nl, zh) - if _, err := os.Stat("templates/content/translations"); !os.IsNotExist(err) { - err := uniTrans.Import(ut.FormatJSON, "templates/content/translations") + // Try to load the Template directory from an environment variable for release images else it's local development. + var transDir string + if ev := os.Getenv("TRANSLATIONS_DIR"); ev != "" { + transDir = ev + } else { + if ev := os.Getenv("SHARED_TEMPLATE_DIR"); ev != "" { + transDir = ev + } else if ev := os.Getenv("TEMPLATE_DIR"); ev != "" { + transDir = ev + } else { + transDir = "./templates" + } + + transDir = filepath.Join(transDir, "translations") + } + + if _, err := os.Stat(transDir); !os.IsNotExist(err) { + err := uniTrans.Import(ut.FormatJSON, transDir) if err != nil { log.Fatal(err) } diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 0000000..32ef14c --- /dev/null +++ b/tools/README.md @@ -0,0 +1,10 @@ +# `/tools` + +Supporting tools for this project. Note that these tools can import code from the /pkg and /internal directories. + +Current Tools: +* [schema](https://gitlab.com/geeks-accelerator/oss/saas-starter-kit/tree/master/tools/schema) - A command line tool for +local development that executes database migrations. + +* [text-translator](https://gitlab.com/geeks-accelerator/oss/saas-starter-kit/tree/master/tools/text-translator) - A +command line tool for supporting internazionalied go-templates.