diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..471328d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +aws.lee diff --git a/example-project/cmd/web-app/main.go b/example-project/cmd/web-app/main.go new file mode 100644 index 0000000..4fa3d18 --- /dev/null +++ b/example-project/cmd/web-app/main.go @@ -0,0 +1,419 @@ +package main + +import ( + "context" + "encoding/json" + "expvar" + "fmt" + "log" + "net/http" + _ "net/http/pprof" + "net/url" + "os" + "os/signal" + "path/filepath" + "reflect" + "strings" + "syscall" + "time" + "html/template" + + template_renderer "geeks-accelerator/oss/saas-starter-kit/example-project/internal/platform/web/template-renderer" + lru "github.com/hashicorp/golang-lru" + "geeks-accelerator/oss/saas-starter-kit/example-project/internal/platform/web" + "geeks-accelerator/oss/saas-starter-kit/example-project/cmd/web-app/handlers" + "geeks-accelerator/oss/saas-starter-kit/example-project/internal/platform/flag" + itrace "geeks-accelerator/oss/saas-starter-kit/example-project/internal/platform/trace" + "github.com/kelseyhightower/envconfig" + "go.opencensus.io/trace" +) + +// build is the git version of this program. It is set using build flags in the makefile. +var build = "develop" + +const LRU_CACHE_ITEMS = 128 + +var ( + localCache *lru.Cache +) + +func init() { + localCache, _ = lru.New(LRU_CACHE_ITEMS) +} + +func main() { + + // ========================================================================= + // Logging + + log := log.New(os.Stdout, "WEB_APP : ", log.LstdFlags|log.Lmicroseconds|log.Lshortfile) + + // ========================================================================= + // Configuration + var cfg struct { + Env string `default:"dev" envconfig:"ENV"` + HTTP struct { + Host string `default:"0.0.0.0:3000" envconfig:"HTTP_HOST"` + DebugHost string `default:"0.0.0.0:4000" envconfig:"DEBUG_HOST"` + ReadTimeout time.Duration `default:"5s" envconfig:"READ_TIMEOUT"` + WriteTimeout time.Duration `default:"5s" envconfig:"WRITE_TIMEOUT"` + ShutdownTimeout time.Duration `default:"5s" envconfig:"SHUTDOWN_TIMEOUT"` + TemplateDir string `default:"./templates" envconfig:"TEMPLATE_DIR"` + StaticDir string `default:"./static" envconfig:"STATIC_DIR"` + } + App struct { + Name string `default:"web-app" envconfig:"APP_NAME"` + StaticS3 struct { + S3Bucket string `envconfig:"APP_STATIC_S3_BUCKET"` + S3KeyPrefix string `envconfig:"APP_STATIC_S3_KEY_PREFIX"` + EnableCloudFront bool `envconfig:"APP_STATIC_S3_ENABLE_CLOUDFRONT"` + } + } + BuildInfo struct { + CiCommitRefName string `envconfig:"CI_COMMIT_REF_NAME"` + CiCommitRefSlug string `envconfig:"CI_COMMIT_REF_SLUG"` + CiCommitSha string `envconfig:"CI_COMMIT_SHA"` + CiCommitTag string `envconfig:"CI_COMMIT_TAG"` + CiCommitTitle string `envconfig:"CI_COMMIT_TITLE"` + CiCommitDescription string `envconfig:"CI_COMMIT_DESCRIPTION"` + CiJobId string `envconfig:"CI_COMMIT_JOB_ID"` + CiJobUrl string `envconfig:"CI_COMMIT_JOB_URL"` + CiPipelineId string `envconfig:"CI_COMMIT_PIPELINE_ID"` + CiPipelineUrl string `envconfig:"CI_COMMIT_PIPELINE_URL"` + } + DB struct { + DialTimeout time.Duration `default:"5s" envconfig:"DIAL_TIMEOUT"` + Host string `default:"mongo:27017/gotraining" envconfig:"HOST"` + } + Trace struct { + Host string `default:"http://tracer:3002/v1/publish" envconfig:"HOST"` + BatchSize int `default:"1000" envconfig:"BATCH_SIZE"` + SendInterval time.Duration `default:"15s" envconfig:"SEND_INTERVAL"` + SendTimeout time.Duration `default:"500ms" envconfig:"SEND_TIMEOUT"` + } + Auth struct { + KeyID string `envconfig:"KEY_ID"` + PrivateKeyFile string `default:"/app/private.pem" envconfig:"PRIVATE_KEY_FILE"` + Algorithm string `default:"RS256" envconfig:"ALGORITHM"` + } + } + + if err := envconfig.Process("WEB_APP", &cfg); err != nil { + log.Fatalf("main : Parsing Config : %v", err) + } + + if err := flag.Process(&cfg); err != nil { + if err != flag.ErrHelp { + log.Fatalf("main : Parsing Command Line : %v", err) + } + return // We displayed help. + } + + // ========================================================================= + // App Starting + + // Print the build version for our logs. Also expose it under /debug/vars. + expvar.NewString("build").Set(build) + log.Printf("main : Started : Application Initializing version %q", build) + defer log.Println("main : Completed") + + cfgJSON, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + log.Fatalf("main : Marshalling Config to JSON : %v", err) + } + + // TODO: Validate what is being written to the logs. We don't + // want to leak credentials or anything that can be a security risk. + log.Printf("main : Config : %v\n", string(cfgJSON)) + + // ========================================================================= + // Template Renderer + // Implements interface web.Renderer to support alternative renderer + + var ( + staticS3BaseUrl string + staticS3CloudFrontOriginPrefix string + ) + if cfg.App.StaticS3.S3Bucket != "" { + // TODO: lookup s3 url/cloud front distribution based on s3 bucket + } + + // Append query string value to break browser cache used for services + // that render responses for a browser with the following: + // 1. when env=dev, the current timestamp will be used to ensure every + // request will skip browser cache. + // 2. all other envs, ie stage and prod. The commit hash will be used to + // ensure that all cache will be reset with each new deployment. + browserCacheBusterQueryString := func() string { + var v string + if cfg.Env == "dev" { + // On dev always break cache. + v = fmt.Sprintf("%d", time.Now().UTC().Unix()) + } else { + // All other envs, use the current commit hash for the build + v = cfg.BuildInfo.CiCommitSha + } + return v + } + + // Helper method for appending the browser cache buster as a query string to + // support breaking browser cache when necessary + browserCacheBusterFunc := browserCacheBuster(browserCacheBusterQueryString) + + // Need defined functions below since they require config values, able to add additional functions + // here to extend functionality. + tmplFuncs := template.FuncMap{ + "BuildInfo": func(k string) string { + r := reflect.ValueOf(cfg.BuildInfo) + f := reflect.Indirect(r).FieldByName(k) + return f.String() + }, + "SiteBaseUrl": func(p string) string { + u, err := url.Parse(cfg.HTTP.Host) + if err != nil { + return "?" + } + u.Path = p + return u.String() + }, + "AssetUrl": func(p string) string { + var u string + if staticS3BaseUrl != "" { + u = template_renderer.S3Url(staticS3BaseUrl, staticS3CloudFrontOriginPrefix, p) + } else { + if !strings.HasPrefix(p, "/") { + p = "/" + p + } + u = p + } + + u = browserCacheBusterFunc( u) + + return u + }, + "SiteAssetUrl": func(p string) string { + var u string + if staticS3BaseUrl != "" { + u = template_renderer.S3Url(staticS3BaseUrl, staticS3CloudFrontOriginPrefix, filepath.Join(cfg.App.Name, p)) + } else { + if !strings.HasPrefix(p, "/") { + p = "/" + p + } + u = p + } + + u = browserCacheBusterFunc( u) + + return u + }, + "SiteS3Url": func(p string) string { + var u string + if staticS3BaseUrl != "" { + u = template_renderer.S3Url(staticS3BaseUrl, staticS3CloudFrontOriginPrefix, filepath.Join(cfg.App.Name, p)) + } else { + u = p + } + return u + }, + "S3Url": func(p string) string { + var u string + if staticS3BaseUrl != "" { + u = template_renderer.S3Url(staticS3BaseUrl, staticS3CloudFrontOriginPrefix, p) + } else { + u = p + } + return u + }, + } + + // + t := template_renderer.NewTemplate(tmplFuncs) + + // global variables exposed for rendering of responses with templates + gvd := map[string]interface{}{ + "_App": map[string]interface{}{ + "ENV": cfg.Env, + "BuildInfo": cfg.BuildInfo, + "BuildVersion": build, + }, + } + + // Custom error handler to support rendering user friendly error page for improved web experience. + eh := func(ctx context.Context, w http.ResponseWriter, r *http.Request, renderer web.Renderer, statusCode int, er error) error { + data := map[string]interface{}{} + + return renderer.Render(ctx, w, r, + "base.tmpl", // base layout file to be used for rendering of errors + "error.tmpl", // generic format for errors, could select based on status code + web.MIMETextHTMLCharsetUTF8, + http.StatusOK, + data, + ) + } + + // Enable template renderer to reload and parse template files when generating a response of dev + // for a more developer friendly process. Any changes to the template files will be included + // without requiring re-build/re-start of service. + // This only supports files that already exist, if a new template file is added, then the + // serivce needs to be restarted, but not rebuilt. + enableHotReload := cfg.Env == "dev" + + // Template Renderer used to generate HTML response for web experience. + renderer, err := template_renderer.NewTemplateRenderer(cfg.HTTP.TemplateDir, enableHotReload, gvd, t, eh) + if err != nil { + log.Fatalf("main : Marshalling Config to JSON : %v", err) + } + + // ========================================================================= + // Start Tracing Support + + logger := func(format string, v ...interface{}) { + log.Printf(format, v...) + } + + log.Printf("main : Tracing Started : %s", cfg.Trace.Host) + exporter, err := itrace.NewExporter(logger, cfg.Trace.Host, cfg.Trace.BatchSize, cfg.Trace.SendInterval, cfg.Trace.SendTimeout) + if err != nil { + log.Fatalf("main : RegiTracingster : ERROR : %v", err) + } + defer func() { + log.Printf("main : Tracing Stopping : %s", cfg.Trace.Host) + batch, err := exporter.Close() + if err != nil { + log.Printf("main : Tracing Stopped : ERROR : Batch[%d] : %v", batch, err) + } else { + log.Printf("main : Tracing Stopped : Flushed Batch[%d]", batch) + } + }() + + trace.RegisterExporter(exporter) + trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()}) + + // ========================================================================= + // Start Debug Service. Not concerned with shutting this down when the + // application is being shutdown. + // + // /debug/vars - Added to the default mux by the expvars package. + // /debug/pprof - Added to the default mux by the net/http/pprof package. + if cfg.HTTP.DebugHost != "" { + go func() { + log.Printf("main : Debug Listening %s", cfg.HTTP.DebugHost) + log.Printf("main : Debug Listener closed : %v", http.ListenAndServe(cfg.HTTP.DebugHost, http.DefaultServeMux)) + }() + } + + // ========================================================================= + // Start APP Service + + // Make a channel to listen for an interrupt or terminate signal from the OS. + // Use a buffered channel because the signal package requires it. + shutdown := make(chan os.Signal, 1) + signal.Notify(shutdown, os.Interrupt, syscall.SIGTERM) + + api := http.Server{ + Addr: cfg.HTTP.Host, + Handler: handlers.APP(shutdown, log, cfg.HTTP.StaticDir, cfg.HTTP.TemplateDir, nil, nil, renderer), + ReadTimeout: cfg.HTTP.ReadTimeout, + WriteTimeout: cfg.HTTP.WriteTimeout, + MaxHeaderBytes: 1 << 20, + } + + // Make a channel to listen for errors coming from the listener. Use a + // buffered channel so the goroutine can exit if we don't collect this error. + serverErrors := make(chan error, 1) + + // Start the service listening for requests. + go func() { + log.Printf("main : APP Listening %s", cfg.HTTP.Host) + serverErrors <- api.ListenAndServe() + }() + + // ========================================================================= + // Shutdown + + // Blocking main and waiting for shutdown. + select { + case err := <-serverErrors: + log.Fatalf("main : Error starting server: %v", err) + + case sig := <-shutdown: + log.Printf("main : %v : Start shutdown..", sig) + + // Create context for Shutdown call. + ctx, cancel := context.WithTimeout(context.Background(), cfg.HTTP.ShutdownTimeout) + defer cancel() + + // Asking listener to shutdown and load shed. + err := api.Shutdown(ctx) + if err != nil { + log.Printf("main : Graceful shutdown did not complete in %v : %v", cfg.HTTP.ShutdownTimeout, err) + err = api.Close() + } + + // Log the status of this shutdown. + switch { + case sig == syscall.SIGSTOP: + log.Fatal("main : Integrity issue caused shutdown") + case err != nil: + log.Fatalf("main : Could not stop server gracefully : %v", err) + } + } +} + +// browserCacheBuster appends a the query string param v to a given url with +// a value based on the value returned from cacheBusterValueFunc +func browserCacheBuster(cacheBusterValueFunc func() string) func(uri string) string { + f := func(uri string) string { + v := cacheBusterValueFunc() + if v == "" { + return uri + } + + u, err := url.Parse(uri) + if err != nil { + return "" + } + q := u.Query() + q.Set("v", v) + u.RawQuery = q.Encode() + + return u.String() + } + + return f +} + +/* + "S3ImgSrcLarge": func(p string) template.HTMLAttr { + res, _ := blower_display.S3ImgSrc(cfg, site, p, []int{320, 480, 800}, true) + return template.HTMLAttr(res) + }, + "S3ImgThumbSrcLarge": func(p string) template.HTMLAttr { + res, _ := blower_display.S3ImgSrc(cfg, site, p, []int{320, 480, 800}, false) + return template.HTMLAttr(res) + }, + "S3ImgSrcMedium": func(p string) template.HTMLAttr { + res, _ := blower_display.S3ImgSrc(cfg, site, p, []int{320, 640}, true) + return template.HTMLAttr(res) + }, + "S3ImgThumbSrcMedium": func(p string) template.HTMLAttr { + res, _ := blower_display.S3ImgSrc(cfg, site, p, []int{320, 640}, false) + return template.HTMLAttr(res) + }, + "S3ImgSrcSmall": func(p string) template.HTMLAttr { + res, _ := blower_display.S3ImgSrc(cfg, site, p, []int{320}, true) + return template.HTMLAttr(res) + }, + "S3ImgThumbSrcSmall": func(p string) template.HTMLAttr { + res, _ := blower_display.S3ImgSrc(cfg, site, p, []int{320}, false) + return template.HTMLAttr(res) + }, + "S3ImgSrc": func(p string, sizes []int) template.HTMLAttr { + res, _ := blower_display.S3ImgSrc(cfg, site, p, sizes, true) + return template.HTMLAttr(res) + }, + "S3ImgUrl": func(p string, size int) string { + res, _ := blower_display.S3ImgUrl(cfg, site, p, size) + return res + }, + */ \ No newline at end of file diff --git a/example-project/internal/platform/web/template-renderer/template_renderer.go b/example-project/internal/platform/web/template-renderer/template_renderer.go index 47bf051..8216369 100644 --- a/example-project/internal/platform/web/template-renderer/template_renderer.go +++ b/example-project/internal/platform/web/template-renderer/template_renderer.go @@ -20,10 +20,6 @@ import ( var ( errInvalidTemplate = errors.New("Invalid template") - - // Base template to support applying custom - // TODO try to remove this - //mainTmpl = `{{define "main" }} {{ template "base" . }} {{ end }}` ) type Template struct { @@ -31,13 +27,15 @@ type Template struct { mainTemplate *template.Template } - +// NewTemplate defines a base set of functions that will be applied to all templates +// being rendered. func NewTemplate(templateFuncs template.FuncMap) *Template { t := &Template{} - // these functions are used and rendered on run-time of web page so don't have to use javascript/jquery - // to for basic template formatting. transformation happens server-side instead of client-side to - // provide base-level consistency. + // Default functions are defined and available for all templates being rendered. + // These base function help with provided basic formatting so don't have to use javascript/jquery, + // transformation happens server-side instead of client-side to provide base-level consistency. + // Any defined function below will be overwritten if a matching function key is included. t.Funcs = template.FuncMap{ // probably could provide examples of each of these "Minus": func(a, b int) int { @@ -125,10 +123,15 @@ type TemplateRenderer struct { enableHotReload bool templates map[string]*template.Template globalViewData map[string]interface{} - //mainTemplate *template.Template + mainTemplate *template.Template errorHandler func(ctx context.Context, w http.ResponseWriter, req *http.Request, renderer web.Renderer, statusCode int, er error) error } +// NewTemplateRenderer implements the interface web.Renderer allowing for execution of +// nested html templates. The templateDir should include three directories: +// 1. layouts: base layouts defined for the entire application +// 2. content: page specific templates that will be nested instead of a layout template +// 3. partials: templates used by multiple layout or content templates func NewTemplateRenderer(templateDir string, enableHotReload bool, globalViewData map[string]interface{}, tmpl *Template, errorHandler func(ctx context.Context, w http.ResponseWriter, req *http.Request, renderer web.Renderer, statusCode int, er error) error) (*TemplateRenderer, error) { r := &TemplateRenderer{ templateDir: templateDir, @@ -141,13 +144,12 @@ func NewTemplateRenderer(templateDir string, enableHotReload bool, globalViewDat errorHandler: errorHandler, } - //r.mainTemplate = template.New("main") - //r.mainTemplate, _ = r.mainTemplate.Parse(mainTmpl) - //r.mainTemplate.Funcs(tmpl.Funcs) - + // Recursively loop through all folders/files in the template directory and group them by their + // template type. They are filename / filepath for lookup on render. err := filepath.Walk(templateDir, func(path string, info os.FileInfo, err error) error { dir := filepath.Base(filepath.Dir(path)) + // Skip directories. if info.IsDir() { return nil } @@ -168,63 +170,79 @@ func NewTemplateRenderer(templateDir string, enableHotReload bool, globalViewDat return r, err } + // Main template used to render execute all templates against. + r.mainTemplate = template.New("main") + r.mainTemplate, _ = r.mainTemplate.Parse( `{{define "main" }} {{ template "base" . }} {{ end }}`) + r.mainTemplate.Funcs(tmpl.Funcs) + // Ensure all layout files render successfully with no errors. for _, f := range r.layoutFiles { - //t, err := r.mainTemplate.Clone() - //if err != nil { - // return r, err - //} - t := template.New("main") - t.Funcs(tmpl.Funcs) + t, err := r.mainTemplate.Clone() + if err != nil { + return r, err + } template.Must(t.ParseFiles(f)) } // Ensure all partial files render successfully with no errors. for _, f := range r.partialFiles { - //t, err := r.mainTemplate.Clone() - //if err != nil { - // return r, err - //} - t := template.New("partial") - t.Funcs(tmpl.Funcs) + t, err := r.mainTemplate.Clone() + if err != nil { + return r, err + } template.Must(t.ParseFiles(f)) } // Ensure all content files render successfully with no errors. for _, f := range r.contentFiles { - //t, err := r.mainTemplate.Clone() - //if err != nil { - // return r, err - //} - t := template.New("content") - t.Funcs(tmpl.Funcs) + t, err := r.mainTemplate.Clone() + if err != nil { + return r, err + } template.Must(t.ParseFiles(f)) } return r, nil } -// Render renders a template document +// Render executes the nested templates and returns the result to the client. +// contentType: supports any content type to allow for rendering text, emails and other formats +// statusCode: the error method calls this function so allow the HTTP Status Code to be set +// data: map[string]interface{} to allow including additional request and globally defined values. func (r *TemplateRenderer) Render(ctx context.Context, w http.ResponseWriter, req *http.Request, templateLayoutName, templateContentName, contentType string, statusCode int, data map[string]interface{}) error { - + // If the template has not been rendered yet or hot reload is enabled, + // then parse the template files. t, ok := r.templates[templateContentName] if !ok || r.enableHotReload { + t, err := r.mainTemplate.Clone() + if err != nil { + return err + } + + // Load the base template file path. layoutFile, ok := r.layoutFiles[templateLayoutName] if !ok { return errors.Wrapf(errInvalidTemplate, "template layout file for %s does not exist", templateLayoutName) } + // The base layout will be the first template. files := []string{layoutFile} + // Append all of the partials that are defined. Not an easy way to determine if the + // layout or content template contain any references to a partial so load all of them. + // This assumes that all partial templates should be uniquely named and not conflict with + // and base layout or content definitions. for _, f := range r.partialFiles { files = append(files, f) } + // Load the content template file path. contentFile, ok := r.contentFiles[templateContentName] if !ok { return errors.Wrapf(errInvalidTemplate, "template content file for %s does not exist", templateContentName) } files = append(files, contentFile) + // Render all of template files t = template.Must(t.ParseFiles(files...)) r.templates[templateContentName] = t } @@ -261,7 +279,6 @@ func (r *TemplateRenderer) Render(ctx context.Context, w http.ResponseWriter, re // to define context.Context as an argument renderData["_Ctx"] = ctx - // Append request data map to render data last so any previous value can be overwritten. if data != nil { for k, v := range data { @@ -278,8 +295,9 @@ func (r *TemplateRenderer) Render(ctx context.Context, w http.ResponseWriter, re return nil } +// Error formats an error and returns the result to the client. func (r *TemplateRenderer) Error(ctx context.Context, w http.ResponseWriter, req *http.Request, statusCode int, er error) error { - // If error hander was defined to support formated response for web, used it. + // If error handler was defined to support formatted response for web, used it. if r.errorHandler != nil { return r.errorHandler(ctx, w, req, r, statusCode, er) } @@ -288,6 +306,8 @@ func (r *TemplateRenderer) Error(ctx context.Context, w http.ResponseWriter, req return web.RespondError(ctx, w, er) } +// Static serves files from the local file exist. +// If an error is encountered, it will handled by TemplateRenderer.Error func (tr *TemplateRenderer) Static(rootDir, prefix string) web.Handler { h := func(ctx context.Context, w http.ResponseWriter, r *http.Request, params map[string]string) error { err := web.StaticHandler(ctx, w, r, params, rootDir, prefix) @@ -299,6 +319,8 @@ func (tr *TemplateRenderer) Static(rootDir, prefix string) web.Handler { return h } +// S3Url formats a path to include either the S3 URL or a CloudFront +// URL instead of serving the file from local file system. func S3Url(baseS3Url, baseS3Origin, p string) string { if strings.HasPrefix(p, "http") { return p