You've already forked golang-saas-starter-kit
mirror of
https://github.com/raseels-repos/golang-saas-starter-kit.git
synced 2025-06-15 00:15:15 +02:00
Finish inital schema migration script
This commit is contained in:
Lee Brown
@ -7,14 +7,17 @@ import (
|
||||
"log"
|
||||
"net/http"
|
||||
_ "net/http/pprof"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/go-redis/redis"
|
||||
"github.com/lib/pq"
|
||||
"geeks-accelerator/oss/saas-starter-kit/example-project/cmd/web-api/handlers"
|
||||
"geeks-accelerator/oss/saas-starter-kit/example-project/internal/platform/auth"
|
||||
"geeks-accelerator/oss/saas-starter-kit/example-project/internal/platform/db"
|
||||
"geeks-accelerator/oss/saas-starter-kit/example-project/internal/platform/flag"
|
||||
itrace "geeks-accelerator/oss/saas-starter-kit/example-project/internal/platform/trace"
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
@ -23,21 +26,11 @@ import (
|
||||
"github.com/kelseyhightower/envconfig"
|
||||
"go.opencensus.io/trace"
|
||||
awstrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/aws/aws-sdk-go/aws"
|
||||
sqltrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/database/sql"
|
||||
redistrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/go-redis/redis"
|
||||
sqlxtrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/jmoiron/sqlx"
|
||||
)
|
||||
|
||||
/*
|
||||
ZipKin: http://localhost:9411
|
||||
AddLoad: hey -m GET -c 10 -n 10000 "http://localhost:3000/v1/users"
|
||||
expvarmon -ports=":3001" -endpoint="/metrics" -vars="requests,goroutines,errors,mem:memstats.Alloc"
|
||||
*/
|
||||
|
||||
/*
|
||||
Need to figure out timeouts for http service.
|
||||
You might want to reset your DB_HOST env var during test tear down.
|
||||
Service should start even without a DB running yet.
|
||||
symbols in profiles: https://github.com/golang/go/issues/23376 / https://github.com/google/pprof/pull/366
|
||||
*/
|
||||
|
||||
// build is the git version of this program. It is set using build flags in the makefile.
|
||||
var build = "develop"
|
||||
|
||||
@ -53,14 +46,56 @@ func main() {
|
||||
var cfg struct {
|
||||
Env string `default:"dev" envconfig:"ENV"`
|
||||
HTTP struct {
|
||||
Host string `default:"0.0.0.0:3001" envconfig:"HTTP_HOST"`
|
||||
DebugHost string `default:"0.0.0.0:4000" envconfig:"HTTP_DEBUG_HOST"`
|
||||
ReadTimeout time.Duration `default:"5s" envconfig:"HTTP_READ_TIMEOUT"`
|
||||
WriteTimeout time.Duration `default:"5s" envconfig:"HTTP_WRITE_TIMEOUT"`
|
||||
ShutdownTimeout time.Duration `default:"5s" envconfig:"HTTP_SHUTDOWN_TIMEOUT"`
|
||||
Host string `default:"0.0.0.0:3000" envconfig:"HTTP_HOST"`
|
||||
ReadTimeout time.Duration `default:"10s" envconfig:"HTTP_READ_TIMEOUT"`
|
||||
WriteTimeout time.Duration `default:"10s" envconfig:"HTTP_WRITE_TIMEOUT"`
|
||||
}
|
||||
HTTPS struct {
|
||||
Host string `default:"" envconfig:"HTTPS_HOST"`
|
||||
ReadTimeout time.Duration `default:"5s" envconfig:"HTTPS_READ_TIMEOUT"`
|
||||
WriteTimeout time.Duration `default:"5s" envconfig:"HTTPS_WRITE_TIMEOUT"`
|
||||
}
|
||||
App struct {
|
||||
Name string `default:"web-app" envconfig:"APP_NAME"`
|
||||
Name string `default:"web-api" envconfig:"APP_NAME"`
|
||||
BaseUrl string `default:"" envconfig:"APP_BASE_URL"`
|
||||
TemplateDir string `default:"./templates" envconfig:"APP_TEMPLATE_DIR"`
|
||||
DebugHost string `default:"0.0.0.0:4000" envconfig:"APP_DEBUG_HOST"`
|
||||
ShutdownTimeout time.Duration `default:"5s" envconfig:"APP_SHUTDOWN_TIMEOUT"`
|
||||
}
|
||||
Redis struct {
|
||||
DialTimeout time.Duration `default:"5s" envconfig:"REDIS_DIAL_TIMEOUT"`
|
||||
Host string `default:":6379" envconfig:"REDIS_HOST"`
|
||||
DB int `default:"1" envconfig:"REDIS_DB"`
|
||||
MaxmemoryPolicy string `envconfig:"REDIS_MAXMEMORY_POLICY"`
|
||||
}
|
||||
DB struct {
|
||||
Host string `default:"127.0.0.1:5433" envconfig:"DB_HOST"`
|
||||
User string `default:"postgres" envconfig:"DB_USER"`
|
||||
Pass string `default:"postgres" envconfig:"DB_PASS" json:"-"` // don't print
|
||||
Database string `default:"shared" envconfig:"DB_DATABASE"`
|
||||
Driver string `default:"postgres" envconfig:"DB_DRIVER"`
|
||||
Timezone string `default:"utc" envconfig:"DB_TIMEZONE"`
|
||||
DisableTLS bool `default:"false" envconfig:"DB_DISABLE_TLS"`
|
||||
}
|
||||
Trace struct {
|
||||
Host string `default:"http://tracer:3002/v1/publish" envconfig:"TRACE_HOST"`
|
||||
BatchSize int `default:"1000" envconfig:"TRACE_BATCH_SIZE"`
|
||||
SendInterval time.Duration `default:"15s" envconfig:"TRACE_SEND_INTERVAL"`
|
||||
SendTimeout time.Duration `default:"500ms" envconfig:"TRACE_SEND_TIMEOUT"`
|
||||
}
|
||||
AwsAccount struct {
|
||||
AccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"`
|
||||
SecretAccessKey string `envconfig:"AWS_SECRET_ACCESS_KEY" json:"-"` // don't print
|
||||
Region string `default:"us-east-1" envconfig:"AWS_REGION"`
|
||||
|
||||
// Get an AWS session from an implicit source if no explicit
|
||||
// configuration is provided. This is useful for taking advantage of
|
||||
// EC2/ECS instance roles.
|
||||
UseRole bool `envconfig:"AWS_USE_ROLE"`
|
||||
}
|
||||
Auth struct {
|
||||
AwsSecretID string `default:"auth-secret-key" envconfig:"AUTH_AWS_SECRET_ID"`
|
||||
KeyExpiration time.Duration `default:"3600s" envconfig:"AUTH_KEY_EXPIRATION"`
|
||||
}
|
||||
BuildInfo struct {
|
||||
CiCommitRefName string `envconfig:"CI_COMMIT_REF_NAME"`
|
||||
@ -74,35 +109,14 @@ func main() {
|
||||
CiPipelineId string `envconfig:"CI_COMMIT_PIPELINE_ID"`
|
||||
CiPipelineUrl string `envconfig:"CI_COMMIT_PIPELINE_URL"`
|
||||
}
|
||||
DB struct {
|
||||
DialTimeout time.Duration `default:"5s" envconfig:"DB_DIAL_TIMEOUT"`
|
||||
Host string `default:"mongo:27017/gotraining" envconfig:"DB_HOST"`
|
||||
}
|
||||
Trace struct {
|
||||
Host string `default:"http://tracer:3002/v1/publish" envconfig:"TRACE_HOST"`
|
||||
BatchSize int `default:"1000" envconfig:"TRACE_BATCH_SIZE"`
|
||||
SendInterval time.Duration `default:"15s" envconfig:"TRACE_SEND_INTERVAL"`
|
||||
SendTimeout time.Duration `default:"500ms" envconfig:"TRACE_SEND_TIMEOUT"`
|
||||
}
|
||||
AwsAccount struct {
|
||||
AccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"`
|
||||
SecretAccessKey string `envconfig:"AWS_SECRET_ACCESS_KEY"`
|
||||
Region string `default:"us-east-1" envconfig:"AWS_REGION"`
|
||||
|
||||
// Get an AWS session from an implicit source if no explicit
|
||||
// configuration is provided. This is useful for taking advantage of
|
||||
// EC2/ECS instance roles.
|
||||
UseRole bool `envconfig:"AWS_USE_ROLE"`
|
||||
}
|
||||
Auth struct {
|
||||
AwsSecretID string `default:"auth-secret-key" envconfig:"AUTH_AWS_SECRET_ID"`
|
||||
KeyExpiration time.Duration `default:"3600s" envconfig:"AUTH_KEY_EXPIRATION"`
|
||||
}
|
||||
}
|
||||
|
||||
// !!! This prefix seems buried, if you copy and paste this main.go
|
||||
// file, its likely you will forget to update this.
|
||||
if err := envconfig.Process("WEB_API", &cfg); err != nil {
|
||||
// The prefix used for loading env variables.
|
||||
// ie: export WEB_API_ENV=dev
|
||||
envKeyPrefix := "WEB_API"
|
||||
|
||||
// For additional details refer to https://github.com/kelseyhightower/envconfig
|
||||
if err := envconfig.Process(envKeyPrefix, &cfg); err != nil {
|
||||
log.Fatalf("main : Parsing Config : %v", err)
|
||||
}
|
||||
|
||||
@ -114,22 +128,43 @@ func main() {
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// App Starting
|
||||
// Config Validation & Defaults
|
||||
|
||||
// If base URL is empty, set the default value from the HTTP Host
|
||||
if cfg.App.BaseUrl == "" {
|
||||
baseUrl := cfg.HTTP.Host
|
||||
if !strings.HasPrefix(baseUrl, "http") {
|
||||
if strings.HasPrefix(baseUrl, "0.0.0.0:") {
|
||||
pts := strings.Split(baseUrl, ":")
|
||||
pts[0] = "127.0.0.1"
|
||||
baseUrl = strings.Join(pts, ":")
|
||||
} else if strings.HasPrefix(baseUrl, ":") {
|
||||
baseUrl = "127.0.0.1" + baseUrl
|
||||
}
|
||||
baseUrl = "http://" + baseUrl
|
||||
}
|
||||
cfg.App.BaseUrl = baseUrl
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Log App Info
|
||||
|
||||
// Print the build version for our logs. Also expose it under /debug/vars.
|
||||
expvar.NewString("build").Set(build)
|
||||
log.Printf("main : Started : Application Initializing version %q", build)
|
||||
defer log.Println("main : Completed")
|
||||
|
||||
cfgJSON, err := json.MarshalIndent(cfg, "", " ")
|
||||
if err != nil {
|
||||
log.Fatalf("main : Marshalling Config to JSON : %v", err)
|
||||
// Print the config for our logs. It's important to any credentials in the config
|
||||
// that could expose a security risk are excluded from being json encoded by
|
||||
// applying the tag `json:"-"` to the struct var.
|
||||
{
|
||||
cfgJSON, err := json.MarshalIndent(cfg, "", " ")
|
||||
if err != nil {
|
||||
log.Fatalf("main : Marshalling Config to JSON : %v", err)
|
||||
}
|
||||
log.Printf("main : Config : %v\n", string(cfgJSON))
|
||||
}
|
||||
|
||||
// TODO: Validate what is being written to the logs. We don't
|
||||
// want to leak credentials or anything that can be a security risk.
|
||||
log.Printf("main : Config : %v\n", string(cfgJSON))
|
||||
|
||||
// =========================================================================
|
||||
// Init AWS Session
|
||||
var awsSession *session.Session
|
||||
@ -144,6 +179,77 @@ func main() {
|
||||
}
|
||||
awsSession = awstrace.WrapSession(awsSession)
|
||||
|
||||
// =========================================================================
|
||||
// Start Redis
|
||||
// Ensure the eviction policy on the redis cluster is set correctly.
|
||||
// AWS Elastic cache redis clusters by default have the volatile-lru.
|
||||
// volatile-lru: evict keys by trying to remove the less recently used (LRU) keys first, but only among keys that have an expire set, in order to make space for the new data added.
|
||||
// allkeys-lru: evict keys by trying to remove the less recently used (LRU) keys first, in order to make space for the new data added.
|
||||
// Recommended to have eviction policy set to allkeys-lru
|
||||
log.Println("main : Started : Initialize Redis")
|
||||
redisClient := redistrace.NewClient(&redis.Options{
|
||||
Addr: cfg.Redis.Host,
|
||||
DB: cfg.Redis.DB,
|
||||
DialTimeout: cfg.Redis.DialTimeout,
|
||||
})
|
||||
defer redisClient.Close()
|
||||
|
||||
evictPolicyConfigKey := "maxmemory-policy"
|
||||
|
||||
// if the maxmemory policy is set for redis, make sure its set on the cluster
|
||||
// default not set and will based on the redis config values defined on the server
|
||||
if cfg.Redis.MaxmemoryPolicy != "" {
|
||||
err := redisClient.ConfigSet(evictPolicyConfigKey, cfg.Redis.MaxmemoryPolicy).Err()
|
||||
if err != nil {
|
||||
log.Fatalf("main : redis : ConfigSet maxmemory-policy : %v", err)
|
||||
}
|
||||
} else {
|
||||
evictPolicy, err := redisClient.ConfigGet(evictPolicyConfigKey).Result()
|
||||
if err != nil {
|
||||
log.Fatalf("main : redis : ConfigGet maxmemory-policy : %v", err)
|
||||
}
|
||||
|
||||
if evictPolicy[1] != "allkeys-lru" {
|
||||
log.Printf("main : redis : ConfigGet maxmemory-policy : recommended to be set to allkeys-lru to avoid OOM")
|
||||
}
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Start Database
|
||||
var dbUrl url.URL
|
||||
{
|
||||
// Query parameters.
|
||||
var q url.Values
|
||||
|
||||
// Handle SSL Mode
|
||||
if cfg.DB.DisableTLS {
|
||||
q.Set("sslmode", "disable")
|
||||
} else {
|
||||
q.Set("sslmode", "require")
|
||||
}
|
||||
|
||||
q.Set("timezone", cfg.DB.Timezone)
|
||||
|
||||
// Construct url.
|
||||
dbUrl = url.URL{
|
||||
Scheme: cfg.DB.Driver,
|
||||
User: url.UserPassword(cfg.DB.User, cfg.DB.Pass),
|
||||
Host: cfg.DB.Host,
|
||||
Path: cfg.DB.Database,
|
||||
RawQuery: q.Encode(),
|
||||
}
|
||||
}
|
||||
|
||||
// Register informs the sqlxtrace package of the driver that we will be using in our program.
|
||||
// It uses a default service name, in the below case "postgres.db". To use a custom service
|
||||
// name use RegisterWithServiceName.
|
||||
sqltrace.Register(cfg.DB.Driver, &pq.Driver{}, sqltrace.WithServiceName("my-service"))
|
||||
masterDb, err := sqlxtrace.Open(cfg.DB.Driver, dbUrl.String())
|
||||
if err != nil {
|
||||
log.Fatalf("main : Register DB : %s : %v", cfg.DB.Driver, err)
|
||||
}
|
||||
defer masterDb.Close()
|
||||
|
||||
// =========================================================================
|
||||
// Load auth keys from AWS and init new Authenticator
|
||||
authenticator, err := auth.NewAuthenticator(awsSession, cfg.Auth.AwsSecretID, time.Now().UTC(), cfg.Auth.KeyExpiration)
|
||||
@ -151,16 +257,6 @@ func main() {
|
||||
log.Fatalf("main : Constructing authenticator : %v", err)
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Start Mongo
|
||||
|
||||
log.Println("main : Started : Initialize Mongo")
|
||||
masterDB, err := db.New(cfg.DB.Host, cfg.DB.DialTimeout)
|
||||
if err != nil {
|
||||
log.Fatalf("main : Register DB : %v", err)
|
||||
}
|
||||
defer masterDB.Close()
|
||||
|
||||
// =========================================================================
|
||||
// Start Tracing Support
|
||||
|
||||
@ -192,10 +288,10 @@ func main() {
|
||||
//
|
||||
// /debug/vars - Added to the default mux by the expvars package.
|
||||
// /debug/pprof - Added to the default mux by the net/http/pprof package.
|
||||
if cfg.HTTP.DebugHost != "" {
|
||||
if cfg.App.DebugHost != "" {
|
||||
go func() {
|
||||
log.Printf("main : Debug Listening %s", cfg.HTTP.DebugHost)
|
||||
log.Printf("main : Debug Listener closed : %v", http.ListenAndServe(cfg.HTTP.DebugHost, http.DefaultServeMux))
|
||||
log.Printf("main : Debug Listening %s", cfg.App.DebugHost)
|
||||
log.Printf("main : Debug Listener closed : %v", http.ListenAndServe(cfg.App.DebugHost, http.DefaultServeMux))
|
||||
}()
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user