go/golang-saas-starter-kit
1
0
Fork 0
mirror of https://github.com/raseels-repos/golang-saas-starter-kit.git synced 2025-08-08 22:36:41 +02:00
Code Issues Releases Activity

Merge branch 'prod' into jsign/aws2

Browse Source
This commit is contained in:
Lee Brown
2019-08-22 11:16:51 -08:00
parent 62f9cff7fe 50151310ad
commit d3b97a6608
64 changed files with 2926 additions and 6718 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
internal/platform/web/webcontext/context.go
View File

@ -45,6 +45,13 @@ var (
Env_Prod Env = "prod"
)
// List of env names.
var EnvNames = []Env{
Env_Dev,
Env_Stage,
Env_Prod,
}
func ContextEnv(ctx context.Context) string {
cv := ctx.Value(KeyValues).(*Values)
if cv != nil {

371
internal/schema/init_schema.go
View File

@ -16,374 +16,3 @@ func initSchema(ctx context.Context, db *sqlx.DB, log *log.Logger, isUnittest bo
return f
}
/*
// initGeonames populates countries and postal codes.
func initGeonamesOld(db *sqlx.DB) error {
schemas := []string{
`DROP TABLE IF EXISTS geoname`,
`create table geoname (
geonameid int,
name varchar(200),
asciiname varchar(200),
alternatenames text,
latitude float,
longitude float,
fclass char(1),
fcode varchar(10),
country varchar(2),
cc2 varchar(600),
admin1 varchar(20),
admin2 varchar(80),
admin3 varchar(20),
admin4 varchar(20),
population bigint,
elevation int,
gtopo30 int,
timezone varchar(40),
moddate date)`,
`DROP TABLE IF EXISTS countryinfo`,
`CREATE TABLE countryinfo (
iso_alpha2 char(2),
iso_alpha3 char(3),
iso_numeric integer,
fips_code character varying(3),
country character varying(200),
capital character varying(200),
areainsqkm double precision,
population integer,
continent char(2),
tld CHAR(10),
currency_code char(3),
currency_name CHAR(20),
phone character varying(20),
postal character varying(60),
postal_format character varying(200),
postal_regex character varying(200),
languages character varying(200),
geonameId int,
neighbours character varying(50),
equivalent_fips_code character varying(3))`,
}
for _, q := range schemas {
_, err := db.Exec(q)
if err != nil {
return errors.WithMessagef(err, "Failed to execute sql query '%s'", q)
}
}
// Load the countryinfo table.
if false {
u := "http://download.geonames.org/export/dump/countryInfo.txt"
resp, err := pester.Get(u)
if err != nil {
return errors.WithMessagef(err, "Failed to read country info from '%s'", u)
}
defer resp.Body.Close()
scanner := bufio.NewScanner(resp.Body)
var prevLine string
var stmt *sql.Stmt
for scanner.Scan() {
line := scanner.Text()
// Skip comments.
if strings.HasPrefix(line, "#") {
prevLine = line
continue
}
// Pull the last comment to load the fields.
if stmt == nil {
prevLine = strings.TrimPrefix(prevLine, "#")
r := csv.NewReader(strings.NewReader(prevLine))
r.Comma = '\t' // Use tab-delimited instead of comma <---- here!
r.FieldsPerRecord = -1
lines, err := r.ReadAll()
if err != nil {
return errors.WithStack(err)
}
var columns []string
for _, fn := range lines[0] {
var cn string
switch fn {
case "ISO":
cn = "iso_alpha2"
case "ISO3":
cn = "iso_alpha3"
case "ISO-Numeric":
cn = "iso_numeric"
case "fips":
cn = "fips_code"
case "Country":
cn = "country"
case "Capital":
cn = "capital"
case "Area(in sq km)":
cn = "areainsqkm"
case "Population":
cn = "population"
case "Continent":
cn = "continent"
case "tld":
cn = "tld"
case "CurrencyCode":
cn = "currency_code"
case "CurrencyName":
cn = "currency_name"
case "Phone":
cn = "phone"
case "Postal":
cn = "postal"
case "Postal Code Format":
cn = "postal_format"
case "Postal Code Regex":
cn = "postal_regex"
case "Languages":
cn = "languages"
case "geonameid":
cn = "geonameId"
case "neighbours":
cn = "neighbours"
case "EquivalentFipsCode":
cn = "equivalent_fips_code"
default :
return errors.Errorf("Failed to map column %s", fn)
}
columns = append(columns, cn)
}
placeholders := []string{}
for i := 0; i < len(columns); i++ {
placeholders = append(placeholders, "?")
}
q := "insert into countryinfo ("+strings.Join(columns, ",")+") values("+strings.Join(placeholders, ",")+")"
q = db.Rebind(q)
stmt, err = db.Prepare(q)
if err != nil {
return errors.WithMessagef(err, "Failed to prepare sql query '%s'", q)
}
}
r := csv.NewReader(strings.NewReader(line))
r.Comma = '\t' // Use tab-delimited instead of comma <---- here!
r.FieldsPerRecord = -1
lines, err := r.ReadAll()
if err != nil {
return errors.WithStack(err)
}
for _, row := range lines {
var args []interface{}
for _, v := range row {
args = append(args, v)
}
_, err = stmt.Exec(args...)
if err != nil {
return errors.WithStack(err)
}
}
}
if err := scanner.Err(); err != nil {
return errors.WithStack(err)
}
}
// Load the geoname table.
{
u := "http://download.geonames.org/export/dump/allCountries.zip"
resp, err := pester.Get(u)
if err != nil {
return errors.WithMessagef(err, "Failed to read countries from '%s'", u)
}
defer resp.Body.Close()
br := bufio.NewReader(resp.Body)
buff := bytes.NewBuffer([]byte{})
size, err := io.Copy(buff, br)
if err != nil {
return err
}
b := bytes.NewReader(buff.Bytes())
zr, err := zip.NewReader(b, size)
if err != nil {
return errors.WithStack(err)
}
q := "insert into geoname " +
"(geonameid,name,asciiname,alternatenames,latitude,longitude,fclass,fcode,country,cc2,admin1,admin2,admin3,admin4,population,elevation,gtopo30,timezone,moddate) " +
"values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
q = db.Rebind(q)
stmt, err := db.Prepare(q)
if err != nil {
return errors.WithMessagef(err, "Failed to prepare sql query '%s'", q)
}
for _, f := range zr.File {
if f.Name == "readme.txt" {
continue
}
fh, err := f.Open()
if err != nil {
return errors.WithStack(err)
}
scanner := bufio.NewScanner(fh)
for scanner.Scan() {
line := scanner.Text()
// Skip comments.
if strings.HasPrefix(line, "#") {
continue
}
if strings.Contains(line, "\"") {
line = strings.Replace(line, "\"", "\\\"", -1)
}
r := csv.NewReader(strings.NewReader(line))
r.Comma = '\t' // Use tab-delimited instead of comma <---- here!
r.LazyQuotes = true
r.FieldsPerRecord = -1
lines, err := r.ReadAll()
if err != nil {
return errors.WithStack(err)
}
for _, row := range lines {
var args []interface{}
for idx, v := range row {
if v == "" {
if idx == 0 || idx == 14 || idx == 15 {
v = "0"
}
}
args = append(args, v)
}
_, err = stmt.Exec(args...)
if err != nil {
return errors.WithStack(err)
}
}
}
if err := scanner.Err(); err != nil {
return errors.WithStack(err)
}
}
}
return errors.New("not finished")
queries := []string{
// Countries...
`DROP TABLE IF EXISTS countries`,
`CREATE TABLE countries(
id serial not null constraint countries_pkey primary key,
geoname_id int,
iso char(2),
country character varying(50),
capital character varying(50),
created_at TIMESTAMP WITH TIME ZONE NOT NULL,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NULL,
archived_at TIMESTAMP WITH TIME ZONE DEFAULT NULL)`,
`create index idx_countries_deleted_at on countries (deleted_at)`,
`insert into countries(geoname_id, iso, country, capital, created_at, updated_at)
select geonameId, iso_alpha2, country, capital, NOW(), NOW()
from countryinfo`,
// Regions...
`DROP TABLE IF EXISTS regions`,
`CREATE TABLE regions (
id serial not null constraint regions_pkey primary key,
country_id int,
geoname_id int,
name varchar(200),
ascii_name varchar(200),
adm varchar(20),
country char(2),
latitude float,
longitude float,
created_at TIMESTAMP WITH TIME ZONE NOT NULL,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NULL,
archived_at TIMESTAMP WITH TIME ZONE DEFAULT NULL)`,
`create index idx_regions_deleted_at on regions (deleted_at)`,
`insert into regions(country_id, geoname_id, name, ascii_name, adm, country, latitude, longitude, created_at, updated_at)
select c.id,
g.geonameid,
g.name,
g.asciiname,
g.admin1,
c.iso,
g.latitude,
g.longitude,
to_timestamp(TO_CHAR(g.moddate, 'YYYY-MM-DD'), 'YYYY-MM-DD'),
to_timestamp(TO_CHAR(g.moddate, 'YYYY-MM-DD'), 'YYYY-MM-DD')
from countries as c
inner join geoname as g on c.iso = g.country and g.fcode like 'ADM1'`,
// cities
`DROP TABLE IF EXISTS cities`,
`CREATE TABLE cities (
id serial not null constraint cities_pkey primary key,
country_id int,
region_id int,
geoname_id int,
name varchar(200),
ascii_name varchar(200),
latitude float,
longitude float,
created_at TIMESTAMP WITH TIME ZONE NOT NULL,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NULL,
archived_at TIMESTAMP WITH TIME ZONE DEFAULT NULL)`,
`create index idx_cities_deleted_at on cities (deleted_at)`,
`insert into cities(country_id, region_id, geoname_id, name, ascii_name, latitude, longitude, created_at, updated_at)
select r.country_id,
r.id,
g.geonameid,
g.name,
g.asciiname,
g.latitude,
g.longitude,
to_timestamp(TO_CHAR(g.moddate, 'YYYY-MM-DD'), 'YYYY-MM-DD'),
to_timestamp(TO_CHAR(g.moddate, 'YYYY-MM-DD'), 'YYYY-MM-DD')
from geoname as g
join regions as r on r.adm = g.admin1
and r.country = g.country
and (g.fcode in ('PPLC', 'PPLA') or (g.fcode like 'PPLA%' and g.population >= 50000));`,
}
tx, err := db.Begin()
if err != nil {
return errors.WithStack(err)
}
for _, q := range queries {
_, err = tx.Exec(q)
if err != nil {
return errors.WithMessagef(err, "Failed to execute sql query '%s'", q)
}
}
err = tx.Commit()
if err != nil {
return errors.WithStack(err)
}
return nil
}
*/

113
internal/schema/migrations.go
View File

@ -5,10 +5,10 @@ import (
"context"
"database/sql"
"encoding/csv"
"fmt"
"log"
"strings"
"time"
"fmt"
"geeks-accelerator/oss/saas-starter-kit/internal/geonames"
"github.com/geeks-accelerator/sqlxmigrate"
@ -26,7 +26,7 @@ func migrationList(ctx context.Context, db *sqlx.DB, log *log.Logger, isUnittest
return []*sqlxmigrate.Migration{
// Create table users.
{
ID: "20190522-01a",
ID: "20190522-01b",
Migrate: func(tx *sql.Tx) error {
q1 := `CREATE TABLE IF NOT EXISTS users (
id char(36) NOT NULL,
@ -57,11 +57,10 @@ func migrationList(ctx context.Context, db *sqlx.DB, log *log.Logger, isUnittest
},
// Create new table accounts.
{
ID: "20190522-01b",
ID: "20190522-01c",
Migrate: func(tx *sql.Tx) error {
q1 := `CREATE TYPE account_status_t as enum('active','pending','disabled')`
if _, err := tx.Exec(q1); err != nil {
return errors.WithMessagef(err, "Query failed %s", q1)
if err := createTypeIfNotExists(tx, "account_status_t", "enum('active','pending','disabled')"); err != nil {
return err
}
q2 := `CREATE TABLE IF NOT EXISTS accounts (
@ -89,7 +88,7 @@ func migrationList(ctx context.Context, db *sqlx.DB, log *log.Logger, isUnittest
return nil
},
Rollback: func(tx *sql.Tx) error {
q1 := `DROP TYPE account_status_t`
q1 := `DROP TYPE IF EXISTS account_status_t`
if _, err := tx.Exec(q1); err != nil {
return errors.WithMessagef(err, "Query failed %s", q1)
}
@ -103,19 +102,17 @@ func migrationList(ctx context.Context, db *sqlx.DB, log *log.Logger, isUnittest
},
// Create new table user_accounts.
{
ID: "20190522-01d",
ID: "20190522-01e",
Migrate: func(tx *sql.Tx) error {
q1 := `CREATE TYPE user_account_role_t as enum('admin', 'user')`
if _, err := tx.Exec(q1); err != nil {
return errors.WithMessagef(err, "Query failed %s", q1)
if err := createTypeIfNotExists(tx, "user_account_role_t", "enum('admin', 'user')"); err != nil {
return err
}
q2 := `CREATE TYPE user_account_status_t as enum('active', 'invited','disabled')`
if _, err := tx.Exec(q2); err != nil {
return errors.WithMessagef(err, "Query failed %s", q2)
if err := createTypeIfNotExists(tx, "user_account_status_t", "enum('active', 'invited','disabled'"); err != nil {
return err
}
q3 := `CREATE TABLE IF NOT EXISTS users_accounts (
q1 := `CREATE TABLE IF NOT EXISTS users_accounts (
id char(36) NOT NULL,
account_id char(36) NOT NULL REFERENCES accounts(id) ON DELETE NO ACTION,
user_id char(36) NOT NULL REFERENCES users(id) ON DELETE NO ACTION,
@ -127,19 +124,19 @@ func migrationList(ctx context.Context, db *sqlx.DB, log *log.Logger, isUnittest
PRIMARY KEY (id),
CONSTRAINT user_account UNIQUE (user_id,account_id)
)`
if _, err := tx.Exec(q3); err != nil {
return errors.WithMessagef(err, "Query failed %s", q3)
if _, err := tx.Exec(q1); err != nil {
return errors.WithMessagef(err, "Query failed %s", q1)
}
return nil
},
Rollback: func(tx *sql.Tx) error {
q1 := `DROP TYPE user_account_role_t`
q1 := `DROP TYPE IF EXISTS user_account_role_t`
if _, err := tx.Exec(q1); err != nil {
return errors.WithMessagef(err, "Query failed %s", q1)
}
q2 := `DROP TYPE userr_account_status_t`
q2 := `DROP TYPE IF EXISTS user_account_status_t`
if _, err := tx.Exec(q2); err != nil {
return errors.WithMessagef(err, "Query failed %s", q2)
}
@ -156,12 +153,11 @@ func migrationList(ctx context.Context, db *sqlx.DB, log *log.Logger, isUnittest
{
ID: "20190622-01",
Migrate: func(tx *sql.Tx) error {
q1 := `CREATE TYPE project_status_t as enum('active','disabled')`
if _, err := tx.Exec(q1); err != nil {
return errors.WithMessagef(err, "Query failed %s", q1)
if err := createTypeIfNotExists(tx, "project_status_t", "enum('active','disabled')"); err != nil {
return err
}
q2 := `CREATE TABLE IF NOT EXISTS projects (
q1 := `CREATE TABLE IF NOT EXISTS projects (
id char(36) NOT NULL,
account_id char(36) NOT NULL REFERENCES accounts(id) ON DELETE SET NULL,
name varchar(255) NOT NULL,
@ -171,19 +167,19 @@ func migrationList(ctx context.Context, db *sqlx.DB, log *log.Logger, isUnittest
archived_at TIMESTAMP WITH TIME ZONE DEFAULT NULL,
PRIMARY KEY (id)
)`
if _, err := tx.Exec(q2); err != nil {
return errors.WithMessagef(err, "Query failed %s", q2)
if _, err := tx.Exec(q1); err != nil {
return errors.WithMessagef(err, "Query failed %s", q1)
}
return nil
},
Rollback: func(tx *sql.Tx) error {
q1 := `DROP TYPE project_status_t`
if _, err := tx.Exec(q1); err != nil {
q1 := `DROP TYPE IF EXISTS project_status_t`
if _, err := tx.Exec(q1); err != nil && !errorIsAlreadyExists(err) {
return errors.WithMessagef(err, "Query failed %s", q1)
}
q2 := `DROP TABLE IF EXISTS projects`
if _, err := tx.Exec(q2); err != nil {
if _, err := tx.Exec(q2); err != nil && !errorIsAlreadyExists(err) {
return errors.WithMessagef(err, "Query failed %s", q2)
}
return nil
@ -277,13 +273,12 @@ func migrationList(ctx context.Context, db *sqlx.DB, log *log.Logger, isUnittest
"VALUES %s", strings.Join(valueStrings, ","))
insertStmt = db.Rebind(insertStmt)
stmt, err := db.Prepare(insertStmt)
_, err := db.Exec(insertStmt, valueArgs...)
if err != nil {
return errors.WithMessagef(err, "Failed to prepare sql query '%s'", insertStmt)
return errors.WithMessagef(err, "Failed to execute sql query '%s'", insertStmt)
}
_, err = stmt.Exec(valueArgs...)
return err
return nil
}
start := time.Now()
for _, country := range countries {
@ -294,7 +289,7 @@ func migrationList(ctx context.Context, db *sqlx.DB, log *log.Logger, isUnittest
}
//fmt.Println("Geoname records: ", len(v))
// Max argument values of Postgres is about 54460. So the batch size for bulk insert is selected 4500*12 (ncol)
batch := 4500
batch := 1000
n := len(v) / batch
//fmt.Println("Number of batch: ", n)
@ -669,3 +664,55 @@ func migrationList(ctx context.Context, db *sqlx.DB, log *log.Logger, isUnittest
},
}
}
// dropTypeIfExists executes drop type.
func dropTypeIfExists(tx *sql.Tx, name string) error {
q := "DROP TYPE IF EXISTS " + name
if _, err := tx.Exec(q); err != nil && !errorIsAlreadyExists(err) {
return errors.WithMessagef(err, "Query failed %s", q)
}
return nil
}
// createTypeIfNotExists checks to ensure a type doesn't exist before creating.
func createTypeIfNotExists(tx *sql.Tx, name, val string) error {
q1 := "select exists (select 1 from pg_type where typname = '"+name+"')"
rows, err := tx.Query(q1)
if err != nil {
return errors.WithMessagef(err, "Query failed %s", q1)
}
defer rows.Close()
var exists bool
for rows.Next() {
err := rows.Scan(&exists)
if err != nil {
return err
}
}
if err := rows.Err(); err != nil {
return err
}
if exists {
return nil
}
q2 := `CREATE TYPE "+name+" AS `+val
if _, err := tx.Exec(q2); err != nil && !errorIsAlreadyExists(err) {
return errors.WithMessagef(err, "Query failed %s", q2)
}
return nil
}
// errorIsAlreadyExists checks an error message for the error "already exists"
func errorIsAlreadyExists(err error) bool {
if strings.Contains(err.Error(), "already exists") {
return true
}
return false
}

14
internal/schema/schema.go
View File

@ -3,12 +3,24 @@ package schema
import (
"context"
"log"
"time"
"geeks-accelerator/oss/saas-starter-kit/internal/platform/web/webcontext"
"github.com/geeks-accelerator/sqlxmigrate"
"github.com/jmoiron/sqlx"
)
func Migrate(ctx context.Context, masterDb *sqlx.DB, log *log.Logger, isUnittest bool) error {
// Migrate is the entry point for performing init schema and running all the migrations.
func Migrate(ctx context.Context, targetEnv webcontext.Env, masterDb *sqlx.DB, log *log.Logger, isUnittest bool) error {
// Set the context with the required values to
// process the request.
v := webcontext.Values{
Now: time.Now(),
Env: targetEnv,
}
ctx = context.WithValue(ctx, webcontext.KeyValues, &v)
// Load list of Schema migrations and init new sqlxmigrate client
migrations := migrationList(ctx, masterDb, log, isUnittest)
m := sqlxmigrate.New(masterDb, sqlxmigrate.DefaultOptions, migrations)