go/golang-saas-starter-kit
1
0
Fork 0
mirror of https://github.com/raseels-repos/golang-saas-starter-kit.git synced 2025-06-27 00:51:13 +02:00
Code Issues Releases Activity

signup autocomplete

Browse Source
This commit is contained in:
Lee Brown
2019-08-01 11:34:03 -08:00
parent 2f51649340
commit 232648a2cc
14 changed files with 1212 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
cmd/web-app/handlers/api_geo.go Normal file
View File

@ -0,0 +1,52 @@
package handlers
import (
"context"
"fmt"
"net/http"
"strings"
"geeks-accelerator/oss/saas-starter-kit/internal/platform/web"
"github.com/jmoiron/sqlx"
"geeks-accelerator/oss/saas-starter-kit/internal/geonames"
"gopkg.in/DataDog/dd-trace-go.v1/contrib/go-redis/redis"
)
// Check provides support for orchestration geo endpoints.
type Geo struct {
MasterDB *sqlx.DB
Redis *redis.Client
}
// RegionsAutocomplete...
func (h *Geo) RegionsAutocomplete(ctx context.Context, w http.ResponseWriter, r *http.Request, params map[string]string) error {
var filters []string
var args []interface{}
if qv := r.URL.Query().Get("postal_code"); qv != "" {
filters = append(filters,"postal_code like ?")
args = append(args, qv+"%")
}
if qv := r.URL.Query().Get("query"); qv != "" {
filters = append(filters,"(state_name like ? or state_code like ?)")
args = append(args, qv+"%", qv+"%")
}
where := strings.Join(filters, " AND ")
res, err := geonames.FindGeonameRegions(ctx, h.MasterDB, where, args)
if err != nil {
fmt.Printf("%+v", err)
return web.RespondJsonError(ctx, w, err)
}
var list []string
for _, c := range res {
list = append(list, c.Name)
}
return web.RespondJson(ctx, w, list, http.StatusOK)
}

8
cmd/web-app/handlers/routes.go
View File

@ -68,6 +68,14 @@ func APP(shutdown chan os.Signal, log *log.Logger, env webcontext.Env, staticDir
app.Handle("POST", "/signup", s.Step1)
app.Handle("GET", "/signup", s.Step1)
// Register geo
g := Geo{
MasterDB: masterDB,
Redis: redis,
}
// These routes are not authenticated
app.Handle("GET", "/geo/regions/autocomplete", g.RegionsAutocomplete)
// Register root
r := Root{
MasterDB: masterDB,

9
cmd/web-app/handlers/signup.go
View File

@ -2,6 +2,7 @@ package handlers
import (
"context"
"geeks-accelerator/oss/saas-starter-kit/internal/geonames"
"net/http"
"time"
@ -80,6 +81,14 @@ func (h *Signup) Step1(ctx context.Context, w http.ResponseWriter, r *http.Reque
// Redirect the user to the dashboard.
http.Redirect(w, r, "/", http.StatusFound)
return nil
}
data["geonameCountries"] = geonames.ValidGeonameCountries
data["countries"], err = geonames.FindCountries(ctx, h.MasterDB, "name", "")
if err != nil {
return err
}
return nil

74
cmd/web-app/templates/content/signup-step1.tmpl
View File

@ -35,17 +35,21 @@
</div>
<div class="form-group row">
<div class="col-sm-6 mb-3 mb-sm-0">
<input type="text" class="form-control form-control-user {{ ValidationFieldClass $.validationErrors "Account.Zipcode" }}" name="Account.Zipcode" value="{{ $.form.Account.Zipcode }}" placeholder="Zipcode" required>
{{template "invalid-feedback" dict "validationDefaults" $.validationDefaults "validationErrors" $.validationErrors "fieldName" "Account.Zipcode" }}
<select class="form-control form-control-user {{ ValidationFieldClass $.validationErrors "Account.Country" }}" id="selectAccountCountry" name="Account.Country" placeholder="Country" required>
{{ range $i := $.countries }}
<option value="{{ $i.Code }}" {{ if eq $.form.Account.Country $i.Code }}selected="selected"{{ end }}>{{ $i.Name }}</option>
{{ end }}
</select>
{{template "invalid-feedback" dict "validationDefaults" $.validationDefaults "validationErrors" $.validationErrors "fieldName" "Account.Country" }}
</div>
</div>
<div class="form-group row">
<div class="col-sm-6 mb-3 mb-sm-0">
<input type="text" class="form-control form-control-user {{ ValidationFieldClass $.validationErrors "Account.Country" }}" name="Account.Country" value="{{ $.form.Account.Country }}" placeholder="Country" required>
{{template "invalid-feedback" dict "validationDefaults" $.validationDefaults "validationErrors" $.validationErrors "fieldName" "Account.Country" }}
<input type="text" class="form-control form-control-user {{ ValidationFieldClass $.validationErrors "Account.Zipcode" }}" id="inputAccountZipcode" name="Account.Zipcode" value="{{ $.form.Account.Zipcode }}" placeholder="Zipcode" required>
{{template "invalid-feedback" dict "validationDefaults" $.validationDefaults "validationErrors" $.validationErrors "fieldName" "Account.Zipcode" }}
</div>
<div class="col-sm-6 mb-3 mb-sm-0">
<input type="text" class="form-control form-control-user {{ ValidationFieldClass $.validationErrors "Account.Region" }}" name="Account.Region" value="{{ $.form.Account.Region }}" placeholder="Region" required>
<input type="text" class="form-control form-control-user {{ ValidationFieldClass $.validationErrors "Account.Region" }}" id="inputAccountRegion" name="Account.Region" value="{{ $.form.Account.Region }}" placeholder="Region" required>
{{template "invalid-feedback" dict "validationDefaults" $.validationDefaults "validationErrors" $.validationErrors "fieldName" "Account.Region" }}
</div>
</div>
@ -103,13 +107,71 @@
</div>
{{end}}
{{define "js"}}
<script src="https://cdn.jsdelivr.net/gh/xcash/bootstrap-autocomplete@v2.2.2/dist/latest/bootstrap-autocomplete.min.js"></script>
<script>
$(document).ready(function() {
$(document).find('body').addClass('bg-gradient-primary');
var regionAutocompleteParams = null;
var countryAutocompleteParams = null;
$('#inputAccountZipcode').on('change', function () {
console.log($(this).val());
regionAutocompleteParams = {postal_code: $(this).val()};
$('#inputAccountRegion').keyup();
countryAutocompleteParams = {postal_code: $(this).val()};
$('#inputAccountCountry').keyup();
});
$('#inputAccountRegion').on('keydown', function () {
regionAutocompleteParams = null;
});
$('#inputAccountCountry').on('keydown', function () {
countryAutocompleteParams = null;
});
$('#inputAccountRegion').autoComplete({
events: {
search: function (qry, callback) {
if (regionAutocompleteParams === null) {
regionAutocompleteParams = {query: qry};
}
$.ajax(
'/geo/regions/autocomplete',
{
data: regionAutocompleteParams
}
).done(function (res) {
callback(res)
});
}
}
});
$('#inputAccountCountry').autoComplete({
events: {
search: function (qry, callback) {
if (countryAutocompleteParams === null) {
countryAutocompleteParams = {query: qry};
}
$.ajax(
'/geo/countries/autocomplete',
{
data: countryAutocompleteParams
}
).done(function (res) {
callback(res)
});
}
}
});
// console.log(($('input[name=\'Account\.Name\']').parent().find('.invalid-feedback').show()));
});
</script>

4
cmd/web-app/templates/content/user-login.tmpl
View File

@ -31,8 +31,8 @@
</div>
<div class="form-group">
<div class="custom-control custom-checkbox small">
<input type="checkbox" class="custom-control-input" id="checkRemberMe" name="RememberMe" value="1" {{ if $.form.RememberMe }}checked="checked"{{end}}>
<label class="custom-control-label" for="checkRemberMe">Remember Me</label>
<input type="checkbox" class="custom-control-input" id="inputRemberMe" name="RememberMe" value="1" {{ if $.form.RememberMe }}checked="checked"{{end}}>
<label class="custom-control-label" for="inputRemberMe">Remember Me</label>
</div>
</div>
<button class="btn btn-primary btn-user btn-block">

2
cmd/web-app/templates/layouts/base.tmpl
View File

@ -17,7 +17,7 @@
<!-- ============================================================== -->
<!-- Custom fonts for this template -->
<!-- ============================================================== -->
<link href="{{ SiteAssetUrl "/assets/vendor/fontawesome-free/css/all.min.css" }}" rel="stylesheet" type="text/css">
<script src="https://kit.fontawesome.com/670ea91c67.js"></script>
<link href="https://fonts.googleapis.com/css?family=Nunito:200,200i,300,300i,400,400i,600,600i,700,700i,800,800i,900,900i" rel="stylesheet">
<!-- ============================================================== -->

1
go.mod
View File

@ -39,6 +39,7 @@ require (
github.com/pkg/errors v0.8.1
github.com/sergi/go-diff v1.0.0
github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24
github.com/stretchr/testify v1.3.0
github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14
github.com/swaggo/swag v1.6.2

1
go.sum
View File

@ -130,6 +130,7 @@ github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0 h1:X9XMOYjxEfAYSy3xK1DzO5dMkkWhs9E9UCcS1IERx2k=
github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0/go.mod h1:Ad7IjTpvzZO8Fl0vh9AzQ+j/jYZfyp2diGwI8m5q+ns=
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=

65
internal/geonames/countries.go Normal file
View File

@ -0,0 +1,65 @@
package geonames
import (
"context"
"github.com/huandu/go-sqlbuilder"
"github.com/jmoiron/sqlx"
"github.com/pkg/errors"
"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
)
const (
// The database table for Country
countriesTableName = "countries"
)
// FindCountries ....
func FindCountries(ctx context.Context, dbConn *sqlx.DB, orderBy, where string, args ...interface{}) ([]*Country, error) {
span, ctx := tracer.StartSpanFromContext(ctx, "internal.geonames.FindCountries")
defer span.Finish()
query := sqlbuilder.NewSelectBuilder()
query.Select("code,iso_alpha3,name,capital,currency_code,currency_name,phone,postal_code_format,postal_code_regex")
query.From(countriesTableName)
if orderBy == "" {
orderBy = "name"
}
query.OrderBy(orderBy)
if where != "" {
query.Where(where)
}
queryStr, queryArgs := query.Build()
queryStr = dbConn.Rebind(queryStr)
args = append(args, queryArgs...)
// fetch all places from the db
rows, err := dbConn.QueryContext(ctx, queryStr, args...)
if err != nil {
err = errors.Wrapf(err, "query - %s", query.String())
err = errors.WithMessage(err, "find countries failed")
return nil, err
}
// iterate over each row
resp := []*Country{}
for rows.Next() {
var (
v Country
err error
)
err = rows.Scan(&v.Code,&v.IsoAlpha3,&v.Name,&v.Capital,&v.CurrencyCode,&v.CurrencyName,&v.Phone,&v.PostalCodeFormat,&v.PostalCodeRegex)
if err != nil {
err = errors.Wrapf(err, "query - %s", query.String())
return nil, err
} else if v.Code == "" {
continue
}
resp = append(resp, &v)
}
return resp, nil
}

324
internal/geonames/geonames.go Normal file
View File

@ -0,0 +1,324 @@
package geonames
import (
"archive/zip"
"bufio"
"bytes"
"context"
"encoding/csv"
"fmt"
"io"
"strconv"
"strings"
"github.com/huandu/go-sqlbuilder"
"github.com/jmoiron/sqlx"
"github.com/pkg/errors"
"github.com/sethgrid/pester"
"github.com/shopspring/decimal"
"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
)
const (
// The database table for Geoname
geonamesTableName = "geonames"
)
var (
// List of country codes that will geonames will be downloaded for.
ValidGeonameCountries = []string{
"AD", "AR", "AS", "AT", "AU", "AX", "BD", "BE", "BG", "BM",
"BR", "BY", "CA", "CH", "CO", "CR", "CZ", "DE", "DK", "DO",
"DZ", "ES", "FI", "FO", "FR", "GB", "GF", "GG", "GL", "GP",
"GT", "GU", "HR", "HU", "IE", "IM", "IN", "IS", "IT", "JE",
"JP", "LI", "LK", "LT", "LU", "LV", "MC", "MD", "MH", "MK",
"MP", "MQ", "MT", "MX", "MY", "NC", "NL", "NO", "NZ", "PH",
"PK", "PL", "PM", "PR", "PT", "RE", "RO", "RU", "SE", "SI",
"SJ", "SK", "SM", "TH", "TR", "UA", "US", "UY", "VA", "VI",
"WF", "YT", "ZA"}
)
// FindGeonames ....
func FindGeonames(ctx context.Context, dbConn *sqlx.DB, orderBy, where string, args ...interface{}) ([]*Geoname, error) {
span, ctx := tracer.StartSpanFromContext(ctx, "internal.geonames.FindGeonames")
defer span.Finish()
query := sqlbuilder.NewSelectBuilder()
query.Select("country_code,postal_code,place_name,state_name,state_code,county_name,county_code,community_name,community_code,latitude,longitude,accuracy")
query.From(geonamesTableName)
if orderBy == "" {
orderBy = "postal_code"
}
query.OrderBy(orderBy)
if where != "" {
query.Where(where)
}
queryStr, queryArgs := query.Build()
queryStr = dbConn.Rebind(queryStr)
args = append(args, queryArgs...)
// fetch all places from the db
rows, err := dbConn.QueryContext(ctx, queryStr, args...)
if err != nil {
err = errors.Wrapf(err, "query - %s", query.String())
err = errors.WithMessage(err, "find regions failed")
return nil, err
}
// iterate over each row
resp := []*Geoname{}
for rows.Next() {
var (
v Geoname
err error
)
err = rows.Scan(&v.CountryCode,&v.PostalCode,&v.PlaceName,&v.StateName,&v.StateCode,&v.CountyName,&v.CountyCode,&v.CommunityName,&v.CommunityCode,&v.Latitude,&v.Longitude,&v.Accuracy)
if err != nil {
return nil, errors.WithStack(err)
}else if v.PostalCode == "" {
continue
}
resp = append(resp, &v)
}
return resp, nil
}
// FindGeonamePostalCodes ....
func FindGeonamePostalCodes(ctx context.Context, dbConn *sqlx.DB, where string, args ...interface{}) ([]string, error) {
span, ctx := tracer.StartSpanFromContext(ctx, "internal.geonames.FindGeonamePostalCodes")
defer span.Finish()
query := sqlbuilder.NewSelectBuilder()
query.Select("postal_code")
query.From(geonamesTableName)
if where != "" {
query.Where(where)
}
queryStr, queryArgs := query.Build()
queryStr = dbConn.Rebind(queryStr)
args = append(args, queryArgs...)
// fetch all places from the db
rows, err := dbConn.QueryContext(ctx, queryStr, args...)
if err != nil {
err = errors.Wrapf(err, "query - %s", query.String())
err = errors.WithMessage(err, "find regions failed")
return nil, err
}
// iterate over each row
resp := []string{}
for rows.Next() {
var (
v string
err error
)
err = rows.Scan(&v)
if err != nil {
return nil, errors.WithStack(err)
} else if v == "" {
continue
}
resp = append(resp, v)
}
return resp, nil
}
// FindGeonameRegions ....
func FindGeonameRegions(ctx context.Context, dbConn *sqlx.DB, orderBy, where string, args ...interface{}) ([]*Region, error) {
span, ctx := tracer.StartSpanFromContext(ctx, "internal.geonames.FindGeonameRegions")
defer span.Finish()
query := sqlbuilder.NewSelectBuilder()
query.Select("distinct state_code", "state_name")
query.From(geonamesTableName)
if orderBy == "" {
orderBy = "state_name"
}
query.OrderBy(orderBy)
if where != "" {
query.Where(where)
}
queryStr, queryArgs := query.Build()
queryStr = dbConn.Rebind(queryStr)
args = append(args, queryArgs...)
// fetch all places from the db
rows, err := dbConn.QueryContext(ctx, queryStr, args...)
if err != nil {
err = errors.Wrapf(err, "query - %s", query.String())
err = errors.WithMessage(err, "find regions failed")
return nil, err
}
// iterate over each row
resp := []*Region{}
for rows.Next() {
var (
v Region
err error
)
err = rows.Scan(&v.Code,&v.Name)
if err != nil {
err = errors.Wrapf(err, "query - %s", query.String())
return nil, err
} else if v.Code == "" {
continue
}
resp = append(resp, &v)
}
return resp, nil
}
// LoadGeonames enables streaming retrieval of GeoNames. The downloaded results
// will be written to the interface{} resultReceiver channel enabling processing the results while
// they're still being fetched. After all pages have been processed the channel is closed.
// Possible types sent to the channel are limited to:
// - error
// - GeoName
func LoadGeonames(ctx context.Context, rr chan<- interface{}, countries ...string) {
defer close(rr)
if len(countries) == 0 {
countries = ValidGeonameCountries
}
for _, country := range countries {
loadGeonameCountry(ctx, rr, country)
}
}
// loadGeonameCountry enables streaming retrieval of GeoNames. The downloaded results
// will be written to the interface{} resultReceiver channel enabling processing the results while
// they're still being fetched.
// Possible types sent to the channel are limited to:
// - error
// - GeoName
func loadGeonameCountry(ctx context.Context, rr chan<- interface{}, country string ) {
u := fmt.Sprintf("http://download.geonames.org/export/zip/%s.zip", country)
resp, err := pester.Get(u)
if err != nil {
rr <- errors.WithMessagef(err, "Failed to read countries from '%s'", u)
return
}
defer resp.Body.Close()
br := bufio.NewReader(resp.Body)
buff := bytes.NewBuffer([]byte{})
size, err := io.Copy(buff, br)
if err != nil {
rr <- errors.WithStack(err)
return
}
b := bytes.NewReader(buff.Bytes())
zr, err := zip.NewReader(b, size)
if err != nil {
rr <- errors.WithStack(err)
return
}
for _, f := range zr.File {
if f.Name == "readme.txt" {
continue
}
fh, err := f.Open()
if err != nil {
rr <- errors.WithStack(err)
return
}
scanner := bufio.NewScanner(fh)
for scanner.Scan() {
line := scanner.Text()
if strings.Contains(line, "\"") {
line = strings.Replace(line, "\"", "\\\"", -1)
}
r := csv.NewReader(strings.NewReader(line))
r.Comma = '\t' // Use tab-delimited instead of comma <---- here!
r.LazyQuotes = true
r.FieldsPerRecord = -1
lines, err := r.ReadAll()
if err != nil {
rr <- errors.WithStack(err)
continue
}
for _, row := range lines {
/*
fmt.Println("CountryCode: row[0]", row[0])
fmt.Println("PostalCode: row[1]", row[1])
fmt.Println("PlaceName: row[2]", row[2])
fmt.Println("StateName: row[3]", row[3])
fmt.Println("StateCode : row[4]", row[4])
fmt.Println("CountyName: row[5]", row[5])
fmt.Println("CountyCode : row[6]", row[6])
fmt.Println("CommunityName: row[7]", row[7])
fmt.Println("CommunityCode: row[8]", row[8])
fmt.Println("Latitude: row[9]", row[9])
fmt.Println("Longitude: row[10]", row[10])
fmt.Println("Accuracy: row[11]", row[11])
*/
gn := Geoname{
CountryCode: row[0],
PostalCode: row[1],
PlaceName: row[2],
StateName: row[3],
StateCode : row[4],
CountyName: row[5],
CountyCode : row[6],
CommunityName: row[7],
CommunityCode: row[8],
}
if row[9] != "" {
gn.Latitude, err = decimal.NewFromString(row[9])
if err != nil {
rr <- errors.WithStack(err)
}
}
if row[10] != "" {
gn.Longitude, err = decimal.NewFromString(row[10])
if err != nil {
rr <- errors.WithStack(err)
}
}
if row[11] != "" {
gn.Accuracy, err = strconv.Atoi(row[11])
if err != nil {
rr <- errors.WithStack(err)
}
}
rr <- gn
}
}
if err := scanner.Err(); err != nil {
rr <- errors.WithStack(err)
}
}
}

37
internal/geonames/models.go Normal file
View File

@ -0,0 +1,37 @@
package geonames
import "github.com/shopspring/decimal"
type Geoname struct {
CountryCode string // US
PostalCode string // 99686
PlaceName string // Valdez
StateName string // Alaska
StateCode string // AK
CountyName string // Valdez-Cordova
CountyCode string // 261
CommunityName string //
CommunityCode string //
Latitude decimal.Decimal // 61.101
Longitude decimal.Decimal // -146.9
Accuracy int // 1
}
type Country struct {
Code string // US
Name string
IsoAlpha3 string
Capital string
CurrencyCode string // .us
CurrencyName string // USD Dollar
Phone string // 1
PostalCodeFormat string // #####-####
PostalCodeRegex string // ^\d{5}(-\d{4})?$
}
type Region struct {
Code string // AK
Name string // Alaska
}

7
internal/platform/web/weberror/error.go
View File

@ -68,7 +68,12 @@ func NewError(ctx context.Context, er error, status int) error {
// Error implements the error interface. It uses the default message of the
// wrapped error. This is what will be shown in the services' logs.
func (err *Error) Error() string {
return err.Err.Error()
if err.Err != nil {
return err.Err.Error()
} else if err.Cause != nil {
return err.Cause.Error()
}
return err.Message
}
// Display renders an error that can be returned as ErrorResponse to the user via the API.

377
internal/schema/init_schema.go
View File

@ -1,17 +1,388 @@
package schema
import (
"github.com/jmoiron/sqlx"
"log"
"github.com/jmoiron/sqlx"
)
// initSchema runs before any migrations are executed. This happens when no other migrations
// have previously been executed.
func initSchema(db *sqlx.DB, log *log.Logger) func(*sqlx.DB) error {
f := func(*sqlx.DB) error {
f := func(db *sqlx.DB) error {
return nil
}
return f
}
/*
// initGeonames populates countries and postal codes.
func initGeonamesOld(db *sqlx.DB) error {
schemas := []string{
`DROP TABLE IF EXISTS geoname`,
`create table geoname (
geonameid int,
name varchar(200),
asciiname varchar(200),
alternatenames text,
latitude float,
longitude float,
fclass char(1),
fcode varchar(10),
country varchar(2),
cc2 varchar(600),
admin1 varchar(20),
admin2 varchar(80),
admin3 varchar(20),
admin4 varchar(20),
population bigint,
elevation int,
gtopo30 int,
timezone varchar(40),
moddate date)`,
`DROP TABLE IF EXISTS countryinfo`,
`CREATE TABLE countryinfo (
iso_alpha2 char(2),
iso_alpha3 char(3),
iso_numeric integer,
fips_code character varying(3),
country character varying(200),
capital character varying(200),
areainsqkm double precision,
population integer,
continent char(2),
tld CHAR(10),
currency_code char(3),
currency_name CHAR(20),
phone character varying(20),
postal character varying(60),
postal_format character varying(200),
postal_regex character varying(200),
languages character varying(200),
geonameId int,
neighbours character varying(50),
equivalent_fips_code character varying(3))`,
}
for _, q := range schemas {
_, err := db.Exec(q)
if err != nil {
return errors.WithMessagef(err, "Failed to execute sql query '%s'", q)
}
}
// Load the countryinfo table.
if false {
u := "http://download.geonames.org/export/dump/countryInfo.txt"
resp, err := pester.Get(u)
if err != nil {
return errors.WithMessagef(err, "Failed to read country info from '%s'", u)
}
defer resp.Body.Close()
scanner := bufio.NewScanner(resp.Body)
var prevLine string
var stmt *sql.Stmt
for scanner.Scan() {
line := scanner.Text()
// Skip comments.
if strings.HasPrefix(line, "#") {
prevLine = line
continue
}
// Pull the last comment to load the fields.
if stmt == nil {
prevLine = strings.TrimPrefix(prevLine, "#")
r := csv.NewReader(strings.NewReader(prevLine))
r.Comma = '\t' // Use tab-delimited instead of comma <---- here!
r.FieldsPerRecord = -1
lines, err := r.ReadAll()
if err != nil {
return errors.WithStack(err)
}
var columns []string
for _, fn := range lines[0] {
var cn string
switch fn {
case "ISO":
cn = "iso_alpha2"
case "ISO3":
cn = "iso_alpha3"
case "ISO-Numeric":
cn = "iso_numeric"
case "fips":
cn = "fips_code"
case "Country":
cn = "country"
case "Capital":
cn = "capital"
case "Area(in sq km)":
cn = "areainsqkm"
case "Population":
cn = "population"
case "Continent":
cn = "continent"
case "tld":
cn = "tld"
case "CurrencyCode":
cn = "currency_code"
case "CurrencyName":
cn = "currency_name"
case "Phone":
cn = "phone"
case "Postal":
cn = "postal"
case "Postal Code Format":
cn = "postal_format"
case "Postal Code Regex":
cn = "postal_regex"
case "Languages":
cn = "languages"
case "geonameid":
cn = "geonameId"
case "neighbours":
cn = "neighbours"
case "EquivalentFipsCode":
cn = "equivalent_fips_code"
default :
return errors.Errorf("Failed to map column %s", fn)
}
columns = append(columns, cn)
}
placeholders := []string{}
for i := 0; i < len(columns); i++ {
placeholders = append(placeholders, "?")
}
q := "insert into countryinfo ("+strings.Join(columns, ",")+") values("+strings.Join(placeholders, ",")+")"
q = db.Rebind(q)
stmt, err = db.Prepare(q)
if err != nil {
return errors.WithMessagef(err, "Failed to prepare sql query '%s'", q)
}
}
r := csv.NewReader(strings.NewReader(line))
r.Comma = '\t' // Use tab-delimited instead of comma <---- here!
r.FieldsPerRecord = -1
lines, err := r.ReadAll()
if err != nil {
return errors.WithStack(err)
}
for _, row := range lines {
var args []interface{}
for _, v := range row {
args = append(args, v)
}
_, err = stmt.Exec(args...)
if err != nil {
return errors.WithStack(err)
}
}
}
if err := scanner.Err(); err != nil {
return errors.WithStack(err)
}
}
// Load the geoname table.
{
u := "http://download.geonames.org/export/dump/allCountries.zip"
resp, err := pester.Get(u)
if err != nil {
return errors.WithMessagef(err, "Failed to read countries from '%s'", u)
}
defer resp.Body.Close()
br := bufio.NewReader(resp.Body)
buff := bytes.NewBuffer([]byte{})
size, err := io.Copy(buff, br)
if err != nil {
return err
}
b := bytes.NewReader(buff.Bytes())
zr, err := zip.NewReader(b, size)
if err != nil {
return errors.WithStack(err)
}
q := "insert into geoname " +
"(geonameid,name,asciiname,alternatenames,latitude,longitude,fclass,fcode,country,cc2,admin1,admin2,admin3,admin4,population,elevation,gtopo30,timezone,moddate) " +
"values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
q = db.Rebind(q)
stmt, err := db.Prepare(q)
if err != nil {
return errors.WithMessagef(err, "Failed to prepare sql query '%s'", q)
}
for _, f := range zr.File {
if f.Name == "readme.txt" {
continue
}
fh, err := f.Open()
if err != nil {
return errors.WithStack(err)
}
scanner := bufio.NewScanner(fh)
for scanner.Scan() {
line := scanner.Text()
// Skip comments.
if strings.HasPrefix(line, "#") {
continue
}
if strings.Contains(line, "\"") {
line = strings.Replace(line, "\"", "\\\"", -1)
}
r := csv.NewReader(strings.NewReader(line))
r.Comma = '\t' // Use tab-delimited instead of comma <---- here!
r.LazyQuotes = true
r.FieldsPerRecord = -1
lines, err := r.ReadAll()
if err != nil {
return errors.WithStack(err)
}
for _, row := range lines {
var args []interface{}
for idx, v := range row {
if v == "" {
if idx == 0 || idx == 14 || idx == 15 {
v = "0"
}
}
args = append(args, v)
}
_, err = stmt.Exec(args...)
if err != nil {
return errors.WithStack(err)
}
}
}
if err := scanner.Err(); err != nil {
return errors.WithStack(err)
}
}
}
return errors.New("not finished")
queries := []string{
// Countries...
`DROP TABLE IF EXISTS countries`,
`CREATE TABLE countries(
id serial not null constraint countries_pkey primary key,
geoname_id int,
iso char(2),
country character varying(50),
capital character varying(50),
created_at TIMESTAMP WITH TIME ZONE NOT NULL,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NULL,
archived_at TIMESTAMP WITH TIME ZONE DEFAULT NULL)`,
`create index idx_countries_deleted_at on countries (deleted_at)`,
`insert into countries(geoname_id, iso, country, capital, created_at, updated_at)
select geonameId, iso_alpha2, country, capital, NOW(), NOW()
from countryinfo`,
// Regions...
`DROP TABLE IF EXISTS regions`,
`CREATE TABLE regions (
id serial not null constraint regions_pkey primary key,
country_id int,
geoname_id int,
name varchar(200),
ascii_name varchar(200),
adm varchar(20),
country char(2),
latitude float,
longitude float,
created_at TIMESTAMP WITH TIME ZONE NOT NULL,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NULL,
archived_at TIMESTAMP WITH TIME ZONE DEFAULT NULL)`,
`create index idx_regions_deleted_at on regions (deleted_at)`,
`insert into regions(country_id, geoname_id, name, ascii_name, adm, country, latitude, longitude, created_at, updated_at)
select c.id,
g.geonameid,
g.name,
g.asciiname,
g.admin1,
c.iso,
g.latitude,
g.longitude,
to_timestamp(TO_CHAR(g.moddate, 'YYYY-MM-DD'), 'YYYY-MM-DD'),
to_timestamp(TO_CHAR(g.moddate, 'YYYY-MM-DD'), 'YYYY-MM-DD')
from countries as c
inner join geoname as g on c.iso = g.country and g.fcode like 'ADM1'`,
// cities
`DROP TABLE IF EXISTS cities`,
`CREATE TABLE cities (
id serial not null constraint cities_pkey primary key,
country_id int,
region_id int,
geoname_id int,
name varchar(200),
ascii_name varchar(200),
latitude float,
longitude float,
created_at TIMESTAMP WITH TIME ZONE NOT NULL,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NULL,
archived_at TIMESTAMP WITH TIME ZONE DEFAULT NULL)`,
`create index idx_cities_deleted_at on cities (deleted_at)`,
`insert into cities(country_id, region_id, geoname_id, name, ascii_name, latitude, longitude, created_at, updated_at)
select r.country_id,
r.id,
g.geonameid,
g.name,
g.asciiname,
g.latitude,
g.longitude,
to_timestamp(TO_CHAR(g.moddate, 'YYYY-MM-DD'), 'YYYY-MM-DD'),
to_timestamp(TO_CHAR(g.moddate, 'YYYY-MM-DD'), 'YYYY-MM-DD')
from geoname as g
join regions as r on r.adm = g.admin1
and r.country = g.country
and (g.fcode in ('PPLC', 'PPLA') or (g.fcode like 'PPLA%' and g.population >= 50000));`,
}
tx, err := db.Begin()
if err != nil {
return errors.WithStack(err)
}
for _, q := range queries {
_, err = tx.Exec(q)
if err != nil {
return errors.WithMessagef(err, "Failed to execute sql query '%s'", q)
}
}
err = tx.Commit()
if err != nil {
return errors.WithStack(err)
}
return nil
}
*/

265
internal/schema/migrations.go
View File

@ -1,9 +1,15 @@
package schema
import (
"bufio"
"context"
"database/sql"
"encoding/csv"
"log"
"strings"
"geeks-accelerator/oss/saas-starter-kit/internal/geonames"
"github.com/sethgrid/pester"
"github.com/geeks-accelerator/sqlxmigrate"
"github.com/jmoiron/sqlx"
_ "github.com/lib/pq"
@ -181,7 +187,7 @@ func migrationList(db *sqlx.DB, log *log.Logger) []*sqlxmigrate.Migration {
},
// Split users.name into first_name and last_name columns.
{
ID: "201907-29-01a",
ID: "20190729-01a",
Migrate: func(tx *sql.Tx) error {
q1 := `ALTER TABLE users
RENAME COLUMN name to first_name;`
@ -205,5 +211,262 @@ func migrationList(db *sqlx.DB, log *log.Logger) []*sqlxmigrate.Migration {
return nil
},
},
// Load new geonames table.
{
ID: "20190731-02b",
Migrate: func(tx *sql.Tx) error {
schemas := []string{
`DROP TABLE IF EXISTS geonames`,
`CREATE TABLE geonames (
country_code char(2),
postal_code character varying(60),
place_name character varying(200),
state_name character varying(200),
state_code character varying(10),
county_name character varying(200),
county_code character varying(10),
community_name character varying(200),
community_code character varying(10),
latitude float,
longitude float,
accuracy int)`,
}
for _, q := range schemas {
_, err := db.Exec(q)
if err != nil {
return errors.WithMessagef(err, "Failed to execute sql query '%s'", q)
}
}
q := "insert into geonames " +
"(country_code,postal_code,place_name,state_name,state_code,county_name,county_code,community_name,community_code,latitude,longitude,accuracy) " +
"values(?,?,?,?,?,?,?,?,?,?,?,?)"
q = db.Rebind(q)
stmt, err := db.Prepare(q)
if err != nil {
return errors.WithMessagef(err, "Failed to prepare sql query '%s'", q)
}
resChan := make(chan interface{})
go geonames.LoadGeonames(context.Background(), resChan)
for r := range resChan {
switch v := r.(type) {
case geonames.Geoname:
_, err = stmt.Exec(v.CountryCode,v.PostalCode,v.PlaceName,v.StateName,v.StateCode,v.CountyName,v.CountyCode,v.CommunityName,v.CommunityCode,v.Latitude,v.Longitude,v.Accuracy)
if err != nil {
return errors.WithStack(err)
}
case error :
return v
}
}
queries := []string{
`create index idx_geonames_country_code on geonames (country_code)`,
`create index idx_geonames_postal_code on geonames (postal_code)`,
}
for _, q := range queries {
_, err := db.Exec(q)
if err != nil {
return errors.WithMessagef(err, "Failed to execute sql query '%s'", q)
}
}
return nil
},
Rollback: func(tx *sql.Tx) error {
return nil
},
},
// Load new countries table.
{
ID: "20190731-02d",
Migrate: func(tx *sql.Tx) error {
prep := []string{
`DROP TABLE IF EXISTS countryinfo`,
`CREATE TABLE countryinfo (
iso_alpha2 char(2),
iso_alpha3 char(3),
iso_numeric integer,
fips_code character varying(3),
country character varying(200),
capital character varying(200),
areainsqkm double precision,
population integer,
continent char(2),
tld CHAR(10),
currency_code char(3),
currency_name CHAR(20),
phone character varying(20),
postal_format character varying(200),
postal_regex character varying(200),
languages character varying(200),
geonameId int,
neighbours character varying(50),
equivalent_fips_code character varying(3))`,
}
for _, q := range prep {
_, err := db.Exec(q)
if err != nil {
return errors.WithMessagef(err, "Failed to execute sql query '%s'", q)
}
}
u := "http://download.geonames.org/export/dump/countryInfo.txt"
resp, err := pester.Get(u)
if err != nil {
return errors.WithMessagef(err, "Failed to read country info from '%s'", u)
}
defer resp.Body.Close()
scanner := bufio.NewScanner(resp.Body)
var prevLine string
var stmt *sql.Stmt
for scanner.Scan() {
line := scanner.Text()
// Skip comments.
if strings.HasPrefix(line, "#") {
prevLine = line
continue
}
// Pull the last comment to load the fields.
if stmt == nil {
prevLine = strings.TrimPrefix(prevLine, "#")
r := csv.NewReader(strings.NewReader(prevLine))
r.Comma = '\t' // Use tab-delimited instead of comma <---- here!
r.FieldsPerRecord = -1
lines, err := r.ReadAll()
if err != nil {
return errors.WithStack(err)
}
var columns []string
for _, fn := range lines[0] {
var cn string
switch fn {
case "ISO":
cn = "iso_alpha2"
case "ISO3":
cn = "iso_alpha3"
case "ISO-Numeric":
cn = "iso_numeric"
case "fips":
cn = "fips_code"
case "Country":
cn = "country"
case "Capital":
cn = "capital"
case "Area(in sq km)":
cn = "areainsqkm"
case "Population":
cn = "population"
case "Continent":
cn = "continent"
case "tld":
cn = "tld"
case "CurrencyCode":
cn = "currency_code"
case "CurrencyName":
cn = "currency_name"
case "Phone":
cn = "phone"
case "Postal Code Format":
cn = "postal_format"
case "Postal Code Regex":
cn = "postal_regex"
case "Languages":
cn = "languages"
case "geonameid":
cn = "geonameId"
case "neighbours":
cn = "neighbours"
case "EquivalentFipsCode":
cn = "equivalent_fips_code"
default :
return errors.Errorf("Failed to map column %s", fn)
}
columns = append(columns, cn)
}
placeholders := []string{}
for i := 0; i < len(columns); i++ {
placeholders = append(placeholders, "?")
}
q := "insert into countryinfo ("+strings.Join(columns, ",")+") values("+strings.Join(placeholders, ",")+")"
q = db.Rebind(q)
stmt, err = db.Prepare(q)
if err != nil {
return errors.WithMessagef(err, "Failed to prepare sql query '%s'", q)
}
}
r := csv.NewReader(strings.NewReader(line))
r.Comma = '\t' // Use tab-delimited instead of comma <---- here!
r.FieldsPerRecord = -1
lines, err := r.ReadAll()
if err != nil {
return errors.WithStack(err)
}
for _, row := range lines {
var args []interface{}
for _, v := range row {
args = append(args, v)
}
_, err = stmt.Exec(args...)
if err != nil {
return errors.WithStack(err)
}
}
}
if err := scanner.Err(); err != nil {
return errors.WithStack(err)
}
queries := []string{
// Countries...
`DROP TABLE IF EXISTS countries`,
`CREATE TABLE countries(
code char(2) not null constraint countries_pkey primary key,
iso_alpha3 char(3),
name character varying(50),
capital character varying(50),
currency_code char(3),
currency_name CHAR(20),
phone character varying(20),
postal_code_format character varying(200),
postal_code_regex character varying(200))`,
`insert into countries(code, iso_alpha3, name, capital, currency_code, currency_name, phone, postal_code_format, postal_code_regex)
select iso_alpha2, iso_alpha3, country, capital, currency_code, currency_name, phone, postal_format, postal_regex
from countryinfo`,
`DROP TABLE IF EXISTS countryinfo`,
}
for _, q := range queries {
_, err := db.Exec(q)
if err != nil {
return errors.WithMessagef(err, "Failed to execute sql query '%s'", q)
}
}
return nil
},
Rollback: func(tx *sql.Tx) error {
return nil
},
},
}
}