go/golang-saas-starter-kit
1
0
Fork 0
mirror of https://github.com/raseels-repos/golang-saas-starter-kit.git synced 2025-07-03 00:58:13 +02:00
Code Issues Releases Activity

Remove a bunch of invalid country codes for geonames

Browse Source
This commit is contained in:
Lee Brown
2020-01-14 10:26:12 -08:00
parent 7ab573c670
commit 8aca18a353
3 changed files with 20 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
internal/geonames/geonames.go
View File

@ -17,9 +17,7 @@ import (
"time"
"geeks-accelerator/oss/saas-starter-kit/internal/platform/web/webcontext"
"github.com/huandu/go-sqlbuilder"
// "github.com/jmoiron/sqlx"
"github.com/pkg/errors"
"github.com/sethgrid/pester"
"github.com/shopspring/decimal"
@ -37,15 +35,15 @@ func ValidGeonameCountries(ctx context.Context) []string {
return []string{"US"}
}
return []string{
"AD", "AR", "AS", "AT", "AU", "AX", "BD", "BE", "BG", "BM",
"BR", "BY", "CA", "CH", "CO", "CR", "CZ", "DE", "DK", "DO",
"AD", "AR", "AS", "AT", "AU", "AX", "BD", "BE", "BG",
"BR", "CA", "CH", "CZ", "DE", "DK", "DO",
"DZ", "ES", "FI", "FO", "FR", "GB", "GF", "GG", "GL", "GP",
"GT", "GU", "HR", "HU", "IE", "IM", "IN", "IS", "IT", "JE",
"JP", "LI", "LK", "LT", "LU", "LV", "MC", "MD", "MH", "MK",
"MP", "MQ", "MT", "MX", "MY", "NC", "NL", "NO", "NZ", "PH",
"GT", "GU", "HR", "HU", "IM", "IN", "IS", "IT", "JE",
"JP", "LI", "LK", "LT", "LU", "MC", "MD", "MH", "MK",
"MP", "MQ", "MX", "MY", "NL", "NO", "NZ", "PH",
"PK", "PL", "PM", "PR", "PT", "RE", "RO", "RU", "SE", "SI",
"SJ", "SK", "SM", "TH", "TR", "UA", "US", "UY", "VA", "VI",
"WF", "YT", "ZA"}
"SJ", "SK", "SM", "TH", "TR", "US", "VA", "VI",
"YT", "ZA"}
}
// FindGeonames ....

15
internal/schema/migrations.go
View File

@ -245,6 +245,9 @@ func migrationList(ctx context.Context, db *sqlx.DB, log *log.Logger, isUnittest
countries = []string{"US"}
}
fmt.Println("isUnittest", isUnittest)
fmt.Println("countries", countries)
ncol := 12
fn := func(geoNames []geonames.Geoname) error {
valueStrings := make([]string, 0, len(geoNames))
@ -282,10 +285,10 @@ func migrationList(ctx context.Context, db *sqlx.DB, log *log.Logger, isUnittest
}
start := time.Now()
for _, country := range countries {
//fmt.Println("LoadGeonames: start country: ", country)
log.Println("LoadGeonames: start country: ", country)
v, err := geoRepo.GetGeonameCountry(context.Background(), country)
if err != nil {
return errors.WithStack(err)
return errors.WithMessagef(err, "Failed to load country %s", country)
}
//fmt.Println("Geoname records: ", len(v))
// Max argument values of Postgres is about 54460. So the batch size for bulk insert is selected 4500*12 (ncol)
@ -297,25 +300,25 @@ func migrationList(ctx context.Context, db *sqlx.DB, log *log.Logger, isUnittest
if n == 0 {
err := fn(v)
if err != nil {
return errors.WithStack(err)
return err
}
} else {
for i := 0; i < n; i++ {
vn := v[i*batch : (i+1)*batch]
err := fn(vn)
if err != nil {
return errors.WithStack(err)
return err
}
if n > 0 && n%25 == 0 {
time.Sleep(200)
}
}
if len(v)%batch > 0 {
fmt.Println("Remain part: ", len(v)-n*batch)
log.Printf("Remain part: %d\n", len(v)-n*batch)
vn := v[n*batch:]
err := fn(vn)
if err != nil {
return errors.WithStack(err)
return err
}
}
}