go/golang-saas-starter-kit
1
0
Fork 0
mirror of https://github.com/raseels-repos/golang-saas-starter-kit.git synced 2025-07-03 00:58:13 +02:00
Code Issues Releases Activity

Remove a bunch of invalid country codes for geonames

Browse Source
This commit is contained in:
Lee Brown
2020-01-14 10:26:12 -08:00
parent 7ab573c670
commit 8aca18a353
3 changed files with 20 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
build/cicd/internal/config/config.go
View File

@ -477,8 +477,8 @@ func NewConfig(log *log.Logger, targetEnv Env, awsCredentials devdeploy.AwsCrede
"s3:ListBucket", "s3:ListBucket",
}, },
Resource: []string{ Resource: []string{
"arn:aws:s3:::"+cfg.AwsS3BucketPublic.BucketName, "arn:aws:s3:::" + cfg.AwsS3BucketPublic.BucketName,
"arn:aws:s3:::"+cfg.AwsS3BucketPrivate.BucketName, "arn:aws:s3:::" + cfg.AwsS3BucketPrivate.BucketName,
}, },
}, },
{ {
@ -489,8 +489,8 @@ func NewConfig(log *log.Logger, targetEnv Env, awsCredentials devdeploy.AwsCrede
"s3:GetObject", "s3:GetObject",
}, },
Resource: []string{ Resource: []string{
"arn:aws:::"+cfg.AwsS3BucketPublic.BucketName+"/*", "arn:aws:::" + cfg.AwsS3BucketPublic.BucketName + "/*",
"arn:aws:::"+cfg.AwsS3BucketPrivate.BucketName+"/*", "arn:aws:::" + cfg.AwsS3BucketPrivate.BucketName + "/*",
}, },
}, },
{ {

16
internal/geonames/geonames.go
View File

@ -17,9 +17,7 @@ import (
"time" "time"
"geeks-accelerator/oss/saas-starter-kit/internal/platform/web/webcontext" "geeks-accelerator/oss/saas-starter-kit/internal/platform/web/webcontext"
"github.com/huandu/go-sqlbuilder" "github.com/huandu/go-sqlbuilder"
// "github.com/jmoiron/sqlx"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/sethgrid/pester" "github.com/sethgrid/pester"
"github.com/shopspring/decimal" "github.com/shopspring/decimal"
@ -37,15 +35,15 @@ func ValidGeonameCountries(ctx context.Context) []string {
return []string{"US"} return []string{"US"}
} }
return []string{ return []string{
"AD", "AR", "AS", "AT", "AU", "AX", "BD", "BE", "BG", "BM", "AD", "AR", "AS", "AT", "AU", "AX", "BD", "BE", "BG",
"BR", "BY", "CA", "CH", "CO", "CR", "CZ", "DE", "DK", "DO", "BR", "CA", "CH", "CZ", "DE", "DK", "DO",
"DZ", "ES", "FI", "FO", "FR", "GB", "GF", "GG", "GL", "GP", "DZ", "ES", "FI", "FO", "FR", "GB", "GF", "GG", "GL", "GP",
"GT", "GU", "HR", "HU", "IE", "IM", "IN", "IS", "IT", "JE", "GT", "GU", "HR", "HU", "IM", "IN", "IS", "IT", "JE",
"JP", "LI", "LK", "LT", "LU", "LV", "MC", "MD", "MH", "MK", "JP", "LI", "LK", "LT", "LU", "MC", "MD", "MH", "MK",
"MP", "MQ", "MT", "MX", "MY", "NC", "NL", "NO", "NZ", "PH", "MP", "MQ", "MX", "MY", "NL", "NO", "NZ", "PH",
"PK", "PL", "PM", "PR", "PT", "RE", "RO", "RU", "SE", "SI", "PK", "PL", "PM", "PR", "PT", "RE", "RO", "RU", "SE", "SI",
"SJ", "SK", "SM", "TH", "TR", "UA", "US", "UY", "VA", "VI", "SJ", "SK", "SM", "TH", "TR", "US", "VA", "VI",
"WF", "YT", "ZA"} "YT", "ZA"}
} }
// FindGeonames .... // FindGeonames ....

15
internal/schema/migrations.go
View File

@ -245,6 +245,9 @@ func migrationList(ctx context.Context, db *sqlx.DB, log *log.Logger, isUnittest
countries = []string{"US"} countries = []string{"US"}
} }
fmt.Println("isUnittest", isUnittest)
fmt.Println("countries", countries)
ncol := 12 ncol := 12
fn := func(geoNames []geonames.Geoname) error { fn := func(geoNames []geonames.Geoname) error {
valueStrings := make([]string, 0, len(geoNames)) valueStrings := make([]string, 0, len(geoNames))
@ -282,10 +285,10 @@ func migrationList(ctx context.Context, db *sqlx.DB, log *log.Logger, isUnittest
} }
start := time.Now() start := time.Now()
for _, country := range countries { for _, country := range countries {
//fmt.Println("LoadGeonames: start country: ", country) log.Println("LoadGeonames: start country: ", country)
v, err := geoRepo.GetGeonameCountry(context.Background(), country) v, err := geoRepo.GetGeonameCountry(context.Background(), country)
if err != nil { if err != nil {
return errors.WithStack(err) return errors.WithMessagef(err, "Failed to load country %s", country)
} }
//fmt.Println("Geoname records: ", len(v)) //fmt.Println("Geoname records: ", len(v))
// Max argument values of Postgres is about 54460. So the batch size for bulk insert is selected 4500*12 (ncol) // Max argument values of Postgres is about 54460. So the batch size for bulk insert is selected 4500*12 (ncol)
@ -297,25 +300,25 @@ func migrationList(ctx context.Context, db *sqlx.DB, log *log.Logger, isUnittest
if n == 0 { if n == 0 {
err := fn(v) err := fn(v)
if err != nil { if err != nil {
return errors.WithStack(err) return err
} }
} else { } else {
for i := 0; i < n; i++ { for i := 0; i < n; i++ {
vn := v[i*batch : (i+1)*batch] vn := v[i*batch : (i+1)*batch]
err := fn(vn) err := fn(vn)
if err != nil { if err != nil {
return errors.WithStack(err) return err
} }
if n > 0 && n%25 == 0 { if n > 0 && n%25 == 0 {
time.Sleep(200) time.Sleep(200)
} }
} }
if len(v)%batch > 0 { if len(v)%batch > 0 {
fmt.Println("Remain part: ", len(v)-n*batch) log.Printf("Remain part: %d\n", len(v)-n*batch)
vn := v[n*batch:] vn := v[n*batch:]
err := fn(vn) err := fn(vn)
if err != nil { if err != nil {
return errors.WithStack(err) return err
} }
} }
} }