checkpoint

example-project/cmd/web-api/handlers/check.go

@@ -28,10 +28,10 @@ func (c *Check) Health(ctx context.Context, w http.ResponseWriter, r *http.Reque
 	}
 
 	// check redis
-	//err = c.Redis.Ping().Err()
+	err = c.Redis.Ping().Err()
-	//if err != nil {
+	if err != nil {
-	//	return errors.Wrap(err, "Redis failed")
+		return errors.Wrap(err, "Redis failed")
-	//}
+	}
 
 	status := struct {
 		Status string `json:"status"`
@@ -41,3 +41,11 @@ func (c *Check) Health(ctx context.Context, w http.ResponseWriter, r *http.Reque
 
 	return web.RespondJson(ctx, w, status, http.StatusOK)
 }
+
+// Ping validates the service is ready to accept requests.
+func (c *Check) Ping(ctx context.Context, w http.ResponseWriter, r *http.Request, params map[string]string) error {
+
+	status := "pong"
+
+	return web.RespondJson(ctx, w, status, http.StatusOK)
+}

example-project/cmd/web-api/handlers/routes.go

@@ -23,8 +23,10 @@ func API(shutdown chan os.Signal, log *log.Logger, masterDB *sqlx.DB, redis *red
 	// Register health check endpoint. This route is not authenticated.
 	check := Check{
 		MasterDB: masterDB,
+		Redis: redis,
 	}
 	app.Handle("GET", "/v1/health", check.Health)
+	app.Handle("GET", "/ping", check.Ping)
 
 	// Register user management and authentication endpoints.
 	u := User{

example-project/go.mod

@@ -1,6 +1,7 @@
 module geeks-accelerator/oss/saas-starter-kit/example-project
 
 require (
+	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
 	github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc
 	github.com/aws/aws-sdk-go v1.20.16
 	github.com/bobesa/go-domain-util v0.0.0-20180815122459-1d708c097a6a
@@ -10,6 +11,7 @@ require (
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/docker v0.7.3-0.20180815000130-e05b657120a6
 	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
 	github.com/dustin/go-humanize v1.0.0
 	github.com/fatih/camelcase v1.0.0
@@ -43,6 +45,7 @@ require (
 	github.com/pborman/uuid v1.2.0
 	github.com/philhofer/fwd v1.0.0 // indirect
 	github.com/pkg/errors v0.8.1
+	github.com/prometheus/client_golang v1.0.0 // indirect
 	github.com/sergi/go-diff v1.0.0
 	github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0
 	github.com/sirupsen/logrus v1.4.2 // indirect
@@ -55,6 +58,7 @@ require (
 	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
 	golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect
 	golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb // indirect
+	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
 	golang.org/x/tools v0.0.0-20190708203411-c8855242db9c // indirect
 	google.golang.org/appengine v1.6.1 // indirect
 	google.golang.org/grpc v1.22.0 // indirect

example-project/go.sum

@@ -1,6 +1,8 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
+github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
+github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
 github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
@@ -8,12 +10,16 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/aws/aws-sdk-go v1.19.33 h1:qz9ZQtxCUuwBKdc5QiY6hKuISYGeRQyLVA2RryDEDaQ=
 github.com/aws/aws-sdk-go v1.19.33/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.20.15 h1:y9ts8MJhB7ReUidS6Rq+0KxdFeL01J+pmOlGq6YqpiQ=
 github.com/aws/aws-sdk-go v1.20.15/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.20.16 h1:Dq68fBH39XnSjjb2hX/iW6mui8JtXcVAuhRYGSRiisY=
 github.com/aws/aws-sdk-go v1.20.16/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/bobesa/go-domain-util v0.0.0-20180815122459-1d708c097a6a/go.mod h1:/mf0HzRK9xVv+1puqGSMzCo7bhEcQhiisuUXlMkq2p4=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc h1:TP+534wVlf61smEIq1nwLLAjQVEK2EADoW3CX9AuT+8=
@@ -37,6 +43,7 @@ github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
 github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
@@ -51,6 +58,8 @@ github.com/geeks-accelerator/sqlxmigrate v0.0.0-20190527223850-4a863a2d30db h1:m
 github.com/geeks-accelerator/sqlxmigrate v0.0.0-20190527223850-4a863a2d30db/go.mod h1:dzpCjo4q7chhMVuHDzs/odROkieZ5Wjp70rNDuX83jU=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
 github.com/go-openapi/jsonpointer v0.19.2 h1:A9+F4Dc/MCNB5jibxf6rRvOvR/iFgQdyNx9eIhnGqq0=
 github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
@@ -74,6 +83,8 @@ github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8w
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -100,6 +111,8 @@ github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kelseyhightower/envconfig v1.3.0 h1:IvRS4f2VcIQy6j4ORGIf9145T/AsUB+oY8LyvN8BXNM=
 github.com/kelseyhightower/envconfig v1.3.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
 github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
@@ -107,6 +120,7 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -128,6 +142,11 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN
 github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o=
 github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -149,14 +168,27 @@ github.com/pborman/uuid v0.0.0-20180122190007-c65b2f87fee3/go.mod h1:VyrYX9gd7ir
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
 github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0 h1:X9XMOYjxEfAYSy3xK1DzO5dMkkWhs9E9UCcS1IERx2k=
 github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0/go.mod h1:Ad7IjTpvzZO8Fl0vh9AzQ+j/jYZfyp2diGwI8m5q+ns=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -177,6 +209,7 @@ github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 h1:zzrxE1FKn5ryBNl9eKOeqQ58Y/Qpo3Q9QNxKHX5uzzQ=
 github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2/go.mod h1:hzfGeIUDq/j97IG+FhNqkowIyEcD88LrW6fyU3K3WqY=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -188,6 +221,7 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225 h1:kNX+jCowfMYzvlSvJu5pQWEmy
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
@@ -197,9 +231,13 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -213,6 +251,7 @@ golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -233,6 +272,7 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA
 google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 gopkg.in/DataDog/dd-trace-go.v1 v1.15.0 h1:2LhklnAJsRSelbnBrrE5QuRleRDkmOh2JWxOtIX6yec=
 gopkg.in/DataDog/dd-trace-go.v1 v1.15.0/go.mod h1:DVp8HmDh8PuTu2Z0fVVlBsyWaC++fzwVCaGWylTe3tg=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

example-project/tools/truss/cmd/devops/README.md

@@ -8,13 +8,13 @@
       "Sid": "ServiceDeployPermissions",    
       "Effect": "Allow",
       "Action": [
-        "cloudwatchlogs:DescribeLogGroups",
-	    "cloudwatchlogs:CreateLogGroup",
         "ec2:DescribeSubnets",
         "ec2:DescribeSubnets",
         "ec2:DescribeSecurityGroups",
         "ec2:CreateSecurityGroup",
         "ec2:AuthorizeSecurityGroupIngress",
+        "ec2:DescribeNetworkInterfaces",
+        "ec2:DescribeVpcs",
         "elasticache:DescribeCacheClusters",
         "elasticache:CreateCacheCluster",
         "elasticache:DescribeCacheParameterGroups",
@@ -29,6 +29,7 @@
         "elasticloadbalancing:ModifyTargetGroupAttributes",
         "ecs:CreateCluster",
         "ecs:CreateService",
+        "ecs:DeleteService",
         "ecs:DescribeClusters",
         "ecs:DescribeServices",
         "ecs:UpdateService",
@@ -64,6 +65,8 @@
 	    "iam:ListPolicies",
 	    "iam:GetPolicyVersion",
 	    "iam:CreatePolicyVersion",
+        "logs:DescribeLogGroups",
+        "logs:CreateLogGroup",
 	    "logs:DescribeLogStreams",
 	    "logs:CreateExportTask",
 	    "logs:DescribeExportTasks",
@@ -85,11 +88,18 @@
 	    "s3:PutBucketPolicy",
         "s3:PutBucketPublicAccessBlock",
         "route53:CreateHostedZone",
+        "route53:ChangeResourceRecordSets",
         "route53:ListHostedZones",
         "secretsmanager:CreateSecret",
         "secretsmanager:ListSecrets",
         "secretsmanager:GetSecretValue",
-        "secretsmanager:UpdateSecret"
+        "secretsmanager:UpdateSecret",
+        "servicediscovery:ListNamespaces",
+        "servicediscovery:CreatePrivateDnsNamespace",
+        "servicediscovery:GetOperation",
+        "servicediscovery:ListServices",
+        "servicediscovery:CreateService",
+        "servicediscovery:GetService"
       ],
       "Resource": "*"
     },

example-project/tools/truss/cmd/devops/models.go

@@ -2,7 +2,9 @@ package devops
 
 import (
 	"github.com/aws/aws-sdk-go/service/elasticache"
+	"github.com/aws/aws-sdk-go/service/iam"
 	"github.com/aws/aws-sdk-go/service/rds"
+	"github.com/aws/aws-sdk-go/service/servicediscovery"
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -54,6 +56,11 @@ type serviceDeployRequest struct {
 	EcsExecutionRoleName   string         `validate:"required"`
 	EcsTaskRoleName        string         `validate:"required"`
 	EcsTaskPolicyName      string         `validate:"required"`
+
+	EcsTaskPolicy *iam.CreatePolicyInput
+	EcsTaskPolicyDocument IamPolicyDocument
+
+
 	EcsServiceDesiredCount int64          `validate:"required"`
 	Ec2SecurityGroupName   string         `validate:"required"`
 	CloudWatchLogGroupName string         `validate:"required"`
@@ -80,6 +87,12 @@ type serviceDeployRequest struct {
 	NoPush                                  bool     `validate:"omitempty"`
 	RecreateService                         bool     `validate:"omitempty"`
 
+
+
+
+	SDNamepsace *servicediscovery.CreatePrivateDnsNamespaceInput
+	SDService *servicediscovery.CreateServiceInput
+
 	CacheCluster *elasticache.CreateCacheClusterInput
 	CacheClusterParameter []*elasticache.ParameterNameValue
 

example-project/tools/truss/cmd/devops/service_deploy.go

@@ -18,6 +18,8 @@ import (
 	"strings"
 	"time"
 
+	"geeks-accelerator/oss/saas-starter-kit/example-project/tools/truss/internal/retry"
+	"github.com/aws/aws-sdk-go/service/servicediscovery"
 	"github.com/aws/aws-sdk-go/service/rds"
 	"github.com/pborman/uuid"
 	"github.com/aws/aws-sdk-go/service/elasticache"
@@ -43,71 +45,6 @@ import (
 	"gopkg.in/go-playground/validator.v9"
 )
 
-// baseServicePolicyDocument defines the default permissions required to access AWS services for all deployed services.
-var baseServicePolicyDocument = IamPolicyDocument{
-	Version: "2012-10-17",
-	Statement: []IamStatementEntry{
-		IamStatementEntry{
-			Sid:    "DefaultServiceAccess",
-			Effect: "Allow",
-			Action: []string{
-				"s3:HeadBucket",
-				"ec2:DescribeNetworkInterfaces",
-				"ec2:DeleteNetworkInterface",
-				"ecs:ListTasks",
-				"ecs:DescribeTasks",
-				"ec2:DescribeNetworkInterfaces",
-				"route53:ListHostedZones",
-				"route53:ListResourceRecordSets",
-				"route53:ChangeResourceRecordSets",
-				"ecs:UpdateService",
-				"ses:SendEmail",
-			},
-			Resource: "*",
-		},
-		IamStatementEntry{
-			Sid:    "ServiceInvokeLambda",
-			Effect: "Allow",
-			Action: []string{
-				"iam:GetRole",
-				"lambda:InvokeFunction",
-				"lambda:ListVersionsByFunction",
-				"lambda:GetFunction",
-				"lambda:InvokeAsync",
-				"lambda:GetFunctionConfiguration",
-				"iam:PassRole",
-				"lambda:GetAlias",
-				"lambda:GetPolicy",
-			},
-			Resource: []string{
-				"arn:aws:iam:::role/*",
-				"arn:aws:lambda:::function:*",
-			},
-		},
-		IamStatementEntry{
-			Sid:    "datadoglambda",
-			Effect: "Allow",
-			Action: []string{
-				"cloudwatch:Get*",
-				"cloudwatch:List*",
-				"ec2:Describe*",
-				"support:*",
-				"tag:GetResources",
-				"tag:GetTagKeys",
-				"tag:GetTagValues",
-			},
-			Resource: "*",
-		},
-	},
-}
-
-/*
-// requiredCmdsBuild proves a list of required executables for completing build.
-var requiredCmdsDeploy = [][]string{
-	[]string{"docker", "version", "-f", "{{.Client.Version}}"},
-}
-*/
-
 // NewServiceDeployRequest generated a new request for executing deploy for a given set of flags.
 func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*serviceDeployRequest, error) {
 
@@ -254,8 +191,73 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
 
 			// Set default AWS ECS Task Policy Name.
 			req.EcsTaskPolicyName = fmt.Sprintf("%s%sServices", req.ProjectNameCamel(), strcase.ToCamel(req.Env))
+			req.EcsTaskPolicy = &iam.CreatePolicyInput{
+				PolicyName:     aws.String(req.EcsTaskPolicyName),
+				Description:    aws.String(fmt.Sprintf("Defines access for %s services. ", req.ProjectName)),
+			}
 			log.Printf("\t\t\tSet ECS Task Policy Name to '%s'.", req.EcsTaskPolicyName)
 
+			// EcsTaskPolicyDocument defines the default document policy used to create the AWS ECS Task Policy. If the
+			// policy already exists, the permissions will be used to add new required actions, but not for removal.
+			// The policy document grants the permissions required for deployed services to access AWS services.
+			req.EcsTaskPolicyDocument = IamPolicyDocument{
+				Version: "2012-10-17",
+				Statement: []IamStatementEntry{
+					IamStatementEntry{
+						Sid:    "DefaultServiceAccess",
+						Effect: "Allow",
+						Action: []string{
+							"s3:HeadBucket",
+							"ec2:DescribeNetworkInterfaces",
+							"ec2:DeleteNetworkInterface",
+							"ecs:ListTasks",
+							"ecs:DescribeTasks",
+							"ec2:DescribeNetworkInterfaces",
+							"route53:ListHostedZones",
+							"route53:ListResourceRecordSets",
+							"route53:ChangeResourceRecordSets",
+							"ecs:UpdateService",
+							"ses:SendEmail",
+						},
+						Resource: "*",
+					},
+					IamStatementEntry{
+						Sid:    "ServiceInvokeLambda",
+						Effect: "Allow",
+						Action: []string{
+							"iam:GetRole",
+							"lambda:InvokeFunction",
+							"lambda:ListVersionsByFunction",
+							"lambda:GetFunction",
+							"lambda:InvokeAsync",
+							"lambda:GetFunctionConfiguration",
+							"iam:PassRole",
+							"lambda:GetAlias",
+							"lambda:GetPolicy",
+						},
+						Resource: []string{
+							"arn:aws:iam:::role/*",
+							"arn:aws:lambda:::function:*",
+						},
+					},
+					IamStatementEntry{
+						Sid:    "datadoglambda",
+						Effect: "Allow",
+						Action: []string{
+							"cloudwatch:Get*",
+							"cloudwatch:List*",
+							"ec2:Describe*",
+							"support:*",
+							"tag:GetResources",
+							"tag:GetTagKeys",
+							"tag:GetTagValues",
+						},
+						Resource: "*",
+					},
+				},
+			}
+
+
 			// Set default Cloudwatch Log Group Name.
 			req.CloudWatchLogGroupName = fmt.Sprintf("logs/env_%s/aws/ecs/cluster_%s/service_%s", req.Env, req.EcsClusterName, req.ServiceName)
 			log.Printf("\t\t\tSet CloudWatch Log Group Name to '%s'.", req.CloudWatchLogGroupName)
@@ -264,6 +266,7 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
 			req.Ec2SecurityGroupName = req.EcsClusterName
 			log.Printf("\t\t\tSet ECS Security Group Name to '%s'.", req.Ec2SecurityGroupName)
 
+
 			// Set default ELB Load Balancer Name when ELB is enabled.
 			if req.EnableEcsElb {
 				if !strings.Contains(req.EcsClusterName, req.Env) && !strings.Contains(req.ServiceName, req.Env) {
@@ -295,6 +298,64 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
 			// S3 temp prefix, a life cycle policy will be applied to this.
 			req.S3BucketTempPrefix = "tmp/"
 
+			// Service Discovery Namespace settings.
+			req.SDNamepsace = &servicediscovery.CreatePrivateDnsNamespaceInput{
+				Name: aws.String(req.EcsClusterName),
+				Description: aws.String(fmt.Sprintf("Private DNS namespace used for services running on the ECS Cluster %s", req.EcsClusterName)),
+
+				// A unique string that identifies the request and that allows failed CreatePrivateDnsNamespace
+				// requests to be retried without the risk of executing the operation twice.
+				// CreatorRequestId can be any unique string, for example, a date/time stamp.
+				CreatorRequestId: aws.String("truss-deploy"),
+			}
+
+			// Service Discovery Service settings.
+			req.SDService = &servicediscovery.CreateServiceInput{
+				Name: aws.String(req.EcsServiceName),
+				Description: aws.String(fmt.Sprintf("Service %s running on the ECS Cluster %s", req.EcsServiceName, req.EcsClusterName)),
+
+				// A complex type that contains information about the Amazon Route 53 records
+				// that you want AWS Cloud Map to create when you register an instance.
+				DnsConfig: &servicediscovery.DnsConfig{
+					DnsRecords: []*servicediscovery.DnsRecord{
+						{
+							// The amount of time, in seconds, that you want DNS resolvers to cache the
+							// settings for this record.
+							TTL: aws.Int64(300),
+
+							// The type of the resource, which indicates the type of value that Route 53
+							// returns in response to DNS queries.
+							Type: aws.String("A"),
+						},
+					},
+				},
+
+				// A complex type that contains information about an optional custom health
+				// check.
+				//
+				// If you specify a health check configuration, you can specify either HealthCheckCustomConfig
+				// or HealthCheckConfig but not both.
+				HealthCheckCustomConfig: &servicediscovery.HealthCheckCustomConfig{
+					// The number of 30-second intervals that you want Cloud Map to wait after receiving
+					// an UpdateInstanceCustomHealthStatus request before it changes the health
+					// status of a service instance. For example, suppose you specify a value of
+					// 2 for FailureTheshold, and then your application sends an UpdateInstanceCustomHealthStatus
+					// request. Cloud Map waits for approximately 60 seconds (2 x 30) before changing
+					// the status of the service instance based on that request.
+					//
+					// Sending a second or subsequent UpdateInstanceCustomHealthStatus request with
+					// the same value before FailureThreshold x 30 seconds has passed doesn't accelerate
+					// the change. Cloud Map still waits FailureThreshold x 30 seconds after the
+					// first request to make the change.
+					FailureThreshold: aws.Int64(3),
+				},
+
+				// A unique string that identifies the request and that allows failed CreatePrivateDnsNamespace
+				// requests to be retried without the risk of executing the operation twice.
+				// CreatorRequestId can be any unique string, for example, a date/time stamp.
+				CreatorRequestId: aws.String("truss-deploy"),
+			}
+
 			// Elastic Cache settings for a Redis cache cluster. Could defined different settings by env.
 			req.CacheCluster = &elasticache.CreateCacheClusterInput{
 				AutoMinorVersionUpgrade:   aws.Bool(true),
@@ -342,6 +403,8 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
 				},
 			}
 
+
+
 			log.Printf("\t%s\tDefaults set.", tests.Success)
 		}
 
@@ -359,26 +422,7 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
 
 // Run is the main entrypoint for deploying a service for a given target env.
 func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
-
+	
-	/*
-		log.Println("Verify required commands are installed.")
-		for _, cmdVals := range requiredCmdsDeploy {
-			cmd := exec.Command(cmdVals[0], cmdVals[1:]...)
-			cmd.Env = os.Environ()
-
-			out, err := cmd.CombinedOutput()
-			if err != nil {
-				return errors.WithMessagef(err, "failed to execute %s - %s\n%s", strings.Join(cmdVals, " "), string(out))
-			}
-
-			log.Printf("\t%s\t%s - %s", tests.Success, cmdVals[0], string(out))
-		}
-
-		// Pull the current env variables to be passed in for command execution.
-		envVars := EnvVars(os.Environ())
-
-	*/
-
 	startTime := time.Now()
 
 	// Load the AWS ECR repository. Try to find by name else create new one.
@@ -526,10 +570,12 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 			return errors.Wrap(err, "failed to create docker build context")
 		}
 
-		_, err = docker.ImageBuild(context.Background(), buildCtx, buildOpts)
+		res, err := docker.ImageBuild(context.Background(), buildCtx, buildOpts)
 		if err != nil {
 			return errors.Wrap(err, "failed to build docker image")
 		}
+		io.Copy(os.Stdout, res.Body)
+		res.Body.Close()
 
 		// Push the newly built docker container to the registry.
 		if req.NoPush == false {
@@ -855,39 +901,22 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 
 		svc := ec2.New(req.awsSession())
 
+		log.Println("\t\tFind all subnets are that default for each available AZ.")
+
 		var subnets []*ec2.Subnet
-		if true { // len(req.ec2SubnetIds) == 0 {
+		err := svc.DescribeSubnetsPages(&ec2.DescribeSubnetsInput{}, func(res *ec2.DescribeSubnetsOutput, lastPage bool) bool {
-			log.Println("\t\tFind all subnets are that default for each available AZ.")
+			for _, s := range res.Subnets {
-
+				if *s.DefaultForAz {
-			err := svc.DescribeSubnetsPages(&ec2.DescribeSubnetsInput{}, func(res *ec2.DescribeSubnetsOutput, lastPage bool) bool {
-				for _, s := range res.Subnets {
-					if *s.DefaultForAz {
-						subnets = append(subnets, s)
-					}
-				}
-				return !lastPage
-			})
-			if err != nil {
-				return errors.Wrap(err, "failed to find default subnets")
-			}
-			/*} else {
-			log.Println("\t\tFind all subnets for the IDs provided.")
-
-			err := svc.DescribeSubnetsPages(&ec2.DescribeSubnetsInput{
-				SubnetIds: aws.StringSlice(flags.Ec2SubnetIds),
-			}, func(res *ec2.DescribeSubnetsOutput, lastPage bool) bool {
-				for _, s := range res.Subnets {
 					subnets = append(subnets, s)
 				}
-				return !lastPage
+			}
-			})
+			return !lastPage
-			if err != nil {
+		})
-				return errors.Wrapf(err, "failed to find subnets: %s", strings.Join(flags.Ec2SubnetIds, ", "))
+		if err != nil {
-			} else if len(flags.Ec2SubnetIds) != len(subnets)  {
+			return errors.Wrap(err, "failed to find default subnets")
-				return errors.Errorf("failed to find all subnets, expected %d, got %d", len(flags.Ec2SubnetIds) != len(subnets))
-			}*/
 		}
 
+
 		if len(subnets) == 0 {
 			return errors.New("failed to find any subnets, expected at least 1")
 		}
@@ -1330,6 +1359,310 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 		log.Printf("\t%s\tUsing Cache Cluster '%s'.\n", tests.Success, *cacheCluster.CacheClusterId)
 	}
 
+	// Route 53 zone lookup when hostname is set. Supports both top level domains or sub domains.
+	var zoneArecNames = map[string][]string{}
+	if req.ServiceDomainName != "" {
+		log.Println("Route 53 - Get or create hosted zones.")
+
+		svc := route53.New(req.awsSession())
+
+		log.Println("\tList all hosted zones.")
+		var zones []*route53.HostedZone
+		err := svc.ListHostedZonesPages(&route53.ListHostedZonesInput{},
+			func(res *route53.ListHostedZonesOutput, lastPage bool) bool {
+				for _, z := range res.HostedZones {
+					zones = append(zones, z)
+				}
+				return !lastPage
+			})
+		if err != nil {
+			return errors.Wrap(err, "failed list route 53 hosted zones")
+		}
+
+		// Generate a slice with the primary domain name and include all the alternative domain names.
+		lookupDomains := []string{req.ServiceDomainName}
+		for _, dn := range req.ServiceDomainNameAliases {
+			lookupDomains = append(lookupDomains, dn)
+		}
+
+		// Loop through all the defined domain names and find the associated zone even when they are a sub domain.
+		for _, dn := range lookupDomains {
+			log.Printf("\t\tFind zone for domain '%s'", dn)
+
+			// Get the top level domain from url.
+			zoneName := domainutil.Domain(dn)
+			var subdomain string
+			if zoneName == "" {
+				// Handle domain names that have weird TDL: ie .tech
+				zoneName = dn
+				log.Printf("\t\t\tNon-standard Level Domain: '%s'", zoneName)
+			} else {
+				log.Printf("\t\t\tTop Level Domain: '%s'", zoneName)
+
+				// Check if url has subdomain.
+				if domainutil.HasSubdomain(dn) {
+					subdomain = domainutil.Subdomain(dn)
+					log.Printf("\t\t\tsubdomain: '%s'", subdomain)
+				}
+			}
+
+			// Start at the top level domain and try to find a hosted zone. Search until a match is found or there are
+			// no more domain levels to search for.
+			var zoneId string
+			for {
+				log.Printf("\t\t\tChecking zone '%s' for associated hosted zone.", zoneName)
+
+				// Loop over each one of hosted zones and try to find match.
+				for _, z := range zones {
+					zn := strings.TrimRight(*z.Name, ".")
+
+					log.Printf("\t\t\t\tChecking if '%s' matches '%s'", zn, zoneName)
+					if zn == zoneName {
+						zoneId = *z.Id
+						break
+					}
+				}
+
+				if zoneId != "" || zoneName == dn {
+					// Found a matching zone or have search all possibilities!
+					break
+				}
+
+				// If we have not found a hosted zone, append the next level from the domain to the zone.
+				pts := strings.Split(subdomain, ".")
+				subs := []string{}
+				for idx, sn := range pts {
+					if idx == len(pts)-1 {
+						zoneName = sn + "." + zoneName
+					} else {
+						subs = append(subs, sn)
+					}
+				}
+				subdomain = strings.Join(subs, ".")
+			}
+
+			var aName string
+			if zoneId == "" {
+
+				// Get the top level domain from url again.
+				zoneName := domainutil.Domain(dn)
+				if zoneName == "" {
+					// Handle domain names that have weird TDL: ie .tech
+					zoneName = dn
+				}
+
+				log.Printf("\t\t\tNo hosted zone found for '%s', create '%s'.", dn, zoneName)
+				createRes, err := svc.CreateHostedZone(&route53.CreateHostedZoneInput{
+					Name: aws.String(zoneName),
+					HostedZoneConfig: &route53.HostedZoneConfig{
+						Comment: aws.String(fmt.Sprintf("Public hosted zone created by saas-starter-kit.")),
+					},
+
+					// A unique string that identifies the request and that allows failed CreateHostedZone
+					// requests to be retried without the risk of executing the operation twice.
+					// You must use a unique CallerReference string every time you submit a CreateHostedZone
+					// request. CallerReference can be any unique string, for example, a date/time
+					// stamp.
+					//
+					// CallerReference is a required field
+					CallerReference: aws.String("truss-deploy"),
+				})
+				if err != nil {
+					return errors.Wrapf(err, "failed to create route 53 hosted zone '%s' for domain '%s'", zoneName, dn)
+				}
+				zoneId = *createRes.HostedZone.Id
+
+				log.Printf("\t\t\tCreated hosted zone '%s'", zoneId)
+
+				// The fully qualified A record name.
+				aName = dn
+			} else {
+				log.Printf("\t\t\tFound hosted zone '%s'", zoneId)
+
+				// The fully qualified A record name.
+				if subdomain != "" {
+					aName = subdomain + "." + zoneName
+				} else {
+					aName = zoneName
+				}
+			}
+
+			// Add the A record to be maintained for the zone.
+			if _, ok := zoneArecNames[zoneId]; !ok {
+				zoneArecNames[zoneId] = []string{}
+			}
+			zoneArecNames[zoneId] = append(zoneArecNames[zoneId], aName)
+
+			log.Printf("\t%s\tZone '%s' found with A record name '%s'.\n", tests.Success, zoneId, aName)
+		}
+	}
+
+	// Setup service discovery.
+	var sdService *servicediscovery.Service
+	{
+		log.Println("SD - Get or Create Namespace")
+
+		svc := servicediscovery.New(req.awsSession())
+
+		log.Println("\t\tList all the private namespaces and try to find an existing entry.")
+
+		listNamespaces := func() (*servicediscovery.NamespaceSummary, error) {
+			var found *servicediscovery.NamespaceSummary
+			err := svc.ListNamespacesPages(&servicediscovery.ListNamespacesInput{
+				Filters: []*servicediscovery.NamespaceFilter{
+					&servicediscovery.NamespaceFilter{
+						Name:      aws.String("TYPE"),
+						Condition: aws.String("EQ"),
+						Values:    aws.StringSlice([]string{"DNS_PRIVATE"}),
+					},
+				},
+			}, func(res *servicediscovery.ListNamespacesOutput, lastPage bool) bool {
+				for _, n := range res.Namespaces {
+					if *n.Name == *req.SDNamepsace.Name {
+						found = n
+						return false
+					}
+				}
+				return !lastPage
+			})
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to list namespaces")
+			}
+
+			return found, nil
+		}
+
+		sdNamespace, err := listNamespaces()
+		if err != nil {
+			return err
+		}
+
+		if sdNamespace == nil {
+			// Link the namespace to the VPC.
+			req.SDNamepsace.Vpc = aws.String(vpcId)
+
+			log.Println("\t\tCreate private namespace.")
+
+			// If no namespace was found, create one.
+			createRes, err := svc.CreatePrivateDnsNamespace(req.SDNamepsace)
+			if err != nil {
+				return errors.Wrapf(err, "failed to create namespace '%s'", *req.SDNamepsace.Name)
+			}
+			operationId := createRes.OperationId
+
+			log.Println("\t\tWait for create operation to finish.")
+			retryFunc := func() (bool, error) {
+				opRes, err := svc.GetOperation(&servicediscovery.GetOperationInput{
+					OperationId: operationId,
+				})
+				if err != nil {
+					return true, err
+				}
+
+				log.Printf("\t\t\tStatus: %s.", *opRes.Operation.Status)
+
+				// The status of the operation. Values include the following:
+				//    * SUBMITTED: This is the initial state immediately after you submit a
+				//    request.
+				//    * PENDING: AWS Cloud Map is performing the operation.
+				//    * SUCCESS: The operation succeeded.
+				//    * FAIL: The operation failed. For the failure reason, see ErrorMessage.
+				if *opRes.Operation.Status == "SUCCESS" {
+					return true, nil
+				} else if *opRes.Operation.Status == "FAIL" {
+					err = errors.Errorf("operation failed")
+					err = awserr.New(*opRes.Operation.ErrorCode, *opRes.Operation.ErrorMessage, err)
+					return true, err
+				}
+
+				return false, nil
+			}
+			err = retry.Retry(context.Background(), nil, retryFunc)
+			if err != nil {
+				return errors.Wrapf(err, "failed to get operation for namespace '%s'", *req.SDNamepsace.Name)
+			}
+
+			// Now that the create operation is complete, try to find the namespace again.
+			sdNamespace, err = listNamespaces()
+			if err != nil {
+				return err
+			}
+
+			log.Printf("\t\tCreated: %s.", *sdNamespace.Arn)
+		} else {
+			log.Printf("\t\tFound: %s.", *sdNamespace.Arn)
+
+			// The number of services that are associated with the namespace.
+			if sdNamespace.ServiceCount != nil {
+				log.Printf("\t\t\tServiceCount: %d.", *sdNamespace.ServiceCount)
+			}
+		}
+
+		log.Printf("\t%s\tUsing Service Discovery Namespace '%s'.\n", tests.Success, *sdNamespace.Id)
+
+
+		// Try to find an existing entry for the current service.
+		var existingService *servicediscovery.ServiceSummary
+		err = svc.ListServicesPages(&servicediscovery.ListServicesInput{
+			Filters: []*servicediscovery.ServiceFilter{
+				&servicediscovery.ServiceFilter{
+					Name:      aws.String("NAMESPACE_ID"),
+					Condition: aws.String("EQ"),
+					Values:    aws.StringSlice([]string{*sdNamespace.Id}),
+				},
+			},
+		}, func(res *servicediscovery.ListServicesOutput, lastPage bool) bool {
+			for _, n := range res.Services {
+				if *n.Name == req.EcsServiceName {
+					existingService = n
+					return false
+				}
+			}
+			return !lastPage
+		})
+		if err != nil {
+			return errors.Wrapf(err, "failed to list services for namespace '%s'", *sdNamespace.Id)
+		}
+
+		if existingService == nil {
+			// Link the service to the namespace.
+			req.SDService.NamespaceId = sdNamespace.Id
+
+			// If no namespace was found, create one.
+			createRes, err := svc.CreateService(req.SDService)
+			if err != nil {
+				return errors.Wrapf(err, "failed to create service '%s'", *req.SDService.Name)
+			}
+			sdService = createRes.Service
+
+
+			log.Printf("\t\tCreated: %s.", *sdService.Arn)
+		} else {
+
+			// If no namespace was found, create one.
+			getRes, err := svc.GetService(&servicediscovery.GetServiceInput{
+				Id: existingService.Id,
+			})
+			if err != nil {
+				return errors.Wrapf(err, "failed to get service '%s'", *req.SDService.Name)
+			}
+			sdService = getRes.Service
+
+
+			log.Printf("\t\tFound: %s.", *sdService.Arn)
+
+			// The number of instances that are currently associated with the service. Instances
+			// that were previously associated with the service but that have been deleted
+			// are not included in the count.
+			if sdService.InstanceCount != nil {
+				log.Printf("\t\t\tInstanceCount: %d.", *sdService.InstanceCount)
+			}
+
+		}
+
+		log.Printf("\t%s\tUsing Service Discovery Service '%s'.\n", tests.Success, *sdService.Id)
+	}
+
 	// Try to find AWS ECS Cluster by name or create new one.
 	var ecsCluster *ecs.Cluster
 	{
@@ -1805,7 +2138,7 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 					}
 
 					var updateDoc bool
-					for _, baseStmt := range baseServicePolicyDocument.Statement {
+					for _, baseStmt := range req.EcsTaskPolicyDocument.Statement {
 						var found bool
 						for curIdx, curStmt := range curDoc.Statement {
 							if baseStmt.Sid != curStmt.Sid {
@@ -1857,17 +2190,14 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 						}
 					}
 				} else {
-					dat, err := json.Marshal(baseServicePolicyDocument)
+					dat, err := json.Marshal(req.EcsTaskPolicyDocument)
 					if err != nil {
 						return errors.Wrap(err, "failed to json encode policy document")
 					}
+					req.EcsTaskPolicy.PolicyDocument = aws.String(string(dat))
 
 					// If no repository was found, create one.
-					res, err := svc.CreatePolicy(&iam.CreatePolicyInput{
+					res, err := svc.CreatePolicy(req.EcsTaskPolicy)
-						PolicyName:     aws.String(req.EcsTaskPolicyName),
-						Description:    aws.String(fmt.Sprintf("Defines access for %s services. ", req.ProjectName)),
-						PolicyDocument: aws.String(string(dat)),
-					})
 					if err != nil {
 						return errors.Wrapf(err, "failed to create task policy '%s'", req.EcsTaskPolicyName)
 					}
@@ -2004,11 +2334,17 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 			recreateService = true
 			forceDelete = true
 		} else if req.EnableEcsElb && (ecsService.LoadBalancers == nil || len(ecsService.LoadBalancers) == 0) {
-			// Service was created with no ELB and now ELB is enabled.
+			// Service was created without ELB and now ELB is enabled.
 			recreateService = true
 		} else if !req.EnableEcsElb && (ecsService.LoadBalancers != nil && len(ecsService.LoadBalancers) > 0) {
 			// Service was created with ELB and now ELB is disabled.
 			recreateService = true
+		} else if sdService != nil && sdService.Arn != nil && (ecsService.ServiceRegistries == nil || len(ecsService.ServiceRegistries) == 0) {
+			// Service was created without Service Discovery and now Service Discovery is enabled.
+			recreateService = true
+		} else if (sdService == nil || sdService.Arn == nil) && (ecsService.ServiceRegistries != nil && len(ecsService.ServiceRegistries) > 0) {
+			// Service was created with Service Discovery and now Service Discovery is disabled.
+			recreateService = true
 		}
 
 		if recreateService {
@@ -2016,7 +2352,30 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 
 			svc := ecs.New(req.awsSession())
 
-			_, err := svc.DeleteService(&ecs.DeleteServiceInput{
+			// The service cannot be stopped while it is scaled above 0
+			if ecsService.DesiredCount  != nil &&  *ecsService.DesiredCount > 0 {
+				log.Println("\t\tScaling service down to zero.")
+				_, err := svc.UpdateService(&ecs.UpdateServiceInput{
+					Cluster: ecsService.ClusterArn,
+					Service: ecsService.ServiceArn,
+					DesiredCount: aws.Int64(int64(0)),
+				})
+				if err != nil {
+					return errors.Wrapf(err, "failed to update service '%s'", ecsService.ServiceName)
+				}
+
+				log.Println("\t\tWait for the service to scale down.")
+				err = svc.WaitUntilServicesStable(&ecs.DescribeServicesInput{
+					Cluster: ecsCluster.ClusterArn,
+					Services: aws.StringSlice([]string{*ecsService.ServiceArn}),
+				})
+				if err != nil {
+					return errors.Wrapf(err, "failed to wait for service '%s' to enter stable state", *ecsService.ServiceName)
+				}
+			}
+
+			log.Println("\t\tDelete Service.")
+			res, err := svc.DeleteService(&ecs.DeleteServiceInput{
 				Cluster: ecsService.ClusterArn,
 				Service: ecsService.ServiceArn,
 
@@ -2026,8 +2385,9 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 				Force: aws.Bool(forceDelete),
 			})
 			if err != nil {
-				return errors.Wrapf(err, "failed to create security group '%s'", req.Ec2SecurityGroupName)
+				return errors.Wrapf(err, "failed to delete service '%s'", ecsService.ServiceName)
 			}
+			ecsService = res.Service
 
 			log.Printf("\t%s\tDelete Service.\n", tests.Success)
 		}
@@ -2611,6 +2971,18 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 				},
 			}
 
+
+			// Add the Service Discovery registry to the ECS service.
+			if sdService != nil {
+				if serviceInput.ServiceRegistries == nil {
+					serviceInput.ServiceRegistries = []*ecs.ServiceRegistry{}
+				}
+				serviceInput.ServiceRegistries = append(serviceInput.ServiceRegistries, &ecs.ServiceRegistry{
+					RegistryArn: sdService.Arn,
+				})
+			}
+
+
 			createRes, err := svc.CreateService(serviceInput)
 
 			// If tags aren't enabled for the account, try the request again without them.
@@ -2867,133 +3239,6 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 		log.Printf("\t%s\tService running.\n", tests.Success)
 	}
 
-	// Route 53 zone lookup when hostname is set. Supports both top level domains or sub domains.
-	var zoneArecNames = map[string][]string{}
-	if req.ServiceDomainName != "" {
-		log.Println("Route 53 - Get or create hosted zones.")
-
-		svc := route53.New(req.awsSession())
-
-		log.Println("\tList all hosted zones.")
-		var zones []*route53.HostedZone
-		err := svc.ListHostedZonesPages(&route53.ListHostedZonesInput{},
-			func(res *route53.ListHostedZonesOutput, lastPage bool) bool {
-			for _, z := range res.HostedZones {
-				zones = append(zones, z)
-			}
-			return !lastPage
-		})
-		if err != nil {
-			return errors.Wrap(err, "failed list route 53 hosted zones")
-		}
-
-		// Generate a slice with the primary domain name and include all the alternative domain names.
-		lookupDomains := []string{req.ServiceDomainName}
-		for _, dn := range req.ServiceDomainNameAliases {
-			lookupDomains = append(lookupDomains, dn)
-		}
-
-		// Loop through all the defined domain names and find the associated zone even when they are a sub domain.
-		for _, dn := range lookupDomains {
-			log.Printf("\t\tFind zone for domain '%s'", dn)
-
-			// Get the top level domain from url.
-			zoneName := domainutil.Domain(dn)
-			log.Printf("\t\t\tTop Level Domain: '%s'", zoneName)
-
-			// Check if url has subdomain.
-			var subdomain string
-			if domainutil.HasSubdomain(dn) {
-				subdomain = domainutil.Subdomain(dn)
-				log.Printf("\t\t\tsubdomain: '%s'", subdomain)
-			}
-
-			// Start at the top level domain and try to find a hosted zone. Search until a match is found or there are
-			// no more domain levels to search for.
-			var zoneId string
-			for {
-				log.Printf("\t\t\tChecking zone '%s' for associated hosted zone.", zoneName)
-
-				// Loop over each one of hosted zones and try to find match.
-				for _, z := range zones {
-					log.Printf("\t\t\t\tChecking if %s matches %s", *z.Name, zoneName)
-
-					if strings.TrimRight(*z.Name, ".") == zoneName {
-						zoneId = *z.Id
-						break
-					}
-				}
-
-				if zoneId != "" || zoneName == dn {
-					// Found a matching zone or have search all possibilities!
-					break
-				}
-
-				// If we have not found a hosted zone, append the next level from the domain to the zone.
-				pts := strings.Split(subdomain, ".")
-				subs := []string{}
-				for idx, sn := range pts {
-					if idx == len(pts)-1 {
-						zoneName = sn + "." + zoneName
-					} else {
-						subs = append(subs, sn)
-					}
-				}
-				subdomain = strings.Join(subs, ".")
-			}
-
-			var aName string
-			if zoneId == "" {
-
-				// Get the top level domain from url again.
-				zoneName := domainutil.Domain(dn)
-
-				log.Printf("\t\t\tNo hosted zone found for '%s', create '%s'.", dn, zoneName)
-				createRes, err := svc.CreateHostedZone(&route53.CreateHostedZoneInput{
-					Name: aws.String(zoneName),
-					HostedZoneConfig: &route53.HostedZoneConfig{
-						Comment: aws.String(fmt.Sprintf("Public hosted zone created by saas-starter-kit.")),
-					},
-
-					// A unique string that identifies the request and that allows failed CreateHostedZone
-					// requests to be retried without the risk of executing the operation twice.
-					// You must use a unique CallerReference string every time you submit a CreateHostedZone
-					// request. CallerReference can be any unique string, for example, a date/time
-					// stamp.
-					//
-					// CallerReference is a required field
-					CallerReference: aws.String("truss-deploy"),
-				})
-				if err != nil {
-					return errors.Wrapf(err, "failed to create route 53 hosted zone '%s' for domain '%s'", zoneName, dn)
-				}
-				zoneId = *createRes.HostedZone.Id
-
-				log.Printf("\t\t\tCreated hosted zone '%s'", zoneId)
-
-				// The fully qualified A record name.
-				aName = dn
-			} else {
-				log.Printf("\t\t\tFound hosted zone '%s'", zoneId)
-
-				// The fully qualified A record name.
-				if subdomain != "" {
-					aName = subdomain + "." + zoneName
-				} else {
-					aName = zoneName
-				}
-			}
-
-			// Add the A record to be maintained for the zone.
-			if _, ok := zoneArecNames[zoneId]; !ok {
-				zoneArecNames[zoneId] = []string{}
-			}
-			zoneArecNames[zoneId] = append(zoneArecNames[zoneId], aName)
-
-			log.Printf("\t%s\tZone '%s' found with A record name '%s'.\n", tests.Success, zoneId, aName)
-		}
-	}
-
 	if req.EnableEcsElb {
 		// TODO: Need to connect ELB to route53
 	} else {
@@ -3006,6 +3251,7 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 			servceTaskRes, err := svc.ListTasks(&ecs.ListTasksInput{
 				Cluster:     aws.String(req.EcsClusterName),
 				ServiceName: aws.String(req.EcsServiceName),
+				DesiredStatus: aws.String("RUNNING"),
 			})
 			if err != nil {
 				return errors.Wrapf(err, "failed to list tasks for cluster '%s' service '%s'", req.EcsClusterName, req.EcsServiceName)
@@ -3020,43 +3266,18 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 				return errors.Wrapf(err, "failed to describe %d tasks for cluster '%s'", len(servceTaskRes.TaskArns), req.EcsClusterName)
 			}
 
-			var failures []*ecs.Failure
 			var taskArns []string
 			for _, t := range taskRes.Tasks {
-				if t.TaskDefinitionArn != taskDef.TaskDefinitionArn {
+				if *t.TaskDefinitionArn != *taskDef.TaskDefinitionArn {
 					continue
 				}
 
+				log.Printf("\t\t\t%s: %s\n", *t.TaskArn, *t.LastStatus)
+
 				taskArns = append(taskArns, *t.TaskArn)
 
-				// Limit failures to only the current task definition.
-				for _, f := range taskRes.Failures {
-					if *f.Arn == *t.TaskArn {
-						failures = append(failures, f)
-					}
-				}
-			}
-
-			if len(failures) > 0 {
-				for _, t := range failures {
-					log.Printf("\t%s\tTask %s failed with %s.\n", tests.Failed, *t.Arn, *t.Reason)
-				}
-			} else {
-				log.Printf("\t%s\tTasks founds.\n", tests.Success)
-			}
-
-			log.Println("\t\tWaiting for tasks to enter running state.")
-			{
-				var err error
-				err = svc.WaitUntilTasksRunning(&ecs.DescribeTasksInput{
-					Cluster: ecsCluster.ClusterArn,
-					Tasks:   aws.StringSlice(taskArns),
-				})
-				if err != nil {
-					return errors.Wrapf(err, "failed to wait for tasks to enter running state for cluster '%s'", req.EcsClusterName)
-				}
-				log.Printf("\t%s\tTasks running.\n", tests.Success)
 			}
+			log.Printf("\t%s\tTasks founds.\n", tests.Success)
 
 			log.Println("\t\tDescribe tasks for running tasks.")
 			{
@@ -3073,6 +3294,8 @@ func ServiceDeploy(log *log.Logger, req *serviceDeployRequest) error {
 						continue
 					}
 
+					// t.Containers[0].NetworkInterfaces[0].Ipv6Address
+
 					for _, a := range t.Attachments {
 						if a.Details == nil {
 							continue

example-project/tools/truss/internal/retry/retry.go

@@ -0,0 +1,86 @@
+// Package retry contains a simple retry mechanism defined by a slice of delay
+// times. There are no maximum retries accounted for here. If retries should be
+// limited, use a Timeout context to keep from retrying forever. This should
+// probably be made into something more robust.
+package retry
+
+import (
+"context"
+"time"
+)
+
+// queryPollIntervals is a slice of the delays before re-checking the status on
+// an executing query, backing off from a short delay at first. This sequence
+// has been selected with Athena queries in mind, which may operate very
+// quickly for things like schema manipulation, or which may run for an
+// extended period of time, when running an actual data analysis query.
+// Long-running queries will exhaust their rapid retries quickly, and fall back
+// to checking every few seconds or longer.
+var DefaultPollIntervals = []time.Duration{
+	time.Millisecond,
+	2 * time.Millisecond,
+	2 * time.Millisecond,
+	5 * time.Millisecond,
+	10 * time.Millisecond,
+	20 * time.Millisecond,
+	50 * time.Millisecond,
+	50 * time.Millisecond,
+	100 * time.Millisecond,
+	100 * time.Millisecond,
+	200 * time.Millisecond,
+	500 * time.Millisecond,
+	time.Second,
+	2 * time.Second,
+	5 * time.Second,
+	10 * time.Second,
+	20 * time.Second,
+	30 * time.Second,
+	time.Minute,
+}
+
+
+// delayer keeps track of the current delay between retries.
+type delayer struct {
+	Delays       []time.Duration
+	currentIndex int
+}
+
+// Delay returns the current delay duration, and advances the index to the next
+// delay defined. If the index has reached the end of the delay slice, then it
+// will continue to return the maximum delay defined.
+func (d *delayer) Delay() time.Duration {
+	t := d.Delays[d.currentIndex]
+	if d.currentIndex < len(d.Delays)-1 {
+		d.currentIndex++
+	}
+	return t
+}
+
+// Retry uses a slice of time.Duration interval delays to retry a function
+// until it either errors or indicates that it is ready to proceed. If f
+// returns true, or an error, the retry loop is broken. Pass a closure as f if
+// you need to record a value from the operation that you are performing inside
+// f.
+func Retry(ctx context.Context, retryIntervals []time.Duration, f func() (bool, error)) (err error) {
+	if retryIntervals == nil || len(retryIntervals) == 0 {
+		retryIntervals = DefaultPollIntervals
+	}
+
+	d := delayer{Delays: retryIntervals}
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+			ok, err := f()
+			if err != nil {
+				return err
+			}
+			if ok {
+				return nil
+			}
+			time.Sleep(d.Delay())
+		}
+	}
+	return err
+}

example-project/tools/truss/internal/retry/retry_test.go

@@ -0,0 +1,86 @@
+package retry
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+)
+
+var errExpectedFailure = errors.New("expected failure for test purposes")
+
+func TestDelayer(t *testing.T) {
+	delays := []time.Duration{
+		time.Millisecond,
+		2 * time.Millisecond,
+		4 * time.Millisecond,
+		10 * time.Millisecond,
+	}
+	tt := []struct {
+		desc       string
+		numRetries int
+		expDelay   time.Duration
+	}{
+		{"first try", 0, time.Millisecond},
+		{"second try", 1, 2 * time.Millisecond},
+		{"len(delays) try", len(delays) - 1, delays[len(delays)-1]},
+		{"len(delays) + 1 try", len(delays), delays[len(delays)-1]},
+		{"len(delays) * 2 try", len(delays) * 2, delays[len(delays)-1]},
+	}
+
+	for _, tc := range tt {
+		t.Run(tc.desc, func(t *testing.T) {
+			var (
+				d     = delayer{Delays: delays}
+				delay time.Duration
+			)
+			for i := tc.numRetries + 1; i > 0; i-- {
+				delay = d.Delay()
+			}
+			if delay != tc.expDelay {
+				t.Fatalf(
+					"expected delay of %s after %d retries, but got %s",
+					tc.expDelay, tc.numRetries, delay)
+			}
+		})
+
+	}
+}
+
+func TestRetry(t *testing.T) {
+	delays := []time.Duration{
+		time.Millisecond,
+		2 * time.Millisecond,
+		3 * time.Millisecond,
+	}
+	tt := []struct {
+		desc    string
+		tries   int
+		success bool
+		err     error
+	}{
+		{"first try", 1, true, nil},
+		{"second try error", 2, false, errExpectedFailure},
+		{"third try success", 3, true, nil},
+	}
+	for _, tc := range tt {
+		t.Run(tc.desc, func(t *testing.T) {
+			tries := 0
+			retryFunc := func() (bool, error) {
+				tries++
+				if tries == tc.tries {
+					return tc.success, tc.err
+				}
+				t.Logf("try #%d unsuccessful: trying again up to %d times", tries, tc.tries)
+				return false, nil
+			}
+			err := Retry(context.Background(), delays, retryFunc)
+			if err != tc.err {
+				t.Fatalf("expected error %s, but got error %s", err, tc.err)
+			}
+			if tries != tc.tries {
+				t.Fatalf("expected %d tries, but tried %d times", tc.tries, tries)
+			}
+		})
+	}
+}