Rolling restart (#619)

* implement rolling restart functionality

bouncing each image individually can ensure that a group of docker
containers launched with docker-compose can stay 100% up during deploy.

* move rolling restart into a function

* honor params.Cleanup

Co-authored-by: Simon Aronsson <simme@arcticbit.se>

cmd/root.go

@@ -30,7 +30,8 @@ var (
 	notifier       *notifications.Notifier
 	timeout        time.Duration
 	lifecycleHooks bool
-	scope	   string
+	rollingRestart bool
+	scope	         string
 )
 
 var rootCmd = &cobra.Command{
@@ -91,6 +92,7 @@ func PreRun(cmd *cobra.Command, args []string) {
 
 	enableLabel, _ = f.GetBool("label-enable")
 	lifecycleHooks, _ = f.GetBool("enable-lifecycle-hooks")
+	rollingRestart, _ = f.GetBool("rolling-restart")
 	scope, _ = f.GetString("scope")
 
 	log.Debug(scope)
@@ -211,6 +213,7 @@ func runUpdatesWithNotifications(filter t.Filter) {
 		Timeout:        timeout,
 		MonitorOnly:    monitorOnly,
 		LifecycleHooks: lifecycleHooks,
+		RollingRestart: rollingRestart,
 	}
 	err := actions.Update(client, updateParams)
 	if err != nil {

docs/arguments.md

@@ -247,7 +247,18 @@ can be defined, but not both. An example: `--schedule "0 0 4 * * *"`
 Environment Variable: WATCHTOWER_SCHEDULE
                 Type: String
              Default: -
-``` 
+```
+
+## Rolling restart
+Restart one image at time instead of stopping and starting all at once.  Useful in conjunction with lifecycle hooks
+to implement zero-downtime deploy.
+
+```
+            Argument: --rolling-restart
+Environment Variable: WATCHTOWER_ROLLING_RESTART
+                Type: Boolean
+             Default: false
+```
 
 ## Wait until timeout
 Timeout before the container is forcefully stopped. When set, this option will change the default (`10s`) wait time to the given value. An example: `--stop-timeout 30s` will set the timeout to 30 seconds.

internal/actions/update.go

@@ -52,15 +52,33 @@ func Update(client container.Client, params types.UpdateParams) error {
 		return nil
 	}
 
-	stopContainersInReversedOrder(containers, client, params)
-	restartContainersInSortedOrder(containers, client, params)
-
+	if params.RollingRestart {
+		performRollingRestart(containers, client, params)
+	} else {
+		stopContainersInReversedOrder(containers, client, params)
+		restartContainersInSortedOrder(containers, client, params)
+	}
 	if params.LifecycleHooks {
 		lifecycle.ExecutePostChecks(client, params)
 	}
 	return nil
 }
 
+func performRollingRestart(containers []container.Container, client container.Client, params types.UpdateParams) {
+	cleanupImageIDs := make(map[string]bool)
+
+	for i := len(containers) - 1; i >= 0; i-- {
+		if containers[i].Stale {
+			stopStaleContainer(containers[i], client, params)
+			restartStaleContainer(containers[i], client, params)
+		}
+	}
+
+	if params.Cleanup {
+		cleanupImages(client, cleanupImageIDs)
+	}
+}
+
 func stopContainersInReversedOrder(containers []container.Container, client container.Client, params types.UpdateParams) {
 	for i := len(containers) - 1; i >= 0; i-- {
 		stopStaleContainer(containers[i], client, params)
@@ -99,11 +117,16 @@ func restartContainersInSortedOrder(containers []container.Container, client con
 		restartStaleContainer(staleContainer, client, params)
 		imageIDs[staleContainer.ImageID()] = true
 	}
+
 	if params.Cleanup {
-		for imageID := range imageIDs {
-			if err := client.RemoveImageByID(imageID); err != nil {
-				log.Error(err)
-			}
+		cleanupImages(client, imageIDs)
+	}
+}
+
+func cleanupImages(client container.Client, imageIDs map[string]bool) {
+	for imageID := range imageIDs {
+		if err := client.RemoveImageByID(imageID); err != nil {
+			log.Error(err)
 		}
 	}
 }

internal/flags/flags.go

@@ -123,6 +123,12 @@ func RegisterSystemFlags(rootCmd *cobra.Command) {
 		viper.GetBool("WATCHTOWER_LIFECYCLE_HOOKS"),
 		"Enable the execution of commands triggered by pre- and post-update lifecycle hooks")
 
+	flags.BoolP(
+		"rolling-restart",
+		"",
+		viper.GetBool("WATCHTOWER_ROLLING_RESTART"),
+		"Restart containers one at a time")
+
 	flags.BoolP(
 		"http-api",
 		"",

pkg/types/update_params.go

@@ -12,4 +12,5 @@ type UpdateParams struct {
 	Timeout        time.Duration
 	MonitorOnly    bool
 	LifecycleHooks bool
+	RollingRestart bool
 }