Feat/lifecycle hooks (#351)

* feat(update): add lifecycle hooks to the update action

* fix(ci): add bash tests for lifecycle-hooks to the ci workflow

* fix(ci): move integration tests to an isolated step

* fix(ci): fix malformed all-contributors json

* fix(ci): disable automatic bash test until we figure out a reasonable way to run it in circleci

.all-contributorsrc

@@ -302,7 +302,7 @@
         "code",
         "doc"
       ]
-    }
+    },
     {
       "login": "zoispag",
       "name": "Zois Pagoulatos",

.circleci/config.yml

@@ -36,9 +36,18 @@ workflows:
               only: /.*/
             tags:
               only: /.*/
+    #  - integration_testing:
+    #      requires:
+    #        - checkout
+    #      filters:
+    #        branches:
+    #          only: /.*/
+    #        tags:
+    #          only: /.*/
       - build:
           requires:
             - testing
+    #        - integration_testing
             - linting
           filters:
             branches:
@@ -90,6 +99,14 @@ jobs:
       - run: go get -u github.com/haya14busa/goverage
       - run: goverage -v -coverprofile=coverage.out ./...
       - run: godacov -t $CODACY_TOKEN -r ./coverage.out -c $CIRCLE_SHA1
+  #integration_testing:
+  #  executor: go
+  #  steps:
+  #    - attach_workspace:
+  #        at: .
+  #    - run: go build .
+  #    - setup_remote_docker
+  #    - run: ./scripts/lifecycle-tests.sh
   build:
     executor: go
     steps:

cmd/root.go

@@ -23,14 +23,15 @@ import (
 const DockerAPIMinVersion string = "1.24"
 
 var (
-	client       container.Client
-	scheduleSpec string
-	cleanup      bool
-	noRestart    bool
-	monitorOnly  bool
-	enableLabel  bool
-	notifier     *notifications.Notifier
-	timeout      time.Duration
+	client         container.Client
+	scheduleSpec   string
+	cleanup        bool
+	noRestart      bool
+	monitorOnly    bool
+	enableLabel    bool
+	notifier       *notifications.Notifier
+	timeout        time.Duration
+	lifecycleHooks bool
 )
 
 var rootCmd = &cobra.Command{
@@ -84,7 +85,9 @@ func PreRun(cmd *cobra.Command, args []string) {
 	if timeout < 0 {
 		log.Fatal("Please specify a positive value for timeout value.")
 	}
+
 	enableLabel, _ = f.GetBool("label-enable")
+	lifecycleHooks, _ = f.GetBool("enable-lifecycle-hooks")
 
 	// configure environment vars for client
 	err := flags.EnvConfig(cmd, DockerAPIMinVersion)
@@ -95,6 +98,7 @@ func PreRun(cmd *cobra.Command, args []string) {
 	noPull, _ := f.GetBool("no-pull")
 	includeStopped, _ := f.GetBool("include-stopped")
 	removeVolumes, _ := f.GetBool("remove-volumes")
+
 	client = container.NewClient(
 		!noPull,
 		includeStopped,
@@ -171,11 +175,12 @@ func runUpgradesOnSchedule(filter t.Filter) error {
 func runUpdatesWithNotifications(filter t.Filter) {
 	notifier.StartNotification()
 	updateParams := actions.UpdateParams{
-		Filter:      filter,
-		Cleanup:     cleanup,
-		NoRestart:   noRestart,
-		Timeout:     timeout,
-		MonitorOnly: monitorOnly,
+		Filter:         filter,
+		Cleanup:        cleanup,
+		NoRestart:      noRestart,
+		Timeout:        timeout,
+		MonitorOnly:    monitorOnly,
+		LifecycleHooks: lifecycleHooks,
 	}
 	err := actions.Update(client, updateParams)
 	if err != nil {

docs/lifecycle-hooks.md

@@ -0,0 +1,45 @@
+
+## Executing commands before and after updating
+
+> **DO NOTE**: Both commands are shell commands executed with `sh`, and therefore require the 
+> container to provide the `sh` executable.
+
+It is possible to execute a *pre-update* command and a *post-update* command 
+**inside** every container updated by watchtower. The *pre-update* command is 
+executed before stopping the container, and the *post-update* command is 
+executed after restarting the container.
+
+This feature is disabled by default. To enable it, you need to set the option
+`--enable-lifecycle-hooks` on the command line, or set the environment variable
+`WATCHTOWER_LIFECYCLE_HOOKS` to true.
+
+ 
+
+### Specifying update commands
+
+The commands are specified using docker container labels, with 
+`com.centurylinklabs.watchtower.pre-update-command` for the *pre-update* 
+command and `com.centurylinklabs.watchtower.lifecycle.post-update` for the
+*post-update* command.
+
+These labels can be declared as instructions in a Dockerfile:
+
+```docker
+LABEL com.centurylinklabs.watchtower.lifecycle.pre-update="/dump-data.sh"
+LABEL com.centurylinklabs.watchtower.lifecycle.post-update="/restore-data.sh"
+```
+
+Or be specified as part of the `docker run` command line:
+
+```bash
+docker run -d \
+  --label=com.centurylinklabs.watchtower.lifecycle.pre-update="/dump-data.sh" \
+  --label=com.centurylinklabs.watchtower.lifecycle.post-update="/restore-data.sh" \
+  someimage
+```
+
+### Execution failure
+
+The failure of a command to execute, identified by an exit code different than 
+0, will not prevent watchtower from updating the container. Only an error
+log statement containing the exit code will be reported.

internal/actions/actions_suite_test.go

@@ -160,7 +160,7 @@ func (client mockClient) StopContainer(c container.Container, d time.Duration) e
 	}
 	return nil
 }
-func (client mockClient) StartContainer(c container.Container) error {
+func (client mockClient) StartContainer(c container.Container) (string, error) {
 	panic("Not implemented")
 }
 
@@ -173,6 +173,14 @@ func (client mockClient) RemoveImage(c container.Container) error {
 	return nil
 }
 
+func (client mockClient) GetContainer(containerID string) (container.Container, error) {
+	return container.Container{}, nil
+}
+
+func (client mockClient) ExecuteCommand(containerID string, command string) error {
+	return nil
+}
+
 func (client mockClient) IsContainerStale(c container.Container) (bool, error) {
 	panic("Not implemented")
 }

internal/actions/update.go

@@ -61,8 +61,9 @@ func stopStaleContainer(container container.Container, client container.Client,
 		return
 	}
 
-	err := client.StopContainer(container, params.Timeout)
-	if err != nil {
+	executePreUpdateCommand(client, container)
+
+	if err := client.StopContainer(container, params.Timeout); err != nil {
 		log.Error(err)
 	}
 }
@@ -89,8 +90,10 @@ func restartStaleContainer(container container.Container, client container.Clien
 	}
 
 	if !params.NoRestart {
-		if err := client.StartContainer(container); err != nil {
+		if newContainerID, err := client.StartContainer(container); err != nil {
 			log.Error(err)
+		} else if container.Stale && params.LifecycleHooks {
+			executePostUpdateCommand(client, newContainerID)
 		}
 	}
 
@@ -104,18 +107,49 @@ func restartStaleContainer(container container.Container, client container.Clien
 func checkDependencies(containers []container.Container) {
 
 	for i, parent := range containers {
-		if parent.Stale {
+		if parent.ToRestart() {
 			continue
 		}
 
 	LinkLoop:
 		for _, linkName := range parent.Links() {
 			for _, child := range containers {
-				if child.Name() == linkName && child.Stale {
-					containers[i].Stale = true
+				if child.Name() == linkName && child.ToRestart() {
+					containers[i].Linked = true
 					break LinkLoop
 				}
 			}
 		}
 	}
 }
+
+func executePreUpdateCommand(client container.Client, container container.Container) {
+
+	command := container.GetLifecyclePreUpdateCommand()
+	if len(command) == 0 {
+		log.Debug("No pre-update command supplied. Skipping")
+	}
+
+	log.Info("Executing pre-update command.")
+	if err := client.ExecuteCommand(container.ID(), command); err != nil {
+		log.Error(err)
+	}
+}
+
+func executePostUpdateCommand(client container.Client, newContainerID string) {
+	newContainer, err := client.GetContainer(newContainerID)
+	if err != nil {
+		log.Error(err)
+		return
+	}
+
+	command := newContainer.GetLifecyclePostUpdateCommand()
+	if len(command) == 0 {
+		log.Debug("No post-update command supplied. Skipping")
+	}
+
+	log.Info("Executing post-update command.")
+	if err := client.ExecuteCommand(newContainerID, command); err != nil {
+		log.Error(err)
+	}
+}

internal/actions/update_params.go

@@ -7,9 +7,10 @@ import (
 
 // UpdateParams contains all different options available to alter the behavior of the Update func
 type UpdateParams struct {
-	Filter      t.Filter
-	Cleanup     bool
-	NoRestart   bool
-	Timeout     time.Duration
-	MonitorOnly bool
+	Filter         t.Filter
+	Cleanup        bool
+	NoRestart      bool
+	Timeout        time.Duration
+	MonitorOnly    bool
+	LifecycleHooks bool
 }

internal/flags/flags.go

@@ -88,6 +88,12 @@ func RegisterSystemFlags(rootCmd *cobra.Command) {
 		"S",
 		viper.GetBool("WATCHTOWER_INCLUDE_STOPPED"),
 		"Will also include created and exited containers")
+
+	flags.BoolP(
+		"enable-lifecycle-hooks",
+		"",
+		viper.GetBool("WATCHTOWER_LIFECYCLE_HOOKS"),
+		"Enable the execution of commands triggered by pre- and post-update lifecycle hooks")
 }
 
 // RegisterNotificationFlags that are used by watchtower to send notifications

pkg/container/client.go

@@ -1,8 +1,10 @@
 package container
 
 import (
+	"bytes"
 	"fmt"
 	"io/ioutil"
+	"strings"
 	"time"
 
 	t "github.com/containrrr/watchtower/pkg/types"
@@ -15,18 +17,18 @@ import (
 	"golang.org/x/net/context"
 )
 
-const (
-	defaultStopSignal = "SIGTERM"
-)
+const defaultStopSignal = "SIGTERM"
 
 // A Client is the interface through which watchtower interacts with the
 // Docker API.
 type Client interface {
 	ListContainers(t.Filter) ([]Container, error)
+	GetContainer(containerID string) (Container, error)
 	StopContainer(Container, time.Duration) error
-	StartContainer(Container) error
+	StartContainer(Container) (string, error)
 	RenameContainer(Container, string) error
 	IsContainerStale(Container) (bool, error)
+	ExecuteCommand(containerID string, command string) error
 	RemoveImage(Container) error
 }
 
@@ -80,18 +82,12 @@ func (client dockerClient) ListContainers(fn t.Filter) ([]Container, error) {
 	}
 
 	for _, runningContainer := range containers {
-		containerInfo, err := client.api.ContainerInspect(bg, runningContainer.ID)
+
+		c, err := client.GetContainer(runningContainer.ID)
 		if err != nil {
 			return nil, err
 		}
 
-		imageInfo, _, err := client.api.ImageInspectWithRaw(bg, containerInfo.Image)
-		if err != nil {
-			return nil, err
-		}
-
-		c := Container{containerInfo: &containerInfo, imageInfo: &imageInfo}
-
 		if fn(c) {
 			cs = append(cs, c)
 		}
@@ -112,6 +108,23 @@ func (client dockerClient) createListFilter() filters.Args {
 	return filterArgs
 }
 
+func (client dockerClient) GetContainer(containerID string) (Container, error) {
+	bg := context.Background()
+
+	containerInfo, err := client.api.ContainerInspect(bg, containerID)
+	if err != nil {
+		return Container{}, err
+	}
+
+	imageInfo, _, err := client.api.ImageInspectWithRaw(bg, containerInfo.Image)
+	if err != nil {
+		return Container{}, err
+	}
+
+	container := Container{containerInfo: &containerInfo, imageInfo: &imageInfo}
+	return container, nil
+}
+
 func (client dockerClient) StopContainer(c Container, timeout time.Duration) error {
 	bg := context.Background()
 	signal := c.StopSignal()
@@ -147,7 +160,7 @@ func (client dockerClient) StopContainer(c Container, timeout time.Duration) err
 	return nil
 }
 
-func (client dockerClient) StartContainer(c Container) error {
+func (client dockerClient) StartContainer(c Container) (string, error) {
 	bg := context.Background()
 	config := c.runtimeConfig()
 	hostConfig := c.hostConfig()
@@ -167,40 +180,40 @@ func (client dockerClient) StartContainer(c Container) error {
 	name := c.Name()
 
 	log.Infof("Creating %s", name)
-	creation, err := client.api.ContainerCreate(bg, config, hostConfig, simpleNetworkConfig, name)
+	createdContainer, err := client.api.ContainerCreate(bg, config, hostConfig, simpleNetworkConfig, name)
 	if err != nil {
-		return err
+		return "", err
 	}
 
 	if !(hostConfig.NetworkMode.IsHost()) {
 
 		for k := range simpleNetworkConfig.EndpointsConfig {
-			err = client.api.NetworkDisconnect(bg, k, creation.ID, true)
+			err = client.api.NetworkDisconnect(bg, k, createdContainer.ID, true)
 			if err != nil {
-				return err
+				return "", err
 			}
 		}
 
 		for k, v := range networkConfig.EndpointsConfig {
-			err = client.api.NetworkConnect(bg, k, creation.ID, v)
+			err = client.api.NetworkConnect(bg, k, createdContainer.ID, v)
 			if err != nil {
-				return err
+				return "", err
 			}
 		}
 
 	}
 
-	return client.startContainerIfPreviouslyRunning(bg, c, creation)
+	if !c.IsRunning() {
+		return createdContainer.ID, nil
+	}
+
+	return createdContainer.ID, client.doStartContainer(bg, c, createdContainer)
 
 }
 
-func (client dockerClient) startContainerIfPreviouslyRunning(bg context.Context, c Container, creation container.ContainerCreateCreatedBody) error {
+func (client dockerClient) doStartContainer(bg context.Context, c Container, creation container.ContainerCreateCreatedBody) error {
 	name := c.Name()
 
-	if !c.IsRunning() {
-		return nil
-	}
-
 	log.Debugf("Starting container %s (%s)", name, creation.ID)
 	err := client.api.ContainerStart(bg, creation.ID, types.ContainerStartOptions{})
 	if err != nil {
@@ -271,6 +284,67 @@ func (client dockerClient) RemoveImage(c Container) error {
 	return err
 }
 
+func (client dockerClient) ExecuteCommand(containerID string, command string) error {
+	bg := context.Background()
+
+	// Create the exec
+	execConfig := types.ExecConfig{
+		Tty:    true,
+		Detach: false,
+		Cmd:    []string{"sh", "-c", command},
+	}
+
+	exec, err := client.api.ContainerExecCreate(bg, containerID, execConfig)
+	if err != nil {
+		return err
+	}
+
+	response, attachErr := client.api.ContainerExecAttach(bg, exec.ID, types.ExecStartCheck{
+		Tty:    true,
+		Detach: false,
+	})
+	if attachErr != nil {
+		log.Errorf("Failed to extract command exec logs: %v", attachErr)
+	}
+
+	// Run the exec
+	execStartCheck := types.ExecStartCheck{Detach: false, Tty: true}
+	err = client.api.ContainerExecStart(bg, exec.ID, execStartCheck)
+	if err != nil {
+		return err
+	}
+
+	var execOutput string
+	if attachErr == nil {
+		defer response.Close()
+		var writer bytes.Buffer
+		written, err := writer.ReadFrom(response.Reader)
+		if err != nil {
+			log.Error(err)
+		} else if written > 0 {
+			execOutput = strings.TrimSpace(writer.String())
+		}
+	}
+
+	// Inspect the exec to get the exit code and print a message if the
+	// exit code is not success.
+	execInspect, err := client.api.ContainerExecInspect(bg, exec.ID)
+	if err != nil {
+		return err
+	}
+
+	if execInspect.ExitCode > 0 {
+		log.Errorf("Command exited with code %v.", execInspect.ExitCode)
+		log.Error(execOutput)
+	} else {
+		if len(execOutput) > 0 {
+			log.Infof("Command output:\n%v", execOutput)
+		}
+	}
+
+	return nil
+}
+
 func (client dockerClient) waitForStopOrTimeout(c Container, waitTime time.Duration) error {
 	bg := context.Background()
 	timeout := time.After(waitTime)

pkg/container/container.go

@@ -10,13 +10,6 @@ import (
 	dockercontainer "github.com/docker/docker/api/types/container"
 )
 
-const (
-	watchtowerLabel = "com.centurylinklabs.watchtower"
-	signalLabel     = "com.centurylinklabs.watchtower.stop-signal"
-	enableLabel     = "com.centurylinklabs.watchtower.enable"
-	zodiacLabel     = "com.centurylinklabs.zodiac.original-image"
-)
-
 // NewContainer returns a new Container instance instantiated with the
 // specified ContainerInfo and ImageInfo structs.
 func NewContainer(containerInfo *types.ContainerJSON, imageInfo *types.ImageInspect) *Container {
@@ -28,7 +21,8 @@ func NewContainer(containerInfo *types.ContainerJSON, imageInfo *types.ImageInsp
 
 // Container represents a running Docker container.
 type Container struct {
-	Stale bool
+	Linked bool
+	Stale  bool
 
 	containerInfo *types.ContainerJSON
 	imageInfo     *types.ImageInspect
@@ -62,7 +56,7 @@ func (c Container) ImageID() string {
 // "latest" tag is assumed.
 func (c Container) ImageName() string {
 	// Compatibility w/ Zodiac deployments
-	imageName, ok := c.containerInfo.Config.Labels[zodiacLabel]
+	imageName, ok := c.getLabelValue(zodiacLabel)
 	if !ok {
 		imageName = c.containerInfo.Config.Image
 	}
@@ -77,7 +71,7 @@ func (c Container) ImageName() string {
 // Enabled returns the value of the container enabled label and if the label
 // was set.
 func (c Container) Enabled() (bool, bool) {
-	rawBool, ok := c.containerInfo.Config.Labels[enableLabel]
+	rawBool, ok := c.getLabelValue(enableLabel)
 	if !ok {
 		return false, false
 	}
@@ -105,6 +99,12 @@ func (c Container) Links() []string {
 	return links
 }
 
+// ToRestart return whether the container should be restarted, either because
+// is stale or linked to another stale container.
+func (c Container) ToRestart() bool {
+	return c.Stale || c.Linked
+}
+
 // IsWatchtower returns a boolean flag indicating whether or not the current
 // container is the watchtower container itself. The watchtower container is
 // identified by the presence of the "com.centurylinklabs.watchtower" label in
@@ -117,11 +117,7 @@ func (c Container) IsWatchtower() bool {
 // container's metadata. If the container has not specified a custom stop
 // signal, the empty string "" is returned.
 func (c Container) StopSignal() string {
-	if val, ok := c.containerInfo.Config.Labels[signalLabel]; ok {
-		return val
-	}
-
-	return ""
+	return c.getLabelValueOrEmpty(signalLabel)
 }
 
 // Ideally, we'd just be able to take the ContainerConfig from the old container
@@ -189,10 +185,3 @@ func (c Container) hostConfig() *dockercontainer.HostConfig {
 
 	return hostConfig
 }
-
-// ContainsWatchtowerLabel takes a map of labels and values and tells
-// the consumer whether it contains a valid watchtower instance label
-func ContainsWatchtowerLabel(labels map[string]string) bool {
-	val, ok := labels[watchtowerLabel]
-	return ok && val == "true"
-}

pkg/container/metadata.go

@@ -0,0 +1,39 @@
+package container
+
+const (
+	watchtowerLabel = "com.centurylinklabs.watchtower"
+	signalLabel     = "com.centurylinklabs.watchtower.stop-signal"
+	enableLabel     = "com.centurylinklabs.watchtower.enable"
+	zodiacLabel     = "com.centurylinklabs.zodiac.original-image"
+	preUpdateLabel  = "com.centurylinklabs.watchtower.lifecycle.pre-update"
+	postUpdateLabel = "com.centurylinklabs.watchtower.lifecycle.post-update"
+)
+
+// GetLifecyclePreUpdateCommand returns the pre-update command set in the container metadata or an empty string
+func (c Container) GetLifecyclePreUpdateCommand() string {
+	return c.getLabelValueOrEmpty(preUpdateLabel)
+}
+
+// GetLifecyclePostUpdateCommand returns the post-update command set in the container metadata or an empty string
+func (c Container) GetLifecyclePostUpdateCommand() string {
+	return c.getLabelValueOrEmpty(postUpdateLabel)
+}
+
+// ContainsWatchtowerLabel takes a map of labels and values and tells
+// the consumer whether it contains a valid watchtower instance label
+func ContainsWatchtowerLabel(labels map[string]string) bool {
+	val, ok := labels[watchtowerLabel]
+	return ok && val == "true"
+}
+
+func (c Container) getLabelValueOrEmpty(label string) string {
+	if val, ok := c.containerInfo.Config.Labels[label]; ok {
+		return val
+	}
+	return ""
+}
+
+func (c Container) getLabelValue(label string) (string, bool) {
+	val, ok := c.containerInfo.Config.Labels[label]
+	return val, ok
+}

scripts/lifecycle-tests.sh

@@ -0,0 +1,208 @@
+#!/usr/bin/env bash
+
+set -e
+
+IMAGE=server
+CONTAINER=server
+LINKED_IMAGE=linked
+LINKED_CONTAINER=linked
+WATCHTOWER_INTERVAL=2
+
+function remove_container {
+	docker kill $1 >> /dev/null || true && docker rm -v $1 >> /dev/null || true
+}
+
+function cleanup {
+  # Do cleanup on exit or error
+  echo "Final cleanup"
+  sleep 2
+  remove_container $CONTAINER
+  remove_container $LINKED_CONTAINER
+  pkill -9 -f watchtower >> /dev/null || true
+}
+trap cleanup EXIT
+
+DEFAULT_WATCHTOWER="$(dirname "${BASH_SOURCE[0]}")/../watchtower"
+WATCHTOWER=$1
+WATCHTOWER=${WATCHTOWER:-$DEFAULT_WATCHTOWER}
+echo "watchtower path is $WATCHTOWER"
+
+##################################################################################
+##### PREPARATION ################################################################
+##################################################################################
+
+#  Create Dockerfile template
+DOCKERFILE=$(cat << EOF
+FROM node:alpine
+
+LABEL com.centurylinklabs.watchtower.lifecycle.pre-update="cat /opt/test/value.txt"
+LABEL com.centurylinklabs.watchtower.lifecycle.post-update="echo image > /opt/test/value.txt"
+
+ENV IMAGE_TIMESTAMP=TIMESTAMP
+
+WORKDIR /opt/test
+ENTRYPOINT ["/usr/local/bin/node", "/opt/test/server.js"]
+
+EXPOSE 8888
+
+RUN mkdir -p /opt/test && echo "default" > /opt/test/value.txt
+COPY server.js /opt/test/server.js
+EOF
+)
+
+# Create temporary directory to build docker image
+TMP_DIR="/tmp/watchtower-commands-test"
+mkdir -p $TMP_DIR
+
+# Create simple http server
+cat > $TMP_DIR/server.js << EOF
+const http = require("http");
+const fs = require("fs");
+
+http.createServer(function(request, response) {
+	const fileContent = fs.readFileSync("/opt/test/value.txt");
+	response.writeHead(200, {"Content-Type": "text/plain"});
+	response.write(fileContent);
+	response.end();
+}).listen(8888, () => { console.log('server is listening on 8888'); });
+EOF
+
+function builddocker {
+	TIMESTAMP=$(date +%s)
+	echo "Building image $TIMESTAMP"
+	echo "${DOCKERFILE/TIMESTAMP/$TIMESTAMP}" > $TMP_DIR/Dockerfile
+	docker build $TMP_DIR -t $IMAGE >> /dev/null
+}
+
+# Start watchtower
+echo "Starting watchtower"
+$WATCHTOWER -i $WATCHTOWER_INTERVAL --no-pull --stop-timeout 2s --enable-lifecycle-hooks $CONTAINER $LINKED_CONTAINER &
+sleep 3
+
+echo "#################################################################"
+echo "##### TEST CASE 1: Execute commands from base image"
+echo "#################################################################"
+
+# Build base image
+builddocker
+
+# Run container
+docker run -d -p 0.0.0.0:8888:8888 --name $CONTAINER $IMAGE:latest >> /dev/null
+sleep 1
+echo "Container $CONTAINER is runnning"
+
+# Test default value
+RESP=$(curl -s http://localhost:8888)
+if [ $RESP != "default" ]; then
+	echo "Default value of container response is invalid" 1>&2
+	exit 1
+fi
+
+# Build updated image to trigger watchtower update
+builddocker
+
+WAIT_AMOUNT=$(($WATCHTOWER_INTERVAL * 3))
+echo "Wait for $WAIT_AMOUNT seconds"
+sleep $WAIT_AMOUNT
+
+# Test value after post-update-command
+RESP=$(curl -s http://localhost:8888)
+if [[ $RESP != "image" ]]; then
+	echo "Value of container response is invalid. Expected: image. Actual: $RESP"
+	exit 1
+fi
+
+remove_container $CONTAINER
+
+echo "#################################################################"
+echo "##### TEST CASE 2: Execute commands from container and base image"
+echo "#################################################################"
+
+# Build base image
+builddocker
+
+# Run container
+docker run -d -p 0.0.0.0:8888:8888 \
+	--label=com.centurylinklabs.watchtower.lifecycle.post-update="echo container > /opt/test/value.txt" \
+	--name $CONTAINER $IMAGE:latest >> /dev/null
+sleep 1
+echo "Container $CONTAINER is runnning"
+
+# Test default value
+RESP=$(curl -s http://localhost:8888)
+if [ $RESP != "default" ]; then
+	echo "Default value of container response is invalid" 1>&2
+	exit 1
+fi
+
+# Build updated image to trigger watchtower update
+builddocker
+
+WAIT_AMOUNT=$(($WATCHTOWER_INTERVAL * 3))
+echo "Wait for $WAIT_AMOUNT seconds"
+sleep $WAIT_AMOUNT
+
+# Test value after post-update-command
+RESP=$(curl -s http://localhost:8888)
+if [[ $RESP != "container" ]]; then
+	echo "Value of container response is invalid. Expected: container. Actual: $RESP"
+	exit 1
+fi
+
+remove_container $CONTAINER
+
+echo "#################################################################"
+echo "##### TEST CASE 3: Execute commands with a linked container"
+echo "#################################################################"
+
+# Tag the current image to keep a version for the linked container
+docker tag $IMAGE:latest $LINKED_IMAGE:latest
+
+# Build base image
+builddocker
+
+# Run container
+docker run -d -p 0.0.0.0:8888:8888 \
+	--label=com.centurylinklabs.watchtower.lifecycle.post-update="echo container > /opt/test/value.txt" \
+	--name $CONTAINER $IMAGE:latest >> /dev/null
+docker run -d -p 0.0.0.0:8989:8888 \
+	--label=com.centurylinklabs.watchtower.lifecycle.post-update="echo container > /opt/test/value.txt" \
+	--link $CONTAINER \
+	--name $LINKED_CONTAINER $LINKED_IMAGE:latest >> /dev/null
+sleep 1
+echo "Container $CONTAINER and $LINKED_CONTAINER are runnning"
+
+# Test default value
+RESP=$(curl -s http://localhost:8888)
+if [ $RESP != "default" ]; then
+	echo "Default value of container response is invalid" 1>&2
+	exit 1
+fi
+
+# Test default value for linked container
+RESP=$(curl -s http://localhost:8989)
+if [ $RESP != "default" ]; then
+	echo "Default value of linked container response is invalid" 1>&2
+	exit 1
+fi
+
+# Build updated image to trigger watchtower update
+builddocker
+
+WAIT_AMOUNT=$(($WATCHTOWER_INTERVAL * 3))
+echo "Wait for $WAIT_AMOUNT seconds"
+sleep $WAIT_AMOUNT
+
+# Test value after post-update-command
+RESP=$(curl -s http://localhost:8888)
+if [[ $RESP != "container" ]]; then
+	echo "Value of container response is invalid. Expected: container. Actual: $RESP"
+	exit 1
+fi
+
+# Test that linked container did not execute pre/post-update-command
+RESP=$(curl -s http://localhost:8989)
+if [[ $RESP != "default" ]]; then
+	echo "Value of linked container response is invalid. Expected: default. Actual: $RESP"
+	exit 1
+fi