From 5ca527066d1b8ff34d2d84b3dfbc9d48c5afad9a Mon Sep 17 00:00:00 2001
From: Tiago Carrondo <2323546+tcarrondo@users.noreply.github.com>
Date: Sat, 27 Jan 2024 16:03:37 +0000
Subject: [PATCH] Fix s3-storage migrate and shell (#3136)

* Fix s3-storage migrate and shell: container needs attachment to postgres network also

* Connect to s3-storage-provider migrate to multiple networks in multiple steps

Multiple `--network` calls lead to:

> docker: Error response from daemon: Container cannot be connected to network endpoints: NETWORK_1 NETWORK_2.

* Connect to s3-storage-provider shell to multiple networks in multiple steps

---------

Co-authored-by: Slavi Pantaleev <slavi@devture.com>
---
 .../ext/s3-storage-provider/bin/migrate.j2        | 14 ++++++++++++--
 .../synapse/ext/s3-storage-provider/bin/shell.j2  | 15 +++++++++++++--
 2 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/bin/migrate.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/bin/migrate.j2
index 5c3a534b5..8d8c4d0d7 100644
--- a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/bin/migrate.j2
+++ b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/bin/migrate.j2
@@ -1,7 +1,9 @@
 #jinja2: lstrip_blocks: "True"
 #!/bin/bash
+set -euo pipefail
 
-{{ devture_systemd_docker_base_host_command_docker }} run \
+container_id=$(\
+	{{ devture_systemd_docker_base_host_command_docker }} create \
 	--rm \
 	--env-file={{ matrix_synapse_ext_s3_storage_provider_base_path }}/env \
 	--mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \
@@ -10,4 +12,12 @@
 	--network={{ matrix_synapse_container_network }} \
 	--entrypoint=/bin/bash \
 	{{ matrix_synapse_docker_image_final }} \
-	-c 's3_media_upload update-db $UPDATE_DB_DURATION && s3_media_upload --no-progress check-deleted $MEDIA_PATH && s3_media_upload --no-progress upload $MEDIA_PATH $BUCKET --delete --storage-class $STORAGE_CLASS --endpoint-url $ENDPOINT {% if matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled %}--sse-customer-algo $SSE_CUSTOMER_ALGO --sse-customer-key $SSE_CUSTOMER_KEY{% endif %}'
+	-c 's3_media_upload update-db $UPDATE_DB_DURATION && s3_media_upload --no-progress check-deleted $MEDIA_PATH && s3_media_upload --no-progress upload $MEDIA_PATH $BUCKET --delete --storage-class $STORAGE_CLASS --endpoint-url $ENDPOINT {% if matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled %}--sse-customer-algo $SSE_CUSTOMER_ALGO --sse-customer-key $SSE_CUSTOMER_KEY{% endif %}' \
+)
+
+{# We need to connect to the Postgres network, which should be in this list. #}
+{% for network in matrix_synapse_container_additional_networks %}
+{{ devture_systemd_docker_base_host_command_docker }} network connect {{ network }} $container_id
+{% endfor %}
+
+{{ devture_systemd_docker_base_host_command_docker }} start --attach $container_id
diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/bin/shell.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/bin/shell.j2
index 6f3804cc4..239554a1c 100644
--- a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/bin/shell.j2
+++ b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/bin/shell.j2
@@ -1,13 +1,24 @@
 #jinja2: lstrip_blocks: "True"
 #!/bin/bash
+set -euo pipefail
 
-{{ devture_systemd_docker_base_host_command_docker }} run \
+container_id=$(\
+	{{ devture_systemd_docker_base_host_command_docker }} create \
 	-it \
 	--rm \
+	--name=matrix-synapse-s3-storage-provider-shell \
 	--env-file={{ matrix_synapse_ext_s3_storage_provider_base_path }}/env \
 	--mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \
 	--mount type=bind,src={{ matrix_synapse_ext_s3_storage_provider_data_path }},dst=/data \
 	--workdir=/data \
 	--network={{ matrix_synapse_container_network }} \
 	--entrypoint=/bin/bash \
-	{{ matrix_synapse_docker_image_final }}
+	{{ matrix_synapse_docker_image_final }} \
+)
+
+{# We need to connect to the Postgres network, which should be in this list. #}
+{% for network in matrix_synapse_container_additional_networks %}
+{{ devture_systemd_docker_base_host_command_docker }} network connect {{ network }} $container_id
+{% endfor %}
+
+{{ devture_systemd_docker_base_host_command_docker }} start --attach -i $container_id