From 353bc7c3624f6b7186d65757e7b66bc8e6e56e2c Mon Sep 17 00:00:00 2001
From: Marcel Partap <mpartap@gmx.net>
Date: Fri, 10 Apr 2020 23:44:53 +0200
Subject: [PATCH 1/8] Add initial support for synapse workers
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

· needs documentation; no checks yet for port clashes or typos in worker name
· according to https://github.com/matrix-org/synapse/wiki/Workers-setup-with-nginx#results
  about 90% of requests go to the synchrotron endpoint
· thus, the synchrotron worker is especially suited to be load-balanced
· most of the other workers are documented to support only a single instance
· https://github.com/matrix-org/synapse/blob/master/docs/workers.md
---
 roles/matrix-synapse/defaults/main.yml        | 37 +++++++++++++++++
 roles/matrix-synapse/tasks/setup_synapse.yml  |  2 +
 roles/matrix-synapse/tasks/workers/setup.yml  |  7 ++++
 .../tasks/workers/setup_install.yml           | 41 +++++++++++++++++++
 .../tasks/workers/setup_uninstall.yml         | 34 +++++++++++++++
 .../templates/synapse/homeserver.yaml.j2      | 38 +++++++++++++++++
 .../systemd/matrix-synapse-worker@.service.j2 | 29 +++++++++++++
 .../templates/synapse/worker.yaml.j2          | 29 +++++++++++++
 8 files changed, 217 insertions(+)
 create mode 100644 roles/matrix-synapse/tasks/workers/setup.yml
 create mode 100644 roles/matrix-synapse/tasks/workers/setup_install.yml
 create mode 100644 roles/matrix-synapse/tasks/workers/setup_uninstall.yml
 create mode 100644 roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2
 create mode 100644 roles/matrix-synapse/templates/synapse/worker.yaml.j2

diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml
index f117fe93d..dcd42ab83 100644
--- a/roles/matrix-synapse/defaults/main.yml
+++ b/roles/matrix-synapse/defaults/main.yml
@@ -258,6 +258,43 @@ matrix_synapse_metrics_port: 9100
 # See https://github.com/matrix-org/synapse/blob/master/docs/manhole.md
 matrix_synapse_manhole_enabled: false
 
+# Enable support for Synapse workers
+matrix_synapse_workers_enabled: false
+
+# List of workers to spawn
+matrix_synapse_workers_enabled_list: []
+
+# Default list of workers to spawn
+matrix_synapse_workers_enabled_list:
+  - { worker: synchrotron, port: 8082 }
+  - { worker: synchrotron, port: 8083 }
+  - { worker: synchrotron, port: 8084 }
+  - { worker: appservice, port: 8085 }
+  - { worker: client_reader, port: 8086 }
+  - { worker: event_creator, port: 8087 }
+  - { worker: federation_reader, port: 8088 }
+  - { worker: federation_sender, port: 8089 }
+  - { worker: frontend_proxy, port: 8090 }
+  - { worker: media_repository, port: 8091 }
+  - { worker: pusher, port: 8092 }
+  - { worker: user_dir, port: 8093 }
+
+# The list of available workers (2020-04-14)
+matrix_synapse_workers_avail_list:
+  - appservice
+  - client_reader
+  - event_creator
+  - federation_reader
+  - federation_sender
+  - frontend_proxy
+  - media_repository
+  - pusher
+  - synchrotron
+  - user_dir
+
+# Ports used for communication between main synapse process and workers
+matrix_synapse_replication_tcp_port: 9092
+matrix_synapse_replication_http_port: 9093
 
 # Send ERROR logs to sentry.io for easier tracking
 # To set this up: go to sentry.io, create a python project, and set
diff --git a/roles/matrix-synapse/tasks/setup_synapse.yml b/roles/matrix-synapse/tasks/setup_synapse.yml
index c40ae170b..ad1b24e6b 100644
--- a/roles/matrix-synapse/tasks/setup_synapse.yml
+++ b/roles/matrix-synapse/tasks/setup_synapse.yml
@@ -19,6 +19,8 @@
 
 - import_tasks: "{{ role_path }}/tasks/ext/setup.yml"
 
+- import_tasks: "{{ role_path }}/tasks/workers/setup.yml"
+
 - import_tasks: "{{ role_path }}/tasks/synapse/setup.yml"
 
 - import_tasks: "{{ role_path }}/tasks/goofys/setup.yml"
diff --git a/roles/matrix-synapse/tasks/workers/setup.yml b/roles/matrix-synapse/tasks/workers/setup.yml
new file mode 100644
index 000000000..faf2899e2
--- /dev/null
+++ b/roles/matrix-synapse/tasks/workers/setup.yml
@@ -0,0 +1,7 @@
+---
+
+- import_tasks: "{{ role_path }}/tasks/workers/setup_install.yml"
+  when: "matrix_synapse_enabled|bool and matrix_synapse_workers_enabled|bool"
+
+- import_tasks: "{{ role_path }}/tasks/workers/setup_uninstall.yml"
+  when: "not matrix_synapse_workers_enabled|bool"
diff --git a/roles/matrix-synapse/tasks/workers/setup_install.yml b/roles/matrix-synapse/tasks/workers/setup_install.yml
new file mode 100644
index 000000000..1f23d0c5a
--- /dev/null
+++ b/roles/matrix-synapse/tasks/workers/setup_install.yml
@@ -0,0 +1,41 @@
+---
+
+- name: Ensure synapse worker base service file installed
+  template:
+    src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker@.service.j2"
+    dest: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service"
+    mode: 0644
+  register: matrix_synapse_worker_systemd_service_result
+
+- name: Ensure previous worker service symlinks are cleaned (FIXME)
+  file:
+    path: "{{ item.root + '/' + item.path }}"
+    state: absent
+  when:
+    - item.state == 'link'
+    - item.path is match('matrix-synapse-worker@*.service')
+  with_filetree:
+    - "{{ matrix_systemd_path }}/matrix-synapse.service.wants"
+
+- name: Ensure systemd reloaded the worker service unit
+  service:
+    daemon_reload: yes
+
+- name: Ensure individual worker service symlinks exist
+  service:
+    name: "matrix-synapse-worker@{{ item.worker }}:{{ item.port }}.service"
+    enabled: true
+  with_items: "{{ matrix_synapse_workers_enabled_list }}"
+
+- name: Ensure creation of specific worker configs
+  template:
+    src: "{{ role_path }}/templates/synapse/worker.yaml.j2"
+    dest: "{{ matrix_synapse_config_dir_path }}/worker.{{ item.worker }}:{{ item.port }}.yaml"
+  with_list: "{{ matrix_synapse_workers_enabled_list }}"
+
+- name: Add workers to synapse.wants list
+  set_fact:
+    matrix_synapse_systemd_wanted_services_list: >
+      {{ matrix_synapse_systemd_wanted_services_list +
+      ['matrix-synapse-worker@' + item.worker + ':' + item.port|string + '.service'] }}
+  with_items: "{{ matrix_synapse_workers_enabled_list }}"
diff --git a/roles/matrix-synapse/tasks/workers/setup_uninstall.yml b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml
new file mode 100644
index 000000000..864308794
--- /dev/null
+++ b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml
@@ -0,0 +1,34 @@
+---
+
+- name: Ensure individual worker services are stopped
+  service:
+    name: "matrix-synapse-worker@{{ item.worker }}:{{ item.port }}.service"
+    state: stopped
+  with_items: "{{ matrix_synapse_workers_enabled_list }}"
+
+# As we cannot know the ports of workers removed from the enabled_list..
+# => .. just kill them all (FIXME?)
+- name: Ensure previous worker service symlinks are cleaned
+  file:
+    path: "{{ item.root + '/' + item.path }}"
+    state: absent
+  when:
+    - item.state == 'link'
+    - item.path is match('matrix-synapse-worker@*.service')
+  with_filetree:
+    - "{{ matrix_systemd_path }}/matrix-synapse.service.wants"
+
+- name: Ensure synapse worker base service file gets removed
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service"
+    state: absent
+  register: matrix_synapse_worker_systemd_service_result
+
+- name: Remove workers from synapse.wants list
+  set_fact:
+    matrix_synapse_systemd_wanted_services_list: "{{ matrix_synapse_systemd_wanted_services_list | reject('search', item) | list }}"
+  with_items: "{{ matrix_synapse_workers_avail_list }}"
+
+- name: Ensure systemd noticed removal of worker service units
+  service:
+    daemon_reload: yes
diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2
index d41313e39..fde097f0c 100644
--- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2
+++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2
@@ -251,6 +251,44 @@ listeners:
     type: manhole
 {% endif %}
 
+{% if matrix_synapse_workers_enabled %}
+  # c.f. https://github.com/matrix-org/synapse/tree/master/docs/workers.md
+  # TCP replication: streaming data from the master to the workers
+  - port: {{ matrix_synapse_replication_tcp_port }}
+    bind_addresses: ['0.0.0.0']
+    type: replication
+
+  # HTTP replication: for the workers to send data to the main synapse process
+  - port: {{ matrix_synapse_replication_http_port }}
+    bind_addresses: ['0.0.0.0']
+    type: http
+    resources:
+      - names: [replication]
+
+# c.f. https://github.com/matrix-org/synapse/tree/master/contrib/systemd-with-workers/README.md
+worker_app: synapse.app.homeserver
+
+# thx https://oznetnerd.com/2017/04/18/jinja2-selectattr-filter/
+# reduce the main worker's offerings to core homeserver business
+{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'appservice')|list  %}
+notify_appservices: false
+{% endif %}
+{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_sender')|list  %}
+send_federation: false
+{% endif %}
+{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|list  %}
+enable_media_repo: false
+{% endif %}
+{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'pusher')|list  %}
+start_pushers: false
+{% endif %}
+{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|list  %}
+update_user_directory: false
+{% endif %}
+
+# rather let systemd handle the forking
+daemonize: false
+{% endif %}
 
 ## Homeserver blocking ##
 
diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2
new file mode 100644
index 000000000..a46517c4e
--- /dev/null
+++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2
@@ -0,0 +1,29 @@
+#jinja2: lstrip_blocks: "True"
+# c.f. https://github.com/matrix-org/synapse/pull/4662
+[Unit]
+Description=Synapse Matrix Worker
+After=matrix-synapse.service
+BindsTo=matrix-synapse.service
+
+[Service]
+Type=simple
+
+# Intentional delay, so that the homeserver (we likely depend on) can manage to start.
+ExecStartPre=/bin/sleep 5
+
+# systemd ftw 🤦‍♂️
+# https://github.com/systemd/systemd/issues/14895#issuecomment-594123923
+ExecStart=/bin/sh -c "WORKER=%i; WORKER=$${WORKER%%:*}; \
+			exec /usr/bin/docker exec \
+			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
+			matrix-synapse \
+			python -m synapse.app.$${WORKER} -c /data/homeserver.yaml -c /data/worker.%i.yaml"
+
+ExecStop=/usr/bin/docker exec matrix-synapse pkill -f %i
+KillMode=process
+Restart=always
+RestartSec=10
+SyslogIdentifier=matrix-synapse-%i
+
+[Install]
+WantedBy=matrix-synapse.service
diff --git a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 b/roles/matrix-synapse/templates/synapse/worker.yaml.j2
new file mode 100644
index 000000000..37a5f87b2
--- /dev/null
+++ b/roles/matrix-synapse/templates/synapse/worker.yaml.j2
@@ -0,0 +1,29 @@
+#jinja2: lstrip_blocks: "True"
+worker_app: synapse.app.{{ item.worker }}
+
+worker_replication_host: 127.0.0.1
+worker_replication_port: {{ matrix_synapse_replication_tcp_port }}
+worker_replication_http_port: {{ matrix_synapse_replication_http_port }}
+
+{% if item.worker not in [ 'appservice', 'federation_sender', 'pusher' ] %}
+worker_listeners:
+  - type: http
+    port: {{ item.port }}
+    resources:
+      - names:
+{% if item.worker in [ 'synchrotron', 'client_reader', 'event_creator', 'frontend_proxy', 'user_dir' ] %}
+        - client
+{% elif item.worker in [ 'federation_reader' ] %}
+        - federation
+{% elif item.worker in [ 'media_repository' ] %}
+        - media
+{% endif %}
+{% endif %}
+
+{% if item.worker == 'frontend_proxy' %}
+worker_main_http_uri: http://127.0.0.1:8008
+{% endif %}
+  
+worker_daemonize: false
+worker_pid_file: /matrix-run/{{ item.worker }}.port{{ item.port }}.pid
+worker_log_config: /data/{{ matrix_server_fqn_matrix }}.log.config

From a14b9c09adee37282b2deb7d0ff63594b5174107 Mon Sep 17 00:00:00 2001
From: Marcel Partap <mpartap@gmx.net>
Date: Sat, 11 Apr 2020 03:08:59 +0200
Subject: [PATCH 2/8] Add to synapse nginx template conditional URL rewrites
 for workers
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

· 😅 How to keep this in sync with the matrix-synapse documentation?
· regex location matching is expensive
· nginx syntax limit: one location only per block / statement
· thus, lots of duplicate statements in this file
---
 .../nginx/conf.d/matrix-synapse.conf.j2       | 110 ++++++++++++++++++
 1 file changed, 110 insertions(+)

diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2
index 996625156..0f4982cc5 100644
--- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2
+++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2
@@ -101,6 +101,93 @@
 	}
 	{% endif %}
 
+	{% if synchrotron_workers %}
+	{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L134 #}
+	location /_matrix/client/r0/sync {
+		proxy_pass http://synchrotron$request_uri;
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+	}
+	location /_matrix/client/r0/events {
+		proxy_pass http://synchrotron$request_uri;
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+	}
+	location /_matrix/client/r0/initialSync {
+		proxy_pass http://synchrotron$request_uri;
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+	}
+	location ~ ^/_matrix/client/r0/rooms/[^/]+/initialSync$ {
+		proxy_pass http://synchrotron$request_uri;
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+	}
+	{% endif %}
+
+	{% set client_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'client_reader')|first %}
+	{% if client_reader_worker %}
+        {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L252 #}
+        location ^/_matrix/client/(versions$|(api/v1|r0|unstable)/(publicRooms$|rooms/.*/joined_me|rooms/.*/context/.|rooms/.*/members$|rooms/.*/messages$|rooms/.*/state$|login$|account/3pid$|keys/query$|keys/changes$|voip/turnServer$|joined_groups$|publicised_groups$|publicised_groups/|pushrules/.*$|groups/.*$|register$|auth/.*/fallback/web$)) {
+                proxy_pass http://127.0.0.1:{{ client_reader_worker.port }}$request_uri;
+                proxy_set_header Host $host;
+                proxy_set_header X-Forwarded-For $remote_addr;
+        }
+	{% endif %}
+
+	{% set media_repository_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|first %}
+	{% if media_repository_worker %}
+        {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L219 #}
+        location /_matrix/media/ {
+                proxy_pass http://127.0.0.1:{{ media_repository_worker.port }}$request_uri;
+                proxy_set_header Host $host;
+                proxy_set_header X-Forwarded-For $remote_addr;
+        }
+        {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L223 #}
+        location ~ ^/_synapse/admin/v1/(purge_media_cache|room/.*/media.*|user/.*/media.*|media/.*|quarantine_media/.*)$ {
+                proxy_pass http://127.0.0.1:{{ media_repository_worker.port }}$request_uri;
+                proxy_set_header Host $host;
+                proxy_set_header X-Forwarded-For $remote_addr;
+        }
+	{% endif %}
+
+	{% set event_creator_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'event_creator')|first %}
+	{% if event_creator_worker %}
+        {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L323 #}
+        location ~ ^/_matrix/client/(api/v1|r0|unstable)/(rooms/.*/send|rooms/.*/state/|rooms/.*/(join|invite|leave|ban|unban|kick)$|join/|profile/) {
+                proxy_pass http://127.0.0.1:{{ event_creator_worker.port }}$request_uri;
+                proxy_set_header Host $host;
+                proxy_set_header X-Forwarded-For $remote_addr;
+        }
+	{% endif %}
+
+	{% set frontend_proxy_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'frontend_proxy')|first %}
+	{% if frontend_proxy_worker %}
+        {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L302 #}
+        location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload {
+                proxy_pass http://127.0.0.1:{{ frontend_proxy_worker.port }}$request_uri;
+                proxy_set_header Host $host;
+                proxy_set_header X-Forwarded-For $remote_addr;
+        }
+        {% if not matrix_synapse_use_presence %}
+                location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status {
+                        proxy_pass http://127.0.0.1:{{ frontend_proxy_worker.port }}$request_uri;
+                        proxy_set_header Host $host;
+                        proxy_set_header X-Forwarded-For $remote_addr;
+                }
+        {% endif %}
+	{% endif %}
+
+	{% set user_dir_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|first %}
+	{% if user_dir_worker %}
+        {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L290 #}
+        location ~ ^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$ {
+                proxy_pass http://127.0.0.1:{{ user_dir_worker.port }}$request_uri;
+                proxy_set_header Host $host;
+                proxy_set_header X-Forwarded-For $remote_addr;
+        }
+	{% endif %}
+
 	{% for configuration_block in matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks %}
 		{{- configuration_block }}
 	{% endfor %}
@@ -174,6 +261,19 @@
 	}
 {% endmacro %}
 
+{% set synchrotron_workers = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'synchrotron')|list %}
+{% if synchrotron_workers %}
+upstream synchrotron {
+	# ensures that requests from the same client will always be passed
+	# to the same server (except when this server is unavailable)
+	ip_hash;
+
+	{% for worker in synchrotron_workers %}
+		server 127.0.0.1:{{ worker.port }};
+	{% endfor %}
+}
+{% endif %}
+
 server {
 	listen {{ 8080 if matrix_nginx_proxy_enabled else 80 }};
 	server_name {{ matrix_nginx_proxy_proxy_matrix_hostname }};
@@ -255,6 +355,16 @@ server {
 		ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH";
 	{% endif %}
 
+	{% set federation_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_reader')|first %}
+	{% if federation_worker %}
+        {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L160 #}
+        location ~ ^(/_matrix/federation/v1/event/|/_matrix/federation/v1/state/|/_matrix/federation/v1/state_ids/|/_matrix/federation/v1/backfill/|/_matrix/federation/v1/get_missing_events/|/_matrix/federation/v1/publicRooms|/_matrix/federation/v1/query/|/_matrix/federation/v1/make_join/|/_matrix/federation/v1/make_leave/|/_matrix/federation/v1/send_join/|/_matrix/federation/v2/send_join/|/_matrix/federation/v1/send_leave/|/_matrix/federation/v2/send_leave/|/_matrix/federation/v1/invite/|/_matrix/federation/v2/invite/|/_matrix/federation/v1/query_auth/|/_matrix/federation/v1/event_auth/|/_matrix/federation/v1/exchange_third_party_invite/|/_matrix/federation/v1/user/devices/|/_matrix/federation/v1/send/|/_matrix/federation/v1/get_groups_publicised$|/_matrix/key/v2/query|/_matrix/federation/v1/groups/) {
+                proxy_pass http://127.0.0.1:{{ federation_worker.port }}$request_uri;
+                proxy_set_header Host $host;
+                proxy_set_header X-Forwarded-For $remote_addr;
+        }
+	{% endif %}
+
 	location / {
 		{% if matrix_nginx_proxy_enabled %}
 			{# Use the embedded DNS resolver in Docker containers to discover the service #}

From cf452fdf0a1ece84ae4c54e6a73a8618114c5132 Mon Sep 17 00:00:00 2001
From: Marcel Partap <mpartap@gmx.net>
Date: Sun, 19 Apr 2020 10:16:01 +0200
Subject: [PATCH 3/8] Fix corner-cases found through testing (aka ansible is
 nuts)

---
 roles/matrix-synapse/tasks/workers/setup.yml         |  1 +
 roles/matrix-synapse/tasks/workers/setup_install.yml |  3 ++-
 .../matrix-synapse/tasks/workers/setup_uninstall.yml | 12 ++++++++----
 3 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/roles/matrix-synapse/tasks/workers/setup.yml b/roles/matrix-synapse/tasks/workers/setup.yml
index faf2899e2..4951ac2d7 100644
--- a/roles/matrix-synapse/tasks/workers/setup.yml
+++ b/roles/matrix-synapse/tasks/workers/setup.yml
@@ -1,4 +1,5 @@
 ---
+# a negative when condition will not actually prevent ansible from executing loops in imported tasks!
 
 - import_tasks: "{{ role_path }}/tasks/workers/setup_install.yml"
   when: "matrix_synapse_enabled|bool and matrix_synapse_workers_enabled|bool"
diff --git a/roles/matrix-synapse/tasks/workers/setup_install.yml b/roles/matrix-synapse/tasks/workers/setup_install.yml
index 1f23d0c5a..0031c236c 100644
--- a/roles/matrix-synapse/tasks/workers/setup_install.yml
+++ b/roles/matrix-synapse/tasks/workers/setup_install.yml
@@ -12,8 +12,9 @@
     path: "{{ item.root + '/' + item.path }}"
     state: absent
   when:
+    - matrix_synapse_workers_enabled|bool
     - item.state == 'link'
-    - item.path is match('matrix-synapse-worker@*.service')
+    - item.path is match('matrix-synapse-worker@.*\\.service')
   with_filetree:
     - "{{ matrix_systemd_path }}/matrix-synapse.service.wants"
 
diff --git a/roles/matrix-synapse/tasks/workers/setup_uninstall.yml b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml
index 864308794..d1e7e3b56 100644
--- a/roles/matrix-synapse/tasks/workers/setup_uninstall.yml
+++ b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml
@@ -1,10 +1,13 @@
 ---
 
-- name: Ensure individual worker services are stopped
+- name: Populate service facts
+  service_facts:
+
+- name: Ensure any worker services are stopped
   service:
-    name: "matrix-synapse-worker@{{ item.worker }}:{{ item.port }}.service"
+    name: "{{ item.key }}"
     state: stopped
-  with_items: "{{ matrix_synapse_workers_enabled_list }}"
+  with_dict: "{{ ansible_facts.services|default({})|dict2items|selectattr('key', 'match', 'matrix-synapse-worker@.+\\.service')|list|items2dict }}"
 
 # As we cannot know the ports of workers removed from the enabled_list..
 # => .. just kill them all (FIXME?)
@@ -13,8 +16,9 @@
     path: "{{ item.root + '/' + item.path }}"
     state: absent
   when:
+    - not matrix_synapse_workers_enabled|bool
     - item.state == 'link'
-    - item.path is match('matrix-synapse-worker@*.service')
+    - item.path is match('matrix-synapse-worker@.*\\.service')
   with_filetree:
     - "{{ matrix_systemd_path }}/matrix-synapse.service.wants"
 

From 66a40735126c361e396dde47e9f96c9fe9664ee4 Mon Sep 17 00:00:00 2001
From: Marcel Partap <mpartap@gmx.net>
Date: Sun, 19 Apr 2020 17:42:51 +0200
Subject: [PATCH 4/8] Publish synapse worker ports, need to be accessible to
 nginx

---
 .../templates/synapse/systemd/matrix-synapse.service.j2      | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2
index 0bd2c25d5..700e41349 100644
--- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2
+++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2
@@ -44,6 +44,11 @@ ExecStart=/usr/bin/docker run --rm --name matrix-synapse \
 			{% if matrix_synapse_manhole_enabled and matrix_synapse_container_manhole_api_host_bind_port %}
 			-p {{ matrix_synapse_container_manhole_api_host_bind_port }}:9000 \
 			{% endif %}
+			{% if matrix_synapse_workers_enabled %}
+			{% for worker in matrix_synapse_workers_enabled_list %}
+			-p {{ worker.port }}:{{ worker.port }} \
+			{% endfor %}
+			{% endif %}
 			-v {{ matrix_synapse_config_dir_path }}:/data:ro \
 			-v {{ matrix_synapse_run_path }}:/matrix-run:rw \
 			-v {{ matrix_synapse_storage_path }}:/matrix-media-store-parent:slave \

From e4763c21bc4a2e3a053629618d7ba7c78e120f92 Mon Sep 17 00:00:00 2001
From: Marcel Partap <mpartap@gmx.net>
Date: Sun, 19 Apr 2020 18:41:27 +0200
Subject: [PATCH 5/8] nginx config: route traffic to workers on matrix-synapse

FIXME: horrid duplication in template file
---
 .../nginx/conf.d/matrix-synapse.conf.j2       | 133 +++++++++++-------
 1 file changed, 80 insertions(+), 53 deletions(-)

diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2
index 0f4982cc5..d7a1e6b3a 100644
--- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2
+++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2
@@ -127,65 +127,86 @@
 
 	{% set client_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'client_reader')|first %}
 	{% if client_reader_worker %}
-        {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L252 #}
-        location ^/_matrix/client/(versions$|(api/v1|r0|unstable)/(publicRooms$|rooms/.*/joined_me|rooms/.*/context/.|rooms/.*/members$|rooms/.*/messages$|rooms/.*/state$|login$|account/3pid$|keys/query$|keys/changes$|voip/turnServer$|joined_groups$|publicised_groups$|publicised_groups/|pushrules/.*$|groups/.*$|register$|auth/.*/fallback/web$)) {
-                proxy_pass http://127.0.0.1:{{ client_reader_worker.port }}$request_uri;
-                proxy_set_header Host $host;
-                proxy_set_header X-Forwarded-For $remote_addr;
-        }
+	{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L252 #}
+	location ^/_matrix/client/(versions$|(api/v1|r0|unstable)/(publicRooms$|rooms/.*/joined_me|rooms/.*/context/.|rooms/.*/members$|rooms/.*/messages$|rooms/.*/state$|login$|account/3pid$|keys/query$|keys/changes$|voip/turnServer$|joined_groups$|publicised_groups$|publicised_groups/|pushrules/.*$|groups/.*$|register$|auth/.*/fallback/web$)) {
+		{# Use the embedded DNS resolver in Docker containers to discover the service #}
+		resolver 127.0.0.11 valid=5s;
+		set $backend "matrix-synapse:{{ client_reader_worker.port }}"
+		proxy_pass http://$backend$request_uri;
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+	}
 	{% endif %}
 
 	{% set media_repository_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|first %}
 	{% if media_repository_worker %}
-        {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L219 #}
-        location /_matrix/media/ {
-                proxy_pass http://127.0.0.1:{{ media_repository_worker.port }}$request_uri;
-                proxy_set_header Host $host;
-                proxy_set_header X-Forwarded-For $remote_addr;
-        }
-        {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L223 #}
-        location ~ ^/_synapse/admin/v1/(purge_media_cache|room/.*/media.*|user/.*/media.*|media/.*|quarantine_media/.*)$ {
-                proxy_pass http://127.0.0.1:{{ media_repository_worker.port }}$request_uri;
-                proxy_set_header Host $host;
-                proxy_set_header X-Forwarded-For $remote_addr;
-        }
+	{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L219 #}
+	location /_matrix/media/ {
+		{# Use the embedded DNS resolver in Docker containers to discover the service #}
+		resolver 127.0.0.11 valid=5s;
+		set $backend "matrix-synapse:{{ media_repository_worker.port }}"
+		proxy_pass http://$backend$request_uri;
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+	}
+	{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L223 #}
+	location ~ ^/_synapse/admin/v1/(purge_media_cache|room/.*/media.*|user/.*/media.*|media/.*|quarantine_media/.*)$ {
+		{# Use the embedded DNS resolver in Docker containers to discover the service #}
+		resolver 127.0.0.11 valid=5s;
+		set $backend "matrix-synapse:{{ media_repository_worker.port }}"
+		proxy_pass http://$backend$request_uri;
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+	}
 	{% endif %}
 
 	{% set event_creator_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'event_creator')|first %}
 	{% if event_creator_worker %}
-        {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L323 #}
-        location ~ ^/_matrix/client/(api/v1|r0|unstable)/(rooms/.*/send|rooms/.*/state/|rooms/.*/(join|invite|leave|ban|unban|kick)$|join/|profile/) {
-                proxy_pass http://127.0.0.1:{{ event_creator_worker.port }}$request_uri;
-                proxy_set_header Host $host;
-                proxy_set_header X-Forwarded-For $remote_addr;
-        }
+	{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L323 #}
+	location ~ ^/_matrix/client/(api/v1|r0|unstable)/(rooms/.*/send|rooms/.*/state/|rooms/.*/(join|invite|leave|ban|unban|kick)$|join/|profile/) {
+		{# Use the embedded DNS resolver in Docker containers to discover the service #}
+		resolver 127.0.0.11 valid=5s;
+		set $backend "matrix-synapse:{{ event_creator_worker.port }}"
+		proxy_pass http://$backend$request_uri;
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+	}
 	{% endif %}
 
 	{% set frontend_proxy_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'frontend_proxy')|first %}
 	{% if frontend_proxy_worker %}
-        {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L302 #}
-        location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload {
-                proxy_pass http://127.0.0.1:{{ frontend_proxy_worker.port }}$request_uri;
-                proxy_set_header Host $host;
-                proxy_set_header X-Forwarded-For $remote_addr;
-        }
-        {% if not matrix_synapse_use_presence %}
-                location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status {
-                        proxy_pass http://127.0.0.1:{{ frontend_proxy_worker.port }}$request_uri;
-                        proxy_set_header Host $host;
-                        proxy_set_header X-Forwarded-For $remote_addr;
-                }
-        {% endif %}
+	{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L302 #}
+	location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload {
+		{# Use the embedded DNS resolver in Docker containers to discover the service #}
+		resolver 127.0.0.11 valid=5s;
+		set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}"
+		proxy_pass http://$backend$request_uri;
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+	}
+	{% if not matrix_synapse_use_presence %}
+		location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status {
+			{# Use the embedded DNS resolver in Docker containers to discover the service #}
+			resolver 127.0.0.11 valid=5s;
+			set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}"
+			proxy_pass http://$backend$request_uri;
+			proxy_set_header Host $host;
+			proxy_set_header X-Forwarded-For $remote_addr;
+		}
+	{% endif %}
 	{% endif %}
 
 	{% set user_dir_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|first %}
 	{% if user_dir_worker %}
-        {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L290 #}
-        location ~ ^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$ {
-                proxy_pass http://127.0.0.1:{{ user_dir_worker.port }}$request_uri;
-                proxy_set_header Host $host;
-                proxy_set_header X-Forwarded-For $remote_addr;
-        }
+	{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L290 #}
+	location ~ ^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$ {
+		{# Use the embedded DNS resolver in Docker containers to discover the service #}
+		resolver 127.0.0.11 valid=5s;
+		set $backend "matrix-synapse:{{ user_dir_worker.port }}"
+		proxy_pass http://$backend$request_uri;
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+	}
 	{% endif %}
 
 	{% for configuration_block in matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks %}
@@ -268,8 +289,11 @@ upstream synchrotron {
 	# to the same server (except when this server is unavailable)
 	ip_hash;
 
-	{% for worker in synchrotron_workers %}
-		server 127.0.0.1:{{ worker.port }};
+	{% for synchrotron_worker in synchrotron_workers %}
+		{# Use the embedded DNS resolver in Docker containers to discover the service #}
+		resolver 127.0.0.11 valid=5s;
+		set $backend "matrix-synapse:{{ synchrotron_worker.port }}"
+		server $backend:{{ synchrotron_worker.port }};
 	{% endfor %}
 }
 {% endif %}
@@ -355,14 +379,17 @@ server {
 		ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH";
 	{% endif %}
 
-	{% set federation_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_reader')|first %}
-	{% if federation_worker %}
-        {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L160 #}
-        location ~ ^(/_matrix/federation/v1/event/|/_matrix/federation/v1/state/|/_matrix/federation/v1/state_ids/|/_matrix/federation/v1/backfill/|/_matrix/federation/v1/get_missing_events/|/_matrix/federation/v1/publicRooms|/_matrix/federation/v1/query/|/_matrix/federation/v1/make_join/|/_matrix/federation/v1/make_leave/|/_matrix/federation/v1/send_join/|/_matrix/federation/v2/send_join/|/_matrix/federation/v1/send_leave/|/_matrix/federation/v2/send_leave/|/_matrix/federation/v1/invite/|/_matrix/federation/v2/invite/|/_matrix/federation/v1/query_auth/|/_matrix/federation/v1/event_auth/|/_matrix/federation/v1/exchange_third_party_invite/|/_matrix/federation/v1/user/devices/|/_matrix/federation/v1/send/|/_matrix/federation/v1/get_groups_publicised$|/_matrix/key/v2/query|/_matrix/federation/v1/groups/) {
-                proxy_pass http://127.0.0.1:{{ federation_worker.port }}$request_uri;
-                proxy_set_header Host $host;
-                proxy_set_header X-Forwarded-For $remote_addr;
-        }
+	{% set federation_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_reader')|first %}
+	{% if federation_reader_worker %}
+	{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L160 #}
+	location ~ ^(/_matrix/federation/v1/event/|/_matrix/federation/v1/state/|/_matrix/federation/v1/state_ids/|/_matrix/federation/v1/backfill/|/_matrix/federation/v1/get_missing_events/|/_matrix/federation/v1/publicRooms|/_matrix/federation/v1/query/|/_matrix/federation/v1/make_join/|/_matrix/federation/v1/make_leave/|/_matrix/federation/v1/send_join/|/_matrix/federation/v2/send_join/|/_matrix/federation/v1/send_leave/|/_matrix/federation/v2/send_leave/|/_matrix/federation/v1/invite/|/_matrix/federation/v2/invite/|/_matrix/federation/v1/query_auth/|/_matrix/federation/v1/event_auth/|/_matrix/federation/v1/exchange_third_party_invite/|/_matrix/federation/v1/user/devices/|/_matrix/federation/v1/send/|/_matrix/federation/v1/get_groups_publicised$|/_matrix/key/v2/query|/_matrix/federation/v1/groups/) {
+		{# Use the embedded DNS resolver in Docker containers to discover the service #}
+		resolver 127.0.0.11 valid=5s;
+		set $backend "matrix-synapse:{{ federation_reader_worker.port }}"
+		proxy_pass http://$backend$request_uri;
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+	}
 	{% endif %}
 
 	location / {

From 5f63d287b7dba70830c0218f18a873e02b9bd79c Mon Sep 17 00:00:00 2001
From: Marcel Partap <mpartap@gmx.net>
Date: Sun, 19 Apr 2020 18:57:00 +0200
Subject: [PATCH 6/8] Move synapse worker ports up 10k

---
 roles/matrix-synapse/defaults/main.yml | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml
index dcd42ab83..9e65d4218 100644
--- a/roles/matrix-synapse/defaults/main.yml
+++ b/roles/matrix-synapse/defaults/main.yml
@@ -266,18 +266,18 @@ matrix_synapse_workers_enabled_list: []
 
 # Default list of workers to spawn
 matrix_synapse_workers_enabled_list:
-  - { worker: synchrotron, port: 8082 }
-  - { worker: synchrotron, port: 8083 }
-  - { worker: synchrotron, port: 8084 }
-  - { worker: appservice, port: 8085 }
-  - { worker: client_reader, port: 8086 }
-  - { worker: event_creator, port: 8087 }
-  - { worker: federation_reader, port: 8088 }
-  - { worker: federation_sender, port: 8089 }
-  - { worker: frontend_proxy, port: 8090 }
-  - { worker: media_repository, port: 8091 }
-  - { worker: pusher, port: 8092 }
-  - { worker: user_dir, port: 8093 }
+  - { worker: synchrotron, port: 18082 }
+  - { worker: synchrotron, port: 18083 }
+  - { worker: synchrotron, port: 18084 }
+  - { worker: appservice, port: 18085 }
+  - { worker: client_reader, port: 18086 }
+  - { worker: event_creator, port: 18087 }
+  - { worker: federation_reader, port: 18088 }
+  - { worker: federation_sender, port: 18089 }
+  - { worker: frontend_proxy, port: 18090 }
+  - { worker: media_repository, port: 18091 }
+  - { worker: pusher, port: 18092 }
+  - { worker: user_dir, port: 18093 }
 
 # The list of available workers (2020-04-14)
 matrix_synapse_workers_avail_list:

From 765c046bebcd071cde1e24d5852f8235cfe17642 Mon Sep 17 00:00:00 2001
From: Christoph Johannes Kleine <linux@cj-k.de>
Date: Sun, 19 Apr 2020 19:50:42 +0200
Subject: [PATCH 7/8] add missing ; to matrix-synapse.conf.j2

---
 .../nginx/conf.d/matrix-synapse.conf.j2        | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2
index d7a1e6b3a..71605105d 100644
--- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2
+++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2
@@ -131,7 +131,7 @@
 	location ^/_matrix/client/(versions$|(api/v1|r0|unstable)/(publicRooms$|rooms/.*/joined_me|rooms/.*/context/.|rooms/.*/members$|rooms/.*/messages$|rooms/.*/state$|login$|account/3pid$|keys/query$|keys/changes$|voip/turnServer$|joined_groups$|publicised_groups$|publicised_groups/|pushrules/.*$|groups/.*$|register$|auth/.*/fallback/web$)) {
 		{# Use the embedded DNS resolver in Docker containers to discover the service #}
 		resolver 127.0.0.11 valid=5s;
-		set $backend "matrix-synapse:{{ client_reader_worker.port }}"
+		set $backend "matrix-synapse:{{ client_reader_worker.port }}";
 		proxy_pass http://$backend$request_uri;
 		proxy_set_header Host $host;
 		proxy_set_header X-Forwarded-For $remote_addr;
@@ -144,7 +144,7 @@
 	location /_matrix/media/ {
 		{# Use the embedded DNS resolver in Docker containers to discover the service #}
 		resolver 127.0.0.11 valid=5s;
-		set $backend "matrix-synapse:{{ media_repository_worker.port }}"
+		set $backend "matrix-synapse:{{ media_repository_worker.port }}";
 		proxy_pass http://$backend$request_uri;
 		proxy_set_header Host $host;
 		proxy_set_header X-Forwarded-For $remote_addr;
@@ -153,7 +153,7 @@
 	location ~ ^/_synapse/admin/v1/(purge_media_cache|room/.*/media.*|user/.*/media.*|media/.*|quarantine_media/.*)$ {
 		{# Use the embedded DNS resolver in Docker containers to discover the service #}
 		resolver 127.0.0.11 valid=5s;
-		set $backend "matrix-synapse:{{ media_repository_worker.port }}"
+		set $backend "matrix-synapse:{{ media_repository_worker.port }}";
 		proxy_pass http://$backend$request_uri;
 		proxy_set_header Host $host;
 		proxy_set_header X-Forwarded-For $remote_addr;
@@ -166,7 +166,7 @@
 	location ~ ^/_matrix/client/(api/v1|r0|unstable)/(rooms/.*/send|rooms/.*/state/|rooms/.*/(join|invite|leave|ban|unban|kick)$|join/|profile/) {
 		{# Use the embedded DNS resolver in Docker containers to discover the service #}
 		resolver 127.0.0.11 valid=5s;
-		set $backend "matrix-synapse:{{ event_creator_worker.port }}"
+		set $backend "matrix-synapse:{{ event_creator_worker.port }}";
 		proxy_pass http://$backend$request_uri;
 		proxy_set_header Host $host;
 		proxy_set_header X-Forwarded-For $remote_addr;
@@ -179,7 +179,7 @@
 	location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload {
 		{# Use the embedded DNS resolver in Docker containers to discover the service #}
 		resolver 127.0.0.11 valid=5s;
-		set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}"
+		set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}";
 		proxy_pass http://$backend$request_uri;
 		proxy_set_header Host $host;
 		proxy_set_header X-Forwarded-For $remote_addr;
@@ -188,7 +188,7 @@
 		location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status {
 			{# Use the embedded DNS resolver in Docker containers to discover the service #}
 			resolver 127.0.0.11 valid=5s;
-			set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}"
+			set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}";
 			proxy_pass http://$backend$request_uri;
 			proxy_set_header Host $host;
 			proxy_set_header X-Forwarded-For $remote_addr;
@@ -202,7 +202,7 @@
 	location ~ ^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$ {
 		{# Use the embedded DNS resolver in Docker containers to discover the service #}
 		resolver 127.0.0.11 valid=5s;
-		set $backend "matrix-synapse:{{ user_dir_worker.port }}"
+		set $backend "matrix-synapse:{{ user_dir_worker.port }}";
 		proxy_pass http://$backend$request_uri;
 		proxy_set_header Host $host;
 		proxy_set_header X-Forwarded-For $remote_addr;
@@ -292,7 +292,7 @@ upstream synchrotron {
 	{% for synchrotron_worker in synchrotron_workers %}
 		{# Use the embedded DNS resolver in Docker containers to discover the service #}
 		resolver 127.0.0.11 valid=5s;
-		set $backend "matrix-synapse:{{ synchrotron_worker.port }}"
+		set $backend "matrix-synapse:{{ synchrotron_worker.port }}";
 		server $backend:{{ synchrotron_worker.port }};
 	{% endfor %}
 }
@@ -385,7 +385,7 @@ server {
 	location ~ ^(/_matrix/federation/v1/event/|/_matrix/federation/v1/state/|/_matrix/federation/v1/state_ids/|/_matrix/federation/v1/backfill/|/_matrix/federation/v1/get_missing_events/|/_matrix/federation/v1/publicRooms|/_matrix/federation/v1/query/|/_matrix/federation/v1/make_join/|/_matrix/federation/v1/make_leave/|/_matrix/federation/v1/send_join/|/_matrix/federation/v2/send_join/|/_matrix/federation/v1/send_leave/|/_matrix/federation/v2/send_leave/|/_matrix/federation/v1/invite/|/_matrix/federation/v2/invite/|/_matrix/federation/v1/query_auth/|/_matrix/federation/v1/event_auth/|/_matrix/federation/v1/exchange_third_party_invite/|/_matrix/federation/v1/user/devices/|/_matrix/federation/v1/send/|/_matrix/federation/v1/get_groups_publicised$|/_matrix/key/v2/query|/_matrix/federation/v1/groups/) {
 		{# Use the embedded DNS resolver in Docker containers to discover the service #}
 		resolver 127.0.0.11 valid=5s;
-		set $backend "matrix-synapse:{{ federation_reader_worker.port }}"
+		set $backend "matrix-synapse:{{ federation_reader_worker.port }}";
 		proxy_pass http://$backend$request_uri;
 		proxy_set_header Host $host;
 		proxy_set_header X-Forwarded-For $remote_addr;

From 46984a4f99f1b3aece5fbce171feb43bf8e4a726 Mon Sep 17 00:00:00 2001
From: Marcel Partap <mpartap@gmx.net>
Date: Sun, 19 Apr 2020 19:55:43 +0200
Subject: [PATCH 8/8] Nginx conf: more testing less b0rk

---
 .../templates/nginx/conf.d/matrix-synapse.conf.j2            | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2
index 71605105d..68edab1da 100644
--- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2
+++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2
@@ -290,10 +290,7 @@ upstream synchrotron {
 	ip_hash;
 
 	{% for synchrotron_worker in synchrotron_workers %}
-		{# Use the embedded DNS resolver in Docker containers to discover the service #}
-		resolver 127.0.0.11 valid=5s;
-		set $backend "matrix-synapse:{{ synchrotron_worker.port }}";
-		server $backend:{{ synchrotron_worker.port }};
+	server "matrix-synapse:{{ synchrotron_worker.port }}";
 	{% endfor %}
 }
 {% endif %}