From db70230ae16ff8351450b21d007f336003ff873e Mon Sep 17 00:00:00 2001
From: Charles Wright <cvwright@futo.org>
Date: Tue, 16 Jan 2024 09:17:24 -0600
Subject: [PATCH 01/17] Add room-workers as a new preset, with new room
 workers, sync workers, client readers, and federation readers.  Based on
 https://tcpipuk.github.io/synapse/index.html

---
 group_vars/matrix_servers                     |  5 ++
 .../defaults/main.yml                         |  5 ++
 ...ix-synapse-reverse-proxy-companion.conf.j2 | 81 ++++++++++++++-----
 roles/custom/matrix-synapse/defaults/main.yml | 47 +++++++++++
 .../tasks/synapse/workers/init.yml            | 52 ++++++++++++
 roles/custom/matrix-synapse/vars/main.yml     | 19 +++++
 6 files changed, 190 insertions(+), 19 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 3ebe649be..d4f4ea1bc 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -4081,6 +4081,11 @@ matrix_synapse_reverse_proxy_companion_container_labels_internal_client_api_trae
 
 matrix_synapse_reverse_proxy_companion_synapse_workers_enabled: "{{ matrix_synapse_workers_enabled }}"
 matrix_synapse_reverse_proxy_companion_synapse_workers_list: "{{ matrix_synapse_workers_enabled_list }}"
+matrix_synapse_reverse_proxy_companion_synapse_room_worker_client_server_locations: "{{ matrix_synapse_workers_room_worker_client_server_endpoints }}"
+matrix_synapse_reverse_proxy_companion_synapse_room_worker_federation_locations: "{{ matrix_synapse_workers_room_worker_federation_endpoints }}"
+matrix_synapse_reverse_proxy_companion_synapse_sync_worker_client_server_locations: "{{ matrix_synapse_workers_sync_worker_client_server_endpoints }}"
+matrix_synapse_reverse_proxy_companion_synapse_client_reader_client_server_locations: "{{ matrix_synapse_workers_client_reader_client_server_endpoints }}"
+matrix_synapse_reverse_proxy_companion_synapse_federation_reader_federation_locations: "{{ matrix_synapse_workers_federation_reader_federation_endpoints }}"
 matrix_synapse_reverse_proxy_companion_synapse_generic_worker_client_server_locations: "{{ matrix_synapse_workers_generic_worker_client_server_endpoints }}"
 matrix_synapse_reverse_proxy_companion_synapse_generic_worker_federation_locations: "{{ matrix_synapse_workers_generic_worker_federation_endpoints }}"
 matrix_synapse_reverse_proxy_companion_synapse_stream_writer_typing_stream_worker_client_server_locations: "{{ matrix_synapse_workers_stream_writer_typing_stream_worker_client_server_endpoints }}"
diff --git a/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml b/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml
index 7a3f4f709..3f19e3a2b 100644
--- a/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml
+++ b/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml
@@ -206,6 +206,11 @@ matrix_synapse_reverse_proxy_companion_synapse_federation_api_additional_server_
 # synapse worker activation and endpoint mappings
 matrix_synapse_reverse_proxy_companion_synapse_workers_enabled: false
 matrix_synapse_reverse_proxy_companion_synapse_workers_list: []
+matrix_synapse_reverse_proxy_companion_synapse_room_worker_client_server_locations: []
+matrix_synapse_reverse_proxy_companion_synapse_room_worker_federation_locations: []
+matrix_synapse_reverse_proxy_companion_synapse_sync_worker_client_server_locations: []
+matrix_synapse_reverse_proxy_companion_synapse_client_reader_client_server_locations: []
+matrix_synapse_reverse_proxy_companion_synapse_federation_reader_federation_locations: []
 matrix_synapse_reverse_proxy_companion_synapse_generic_worker_client_server_locations: []
 matrix_synapse_reverse_proxy_companion_synapse_generic_worker_federation_locations: []
 matrix_synapse_reverse_proxy_companion_synapse_stream_writer_typing_stream_worker_client_server_locations: []
diff --git a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2 b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
index c94855d59..c3164e963 100644
--- a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
+++ b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
@@ -1,5 +1,9 @@
 #jinja2: lstrip_blocks: "True"
 
+{% set room_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'room_worker') | list %}
+{% set sync_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'sync_worker') | list %}
+{% set client_reader_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'client_reader') | list %}
+{% set federation_reader_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'federation_reader') | list %}
 {% set generic_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'generic_worker') | list %}
 {% set stream_writer_typing_stream_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'stream_writer') | selectattr('stream_writer_stream', 'equalto', 'typing') | list %}
 {% set stream_writer_to_device_stream_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'stream_writer') | selectattr('stream_writer_stream', 'equalto', 'to_device') | list %}
@@ -9,9 +13,31 @@
 {% set media_repository_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'media_repository') | list %}
 {% set user_dir_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'user_dir') | list %}
 
-{% macro render_worker_upstream(name, workers) %}
+
+# Maps from https://tcpipuk.github.io/synapse/deployment/nginx.html#mapsconf
+# Client username from MXID
+map $http_authorization $mxid_localpart {
+  default                           $http_authorization;
+  "~Bearer syt_(?<username>.*?)_.*" $username;
+  ""                                $accesstoken_from_urlparam;
+}
+
+# Whether to upgrade HTTP connection
+map $http_upgrade $connection_upgrade {
+  default upgrade;
+  '' close;
+}
+
+#Extract room name from URI
+map $request_uri $room_name {
+  ~^/_matrix/(client|federation)/.*?(?:%21|!)(?<room>[A-Za-z0-9._=\-\/]+)(?::|%3A)[A-Za-z0-9._=\-\/]+ $room;
+}
+# End maps
+
+{% macro render_worker_upstream(name, workers, load_balance) %}
 {% if workers | length > 0 %}
 	upstream {{ name }} {
+		{{ load_balance }}
 		keepalive {{ workers | length * 2 }};
 		{% for worker in workers %}
 			server "{{ worker.name }}:{{ worker.port }}";
@@ -37,28 +63,22 @@
 	{% endif %}
 	# Round Robin "upstream" pools for workers
 
-	{% if generic_workers |length > 0 %}
-	upstream generic_workers_upstream {
-		# ensures that requests from the same client will always be passed
-		# to the same server (except when this server is unavailable)
-		hash $http_x_forwarded_for;
-		keepalive {{ generic_workers | length * 2 }};
+	{{ render_worker_upstream('room_workers_upstream', room_workers, 'hash $room_name consistent;')
+	{{ render_worker_upstream('sync_workers_upstream', room_workers, 'hash $mxid_localpart consistent;')
+	{{ render_worker_upstream('client_reader_workers_upstream', client_reader_workers, 'least_conn;')
+	{{ render_worker_upstream('federation_reader_workers_upstream', federation_reader_workers, 'hash $http_x_forwarded_for;') }}
 
-		{% for worker in generic_workers %}
-			server "{{ worker.name }}:{{ worker.port }}";
-		{% endfor %}
-	}
-	{% endif %}
+	{{ render_worker_upstream('generic_workers_upstream', generic_workers, 'hash $http_x_forwarded_for;') }}
 
-	{{ render_worker_upstream('stream_writer_typing_stream_workers_upstream', stream_writer_typing_stream_workers) }}
-	{{ render_worker_upstream('stream_writer_to_device_stream_workers_upstream', stream_writer_to_device_stream_workers) }}
-	{{ render_worker_upstream('stream_writer_account_data_stream_workers_upstream', stream_writer_account_data_stream_workers) }}
-	{{ render_worker_upstream('stream_writer_receipts_stream_workers_upstream', stream_writer_receipts_stream_workers) }}
-	{{ render_worker_upstream('stream_writer_presence_stream_workers_upstream', stream_writer_presence_stream_workers) }}
+	{{ render_worker_upstream('stream_writer_typing_stream_workers_upstream', stream_writer_typing_stream_workers, '') }}
+	{{ render_worker_upstream('stream_writer_to_device_stream_workers_upstream', stream_writer_to_device_stream_workers, '') }}
+	{{ render_worker_upstream('stream_writer_account_data_stream_workers_upstream', stream_writer_account_data_stream_workers, '') }}
+	{{ render_worker_upstream('stream_writer_receipts_stream_workers_upstream', stream_writer_receipts_stream_workers, '') }}
+	{{ render_worker_upstream('stream_writer_presence_stream_workers_upstream', stream_writer_presence_stream_workers, '') }}
 
-	{{ render_worker_upstream('media_repository_workers_upstream', media_repository_workers) }}
+	{{ render_worker_upstream('media_repository_workers_upstream', media_repository_workers, 'least_conn;') }}
 
-	{{ render_worker_upstream('user_dir_workers_upstream', user_dir_workers) }}
+	{{ render_worker_upstream('user_dir_workers_upstream', user_dir_workers, '') }}
 {% endif %}
 
 server {
@@ -74,6 +94,21 @@ server {
 	{% if matrix_synapse_reverse_proxy_companion_synapse_workers_enabled %}
 		{# Workers redirects BEGIN #}
 
+		{% if room_workers | length > 0 %}
+			# https://tcpipuk.github.io/synapse/deployment/workers.html
+			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_room_worker_client_server_locations, 'room_workers_upstream') }}
+		{% endif %}
+
+		{% if sync_workers | length > 0 %}
+			# https://tcpipuk.github.io/synapse/deployment/workers.html
+			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_sync_worker_client_server_locations, 'sync_workers_upstream') }}
+		{% endif %}
+
+		{% if client_reader_workers | length > 0 %}
+			# https://tcpipuk.github.io/synapse/deployment/workers.html
+			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_client_reader_client_server_locations, 'client_reader_workers_upstream') }}
+		{% endif %}
+
 		{% if generic_workers | length > 0 %}
 			# https://matrix-org.github.io/synapse/latest/workers.html#synapseappgeneric_worker
 			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_generic_worker_client_server_locations, 'generic_workers_upstream') }}
@@ -165,6 +200,14 @@ server {
 	gzip_types text/plain application/json;
 
 	{% if matrix_synapse_reverse_proxy_companion_synapse_workers_enabled %}
+		{% if room_workers | length > 0 %}
+			# https://tcpipuk.github.io/synapse/deployment/workers.html
+			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_room_worker_federation_locations, 'room_workers_upstream') }}
+		{% endif %}
+		{% if federation_reader_workers | length > 0 %}
+			# https://tcpipuk.github.io/synapse/deployment/workers.html
+			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_federation_reader_federation_locations, 'federation_reader_workers_upstream') }}
+		{% endif %}
 		{% if generic_workers | length > 0 %}
 			# https://matrix-org.github.io/synapse/latest/workers.html#synapseappgeneric_worker
 			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_generic_worker_federation_locations, 'generic_workers_upstream') }}
diff --git a/roles/custom/matrix-synapse/defaults/main.yml b/roles/custom/matrix-synapse/defaults/main.yml
index 4b1513fa5..7dd7ec227 100644
--- a/roles/custom/matrix-synapse/defaults/main.yml
+++ b/roles/custom/matrix-synapse/defaults/main.yml
@@ -666,6 +666,10 @@ matrix_synapse_workers_preset: one-of-each
 
 matrix_synapse_workers_presets:
   little-federation-helper:
+    room_workers_count: 0
+    sync_workers_count: 0
+    client_reader_workers_count: 0
+    federation_reader_workers_count: 0
     generic_workers_count: 0
     pusher_workers_count: 0
     federation_sender_workers_count: 1
@@ -680,6 +684,10 @@ matrix_synapse_workers_presets:
     stream_writer_receipts_stream_workers_count: 0
     stream_writer_presence_stream_workers_count: 0
   one-of-each:
+    room_workers_count: 0
+    sync_workers_count: 0
+    client_reader_workers_count: 0
+    federation_reader_workers_count: 0
     generic_workers_count: 1
     pusher_workers_count: 1
     federation_sender_workers_count: 1
@@ -693,6 +701,24 @@ matrix_synapse_workers_presets:
     stream_writer_account_data_stream_workers_count: 1
     stream_writer_receipts_stream_workers_count: 1
     stream_writer_presence_stream_workers_count: 1
+  room-workers:
+    room_workers_count: 1
+    sync_workers_count: 1
+    client_reader_workers_count: 1
+    federation_reader_workers_count: 1
+    generic_workers_count: 0
+    pusher_workers_count: 1
+    federation_sender_workers_count: 1
+    media_repository_workers_count: 1
+    appservice_workers_count: 1
+    user_dir_workers_count: 1
+    background_workers_count: 1
+    stream_writer_events_stream_workers_count: 1
+    stream_writer_typing_stream_workers_count: 1
+    stream_writer_to_device_stream_workers_count: 1
+    stream_writer_account_data_stream_workers_count: 1
+    stream_writer_receipts_stream_workers_count: 1
+    stream_writer_presence_stream_workers_count: 1
 
 # Controls whether the matrix-synapse container exposes the various worker ports
 # (see `port` and `metrics_port` in `matrix_synapse_workers_enabled_list`) outside of the container.
@@ -733,6 +759,27 @@ matrix_synapse_worker_container_labels_public_metrics_middleware_basic_auth_user
 #   another.label="here"
 matrix_synapse_worker_container_labels_additional_labels: ''
 
+# Room workers
+matrix_synapse_workers_room_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['room_workers_count'] }}"
+matrix_synapse_workers_room_workers_port_range_start: 28111
+matrix_synapse_workers_room_workers_metrics_range_start: 29111
+
+# Sync workers
+matrix_synapse_workers_sync_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['sync_workers_count'] }}"
+matrix_synapse_workers_sync_workers_port_range_start: 28211
+matrix_synapse_workers_sync_workers_metrics_range_start: 29211
+
+# Client reader workers
+matrix_synapse_workers_client_reader_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['client_reader_workers_count'] }}"
+matrix_synapse_workers_client_reader_workers_port_range_start: 28311
+matrix_synapse_workers_client_reader_workers_metrics_range_start: 29311
+
+# Federation reader workers
+matrix_synapse_workers_federation_reader_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['federation_reader_workers_count'] }}"
+matrix_synapse_workers_federation_reader_workers_port_range_start: 28411
+matrix_synapse_workers_federation_reader_workers_metrics_range_start: 29411
+
+# Generic workers
 matrix_synapse_workers_generic_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['generic_workers_count'] }}"
 matrix_synapse_workers_generic_workers_port_range_start: 18111
 matrix_synapse_workers_generic_workers_metrics_range_start: 19111
diff --git a/roles/custom/matrix-synapse/tasks/synapse/workers/init.yml b/roles/custom/matrix-synapse/tasks/synapse/workers/init.yml
index 68b8f37e3..a1bc4cc72 100644
--- a/roles/custom/matrix-synapse/tasks/synapse/workers/init.yml
+++ b/roles/custom/matrix-synapse/tasks/synapse/workers/init.yml
@@ -4,6 +4,58 @@
 # set_fact within a loop does not work reliably in Ansible (it only executes on the first iteration for some reason),
 # so we're forced to do something much uglier.
 
+- name: Build room workers
+  ansible.builtin.set_fact:
+    worker:
+      id: "room-worker-{{ item }}"
+      name: "matrix-synapse-worker-room-{{ item }}"
+      type: 'room_worker'
+      app: 'generic_worker'
+      webserving: true
+      port: "{{ matrix_synapse_workers_room_workers_port_range_start + item }}"
+      metrics_port: "{{ matrix_synapse_workers_room_workers_metrics_range_start + item }}"
+  register: "matrix_synapse_workers_list_results_room_workers"
+  loop: "{{ range(0, matrix_synapse_workers_room_workers_count | int) | list }}"
+
+- name: Build client sync workers
+  ansible.builtin.set_fact:
+    worker:
+      id: "sync-worker-{{ item }}"
+      name: "matrix-synapse-worker-sync-{{ item }}"
+      type: 'sync_worker'
+      app: 'generic_worker'
+      webserving: true
+      port: "{{ matrix_synapse_workers_sync_workers_port_range_start + item }}"
+      metrics_port: "{{ matrix_synapse_workers_sync_workers_metrics_range_start + item }}"
+  register: "matrix_synapse_workers_list_results_sync_workers"
+  loop: "{{ range(0, matrix_synapse_workers_sync_workers_count | int) | list }}"
+
+- name: Build client reader workers
+  ansible.builtin.set_fact:
+    worker:
+      id: "client-reader-{{ item }}"
+      name: "matrix-synapse-worker-client-reader-{{ item }}"
+      type: 'client_reader'
+      app: 'generic_worker'
+      webserving: true
+      port: "{{ matrix_synapse_workers_client_reader_workers_port_range_start + item }}"
+      metrics_port: "{{ matrix_synapse_workers_client_reader_workers_metrics_range_start + item }}"
+  register: "matrix_synapse_workers_list_results_client_reader_workers"
+  loop: "{{ range(0, matrix_synapse_workers_client_reader_workers_count | int) | list }}"
+
+- name: Build federation reader workers
+  ansible.builtin.set_fact:
+    worker:
+      id: "federation-reader-{{ item }}"
+      name: "matrix-synapse-worker-federation-reader-{{ item }}"
+      type: 'federation_reader'
+      app: 'generic_worker'
+      webserving: true
+      port: "{{ matrix_synapse_workers_federation_reader_workers_port_range_start + item }}"
+      metrics_port: "{{ matrix_synapse_workers_federation_reader_workers_metrics_range_start + item }}"
+  register: "matrix_synapse_workers_list_results_federation_reader_workers"
+  loop: "{{ range(0, matrix_synapse_workers_federation_reader_workers_count | int) | list }}"
+
 - name: Build generic workers
   ansible.builtin.set_fact:
     worker:
diff --git a/roles/custom/matrix-synapse/vars/main.yml b/roles/custom/matrix-synapse/vars/main.yml
index 137a59737..63ad48550 100644
--- a/roles/custom/matrix-synapse/vars/main.yml
+++ b/roles/custom/matrix-synapse/vars/main.yml
@@ -8,6 +8,25 @@ matrix_synapse_media_store_directory_name: "{{ matrix_synapse_media_store_path |
 # Optionally: `false` to fully disable tls on outbound smtp
 matrix_synapse_email_smtp_enable_tls: true
 
+# Room workers handle any URL that contains a room id, either through the client-server API or the federation API
+# - see https://tcpipuk.github.io/synapse/deployment/nginx.html#locationsconf
+matrix_synapse_workers_room_worker_client_server_endpoints:
+  - ^/_matrix/client/.*?!(?<room>[A-Za-z0-9._=\-\/]+):[A-Za-z0-9.\-]+
+matrix_synapse_workers_room_worker_federation_endpoints:
+  - ^/_matrix/federation/v[12]/(?:state_ids|get_missing_events)/(?:%21|!)(?<room>[A-Za-z0-9._=\-\/]+)(:|%3A)[A-Za-z0-9.\-]+
+
+# Sync workers handle /sync and the (now deprecated) related endpoints
+matrix_synapse_workers_sync_worker_client_server_endpoints:
+  - ^/_matrix/client/(api/v1|r0|v3|unstable)/(sync|events|initialSync|rooms/[^/]+/initialSync)$
+
+# Client reader workers handle generic client-server endpoints that don't contain a roomid or sync
+matrix_synapse_workers_client_reader_client_server_endpoints:
+  - ^/_matrix/client/(api/v1|r0|v3|unstable)/(room_keys/|keys/(query|changes|claim|upload/|room_keys/)|login|register(/available|/m.login.registration_token/validity|)|password_policy|profile|rooms/.*/(joined_members|context/.*|members|state|hierarchy|relations/|event/|aliases|timestamp_to_event|redact|send|state/|(join|invite|leave|ban|unban|kick))|createRoom|publicRooms|account/(3pid|whoami|devices)|versions|voip/turnServer|joined_rooms|search|user/.*/filter(/|$)|directory/room/.*|capabilities)
+
+# Federation reader workers handle generic federation endpoints that don't contain a roomid
+matrix_synapse_workers_federation_reader_federation_endpoints:
+  - ^/_matrix/(federation/(v1|v2)|key/v2)/
+
 # A Synapse generic worker can handle both federation and client-server API endpoints.
 # We wish to split these, as we normally serve federation separately and don't want them mixed up.
 #

From 0175a472d73556eb25ab0cededf25c5e891ac948 Mon Sep 17 00:00:00 2001
From: Charles Wright <cvwright@futo.org>
Date: Tue, 16 Jan 2024 10:02:36 -0600
Subject: [PATCH 02/17] Typo: forgot closing }}'s

---
 .../conf.d/matrix-synapse-reverse-proxy-companion.conf.j2   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2 b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
index c3164e963..f44823aa8 100644
--- a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
+++ b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
@@ -63,9 +63,9 @@ map $request_uri $room_name {
 	{% endif %}
 	# Round Robin "upstream" pools for workers
 
-	{{ render_worker_upstream('room_workers_upstream', room_workers, 'hash $room_name consistent;')
-	{{ render_worker_upstream('sync_workers_upstream', room_workers, 'hash $mxid_localpart consistent;')
-	{{ render_worker_upstream('client_reader_workers_upstream', client_reader_workers, 'least_conn;')
+	{{ render_worker_upstream('room_workers_upstream', room_workers, 'hash $room_name consistent;') }}
+	{{ render_worker_upstream('sync_workers_upstream', room_workers, 'hash $mxid_localpart consistent;') }}
+	{{ render_worker_upstream('client_reader_workers_upstream', client_reader_workers, 'least_conn;') }}
 	{{ render_worker_upstream('federation_reader_workers_upstream', federation_reader_workers, 'hash $http_x_forwarded_for;') }}
 
 	{{ render_worker_upstream('generic_workers_upstream', generic_workers, 'hash $http_x_forwarded_for;') }}

From 12a8d535e81ad95fef232aa997c1afc5aa7dc624 Mon Sep 17 00:00:00 2001
From: Charles Wright <cvwright@futo.org>
Date: Tue, 16 Jan 2024 10:53:20 -0600
Subject: [PATCH 03/17] Move maps inside the if-workers block; Add Tom's map to
 extract access token from the URI arg

---
 ...ix-synapse-reverse-proxy-companion.conf.j2 | 45 ++++++++++---------
 1 file changed, 24 insertions(+), 21 deletions(-)

diff --git a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2 b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
index f44823aa8..11619f204 100644
--- a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
+++ b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
@@ -13,27 +13,6 @@
 {% set media_repository_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'media_repository') | list %}
 {% set user_dir_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'user_dir') | list %}
 
-
-# Maps from https://tcpipuk.github.io/synapse/deployment/nginx.html#mapsconf
-# Client username from MXID
-map $http_authorization $mxid_localpart {
-  default                           $http_authorization;
-  "~Bearer syt_(?<username>.*?)_.*" $username;
-  ""                                $accesstoken_from_urlparam;
-}
-
-# Whether to upgrade HTTP connection
-map $http_upgrade $connection_upgrade {
-  default upgrade;
-  '' close;
-}
-
-#Extract room name from URI
-map $request_uri $room_name {
-  ~^/_matrix/(client|federation)/.*?(?:%21|!)(?<room>[A-Za-z0-9._=\-\/]+)(?::|%3A)[A-Za-z0-9._=\-\/]+ $room;
-}
-# End maps
-
 {% macro render_worker_upstream(name, workers, load_balance) %}
 {% if workers | length > 0 %}
 	upstream {{ name }} {
@@ -58,6 +37,30 @@ map $request_uri $room_name {
 {% endmacro %}
 
 {% if matrix_synapse_reverse_proxy_companion_synapse_workers_enabled %}
+
+# Maps from https://tcpipuk.github.io/synapse/deployment/nginx.html#mapsconf
+# Client username from access token
+map $arg_access_token $accesstoken_from_urlparam {
+  default                    $arg_access_token;
+  "~syt_(?<username>.*?)_.*" $username;
+}
+# Client username from MXID
+map $http_authorization $mxid_localpart {
+  default                           $http_authorization;
+  "~Bearer syt_(?<username>.*?)_.*" $username;
+  ""                                $accesstoken_from_urlparam;
+}
+# Whether to upgrade HTTP connection
+map $http_upgrade $connection_upgrade {
+  default upgrade;
+  '' close;
+}
+#Extract room name from URI
+map $request_uri $room_name {
+  ~^/_matrix/(client|federation)/.*?(?:%21|!)(?<room>[A-Za-z0-9._=\-\/]+)(?::|%3A)[A-Za-z0-9._=\-\/]+ $room;
+}
+# End maps
+
 	{% if matrix_synapse_reverse_proxy_companion_synapse_cache_enabled %}
     	proxy_cache_path  {{ matrix_synapse_reverse_proxy_companion_synapse_cache_path }} levels=1:2 keys_zone={{ matrix_synapse_reverse_proxy_companion_synapse_cache_keys_zone_name }}:{{ matrix_synapse_reverse_proxy_companion_synapse_cache_keys_zone_size }} inactive={{ matrix_synapse_reverse_proxy_companion_synapse_cache_inactive_time }} max_size={{ matrix_synapse_reverse_proxy_companion_synapse_cache_max_size_mb }}m;
 	{% endif %}

From 5ca9a7269a3ad6aee0c0c654ca348161cfdc2b2c Mon Sep 17 00:00:00 2001
From: Charles Wright <cvwright@futo.org>
Date: Tue, 16 Jan 2024 10:58:46 -0600
Subject: [PATCH 04/17] Add the new worker types to the list of available
 worker types

---
 roles/custom/matrix-synapse/vars/main.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/roles/custom/matrix-synapse/vars/main.yml b/roles/custom/matrix-synapse/vars/main.yml
index 63ad48550..828dcbeab 100644
--- a/roles/custom/matrix-synapse/vars/main.yml
+++ b/roles/custom/matrix-synapse/vars/main.yml
@@ -261,9 +261,13 @@ matrix_synapse_workers_user_dir_endpoints:
 
 matrix_synapse_workers_avail_list:
   - appservice
+  - client_reader
+  - federation_reader
   - federation_sender
   - generic_worker
   - media_repository
   - pusher
+  - room_worker
+  - sync_worker
   - user_dir
 ### workers:end

From 1379200e9dc8a8064d91a3a7ab14baba3e4ec5d3 Mon Sep 17 00:00:00 2001
From: Charles Wright <cvwright@futo.org>
Date: Tue, 16 Jan 2024 11:13:51 -0600
Subject: [PATCH 05/17] Add new worker types to the dynamic workers list

---
 .../custom/matrix-synapse/tasks/synapse/workers/init.yml  | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/roles/custom/matrix-synapse/tasks/synapse/workers/init.yml b/roles/custom/matrix-synapse/tasks/synapse/workers/init.yml
index a1bc4cc72..8bf8201a0 100644
--- a/roles/custom/matrix-synapse/tasks/synapse/workers/init.yml
+++ b/roles/custom/matrix-synapse/tasks/synapse/workers/init.yml
@@ -182,6 +182,14 @@
     matrix_synapse_dynamic_workers_list: "{{ matrix_synapse_dynamic_workers_list | default([]) + [item.ansible_facts.worker] }}"
   with_items: |
     {{
+      matrix_synapse_workers_list_results_room_workers.results
+      +
+      matrix_synapse_workers_list_results_sync_workers.results
+      +
+      matrix_synapse_workers_list_results_client_reader_workers.results
+      +
+      matrix_synapse_workers_list_results_federation_reader_workers.results
+      +
       matrix_synapse_workers_list_results_generic_workers.results
       +
       matrix_synapse_workers_list_results_stream_writer_workers.results

From 124524ea1fa28f3a2c9573ace834d9f1aa190f26 Mon Sep 17 00:00:00 2001
From: Charles Wright <cvwright@futo.org>
Date: Tue, 16 Jan 2024 11:22:46 -0600
Subject: [PATCH 06/17] Typo: Send sync endpoints to sync workers, not room
 workers

---
 .../nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2 b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
index 11619f204..781b2edb3 100644
--- a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
+++ b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
@@ -67,7 +67,7 @@ map $request_uri $room_name {
 	# Round Robin "upstream" pools for workers
 
 	{{ render_worker_upstream('room_workers_upstream', room_workers, 'hash $room_name consistent;') }}
-	{{ render_worker_upstream('sync_workers_upstream', room_workers, 'hash $mxid_localpart consistent;') }}
+	{{ render_worker_upstream('sync_workers_upstream', sync_workers, 'hash $mxid_localpart consistent;') }}
 	{{ render_worker_upstream('client_reader_workers_upstream', client_reader_workers, 'least_conn;') }}
 	{{ render_worker_upstream('federation_reader_workers_upstream', federation_reader_workers, 'hash $http_x_forwarded_for;') }}
 

From a1cbe7f39b43b6016543db02b8d6e102e62f2ff9 Mon Sep 17 00:00:00 2001
From: Charles Wright <cvwright@futo.org>
Date: Tue, 16 Jan 2024 16:32:32 -0600
Subject: [PATCH 07/17] Add overrides for locations that must go to the main
 Synapse process

---
 .../defaults/main.yml                         |  4 +-
 ...ix-synapse-reverse-proxy-companion.conf.j2 | 45 +++++++++++++++++++
 2 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml b/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml
index 3f19e3a2b..4a330e7a8 100644
--- a/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml
+++ b/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml
@@ -220,7 +220,9 @@ matrix_synapse_reverse_proxy_companion_synapse_stream_writer_receipts_stream_wor
 matrix_synapse_reverse_proxy_companion_synapse_stream_writer_presence_stream_worker_client_server_locations: []
 matrix_synapse_reverse_proxy_companion_synapse_media_repository_locations: []
 matrix_synapse_reverse_proxy_companion_synapse_user_dir_locations: []
-
+matrix_synapse_reverse_proxy_companion_client_server_main_override_locations_regex: ^/_matrix/client/(api/v1|r0|v3|unstable)/(account/3pid/|directory/list/room/|pushrules/|rooms/[^/]+/(forget|upgrade)|login/sso/redirect/|register)
+matrix_synapse_reverse_proxy_companion_client_server_sso_override_locations_regex: ^(/_matrix/client/(api/v1|r0|v3|unstable)/login/sso/redirect|/_synapse/client/(pick_username|(new_user_consent|oidc/callback|pick_idp|sso_register)$))
+matrix_synapse_reverse_proxy_companion_federation_override_locations_regex: ^/_matrix/federation/v1/openid/userinfo$
 
 # synapse content caching
 matrix_synapse_reverse_proxy_companion_synapse_cache_enabled: false
diff --git a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2 b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
index 781b2edb3..81c9ced20 100644
--- a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
+++ b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
@@ -95,6 +95,36 @@ server {
 	gzip_types text/plain application/json;
 
 	{% if matrix_synapse_reverse_proxy_companion_synapse_workers_enabled %}
+		# Client-server overrides -- These locations must go to the main Synapse process
+		location ~ {{ matrix_synapse_reverse_proxy_companion_client_server_main_override_locations_regex }} {
+			{# FIXME: This block was copied from the main Synapse fallback below.  It would be better to have it in one place and avoid duplication. #}
+			{# Use the embedded DNS resolver in Docker containers to discover the service #}
+			resolver {{ matrix_synapse_reverse_proxy_companion_http_level_resolver }} valid=5s;
+			set $backend "{{ matrix_synapse_reverse_proxy_companion_client_api_addr }}";
+			proxy_pass http://$backend;
+
+			proxy_set_header Host $host;
+
+			client_body_buffer_size 25M;
+			client_max_body_size {{ matrix_synapse_reverse_proxy_companion_client_api_client_max_body_size_mb }}M;
+			proxy_max_temp_file_size 0;
+		}
+
+		# Client-server SSO overrides -- These locations must go to the main Synapse process
+		location ~ {{ matrix_synapse_reverse_proxy_companion_client_server_sso_override_locations_regex }} {
+			{# FIXME: This block was copied from the main Synapse fallback below.  It would be better to have it in one place and avoid duplication. #}
+			{# Use the embedded DNS resolver in Docker containers to discover the service #}
+			resolver {{ matrix_synapse_reverse_proxy_companion_http_level_resolver }} valid=5s;
+			set $backend "{{ matrix_synapse_reverse_proxy_companion_client_api_addr }}";
+			proxy_pass http://$backend;
+
+			proxy_set_header Host $host;
+
+			client_body_buffer_size 25M;
+			client_max_body_size {{ matrix_synapse_reverse_proxy_companion_client_api_client_max_body_size_mb }}M;
+			proxy_max_temp_file_size 0;
+		}
+
 		{# Workers redirects BEGIN #}
 
 		{% if room_workers | length > 0 %}
@@ -203,6 +233,21 @@ server {
 	gzip_types text/plain application/json;
 
 	{% if matrix_synapse_reverse_proxy_companion_synapse_workers_enabled %}
+		# Federation overrides -- These locations must go to the main Synapse process
+		location ~ {{ matrix_synapse_reverse_proxy_companion_federation_override_locations_regex }} {
+			{# FIXME: This block was copied from the fallback location below.  It would be better to have it in one place and avoid duplication. #}
+			{# Use the embedded DNS resolver in Docker containers to discover the service #}
+			resolver {{ matrix_synapse_reverse_proxy_companion_http_level_resolver }} valid=5s;
+			set $backend "{{ matrix_synapse_reverse_proxy_companion_federation_api_addr }}";
+			proxy_pass http://$backend;
+
+			proxy_set_header Host $host;
+
+			client_body_buffer_size 25M;
+			client_max_body_size {{ matrix_synapse_reverse_proxy_companion_federation_api_client_max_body_size_mb }}M;
+			proxy_max_temp_file_size 0;
+		}
+
 		{% if room_workers | length > 0 %}
 			# https://tcpipuk.github.io/synapse/deployment/workers.html
 			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_room_worker_federation_locations, 'room_workers_upstream') }}

From 0dbdaf5b9f2e339684e8c916787cf623b20a2cab Mon Sep 17 00:00:00 2001
From: Charles Wright <cvwright@futo.org>
Date: Tue, 16 Jan 2024 16:51:23 -0600
Subject: [PATCH 08/17] Enable HTTP resources for new worker types

---
 .../matrix-synapse/templates/synapse/worker.yaml.j2 | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/roles/custom/matrix-synapse/templates/synapse/worker.yaml.j2 b/roles/custom/matrix-synapse/templates/synapse/worker.yaml.j2
index 18b96a55f..738f0fa33 100644
--- a/roles/custom/matrix-synapse/templates/synapse/worker.yaml.j2
+++ b/roles/custom/matrix-synapse/templates/synapse/worker.yaml.j2
@@ -7,10 +7,17 @@ worker_log_config: /data/{{ matrix_server_fqn_matrix }}.log.config
 
 {% set http_resources = [] %}
 
-{% if matrix_synapse_worker_details.type == 'user_dir' %}
+{% if matrix_synapse_worker_details.type == 'room_worker' %}
+  {% set http_resources = http_resources + ['client', 'federation'] %}
+{% elif matrix_synapse_worker_details.type == 'sync_worker' %}
   {% set http_resources = http_resources + ['client'] %}
-{% endif %}
-{% if matrix_synapse_worker_details.type == 'generic_worker' %}
+{% elif matrix_synapse_worker_details.type == 'client_reader' %}
+  {% set http_resources = http_resources + ['client'] %}
+{% elif matrix_synapse_worker_details.type == 'federation_reader' %}
+  {% set http_resources = http_resources + ['federation'] %}
+{% elif matrix_synapse_worker_details.type == 'user_dir' %}
+  {% set http_resources = http_resources + ['client'] %}
+{% elif matrix_synapse_worker_details.type == 'generic_worker' %}
   {% set http_resources = http_resources + ['client', 'federation'] %}
 {% endif %}
 {#

From 55604f73c5d0726e006a936af8d302bdcafa0cff Mon Sep 17 00:00:00 2001
From: Charles Wright <cvwright@futo.org>
Date: Tue, 16 Jan 2024 17:24:13 -0600
Subject: [PATCH 09/17] Bugfix: Locations for new workers must go *after* the
 stream writers

---
 ...ix-synapse-reverse-proxy-companion.conf.j2 | 30 +++++++++----------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2 b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
index 81c9ced20..dcb6ebeea 100644
--- a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
+++ b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
@@ -127,21 +127,6 @@ server {
 
 		{# Workers redirects BEGIN #}
 
-		{% if room_workers | length > 0 %}
-			# https://tcpipuk.github.io/synapse/deployment/workers.html
-			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_room_worker_client_server_locations, 'room_workers_upstream') }}
-		{% endif %}
-
-		{% if sync_workers | length > 0 %}
-			# https://tcpipuk.github.io/synapse/deployment/workers.html
-			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_sync_worker_client_server_locations, 'sync_workers_upstream') }}
-		{% endif %}
-
-		{% if client_reader_workers | length > 0 %}
-			# https://tcpipuk.github.io/synapse/deployment/workers.html
-			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_client_reader_client_server_locations, 'client_reader_workers_upstream') }}
-		{% endif %}
-
 		{% if generic_workers | length > 0 %}
 			# https://matrix-org.github.io/synapse/latest/workers.html#synapseappgeneric_worker
 			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_generic_worker_client_server_locations, 'generic_workers_upstream') }}
@@ -172,6 +157,21 @@ server {
 			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_stream_writer_presence_stream_worker_client_server_locations, 'stream_writer_presence_stream_workers_upstream') }}
 		{% endif %}
 
+		{% if room_workers | length > 0 %}
+			# https://tcpipuk.github.io/synapse/deployment/workers.html
+			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_room_worker_client_server_locations, 'room_workers_upstream') }}
+		{% endif %}
+
+		{% if sync_workers | length > 0 %}
+			# https://tcpipuk.github.io/synapse/deployment/workers.html
+			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_sync_worker_client_server_locations, 'sync_workers_upstream') }}
+		{% endif %}
+
+		{% if client_reader_workers | length > 0 %}
+			# https://tcpipuk.github.io/synapse/deployment/workers.html
+			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_client_reader_client_server_locations, 'client_reader_workers_upstream') }}
+		{% endif %}
+
 		{% if media_repository_workers | length > 0 %}
 			# https://matrix-org.github.io/synapse/latest/workers.html#synapseappmedia_repository
 			{% for location in matrix_synapse_reverse_proxy_companion_synapse_media_repository_locations %}

From 7d9eb561644811c912d9e074d4b7f70198060f10 Mon Sep 17 00:00:00 2001
From: Charles Wright <cvwright@futo.org>
Date: Wed, 17 Jan 2024 15:22:27 -0600
Subject: [PATCH 10/17] Add a validation step to fail when the user allocates
 generic workers together with all of the new worker types

---
 roles/custom/matrix-synapse/tasks/validate_config.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/roles/custom/matrix-synapse/tasks/validate_config.yml b/roles/custom/matrix-synapse/tasks/validate_config.yml
index ab389a432..fdf208e8d 100644
--- a/roles/custom/matrix-synapse/tasks/validate_config.yml
+++ b/roles/custom/matrix-synapse/tasks/validate_config.yml
@@ -47,6 +47,12 @@
     - "matrix_synapse_workers_stream_writer_receipts_stream_workers_count"
     - "matrix_synapse_workers_stream_writer_presence_stream_workers_count"
 
+- name: Fail when mixing generic workers with new specialized workers
+  ansible.builtin.fail:
+    msg: >-
+      Generic workers should not be mixed with the new specialized worker types (room workers, sync workers, client readers, and federation readers)
+  when: matrix_synapse_workers_generic_workers_count | int > 0 and matrix_synapse_workers_room_workers_count | int > 0 and matrix_synapse_workers_sync_workers_count | int > 0 and matrix_synapse_workers_client_reader_workers_count | int > 0 and matrix_synapse_workers_federation_reader_workers_count | int > 0
+
 - name: (Deprecation) Catch and report renamed settings
   ansible.builtin.fail:
     msg: >-

From 3c7f89624679e2d59a524a9070ba82915cc2f3d5 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sat, 20 Jan 2024 11:10:34 +0200
Subject: [PATCH 11/17] Prevent generic workers being combined with any of the
 other types

Until now, the validation check would only get tripped up
if generic workers are used, combined with at least one EACH
other type of specialized workers.

This means that someone doing this:

```
matrix_synapse_workers_preset: one-of-each
matrix_synapse_workers_client_reader_workers_count: 5
```

.. would not have triggered this safety check.

Related to https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/3100
---
 roles/custom/matrix-synapse/tasks/validate_config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/custom/matrix-synapse/tasks/validate_config.yml b/roles/custom/matrix-synapse/tasks/validate_config.yml
index fdf208e8d..2f1a5e1c0 100644
--- a/roles/custom/matrix-synapse/tasks/validate_config.yml
+++ b/roles/custom/matrix-synapse/tasks/validate_config.yml
@@ -51,7 +51,7 @@
   ansible.builtin.fail:
     msg: >-
       Generic workers should not be mixed with the new specialized worker types (room workers, sync workers, client readers, and federation readers)
-  when: matrix_synapse_workers_generic_workers_count | int > 0 and matrix_synapse_workers_room_workers_count | int > 0 and matrix_synapse_workers_sync_workers_count | int > 0 and matrix_synapse_workers_client_reader_workers_count | int > 0 and matrix_synapse_workers_federation_reader_workers_count | int > 0
+  when: matrix_synapse_workers_generic_workers_count | int > 0 and ((matrix_synapse_workers_room_workers_count | int + matrix_synapse_workers_sync_workers_count | int + matrix_synapse_workers_client_reader_workers_count | int + matrix_synapse_workers_federation_reader_workers_count | int) > 0)
 
 - name: (Deprecation) Catch and report renamed settings
   ansible.builtin.fail:

From 16ca50c6ef4f37759ff7c1fce040958158e9e011 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sat, 20 Jan 2024 11:24:59 +0200
Subject: [PATCH 12/17] Add a few more comments in
 matrix-synapse-reverse-proxy-companion.conf.j2

Related to https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/3100
---
 .../conf.d/matrix-synapse-reverse-proxy-companion.conf.j2   | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2 b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
index b3b8165e8..0dac97120 100644
--- a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
+++ b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
@@ -158,17 +158,23 @@ server {
 		{% endif %}
 
 		{% if room_workers | length > 0 %}
+			# room workers
 			# https://tcpipuk.github.io/synapse/deployment/workers.html
+			# https://tcpipuk.github.io/synapse/deployment/nginx.html#locationsconf
 			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_room_worker_client_server_locations, 'room_workers_upstream') }}
 		{% endif %}
 
 		{% if sync_workers | length > 0 %}
+			# sync workers
 			# https://tcpipuk.github.io/synapse/deployment/workers.html
+			# https://tcpipuk.github.io/synapse/deployment/nginx.html#locationsconf
 			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_sync_worker_client_server_locations, 'sync_workers_upstream') }}
 		{% endif %}
 
 		{% if client_reader_workers | length > 0 %}
+			# client_reader workers
 			# https://tcpipuk.github.io/synapse/deployment/workers.html
+			# https://tcpipuk.github.io/synapse/deployment/nginx.html#locationsconf
 			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_client_reader_client_server_locations, 'client_reader_workers_upstream') }}
 		{% endif %}
 

From 7cb33da46ad4645709f8809506f99b8b03aaa232 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sat, 20 Jan 2024 11:35:20 +0200
Subject: [PATCH 13/17] Add some clarification comment in
 matrix-synapse-reverse-proxy-companion/defaults/main.yml

---
 .../matrix-synapse-reverse-proxy-companion/defaults/main.yml   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml b/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml
index cebb259eb..f480f2891 100644
--- a/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml
+++ b/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml
@@ -192,7 +192,8 @@ matrix_synapse_reverse_proxy_companion_synapse_client_api_additional_server_conf
 matrix_synapse_reverse_proxy_companion_synapse_federation_api_additional_server_configuration_blocks: []
 
 
-# synapse worker activation and endpoint mappings
+# synapse worker activation and endpoint mappings.
+# These are all populated via Ansible group variables.
 matrix_synapse_reverse_proxy_companion_synapse_workers_enabled: false
 matrix_synapse_reverse_proxy_companion_synapse_workers_list: []
 matrix_synapse_reverse_proxy_companion_synapse_room_worker_client_server_locations: []

From 84446e52e983f9cf1a1441f980d55773406b0a8c Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sat, 20 Jan 2024 12:39:20 +0200
Subject: [PATCH 14/17] Rename Synapse worker preset name (room-workers ->
 specialized-workers)

I believe `specialized-workers` is a better name than `room-workers`,
because when enabled, 4 different types of specialized workers are
created:

- Room workers
- Sync workers
- Client readers
- Federation readers

Only one of these is called room-workers.

In the future, more specialized workers may be added, making the
`room-workers` preset name an even poorer choice.

Related to https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/3100
---
 roles/custom/matrix-synapse/defaults/main.yml |  2 +-
 roles/custom/matrix-synapse/tasks/init.yml    | 11 +++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/roles/custom/matrix-synapse/defaults/main.yml b/roles/custom/matrix-synapse/defaults/main.yml
index 84bc9ff8a..ac382622b 100644
--- a/roles/custom/matrix-synapse/defaults/main.yml
+++ b/roles/custom/matrix-synapse/defaults/main.yml
@@ -706,7 +706,7 @@ matrix_synapse_workers_presets:
     stream_writer_account_data_stream_workers_count: 1
     stream_writer_receipts_stream_workers_count: 1
     stream_writer_presence_stream_workers_count: 1
-  room-workers:
+  specialized-workers:
     room_workers_count: 1
     sync_workers_count: 1
     client_reader_workers_count: 1
diff --git a/roles/custom/matrix-synapse/tasks/init.yml b/roles/custom/matrix-synapse/tasks/init.yml
index 234eb78e5..341350a49 100644
--- a/roles/custom/matrix-synapse/tasks/init.yml
+++ b/roles/custom/matrix-synapse/tasks/init.yml
@@ -1,5 +1,16 @@
 ---
 
+# This validation task is here, not in validate_config.yml,
+# because init.yml runs before it.
+#
+# validate_config.yml aims to validate the configuration based on the work we do,
+# so we can't change the order.
+- name: Fail when using the old (renamed) room-workers preset
+  ansible.builtin.fail:
+    msg: >-
+      The `room-workers` preset has been renamed to `specialized-workers`. Update your `matrix_synapse_workers_preset` variable to use the new name.
+  when: matrix_synapse_workers_preset == 'room-workers'
+
 # Unless `matrix_synapse_workers_enabled_list` is explicitly defined,
 # we'll generate it dynamically.
 - ansible.builtin.include_tasks: "{{ role_path }}/tasks/synapse/workers/init.yml"

From 9fb2d53b542820165295b526fd6bfe257b0afdd5 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sat, 20 Jan 2024 12:41:21 +0200
Subject: [PATCH 15/17] Rework Synapse workers documentation

Related to: https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/3100
---
 docs/configuring-playbook-synapse.md          | 53 +++++++++++++++++--
 roles/custom/matrix-synapse/defaults/main.yml |  3 +-
 2 files changed, 50 insertions(+), 6 deletions(-)

diff --git a/docs/configuring-playbook-synapse.md b/docs/configuring-playbook-synapse.md
index e9a3c3520..e99a56e7f 100644
--- a/docs/configuring-playbook-synapse.md
+++ b/docs/configuring-playbook-synapse.md
@@ -20,22 +20,65 @@ Alternatively, **if there is no pre-defined variable** for a Synapse setting you
 
 ## Load balancing with workers
 
-To have Synapse gracefully handle thousands of users, worker support should be enabled. It factors out some homeserver tasks and spreads the load of incoming client and server-to-server traffic between multiple processes. More information can be found in the [official Synapse workers documentation](https://github.com/element-hq/synapse/blob/master/docs/workers.md).
+To have Synapse gracefully handle thousands of users, worker support should be enabled. It factors out some homeserver tasks and spreads the load of incoming client and server-to-server traffic between multiple processes. More information can be found in the [official Synapse workers documentation](https://github.com/element-hq/synapse/blob/master/docs/workers.md) and [Tom Foster](https://github.com/tcpipuk)'s [Synapse homeserver guide](https://tcpipuk.github.io/synapse/index.html).
 
 To enable Synapse worker support, update your `inventory/host_vars/matrix.DOMAIN/vars.yml` file:
 
 ```yaml
 matrix_synapse_workers_enabled: true
+
+matrix_synapse_workers_preset: one-of-each
 ```
 
-We support a few configuration presets (`matrix_synapse_workers_preset: one-of-each` being the default configuration):
-- `little-federation-helper` - a very minimal worker configuration to improve federation performance
-- `one-of-each` - one worker of each supported type
+By default, this enables the `one-of-each` [worker preset](#worker-presets), but you may wish to use another preset or [control the number of worker instances](#controlling-the-number-of-worker-instances).
 
-If you'd like more customization power, you can start with one of the presets and tweak various `matrix_synapse_workers_*_count` variables manually.
+### Worker presets
+
+We support a few configuration presets (`matrix_synapse_workers_preset: one-of-each` being the default configuration right now):
+
+- (federation-only) `little-federation-helper` - a very minimal worker configuration to improve federation performance
+- (generic) `one-of-each` - defaults to one worker of each supported type - no smart routing, just generic workers
+- (specialized) `specialized-workers` - defaults to one worker of each supported type, but disables generic workers and uses [specialized workers](#specialized-workers) instead
+
+These presets represent a few common configurations. There are many worker types which can be mixed and matched based on your needs.
+
+#### Generic workers
+
+Previously, the playbook only supported the most basic type of load-balancing. We call it **generic load-balancing** below, because incoming HTTP requests are sent to a generic worker. Load-balancing was done based on the requestor's IP address. This is simple, but not necessarily optimal. If you're accessing your account from multiple IP addresses (e.g. your mobile phone being on a different network than your PC), these separate requests may potentially be routed to different workers, each of which would need to cache roughly the same data.
+
+This is **still the default load-balancing method (preset) used by the playbook**.
+
+To use generic load-balancing, do not specify `matrix_synapse_workers_preset` to make it use the default value (`one-of-each`), or better yet - explicitly set it as `one-of-each`.
+
+You may also consider [tweaking the number of workers of each type](#controlling-the-number-of-worker-instances) from the default (one of each).
+
+#### Specialized workers
+
+The playbook now supports a smarter **specialized load-balancing** inspired by [Tom Foster](https://github.com/tcpipuk)'s [Synapse homeserver guide](https://tcpipuk.github.io/synapse/index.html). Instead of routing requests to one or more [generic workers](#generic-workers) based only on the requestor's IP adddress, specialized load-balancing routes to **4 different types of specialized workers** based on **smarter criteria** - the access token (username) of the requestor and/or on the resource (room, etc.) being requested.
+
+The playbook supports these **4 types** of specialized workers:
+
+- Room workers - handles various [Client-Server](https://spec.matrix.org/v1.9/client-server-api/) & [Federation](https://spec.matrix.org/v1.9/server-server-api) APIs dedicated to handling specific rooms
+- Sync workers - handles various [Client-Server](https://spec.matrix.org/v1.9/client-server-api/) APIs related to synchronization (most notably [the `/sync` endpoint](https://spec.matrix.org/v1.9/client-server-api/#get_matrixclientv3sync))
+- Client readers - handles various [Client-Server](https://spec.matrix.org/v1.9/client-server-api/) APIs which are not for specific rooms (handled by **room workers**) or for synchronization (handled by **sync workers**)
+- Federation readers - handles various [Federation](https://spec.matrix.org/v1.9/server-server-api) APIs which are not for specific rooms (handled by **room workers**)
+
+To use specialized load-balancing, consider enabling the `specialized-workers` [worker preset](#worker-presets) and potentially [tweaking the number of workers of each type](#controlling-the-number-of-worker-instances) from the default (one of each).
+
+#### Controlling the number of worker instances
+
+If you'd like more customization power, you can start with one of the [worker presets](#worker-presets) and then tweak various `matrix_synapse_workers_*_count` variables manually.
+
+To find what variables are available for you to override in your own `vars.yml` configuration file, see the [`defaults/main.yml` file for the `matrix-synapse` Ansible role](../roles/custom/matrix-synapse/defaults/main.yml).
+
+The only thing you **cannot** do is mix [generic workers](#generic-workers) and [specialized workers](#specialized-workers).
+
+#### Effect of enabling workers on the rest of your server
 
 When Synapse workers are enabled, the integrated [Postgres database is tuned](maintenance-postgres.md#tuning-postgresql), so that the maximum number of Postgres connections are increased from `200` to `500`. If you need to decrease or increase the number of maximum Postgres connections further, use the `devture_postgres_max_connections` variable.
 
+A separate Ansible role (`matrix-synapse-reverse-proxy-companion`) and component handles load-balancing for workers. This role/component is automatically enabled when you enable workers. Make sure to use the `setup-all` tag (not `install-all`!) during the playbook's [installation](./installing.md) process, especially if you're disabling workers, so that components may be installed/uninstalled correctly.
+
 In case any problems occur, make sure to have a look at the [list of synapse issues about workers](https://github.com/matrix-org/synapse/issues?q=workers+in%3Atitle) and your `journalctl --unit 'matrix-*'`.
 
 
diff --git a/roles/custom/matrix-synapse/defaults/main.yml b/roles/custom/matrix-synapse/defaults/main.yml
index ac382622b..dd4ac05a2 100644
--- a/roles/custom/matrix-synapse/defaults/main.yml
+++ b/roles/custom/matrix-synapse/defaults/main.yml
@@ -663,7 +663,8 @@ matrix_synapse_workers_enabled: false
 #
 # The posible values (as seen in `matrix_synapse_workers_presets`) are:
 # - "little-federation-helper" - a very minimal worker configuration to improve federation performance
-# - "one-of-each" - one worker of each supported type
+# - "one-of-each" - one worker of each supported type + a generic worker
+# - "specialized-workers" - one worker of each supported type + specialized workers
 #
 # You can override `matrix_synapse_workers_presets` to define your own presets, which is ill-advised, because it's fragile.
 # To use a more custom configuration, start with one of these presets as a base and configure `matrix_synapse_workers_*_count` variables manually, to suit your liking.

From 24394d3ec42b5381e2fcd0f671b88cd7cc35beca Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sat, 20 Jan 2024 12:41:46 +0200
Subject: [PATCH 16/17] Announce support for specialized Synapse workers

Related to https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/3100
---
 CHANGELOG.md | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7bb44f665..aecd10d5b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,16 @@
+# 2024-01-20
+
+## Support for more efficient (specialized) Synapse workers
+
+Thanks to [Charles Wright](https://github.com/cvwright) from [FUTO](https://www.futo.org/), the creators of the [Circles app](https://circu.li/), the playbook has [received support](https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/3100) for load-balancing the Synapse workload via [specialized workers](./docs/configuring-playbook-synapse.md#specialized-workers) which are supposed to work better than our old [generic workers]((./docs/configuring-playbook-synapse.md#generic-workers)) implementation.
+
+For now, playbook defaults remain unchanged and the `one-of-each` [workers preset](./docs/configuring-playbook-synapse.md#worker-presets) continues being the default. However, the default may change in the future. If you'd like to remain on this preset even if/when the defaults change, consider explicitly adding `matrix_synapse_workers_preset: one-of-each` to your `vars.yml` configuration.
+
+Our specialized workers setup is based on recommendations found in [Tom Foster](https://github.com/tcpipuk)'s [Synapse homeserver guide](https://tcpipuk.github.io/synapse/index.html). What's special about our new setup is that we try to parse information out of the request (who the user is; which room is being operated on) and try to forward similar requests to the same worker. As an example, this means that once a worker caches some room information, subsequent requests for the same room will be routed to the same worker (which supposedly still has the room's state cached).
+
+To get started, refer to our [Specialized workers](./docs/configuring-playbook-synapse.md#specialized-workers) documentation section.
+
+
 # 2024-01-17
 
 ## Switching to Element's AGPLv3-licensed Synapse release

From 9a7cb0f716e0ea73cf956ca84e736ab37b6e29bf Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sat, 20 Jan 2024 12:45:10 +0200
Subject: [PATCH 17/17] Fix broken link in changelog entry

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index aecd10d5b..0e8ece22b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,7 +2,7 @@
 
 ## Support for more efficient (specialized) Synapse workers
 
-Thanks to [Charles Wright](https://github.com/cvwright) from [FUTO](https://www.futo.org/), the creators of the [Circles app](https://circu.li/), the playbook has [received support](https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/3100) for load-balancing the Synapse workload via [specialized workers](./docs/configuring-playbook-synapse.md#specialized-workers) which are supposed to work better than our old [generic workers]((./docs/configuring-playbook-synapse.md#generic-workers)) implementation.
+Thanks to [Charles Wright](https://github.com/cvwright) from [FUTO](https://www.futo.org/), the creators of the [Circles app](https://circu.li/), the playbook has [received support](https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/3100) for load-balancing the Synapse workload via [specialized workers](./docs/configuring-playbook-synapse.md#specialized-workers) which are supposed to work better than our old [generic workers](./docs/configuring-playbook-synapse.md#generic-workers) implementation.
 
 For now, playbook defaults remain unchanged and the `one-of-each` [workers preset](./docs/configuring-playbook-synapse.md#worker-presets) continues being the default. However, the default may change in the future. If you'd like to remain on this preset even if/when the defaults change, consider explicitly adding `matrix_synapse_workers_preset: one-of-each` to your `vars.yml` configuration.