Add Synapse worker presets for easier configuration

Adding more presets in the future would be nice.

docs/configuring-playbook-synapse.md

@@ -19,18 +19,30 @@ Alternatively, **if there is no pre-defined variable** for a Synapse setting you
 
 
 ## Load balancing with workers
-To have synapse gracefully handle thousands of users, worker support should be enabled. It factors out some homeserver tasks and spreads the load of incoming client and server-to-server traffic between multiple processes. More information can be found at https://github.com/matrix-org/synapse/blob/master/docs/workers.md (which, coincidentally, also is the file which an awk script extracts the endpoint URLs from when running with tag `setup-synapse`).
 
-To enable synapse worker support, set
+To have Synapse gracefully handle thousands of users, worker support should be enabled. It factors out some homeserver tasks and spreads the load of incoming client and server-to-server traffic between multiple processes. More information can be found in the [official Synapse workers documentation](https://github.com/matrix-org/synapse/blob/master/docs/workers.md).
+
+To enable Synapse worker support, update your `inventory/host_vars/matrix.DOMAIN/vars.yml` file:
 
 ```yaml
 matrix_synapse_workers_enabled: true
 ```
 
-in your `inventory/host_vars/matrix.DOMAIN/vars.yml` file.
+We support a few configuration presets (`matrix_synapse_workers_preset: one-of-each` being the default configuration):
-There, you can also override the default `matrix_synapse_workers_enabled_list` from [`roles/matrix-synapse/defaults/main.yml`](../roles/matrix-synapse/defaults/main.yml).
+- `little-federation-helper` - a very minimal worker configuration to improve federation performance
+- `one-of-each` - one worker of each supported type
 
-If you are not using the inbuilt nginx proxy container but an instance managed by yourself, you are currently on your own as the template needs yet to be adapted to better support this use case.
+If you'd like more customization power, you can start with one of the presets and tweak various `matrix_synapse_workers_*_count` variables manually.
+
+If you increase worker counts too much, you may need to increase the maximum number of Postgres connections too (example):
+
+```yaml
+matrix_postgres_process_extra_arguments: [
+  "-c 'max_connections=200'"
+]
+```
+
+If you're using the default setup (the `matrix-nginx-proxy` webserver being enabled) or you're using your own `nginx` server (which imports the configuration files generated by the playbook), you're good to go. If you use some other webserver, you may need to tweak your reverse-proxy setup manually to forward traffic to the various workers.
 
 In case any problems occur, make sure to have a look at the [list of synapse issues about workers](https://github.com/matrix-org/synapse/issues?q=workers+in%3Atitle) and your `journalctl --unit 'matrix-*'`.
 

roles/matrix-synapse/defaults/main.yml

@@ -301,6 +301,35 @@ matrix_synapse_manhole_enabled: false
 # Enable support for Synapse workers
 matrix_synapse_workers_enabled: false
 
+# Specifies worker configuration that should be used when workers are enabled.
+#
+# The posible values (as seen in `matrix_synapse_workers_presets`) are:
+# - "little-federation-helper" - a very minimal worker configuration to improve federation performance
+# - "one-of-each" - one worker of each supported type
+#
+# You can override `matrix_synapse_workers_presets` to define your own presets, which is ill-advised, because it's fragile.
+# To use a more custom configuration, start with one of these presets as a base and configure `matrix_synapse_workers_*_count` variables manually, to suit your liking.
+matrix_synapse_workers_preset: one-of-each
+
+matrix_synapse_workers_presets:
+  little-federation-helper:
+    generic_workers_count: 0
+    pusher_workers_count: 0
+    appservice_workers_count: 0
+    federation_sender_workers_count: 1
+    media_repository_workers_count: 0
+    user_dir_workers_count: 0
+    frontend_proxy_workers_count: 0
+  one-of-each:
+    generic_workers_count: 1
+    pusher_workers_count: 1
+    appservice_workers_count: 1
+    federation_sender_workers_count: 1
+    media_repository_workers_count: 1
+    # Disabled until https://github.com/matrix-org/synapse/issues/8787 is resolved.
+    user_dir_workers_count: 0
+    frontend_proxy_workers_count: 1
+
 # Controls whether the matrix-synapse container exposes the various worker ports
 # (see `port` and `metrics_port` in `matrix_synapse_workers_enabled_list`) outside of the container.
 #
@@ -308,36 +337,36 @@ matrix_synapse_workers_enabled: false
 # It takes "*" to signify "bind on all interfaces" ("0.0.0.0" is IPv4-only).
 matrix_synapse_workers_container_host_bind_address: ''
 
-matrix_synapse_workers_generic_workers_count: 3
+matrix_synapse_workers_generic_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['generic_workers_count'] }}"
 matrix_synapse_workers_generic_workers_port_range_start: 18111
 matrix_synapse_workers_generic_workers_metrics_range_start: 19111
 
 # matrix_synapse_workers_pusher_workers_count can only be 0 or 1 for now.
 # More instances are not supported due to a playbook limitation having to do with keeping `pusher_instances` in `homeserver.yaml` updated.
 # See https://github.com/matrix-org/synapse/commit/ddfdf945064925eba761ae3748e38f3a1c73c328
-matrix_synapse_workers_pusher_workers_count: 1
+matrix_synapse_workers_pusher_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['pusher_workers_count'] }}"
 matrix_synapse_workers_pusher_workers_metrics_range_start: 19200
 
 # matrix_synapse_workers_appservice_workers_count can only be 0 or 1. More instances are not supported.
-matrix_synapse_workers_appservice_workers_count: 1
+matrix_synapse_workers_appservice_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['appservice_workers_count'] }}"
 matrix_synapse_workers_appservice_workers_metrics_range_start: 19300
 
 # matrix_synapse_workers_federation_sender_workers_count can only be 0 or 1 for now.
 # More instances are not supported due to a playbook limitation having to do with keeping `federation_sender_instances` in `homeserver.yaml` updated.
 # See https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappfederation_sender
-matrix_synapse_workers_federation_sender_workers_count: 1
+matrix_synapse_workers_federation_sender_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['federation_sender_workers_count'] }}"
 matrix_synapse_workers_federation_sender_workers_metrics_range_start: 19400
 
-matrix_synapse_workers_media_repository_workers_count: 1
+matrix_synapse_workers_media_repository_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['media_repository_workers_count'] }}"
 matrix_synapse_workers_media_repository_workers_port_range_start: 18551
 matrix_synapse_workers_media_repository_workers_metrics_range_start: 19551
 
 # Disabled until https://github.com/matrix-org/synapse/issues/8787 is resolved.
-matrix_synapse_workers_user_dir_workers_count: 0
+matrix_synapse_workers_user_dir_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['user_dir_workers_count'] }}"
 matrix_synapse_workers_user_dir_workers_port_range_start: 18661
 matrix_synapse_workers_user_dir_workers_metrics_range_start: 19661
 
-matrix_synapse_workers_frontend_proxy_workers_count: 1
+matrix_synapse_workers_frontend_proxy_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['frontend_proxy_workers_count'] }}"
 matrix_synapse_workers_frontend_proxy_workers_port_range_start: 18771
 matrix_synapse_workers_frontend_proxy_workers_metrics_range_start: 19771
 

roles/matrix-synapse/tasks/synapse/workers/init.yml

@@ -11,7 +11,7 @@
       port: "{{ matrix_synapse_workers_generic_workers_port_range_start + item }}"
       metrics_port: "{{ matrix_synapse_workers_generic_workers_metrics_range_start + item }}"
   register: "matrix_synapse_workers_list_results_generic_workers"
-  loop: "{{ range(0, matrix_synapse_workers_generic_workers_count)|list }}"
+  loop: "{{ range(0, matrix_synapse_workers_generic_workers_count|int)|list }}"
 
 - name: Build federation sender workers
   set_fact:
@@ -21,7 +21,7 @@
       port: 0
       metrics_port: "{{ matrix_synapse_workers_federation_sender_workers_metrics_range_start + item }}"
   register: "matrix_synapse_workers_list_results_federation_sender_workers"
-  loop: "{{ range(0, matrix_synapse_workers_federation_sender_workers_count)|list }}"
+  loop: "{{ range(0, matrix_synapse_workers_federation_sender_workers_count|int)|list }}"
 
 # This type of worker can only have a count of 1, at most
 - name: Build pusher workers
@@ -32,7 +32,7 @@
       port: 0
       metrics_port: "{{ matrix_synapse_workers_pusher_workers_metrics_range_start + item }}"
   register: "matrix_synapse_workers_list_results_pusher_workers"
-  loop: "{{ range(0, matrix_synapse_workers_pusher_workers_count)|list }}"
+  loop: "{{ range(0, matrix_synapse_workers_pusher_workers_count|int)|list }}"
 
 # This type of worker can only have a count of 1, at most
 - name: Build appservice workers
@@ -43,7 +43,7 @@
       port: 0
       metrics_port: "{{ matrix_synapse_workers_appservice_workers_metrics_range_start + item }}"
   register: "matrix_synapse_workers_list_results_appservice_workers"
-  loop: "{{ range(0, matrix_synapse_workers_appservice_workers_count)|list }}"
+  loop: "{{ range(0, matrix_synapse_workers_appservice_workers_count|int)|list }}"
 
 - name: Build media_repository workers
   set_fact:
@@ -53,7 +53,7 @@
       port: "{{ matrix_synapse_workers_media_repository_workers_port_range_start + item }}"
       metrics_port: "{{ matrix_synapse_workers_media_repository_workers_metrics_range_start + item }}"
   register: "matrix_synapse_workers_list_results_media_repository_workers"
-  loop: "{{ range(0, matrix_synapse_workers_media_repository_workers_count)|list }}"
+  loop: "{{ range(0, matrix_synapse_workers_media_repository_workers_count|int)|list }}"
 
 - name: Build frontend_proxy workers
   set_fact:
@@ -63,7 +63,7 @@
       port: "{{ matrix_synapse_workers_frontend_proxy_workers_port_range_start + item }}"
       metrics_port: "{{ matrix_synapse_workers_frontend_proxy_workers_metrics_range_start + item }}"
   register: "matrix_synapse_workers_list_results_frontend_proxy_workers"
-  loop: "{{ range(0, matrix_synapse_workers_frontend_proxy_workers_count)|list }}"
+  loop: "{{ range(0, matrix_synapse_workers_frontend_proxy_workers_count|int)|list }}"
 
 - set_fact:
     matrix_synapse_dynamic_workers_list: "{{ matrix_synapse_dynamic_workers_list|default([]) + [item.ansible_facts.worker] }}"

roles/matrix-synapse/tasks/validate_config.yml

@@ -16,7 +16,7 @@
   fail:
     msg: >-
       `{{ item }}` cannot be more than 1. This is a single-instance worker.
-  when: "vars[item] > 1"
+  when: "vars[item]|int > 1"
   with_items:
     - "matrix_synapse_workers_appservice_workers_count"
     - "matrix_synapse_workers_pusher_workers_count"