Stabilize Matrix Authentication Service integration for Synapse

Related to https://github.com/element-hq/synapse/pull/18759

Currently problematic (leading to failures to start for Synapse) because of:
https://github.com/element-hq/synapse/pull/18759#issuecomment-3172744530

group_vars/matrix_servers

@@ -669,17 +669,6 @@ matrix_authentication_service_config_passwords_schemes:
   - version: 2
     algorithm: argon2id
 
-matrix_authentication_service_config_clients_auto: |-
-  {{
-    ([
-      {
-        'client_id': matrix_synapse_experimental_features_msc3861_client_id,
-        'client_auth_method': matrix_synapse_experimental_features_msc3861_client_auth_method,
-        'client_secret': matrix_synapse_experimental_features_msc3861_client_secret,
-      }
-    ] if matrix_synapse_experimental_features_msc3861_enabled else [])
-  }}
-
 matrix_authentication_service_config_email_transport: "{{ 'smtp' if exim_relay_enabled else 'blackhole' }}"
 matrix_authentication_service_config_email_hostname: "{{ exim_relay_identifier if exim_relay_enabled else '' }}"
 matrix_authentication_service_config_email_port: "{{ 8025 if exim_relay_enabled else 587 }}"
@@ -4911,7 +4900,7 @@ matrix_synapse_systemd_required_services_list_auto: |
     +
     (['matrix-goofys.service'] if matrix_s3_media_store_enabled else [])
     +
-    (['matrix-authentication-service.service'] if (matrix_authentication_service_enabled and matrix_synapse_experimental_features_msc3861_enabled) else [])
+    (['matrix-authentication-service.service'] if (matrix_synapse_matrix_authentication_service_enabled and matrix_synapse_matrix_authentication_service_endpoint == matrix_authentication_service_http_base_container_url) else [])
   }}
 
 matrix_synapse_systemd_wanted_services_list_auto: |
@@ -4945,11 +4934,9 @@ matrix_synapse_report_stats_endpoint: "{{ (('http://' + matrix_synapse_usage_exp
 
 matrix_synapse_experimental_features_msc3266_enabled: "{{ matrix_rtc_enabled }}"
 
-matrix_synapse_experimental_features_msc3861_enabled: "{{ matrix_authentication_service_enabled and not matrix_authentication_service_migration_in_progress }}"
+matrix_synapse_matrix_authentication_service_enabled: "{{ matrix_authentication_service_enabled }}"
-matrix_synapse_experimental_features_msc3861_issuer: "{{ matrix_authentication_service_http_base_container_url if matrix_authentication_service_enabled else '' }}"
+matrix_synapse_matrix_authentication_service_endpoint: "{{ matrix_authentication_service_http_base_container_url if matrix_authentication_service_enabled else '' }}"
-matrix_synapse_experimental_features_msc3861_client_secret: "{{ '%s' | format(matrix_homeserver_generic_secret_key) | password_hash('sha512', 'syn.ngauth.cs', rounds=655555) | to_uuid }}"
+matrix_synapse_matrix_authentication_service_secret: "{{ matrix_authentication_service_config_matrix_secret if matrix_authentication_service_enabled else '' }}"
-matrix_synapse_experimental_features_msc3861_admin_token: "{{ matrix_authentication_service_config_matrix_secret if matrix_authentication_service_enabled else '' }}"
-matrix_synapse_experimental_features_msc3861_account_management_url: "{{ matrix_authentication_service_account_management_url if matrix_authentication_service_enabled else '' }}"
 
 matrix_synapse_experimental_features_msc4108_enabled: "{{ matrix_authentication_service_enabled and not matrix_authentication_service_migration_in_progress }}"
 
@@ -4961,7 +4948,7 @@ matrix_synapse_experimental_features_msc4222_enabled: "{{ matrix_rtc_enabled }}"
 # Unless this is done, Synapse fails on startup with:
 # > Error in configuration at 'password_config.enabled':
 # > Password auth cannot be enabled when OAuth delegation is enabled
-matrix_synapse_password_config_enabled: "{{ not matrix_synapse_experimental_features_msc3861_enabled }}"
+matrix_synapse_password_config_enabled: "{{ not matrix_synapse_matrix_authentication_service_enabled }}"
 
 matrix_synapse_register_user_script_matrix_authentication_service_path: "{{ matrix_authentication_service_bin_path }}/register-user"
 

roles/custom/matrix-synapse/defaults/main.yml

@@ -1216,13 +1216,6 @@ matrix_synapse_email_app_name: Matrix
 matrix_synapse_email_client_base_url: "{{ 'https' if matrix_playbook_ssl_enabled else 'http' }}://{{ matrix_server_fqn_element }}"
 matrix_synapse_email_invite_client_location: "https://app.element.io"
 
-
-################################################################################
-#
-# Next-generation auth for Matrix, based on OAuth 2.0/OIDC
-#
-################################################################################
-
 # Controls whether to enable the "send typing, presence and receipts to appservices" experimental feature.
 #
 # See:
@@ -1244,50 +1237,29 @@ matrix_synapse_experimental_features_msc3202_device_masquerading_enabled: false
 # - https://matrix-org.github.io/matrix-hookshot/latest/advanced/encryption.html#running-with-synapse
 matrix_synapse_experimental_features_msc3202_transaction_extensions_enabled: false
 
-# Controls whether to enable the "Next-generation auth for Matrix, based on OAuth 2.0/OIDC" experimental feature.
+################################################################################
 #
+# Next-generation auth for Matrix, based on OAuth 2.0/OIDC
+#
+################################################################################
+
+# Controls whether to enable "Matrix Authentication Service" integration ("Next-generation auth for Matrix, based on OAuth 2.0/OIDC").
 # See:
+# - https://github.com/element-hq/matrix-authentication-service
 # - https://matrix.org/blog/2023/09/better-auth/
 # - https://github.com/matrix-org/matrix-spec-proposals/pull/3861
-matrix_synapse_experimental_features_msc3861_enabled: false
+matrix_synapse_matrix_authentication_service_enabled: false
 
-# Specifies the issuer URL for the OAuth 2.0/OIDC authentication provider.
+# Specifies the base URL where the Matrix Authentication Service is running.
-#
+matrix_synapse_matrix_authentication_service_endpoint: ""
-# This can be set to a private (container) URL.
-#
-# Example: https://matrix.example.com/auth/
-matrix_synapse_experimental_features_msc3861_issuer: ''
 
-# Specifies the introspection endpoint URL for the OAuth 2.0/OIDC authentication provider.
+# Specifies the shared secret used to authenticate Matrix Authentication Service requests.
-#
+# Must be the same as `matrix.secret` in the Matrix Authentication Service configuration.
-# This can be set to a private (container) URL.
+# See https://element-hq.github.io/matrix-authentication-service/reference/configuration.html#matrix
-#
+matrix_synapse_matrix_authentication_service_secret: ""
-# If this is left empty, `{issuer}/.well-known/openid-configuration` will be fetched and the `introspection_endpoint` will be extracted from there.
-# We define it explicitly, because this allows us to override it and use an internal (container network) URL instead of using the public one.
-# Avoiding public addresses is an optimization that decreases overhead due to public networking and SSL termination.
-#
-# Example: https://matrix.example.com/auth/oauth2/introspect
-matrix_synapse_experimental_features_msc3861_introspection_endpoint: "{{ matrix_synapse_experimental_features_msc3861_issuer + 'oauth2/introspect' }}"
-
-# A unique identifier for the client.
-#
-# It must be a valid ULID (https://github.com/ulid/spec),
-# and it happens that 0000000000000000000SYNAPSE is a valid ULID.
-matrix_synapse_experimental_features_msc3861_client_id: '0000000000000000000SYNAPSE'
-
-matrix_synapse_experimental_features_msc3861_client_auth_method: client_secret_basic
-
-matrix_synapse_experimental_features_msc3861_client_secret: ''
-
-# A token that can be used to make admin API calls.
-# Matches `matrix.secret` in the matrix-authentication-service config
-matrix_synapse_experimental_features_msc3861_admin_token: ''
-
-# URL to advertise to clients where users can self-manage their account.
-matrix_synapse_experimental_features_msc3861_account_management_url: ''
 
 # Controls whether to enable the "QR code login" experimental feature.
-# Enabling this requires that MSC3861 (see `matrix_synapse_experimental_features_msc3861_enabled`) is also enabled.
+# Enabling this requires that Matrix Authentication Service integration (see `matrix_synapse_matrix_authentication_service_enabled`) is also enabled.
 matrix_synapse_experimental_features_msc4108_enabled: false
 
 ################################################################################

roles/custom/matrix-synapse/tasks/main.yml

@@ -62,7 +62,7 @@
 - tags:
     - register-user
   block:
-    - when: matrix_synapse_enabled and not matrix_synapse_experimental_features_msc3861_enabled
+    - when: matrix_synapse_enabled and not matrix_synapse_matrix_authentication_service_enabled
       ansible.builtin.include_tasks: "{{ role_path }}/tasks/register_user.yml"
 
 - tags:

roles/custom/matrix-synapse/tasks/validate_config.yml

@@ -39,23 +39,11 @@
     - {'name': 'matrix_synapse_metrics_proxying_hostname', when: "{{ matrix_synapse_metrics_proxying_enabled }}"}
     - {'name': 'matrix_synapse_metrics_proxying_path_prefix', when: "{{ matrix_synapse_metrics_proxying_enabled }}"}
 
-    - {'name': 'matrix_synapse_experimental_features_msc3861_issuer', when: "{{ matrix_synapse_experimental_features_msc3861_enabled }}"}
+    - {'name': 'matrix_synapse_matrix_authentication_service_endpoint', when: "{{ matrix_synapse_matrix_authentication_service_enabled }}"}
-    - {'name': 'matrix_synapse_experimental_features_msc3861_client_id', when: "{{ matrix_synapse_experimental_features_msc3861_enabled }}"}
+    - {'name': 'matrix_synapse_matrix_authentication_service_secret', when: "{{ matrix_synapse_matrix_authentication_service_enabled }}"}
-    - {'name': 'matrix_synapse_experimental_features_msc3861_client_auth_method', when: "{{ matrix_synapse_experimental_features_msc3861_enabled }}"}
-    - {'name': 'matrix_synapse_experimental_features_msc3861_client_secret', when: "{{ matrix_synapse_experimental_features_msc3861_enabled }}"}
-    - {'name': 'matrix_synapse_experimental_features_msc3861_admin_token', when: "{{ matrix_synapse_experimental_features_msc3861_enabled }}"}
-    - {'name': 'matrix_synapse_experimental_features_msc3861_account_management_url', when: "{{ matrix_synapse_experimental_features_msc3861_enabled }}"}
 
     - {'name': 'matrix_synapse_container_labels_traefik_compression_middleware_name', when: "{{ matrix_synapse_container_labels_traefik_compression_middleware_enabled }}"}
 
-# If only MSC 4108 is enabled, Synapse fails with: "MSC4108 requires MSC3861 to be enabled"
-- name: Fail if Synapse experimental feature QR code login (MSC4108) is enabled while Next-Gen Auth (MSC3861) is not
-  ansible.builtin.fail:
-    msg: >-
-      QR code login (MSC4108) requires Next-Gen Auth (MSC3861) to be enabled or Synapse will fail to start.
-      Enable `matrix_synapse_experimental_features_msc3861_enabled` when using `matrix_synapse_experimental_features_msc4108_enabled`.
-  when: "matrix_synapse_experimental_features_msc4108_enabled and not matrix_synapse_experimental_features_msc3861_enabled"
-
 - name: Fail if asking for more than 1 instance of single-instance workers
   ansible.builtin.fail:
     msg: >-
@@ -121,6 +109,14 @@
     - {'old': 'matrix_s3_goofys_docker_image_name_prefix', 'new': 'matrix_s3_goofys_docker_image_registry_prefix'}
     - {'old': 'matrix_synapse_rust_synapse_compress_state_docker_image_name_prefix', 'new': 'matrix_synapse_rust_synapse_compress_state_docker_image_registry_prefix'}
 
+    - {'old': 'matrix_synapse_experimental_features_msc3861_enabled', 'new': 'matrix_synapse_matrix_authentication_service_enabled'}
+    - {'old': 'matrix_synapse_experimental_features_msc3861_issuer', 'new': '<superseded by matrix_synapse_matrix_authentication_service_endpoint>'}
+    - {'old': 'matrix_synapse_experimental_features_msc3861_client_id', 'new': '<removed>'}
+    - {'old': 'matrix_synapse_experimental_features_msc3861_client_auth_method', 'new': '<removed>'}
+    - {'old': 'matrix_synapse_experimental_features_msc3861_client_secret', 'new': '<removed>'}
+    - {'old': 'matrix_synapse_experimental_features_msc3861_admin_token', 'new': '<removed>'}
+    - {'old': 'matrix_synapse_experimental_features_msc3861_account_management_url', 'new': '<removed>'}
+
 - name: (Deprecation) Catch and report renamed settings in matrix_synapse_configuration_extension_yaml
   ansible.builtin.fail:
     msg: >-
@@ -163,8 +159,8 @@
 
 - name: Fail if known Synapse password provider modules are enabled when auth is delegated to Matrix Authentication Service
   ansible.builtin.fail:
-    msg: "When Synapse is delegating authentication to Matrix Authentication Service, it does not make sense to enable password provider modules, because it is not Synapse that is handling authentication. Please disable {{ item }} before enabling Matrix Authentication Service integration for Synapse. Synapse will refuse to start otherwise."
+    msg: "When Synapse is delegating authentication to Matrix Authentication Service (`matrix_synapse_matrix_authentication_service_enabled: true`), it does not make sense to enable password provider modules, because it is not Synapse that is handling authentication. Please disable {{ item }} before enabling Matrix Authentication Service integration for Synapse. Synapse will refuse to start otherwise."
-  when: matrix_synapse_experimental_features_msc3861_enabled and vars[item] | bool
+  when: matrix_synapse_matrix_authentication_service_enabled and vars[item] | bool
   with_items:
     - matrix_synapse_ext_password_provider_rest_auth_enabled
     - matrix_synapse_ext_password_provider_shared_secret_auth_enabled
@@ -172,10 +168,30 @@
 
 - name: Fail if password config is enabled for Synapse when auth is delegated to Matrix Authentication Service
   ansible.builtin.fail:
-    msg: "When Synapse is delegating authentication to Matrix Authentication Service, it doesn't make sense to enable the password config (`matrix_synapse_password_config_enabled: true`), because it is not Synapse that is handling authentication. Please remove your `matrix_synapse_password_config_enabled: true` setting before enabling Matrix Authentication Service integration for Synapse. Synapse will refuse to start otherwise."
+    msg: "When Synapse is delegating authentication to Matrix Authentication Service (`matrix_synapse_matrix_authentication_service_enabled: true`), it doesn't make sense to enable the password config (`matrix_synapse_password_config_enabled: true`), because it is not Synapse that is handling authentication. Please remove your `matrix_synapse_password_config_enabled: true` setting before enabling Matrix Authentication Service integration for Synapse. Synapse will refuse to start otherwise."
-  when: matrix_synapse_experimental_features_msc3861_enabled and matrix_synapse_password_config_enabled
+  when: matrix_synapse_matrix_authentication_service_enabled and matrix_synapse_password_config_enabled
 
-- name: Fail if QR code login (MSC4108) is enabled while Next-Gen Auth (MSC3861) is not
+- name: Fail if registration is enabled for Synapse when auth is delegated to Matrix Authentication Service
   ansible.builtin.fail:
-    msg: "When Synapse QR code login is enabled (MSC4108 via `matrix_synapse_experimental_features_msc4108_enabled`), Next-Gen auth (MSC3861 via `matrix_synapse_experimental_features_msc3861_enabled`) must also be enabled."
+    msg: "When Synapse is delegating authentication to Matrix Authentication Service (`matrix_synapse_matrix_authentication_service_enabled: true`), it doesn't make sense to enable registration (`matrix_synapse_enable_registration: true`), because it is not Synapse that is handling authentication. Synapse will refuse to start otherwise."
-  when: matrix_synapse_experimental_features_msc4108_enabled and not matrix_synapse_experimental_features_msc3861_enabled
+  when: matrix_synapse_matrix_authentication_service_enabled and matrix_synapse_enable_registration
+
+- name: Fail if registration CAPTCHA is enabled for Synapse when auth is delegated to Matrix Authentication Service
+  ansible.builtin.fail:
+    msg: "When Synapse is delegating authentication to Matrix Authentication Service (`matrix_synapse_matrix_authentication_service_enabled: true`), it doesn't make sense to enable registration CAPTCHA (`matrix_synapse_enable_registration_captcha: true`), because it is not Synapse that is handling authentication. Synapse will refuse to start otherwise."
+  when: matrix_synapse_matrix_authentication_service_enabled and matrix_synapse_enable_registration_captcha
+
+- name: Fail if OpenID Connect is enabled for Synapse when auth is delegated to Matrix Authentication Service
+  ansible.builtin.fail:
+    msg: "When Synapse is delegating authentication to Matrix Authentication Service (`matrix_synapse_matrix_authentication_service_enabled: true`), it doesn't make sense to enable OpenID Connect (`matrix_synapse_oidc_enabled: true`), because it is not Synapse that is handling authentication. Synapse will refuse to start otherwise."
+  when: matrix_synapse_matrix_authentication_service_enabled and matrix_synapse_oidc_enabled
+
+- name: Fail if CAS config is enabled for Synapse when auth is delegated to Matrix Authentication Service
+  ansible.builtin.fail:
+    msg: "When Synapse is delegating authentication to Matrix Authentication Service (`matrix_synapse_matrix_authentication_service_enabled: true`), it doesn't make sense to enable CAS config (`matrix_synapse_cas_config_enabled: true`), because it is not Synapse that is handling authentication. Synapse will refuse to start otherwise."
+  when: matrix_synapse_matrix_authentication_service_enabled and matrix_synapse_cas_config_enabled
+
+- name: Fail if QR code login (MSC4108) is enabled while Matrix Authentication Service is not
+  ansible.builtin.fail:
+    msg: "When Synapse QR code login is enabled (MSC4108 via `matrix_synapse_experimental_features_msc4108_enabled`), Matrix Authentication Service integration (`matrix_synapse_matrix_authentication_service_enabled`) must also be enabled."
+  when: matrix_synapse_experimental_features_msc4108_enabled and not matrix_synapse_matrix_authentication_service_enabled

roles/custom/matrix-synapse/templates/synapse/bin/register-user.j2

@@ -1,7 +1,7 @@
 #jinja2: lstrip_blocks: True
 #!/bin/bash
 
-{% if matrix_synapse_experimental_features_msc3861_enabled %}
+{% if matrix_synapse_matrix_authentication_service_enabled %}
 	echo "Registering users is handled by the Matrix Authentication Service, so you cannot use this script anymore."
 	echo "Consider using the {{ matrix_synapse_register_user_script_matrix_authentication_service_path }} script instead."
 	exit 2

roles/custom/matrix-synapse/templates/synapse/homeserver.yaml.j2

@@ -2971,6 +2971,14 @@ background_updates:
     #
     #default_batch_size: 50
 
+
+{% if matrix_synapse_matrix_authentication_service_enabled %}
+matrix_authentication_service:
+  enabled: true
+  endpoint: {{ matrix_synapse_matrix_authentication_service_endpoint | to_json }}
+  secret: {{ matrix_synapse_matrix_authentication_service_secret | to_json }}
+{% endif %}
+
 experimental_features:
   {% if matrix_synapse_experimental_features_msc2409_to_device_messages_enabled %}
   msc2409_to_device_messages_enabled: true
@@ -2984,17 +2992,6 @@ experimental_features:
   {% if matrix_synapse_experimental_features_msc3266_enabled %}
   msc3266_enabled: true
   {% endif %}
-  {% if matrix_synapse_experimental_features_msc3861_enabled %}
-  msc3861:
-    enabled: true
-    issuer: {{ matrix_synapse_experimental_features_msc3861_issuer | to_json }}
-    introspection_endpoint: {{ matrix_synapse_experimental_features_msc3861_introspection_endpoint | to_json }}
-    client_id: {{ matrix_synapse_experimental_features_msc3861_client_id | to_json }}
-    client_auth_method: {{ matrix_synapse_experimental_features_msc3861_client_auth_method | to_json }}
-    client_secret: {{ matrix_synapse_experimental_features_msc3861_client_secret | to_json }}
-    admin_token: {{ matrix_synapse_experimental_features_msc3861_admin_token | to_json }}
-    account_management_url: {{ matrix_synapse_experimental_features_msc3861_account_management_url | to_json }}
-  {% endif %}
   {% if matrix_synapse_experimental_features_msc4108_enabled %}
   msc4108_enabled: true
   {% endif %}