From 9f1cf3f482c7b9cbe011c163bbd4de369adebd92 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 11 Aug 2026 17:31:15 +0100 Subject: [PATCH 01/28] Fix the documentation on the `federation_domain_whitelist` config option. (#20089) [As discussed in Backend Lobby](https://matrix.to/#/!SGNQGPGUwtcPBUotTL:matrix.org/$jz87yx9uFcwKYhwejCLiVlfRLKseUHRnRrz2jZcGvOs?via=jki.re&via=element.io&via=matrix.org) As for justification for calling this the recommended way, - from memory this is accurate - a previous changelog implies this: https://github.com/element-hq/synapse/blob/287904c03ac8892407be960d475c4d25007e8917/docs/changelogs/CHANGES-2022.md?plain=1#L1296 - this is what we are doing internally ([Backend Lobby example](https://matrix.to/#/!SGNQGPGUwtcPBUotTL:matrix.org/$MTCVg5-D0k_jq9QZxVSSy1CeMGs8mbG6ld-BWOOy9JI?via=jki.re&via=element.io&via=matrix.org)) I can't find a definitive source though --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/20089.doc | 1 + docs/usage/configuration/config_documentation.md | 9 ++++----- schema/synapse-config.schema.yaml | 12 +++++++----- 3 files changed, 12 insertions(+), 10 deletions(-) create mode 100644 changelog.d/20089.doc diff --git a/changelog.d/20089.doc b/changelog.d/20089.doc new file mode 100644 index 0000000000..e72db0dbde --- /dev/null +++ b/changelog.d/20089.doc @@ -0,0 +1 @@ +Fix the documentation on the `federation_domain_whitelist` config option. \ No newline at end of file diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 3fb961d6f8..7654039e5c 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1287,11 +1287,10 @@ Options related to federation. --- ### `federation_domain_whitelist` -*(array)* Restrict federation to the given whitelist of domains. N.B. we recommend also firewalling your federation listener to limit inbound federation traffic as early as possible, rather than relying purely on this application-layer restriction. If not specified, the default is to whitelist everything. - -Note: this does not stop a server from joining rooms that servers not on the whitelist are in. As such, this option is really only useful to establish a "private federation", where a group of servers all whitelist each other and have the same whitelist. - -Defaults to `[]`. +*(array)* Restrict federation to the given whitelist of domains. N.B. we recommend also firewalling your federation listener to limit inbound federation traffic as early as possible, rather than relying purely on this application-layer restriction. +If specified as an empty list (`[]`), federation will be denied with all servers. Specifying an empty list (`[]`) here is the recommended way of disabling federation. +If not specified, the default is to allow federation with all servers. +Note: this does not stop a server from joining rooms that servers not on the whitelist are in. As such, this option is really only useful to establish a "private federation", where a group of servers all whitelist each other and have the same whitelist. There is no default for this option. Example configuration: ```yaml diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 6e7880a900..7ab9191181 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -280,10 +280,10 @@ properties: description: >- Use this option to include updates of other users' profiles in sync responses, for users who share rooms. - + Requires an [MSC4429](https://github.com/matrix-org/matrix-spec-proposals/pull/4429) compatible client, and is currently limited to legacy sync and local users only. - + This feature is under development and should be used with caution on busy servers or servers which depend on `limit_profile_requests_to_users_who_share_rooms` for ensuring profile information doesn't leak across rooms. @@ -1578,9 +1578,12 @@ properties: Restrict federation to the given whitelist of domains. N.B. we recommend also firewalling your federation listener to limit inbound federation traffic as early as possible, rather than relying purely on this - application-layer restriction. If not specified, the default is to - whitelist everything. + application-layer restriction. + If specified as an empty list (`[]`), federation will be denied with all servers. + Specifying an empty list (`[]`) here is the recommended way of disabling federation. + + If not specified, the default is to allow federation with all servers. Note: this does not stop a server from joining rooms that servers not on the whitelist are in. As such, this option is really only useful to @@ -1588,7 +1591,6 @@ properties: each other and have the same whitelist. items: type: string - default: [] examples: - - lon.example.com - nyc.example.com From e6cc157cbdca31c9457baee1a373103a3f354379 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 12 Aug 2026 11:54:53 -0500 Subject: [PATCH 02/28] Update release script to check more often for actions being completed (every 1m) (#20093) (`_wait_for_actions`) Spawning from seeing the release CI being complete but needing to wait up to 5 minutes longer to continue on. ### Dev notes Originally the waiting was introduced in https://github.com/matrix-org/synapse/pull/13483 GitHub rate limit: > The primary rate limit for unauthenticated requests is 60 requests per hour. > > *-- https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2026-03-10#primary-rate-limit-for-unauthenticated-users* --- changelog.d/20093.misc | 1 + scripts-dev/release.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20093.misc diff --git a/changelog.d/20093.misc b/changelog.d/20093.misc new file mode 100644 index 0000000000..60c745eb47 --- /dev/null +++ b/changelog.d/20093.misc @@ -0,0 +1 @@ +Update release script to check more often for actions being completed so you don't have to wait around as much. diff --git a/scripts-dev/release.py b/scripts-dev/release.py index f78c2c0ab0..58d36f7dcc 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -600,9 +600,15 @@ def _wait_for_actions(gh_token: str | None) -> None: headers["authorization"] = f"token {gh_token}" req = urllib.request.Request(url, headers=headers) + # Initially, wait 10 minutes as we know the CI typically takes 15m+ anyway (no need + # to check over and over when we know it won't be finished yet) time.sleep(10 * 60) while True: - time.sleep(5 * 60) + # Then check once every minute. Short enough to not have to wait around too long + # while not spamming the GitHub API and running into the unauthenticated API + # request rate limit (60 requests per hour so 1 request/minute perfectly aligns + # to not run into any problems) + time.sleep(1 * 60) response = urllib.request.urlopen(req) resp = json.loads(response.read()) From 4bbc6ad74fedb5af556655a9e6d986a32b429b5a Mon Sep 17 00:00:00 2001 From: William L Thomson Jr Date: Thu, 13 Aug 2026 06:12:32 -0400 Subject: [PATCH 03/28] Document lighttpd configuration example from matrix.jaxlug.ngo (#19875) --- changelog.d/19875.doc | 1 + docs/reverse_proxy.md | 85 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 changelog.d/19875.doc diff --git a/changelog.d/19875.doc b/changelog.d/19875.doc new file mode 100644 index 0000000000..6ab67a08a2 --- /dev/null +++ b/changelog.d/19875.doc @@ -0,0 +1 @@ +Document lighttpd reverse proxy configuration example from matrix.jaxlug.ngo, a contribution from the JaxLUG, the Jacksonville Linux Users Group Inc.. diff --git a/docs/reverse_proxy.md b/docs/reverse_proxy.md index 0e3303df57..df953bbccc 100644 --- a/docs/reverse_proxy.md +++ b/docs/reverse_proxy.md @@ -4,8 +4,10 @@ It is recommended to put a reverse proxy such as [nginx](https://nginx.org/en/docs/http/ngx_http_proxy_module.html), [Apache](https://httpd.apache.org/docs/current/mod/mod_proxy_http.html), [Caddy](https://caddyserver.com/docs/quick-starts/reverse-proxy), -[HAProxy](https://www.haproxy.org/) or -[relayd](https://man.openbsd.org/relayd.8) in front of Synapse. +[HAProxy](https://www.haproxy.org/), +[relayd](https://man.openbsd.org/relayd.8) or +[lighttpd](https://www.lighttpd.net/) +in front of Synapse. This has the advantage of being able to expose the default HTTPS port (443) to Matrix clients without requiring Synapse to bind to a privileged port (port numbers less than 1024), avoiding the need for `CAP_NET_BIND_SERVICE` or running as root. @@ -312,6 +314,85 @@ relay "matrix_federation" { } ``` +### lighttpd +```conf +server.modules = ( + "mod_rewrite", + "mod_redirect", + "mod_access", + "mod_setenv", + "mod_openssl", + "mod_proxy", + "mod_accesslog" +) + +server.username = "lighttpd" +server.groupname = "lighttpd" + +# disable for wildcard IPv6 on all, enable for specific IPv6 addresses +# see below IPv4 0.0.0.0 & IPv6 [::] +server.use-ipv6 = "disable" + +ssl.pemfile = "/etc/lighttpd/cert+privkey.pem" +ssl.ca-file = "/etc/lighttpd/fullchain.pem" + +# redirect HTTP traffic to HTTPS, same for IPv6 below +$SERVER["socket"] == "0.0.0.0:80" { + url.redirect = ( + "" => "https://${url.authority.noport}${url.path}${qsa}" + ) +} +$SERVER["socket"] == "0.0.0.0:443" { ssl.engine = "enable" } +$SERVER["socket"] == "0.0.0.0:8448" { ssl.engine = "enable" } +$SERVER["socket"] == "[::]:80" { + url.redirect = ( + "" => "https://${url.authority.noport}${url.path}${qsa}" + ) +} +$SERVER["socket"] == "[::]:443" { ssl.engine = "enable" } +$SERVER["socket"] == "[::]:8448" { ssl.engine = "enable" } + + + +# both lighttpd and synapse need permissions for socket r/w +$HTTP["url"] =~ "(/_matrix|_synapse/admin|/_synapse/client)" { + proxy.balance = "hash" + proxy.server = ( + "" => ( + "backend-socket" => ( + "host" => "/var/lib/synapse/main_public.sock", + "port" => 0 + ) + ) + ) + proxy.forwarded = ( + "for" => 1, + "proto" => 1, + "host" => 1, + ) +} +# protect admin access IPv6 ULA only +$HTTP["remoteip"] !="fd00::/8" { + $HTTP["url"] =~ "^/_synapse/admin/" { + url.access-deny = ( "" ) + } +} +``` + +[Delegation](delegate.md) example: +```conf +url.rewrite-once = ( + "^/\.well-known/matrix/client$" => "/.well-known/matrix/client.json", + "^/\.well-known/matrix/server$" => "/.well-known/matrix/server.json" +) + +# This condition intentionally matches the post-rewrite URLs. +$HTTP["url"] =~ "^/\.well-known/matrix/(client|server)\.json$" { + mimetype.assign = ( ".json" => "application/json" ) + setenv.set-response-header = ( "Access-Control-Allow-Origin" => "*" ) +} +``` + ## Health check endpoint From b7db66c21e845f2ada12d38e5ec848c5bbfe6f07 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 13 Aug 2026 11:13:36 +0100 Subject: [PATCH 04/28] Clarify comment From https://github.com/element-hq/synapse/pull/19875#discussion_r3774386943. I was unable to do so on the PR quickly as the branch was not available for maintainers to edit. --- docs/reverse_proxy.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/reverse_proxy.md b/docs/reverse_proxy.md index df953bbccc..fc38c06335 100644 --- a/docs/reverse_proxy.md +++ b/docs/reverse_proxy.md @@ -329,8 +329,11 @@ server.modules = ( server.username = "lighttpd" server.groupname = "lighttpd" -# disable for wildcard IPv6 on all, enable for specific IPv6 addresses -# see below IPv4 0.0.0.0 & IPv6 [::] +# We set this to "disable" and use IPv6 `[::]` explicitly below, +# in order to listen on all incoming IPv6 addresses. +# +# If you only want to listen on specific IPv6 addresses, set this +# to "enable" and specify said addresses below. server.use-ipv6 = "disable" ssl.pemfile = "/etc/lighttpd/cert+privkey.pem" From 0c6714d0d291b1cc2d450b11964038bca424814a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 13 Aug 2026 14:02:07 +0100 Subject: [PATCH 05/28] Allow specifying multiple `action_name` and `status` params in the scheduled tasks admin API (#20067) We have an internal usage of `/scheduled_tasks` that would like to fetch multiple actions at once (janitor). We also make it so that invalid `status` values now return a 400 rather than a 500. Co-authored-by: Claude Fable 5 --- changelog.d/20067.feature | 1 + docs/admin_api/scheduled_tasks.md | 4 +- synapse/rest/admin/scheduled_tasks.py | 33 +++++++++++---- tests/rest/admin/test_scheduled_tasks.py | 53 ++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 8 deletions(-) create mode 100644 changelog.d/20067.feature diff --git a/changelog.d/20067.feature b/changelog.d/20067.feature new file mode 100644 index 0000000000..123d4df745 --- /dev/null +++ b/changelog.d/20067.feature @@ -0,0 +1 @@ +Allow specifying multiple `action_name` and `status` query parameters when listing scheduled tasks via the admin API. diff --git a/docs/admin_api/scheduled_tasks.md b/docs/admin_api/scheduled_tasks.md index 949a03ee39..7d7c68d987 100644 --- a/docs/admin_api/scheduled_tasks.md +++ b/docs/admin_api/scheduled_tasks.md @@ -31,8 +31,10 @@ It returns a JSON body like the following: **Query parameters:** * `action_name`: string - Is optional. Returns only the scheduled tasks with the given action name. + May be given multiple times to return tasks matching any of the given action names. * `resource_id`: string - Is optional. Returns only the scheduled tasks with the given resource id. -* `status`: string - Is optional. Returns only the scheduled tasks matching the given status, one of +* `status`: string - Is optional. Returns only the scheduled tasks matching the given status. + May be given multiple times to return tasks matching any of the given statuses. The status must be one of - "scheduled" - Task is scheduled but not active - "active" - Task is active and probably running, and if not will be run on next scheduler loop run - "complete" - Task has completed successfully diff --git a/synapse/rest/admin/scheduled_tasks.py b/synapse/rest/admin/scheduled_tasks.py index 5b3526c7e5..08c7bec783 100644 --- a/synapse/rest/admin/scheduled_tasks.py +++ b/synapse/rest/admin/scheduled_tasks.py @@ -15,7 +15,12 @@ # from typing import TYPE_CHECKING -from synapse.http.servlet import RestServlet, parse_integer, parse_string +from synapse.http.servlet import ( + RestServlet, + parse_integer, + parse_string, + parse_strings_from_args, +) from synapse.http.site import SynapseRequest from synapse.rest.admin import admin_patterns, assert_requester_is_admin from synapse.types import JsonDict, TaskStatus @@ -38,19 +43,33 @@ def __init__(self, hs: "HomeServer"): async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: await assert_requester_is_admin(self._auth, request) + # twisted.web.server.Request.args is incorrectly defined as Any | None + args: dict[bytes, list[bytes]] = request.args # type: ignore + # extract query params - action_name = parse_string(request, "action_name") + actions = parse_strings_from_args(args, "action_name") resource_id = parse_string(request, "resource_id") - status = parse_string(request, "status") + status_strings = parse_strings_from_args( + args, + "status", + allowed_values=[status.value for status in TaskStatus], + ) # This parameter was historically called `job_status`, while the Admin API docs # defined it as `status`. We now support both, as `status` is generally # a nicer name. A v2 of this endpoint should keep only `status`. - if status is None: - status = parse_string(request, "job_status") + if status_strings is None: + status_strings = parse_strings_from_args( + args, + "job_status", + allowed_values=[status.value for status in TaskStatus], + ) max_timestamp = parse_integer(request, "max_timestamp") - actions = [action_name] if action_name else None - statuses = [TaskStatus(status)] if status else None + statuses = ( + [TaskStatus(status) for status in status_strings] + if status_strings + else None + ) tasks = await self._store.get_scheduled_tasks( actions=actions, diff --git a/tests/rest/admin/test_scheduled_tasks.py b/tests/rest/admin/test_scheduled_tasks.py index 4b7adb6b89..388570df0b 100644 --- a/tests/rest/admin/test_scheduled_tasks.py +++ b/tests/rest/admin/test_scheduled_tasks.py @@ -190,3 +190,56 @@ def test_filtering_scheduled_tasks(self) -> None: # only the task with the matching resource id should have been returned self.assertEqual(len(found_tasks), 1) self.assertEqual(found_tasks[0]["resource_id"], "failed_task") + + def test_filtering_scheduled_tasks_multiple_values(self) -> None: + """ + Test that the `action_name` and `status` filters can be given multiple + times, returning tasks matching any of the given values. + """ + # filter via multiple statuses + channel = self.make_request( + "GET", + "/_synapse/admin/v1/scheduled_tasks?status=active&status=failed", + content={}, + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + found_tasks = self.check_scheduled_tasks_response( + channel.json_body["scheduled_tasks"] + ) + + # the active and failed tasks should have been returned + self.assertEqual(len(found_tasks), 2) + self.assertEqual({task["status"] for task in found_tasks}, {"active", "failed"}) + + # filter via multiple action names + channel = self.make_request( + "GET", + "/_synapse/admin/v1/scheduled_tasks?action_name=test_task&action_name=finished_test_task", + content={}, + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + found_tasks = self.check_scheduled_tasks_response( + channel.json_body["scheduled_tasks"] + ) + + # only the tasks with the given action names should have been returned + self.assertEqual(len(found_tasks), 2) + self.assertEqual( + {task["action"] for task in found_tasks}, + {"test_task", "finished_test_task"}, + ) + + def test_filtering_scheduled_tasks_invalid_status(self) -> None: + """ + Test that an invalid `status` value is rejected with a 400 error. + """ + channel = self.make_request( + "GET", + "/_synapse/admin/v1/scheduled_tasks?status=unknown_status", + content={}, + access_token=self.admin_user_tok, + ) + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) From c78c274f172a05439bd8267eae139773abfdf499 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Thu, 13 Aug 2026 16:17:40 +0100 Subject: [PATCH 06/28] Fix the schema diff CI not using `faketime` for SQLite. (#20099) Issue spotted in: https://github.com/element-hq/synapse/pull/20098 Follows: #20027 We already use `faketime` for Postgres, but I forgot that the SQLite schema delta would have the same problem and somehow tuned it out of the preview diff on the original PR. --------- Signed-off-by: Olivier 'reivilibre --- .github/workflows/schema_diff.yml | 9 ++++++--- changelog.d/20099.misc | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 changelog.d/20099.misc diff --git a/.github/workflows/schema_diff.yml b/.github/workflows/schema_diff.yml index 0c32ef33f4..3dfc02a469 100644 --- a/.github/workflows/schema_diff.yml +++ b/.github/workflows/schema_diff.yml @@ -30,6 +30,7 @@ jobs: - name: Start postgres with a faked clock background: true id: postgres + # Use faketime here for schema deltas that are wall-clock sensitive under Postgres run: | # Build a docker image with faketime mkdir /tmp/postgres-faketime @@ -58,8 +59,8 @@ jobs: with: fetch-depth: 0 - - name: Install PostgreSQL client - run: sudo apt-get -qq install postgresql-client + - name: Install PostgreSQL client and faketime + run: sudo apt-get -qq install postgresql-client faketime - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -77,8 +78,10 @@ jobs: PGHOST: localhost PGUSER: postgres PGPASSWORD: postgres + # Use faketime here for schema deltas that are wall-clock sensitive under SQLite run: | - poetry run python .ci/scripts/schema_diff.py \ + faketime -f "2001-05-25 12:42:42" \ + poetry run python .ci/scripts/schema_diff.py \ --base origin/develop \ > "${{ runner.temp }}/schema_diff.md" diff --git a/changelog.d/20099.misc b/changelog.d/20099.misc new file mode 100644 index 0000000000..f17b66915b --- /dev/null +++ b/changelog.d/20099.misc @@ -0,0 +1 @@ +Fix the schema diff CI not using `faketime` for SQLite. \ No newline at end of file From c0357de4eda2f32919d89b4cffa306d68562c022 Mon Sep 17 00:00:00 2001 From: FrenchGithubUser Date: Thu, 13 Aug 2026 17:30:51 +0200 Subject: [PATCH 07/28] fix: presence stream stalling intermittently (#20090) This is a fix for presence updates silently stalling when a `/sync` request is cancelled mid-write, causing a stream ID to be leaked into `_unfinished_ids` and permanently pinning the persisted stream position. Fixes https://github.com/element-hq/synapse/issues/19800 ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/20090.bugfix | 1 + synapse/storage/util/id_generators.py | 42 +++++++++++-- tests/storage/test_id_generators.py | 88 +++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 changelog.d/20090.bugfix diff --git a/changelog.d/20090.bugfix b/changelog.d/20090.bugfix new file mode 100644 index 0000000000..69b8035a44 --- /dev/null +++ b/changelog.d/20090.bugfix @@ -0,0 +1 @@ +Fix a bug where presence updates could stop being sent to clients (the presence stream position becoming stuck) if a `/sync` request was cancelled while a presence write was allocating a stream ID. Contributed by @FrenchGithubUser @Famedly. diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index c9c339b235..d294119bb2 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -906,14 +906,44 @@ class _MultiWriterCtxManager: stream_ids: list[int] = attr.Factory(list) async def __aenter__(self) -> int | list[int]: + def _load(txn: LoggingTransaction) -> list[int]: + ids = self.id_gen._load_next_mult_id_txn(txn, self.multiple_ids or 1) + # Record the allocated IDs on the context manager as a side effect + # (rather than only via the return value), so that if this coroutine + # is cancelled after the transaction has committed we still know + # which IDs to release below. + self.stream_ids = ids + return ids + # It's safe to run this in autocommit mode as fetching values from a # sequence ignores transaction semantics anyway. - self.stream_ids = await self.id_gen._db.runInteraction( - "_load_next_mult_id", - self.id_gen._load_next_mult_id_txn, - self.multiple_ids or 1, - db_autocommit=True, - ) + try: + await self.id_gen._db.runInteraction( + "_load_next_mult_id", + _load, + db_autocommit=True, + ) + except BaseException: + # We catch `BaseException` rather than `Exception`, + # because request cancellation surfaces here as exceptions that are + # not `Exception` subclasses: `asyncio.CancelledError` + # and `GeneratorExit` (raised when a paused coroutine is garbage + # collected). + # + # If we're interrupted (e.g. the enclosing request was cancelled) + # after the transaction allocated the IDs but before we returned, + # then `__aexit__` will never run, because Python only invokes it + # once `__aenter__` has returned. The allocated IDs would then be + # leaked into `_unfinished_ids` forever, permanently pinning the + # persisted stream position and, e.g., wedging presence. + # + # So mark them as finished here to unblock the position. This mirrors + # what `__aexit__` does on the failure path (marking the IDs finished + # and notifying replication, but not persisting a new position). + if self.stream_ids: + self.id_gen._mark_ids_as_finished(self.stream_ids) + self.notifier.notify_replication() + raise if self.multiple_ids is None: return self.stream_ids[0] * self.id_gen._return_factor diff --git a/tests/storage/test_id_generators.py b/tests/storage/test_id_generators.py index 9a338607ee..a42247b498 100644 --- a/tests/storage/test_id_generators.py +++ b/tests/storage/test_id_generators.py @@ -19,8 +19,12 @@ # # +from unittest import mock + +from twisted.internet.defer import CancelledError, Deferred, ensureDeferred from twisted.internet.testing import MemoryReactor +from synapse.logging.context import LoggingContext, make_deferred_yieldable from synapse.server import HomeServer from synapse.storage.database import ( DatabasePool, @@ -225,6 +229,90 @@ async def _get_next_async() -> None: self.assertEqual(id_gen.get_positions(), {"master": 8}) self.assertEqual(id_gen.get_current_token_for_writer("master"), 8) + def test_cancelled_enter_does_not_wedge_position(self) -> None: + """Reproduces presence getting stuck. + + If the `get_next()` async context manager is cancelled while + `__aenter__` is allocating a stream ID, the DB interaction that runs the + sequence has already added the ID to `_unfinished_ids`, but `__aexit__` + is never called (Python only invokes `__aexit__` if `__aenter__` + returned). The abandoned ID is therefore leaked into `_unfinished_ids` + forever, which permanently pins the persisted stream position: new rows + keep getting higher IDs, but `get_current_token()` can never advance past + `leaked_id - 1` until the process restarts. + + This mirrors a `/sync` request being cancelled part-way through + persisting a presence update. `/sync` became `@cancellable` in #19499, + and on a monolith the presence write in `PresenceStore.update_presence` + is awaited inside that cancellable request scope. + """ + # Prefill table with 7 rows written by 'master'; position starts at 7. + self._insert_rows("master", 7) + + id_gen = self._create_id_generator() + self.assertEqual(id_gen.get_current_token_for_writer("master"), 7) + + # We model the cancellation at the seam it actually happens in + # production: `__aenter__` awaits `runInteraction("_load_next_mult_id")`, + # whose transaction runs in a thread pool and so *always* completes - + # allocating stream ID 8 and adding it to `_unfinished_ids` - but the + # awaiting coroutine is handed a `CancelledError` because the enclosing + # `/sync` request was cancelled. We reproduce that by letting the real + # interaction run (applying its side effects) and then failing the + # awaited deferred with `CancelledError`. + cancel_enter: "Deferred[None]" = Deferred() + original_run_interaction = id_gen._db.runInteraction + + async def blocking_run_interaction(desc, func, *args, **kwargs): # type: ignore[no-untyped-def] + result = await original_run_interaction(desc, func, *args, **kwargs) + if desc == "_load_next_mult_id": + # Stream ID 8 is now allocated and recorded in `_unfinished_ids`. + # Deliver the cancellation here, exactly as a cancelled `/sync` + # would land it on this `await`. + await make_deferred_yieldable(cancel_enter) + return result + + async def presence_like_write() -> None: + # Mirrors `PresenceStore.update_presence`: allocate an ID and + # "persist" under the context manager. + with LoggingContext(name="sync", server_name=self.hs.hostname): + async with id_gen.get_next(): + pass + + with mock.patch.object( + id_gen._db, "runInteraction", new=blocking_run_interaction + ): + write = ensureDeferred(presence_like_write()) + + # The write is now blocked inside `__aenter__`, i.e. after stream ID + # 8 has been allocated and added to `_unfinished_ids`. + self.assertNoResult(write) + + # The client goes away and the `/sync` request is cancelled. + cancel_enter.errback(CancelledError()) + + # The cancellation must surface as a `CancelledError`. + self.get_failure(write, CancelledError) + + # The cancelled write never persisted a row for ID 8, so the generator + # must not let that abandoned ID wedge the position. A subsequent + # *successful* write should be able to advance the persisted token. + async def _successful_write() -> None: + async with id_gen.get_next(): + pass + + self.get_success(_successful_write()) + + # On the buggy code the token is still stuck at 7 (ID 8 is leaked in + # `_unfinished_ids`, blocking everything behind it). Once the leak is + # fixed, the token advances to 9: ID 8 was allocated (and abandoned) by + # the cancelled write, so the successful write above takes ID 9. + self.assertEqual( + id_gen.get_current_token_for_writer("master"), + 9, + "presence stream position is wedged by the cancelled allocation", + ) + def test_out_of_order_finish(self) -> None: """Test that IDs persisted out of order are correctly handled""" From 5a782430ec67e45dbb2c2c035f8db33aa4855179 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Mon, 17 Aug 2026 12:47:35 +0100 Subject: [PATCH 08/28] Fix the schema diff CI breaking when the Rust module was changed. (#20117) Follows: #20027 Noticed in https://github.com/element-hq/synapse/pull/20003 The problem was that `--no-root` prevents reinstalling the root project (Synapse). However, since we just did a `git checkout`, we need to reinstall the root project in case the Rust code changed, as the `poetry install` command is what causes the Rust to be recompiled. Doing otherwise causes the 'Rust module outdated' error when importing `synapse`. --------- Signed-off-by: Olivier 'reivilibre --- .ci/scripts/schema_diff.py | 2 +- changelog.d/20117.misc | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20117.misc diff --git a/.ci/scripts/schema_diff.py b/.ci/scripts/schema_diff.py index 9b7af72434..8354b64233 100755 --- a/.ci/scripts/schema_diff.py +++ b/.ci/scripts/schema_diff.py @@ -193,7 +193,7 @@ def main() -> None: # Refresh dependencies print("Installing dependencies for base commit...", file=sys.stderr) subprocess.run( - ["poetry", "install", "--no-root", "--extras", "postgres"], + ["poetry", "install", "--extras", "postgres"], cwd=REPO_ROOT, check=True, # Poetry install is noisy, so pipe its stdout to stderr diff --git a/changelog.d/20117.misc b/changelog.d/20117.misc new file mode 100644 index 0000000000..92d421117e --- /dev/null +++ b/changelog.d/20117.misc @@ -0,0 +1 @@ +Fix the schema diff CI breaking when the Rust module was changed. \ No newline at end of file From 94a5f2afb36a4afdc36813fd1c24b9a1c4aec252 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 17 Aug 2026 12:52:43 +0100 Subject: [PATCH 09/28] Speed up marking device list changes as converted (#20098) The conversion of `device_lists_changes_in_room` rows into `device_lists_outbound_pokes` is DB-bound on `mark_redundant_device_lists_pokes`. The `UPDATE` uses the `(room_id, stream_id)` index, so each call scans the unconverted backlog, getting slower the further behind the conversion is. Add a partial index matching the query via a background update, and skip the (safe-to-skip) `UPDATE` until the index has been built. Also add a metric reporting how far behind the conversion is, using the existing `inserted_ts` column. Fixes https://github.com/element-hq/backend-internal/issues/286 --------- Co-authored-by: Claude Fable 5 Co-authored-by: Eric Eastwood --- changelog.d/20098.misc | 1 + synapse/handlers/device.py | 54 ++++++++++++ synapse/storage/background_updates.py | 16 ++++ synapse/storage/databases/main/devices.py | 86 ++++++++++++++++++- ..._lists_changes_in_room_unconverted_idx.sql | 22 +++++ 5 files changed, 176 insertions(+), 3 deletions(-) create mode 100644 changelog.d/20098.misc create mode 100644 synapse/storage/schema/main/delta/94/08_device_lists_changes_in_room_unconverted_idx.sql diff --git a/changelog.d/20098.misc b/changelog.d/20098.misc new file mode 100644 index 0000000000..0504be872d --- /dev/null +++ b/changelog.d/20098.misc @@ -0,0 +1 @@ +Speed up the conversion of device list changes into outbound federation pokes, and add a metric for how far behind the conversion is. diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 2225466648..ae61bc24bf 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -30,6 +30,8 @@ cast, ) +from prometheus_client import Gauge + from synapse.api import errors from synapse.api.constants import EduTypes, EventTypes, Membership from synapse.api.errors import ( @@ -41,6 +43,7 @@ SynapseError, ) from synapse.logging.opentracing import log_kv, set_tag, trace +from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import ( wrap_as_background_process, ) @@ -89,6 +92,21 @@ MAX_DEVICE_DISPLAY_NAME_LEN = 100 DELETE_STALE_DEVICES_INTERVAL = Duration(days=1) +device_list_conversion_lag_gauge = Gauge( + "synapse_device_lists_changes_conversion_lag_seconds", + "Age of the oldest device list change that has yet to be converted to outbound federation pokes", + labelnames=[SERVER_NAME_LABEL], +) + +device_list_conversion_stream_lag_gauge = Gauge( + "synapse_device_lists_changes_conversion_stream_lag", + "Number of stream IDs between the current device lists stream position and the position converted to outbound federation pokes", + labelnames=[SERVER_NAME_LABEL], +) + +# How often to update the device list conversion lag gauges. +DEVICE_LIST_CONVERSION_LAG_GAUGE_METRIC_UPDATE_INTERVAL = Duration(seconds=30) + def _check_device_name_length(name: str | None) -> None: """ @@ -960,6 +978,13 @@ def __init__(self, hs: "HomeServer"): self.device_list_updater.incoming_device_list_update, ) + # Report how far behind we are at converting device list changes + # into outbound pokes. + self.clock.looping_call( + self._report_device_list_conversion_lag, + DEVICE_LIST_CONVERSION_LAG_GAUGE_METRIC_UPDATE_INTERVAL, + ) + @trace @measure_func("notify_device_update") async def notify_device_update( @@ -1033,6 +1058,35 @@ async def handle_new_device_update(self) -> None: self._handle_new_device_update_async() return + @wrap_as_background_process("_report_device_list_conversion_lag") + async def _report_device_list_conversion_lag(self) -> None: + """Report how far behind we are at converting rows in + `device_lists_changes_in_room` to `device_lists_outbound_pokes`. + """ + ( + oldest_ts, + last_converted_pos, + ) = await self.store.get_device_list_conversion_lag() + + if oldest_ts is None: + device_list_conversion_lag_ms = 0 + else: + device_list_conversion_lag_ms = max(0, self.clock.time_msec() - oldest_ts) + + device_list_conversion_lag_gauge.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).set(device_list_conversion_lag_ms / 1000.0) # convert to seconds + + # The stream ID lag is only an approximation of the conversion + # backlog: the converted position only advances when the conversion + # loop runs, and stream IDs in the gap may not have rows needing + # conversion at all. + current_pos = self.store.get_device_stream_token().stream + + device_list_conversion_stream_lag_gauge.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).set(max(0, current_pos - last_converted_pos)) + @wrap_as_background_process("_handle_new_device_update_async") async def _handle_new_device_update_async(self) -> None: """Called when we have a new local device list update that we need to diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index 311534c5e7..8137b02036 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -262,6 +262,10 @@ def __init__(self, hs: "HomeServer", database: "DatabasePool"): # enum? self._all_done = False + # A set of background updates that we have queried the database for and + # found to be completed. + self._completed_background_updates: set[str] = set() + # Whether we're currently running updates self._running = False @@ -394,9 +398,15 @@ def get_current_update(self) -> BackgroundUpdatePerformance | None: return perf def start_doing_background_updates(self) -> None: + """Start doing background updates in the background. + + This gets called both on startup and when the admin API is used to + reschedule background updates. + """ if self.enabled: # if we start a new background update, not all updates are done. self._all_done = False + self._completed_background_updates.clear() sleep = self.sleep_enabled self.hs.run_as_background_process( "background_updates", @@ -478,6 +488,9 @@ async def has_completed_background_update(self, update_name: str) -> bool: if update_name == self._current_background_update: return False + if update_name in self._completed_background_updates: + return True + update_exists = await self.db_pool.simple_select_one_onecol( "background_updates", keyvalues={"update_name": update_name}, @@ -486,6 +499,9 @@ async def has_completed_background_update(self, update_name: str) -> bool: allow_none=True, ) + if not update_exists: + self._completed_background_updates.add(update_name) + return not update_exists async def have_completed_background_updates( diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 0e4c8ac491..69470ae6e6 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -83,6 +83,10 @@ # `device_lists_changes_in_room.inserted_ts`. BG_UPDATE_ADD_INSERTED_TS_INDEX = "device_lists_changes_in_room_inserted_ts_idx" +# Background update name for adding an index on unconverted rows in +# `device_lists_changes_in_room`. +BG_UPDATE_ADD_UNCONVERTED_IDX = "device_lists_changes_in_room_unconverted_idx" + # Prunes entries out of the `device_lists_changes_in_room` table that are more # than this old. @@ -2204,7 +2208,22 @@ async def mark_redundant_device_lists_pokes( converted_upto_stream_id: int, ) -> None: """If we've calculated the outbound pokes for a given room/device list - update, mark any subsequent changes as already converted""" + update, mark any subsequent changes as already converted. + + This is an optimization only. Skipping it is always safe, and just + means the subsequent changes get converted individually. + """ + + # Without the index added by `BG_UPDATE_ADD_UNCONVERTED_IDX`, the + # UPDATE below scans the unconverted backlog on every call, getting + # slower the further behind we are. Skip it until the index exists. + unconverted_idx_ready = ( + await self.db_pool.updates.has_completed_background_update( + BG_UPDATE_ADD_UNCONVERTED_IDX + ) + ) + if not unconverted_idx_ready: + return sql = """ UPDATE device_lists_changes_in_room @@ -2446,17 +2465,69 @@ async def get_device_change_last_converted_pos(self) -> tuple[int, str]: `FALSE` have not been converted. """ + return await self.db_pool.runInteraction( + desc="get_device_change_last_converted_pos", + func=self.get_device_change_last_converted_pos_txn, + db_autocommit=True, + ) + + def get_device_change_last_converted_pos_txn( + self, txn: LoggingTransaction + ) -> tuple[int, str]: + """Get the position of the last row in `device_list_changes_in_room` that has been + converted to `device_lists_outbound_pokes`. + + Rows with a strictly greater position where `converted_to_destinations` is + `FALSE` have not been converted.""" + # There should be only one row in this table, though we want to # future-proof ourselves for when we have multiple rows (one for each # instance). So to handle that case we take the minimum of all rows. - rows = await self.db_pool.simple_select_list( + rows = self.db_pool.simple_select_list_txn( + txn, table="device_lists_changes_converted_stream_position", keyvalues={}, retcols=["stream_id", "room_id"], - desc="get_device_change_last_converted_pos", ) return cast(tuple[int, str], min(rows)) + async def get_device_list_conversion_lag(self) -> tuple[int | None, int]: + """Get how far behind we are at converting rows in + `device_lists_changes_in_room` to `device_lists_outbound_pokes`. + + Returns: + A tuple of: + 1. the timestamp (ms) at which the oldest unconverted change + was inserted. None if there is nothing to convert, or if + the oldest row predates the `inserted_ts` column. + 2. the stream ID of the last converted position. + """ + + # Rows for one device list update share a `stream_id` (and insertion + # time), so ordering by `stream_id` alone is fine. + sql = """ + SELECT inserted_ts FROM device_lists_changes_in_room + WHERE + (stream_id, room_id) > (?, ?) AND + NOT converted_to_destinations + ORDER BY stream_id ASC + LIMIT 1 + """ + + def get_device_list_conversion_lag_txn( + txn: LoggingTransaction, + ) -> tuple[int | None, int]: + stream_id, room_id = self.get_device_change_last_converted_pos_txn(txn) + + txn.execute(sql, (stream_id, room_id)) + row = txn.fetchone() + return row[0] if row else None, stream_id + + return await self.db_pool.runInteraction( + "get_device_list_conversion_lag", + get_device_list_conversion_lag_txn, + ) + async def set_device_change_last_converted_pos( self, stream_id: int, @@ -2699,6 +2770,15 @@ def __init__( where_clause="inserted_ts IS NOT NULL", ) + # Add an index to speed up `mark_redundant_device_lists_pokes`. + self.db_pool.updates.register_background_index_update( + BG_UPDATE_ADD_UNCONVERTED_IDX, + index_name="device_lists_changes_in_room_unconverted_idx", + table="device_lists_changes_in_room", + columns=["user_id", "device_id", "room_id", "stream_id"], + where_clause="NOT converted_to_destinations", + ) + async def _drop_device_list_streams_non_unique_indexes( self, progress: JsonDict, batch_size: int ) -> int: diff --git a/synapse/storage/schema/main/delta/94/08_device_lists_changes_in_room_unconverted_idx.sql b/synapse/storage/schema/main/delta/94/08_device_lists_changes_in_room_unconverted_idx.sql new file mode 100644 index 0000000000..6bebcab11d --- /dev/null +++ b/synapse/storage/schema/main/delta/94/08_device_lists_changes_in_room_unconverted_idx.sql @@ -0,0 +1,22 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + + +-- Add an index on `device_lists_changes_in_room(user_id, device_id, room_id, +-- stream_id)` for unconverted rows, so that marking redundant rows as +-- converted (in `mark_redundant_device_lists_pokes`) does not require a scan +-- of the unconverted backlog. +-- +-- This is a partial index as we only ever query for unconverted rows. +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9408, 'device_lists_changes_in_room_unconverted_idx', '{}'); From ba3ab1876acd326fa5cc9b161e87b064ac54f3ed Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 18 Aug 2026 03:27:50 -0500 Subject: [PATCH 10/28] Optimize `mark_as_sent_devices_by_remote` to do less CPU work on the database (#20120) ### Background On `matrix.org`, since 2026-08-15, we are seeing the database CPU being saturated more than usual ([grafana](https://grafana.matrix.org/d/rYdddlPWk/node-exporter?orgId=1&from=2026-08-10T23:23:14.237Z&to=2026-08-17T23:23:14.237Z&timezone=browser&var-DS_PROMETHEUS=default&var-job=machine&var-node=matrix-db-01.matrix.org:9100&var-diskdevices=%5Ba-z%5D%2B%7Cnvme%5B0-9%5D%2Bn%5B0-9%5D%2B&refresh=1m&viewPanel=panel-77)) CPU of database server @reivilibre [found](https://matrix.to/#/!yHWhpxlXVaLcsgDUKb:matrix.org/$fUMf60IbFocpEuJNbmbEzFiCyKCFnAl4I4XpB27DUQs?via=banzan.uk&via=element.io&via=matrix.org) `mark_as_sent_devices_by_remote` spiking in the `DB transactions by total txn time` graph ([grafana](https://grafana.matrix.org/d/000000012/synapse?var-bucket_size=$__auto&orgId=1&from=2026-08-10T22:26:07.336Z&to=2026-08-17T22:26:07.336Z&timezone=browser&var-datasource=default&var-instance=matrix.org&var-job=synapse_federation_sender&var-index=$__all&showCategory=Thresholds&viewPanel=panel-11)) 'mark_as_sent_devices_by_remote' spiking in the 'DB transactions by total txn time' graph And we indeed see a bunch of time being spent on `mark_as_sent_devices_by_remote` by looking at `pg_stat_statements`. The top two queries we're spending CPU on are the `SELECT` and `DELETE` statements in [`mark_as_sent_devices_by_remote`](https://github.com/element-hq/synapse/blob/94a5f2afb36a4afdc36813fd1c24b9a1c4aec252/synapse/storage/databases/main/devices.py#L922-L950). We also did some related work in this area recently with https://github.com/element-hq/synapse/pull/20098 although it was tackling a different bottle-neck. ### This PR This PRs combines the two separate `SELECT` and `DELETE` queries which touch the same data into one `DELETE ... RETURNING ...` query. this means we get to save the cost of one of those statements (less CPU on the database) and fewer statements per transaction (less round-trips) means connections turn over faster, and can move on to process the next thing. This is a micro-optimization I spotted while reading [`_mark_as_sent_devices_by_remote_txn`](https://github.com/element-hq/synapse/blob/develop/synapse/storage/databases/main/devices.py#L922-L950) rather than a structural fix. I'm sure there are even better things to do to where we could even avoid this kind of work altogether but this seemed like a quick win especially given how much this particular transaction is saturating the database. Doing things faster doesn't necessarily mean we solve the saturated/starved CPU problem but it does mean the same task costs less CPU. From the `pg_stat_statements` samples above, we can expect to be up to ~27% more efficient with database CPU on this code path (derived from the `cpus_busy` numbers above `22.932 / (22.932 + 61.956)`). We're removing the `SELECT` (`61.956` `cpus_busy`) but I still expect the `DELETE` (`22.932` `cpus_busy`) to take on a similar cost as I'm sure it's a warm cache situation between them. --- changelog.d/20120.misc | 1 + synapse/storage/databases/main/devices.py | 55 ++++++++++++++++------- 2 files changed, 40 insertions(+), 16 deletions(-) create mode 100644 changelog.d/20120.misc diff --git a/changelog.d/20120.misc b/changelog.d/20120.misc new file mode 100644 index 0000000000..d59db030c6 --- /dev/null +++ b/changelog.d/20120.misc @@ -0,0 +1 @@ +Reduce database CPU usage when marking device list changes as sent over federation. diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 69470ae6e6..6cf9270ff2 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -922,33 +922,56 @@ async def mark_as_sent_devices_by_remote( def _mark_as_sent_devices_by_remote_txn( self, txn: LoggingTransaction, destination: str, stream_id: int ) -> None: - # We update the device_lists_outbound_last_success with the successfully - # poked users. + # Delete all sent outbound pokes, returning them so that we can update + # `device_lists_outbound_last_success` with the successfully poked users. + # + # This is a high frequency transaction (runs very often when processing a + # backlog of device list changes) and can bog down the database CPU with the + # sheer number of statements. + # + # We prefer to trade a little bit of processing time on the Python side + # (aggregating `max_stream_id_by_user_id`) as the alternative would be to have + # two separate queries; a `SELECT ... GROUP BY user_id` with the aggregation and + # then a `DELETE` which means we touch the same rows twice. We get to save the + # cost of one of those statements (less CPU on the database) and fewer + # statements per transaction (less round-trips) means connections turn over + # faster, and can move on to process the next thing. + # + # By the nature of `MAX_EDUS_PER_TRANSACTION`, we're only dealing with 100 rows + # at max which is pretty trivial for us to process on the Python side. sql = """ - SELECT user_id, coalesce(max(o.stream_id), 0) - FROM device_lists_outbound_pokes as o - WHERE destination = ? AND o.stream_id <= ? - GROUP BY user_id + DELETE FROM device_lists_outbound_pokes + WHERE destination = ? AND stream_id <= ? + RETURNING user_id, stream_id """ txn.execute(sql, (destination, stream_id)) - rows = txn.fetchall() + # Aggregate `max_stream_id_by_user_id` + max_stream_id_by_user_id: dict[str, int] = {} + for user_id, poke_stream_id in txn: + max_stream_id_by_user_id[user_id] = max( + max_stream_id_by_user_id.get(user_id, 0), poke_stream_id + ) + + # Update `device_lists_outbound_last_success` with the successfully poked + # users. + # + # We could potentially combine this in one big CTE with the query above but it + # isn't supported by SQLite (SQLite doesn't support `DELETE` in a CTE). self.db_pool.simple_upsert_many_txn( txn=txn, table="device_lists_outbound_last_success", key_names=("destination", "user_id"), - key_values=[(destination, user_id) for user_id, _ in rows], + key_values=[ + (destination, user_id) for user_id in max_stream_id_by_user_id.keys() + ], value_names=("stream_id",), - value_values=[(stream_id,) for _, stream_id in rows], + value_values=[ + (user_stream_id,) + for user_stream_id in max_stream_id_by_user_id.values() + ], ) - # Delete all sent outbound pokes - sql = """ - DELETE FROM device_lists_outbound_pokes - WHERE destination = ? AND stream_id <= ? - """ - txn.execute(sql, (destination, stream_id)) - async def add_user_signature_change_to_streams( self, from_user_id: str, user_ids: list[str] ) -> int: From 11de9503fb053dcc2ad47b8038a4b98631842950 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Thu, 20 Aug 2026 07:49:03 -0400 Subject: [PATCH 11/28] Update Rust dependencies for security fixes (#20131) --- Cargo.lock | 29 ++++++++++++++--------------- changelog.d/20131.misc | 1 + changelog.d/20131.misc.1 | 1 + rust/Cargo.toml | 6 +++--- rust/src/deferred.rs | 2 +- rust/src/storage/db/mod.rs | 3 ++- 6 files changed, 22 insertions(+), 20 deletions(-) create mode 100644 changelog.d/20131.misc create mode 100644 changelog.d/20131.misc.1 diff --git a/Cargo.lock b/Cargo.lock index 50daff2dd9..3cb739d495 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -843,9 +843,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" +checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" dependencies = [ "anyhow", "bytes", @@ -859,18 +859,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" +checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" +checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327" dependencies = [ "libc", "pyo3-build-config", @@ -889,9 +889,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" +checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -901,22 +901,21 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" +checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952" dependencies = [ "heck", "proc-macro2", - "pyo3-build-config", "quote", "syn 2.0.104", ] [[package]] name = "pythonize" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b79f670c9626c8b651c0581011b57b6ba6970bb69faf01a7c4c0cfc81c43f95" +checksum = "6ec376e1216e0c929a74964ce2020012a1a39f32d80e78aa688721219ea7fb89" dependencies = [ "pyo3", "serde", @@ -1193,9 +1192,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a" dependencies = [ "ring", "rustls-pki-types", diff --git a/changelog.d/20131.misc b/changelog.d/20131.misc new file mode 100644 index 0000000000..a90fd7d4ad --- /dev/null +++ b/changelog.d/20131.misc @@ -0,0 +1 @@ +Update rustls-webpki to address GHSA-82j2-j2ch-gfr8. diff --git a/changelog.d/20131.misc.1 b/changelog.d/20131.misc.1 new file mode 100644 index 0000000000..c0c211ed8c --- /dev/null +++ b/changelog.d/20131.misc.1 @@ -0,0 +1 @@ +Update pyo3 to address GHSA-36hh-v3qg-5jq4 and GHSA-chgr-c6px-7xpp. diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 612ab09f6d..f6b12c4c1b 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -31,7 +31,7 @@ http = "1.1.0" lazy_static = "1.4.0" log = "0.4.17" mime = "0.3.17" -pyo3 = { version = "0.28.3", features = [ +pyo3 = { version = "0.29.0", features = [ "macros", "anyhow", "abi3", @@ -40,8 +40,8 @@ pyo3 = { version = "0.28.3", features = [ # https://docs.rs/pyo3/latest/pyo3/bytes/index.html "bytes", ] } -pyo3-log = "0.13.3" -pythonize = { version = "0.28.0", features = ["arbitrary_precision"] } +pyo3-log = "0.13.4" +pythonize = { version = "0.29.0", features = ["arbitrary_precision"] } regex = "1.6.0" sha2 = "0.10.8" serde = { version = "1.0.144", features = ["derive", "rc"] } diff --git a/rust/src/deferred.rs b/rust/src/deferred.rs index 62a1ce6b90..3da5defb83 100644 --- a/rust/src/deferred.rs +++ b/rust/src/deferred.rs @@ -141,7 +141,7 @@ pub(crate) async fn run_python_awaitable( make_awaitable: F, ) -> PyResult> where - F: for<'py> Fn(Python<'py>) -> PyResult> + Send + 'static, + F: for<'py> Fn(Python<'py>) -> PyResult> + Send + Sync + 'static, { // Resolves when the awaitable completes; carries the resolved value or error. let (tx, rx) = oneshot::channel::>>(); diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs index fd40d52f08..175f66e76d 100644 --- a/rust/src/storage/db/mod.rs +++ b/rust/src/storage/db/mod.rs @@ -32,7 +32,7 @@ pub mod python_db_pool; /// It may be invoked multiple times under certain failure modes (serialization /// and deadlock errors), so it is `Fn` rather than `FnOnce`. pub type ErasedInteraction = - Box Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, ErasedResult> + Send>; + Box Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, ErasedResult> + Send + Sync>; /// The type-erased *result* of an [`ErasedInteraction`] /// [`DatabasePool::run_interaction_erased`]. @@ -114,6 +114,7 @@ pub trait DatabasePoolExt: DatabasePool { R: Send + 'static, F: for<'txn> Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, anyhow::Result> + Send + + Sync + 'static, { // Erase the concrete return type `R` into `Box` so we can call From d0b105757ed706c2a2cd543583561f4990b8345e Mon Sep 17 00:00:00 2001 From: Johannes Marbach Date: Thu, 20 Aug 2026 16:27:11 +0200 Subject: [PATCH 12/28] Add experimental support for MSC4502: Targeted and unrestricted room member queries (#19974) This is another stopgap towards https://github.com/element-hq/voip-internal/issues/641 and adds experimental support for [MSC4502](https://github.com/matrix-org/matrix-spec-proposals/pull/4502). This adds a new endpoint `/rooms/{roomId}/is_joined` to query if a user or server is joined to a room known to the homeserver. Access to the endpoint is guarded behind a new OAuth scope that can be assigned when registering application services. Tracking issue: https://github.com/element-hq/synapse/issues/20118 --------- Signed-off-by: Johannes Marbach --- changelog.d/19974.feature | 1 + rust/src/config/mod.rs | 1 + rust/src/handlers/versions.rs | 4 + synapse/api/auth/__init__.py | 8 +- synapse/api/auth/base.py | 18 +- synapse/appservice/__init__.py | 17 ++ synapse/config/appservice.py | 9 + synapse/config/experimental.py | 3 + synapse/rest/__init__.py | 2 + synapse/rest/client/room_membership.py | 103 ++++++++++ tests/appservice/test_appservice.py | 36 +++- tests/rest/client/test_room_membership.py | 235 ++++++++++++++++++++++ tests/rest/client/test_versions.py | 11 + tests/storage/test_appservice.py | 80 +++++++- 14 files changed, 520 insertions(+), 8 deletions(-) create mode 100644 changelog.d/19974.feature create mode 100644 synapse/rest/client/room_membership.py create mode 100644 tests/rest/client/test_room_membership.py diff --git a/changelog.d/19974.feature b/changelog.d/19974.feature new file mode 100644 index 0000000000..4a9290ea76 --- /dev/null +++ b/changelog.d/19974.feature @@ -0,0 +1 @@ +Add experimental support for MSC4502: Targeted and unrestricted room member queries. diff --git a/rust/src/config/mod.rs b/rust/src/config/mod.rs index 1c97373c27..db00197107 100644 --- a/rust/src/config/mod.rs +++ b/rust/src/config/mod.rs @@ -74,4 +74,5 @@ pub struct ExperimentalConfig { pub msc4491_enabled: bool, pub msc4143_enabled: bool, pub msc4446_enabled: bool, + pub msc4502_enabled: bool, } diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index 5d35b052bd..dc9e55e349 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -272,6 +272,9 @@ pub struct UnstableFeatureMap { /// MSC4446: Allow moving the fully read marker backwards. #[serde(rename = "com.beeper.msc4446")] msc4446_enabled: bool, + /// MSC4502: Targeted and unrestricted room member queries + #[serde(rename = "io.element.msc4502")] + msc4502: bool, // Whether new rooms will be set to encrypted or not (based on presets). #[serde(rename = "io.element.e2ee_forced.public")] @@ -324,6 +327,7 @@ pub fn synapse_config_to_global_unstable_feature_map( msc4491_enabled: config.experimental.msc4491_enabled, msc4143_enabled: config.experimental.msc4143_enabled, msc4446_enabled: config.experimental.msc4446_enabled, + msc4502: config.experimental.msc4502_enabled, e2ee_forced_public: config .room .encryption_enabled_by_default_for_room_presets diff --git a/synapse/api/auth/__init__.py b/synapse/api/auth/__init__.py index 201c295f06..d8d3b31b9d 100644 --- a/synapse/api/auth/__init__.py +++ b/synapse/api/auth/__init__.py @@ -24,7 +24,7 @@ from twisted.web.server import Request -from synapse.appservice import ApplicationService +from synapse.appservice import ApplicationService, Scopes from synapse.http.site import SynapseRequest from synapse.metrics import SERVER_NAME_LABEL from synapse.types import Requester @@ -205,3 +205,9 @@ async def check_user_in_room_or_world_readable( membership event ID of the user. If the user is not in the room and never has been, then `(Membership.JOIN, None)` is returned. """ + + def assert_requester_has_scope(self, requester: Requester, scope: Scopes) -> None: + """Asserts that the requester has the given scope, either directly + (e.g. via an OAuth token) or via the scopes registered against the + application service. + """ diff --git a/synapse/api/auth/base.py b/synapse/api/auth/base.py index 14e76b0cff..0b10640b50 100644 --- a/synapse/api/auth/base.py +++ b/synapse/api/auth/base.py @@ -19,6 +19,7 @@ # # import logging +from http import HTTPStatus from typing import TYPE_CHECKING from netaddr import IPAddress @@ -33,7 +34,7 @@ MissingClientTokenError, UnstableSpecAuthError, ) -from synapse.appservice import ApplicationService +from synapse.appservice import ApplicationService, Scopes from synapse.http import get_request_user_agent from synapse.http.site import SynapseRequest from synapse.logging.opentracing import trace @@ -362,6 +363,21 @@ async def get_appservice_user( effective_user_id, app_service=app_service, device_id=effective_device_id ) + def assert_requester_has_scope(self, requester: Requester, scope: Scopes) -> None: + """Asserts that the requester has the given scope, either directly + (e.g. via an OAuth token) or via the scopes registered against the + application service. + """ + if scope in requester.scope: + return + + if requester.app_service_id is not None: + app_service = self.store.get_app_service_by_id(requester.app_service_id) + if app_service is not None and app_service.has_scope(scope): + return + + raise AuthError(HTTPStatus.FORBIDDEN, f"Missing {scope} scope") + async def _record_request( self, request: SynapseRequest, requester: Requester ) -> None: diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index c55a83a879..19fff7f009 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -63,6 +63,14 @@ TransactionUnusedFallbackKeys = dict[str, dict[str, list[str]]] +class Scopes(str, Enum): + """ + All known scopes assignable to application services for extended privileges. + """ + + QUERY_ROOM_MEMBERSHIP = "urn:matrix:client:io.element.msc4502:rooms:is_joined" + + class ApplicationServiceState(Enum): DOWN = "down" UP = "up" @@ -104,6 +112,7 @@ def __init__( supports_unstable_ephemeral: bool = False, msc3202_transaction_extensions: bool = False, msc4190_device_management: bool = False, + scopes: Iterable[str] = frozenset(), ): self.token = token self.url = ( @@ -140,6 +149,11 @@ def __init__( else: self.protocols = set() + self.scopes = set(scopes) + unknown_scopes = self.scopes - frozenset(Scopes) + if unknown_scopes: + raise ValueError(f"Unknown application service scope(s): {unknown_scopes}") + self.rate_limited = rate_limited def _check_namespaces( @@ -379,6 +393,9 @@ def is_exclusive_user(self, user_id: str) -> bool: def is_interested_in_protocol(self, protocol: str) -> bool: return protocol in self.protocols + def has_scope(self, scope: Scopes) -> bool: + return scope in self.scopes + def is_exclusive_alias(self, alias: str) -> bool: return self._is_exclusive(ApplicationService.NS_ALIASES, alias) diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py index 7a629d10bf..4e61ef694f 100644 --- a/synapse/config/appservice.py +++ b/synapse/config/appservice.py @@ -199,6 +199,14 @@ def _load_appservice( "The `io.element.msc4190` option should be true or false if specified." ) + # Opt-in list of scopes granted to this appservice for restricted C-S API + # functionality. + scopes = as_info.get("io.element.msc4502.scopes", []) + if not isinstance(scopes, list) or not all(isinstance(s, str) for s in scopes): + raise ValueError( + "The `io.element.msc4502.scopes` option should be a list of strings if specified." + ) + return ApplicationService( token=as_info["as_token"], url=as_info["url"], @@ -213,4 +221,5 @@ def _load_appservice( supports_ephemeral=supports_ephemeral, msc3202_transaction_extensions=msc3202_transaction_extensions, msc4190_device_management=msc4190_enabled, + scopes=scopes, ) diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 6ad9f53517..97dc803ab7 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -203,6 +203,9 @@ def read_config( # See https://github.com/element-hq/synapse/issues/19524 self.msc4370_enabled = experimental.get("msc4370_enabled", False) + # MSC4502: Targeted and unrestricted room member queries + self.msc4502_enabled: bool = experimental.get("msc4502_enabled", False) + auth_delegated = (config.get("matrix_authentication_service") or {}).get( "enabled", False ) diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index a56a81a8e9..c8ede662aa 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -60,6 +60,7 @@ retention, room, room_keys, + room_membership, room_upgrade_rest_servlet, sendtodevice, sync, @@ -128,6 +129,7 @@ rendezvous.register_servlets, auth_metadata.register_servlets, thread_subscriptions.register_servlets, + room_membership.register_servlets, ) SERVLET_GROUPS: dict[str, Iterable[RegisterServletsFunc]] = { diff --git a/synapse/rest/client/room_membership.py b/synapse/rest/client/room_membership.py new file mode 100644 index 0000000000..026bee8214 --- /dev/null +++ b/synapse/rest/client/room_membership.py @@ -0,0 +1,103 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import logging +from http import HTTPStatus +from typing import TYPE_CHECKING + +from synapse.api.constants import EventTypes, Membership +from synapse.api.errors import Codes, SynapseError +from synapse.appservice import Scopes +from synapse.http.server import HttpServer +from synapse.http.servlet import RestServlet, parse_string +from synapse.http.site import SynapseRequest +from synapse.rest.client._base import client_patterns +from synapse.types import JsonDict, RoomID, UserID +from synapse.util.stringutils import parse_and_validate_server_name + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +class AppserviceRoomMembershipRestServlet(RestServlet): + PATTERNS = client_patterns( + r"/io\.element\.msc4502/rooms/(?P[^/]*)/is_joined$", releases=() + ) + CATEGORY = "Client API requests" + + def __init__(self, hs: "HomeServer"): + super().__init__() + self.auth = hs.get_auth() + self.store = hs.get_datastores().main + self.storage_controllers = hs.get_storage_controllers() + self.is_mine_id = hs.is_mine_id + + async def on_GET( + self, request: SynapseRequest, room_id: str + ) -> tuple[int, JsonDict]: + requester = await self.auth.get_user_by_req(request, allow_guest=False) + self.auth.assert_requester_has_scope(requester, Scopes.QUERY_ROOM_MEMBERSHIP) + + if not RoomID.is_valid(room_id): + raise SynapseError( + HTTPStatus.BAD_REQUEST, "Invalid room ID", Codes.INVALID_PARAM + ) + + mxid = parse_string(request, "mxid") + server_name = parse_string(request, "server_name") + + if (mxid is None) == (server_name is None): + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "Exactly one of 'mxid' or 'server_name' query parameters must be given", + Codes.MISSING_PARAM, + ) + + if mxid is not None: + if not UserID.is_valid(mxid): + raise SynapseError( + HTTPStatus.BAD_REQUEST, + f"Invalid MXID: {mxid}", + Codes.INVALID_PARAM, + ) + if self.is_mine_id(mxid): + joined = await self.store.check_local_user_in_room(mxid, room_id) + else: + event = await self.storage_controllers.state.get_current_state_event( + room_id, EventTypes.Member, mxid + ) + joined = ( + event is not None + and event.content.get("membership") == Membership.JOIN + ) + else: + assert server_name is not None + try: + parse_and_validate_server_name(server_name) + except ValueError: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + f"Invalid server name: {server_name}", + Codes.INVALID_PARAM, + ) + joined = await self.store.is_host_joined(room_id, server_name) + + return HTTPStatus.OK, {"joined": joined} + + +def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: + if hs.config.experimental.msc4502_enabled: + AppserviceRoomMembershipRestServlet(hs).register(http_server) diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py index 620c2b907b..3f124d9a2c 100644 --- a/tests/appservice/test_appservice.py +++ b/tests/appservice/test_appservice.py @@ -24,7 +24,11 @@ from twisted.internet import defer -from synapse.appservice import ApplicationService, Namespace +from synapse.appservice import ( + ApplicationService, + Namespace, + Scopes, +) from synapse.types import UserID from tests import unittest @@ -257,3 +261,33 @@ def test_member_list_match(self) -> Generator["defer.Deferred[Any]", object, Non ) ) ) + + +class ApplicationServiceScopesTestCase(unittest.TestCase): + def test_has_no_scopes_by_default(self) -> None: + service = ApplicationService( + id="unique_identifier", + sender=UserID.from_string("@as:test"), + token="some_token", + ) + self.assertEqual(len(service.scopes), 0) + self.assertFalse(service.has_scope(Scopes.QUERY_ROOM_MEMBERSHIP)) + + def test_has_valid_scope_if_specified(self) -> None: + service = ApplicationService( + id="unique_identifier", + sender=UserID.from_string("@as:test"), + token="some_token", + scopes=[Scopes.QUERY_ROOM_MEMBERSHIP], + ) + self.assertEqual(len(service.scopes), 1) + self.assertTrue(service.has_scope(Scopes.QUERY_ROOM_MEMBERSHIP)) + + def test_unknown_scope_raises(self) -> None: + with self.assertRaises(ValueError): + ApplicationService( + id="unique_identifier", + sender=UserID.from_string("@as:test"), + token="some_token", + scopes=["does:not:exist"], + ) diff --git a/tests/rest/client/test_room_membership.py b/tests/rest/client/test_room_membership.py new file mode 100644 index 0000000000..265c1f0f49 --- /dev/null +++ b/tests/rest/client/test_room_membership.py @@ -0,0 +1,235 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +from http import HTTPStatus +from unittest.mock import AsyncMock, patch + +from twisted.internet.testing import MemoryReactor + +from synapse.api.errors import Codes +from synapse.appservice import ApplicationService, Scopes +from synapse.rest import admin +from synapse.rest.client import login, room, room_membership +from synapse.server import HomeServer +from synapse.types import JsonDict, UserID, create_requester +from synapse.util.clock import Clock + +from tests import unittest +from tests.test_utils import event_injection +from tests.unittest import override_config + +AS_TOKEN = "i_am_an_app_service" +AS_TOKEN_NO_SCOPE = "i_am_an_app_service_without_scope" + + +class AppserviceRoomMembershipRestServletTestCase(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets_for_client_rest_resource, + login.register_servlets, + room.register_servlets, + room_membership.register_servlets, + ] + + def default_config(self) -> JsonDict: + config = super().default_config() + config["experimental_features"] = { + "msc4502_enabled": True, + # Merge in this order to allow `override_config` to override the flag + **config.get("experimental_features", {}), + } + return config + + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + self.creator = self.register_user("owner", "pass") + self.creator_tok = self.login("owner", "pass") + self.room_id = self.helper.create_room_as(self.creator, tok=self.creator_tok) + + self.joined_user = self.register_user("joined_user", "pass") + self.joined_user_tok = self.login("joined_user", "pass") + self.helper.join(self.room_id, self.joined_user, tok=self.joined_user_tok) + + self.not_joined_user = self.register_user("not_joined_user", "pass") + self.not_joined_user_tok = self.login("not_joined_user", "pass") + + self.remote_server = "elsewhere.com" + self.remote_user = UserID.from_string(f"@joined_user:{self.remote_server}") + self.get_success( + event_injection.inject_member_event( + self.hs, self.room_id, self.remote_user.to_string(), "join" + ) + ) + self.not_joined_remote_user = UserID.from_string( + f"@not_joined_user:{self.remote_server}" + ) + + self.unknown_server = "unknown.org" + self.unknown_room_id = "!unknown:unknown.org" + + main_store = self.hs.get_datastores().main + main_store.services_cache.append( + ApplicationService( + AS_TOKEN, + id="as_with_scope", + sender=UserID.from_string("@as:test"), + scopes=[Scopes.QUERY_ROOM_MEMBERSHIP], + ) + ) + main_store.services_cache.append( + ApplicationService( + AS_TOKEN_NO_SCOPE, + id="as_without_scope", + sender=UserID.from_string("@as2:test"), + ) + ) + + def _get_joined( + self, room_id: str, params: str, access_token: str | None + ) -> tuple[int, JsonDict]: + channel = self.make_request( + "GET", + f"/_matrix/client/unstable/io.element.msc4502/rooms/{room_id}/is_joined?{params}", + access_token=access_token, + ) + return channel.code, channel.json_body + + def test_invalid_room_id_format(self) -> None: + code, body = self._get_joined( + "not-a-room-id", f"mxid={self.joined_user}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.BAD_REQUEST, body) + self.assertEqual(body["errcode"], Codes.INVALID_PARAM) + + def test_both_mxid_and_server_name_given(self) -> None: + code, body = self._get_joined( + self.room_id, + f"mxid={self.joined_user}&server_name={self.hs.hostname}", + AS_TOKEN, + ) + self.assertEqual(code, HTTPStatus.BAD_REQUEST, body) + self.assertEqual(body["errcode"], Codes.MISSING_PARAM) + + def test_neither_mxid_nor_server_name_given(self) -> None: + code, body = self._get_joined(self.room_id, "", AS_TOKEN) + self.assertEqual(code, HTTPStatus.BAD_REQUEST, body) + self.assertEqual(body["errcode"], Codes.MISSING_PARAM) + + def test_invalid_mxid_format(self) -> None: + code, body = self._get_joined(self.room_id, "mxid=not-a-userid", AS_TOKEN) + self.assertEqual(code, HTTPStatus.BAD_REQUEST, body) + self.assertEqual(body["errcode"], Codes.INVALID_PARAM) + + def test_invalid_server_name_format(self) -> None: + code, body = self._get_joined(self.room_id, "server_name=foo_bar", AS_TOKEN) + self.assertEqual(code, HTTPStatus.BAD_REQUEST, body) + self.assertEqual(body["errcode"], Codes.INVALID_PARAM) + + def test_local_user_joined(self) -> None: + code, body = self._get_joined( + self.room_id, f"mxid={self.joined_user}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": True}) + + def test_local_user_not_joined(self) -> None: + code, body = self._get_joined( + self.room_id, f"mxid={self.not_joined_user}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": False}) + + def test_remote_user_joined(self) -> None: + code, body = self._get_joined( + self.room_id, f"mxid={self.remote_user.to_string()}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": True}) + + def test_remote_user_not_joined(self) -> None: + code, body = self._get_joined( + self.room_id, f"mxid={self.not_joined_remote_user.to_string()}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": False}) + + def test_local_server_name_joined(self) -> None: + code, body = self._get_joined( + self.room_id, f"server_name={self.hs.hostname}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": True}) + + def test_remote_server_name_joined(self) -> None: + code, body = self._get_joined( + self.room_id, f"server_name={self.remote_server}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": True}) + + def test_remote_server_name_not_joined(self) -> None: + code, body = self._get_joined( + self.room_id, f"server_name={self.unknown_server}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": False}) + + def test_nonexistent_room_returns_false(self) -> None: + code, body = self._get_joined( + self.unknown_room_id, f"server_name={self.unknown_server}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": False}) + + def test_no_token_unauthorized(self) -> None: + code, body = self._get_joined(self.room_id, f"mxid={self.joined_user}", None) + self.assertEqual(code, HTTPStatus.UNAUTHORIZED, body) + self.assertEqual(body["errcode"], Codes.MISSING_TOKEN) + + def test_normal_user_token_forbidden(self) -> None: + code, body = self._get_joined( + self.room_id, f"mxid={self.joined_user}", self.creator_tok + ) + self.assertEqual(code, HTTPStatus.FORBIDDEN, body) + self.assertEqual(body["errcode"], Codes.FORBIDDEN) + + def test_same_user_token_forbidden(self) -> None: + code, body = self._get_joined( + self.room_id, f"mxid={self.joined_user}", self.joined_user_tok + ) + self.assertEqual(code, HTTPStatus.FORBIDDEN, body) + self.assertEqual(body["errcode"], Codes.FORBIDDEN) + + def test_user_with_oauth_scope_allowed(self) -> None: + requester = create_requester(self.creator, scope={Scopes.QUERY_ROOM_MEMBERSHIP}) + with patch.object( + self.hs.get_auth(), "get_user_by_req", AsyncMock(return_value=requester) + ): + code, body = self._get_joined( + self.room_id, f"mxid={self.joined_user}", "doesnt-matter" + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": True}) + + def test_appservice_without_scope_forbidden(self) -> None: + code, body = self._get_joined( + self.room_id, f"mxid={self.joined_user}", AS_TOKEN_NO_SCOPE + ) + self.assertEqual(code, HTTPStatus.FORBIDDEN, body) + self.assertEqual(body["errcode"], Codes.FORBIDDEN) + + @override_config({"experimental_features": {"msc4502_enabled": False}}) + def test_unreachable_when_experimental_flag_disabled(self) -> None: + code, _ = self._get_joined(self.room_id, f"mxid={self.joined_user}", AS_TOKEN) + self.assertEqual(code, HTTPStatus.NOT_FOUND) diff --git a/tests/rest/client/test_versions.py b/tests/rest/client/test_versions.py index bbdbe38e07..1ed6bb145b 100644 --- a/tests/rest/client/test_versions.py +++ b/tests/rest/client/test_versions.py @@ -153,6 +153,17 @@ def test_msc4446_true_if_enabled(self) -> None: self.assertEqual(channel.code, 200, channel.result) self.assertTrue(channel.json_body["unstable_features"]["com.beeper.msc4446"]) + def test_msc4502_false_by_default(self) -> None: + channel = self.make_request("GET", "/_matrix/client/versions") + self.assertEqual(channel.code, 200, channel.result) + self.assertFalse(channel.json_body["unstable_features"]["io.element.msc4502"]) + + @unittest.override_config({"experimental_features": {"msc4502_enabled": True}}) + def test_msc4502_true_if_enabled(self) -> None: + channel = self.make_request("GET", "/_matrix/client/versions") + self.assertEqual(channel.code, 200, channel.result) + self.assertTrue(channel.json_body["unstable_features"]["io.element.msc4502"]) + def _sanity_check_versions_response(self, versions_response: JsonDict) -> None: """ Make sure this looks like a `/_matrix/client/versions` response diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index 4b9d069d6a..8430e54b92 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -21,7 +21,7 @@ import json import os import tempfile -from typing import cast +from typing import Any, cast from unittest.mock import AsyncMock, Mock import yaml @@ -29,7 +29,11 @@ from twisted.internet import defer from twisted.internet.testing import MemoryReactor -from synapse.appservice import ApplicationService, ApplicationServiceState +from synapse.appservice import ( + ApplicationService, + ApplicationServiceState, + Scopes, +) from synapse.config._base import ConfigError from synapse.events import EventBase from synapse.server import HomeServer @@ -38,7 +42,7 @@ ApplicationServiceStore, ApplicationServiceTransactionStore, ) -from synapse.types import DeviceListUpdates +from synapse.types import DeviceListUpdates, JsonDict from synapse.util.clock import Clock from tests import unittest @@ -479,8 +483,8 @@ def __init__( class ApplicationServiceStoreConfigTestCase(unittest.HomeserverTestCase): - def _write_config(self, suffix: str, **kwargs: str) -> str: - vals = { + def _write_config(self, suffix: str, **kwargs: Any) -> str: + vals: JsonDict = { "id": "id" + suffix, "url": "url" + suffix, "as_token": "as_token" + suffix, @@ -566,3 +570,69 @@ def test_duplicate_as_tokens(self) -> None: self.assertIn(f1, str(e)) self.assertIn(f2, str(e)) self.assertIn("as_token", str(e)) + + def test_invalid_scopes_raises(self) -> None: + f = self._write_config( + suffix="1", **{"io.element.msc4502.scopes": "not-a-list"} + ) + + self.hs.config.appservice.app_service_config_files = [f] + self.hs.config.caches.event_cache_size = 1 + + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + with self.assertRaises(ValueError): + ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) + + def test_known_scope_works(self) -> None: + f = self._write_config( + suffix="1", + **{"io.element.msc4502.scopes": [Scopes.QUERY_ROOM_MEMBERSHIP.value]}, + ) + + self.hs.config.appservice.app_service_config_files = [f] + self.hs.config.caches.event_cache_size = 1 + + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) + + def test_unknown_scope_raises(self) -> None: + f = self._write_config( + suffix="1", **{"io.element.msc4502.scopes": ["does:not:exist"]} + ) + + self.hs.config.appservice.app_service_config_files = [f] + self.hs.config.caches.event_cache_size = 1 + + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + with self.assertRaises(ValueError): + ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) From ea4218b16b30ee107ecd7c3b7f57257221846dec Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Thu, 20 Aug 2026 16:43:24 +0100 Subject: [PATCH 13/28] Schema diff CI: Fix hanging when Rust module changes (#20129) Follows: #20117 schema_diff: Move faketime so it doesn't affect `poetry install` Caused a hang in the CI for https://github.com/element-hq/synapse/pull/20003 whilst recompiling the Rust module (it seems) --------- Signed-off-by: Olivier 'reivilibre --- .ci/scripts/schema_diff.py | 6 ++++++ .github/workflows/schema_diff.yml | 5 ++--- changelog.d/20129.misc | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 changelog.d/20129.misc diff --git a/.ci/scripts/schema_diff.py b/.ci/scripts/schema_diff.py index 8354b64233..5dcc248f80 100755 --- a/.ci/scripts/schema_diff.py +++ b/.ci/scripts/schema_diff.py @@ -33,6 +33,12 @@ def run_make_full_schema(output_dir: Path) -> None: sys.exit(1) cmd: list[str] = [ + # Use faketime here for schema deltas that are wall-clock sensitive under SQLite + # We must only use faketime at this level because freezing the clock + # seems to cause `poetry install` to hang when recompiling our Rust module + "faketime", + "-f", + "2001-05-25 12:42:42", "poetry", "run", str(MAKE_FULL_SCHEMA_SCRIPT), diff --git a/.github/workflows/schema_diff.yml b/.github/workflows/schema_diff.yml index 3dfc02a469..58aa720b9a 100644 --- a/.github/workflows/schema_diff.yml +++ b/.github/workflows/schema_diff.yml @@ -31,6 +31,7 @@ jobs: background: true id: postgres # Use faketime here for schema deltas that are wall-clock sensitive under Postgres + # For SQLite, faketime is used when invoking `make_full_schema.sh` within the script run: | # Build a docker image with faketime mkdir /tmp/postgres-faketime @@ -78,10 +79,8 @@ jobs: PGHOST: localhost PGUSER: postgres PGPASSWORD: postgres - # Use faketime here for schema deltas that are wall-clock sensitive under SQLite run: | - faketime -f "2001-05-25 12:42:42" \ - poetry run python .ci/scripts/schema_diff.py \ + poetry run python .ci/scripts/schema_diff.py \ --base origin/develop \ > "${{ runner.temp }}/schema_diff.md" diff --git a/changelog.d/20129.misc b/changelog.d/20129.misc new file mode 100644 index 0000000000..92d421117e --- /dev/null +++ b/changelog.d/20129.misc @@ -0,0 +1 @@ +Fix the schema diff CI breaking when the Rust module was changed. \ No newline at end of file From 57cf536ee486b996309dd8b1e3dcd60664d25c37 Mon Sep 17 00:00:00 2001 From: Christian Paul Date: Fri, 21 Aug 2026 23:17:54 +0800 Subject: [PATCH 14/28] Update Synapse repo link in inconsistent stream error (#20128) --- changelog.d/20128.misc | 1 + synapse/storage/util/sequence.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20128.misc diff --git a/changelog.d/20128.misc b/changelog.d/20128.misc new file mode 100644 index 0000000000..1361eb0e5e --- /dev/null +++ b/changelog.d/20128.misc @@ -0,0 +1 @@ +Update Synapse repo link in inconsistent stream error. diff --git a/synapse/storage/util/sequence.py b/synapse/storage/util/sequence.py index 5bee3cf34f..0f3cfa7a7e 100644 --- a/synapse/storage/util/sequence.py +++ b/synapse/storage/util/sequence.py @@ -41,7 +41,7 @@ of '%(stream_name)s' in the 'stream_positions' table. This is likely a programming error and should be reported at -https://github.com/matrix-org/synapse. +https://github.com/element-hq/synapse. A temporary workaround to fix this error is to shut down Synapse (including any and all workers) and run the following SQL: From 3debaff0f86c1f34d5c5970bd5a10bea8a67b266 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 21 Aug 2026 18:56:03 +0300 Subject: [PATCH 15/28] Fix `__len__` of Sliding Sync `PerConnectionState` ignoring account data (#20124) Noticed while working on https://github.com/element-hq/synapse/pull/20003, submitting separately. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Eric Eastwood --- changelog.d/20124.misc | 1 + synapse/types/handlers/sliding_sync.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20124.misc diff --git a/changelog.d/20124.misc b/changelog.d/20124.misc new file mode 100644 index 0000000000..d92a12ce08 --- /dev/null +++ b/changelog.d/20124.misc @@ -0,0 +1 @@ +Fix cache `__len__` of Sliding Sync `PerConnectionState` ignoring account data entries. \ No newline at end of file diff --git a/synapse/types/handlers/sliding_sync.py b/synapse/types/handlers/sliding_sync.py index dd913250ba..d8c90c0fdd 100644 --- a/synapse/types/handlers/sliding_sync.py +++ b/synapse/types/handlers/sliding_sync.py @@ -919,6 +919,7 @@ class PerConnectionState: receipts: The status of each room for the receipts stream. room_configs: Map from room_id to the `RoomSyncConfig` of all rooms that we have previously sent down. + account_data: The status of each room for the account_data stream. """ last_used_ts: int | None = None @@ -951,7 +952,12 @@ def copy(self) -> "PerConnectionState": ) def __len__(self) -> int: - return len(self.rooms) + len(self.receipts) + len(self.room_configs) + return ( + len(self.account_data) + + len(self.rooms) + + len(self.receipts) + + len(self.room_configs) + ) @attr.s(auto_attribs=True) From 3db77e80a5a9bea0d36830906661f568314d04d0 Mon Sep 17 00:00:00 2001 From: catfromplan9 <104175360+catfromplan9@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:02:44 +0000 Subject: [PATCH 16/28] Fix WebP thumbnails losing their transparency (#20094) Signed-off-by: cat --- changelog.d/20094.bugfix | 1 + synapse/media/media_repository.py | 20 +++-- synapse/media/thumbnailer.py | 30 ++++++- tests/media/test_media_storage.py | 140 +++++++++++++++++++++++++++++- 4 files changed, 181 insertions(+), 10 deletions(-) create mode 100644 changelog.d/20094.bugfix diff --git a/changelog.d/20094.bugfix b/changelog.d/20094.bugfix new file mode 100644 index 0000000000..ebea9aaa04 --- /dev/null +++ b/changelog.d/20094.bugfix @@ -0,0 +1 @@ +Thumbnail WebP images that use transparency as PNG rather than JPEG, to preserve transparency during thumbnailing. Contributed by @catfromplan9. diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 180adc20a0..31bb401e65 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -1387,17 +1387,23 @@ async def _generate_thumbnails( self.hs.get_reactor(), thumbnailer.transpose ) + # JPEG has no alpha channel, so it would flatten a transparent + # image onto a solid color background. + needs_alpha = await defer_to_thread( + self.hs.get_reactor(), lambda: thumbnailer.has_transparency + ) + # We deduplicate the thumbnail sizes by ignoring the cropped versions if # they have the same dimensions of a scaled one. thumbnails: dict[tuple[int, int, str], str] = {} for requirement in requirements: + t_type = requirement.media_type + if needs_alpha and t_type == "image/jpeg": + t_type = "image/png" + if requirement.method == "crop": thumbnails.setdefault( - ( - requirement.width, - requirement.height, - requirement.media_type, - ), + (requirement.width, requirement.height, t_type), requirement.method, ) elif requirement.method == "scale": @@ -1406,9 +1412,7 @@ async def _generate_thumbnails( ) t_width = min(m_width, t_width) t_height = min(m_height, t_height) - thumbnails[(t_width, t_height, requirement.media_type)] = ( - requirement.method - ) + thumbnails[(t_width, t_height, t_type)] = requirement.method # Now we generate the thumbnails for each dimension, store it # diff --git a/synapse/media/thumbnailer.py b/synapse/media/thumbnailer.py index 27f016931a..a75a420750 100644 --- a/synapse/media/thumbnailer.py +++ b/synapse/media/thumbnailer.py @@ -23,7 +23,7 @@ from collections.abc import Callable from io import BytesIO from types import TracebackType -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast from PIL import Image, ImageSequence @@ -174,6 +174,34 @@ def _resize_image(self, image: Image.Image, width: int, height: int) -> Image.Im converted = image return converted.resize((width, height), Image.LANCZOS) + @property + def has_transparency(self) -> bool: + """Whether the current frame actually makes use of transparency. + + Having an alpha channel isn't enough: fully opaque RGBA is common. + + Note that this can block for a while on large images, as `getextrema()` + scans the entire alpha plane. Consider calling it via `defer_to_thread`. + """ + image = self.image + + if image.mode == "P" and "transparency" in image.info: + return True + + if "A" not in image.getbands(): + return False + + try: + with image.getchannel("A") as alpha: + # Single band, so `getextrema` returns a plain (min, max) pair. + min_alpha = cast(float, alpha.getextrema()[0]) + except Exception: + # Assume transparency, since dropping it is the destructive option. + logger.exception("Error inspecting image alpha channel") + return True + + return min_alpha < 255 + @property def is_animated(self) -> bool: if self._animation_broken: diff --git a/tests/media/test_media_storage.py b/tests/media/test_media_storage.py index 430a1d6789..da5fa8a140 100644 --- a/tests/media/test_media_storage.py +++ b/tests/media/test_media_storage.py @@ -62,7 +62,7 @@ from synapse.rest import admin from synapse.rest.client import login, media from synapse.server import HomeServer -from synapse.types import JsonDict, RoomAlias +from synapse.types import JsonDict, RoomAlias, UserID from synapse.util.clock import Clock from tests import unittest @@ -1444,6 +1444,15 @@ def _make_stale_mpo() -> bytes: return data[:primary_size] +def _make_webp(alpha: int) -> bytes: + """Build a small WebP whose pixels all have the given alpha.""" + out = BytesIO() + Image.new("RGBA", (64, 64), (255, 0, 0, alpha)).save( + out, format="WEBP", lossless=True + ) + return out.getvalue() + + class ThumbnailerAnimatedTestCase(unittest.TestCase): """Tests that the thumbnailer only animates when explicitly asked to.""" @@ -1589,3 +1598,132 @@ def test_undecodable_animation_falls_back_to_static(self, method: str) -> None: self.assertEqual(Image.open(out).format, "WEBP") self.assertFalse(getattr(Image.open(out), "is_animated", False)) self.assert_is_first_frame(out) + + +class ThumbnailerTransparencyTestCase(unittest.TestCase): + """Tests the transparency detection that picks the thumbnail format.""" + + def setUp(self) -> None: + super().setUp() + self.tempdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tempdir, ignore_errors=True) + + def _thumbnailer(self, name: str, data: bytes) -> Thumbnailer: + path = os.path.join(self.tempdir, name) + with open(path, "wb") as f: + f.write(data) + thumbnailer = Thumbnailer(path) + self.addCleanup(thumbnailer.close) + return thumbnailer + + def test_transparent_webp(self) -> None: + thumbnailer = self._thumbnailer("transparent.webp", _make_webp(0)) + self.assertTrue(thumbnailer.has_transparency) + + def test_partially_transparent_webp(self) -> None: + thumbnailer = self._thumbnailer("partial.webp", _make_webp(128)) + self.assertTrue(thumbnailer.has_transparency) + + def test_opaque_webp(self) -> None: + thumbnailer = self._thumbnailer("opaque.webp", _make_webp(255)) + self.assertFalse(thumbnailer.has_transparency) + + def test_unused_alpha_channel(self) -> None: + """An alpha channel that is fully opaque doesn't count as transparency.""" + out = BytesIO() + Image.new("RGBA", (64, 64), (255, 0, 0, 255)).save(out, format="PNG") + thumbnailer = self._thumbnailer("opaque_rgba.png", out.getvalue()) + self.assertEqual(thumbnailer.image.mode, "RGBA") + self.assertFalse(thumbnailer.has_transparency) + + def test_palette_transparency(self) -> None: + """Palette images signal transparency through an index, not a channel.""" + out = BytesIO() + Image.new("P", (64, 64)).save(out, format="PNG", transparency=0) + thumbnailer = self._thumbnailer("palette.png", out.getvalue()) + self.assertEqual(thumbnailer.image.mode, "P") + self.assertTrue(thumbnailer.has_transparency) + + def test_cmyk_jpeg(self) -> None: + thumbnailer = self._thumbnailer("opaque.jpg", SMALL_CMYK_JPEG) + self.assertFalse(thumbnailer.has_transparency) + + def test_png_thumbnail_keeps_alpha(self) -> None: + """The PNG we switch to actually retains the transparency.""" + thumbnailer = self._thumbnailer("transparent.webp", _make_webp(0)) + result = Image.open(thumbnailer.scale(32, 32, "image/png")) + self.assertEqual(result.format, "PNG") + pixel = result.convert("RGBA").getpixel((16, 16)) + assert isinstance(pixel, tuple) + self.assertEqual(pixel[3], 0) + + +class ThumbnailFormatTestCase(unittest.HomeserverTestCase): + """Tests that transparent sources aren't flattened onto a black background.""" + + servlets = [ + admin.register_servlets, + login.register_servlets, + media.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + self.media_repo = hs.get_media_repository() + self.user = self.register_user("user", "pass") + self.tok = self.login("user", "pass") + + def create_resource_dict(self) -> dict[str, Resource]: + resources = super().create_resource_dict() + resources["/_matrix/media"] = self.hs.get_media_repository_resource() + return resources + + def _upload(self, data: bytes, media_type: str) -> str: + """Upload the given media and return its media ID.""" + mxc = self.get_success( + self.media_repo.create_or_update_content( + media_type, + "test", + BytesIO(data), + len(data), + UserID.from_string(self.user), + ) + ) + return mxc.media_id + + def _thumbnail_types(self, media_id: str) -> set[str]: + thumbnails = self.get_success(self.store.get_local_media_thumbnails(media_id)) + self.assertTrue(thumbnails, "no thumbnails were generated") + return {thumbnail.type for thumbnail in thumbnails} + + def test_transparent_webp_thumbnails_as_png(self) -> None: + media_id = self._upload(_make_webp(0), "image/webp") + self.assertEqual(self._thumbnail_types(media_id), {"image/png"}) + + def test_opaque_webp_thumbnails_as_jpeg(self) -> None: + media_id = self._upload(_make_webp(255), "image/webp") + self.assertEqual(self._thumbnail_types(media_id), {"image/jpeg"}) + + def test_animated_thumbnail_is_still_webp(self) -> None: + """Transparency detection doesn't disturb the animated thumbnails.""" + media_id = self._upload(_make_animated_gif(), "image/gif") + self.assertIn(ANIMATED_THUMBNAIL_TYPE, self._thumbnail_types(media_id)) + + def test_served_thumbnail_keeps_transparency(self) -> None: + """The thumbnail a client actually receives still has its alpha channel.""" + media_id = self._upload(_make_webp(0), "image/webp") + + channel = self.make_request( + "GET", + f"/_matrix/client/v1/media/thumbnail/test/{media_id}" + "?width=32&height=32&method=scale", + shorthand=False, + access_token=self.tok, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.headers.getRawHeaders(b"Content-Type"), [b"image/png"]) + + thumbnail = Image.open(BytesIO(channel.result["body"])) + pixel = thumbnail.convert("RGBA").getpixel((16, 16)) + assert isinstance(pixel, tuple) + self.assertEqual(pixel[3], 0) From 4bb07152c8784a1052da87e94a5ea463e143956d Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 25 Aug 2026 17:37:03 +0300 Subject: [PATCH 17/28] Support for profile updates over Sliding Sync (MSC4262) (#20003) This PR implements support for profile updates over Sliding Sync: https://github.com/matrix-org/matrix-spec-proposals/pull/4262. This pr may be easier to review as a whole than commit by commit. This builds on the legacy sync profile updates feature https://github.com/element-hq/synapse/pull/19556, specifically the profile updates stream it added. Submitting for early review to get consensus on implementation. There are some things we would like to add still, from spec, mainly: * > Homeservers should only consider a profile field update "accepted" by a client > once the client returns with a new /sync request with the next /sync token, > NOT just after sending down the profile update. The client may never receive > response due to network conditions, or a bug in the client implementation. * > When a room enters this subset in this connection for the first time, all requested > fields from profiles of users in that room MAY be sent down. This gives the client > a base set of information for which future field updates can be applied on top of. > The homeserver MAY omit some fields and profiles if it believes that the client has > already received them, likewise repeat profiles MAY be sent down based on homeserver > implementation. * > Finally, if the list of fields expands to cover a new field ID, those fields should > be sent down for all users that are within the current room subset. Future incremental > updates will then include changes to this field. * Additionally, we would need to implement a lazy loading cache similar to the legacy sync. (not part of MSC as such) Depending on review these could either be added to this pr, or to keep this pr from not growing too much, be added in a follow-up pr, as they are more enhancement to this base sliding sync profile updates functionality than a part of the core functionality. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: Olivier 'reivilibre' Co-authored-by: Olivier 'reivilibre --- changelog.d/20003.feature | 2 + .../conf/workers-shared-extra.yaml.j2 | 2 +- .../configuration/config_documentation.md | 2 +- rust/src/handlers/versions.rs | 4 + schema/synapse-config.schema.yaml | 8 +- synapse/api/constants.py | 6 +- synapse/config/server.py | 2 +- synapse/handlers/profile.py | 9 +- synapse/handlers/sliding_sync/extensions.py | 406 +++++- synapse/rest/client/sync.py | 39 +- synapse/storage/databases/main/events.py | 6 +- synapse/storage/databases/main/profile.py | 77 +- synapse/storage/databases/main/roommember.py | 18 +- synapse/storage/schema/__init__.py | 2 +- .../main/delta/94/07_profile_updates.sql | 2 +- synapse/types/handlers/sliding_sync.py | 15 + synapse/types/rest/client/__init__.py | 15 + synapse/util/async_helpers.py | 27 + tests/handlers/test_profile.py | 22 +- tests/handlers/test_sync.py | 77 +- .../sliding_sync/test_extension_profiles.py | 1231 +++++++++++++++++ 21 files changed, 1874 insertions(+), 98 deletions(-) create mode 100644 changelog.d/20003.feature create mode 100644 tests/rest/client/sliding_sync/test_extension_profiles.py diff --git a/changelog.d/20003.feature b/changelog.d/20003.feature new file mode 100644 index 0000000000..add6b613c8 --- /dev/null +++ b/changelog.d/20003.feature @@ -0,0 +1,2 @@ +Add optional support for [MSC4262: Profile Updates for Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4262). +Currently defaults to not enabled, and is limited to local users only for the sync results. \ No newline at end of file diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2 index 64a36522fa..4dc4eb932b 100644 --- a/docker/complement/conf/workers-shared-extra.yaml.j2 +++ b/docker/complement/conf/workers-shared-extra.yaml.j2 @@ -15,7 +15,7 @@ enable_registration_without_verification: true bcrypt_rounds: 4 url_preview_enabled: true url_preview_ip_range_blacklist: [] -# MSC4429 Profile updates down legacy /sync +# MSC4429 and MSC4262 Profile updates down sync include_profile_updates_in_sync: true ## Registration ## diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 7654039e5c..00a31ff447 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -339,7 +339,7 @@ include_profile_data_on_invite: false ### `include_profile_updates_in_sync` *(boolean)* Use this option to include updates of other users' profiles in sync responses, for users who share rooms. -Requires an [MSC4429](https://github.com/matrix-org/matrix-spec-proposals/pull/4429) compatible client, and is currently limited to legacy sync and local users only. +For legacy sync clients, requires [MSC4429](https://github.com/matrix-org/matrix-spec-proposals/pull/4429) compatibility. For sliding sync clients, requires [MSC4262](https://github.com/matrix-org/matrix-spec-proposals/pull/4262) compatibility. Note, profile updates via sync are currently limited to local users only. This feature is under development and should be used with caution on busy servers or servers which depend on `limit_profile_requests_to_users_who_share_rooms` for ensuring profile information doesn't leak across rooms. Defaults to `false`. Example configuration: diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index dc9e55e349..ab4bd9a160 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -251,6 +251,9 @@ pub struct UnstableFeatureMap { /// MSC4169: Backwards-compatible redaction sending using `/send` #[serde(rename = "com.beeper.msc4169")] msc4169: bool, + /// MSC4262: Profile updates for simplified sliding sync. + #[serde(rename = "org.matrix.msc4262")] + msc4262: bool, /// MSC4354: Sticky events #[serde(rename = "org.matrix.msc4354")] msc4354: bool, @@ -320,6 +323,7 @@ pub fn synapse_config_to_global_unstable_feature_map( msc4155: config.experimental.msc4155_enabled, msc4306: config.experimental.msc4306_enabled, msc4169: config.experimental.msc4169_enabled, + msc4262: config.server.include_profile_updates_in_sync, msc4354: config.experimental.msc4354_enabled, msc4380: true, msc4429: config.server.include_profile_updates_in_sync, diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 6a6210fcb2..a8d1719288 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -281,9 +281,11 @@ properties: Use this option to include updates of other users' profiles in sync responses, for users who share rooms. - Requires an [MSC4429](https://github.com/matrix-org/matrix-spec-proposals/pull/4429) - compatible client, and is currently limited to legacy sync and local users only. - + For legacy sync clients, requires [MSC4429](https://github.com/matrix-org/matrix-spec-proposals/pull/4429) + compatibility. For sliding sync clients, requires + [MSC4262](https://github.com/matrix-org/matrix-spec-proposals/pull/4262) compatibility. Note, profile updates + via sync are currently limited to local users only. + This feature is under development and should be used with caution on busy servers or servers which depend on `limit_profile_requests_to_users_who_share_rooms` for ensuring profile information doesn't leak across rooms. diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 041a7f284a..8e91b2e203 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -441,6 +441,7 @@ class ProfileUpdateAction(str, enum.Enum): normally includes. This update action currently has no meaning for sync responses that are not incremental and non-lazy. """ + LEFT_ROOM = "left_room" """ This profile update row action represents a user leaving a room. @@ -450,9 +451,12 @@ class ProfileUpdateAction(str, enum.Enum): profiles, so clients can clear their cache containing the users profile data they are no longer interested in. """ + UPDATE = "update" """ - This profile update row action represents a user updating a profile field. + This profile update row action represents a user updating one or more + profile fields. + 'Updating' could mean creating, changing the value of, or deleting a field. Depending on the type of sync (initial/incremental, lazy/non-lazy), either the diff of profile field updates or all the current profile fields are included diff --git a/synapse/config/server.py b/synapse/config/server.py index 42d43ea7f6..00b414ea30 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -585,7 +585,7 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: " 'allow_public_rooms_over_federation' is set." ) - # Whether to support MSC4429 profile updates down legacy /sync + # Whether to support MSC4429 and MSC4262 Profile updates down sync self.include_profile_updates_in_sync = config.get( "include_profile_updates_in_sync", False, diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 5f2cec2366..3d052bc893 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -106,7 +106,9 @@ def __init__(self, hs: "HomeServer"): self._worker_locks = hs.get_worker_locks_handler() # Profile updates stream - self._msc4429_enabled = hs.config.server.include_profile_updates_in_sync + self._include_profile_updates_in_sync = ( + hs.config.server.include_profile_updates_in_sync + ) self._is_events_writer = ( hs.get_instance_name() in hs.config.worker.writers.events ) @@ -767,6 +769,9 @@ async def delete_profile_field( ) -> None: """Delete a field from a user's profile. + This should only be called for custom profile fields, + not displayname or avatar_url. + Preconditions: - This must NOT be called as part of deactivating the user, because we will notify modules about the change whilst claiming it is not related @@ -780,6 +785,8 @@ async def delete_profile_field( field_name: The name of the profile field to remove. by_admin: Whether this change was made by an administrator. """ + assert field_name not in (ProfileFields.DISPLAYNAME, ProfileFields.AVATAR_URL) + if not self.hs.is_mine(target_user): raise SynapseError(400, "User is not hosted on this homeserver") diff --git a/synapse/handlers/sliding_sync/extensions.py b/synapse/handlers/sliding_sync/extensions.py index b3342de778..7ee26079ed 100644 --- a/synapse/handlers/sliding_sync/extensions.py +++ b/synapse/handlers/sliding_sync/extensions.py @@ -25,20 +25,31 @@ from typing_extensions import TypeAlias, assert_never -from synapse.api.constants import AccountDataTypes, EduTypes, StickyEvent +from synapse.api.constants import ( + AccountDataTypes, + EduTypes, + EventTypes, + ProfileFields, + ProfileUpdateAction, + StickyEvent, +) from synapse.events.utils import FilteredEvent from synapse.handlers.receipts import ReceiptEventSource from synapse.logging.opentracing import trace from synapse.storage.databases.main.receipts import ReceiptInRoom from synapse.types import ( Absent, + AbsentType, DeviceListUpdates, + JsonDict, JsonMapping, + JsonValue, MultiWriterStreamToken, SlidingSyncStreamToken, StrCollection, StreamToken, ThreadSubscriptionsToken, + UserID, ) from synapse.types.handlers.sliding_sync import ( HaveSentRoomFlag, @@ -47,6 +58,7 @@ PerConnectionState, SlidingSyncConfig, SlidingSyncResult, + StateValues, ) from synapse.types.rest.client import SlidingSyncStickyEventsToken from synapse.util.async_helpers import ( @@ -80,6 +92,7 @@ def __init__(self, hs: "HomeServer"): self._storage_controllers = hs.get_storage_controllers() self._enable_thread_subscriptions = hs.config.experimental.msc4306_enabled self._enable_sticky_events = hs.config.experimental.msc4354_enabled + self._enable_profiles = hs.config.server.include_profile_updates_in_sync @trace async def get_extensions_response( @@ -197,6 +210,18 @@ async def get_extensions_response( from_token=from_token, ) + profiles_coro = None + if sync_config.extensions.profiles is not Absent and self._enable_profiles: + profiles_coro = self.get_profiles_extension_response( + sync_config=sync_config, + profiles_request=sync_config.extensions.profiles, + actual_room_ids=actual_room_ids, + to_token=to_token, + from_token=from_token, + actual_room_response_map=actual_room_response_map, + actual_lists=actual_lists, + ) + ( to_device_response, e2ee_response, @@ -205,6 +230,7 @@ async def get_extensions_response( typing_response, thread_subs_response, sticky_events_response, + profiles_response, ) = await gather_optional_coroutines( to_device_coro, e2ee_coro, @@ -213,6 +239,7 @@ async def get_extensions_response( typing_coro, thread_subs_coro, sticky_events_coro, + profiles_coro, ) return SlidingSyncResult.Extensions( @@ -223,6 +250,7 @@ async def get_extensions_response( typing=typing_response, thread_subscriptions=thread_subs_response, sticky_events=sticky_events_response, + profiles=profiles_response, ) def find_relevant_room_ids_for_extension( @@ -1055,3 +1083,379 @@ async def get_sticky_events_extension_response( sticky_events_stream_id=sticky_events_to_id ), ) + + async def _get_profile_ids_for_profiles_extension( + self, + user_id: str, + actual_room_ids: set[str], + sync_config: SlidingSyncConfig, + actual_room_response_map: Mapping[str, SlidingSyncResult.RoomResult], + actual_lists: Mapping[str, SlidingSyncResult.SlidingWindowList], + ) -> tuple[set[str], set[str]]: + """ + Calculate target user profiles as candiates to include in the profile + extension sync response. + + This function looks at both the sync config and the already calculated + rooms response, and pieces together the full set of user IDs to include + profiles for, based on sync config rooms being lazy loading or not. + + For rooms with lazy loading, only profiles for those users who have sent events + into the timeline will be included, unless they would be included otherwise. + For other rooms, all members of the room will be included as candidates. + + Note, this does not collect user IDs from the profile updates stream. + + Args: + user_id: The full user ID syncing. + actual_room_ids: The actual room IDs in the the Sliding Sync response. + sync_config: The Sliding Sync config object. + actual_room_response_map: A calculated map of responses per room. + actual_lists: Sliding window API. A map of list key to list results in the + Sliding Sync response. + + Returns: + Tuple containing two sets: + - first including all found user IDs, + - second containing user IDs calculated via lazy configured rooms. + """ + lazy_profile_user_ids = set() + non_lazy_profile_user_ids = set() + + # Separate rooms into lazy and non-lazy based on sync config. + # Look at subscriptions first + lazy_rooms = ( + { + room_id + for room_id, room_config in sync_config.room_subscriptions.items() + if (EventTypes.Member, StateValues.LAZY) in room_config.required_state + } + if sync_config.room_subscriptions + else set() + ) + # Iterate lists to find lazy rooms + if sync_config.lists: + for list_name, list_data in sync_config.lists.items(): + if (EventTypes.Member, StateValues.LAZY) in list_data.required_state: + for op in actual_lists[list_name].ops: + lazy_rooms.update(op.room_ids) + + if lazy_rooms: + # For rooms configured as lazy, include users based on room response. + for room_id, room_data in actual_room_response_map.items(): + if room_id not in lazy_rooms: + continue + # Include users from timeline events + for timeline_event in room_data.timeline_events: + lazy_profile_user_ids.add(timeline_event.event.sender) + # Include users from required state + for state_event in room_data.required_state: + if state_event.type == EventTypes.Member: + lazy_profile_user_ids.add(state_event.state_key) + # Include heroes + if room_data.heroes: + for hero in room_data.heroes: + lazy_profile_user_ids.add(hero.user_id) + + non_lazy_rooms = actual_room_ids.difference(lazy_rooms) + # If we still have non-lazy rooms, get their members. + if non_lazy_rooms: + non_lazy_profile_user_ids = ( + # TODO we should consider adding a limit to how many profiles + # of room members we push down the line. However, this produces + # a problem for clients in that they won't know which users + # just don't have any profile information, and which users were limited + # out. If we had an endpoint to fetch a list of profiles at once, + # we could have a hard limit here and clients could fetch the missing + # profiles separately for non-lazy initial sync cases. + await self.store.get_local_users_who_share_room_with_user( + user_id, + limit_to_rooms=non_lazy_rooms, + ) + ) + + # Unify the two lists + profile_user_ids = lazy_profile_user_ids.union(non_lazy_profile_user_ids) + + # Return a tuple containing the full list of user IDs and the lazy subset. + return ( + profile_user_ids, + lazy_profile_user_ids, + ) + + async def _get_profiles_extension_initial_sync_response( + self, + user_id: UserID, + fields: set[str] | None, + profile_user_ids: set[str], + ) -> dict[str, JsonDict]: + """ + Build an initial sync response for the profiles extension. + + Args: + user_id: The syncing user UserID + fields: A set of fields to include in the response. + `None` means all fields. + profile_user_ids: Set of user IDs whose profiles are related to this sync response. + + Returns: + A dictionary (in API response format) mapping users to their + profile updates in an `updated` dictionary. + + { + "@user:example.org": { + "updated": { + "displayname": "Somebody", + "avatar_url": "mxc://example.org/123123123", + "org.example.field": "hiss", + ... + } + }, + ... + } + """ + response: dict[str, JsonDict] = {} + + # This doesn't return entries for the users with no profile data, + # which is good as we don't want to generate anything for users + # with no profile data in initial sync. + profile_data_by_user = await self.store.get_profile_data_for_users( + # Force our own user to be in the set, as we should + # always watch our own profile updates + profile_user_ids | {user_id.to_string()} + ) + + # Serialise the profile updates into the sync response format. + for profile_user_id, profile_data in profile_data_by_user.items(): + per_user_updates: dict[str, JsonValue | dict[str, JsonValue]] + # Include the fields the client asked for, or all, if not specified + if fields is not None: + per_user_updates = { + k: v for k, v in profile_data.items() if k in fields + } + else: + per_user_updates = profile_data + + if per_user_updates: + response[profile_user_id] = { + "updated": per_user_updates, + } + + return response + + async def get_profiles_extension_response( + self, + sync_config: SlidingSyncConfig, + profiles_request: SlidingSyncConfig.Extensions.ProfilesExtension, + actual_room_ids: set[str], + to_token: StreamToken, + from_token: SlidingSyncStreamToken | None, + actual_room_response_map: Mapping[str, SlidingSyncResult.RoomResult], + actual_lists: Mapping[str, SlidingSyncResult.SlidingWindowList], + ) -> SlidingSyncResult.Extensions.ProfilesExtension | None: + """ + Generate a response for the profiles extension. + + Args: + sync_config: The Sliding Sync config. + profiles_request: The profiles extension request. + actual_room_ids: The actual room IDs in the the Sliding Sync response. + to_token: The stream token to generate a response until. + from_token: The stream token to generate a response from. + actual_room_response_map: A calculated map of responses per room. + actual_lists: Sliding window API. A map of list key to list results in the + Sliding Sync response. + + Returns: + - A SlidingSyncResult.Extensions.ProfilesExtension object containing + all the users who have profile updates. + - None if the extension is disabled. + """ + if not profiles_request.enabled: + return None + + user_id = sync_config.user.to_string() + fields = ( + set(profiles_request.fields) + if profiles_request.fields is not Absent + else None + ) + + response: dict[str, JsonDict | None] = {} + + ( + profile_user_ids, + lazy_profile_user_ids, + ) = await self._get_profile_ids_for_profiles_extension( + user_id=user_id, + actual_room_ids=actual_room_ids, + sync_config=sync_config, + actual_room_response_map=actual_room_response_map, + actual_lists=actual_lists, + ) + + if from_token is None: + # Initial sync + return SlidingSyncResult.Extensions.ProfilesExtension( + users=await self._get_profiles_extension_initial_sync_response( + user_id=sync_config.user, + fields=fields, + profile_user_ids=profile_user_ids, + ), + ) + + # Incremental sync + updates = await self.store.get_profile_updates_for_user_and_fields( + from_id=from_token.stream_token.profile_updates_key, + to_id=to_token.profile_updates_key, + user_id=user_id, + field_names=fields, + ) + + # Set of users that just joined their first room that we share with them + joined_room_user_ids: set[str] = set() + # Set of tracked users that have updated their profile + updated_user_ids: set[str] = set() + # Set of tracked users that just left their last room that we share with them + left_room_user_ids: set[str] = set() + + # Process updates in stream order + # We need to be careful of users that have multiple types of updates + # within this sequence of stream rows. + for update in updates: + if update.action == ProfileUpdateAction.JOINED_ROOM: + joined_room_user_ids.add(update.user_id) + # If the user joins a shared room, that overrides + # the fact that they previously left the last shared room + left_room_user_ids.discard(update.user_id) + elif update.action == ProfileUpdateAction.UPDATE: + updated_user_ids.add(update.user_id) + elif update.action == ProfileUpdateAction.LEFT_ROOM: + left_room_user_ids.add(update.user_id) + # If the user leaves their last shared room, that overrides + # the fact that they previously joined a shared room + # and perhaps updated their profile whilst they were in it + joined_room_user_ids.discard(update.user_id) + updated_user_ids.discard(update.user_id) + + # Add the users who joined a shared room or updated their profile to the set of + # users we will serialise profiles for + profile_user_ids.update(joined_room_user_ids) + profile_user_ids.update(updated_user_ids) + + # Process left rooms + for other_user_id in left_room_user_ids: + # Return a null response to the client + # This tells the client that it will no longer receive updates for the user + response[other_user_id] = None + + updated_user_fields: dict[str, set[str]] = {} + # Set fields from updates + for update in updates: + if ( + update.action != ProfileUpdateAction.UPDATE + or not update.affected_fields + or update.user_id in left_room_user_ids + # Skip if not interested in this user + or update.user_id not in profile_user_ids + ): + continue + interesting_changed_fields: set[str] + if fields is not None: + interesting_changed_fields = set(update.affected_fields) & fields + else: + interesting_changed_fields = set(update.affected_fields) + + if not interesting_changed_fields: + # Skip the update as the client is not interested in these fields + continue + + updated_user_fields.setdefault(update.user_id, set()).update( + interesting_changed_fields + ) + + profile_data_by_user = await self.store.get_profile_data_for_users( + profile_user_ids, + ) + + # Serialise the profile updates into the sync response format. + for profile_user_id in profile_user_ids: + if profile_user_id in left_room_user_ids: + continue + profile_data = profile_data_by_user.get(profile_user_id) + if profile_data is None: + # We don't have profile data for this user + # (This is different from having an empty profile) + # Return a null in incremental sync, telling the client to + # remove all profile information for this user. + response[profile_user_id] = None + continue + + # Calculate which fields had updates + updated_fields: set[str] = updated_user_fields.get(profile_user_id, set()) + # Calculate the full available field list + user_fields = set(profile_data.keys()).union(updated_fields) + + # If the user joined the room or is included via lazy loading events, + # include all fields the client wants. This happens because when lazy + # a room, clients will not necessarily have the profile for the user that + # sent an event in the room, and thus we deliver all the fields. The same + # is true if another user joins the room - we need to deliver an initial + # state for clients to work on. + # For non-lazy-loaded users, include only updated fields. We assume clients + # with non-lazy loaded rooms have received the profiles for all the members + # in the room, and thus only need updates. + user_fields = ( + user_fields + if profile_user_id in joined_room_user_ids + or profile_user_id in lazy_profile_user_ids + else updated_fields + ) + # Filter down if the client only wants a subset + if fields: + user_fields = user_fields.intersection(fields) + + if not user_fields: + continue + + per_user_updates: dict[str, JsonValue | dict[str, JsonValue]] = {} + per_user_removals: set[str] = set() + for field_name in user_fields: + # For custom fields the lack of a field means it will be `Absent`, + # for displayname/avatar_url it will be `None`, due to way we store + # things differently. + # FIXME: I intend to simplify this by pushing the special-case logic + # for these 'original' profile fields into the storage layer instead. + absent_type = ( + Absent + if field_name + not in (ProfileFields.DISPLAYNAME, ProfileFields.AVATAR_URL) + else None + ) + field_value: JsonValue | dict[str, JsonValue] | AbsentType = ( + profile_data.get(field_name, absent_type) + ) + if ( + # If the field isn't found on the profile and it is present in + # `updated_fields`, that means an existing field has been removed. + # We need the check against `updated_fields` as some profile fields + # are `None` by default, for example each and every user created + # by Synapse will have `avatar_url: None`, and we don't want to + # constantly send that to the clients. + field_value is absent_type and field_name in updated_fields + ): + per_user_removals.add(field_name) + else: + per_user_updates[field_name] = cast(JsonValue, field_value) + + if per_user_updates or per_user_removals: + entry: dict[str, JsonValue | JsonDict] = {} + response[profile_user_id] = entry + if per_user_updates: + entry["updated"] = per_user_updates + if per_user_removals: + entry["removed"] = list(per_user_removals) + + return SlidingSyncResult.Extensions.ProfilesExtension( + users=response, + ) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 4e437c5e98..08002a6708 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -55,7 +55,13 @@ from synapse.http.site import SynapseRequest from synapse.logging.opentracing import log_kv, set_tag, trace_with_opname from synapse.rest.admin.experimental_features import ExperimentalFeature -from synapse.types import JsonDict, Requester, SlidingSyncStreamToken, StreamToken +from synapse.types import ( + JsonDict, + JsonMapping, + Requester, + SlidingSyncStreamToken, + StreamToken, +) from synapse.types.rest.client import SlidingSyncBody from synapse.util.caches.lrucache import LruCache from synapse.util.cancellation import cancellable @@ -123,7 +129,9 @@ def __init__(self, hs: "HomeServer"): self._event_serializer = hs.get_event_client_serializer() self._msc2654_enabled = hs.config.experimental.msc2654_enabled self._msc3773_enabled = hs.config.experimental.msc3773_enabled - self._msc4429_enabled = hs.config.server.include_profile_updates_in_sync + self._include_profile_updates_in_sync = ( + hs.config.server.include_profile_updates_in_sync + ) self._json_filter_cache: LruCache[str, bool] = LruCache( max_size=1000, @@ -352,7 +360,7 @@ async def encode_response( if sync_result.to_device: response["to_device"] = {"events": sync_result.to_device} - if self._msc4429_enabled and sync_result.profile_updates: + if self._include_profile_updates_in_sync and sync_result.profile_updates: # FIXME: See issue https://github.com/element-hq/synapse/issues/19981 # for concerns around the current implementation of the profile # updates stream. @@ -1142,8 +1150,33 @@ async def encode_extensions( requester, extensions.sticky_events, ref_rooms_results ) + if extensions.profiles: + serialized_extensions[ + "org.matrix.msc4262.profiles" + ] = await self._serialise_profiles( + extensions.profiles, + ) + return serialized_extensions + async def _serialise_profiles( + self, + profiles: SlidingSyncResult.Extensions.ProfilesExtension, + ) -> JsonMapping: + """ + Serialise the profiles extension response. + + Args: + profiles: The generated profiles response object. + + Returns: + A dictionary containing the response `users` with the + generated profile updates. + """ + return { + "users": profiles.users, + } + async def _serialise_sticky_events( self, requester: Requester, diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 8211612e2d..35f387576f 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -269,7 +269,9 @@ def __init__( self._clock = hs.get_clock() self._instance_name = hs.get_instance_name() self._msc4354_enabled = hs.config.experimental.msc4354_enabled - self._msc4429_enabled = hs.config.server.include_profile_updates_in_sync + self._include_profile_updates_in_sync = ( + hs.config.server.include_profile_updates_in_sync + ) self._ephemeral_messages_enabled = hs.config.server.enable_ephemeral_messages self.is_mine_id = hs.is_mine_id @@ -2121,7 +2123,7 @@ def _update_current_state_txn( txn, {m for m in members_to_cache_bust if not self.hs.is_mine_id(m)} ) - if self._msc4429_enabled: + if self._include_profile_updates_in_sync: # Handle changes to the profile updates stream. # We've already done a bunch of work calculating the changes needed # for the sliding sync tables, so we may as well re-use that information diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 05faad5b26..d5e35fa7ca 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -86,7 +86,9 @@ def __init__( "populate_full_user_id_profiles", self.populate_full_user_id_profiles ) - self._msc4429_enabled = hs.config.server.include_profile_updates_in_sync + self._include_profile_updates_in_sync = ( + hs.config.server.include_profile_updates_in_sync + ) self._is_events_writer = self._instance_name in hs.config.worker.writers.events self._profile_updates_id_gen: MultiWriterIdGenerator = MultiWriterIdGenerator( db_conn=db_conn, @@ -403,6 +405,7 @@ def _get_updated_profile_updates_txn( "get_updated_profile_updates", _get_updated_profile_updates_txn ) + # FIXME this function should be deleted, it's not used. async def get_profile_updates_for_fields( self, *, @@ -500,7 +503,7 @@ async def get_profile_updates_for_user_and_fields( from_id: int, to_id: int, user_id: str, - field_names: Set[str], + field_names: Set[str] | None, include_users: set[str] | None = None, ) -> list[ProfileUpdate]: """Get profile update markers for a user in a stream range. @@ -515,15 +518,16 @@ async def get_profile_updates_for_user_and_fields( to_id: The ending stream ID (inclusive). user_id: The full user ID to filter on. field_names: Set of field names to filter update actions against. + `None` means "include all fields". include_users: If given, only include updates for these user IDs. Returns: - A list of ProfileUpdates update rows. + A list of ProfileUpdate update rows, in stream order """ if from_id >= to_id: return [] - if len(field_names) == 0: + if field_names is not None and len(field_names) == 0: return [] if include_users is not None and len(include_users) == 0: @@ -533,22 +537,27 @@ async def get_profile_updates_for_user_and_fields( def _get_profile_updates_for_user_and_fields_txn( txn: LoggingTransaction, ) -> list[ProfileUpdate]: - wanted_field_in_elems_clause, wanted_field_in_elems_args = ( - make_in_list_sql_clause( + # Build a `field_clause` that matches updates containing the fields we are interested in + if field_names is None: + # We are interested in all fields, so match any update with fields + field_clause = "pu.affected_fields IS NOT NULL" + field_args: list[str] = [] + else: + wanted_field_in_elems_clause, field_args = make_in_list_sql_clause( txn.database_engine, "field_names.value", field_names ) - ) - if isinstance(txn.database_engine, PostgresEngine): - # Note that if we had a GIN index on `affected_fields`, this would defeat it. - # If we decide we want one, we should consider using the `?|` operator or its - # clearer-named `jsonb_exists_any` equivalent. - all_field_names_table_expression = "jsonb_array_elements_text(pu.affected_fields) AS field_names(value)" - else: - # json_each is a table-valued function that gives `value` as one of its column names - all_field_names_table_expression = ( - "json_each(pu.affected_fields) AS field_names" - ) + if isinstance(txn.database_engine, PostgresEngine): + # Note that if we had a GIN index on `affected_fields`, this would defeat it. + # If we decide we want one, we should consider using the `?|` operator or its + # clearer-named `jsonb_exists_any` equivalent. + all_field_names_table_expression = "jsonb_array_elements_text(pu.affected_fields) AS field_names(value)" + else: + # json_each is a table-valued function that gives `value` as one of its column names + all_field_names_table_expression = ( + "json_each(pu.affected_fields) AS field_names" + ) + field_clause = f"(EXISTS (SELECT 1 FROM {all_field_names_table_expression} WHERE {wanted_field_in_elems_clause}))" user_clause = "" user_args: list[str] = [] @@ -573,7 +582,7 @@ def _get_profile_updates_for_user_and_fields_txn( AND puf.user_id = ? {user_clause} AND ( - (EXISTS (SELECT 1 FROM {all_field_names_table_expression} WHERE {wanted_field_in_elems_clause})) + {field_clause} OR pu.action != ? ) ORDER BY pu.stream_id ASC @@ -583,7 +592,7 @@ def _get_profile_updates_for_user_and_fields_txn( to_id, user_id, *user_args, - *wanted_field_in_elems_args, + *field_args, ProfileUpdateAction.UPDATE.value, ), ) @@ -591,18 +600,20 @@ def _get_profile_updates_for_user_and_fields_txn( updates: list[ProfileUpdate] = [] for stream_id, updated_user_id, action, affected_fields_dbjson in rows: + if affected_fields_dbjson is not None: + # Get the field names that were affected by this update + affected_fields = frozenset(db_to_json(affected_fields_dbjson)) + if field_names is not None: + # Only include the field names that we care about + affected_fields &= field_names + else: + affected_fields = None updates.append( ProfileUpdate( stream_id=stream_id, user_id=updated_user_id, action=action, - affected_fields=( - # Get the field names that were affected by this update - # and intersect with the field names we care about - frozenset(db_to_json(affected_fields_dbjson)) & field_names - ) - if affected_fields_dbjson is not None - else None, + affected_fields=affected_fields, ) ) @@ -755,7 +766,7 @@ def _set_profile_field_txn( Returns: The profile updates stream ID that was created in this transaction """ - if self._msc4429_enabled: + if self._include_profile_updates_in_sync: assert self._is_events_writer self._check_profile_size(txn, user_id, field_name, new_value) @@ -818,7 +829,7 @@ def _set_profile_field_txn( ), ) - if not self._msc4429_enabled: + if not self._include_profile_updates_in_sync: return None # Record updates in the profile updates stream @@ -847,7 +858,7 @@ def record_profile_updates_for_user_joined_room_txn( users profile should be pushed to the client, should they need it already even if the user hasn't actually joined the room. """ - if not self._msc4429_enabled: + if not self._include_profile_updates_in_sync: return assert self._is_events_writer @@ -884,7 +895,7 @@ def record_profile_updates_txn( txn: Transaction to use user_id: User ID that made the profile update action: The profile update action, either `update`, `left_room` or - `joined_room` + `joined_room`. field_names: A list of fields that were set, if ProfileUpdateAction.UPDATE user_rooms: Optionally, a set of rooms that the update concerns. If not given, a database lookup will be done to fetch all the users rooms. @@ -895,7 +906,7 @@ def record_profile_updates_txn( Returns: The latest stream ID created in this transaction """ - if not self._msc4429_enabled: + if not self._include_profile_updates_in_sync: return None if action == ProfileUpdateAction.UPDATE: @@ -1010,7 +1021,7 @@ async def delete_profile_field( field_name: The name of the custom profile field. """ - if self._msc4429_enabled: + if self._include_profile_updates_in_sync: assert self._is_events_writer def delete_profile_field(txn: LoggingTransaction) -> int | None: @@ -1032,7 +1043,7 @@ def delete_profile_field(txn: LoggingTransaction) -> int | None: (f'$."{field_name}"', user_id.localpart), ) - if not self._msc4429_enabled: + if not self._include_profile_updates_in_sync: return None stream_id = self.record_profile_updates_txn( diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 499b114cb6..72ac8c7ed2 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -1041,12 +1041,26 @@ async def get_users_who_share_room_with_user( return user_who_share_room - async def get_local_users_who_share_room_with_user(self, user_id: str) -> set[str]: - """Returns the set of local users who share a room with `user_id`. + async def get_local_users_who_share_room_with_user( + self, + user_id: str, + limit_to_rooms: set[str] | None = None, + ) -> set[str]: + """ + Returns the set of local users who share a room with `user_id`. This also includes the `user_id` themselves. + + Args: + user_id: The user ID to find the local users who share rooms. + limit_to_rooms: Optional set of rooms to limit to. + + Returns: + Set of local user ID's who share a room with the given user. """ room_ids = await self.get_rooms_for_user(user_id) + if limit_to_rooms is not None: + room_ids = room_ids.intersection(limit_to_rooms) user_who_share_room: set[str] = set() for room_id in room_ids: diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 3495dce866..0d9022116d 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -175,7 +175,7 @@ Changes in SCHEMA_VERSION = 94 - Add `recheck` column (boolean, default true) to the `redactions` table. - MSC4242: Add state DAG tables. - - MSC4429: Track updates to user profile fields via a new stream. + - MSC4429/MSC4262: Track updates to user profile fields via a new stream. """ diff --git a/synapse/storage/schema/main/delta/94/07_profile_updates.sql b/synapse/storage/schema/main/delta/94/07_profile_updates.sql index 720f958e09..e053421be3 100644 --- a/synapse/storage/schema/main/delta/94/07_profile_updates.sql +++ b/synapse/storage/schema/main/delta/94/07_profile_updates.sql @@ -12,7 +12,7 @@ -- . -- Track updates to profile fields. --- For MSC4429 legacy /sync and others. +-- For MSC4429 and MSC4262 down sync and others. -- See https://github.com/element-hq/synapse/issues/19981 for potential future directions of this table. CREATE TABLE IF NOT EXISTS profile_updates ( stream_id BIGINT NOT NULL PRIMARY KEY, diff --git a/synapse/types/handlers/sliding_sync.py b/synapse/types/handlers/sliding_sync.py index d8c90c0fdd..f6d30460b5 100644 --- a/synapse/types/handlers/sliding_sync.py +++ b/synapse/types/handlers/sliding_sync.py @@ -442,6 +442,19 @@ class StickyEventsExtension: def __bool__(self) -> bool: return bool(self.room_id_to_sticky_events) + @attr.s(slots=True, frozen=True, auto_attribs=True) + class ProfilesExtension: + """The Profile Updates extension (MSC4262) + + Attributes: + users: map (user_id -> [profile_updates]) + """ + + users: Mapping[str, JsonMapping | None] + + def __bool__(self) -> bool: + return bool(self.users) + to_device: ToDeviceExtension | None = None e2ee: E2eeExtension | None = None account_data: AccountDataExtension | None = None @@ -449,6 +462,7 @@ def __bool__(self) -> bool: typing: TypingExtension | None = None thread_subscriptions: ThreadSubscriptionsExtension | None = None sticky_events: StickyEventsExtension | None = None + profiles: ProfilesExtension | None = None def __bool__(self) -> bool: """Are there any updates that should be returned immediately to @@ -461,6 +475,7 @@ def __bool__(self) -> bool: or self.typing or self.thread_subscriptions or self.sticky_events + or self.profiles ) next_pos: SlidingSyncStreamToken diff --git a/synapse/types/rest/client/__init__.py b/synapse/types/rest/client/__init__.py index a7cb4d8b08..353367c0d7 100644 --- a/synapse/types/rest/client/__init__.py +++ b/synapse/types/rest/client/__init__.py @@ -478,6 +478,18 @@ class StickyEventsExtension(RequestBodyModel): limit: NonNegativeStrictInt = 100 since: SlidingSyncStickyEventsToken | AbsentType = Absent + class ProfilesExtension(RequestBodyModel): + """The Profile Updates extension (MSC4262) + + Attributes: + enabled + fields: List of fields to filter upon (optional) + """ + + enabled: StrictBool = False + # Optionally filter on specific fields + fields: list[StrictStr] | AbsentType = Absent + to_device: ToDeviceExtension | None = None e2ee: E2eeExtension | None = None account_data: AccountDataExtension | None = None @@ -489,6 +501,9 @@ class StickyEventsExtension(RequestBodyModel): sticky_events: StickyEventsExtension | AbsentType = Field( Absent, alias="org.matrix.msc4354.sticky_events" ) + profiles: ProfilesExtension | AbsentType = Field( + Absent, alias="org.matrix.msc4262.profiles" + ) conn_id: StrictStr | None = None lists: ( diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 5211bcc8e1..4ef1620a84 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -391,6 +391,7 @@ async def yieldable_gather_results_delaying_cancellation( T5 = TypeVar("T5") T6 = TypeVar("T6") T7 = TypeVar("T7") +T8 = TypeVar("T8") @overload @@ -544,6 +545,32 @@ async def gather_optional_coroutines( ]: ... +@overload +async def gather_optional_coroutines( + *coroutines: Unpack[ + tuple[ + Coroutine[Any, Any, T1] | None, + Coroutine[Any, Any, T2] | None, + Coroutine[Any, Any, T3] | None, + Coroutine[Any, Any, T4] | None, + Coroutine[Any, Any, T5] | None, + Coroutine[Any, Any, T6] | None, + Coroutine[Any, Any, T7] | None, + Coroutine[Any, Any, T8] | None, + ] + ], +) -> tuple[ + T1 | None, + T2 | None, + T3 | None, + T4 | None, + T5 | None, + T6 | None, + T7 | None, + T8 | None, +]: ... + + async def gather_optional_coroutines( *coroutines: Unpack[tuple[Coroutine[Any, Any, T1] | None, ...]], ) -> tuple[T1 | None, ...]: diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index c53c04fbc8..590c7e62f5 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -199,13 +199,13 @@ def test_update_room_membership_on_set_displayname(self) -> None: ["m.status", '{"text": "Holiday", "emoji": "🏖"}'], ] ) - def test_update_profile_does_not_update_stream_on_set_field_if_msc4429_not_enabled( + def test_update_profile_does_not_update_stream_on_set_field_if_include_profile_updates_in_sync_not_enabled( self, field_name: str, new_value: str, ) -> None: """Test that profile updates don't get recorded in the profile updates stream - if MSC4429 is not enabled.""" + if `include_profile_updates_in_sync` is not enabled.""" self.get_success( self.handler.set_field( target_user=self.frank, @@ -230,13 +230,13 @@ def test_update_profile_does_not_update_stream_on_set_field_if_msc4429_not_enabl ["m.status", '{"text": "Holiday", "emoji": "🏖"}'], ] ) - def test_update_profile_does_not_notify_notifier_on_set_field_if_msc4429_not_enabled( + def test_update_profile_does_not_notify_notifier_on_set_field_if_include_profile_updates_in_sync_not_enabled( self, field_name: str, new_value: str, ) -> None: """Test that profile updates do not cause the profile updates stream notifier - to wake up if MSC4429 is not enabled.""" + to wake up if `include_profile_updates_in_sync` is not enabled.""" self.get_success( self.handler.set_field( target_user=self.frank, @@ -265,7 +265,8 @@ def test_update_profile_does_not_notify_notifier_on_set_field_if_user_not_in_roo self, field_name: str, new_value: str ) -> None: """Test that profile updates do not cause the profile updates stream notifier - to wake up if the user is not in any rooms, if MSC4429 is enabled.""" + to wake up if the user is not in any rooms, if `include_profile_updates_in_sync` + is enabled.""" self.get_success( self.handler.set_field( target_user=self.frank, @@ -293,7 +294,7 @@ def test_update_profile_updates_stream_on_set_field( self, field_name: str, new_value: str ) -> None: """Test that profile updates get recorded in the profile updates stream if - MSC4429 is enabled.""" + `include_profile_updates_in_sync` is enabled.""" self.get_success( self.handler.set_field( target_user=self.frank, @@ -320,6 +321,8 @@ def test_update_profile_updates_stream_on_set_field( ) fields_updates = self.get_success( + # FIXME this function should be deleted, it's not used. + # Adapt this test to use the right one. self.store.get_profile_updates_for_fields( from_id=1, to_id=2, @@ -361,7 +364,7 @@ def test_update_profile_set_field_writes_to_per_user_profile_tracking_table( self, ) -> None: """Test that profiles updates get recorded in the 'per user' profile updates - stream tracking table, if MSC4429 is enabled.""" + stream tracking table, if `include_profile_updates_in_sync` is enabled.""" self.register_user("roger", "password") roger_token = self.login("roger", "password") self.register_user("millie", "password") @@ -501,7 +504,8 @@ def test_previous_profile_updates_stream_rows_cleared_if_no_longer_sharing_a_roo self, ) -> None: """Test that previous profile update stream rows are removed for a user if - the user no longer shares rooms with another user, if MSC4429 is enabled. + the user no longer shares rooms with another user, if + `include_profile_updates_in_sync` is enabled. This test ensures that when a user leaves a room, we clear all old profile update rows of users who the user no longer shares rooms with, to avoid @@ -668,7 +672,7 @@ def test_update_profile_notifies_notifier_on_set_field( new_value: str, ) -> None: """Test that profile updates wake up the profile updates stream on profile - field updates, if MSC4429 is enabled.""" + field updates, if `include_profile_updates_in_sync` is enabled.""" self.helper.create_room_as( room_creator=self.frank.to_string(), tok=self.frank_token, diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index bfb687a6c7..5771e60533 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -1187,8 +1187,8 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: ) def test_initial_sync_no_profile_updates_if_not_enabled(self) -> None: - """Test that without MSC4429 enabled the initial sync response does not - contain any profile updates.""" + """Test that without `include_profile_updates_in_sync` enabled the initial sync + response does not contain any profile updates.""" self.get_success( self.profile_handler.set_field( target_user=UserID.from_string(self.other_user), @@ -1212,8 +1212,8 @@ def test_initial_sync_no_profile_updates_if_not_enabled(self) -> None: @override_config({"include_profile_updates_in_sync": True}) def test_initial_sync_no_profile_updates_if_not_filtered_for(self) -> None: - """Test that with MSC4429 enabled the initial sync response does not - contain any profile updates, if fields are not filtered for.""" + """Test that with `include_profile_updates_in_sync` enabled the initial sync + response does not contain any profile updates, if fields are not filtered for.""" self.get_success( self.profile_handler.set_field( target_user=UserID.from_string(self.other_user), @@ -1240,9 +1240,9 @@ def test_initial_sync_no_profile_updates_if_not_filtered_for(self) -> None: @override_config({"include_profile_updates_in_sync": True}) def test_initial_sync_responds_with_tracked_profile_updates(self) -> None: - """Test that with MSC4429 enabled the initial sync response does - contain profile updates for users who share rooms, for the fields the - client requests. This response should include our syncing user.""" + """Test that with `include_profile_updates_in_sync` enabled the initial sync + response does contain profile updates for users who share rooms, for the fields + the client requests. This response should include our syncing user.""" self.get_success( self.profile_handler.set_field( target_user=UserID.from_string(self.other_user), @@ -1304,8 +1304,8 @@ def test_initial_sync_responds_with_tracked_profile_updates(self) -> None: def test_initial_sync_does_not_include_untracked_users_profile_updates( self, is_lazy: bool ) -> None: - """Test that with MSC4429 enabled the initial sync response does not - contain profile updates for users who do not share rooms.""" + """Test that with `include_profile_updates_in_sync` enabled the initial sync + response does not contain profile updates for users who do not share rooms.""" third_user = self.register_user("third_user", "password") self.get_success( self.profile_handler.set_field( @@ -1347,8 +1347,8 @@ def test_initial_sync_does_not_include_untracked_users_profile_updates( def test_initial_sync_lazy_loading_responds_with_only_profiles_with_events( self, ) -> None: - """Test that with MSC4429 enabled the initial sync lazy loading response does - contain profile updates for events in the timeline. + """Test that with `include_profile_updates_in_sync` enabled the initial sync + lazy loading response does contain profile updates for events in the timeline. This test ensures lazy loading sync only returns profiles that we also have events for in the sync response. The second room in this test has the most @@ -1424,8 +1424,8 @@ def test_initial_sync_lazy_loading_responds_with_only_profiles_with_events( def test_incremental_sync_sends_down_profile_update_diffs( self, ) -> None: - """Test that with MSC4429 enabled the incremental sync response does - contain profile update diffs.""" + """Test that with `include_profile_updates_in_sync` enabled the incremental + sync response does contain profile update diffs.""" requester = create_requester(self.user) initial_result = self.get_success( self.sync_handler.wait_for_sync_for_user( @@ -1499,9 +1499,9 @@ def test_incremental_sync_sends_down_profile_update_diffs( def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( self, ) -> None: - """Test that with MSC4429 enabled the incremental sync lazy loading response - does contain profile updates even if the user would be filtered out by lazy - loading. + """Test that with `include_profile_updates_in_sync` enabled the incremental + sync lazy loading response does contain profile updates even if the user would + be filtered out by lazy loading. """ third_user = self.register_user("third_user", "password") third_tok = self.login("third_user", "password") @@ -1652,7 +1652,7 @@ def test_sync_filters_out_profile_updates_from_federated_users( is_initial: bool, is_lazy: bool, ) -> None: - """Test that with MSC4429 enabled any sync response + """Test that with `include_profile_updates_in_sync` enabled any sync response doesn't contain federated users even if there are timeline events from them. """ # Join a federated user to the room, causing a membership event into @@ -1742,8 +1742,8 @@ def test_sync_response_always_includes_the_user_themselves( is_initial: bool, is_lazy: bool, ) -> None: - """Test that with MSC4429 enabled any sync response always contains the users - own updates. + """Test that with `include_profile_updates_in_sync` enabled any sync response + always contains the users own updates. This test is made with a user that is not in any rooms, to prove our code to collect interested users from the profile updates always collect the user. @@ -1830,8 +1830,8 @@ def test_sync_profile_updates_works_correctly_with_falsey_values( is_initial: bool, is_lazy: bool, ) -> None: - """Test that with MSC4429 enabled a sync response correctly includes falsey - profile field values. + """Test that with `include_profile_updates_in_sync` enabled a sync response + correctly includes falsey profile field values. """ requester = create_requester(self.user) filter_json: dict[str, dict] = { @@ -1911,8 +1911,9 @@ def test_sync_profile_updates_works_correctly_with_falsey_values( def test_incremental_sync_lazy_loading_cache_filters_recently_sent_profiles_and_fields( self, ) -> None: - """Test that with MSC4429 enabled the incremental sync lazy loading response - filters out unchanged profiles or fields we have recently sent to the client. + """Test that with `include_profile_updates_in_sync` enabled the incremental + sync lazy loading response filters out unchanged profiles or fields we have + recently sent to the client. """ requester = create_requester(self.user) self.get_success( @@ -2062,8 +2063,8 @@ def test_incremental_sync_lazy_loading_cache_filters_recently_sent_profiles_and_ def test_incremental_sync_sends_down_null_profile_if_user_no_longer_sharing_rooms( self, ) -> None: - """Test that with MSC4429 enabled the incremental sync response - includes a 'null' for users who are no longer sharing rooms. + """Test that with `include_profile_updates_in_sync` enabled the incremental + sync response includes a 'null' for users who are no longer sharing rooms. """ requester = create_requester(self.user) initial_result = self.get_success( @@ -2112,9 +2113,9 @@ def test_incremental_sync_sends_down_null_profile_if_user_no_longer_sharing_room def test_incremental_sync_sends_down_all_requested_fields_for_users_who_have_joined( self, ) -> None: - """Test that with MSC4429 enabled the incremental sync response - includes all the requested fields of a user who has joined a room with the - syncing user. + """Test that with `include_profile_updates_in_sync` enabled the incremental + sync response includes all the requested fields of a user who has joined a room + with the syncing user. """ requester = create_requester(self.user) initial_result = self.get_success( @@ -2211,8 +2212,8 @@ def test_incremental_sync_sends_down_all_requested_fields_for_users_who_have_joi ) @override_config({"include_profile_updates_in_sync": True}) def test_incremental_sync_includes_own_profile_updates(self, is_lazy: bool) -> None: - """Test that with MSC4429 enabled the incremental sync response includes - ones own profile updates.""" + """Test that with `include_profile_updates_in_sync` enabled the incremental + sync response includes ones own profile updates.""" requester = create_requester(self.user) filter_json: dict[str, dict] = { "org.matrix.msc4429.profile_fields": {"ids": ["m.status", "avatar_url"]} @@ -2275,8 +2276,8 @@ def test_incremental_sync_join_leave_join_leave_includes_user_joining_and_leavin eager_sync: bool, is_lazy: bool, ) -> None: - """Test that with MSC4429 enabled the incremental sync response - correctly handles multiple join / leave / join / leave in a row. + """Test that with `include_profile_updates_in_sync` enabled the incremental + sync response correctly handles multiple join / leave / join / leave in a row. In the first variant we sync and check after each iteration of join/leave. In the second variant we only sync at the end of all the join/leaves. @@ -2440,9 +2441,9 @@ def test_profile_updates_dont_get_silenced_by_cache( value: str | bool | list | dict | int | float | None, new_value: str | bool | list | dict | int | float | None, ) -> None: - """Test that with MSC4429 enabled the incremental lazy sync response - includes all the profile update changes for the user, even if the profile - field has been recently sent and is in our lazy loading cache. + """Test that with `include_profile_updates_in_sync` enabled the incremental + lazy sync response includes all the profile update changes for the user, even + if the profile field has been recently sent and is in our lazy loading cache. Parameterize across different types of potential value types that profile field updates could have to ensure robustness. @@ -2559,9 +2560,9 @@ def test_profile_updates_dont_get_silenced_by_cache( def test_lazy_loading_cache_and_multiple_updates_to_the_same_field( self, ) -> None: - """Test that with MSC4429 enabled the incremental lazy sync response - includes an update to a field, even when the value changes back to a - value set and cached previously. + """Test that with `include_profile_updates_in_sync` enabled the incremental + lazy sync response includes an update to a field, even when the value changes + back to a value set and cached previously. """ requester = create_requester(self.user) filter_json = { diff --git a/tests/rest/client/sliding_sync/test_extension_profiles.py b/tests/rest/client/sliding_sync/test_extension_profiles.py new file mode 100644 index 0000000000..40d426fb87 --- /dev/null +++ b/tests/rest/client/sliding_sync/test_extension_profiles.py @@ -0,0 +1,1231 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2024 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +import logging + +from parameterized import parameterized, parameterized_class + +from twisted.internet.testing import MemoryReactor + +import synapse.rest.admin +from synapse.api.constants import ProfileFields +from synapse.rest.client import knock, login, profile, room, sync +from synapse.server import HomeServer +from synapse.types import UserID, create_requester +from synapse.util.clock import Clock + +from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase +from tests.unittest import override_config + +logger = logging.getLogger(__name__) + + +# FIXME: This can be removed once we bump `SCHEMA_COMPAT_VERSION` and run the +# foreground update for +# `sliding_sync_joined_rooms`/`sliding_sync_membership_snapshots` (tracked by +# https://github.com/element-hq/synapse/issues/17623) +@parameterized_class( + ("use_new_tables",), + [ + (True,), + (False,), + ], + class_name_func=lambda cls, + num, + params_dict: f"{cls.__name__}_{'new' if params_dict['use_new_tables'] else 'fallback'}", +) +class SlidingSyncProfilesTestCase(SlidingSyncBase): + """Tests for the profile updates sliding sync extension""" + + servlets = [ + synapse.rest.admin.register_servlets, + knock.register_servlets, + login.register_servlets, + profile.register_servlets, + room.register_servlets, + sync.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + self.profile_handler = self.hs.get_profile_handler() + self.user = self.register_user("user", "password") + self.tok = self.login("user", "password") + self.other_user = self.register_user("other_user", "password") + self.other_tok = self.login("other_user", "password") + self.joined_room = self.helper.create_room_as(self.user, tok=self.tok) + self.helper.join( + room=self.joined_room, user=self.other_user, tok=self.other_tok + ) + super().prepare(reactor, clock, hs) + + @parameterized.expand( + [ + True, + False, + ] + ) + def test_no_data_when_not_enabled(self, is_initial: bool) -> None: + """ + Test that no profile extension response is returned + if the feature is not enabled. + """ + if is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + "fields": ["field"], + }, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + self.assertIsNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + + if not is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + + self.assertIsNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_no_data_initial_sync(self) -> None: + """ + Test that enabling the profiles extension works during an initial sync, + even if there is no-data. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + "fields": ["field"], + }, + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + self.assertIsNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_no_data_incremental_sync(self) -> None: + """ + Test that enabling profiles extension works during an incremental sync, even + if there is no-data. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + "fields": ["field"], + } + }, + } + _, from_token = self.do_sync(sync_body, tok=user1_tok) + + # Make an incremental Sliding Sync request with the profiles extension enabled + response_body, _ = self.do_sync(sync_body, since=from_token, tok=user1_tok) + + self.assertIsNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_updated_fields_are_sent(self, is_initial: bool) -> None: + """ + Test that profile extension response returns field updates + in incremental and initial sync. + """ + if is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + "room_subscriptions": { + self.joined_room: { + "required_state": [], + "timeline_limit": 10, + }, + }, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + "fields": ["field"], + }, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + if is_initial: + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "updated": { + "field": "value", + } + }, + ) + else: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # We don't include room subscriptions, as we want to see updates coming + # through even without room subscriptions + del sync_body["room_subscriptions"] + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "updated": { + "field": "value", + } + }, + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_updated_field_then_deleted_does_not_error(self) -> None: + """ + Test that profile extension response does not crash if the user first + updates a field, then deletes it, and then the sync happens seeing both + the update and delete in the stream. + """ + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + "fields": ["field"], + }, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + + # Update field + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="new value", + ) + ) + # Delete field + self.get_success( + self.profile_handler.delete_profile_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + ) + ) + + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "removed": [ + "field", + ], + }, + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_updated_fields_are_not_sent_if_not_requested( + self, is_initial: bool + ) -> None: + """ + Test that profile extension response doesn't return field updates we didn't + request in initial and incremental sync. + """ + if is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="anotherfield", + new_value="value", + ) + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + "fields": ["field"], + }, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + if is_initial: + # Nothing returned since we didn't ask for the updated field + self.assertIsNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + + if not is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="anotherfield", + new_value="value", + ) + ) + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + # Nothing returned since we didn't ask for the updated field + self.assertIsNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_updated_fields_are_not_included_if_not_in_requested_rooms( + self, is_initial: bool + ) -> None: + """ + Test that profile extension response respects the room subscriptions, by: + * for initial sync returning updates for only those users in the given rooms + * for incremental sync returning all updates in shared rooms + """ + new_room = self.helper.create_room_as(self.user, tok=self.tok) + if is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + "room_subscriptions": { + new_room: { + "required_state": [], + "timeline_limit": 10, + }, + }, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + "fields": ["field"], + }, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + if is_initial: + # Nothing returned since even though user and other_user share a room, + # we didn't ask for that room. + self.assertIsNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + + if not is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + # Even though we only asked for a room other_user is not in, + # since these users share a room, updates are always sent via incremental + # sync. + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "updated": { + "field": "value", + } + }, + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_all_fields_returned_if_no_fields_specified(self, is_initial: bool) -> None: + """ + Test that profile extension response returns all profile fields if we didn't + request any particular fields in initial and incremental sync. + """ + if is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + # We need to ensure a room is included to get things back in initial sync + "room_subscriptions": { + self.joined_room: { + "required_state": [], + "timeline_limit": 10, + }, + }, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + }, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + if is_initial: + # As this is an initial sync, we get all profile fields + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "updated": { + "avatar_url": None, + "displayname": "other_user", + "field": "value", + } + }, + ) + + else: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # We don't include room subscriptions, as we want to see updates coming + # through even without room subscriptions + del sync_body["room_subscriptions"] + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + # As this is an incremental sync, we only get actual updates back + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "updated": { + "field": "value", + } + }, + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_null_profile_returned_if_user_left_all_rooms( + self, + request_fields: bool, + ) -> None: + """ + Test that profile extension response returns a null for the user in + incremental sync. + """ + # Make an initial Sliding Sync request with the profiles extension enabled + profiles_config: dict = { + "enabled": True, + } + if request_fields: + profiles_config["fields"] = ["field"] + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": profiles_config, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + + self.helper.leave(self.joined_room, self.other_user, tok=self.other_tok) + + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + # We should see a null profile + self.assertIsNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_profile_returned_if_user_left_then_rejoined(self) -> None: + """ + Test that the profile extension response returns a profile, rather than a + null, for a user that left their last shared room and then rejoined it + within the same incremental sync window. + """ + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": {"enabled": True}, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + + self.helper.leave(self.joined_room, self.other_user, tok=self.other_tok) + self.helper.join(self.joined_room, self.other_user, tok=self.other_tok) + + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + # The rejoin overrides the leave, so we should see the full profile rather + # than a null profile. + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "updated": { + "displayname": "other_user", + # FIXME: This shouldn't be returned, but currently is + "avatar_url": None, + } + }, + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_all_fields_returned_in_incremental_non_lazy_sync_if_someone_joined( + self, request_fields: bool + ) -> None: + """ + Test that profile extension response returns all profile fields in + incremental non-lazy sync, if someone joined the room.. + """ + # Make an initial Sliding Sync request with the profiles extension enabled + profiles_config: dict = { + "enabled": True, + } + if request_fields: + profiles_config["fields"] = ["displayname"] + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": profiles_config, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + + third_user = self.register_user("third_user", "third_user") + third_tok = self.login(third_user, "third_user") + self.helper.join(self.joined_room, third_user, tok=third_tok) + + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + + expectation = { + "updated": { + "avatar_url": None, + "displayname": "third_user", + } + } + if request_fields: + expectation = { + "updated": { + "displayname": "third_user", + }, + } + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@third_user:test" + ], + expectation, + ) + + @parameterized.expand(["displayname", "avatar_url", "someotherfield"]) + @override_config({"include_profile_updates_in_sync": True}) + def test_removed_fields_get_sent_down_as_removed( + self, + field_name: str, + ) -> None: + """ + Test that we deliver clear/removed fields in the "removed" key in the response. + """ + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name=field_name, + new_value="value", + ) + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + }, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + + # Delete the field + if field_name in (ProfileFields.DISPLAYNAME, ProfileFields.AVATAR_URL): + self.get_success( + self.profile_handler.set_profile_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name=field_name, + new_value=None, + ) + ) + else: + self.get_success( + self.profile_handler.delete_profile_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name=field_name, + ) + ) + + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + # We should see the removed field + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "removed": [ + field_name, + ], + }, + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_updated_key_only_present_if_updates(self) -> None: + """ + > The updated field SHOULD only be present if there are changes to existing fields on a user's profile. + """ + self.skipTest("Not yet implemented") + + @override_config({"include_profile_updates_in_sync": True}) + def test_rooms_subset_changing_includes_full_profile(self) -> None: + """ + > When a room enters this subset in this connection for the first time, all requested + > fields from profiles of users in that room MAY be sent down. This gives the client + > a base set of information for which future field updates can be applied on top of. + > The homeserver MAY omit some fields and profiles if it believes that the client has + > already received them, likewise repeat profiles MAY be sent down based on homeserver + > implementation. + """ + self.skipTest("Not yet implemented") + + @override_config({"include_profile_updates_in_sync": True}) + def test_fields_subset_changing_sends_down_field_even_if_not_changed(self) -> None: + """ + > Finally, if the list of fields expands to cover a new field ID, those fields should + > be sent down for all users that are within the current room subset. Future incremental + > updates will then include changes to this field. + """ + self.skipTest("Not yet implemented") + + @parameterized.expand( + [ + [True, True], + [True, False], + [False, False], + [False, True], + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_lazy_loading_sends_down_full_profile_if_events_in_timeline( + self, + is_initial: bool, + use_room_subsciptions: bool, + ) -> None: + """ + Test that when lazy loading, only those members who have events in + the timeline get their profiles sent down in the sync response, for + rooms configured with lazy loading. + + Rooms without lazy loading should include all the members in initial sync, + none in incremental. + """ + # Create three users to fill the heroes + # Our heroes will thus be user, other_user and the three heroes here. + for i in range(3): + user = self.register_user(f"hero{i}", "password") + tok = self.login(f"hero{i}", "password") + self.helper.join(self.joined_room, user=user, tok=tok) + third_user = self.register_user("third_user", "password") + third_tok = self.login("third_user", "password") + fourth_user = self.register_user("fourth_user", "password") + fourth_tok = self.login("fourth_user", "password") + fifth_user = self.register_user("fifth_user", "password") + fifth_tok = self.login("fifth_user", "password") + self.helper.join( + room=self.joined_room, + user=third_user, + tok=third_tok, + ) + self.helper.join( + room=self.joined_room, + user=fifth_user, + tok=fifth_tok, + ) + new_room = self.helper.create_room_as(self.user, tok=self.tok) + self.helper.join( + room=new_room, + user=fourth_user, + tok=fourth_tok, + ) + if is_initial: + self.helper.send_messages( + room_id=self.joined_room, num_events=1, tok=self.other_tok + ) + self.helper.send_messages( + room_id=self.joined_room, num_events=10, tok=third_tok + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body: dict[str, dict] = { + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + }, + }, + } + if use_room_subsciptions: + sync_body["room_subscriptions"] = { + self.joined_room: { + "required_state": [], + "timeline_limit": 10, + }, + new_room: { + "required_state": [], + "timeline_limit": 10, + }, + } + else: + sync_body["lists"] = { + "foo-list": { + "ranges": [[0, 0]], + "required_state": [], + "timeline_limit": 10, + } + } + # We also need to specifically request the non-lazy room, otherwise + # our test to see if non-lazy members are also included will fail + sync_body["room_subscriptions"] = { + new_room: { + "required_state": [], + "timeline_limit": 10, + }, + } + if is_initial: + if use_room_subsciptions: + sync_body["room_subscriptions"][self.joined_room]["required_state"] = [ + ["m.room.member", "$LAZY"], + # Don't request other state as we're checking timeline events + # ["*", "*"], + ] + else: + sync_body["lists"]["foo-list"]["required_state"] = [ + ["m.room.member", "$LAZY"], + # Don't request other state as we're checking timeline events + # ["*", "*"], + ] + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + if is_initial: + self.assertIsNotNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + # Other user is a hero so should be included. + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@other_user:test" + ) + ) + # Third user has events in the timeline, so should be here. + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@third_user:test" + ) + ) + # Initial sync always includes ourselves + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@user:test" + ) + ) + # Fourth user is a member of a non-lazy configured room, so should be here. + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@fourth_user:test" + ) + ) + # Fifth user should be filtered out as they have no events in the room. + self.assertIsNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@fifth_user:test" + ) + ) + + if not is_initial: + self.helper.send_messages( + room_id=self.joined_room, num_events=1, tok=self.other_tok + ) + self.helper.send_messages( + room_id=self.joined_room, num_events=10, tok=third_tok + ) + if use_room_subsciptions: + sync_body["room_subscriptions"][self.joined_room]["required_state"] = [ + ["m.room.member", "$LAZY"], + # Don't request other state as we're checking timeline events + # ["*", "*"], + ] + else: + sync_body["lists"]["foo-list"]["required_state"] = [ + ["m.room.member", "$LAZY"], + # Don't request other state as we're checking timeline events + # ["*", "*"], + ] + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + self.assertIsNotNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + # TODO check this if it's expected that heroes come down differently + # depending on if using a room subscription or a list + if use_room_subsciptions: + # Other user should be filtered out as heroes don't come down + # in incremental sync in the same way as initial sync, if the + # room is included via a room subscription. + self.assertIsNone( + response_body["extensions"]["org.matrix.msc4262.profiles"][ + "users" + ].get("@other_user:test") + ) + else: + # Other user should be included as heroes do come down + # in incremental sync in the same way as initial sync when the + # room is included in a list + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"][ + "users" + ].get("@other_user:test") + ) + # Third user has events in the timeline, so should be here. + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@third_user:test" + ) + ) + # We are not included ourselves in incremental sync without updates. + self.assertIsNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@user:test" + ) + ) + # Fourth user is a member of a non-lazy configured room, but had no updates, + # so shouldn't be here. + self.assertIsNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@fourth_user:test" + ) + ) + # Fifth user should be excluded as they have no events. + self.assertIsNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@fifth_user:test" + ) + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_lazy_loading_sends_full_profile_even_if_no_events_if_otherwise_included( + self, + use_room_subsciptions: bool, + ) -> None: + """ + Test that when lazy loading, if a user is in both a lazy loading room + and a non-lazy configured room, even if there are no events in the timeline, + their profile is sent down. + + This test only makes sense for initial sync, as for incremental we would + not expect to see users without timeline events if they had no profile updates. + """ + # Create some users to fill the heroes so they don't pollute the test. + for i in range(3): + user = self.register_user(f"hero{i}", "password") + tok = self.login(f"hero{i}", "password") + self.helper.join(self.joined_room, user=user, tok=tok) + new_user = self.register_user("new_user", password="password") + new_tok = self.login("new_user", password="password") + new_room = self.helper.create_room_as(self.user, tok=self.tok) + self.helper.join( + room=self.joined_room, + user=new_user, + tok=new_tok, + ) + self.helper.join( + room=new_room, + user=new_user, + tok=new_tok, + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body: dict[str, dict] = { + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + }, + }, + } + if use_room_subsciptions: + sync_body["room_subscriptions"] = { + self.joined_room: { + "required_state": [ + ["m.room.member", "$LAZY"], + # Don't request any events for this room + # ["*", "*"], + ], + # Force zero timeline events in the response, otherwise + # this test wont work, as the timeline_events in the room + # response will contain all the create/join etc events too. + "timeline_limit": 0, + }, + new_room: { + "required_state": [], + "timeline_limit": 10, + }, + } + else: + sync_body["lists"] = { + "foo-list": { + "ranges": [[0, 0]], + "required_state": [ + ["m.room.member", "$LAZY"], + # Don't request any events for this room + # ["*", "*"], + ], + # Force zero timeline events in the response, otherwise + # this test wont work, as the timeline_events in the room + # response will contain all the create/join etc events too. + "timeline_limit": 0, + } + } + # We also need to specifically request the non-lazy room, otherwise + # our test to see if non-lazy members are also included will fail + sync_body["room_subscriptions"] = { + new_room: { + "required_state": [], + "timeline_limit": 10, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + self.assertIsNotNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + # New user should be included as they are in a non-lazy room too, + # even though the lazy configured room had no events. + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@new_user:test" + ) + ) + # Initial sync always includes ourselves + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@user:test" + ) + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_lazy_loading_sends_full_profile_for_required_state_member_events( + self, + use_room_subsciptions: bool, + ) -> None: + """ + Test that when lazy loading for lazy rooms, even without timeline events, + we get profiles for users who have membership events in required_state. + + This test only makes sense for initial sync, as for incremental this would + happen via the `ProfileUpdateAction.JOINED_ROOM` events. + """ + # Create some users to fill the heroes so they don't pollute the test. + for i in range(3): + user = self.register_user(f"hero{i}", "password") + tok = self.login(f"hero{i}", "password") + self.helper.join(self.joined_room, user=user, tok=tok) + new_user = self.register_user("new_user", password="password") + new_tok = self.login("new_user", password="password") + self.helper.join( + room=self.joined_room, + user=new_user, + tok=new_tok, + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body: dict[str, dict] = { + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + }, + }, + } + if use_room_subsciptions: + sync_body["room_subscriptions"] = { + self.joined_room: { + "required_state": [ + ["m.room.member", "$LAZY"], + ["*", "*"], + ], + # Force zero timeline events in the response, otherwise + # this test wont work, as the timeline_events in the room + # response will contain all the create/join etc events too. + "timeline_limit": 0, + }, + } + else: + sync_body["lists"] = { + "foo-list": { + "ranges": [[0, 0]], + "required_state": [ + ["m.room.member", "$LAZY"], + ["*", "*"], + ], + # Force zero timeline events in the response, otherwise + # this test wont work, as the timeline_events in the room + # response will contain all the create/join etc events too. + "timeline_limit": 0, + } + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + self.assertIsNotNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + # New user should be included as they joined the room and as such + # have membership events in required_state. + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@new_user:test" + ) + ) + # Initial sync always includes ourselves + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@user:test" + ) + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_lazy_loading_sends_full_profile_for_heroes( + self, + use_room_subsciptions: bool, + ) -> None: + """ + Test that when lazy loading for lazy rooms, even without timeline events or + required_state, we get profiles for room heroes. + + This test must ensure heroes don't get included in timeline_events + or required_state. + + This test only makes sense for initial sync, as for incremental sync + Synapse doesn't generate a room response without requesting state or + timeline events, thus no heroes either. + """ + # Create some users to fill the heroes + for i in range(4): + user = self.register_user(f"hero{i}", "password") + tok = self.login(f"hero{i}", "password") + self.helper.join(self.joined_room, user=user, tok=tok) + not_hero = self.register_user("not_hero", "password") + not_hero_tok = self.login("not_hero", "password") + self.helper.join(self.joined_room, user=not_hero, tok=not_hero_tok) + + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body: dict[str, dict] = { + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + }, + }, + } + if use_room_subsciptions: + sync_body["room_subscriptions"] = { + self.joined_room: { + "required_state": [ + ["m.room.member", "$LAZY"], + # Don't request any events for this room + # ["*", "*"], + ], + # Force zero timeline events in the response, otherwise + # this test wont work, as the timeline_events in the room + # response will contain all the create/join etc events too. + "timeline_limit": 0, + }, + } + else: + sync_body["lists"] = { + "foo-list": { + "ranges": [[0, 0]], + "required_state": [ + ["m.room.member", "$LAZY"], + # Don't request any events for this room + # ["*", "*"], + ], + # Force zero timeline events in the response, otherwise + # this test wont work, as the timeline_events in the room + # response will contain all the create/join etc events too. + "timeline_limit": 0, + } + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + self.assertIsNotNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + # Other user should be included as they are a room hero + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@other_user:test" + ) + ) + # Not hero user should be excluded as they're not a hero + self.assertIsNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@not_hero:test" + ) + ) + # Initial sync always includes ourselves + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@user:test" + ) + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_repeat_of_sync_correctly_includes_profile_information_again(self) -> None: + """ + > Homeservers should only consider a profile field update "accepted" by a client + > once the client returns with a new /sync request with the next /sync token, + > NOT just after sending down the profile update. The client may never receive + > response due to network conditions, or a bug in the client implementation. + """ + self.skipTest("Not yet implemented") From d59856b9a79dbc52cd1740ae2235f0350dde6d28 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 25 Aug 2026 19:14:17 +0300 Subject: [PATCH 18/28] Fix sync stream not being woken up when a user updates a profile field without belonging to any rooms (#20135) Fix sync stream not being woken up when a user updates a profile field without belonging to any rooms. Fixes https://github.com/element-hq/synapse/issues/20110 --- changelog.d/20135.bugfix | 1 + synapse/handlers/profile.py | 18 +++++++++++++++++ tests/handlers/test_profile.py | 36 +++++++++++++++++++++++++++++++--- 3 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 changelog.d/20135.bugfix diff --git a/changelog.d/20135.bugfix b/changelog.d/20135.bugfix new file mode 100644 index 0000000000..1b51b21fa2 --- /dev/null +++ b/changelog.d/20135.bugfix @@ -0,0 +1 @@ +Fix sync stream not being woken up when a user updates a profile field without belonging to any rooms. \ No newline at end of file diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 3d052bc893..067a16262c 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -668,11 +668,20 @@ async def set_field( if stream_id is not None: room_ids = await self.store.get_rooms_for_user(target_user.to_string()) if room_ids: + # Wake up the stream for the rooms involved self._notifier.on_new_event( StreamKeyType.PROFILE_UPDATES, stream_id, rooms=room_ids, ) + else: + # Wake up the stream for ourselves, as we might be updating our + # profile even if we don't have rooms + self._notifier.on_new_event( + StreamKeyType.PROFILE_UPDATES, + stream_id, + users=[target_user], + ) async def set_profile_field( self, @@ -807,11 +816,20 @@ async def delete_profile_field( if stream_id: room_ids = await self.store.get_rooms_for_user(target_user.to_string()) if room_ids: + # Wake up the stream for the rooms involved self._notifier.on_new_event( StreamKeyType.PROFILE_UPDATES, stream_id, rooms=room_ids, ) + else: + # Wake up the stream for ourselves, as we might be updating our + # profile even if we don't have rooms + self._notifier.on_new_event( + StreamKeyType.PROFILE_UPDATES, + stream_id, + users=[target_user], + ) async def on_profile_query(self, args: JsonDict) -> JsonDict: """Handles federation profile query requests.""" diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 590c7e62f5..31e7e1c0a7 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -261,10 +261,10 @@ def test_update_profile_does_not_notify_notifier_on_set_field_if_include_profile ] ) @override_config({"include_profile_updates_in_sync": True}) - def test_update_profile_does_not_notify_notifier_on_set_field_if_user_not_in_rooms( + def test_update_profile_does_notify_notifier_on_set_field_if_user_not_in_rooms( self, field_name: str, new_value: str ) -> None: - """Test that profile updates do not cause the profile updates stream notifier + """Test that profile updates does cause the profile updates stream notifier to wake up if the user is not in any rooms, if `include_profile_updates_in_sync` is enabled.""" self.get_success( @@ -280,7 +280,37 @@ def test_update_profile_does_not_notify_notifier_on_set_field_if_user_not_in_roo for call in self.on_new_event.mock_calls if call.args[0] == StreamKeyType.PROFILE_UPDATES ] - self.assertEqual(len(calls_found), 0) + self.assertEqual(len(calls_found), 1) + + @override_config({"include_profile_updates_in_sync": True}) + def test_update_profile_does_notify_notifier_on_delete_profile_field_if_user_not_in_rooms( + self, + ) -> None: + """Test that profile updates does cause the profile updates stream notifier + to wake up if the user is not in any rooms, if `include_profile_updates_in_sync` + is enabled.""" + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name="field", + new_value="value", + ) + ) + self.on_new_event.reset_mock() + self.get_success( + self.handler.delete_profile_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name="field", + ) + ) + calls_found = [ + call + for call in self.on_new_event.mock_calls + if call.args[0] == StreamKeyType.PROFILE_UPDATES + ] + self.assertEqual(len(calls_found), 1) @parameterized.expand( [ From 37b59059edf383b7ccd5586804b7c0455cf3a21f Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 25 Aug 2026 11:18:21 -0500 Subject: [PATCH 19/28] 1.160.0rc1 --- CHANGES.md | 35 +++++++++++++++++++++++++++++++ changelog.d/19875.doc | 1 - changelog.d/19974.feature | 1 - changelog.d/20003.feature | 2 -- changelog.d/20067.feature | 1 - changelog.d/20089.doc | 1 - changelog.d/20090.bugfix | 1 - changelog.d/20093.misc | 1 - changelog.d/20094.bugfix | 1 - changelog.d/20098.misc | 1 - changelog.d/20099.misc | 1 - changelog.d/20117.misc | 1 - changelog.d/20120.misc | 1 - changelog.d/20124.misc | 1 - changelog.d/20128.misc | 1 - changelog.d/20129.misc | 1 - changelog.d/20131.misc | 1 - changelog.d/20131.misc.1 | 1 - changelog.d/20135.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- schema/synapse-config.schema.yaml | 2 +- 22 files changed, 43 insertions(+), 21 deletions(-) delete mode 100644 changelog.d/19875.doc delete mode 100644 changelog.d/19974.feature delete mode 100644 changelog.d/20003.feature delete mode 100644 changelog.d/20067.feature delete mode 100644 changelog.d/20089.doc delete mode 100644 changelog.d/20090.bugfix delete mode 100644 changelog.d/20093.misc delete mode 100644 changelog.d/20094.bugfix delete mode 100644 changelog.d/20098.misc delete mode 100644 changelog.d/20099.misc delete mode 100644 changelog.d/20117.misc delete mode 100644 changelog.d/20120.misc delete mode 100644 changelog.d/20124.misc delete mode 100644 changelog.d/20128.misc delete mode 100644 changelog.d/20129.misc delete mode 100644 changelog.d/20131.misc delete mode 100644 changelog.d/20131.misc.1 delete mode 100644 changelog.d/20135.bugfix diff --git a/CHANGES.md b/CHANGES.md index 51c41cddb7..3fc546caea 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,38 @@ +# Synapse 1.160.0rc1 (2026-08-25) + +## Features + +- Add experimental support for MSC4502: Targeted and unrestricted room member queries. ([\#19974](https://github.com/element-hq/synapse/issues/19974)) +- Add optional support for [MSC4262: Profile Updates for Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4262). + Currently defaults to not enabled, and is limited to local users only for the sync results. ([\#20003](https://github.com/element-hq/synapse/issues/20003)) +- Allow specifying multiple `action_name` and `status` query parameters when listing scheduled tasks via the admin API. ([\#20067](https://github.com/element-hq/synapse/issues/20067)) + +## Bugfixes + +- Fix a bug where presence updates could stop being sent to clients (the presence stream position becoming stuck) if a `/sync` request was cancelled while a presence write was allocating a stream ID. Contributed by @FrenchGithubUser @Famedly. ([\#20090](https://github.com/element-hq/synapse/issues/20090)) +- Thumbnail WebP images that use transparency as PNG rather than JPEG, to preserve transparency during thumbnailing. Contributed by @catfromplan9. ([\#20094](https://github.com/element-hq/synapse/issues/20094)) +- Fix sync stream not being woken up when a user updates a profile field without belonging to any rooms. ([\#20135](https://github.com/element-hq/synapse/issues/20135)) + +## Improved Documentation + +- Document lighttpd reverse proxy configuration example from matrix.jaxlug.ngo, a contribution from the JaxLUG, the Jacksonville Linux Users Group Inc.. ([\#19875](https://github.com/element-hq/synapse/issues/19875)) +- Fix the documentation on the `federation_domain_whitelist` config option. ([\#20089](https://github.com/element-hq/synapse/issues/20089)) + +## Internal Changes + +- Update release script to check more often for actions being completed so you don't have to wait around as much. ([\#20093](https://github.com/element-hq/synapse/issues/20093)) +- Speed up the conversion of device list changes into outbound federation pokes, and add a metric for how far behind the conversion is. ([\#20098](https://github.com/element-hq/synapse/issues/20098)) +- Fix the schema diff CI not using `faketime` for SQLite. ([\#20099](https://github.com/element-hq/synapse/issues/20099)) +- Fix the schema diff CI breaking when the Rust module was changed. ([\#20117](https://github.com/element-hq/synapse/issues/20117), [\#20129](https://github.com/element-hq/synapse/issues/20129)) +- Reduce database CPU usage when marking device list changes as sent over federation. ([\#20120](https://github.com/element-hq/synapse/issues/20120)) +- Fix cache `__len__` of Sliding Sync `PerConnectionState` ignoring account data entries. ([\#20124](https://github.com/element-hq/synapse/issues/20124)) +- Update Synapse repo link in inconsistent stream error. ([\#20128](https://github.com/element-hq/synapse/issues/20128)) +- Update rustls-webpki to address GHSA-82j2-j2ch-gfr8. ([\#20131](https://github.com/element-hq/synapse/issues/20131)) +- Update pyo3 to address GHSA-36hh-v3qg-5jq4 and GHSA-chgr-c6px-7xpp. ([\#20131](https://github.com/element-hq/synapse/issues/20131)) + + + + # Synapse 1.159.0 (2026-08-18) No significant changes since 1.159.0rc1. diff --git a/changelog.d/19875.doc b/changelog.d/19875.doc deleted file mode 100644 index 6ab67a08a2..0000000000 --- a/changelog.d/19875.doc +++ /dev/null @@ -1 +0,0 @@ -Document lighttpd reverse proxy configuration example from matrix.jaxlug.ngo, a contribution from the JaxLUG, the Jacksonville Linux Users Group Inc.. diff --git a/changelog.d/19974.feature b/changelog.d/19974.feature deleted file mode 100644 index 4a9290ea76..0000000000 --- a/changelog.d/19974.feature +++ /dev/null @@ -1 +0,0 @@ -Add experimental support for MSC4502: Targeted and unrestricted room member queries. diff --git a/changelog.d/20003.feature b/changelog.d/20003.feature deleted file mode 100644 index add6b613c8..0000000000 --- a/changelog.d/20003.feature +++ /dev/null @@ -1,2 +0,0 @@ -Add optional support for [MSC4262: Profile Updates for Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4262). -Currently defaults to not enabled, and is limited to local users only for the sync results. \ No newline at end of file diff --git a/changelog.d/20067.feature b/changelog.d/20067.feature deleted file mode 100644 index 123d4df745..0000000000 --- a/changelog.d/20067.feature +++ /dev/null @@ -1 +0,0 @@ -Allow specifying multiple `action_name` and `status` query parameters when listing scheduled tasks via the admin API. diff --git a/changelog.d/20089.doc b/changelog.d/20089.doc deleted file mode 100644 index e72db0dbde..0000000000 --- a/changelog.d/20089.doc +++ /dev/null @@ -1 +0,0 @@ -Fix the documentation on the `federation_domain_whitelist` config option. \ No newline at end of file diff --git a/changelog.d/20090.bugfix b/changelog.d/20090.bugfix deleted file mode 100644 index 69b8035a44..0000000000 --- a/changelog.d/20090.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where presence updates could stop being sent to clients (the presence stream position becoming stuck) if a `/sync` request was cancelled while a presence write was allocating a stream ID. Contributed by @FrenchGithubUser @Famedly. diff --git a/changelog.d/20093.misc b/changelog.d/20093.misc deleted file mode 100644 index 60c745eb47..0000000000 --- a/changelog.d/20093.misc +++ /dev/null @@ -1 +0,0 @@ -Update release script to check more often for actions being completed so you don't have to wait around as much. diff --git a/changelog.d/20094.bugfix b/changelog.d/20094.bugfix deleted file mode 100644 index ebea9aaa04..0000000000 --- a/changelog.d/20094.bugfix +++ /dev/null @@ -1 +0,0 @@ -Thumbnail WebP images that use transparency as PNG rather than JPEG, to preserve transparency during thumbnailing. Contributed by @catfromplan9. diff --git a/changelog.d/20098.misc b/changelog.d/20098.misc deleted file mode 100644 index 0504be872d..0000000000 --- a/changelog.d/20098.misc +++ /dev/null @@ -1 +0,0 @@ -Speed up the conversion of device list changes into outbound federation pokes, and add a metric for how far behind the conversion is. diff --git a/changelog.d/20099.misc b/changelog.d/20099.misc deleted file mode 100644 index f17b66915b..0000000000 --- a/changelog.d/20099.misc +++ /dev/null @@ -1 +0,0 @@ -Fix the schema diff CI not using `faketime` for SQLite. \ No newline at end of file diff --git a/changelog.d/20117.misc b/changelog.d/20117.misc deleted file mode 100644 index 92d421117e..0000000000 --- a/changelog.d/20117.misc +++ /dev/null @@ -1 +0,0 @@ -Fix the schema diff CI breaking when the Rust module was changed. \ No newline at end of file diff --git a/changelog.d/20120.misc b/changelog.d/20120.misc deleted file mode 100644 index d59db030c6..0000000000 --- a/changelog.d/20120.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce database CPU usage when marking device list changes as sent over federation. diff --git a/changelog.d/20124.misc b/changelog.d/20124.misc deleted file mode 100644 index d92a12ce08..0000000000 --- a/changelog.d/20124.misc +++ /dev/null @@ -1 +0,0 @@ -Fix cache `__len__` of Sliding Sync `PerConnectionState` ignoring account data entries. \ No newline at end of file diff --git a/changelog.d/20128.misc b/changelog.d/20128.misc deleted file mode 100644 index 1361eb0e5e..0000000000 --- a/changelog.d/20128.misc +++ /dev/null @@ -1 +0,0 @@ -Update Synapse repo link in inconsistent stream error. diff --git a/changelog.d/20129.misc b/changelog.d/20129.misc deleted file mode 100644 index 92d421117e..0000000000 --- a/changelog.d/20129.misc +++ /dev/null @@ -1 +0,0 @@ -Fix the schema diff CI breaking when the Rust module was changed. \ No newline at end of file diff --git a/changelog.d/20131.misc b/changelog.d/20131.misc deleted file mode 100644 index a90fd7d4ad..0000000000 --- a/changelog.d/20131.misc +++ /dev/null @@ -1 +0,0 @@ -Update rustls-webpki to address GHSA-82j2-j2ch-gfr8. diff --git a/changelog.d/20131.misc.1 b/changelog.d/20131.misc.1 deleted file mode 100644 index c0c211ed8c..0000000000 --- a/changelog.d/20131.misc.1 +++ /dev/null @@ -1 +0,0 @@ -Update pyo3 to address GHSA-36hh-v3qg-5jq4 and GHSA-chgr-c6px-7xpp. diff --git a/changelog.d/20135.bugfix b/changelog.d/20135.bugfix deleted file mode 100644 index 1b51b21fa2..0000000000 --- a/changelog.d/20135.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix sync stream not being woken up when a user updates a profile field without belonging to any rooms. \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index 8a6da09864..a0325ac3bc 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.160.0~rc1) stable; urgency=medium + + * New synapse release 1.160.0rc1. + + -- Synapse Packaging team Tue, 25 Aug 2026 16:17:54 +0000 + matrix-synapse-py3 (1.159.0) stable; urgency=medium * New synapse release 1.159.0. diff --git a/pyproject.toml b/pyproject.toml index 8e84e6d37d..751b2805a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.159.0" +version = "1.160.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index a8d1719288..59791a4a1f 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://element-hq.github.io/synapse/schema/synapse/v1.159/synapse-config.schema.json +$id: https://element-hq.github.io/synapse/schema/synapse/v1.160/synapse-config.schema.json type: object properties: modules: From 5c0e07221415d964b1a34b6f15ba421c30f8573f Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 25 Aug 2026 11:19:31 -0500 Subject: [PATCH 20/28] Link MSC --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 3fc546caea..270a8eb013 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,7 +2,7 @@ ## Features -- Add experimental support for MSC4502: Targeted and unrestricted room member queries. ([\#19974](https://github.com/element-hq/synapse/issues/19974)) +- Add experimental support for [MSC4502](https://github.com/matrix-org/matrix-spec-proposals/pull/4502): Targeted and unrestricted room member queries. ([\#19974](https://github.com/element-hq/synapse/issues/19974)) - Add optional support for [MSC4262: Profile Updates for Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4262). Currently defaults to not enabled, and is limited to local users only for the sync results. ([\#20003](https://github.com/element-hq/synapse/issues/20003)) - Allow specifying multiple `action_name` and `status` query parameters when listing scheduled tasks via the admin API. ([\#20067](https://github.com/element-hq/synapse/issues/20067)) From 2f2ee47d0a0c2b02b068ea3ce85e0ce5cd774b43 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 25 Aug 2026 11:20:18 -0500 Subject: [PATCH 21/28] Remove negative phrasing --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 270a8eb013..d0e85623cc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,7 @@ - Add experimental support for [MSC4502](https://github.com/matrix-org/matrix-spec-proposals/pull/4502): Targeted and unrestricted room member queries. ([\#19974](https://github.com/element-hq/synapse/issues/19974)) - Add optional support for [MSC4262: Profile Updates for Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4262). - Currently defaults to not enabled, and is limited to local users only for the sync results. ([\#20003](https://github.com/element-hq/synapse/issues/20003)) + Currently defaults to disabled, and is limited to local users only for the sync results. ([\#20003](https://github.com/element-hq/synapse/issues/20003)) - Allow specifying multiple `action_name` and `status` query parameters when listing scheduled tasks via the admin API. ([\#20067](https://github.com/element-hq/synapse/issues/20067)) ## Bugfixes From 4172c8284198ac83b8fdf631d5fde4d1a0664b13 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 25 Aug 2026 11:22:06 -0500 Subject: [PATCH 22/28] Standardize attribution --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index d0e85623cc..c265a0a839 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,7 +15,7 @@ ## Improved Documentation -- Document lighttpd reverse proxy configuration example from matrix.jaxlug.ngo, a contribution from the JaxLUG, the Jacksonville Linux Users Group Inc.. ([\#19875](https://github.com/element-hq/synapse/issues/19875)) +- Document lighttpd reverse proxy configuration example from matrix.jaxlug.ngo. Contributed by JaxLUG from the Jacksonville Linux Users Group Inc.. ([\#19875](https://github.com/element-hq/synapse/issues/19875)) - Fix the documentation on the `federation_domain_whitelist` config option. ([\#20089](https://github.com/element-hq/synapse/issues/20089)) ## Internal Changes From 44188b5ceb54fb3a81cb2bcefc030fae4730ac79 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 25 Aug 2026 11:25:26 -0500 Subject: [PATCH 23/28] Link GitHub security advisories --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c265a0a839..edfe161d22 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -27,8 +27,8 @@ - Reduce database CPU usage when marking device list changes as sent over federation. ([\#20120](https://github.com/element-hq/synapse/issues/20120)) - Fix cache `__len__` of Sliding Sync `PerConnectionState` ignoring account data entries. ([\#20124](https://github.com/element-hq/synapse/issues/20124)) - Update Synapse repo link in inconsistent stream error. ([\#20128](https://github.com/element-hq/synapse/issues/20128)) -- Update rustls-webpki to address GHSA-82j2-j2ch-gfr8. ([\#20131](https://github.com/element-hq/synapse/issues/20131)) -- Update pyo3 to address GHSA-36hh-v3qg-5jq4 and GHSA-chgr-c6px-7xpp. ([\#20131](https://github.com/element-hq/synapse/issues/20131)) +- Update rustls-webpki to address [GHSA-82j2-j2ch-gfr8](https://github.com/advisories/GHSA-82j2-j2ch-gfr8). ([\#20131](https://github.com/element-hq/synapse/issues/20131)) +- Update pyo3 to address [GHSA-36hh-v3qg-5jq4](https://github.com/advisories/GHSA-36hh-v3qg-5jq4) and [GHSA-chgr-c6px-7xpp](https://github.com/advisories/GHSA-chgr-c6px-7xpp). ([\#20131](https://github.com/element-hq/synapse/issues/20131)) From 9806ac4dbbcaf2a62c3c25651c0a2b9b4637916c Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 25 Aug 2026 11:36:35 -0500 Subject: [PATCH 24/28] Generic stream ID fix --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index edfe161d22..dd9f64ed03 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,7 +9,7 @@ ## Bugfixes -- Fix a bug where presence updates could stop being sent to clients (the presence stream position becoming stuck) if a `/sync` request was cancelled while a presence write was allocating a stream ID. Contributed by @FrenchGithubUser @Famedly. ([\#20090](https://github.com/element-hq/synapse/issues/20090)) +- Fix a bug where stream positions (presence, to-device message, etc.) could stop being sent to clients if a request was cancelled while a write was allocating a stream ID. Contributed by @FrenchGithubUser @Famedly. ([\#20090](https://github.com/element-hq/synapse/issues/20090)) - Thumbnail WebP images that use transparency as PNG rather than JPEG, to preserve transparency during thumbnailing. Contributed by @catfromplan9. ([\#20094](https://github.com/element-hq/synapse/issues/20094)) - Fix sync stream not being woken up when a user updates a profile field without belonging to any rooms. ([\#20135](https://github.com/element-hq/synapse/issues/20135)) From d27933f121a877313beff4b0008c51c327cb721a Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 25 Aug 2026 11:38:07 -0500 Subject: [PATCH 25/28] Remove specific example source that doesn't read well --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index dd9f64ed03..670a61782e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,7 +15,7 @@ ## Improved Documentation -- Document lighttpd reverse proxy configuration example from matrix.jaxlug.ngo. Contributed by JaxLUG from the Jacksonville Linux Users Group Inc.. ([\#19875](https://github.com/element-hq/synapse/issues/19875)) +- Document lighttpd reverse proxy configuration example. Contributed by JaxLUG from the Jacksonville Linux Users Group Inc.. ([\#19875](https://github.com/element-hq/synapse/issues/19875)) - Fix the documentation on the `federation_domain_whitelist` config option. ([\#20089](https://github.com/element-hq/synapse/issues/20089)) ## Internal Changes From 4245bdc6f048e8682751fd8092666184bb4033fb Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 27 Aug 2026 09:29:50 +0300 Subject: [PATCH 26/28] Fix sending down removed custom profile fields in legacy sync (#20147) When a client (correctly) calls the `DELETE` endpoint to remove a custom profile field (like Element X does with `m.status`), we incorrectly don't include it in the sync response in legacy sync. This was due to the fact that we cleaned up the sent fields down to what fields the profile currently has. Always ensure any fields in `ProfileUpdateAction.UPDATE` are sent down, as `null` values for profile fields which have been deleted. Fixes an issue where clearing a user status from Element X does not reflect in the user status being cleared on Element Web. Note, target is the v1.160.0 release branch due to customer commitments, and this fixes web and mobile clients not working together correctly. ### Pull Request Checklist * [ ] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/20147.bugfix | 1 + synapse/handlers/sync.py | 37 +++++++++++-- tests/handlers/test_sync.py | 100 +++++++++++++++++++++++++++++++++++- 3 files changed, 133 insertions(+), 5 deletions(-) create mode 100644 changelog.d/20147.bugfix diff --git a/changelog.d/20147.bugfix b/changelog.d/20147.bugfix new file mode 100644 index 0000000000..74768dc63e --- /dev/null +++ b/changelog.d/20147.bugfix @@ -0,0 +1 @@ +Fix sending custom profile field removals to legacy sync clients when the field is deleted using the profile field delete endpoint. \ No newline at end of file diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 943105415a..f8a766feee 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2420,8 +2420,22 @@ async def _generate_sync_entry_for_profile_updates( if include_users and other_user_id in include_users: # Include all the fields the client asked for, as this user # has events in a lazy loaded sync response, except for - # fields we've recently sent in a previous lazy loaded sync response - fields = set(profile_data.keys()).intersection(profile_fields) + # fields we've recently sent in a previous lazy loaded sync response. + # We must include _updated_ fields even if the profile doesn't have + # this field. The value will be sent down as `None`. We must do + # this as currently legacy sync delivers field removals by + # delivering a null value to clients, and if a field is completely + # deleted, we can't otherwise do that. The fact this field has + # a `ProfileUpdateAction.UPDATE` is enough to tell us it should + # be sent down. + # TODO once removals are sent down in a dedicated key instead of + # null values, the `.union(updated_user_fields.get(other_user_id, []))` + # part here can be removed. + fields = ( + set(profile_data.keys()) + .union(updated_user_fields.get(other_user_id, [])) + .intersection(profile_fields) + ) for field_name in fields: cache_key = ( sync_config.user.to_string(), @@ -2469,9 +2483,24 @@ async def _generate_sync_entry_for_profile_updates( if other_user_id in joined_room_user_ids else set(updated_user_fields.get(other_user_id, [])) ) - fields = set(profile_data.keys()).intersection(fields) + # We must include _updated_ fields even if the profile doesn't have + # this field. The value will be sent down as `None`. We must do + # this as currently legacy sync delivers field removals by + # delivering a null value to clients, and if a field is completely + # deleted, we can't otherwise do that. The fact this field has + # a `ProfileUpdateAction.UPDATE` is enough to tell us it should + # be sent down. + # TODO once removals are sent down in a dedicated key instead of + # null values, the `.union(updated_user_fields.get(other_user_id, []))` + # part here can be removed. + fields = ( + set(profile_data.keys()) + .union(updated_user_fields.get(other_user_id, [])) + .intersection(fields) + ) + # fields.update(set(updated_user_fields.get(other_user_id, []))) for field_name in fields: - per_user_updates[field_name] = profile_data[field_name] + per_user_updates[field_name] = profile_data.get(field_name) if per_user_updates: profile_updates[other_user_id] = per_user_updates diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index 5771e60533..b732c501d9 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -26,7 +26,7 @@ from twisted.internet import defer from twisted.internet.testing import MemoryReactor -from synapse.api.constants import AccountDataTypes, EventTypes, JoinRules +from synapse.api.constants import AccountDataTypes, EventTypes, JoinRules, ProfileFields from synapse.api.errors import Codes, ResourceLimitError from synapse.api.filtering import FilterCollection, Filtering from synapse.api.room_versions import RoomVersion, RoomVersions @@ -2109,6 +2109,104 @@ def test_incremental_sync_sends_down_null_profile_if_user_no_longer_sharing_room incremental_result.profile_updates["@other_user:test"], ) + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_incremental_sync_sends_down_deleted_fields(self, is_lazy: bool) -> None: + """ + Tests that, with `include_profile_updates_in_sync` enabled, + an incremental sync returns deleted fields as a `null` value, both for + `displayname` (stored as its own column) and for + generic custom fields (stored as JSON). + """ + # Set up a user with `displayname` and `m.status` profile fields + requester = create_requester(self.user) + other_requester = create_requester(self.other_user) + other_user = UserID.from_string(self.other_user) + filter_json: dict = { + "org.matrix.msc4429.profile_fields": {"ids": ["displayname", "m.status"]}, + } + if is_lazy: + filter_json["room"] = { + "state": { + "lazy_load_members": True, + }, + } + sync_config = generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ) + self.get_success( + self.profile_handler.set_field( + target_user=other_user, + requester=other_requester, + field_name=ProfileFields.DISPLAYNAME, + new_value="Bob", + ) + ) + self.get_success( + self.profile_handler.set_field( + target_user=other_user, + requester=other_requester, + field_name="m.status", + new_value={"text": "On holiday"}, + ) + ) + + # Do an initial sync after the point of those fields being set + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=sync_config, + request_key=generate_request_key(), + ) + ) + self.assertEqual( + initial_result.profile_updates["@other_user:test"], + {"displayname": "Bob", "m.status": {"text": "On holiday"}}, + ) + + # Delete the displayname and the `m.status` profile fields + self.get_success( + self.profile_handler.set_field( + target_user=other_user, + requester=other_requester, + field_name=ProfileFields.DISPLAYNAME, + new_value="", + ) + ) + self.get_success( + self.profile_handler.delete_profile_field( + target_user=other_user, + requester=other_requester, + field_name="m.status", + ) + ) + + # Do an incremental sync. + # Expect the deletion of both fields to be communicated in it. + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=sync_config, + request_key=generate_request_key(), + ) + ) + self.assertEqual( + incremental_result.profile_updates, + # We currently represent deleted fields as `null`, even though + # it's ambiguous (TODO MSC change) + {"@other_user:test": {"displayname": None, "m.status": None}}, + ) + @override_config({"include_profile_updates_in_sync": True}) def test_incremental_sync_sends_down_all_requested_fields_for_users_who_have_joined( self, From 955d0cf70d4ff2a2a754619eb9223cfd5be1e6b5 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 31 Aug 2026 17:25:54 -0500 Subject: [PATCH 27/28] 1.160.0rc2 --- CHANGES.md | 8 ++++++++ changelog.d/20147.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 15 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/20147.bugfix diff --git a/CHANGES.md b/CHANGES.md index 670a61782e..d6d9b22f2f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,11 @@ +# Synapse 1.160.0rc2 (2026-08-31) + +## Bugfixes + +- Fix sending custom profile field removals to legacy sync clients when the field is deleted using the profile field delete endpoint. ([\#20147](https://github.com/element-hq/synapse/issues/20147)) + + + # Synapse 1.160.0rc1 (2026-08-25) ## Features diff --git a/changelog.d/20147.bugfix b/changelog.d/20147.bugfix deleted file mode 100644 index 74768dc63e..0000000000 --- a/changelog.d/20147.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix sending custom profile field removals to legacy sync clients when the field is deleted using the profile field delete endpoint. \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index a0325ac3bc..63c2d5fcad 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.160.0~rc2) stable; urgency=medium + + * New synapse release 1.160.0rc2. + + -- Synapse Packaging team Mon, 31 Aug 2026 22:25:15 +0000 + matrix-synapse-py3 (1.160.0~rc1) stable; urgency=medium * New synapse release 1.160.0rc1. diff --git a/pyproject.toml b/pyproject.toml index 751b2805a5..69a493ee09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.160.0rc1" +version = "1.160.0rc2" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From 92fb8a06dc351c184eb96360c67b124f26c2d54b Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 2 Sep 2026 16:23:38 -0500 Subject: [PATCH 28/28] 1.160.0 --- CHANGES.md | 7 +++++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index d6d9b22f2f..6fbfa1adfd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,10 @@ +# Synapse 1.160.0 (2026-09-02) + +No significant changes since 1.160.0rc2. + + + + # Synapse 1.160.0rc2 (2026-08-31) ## Bugfixes diff --git a/debian/changelog b/debian/changelog index 63c2d5fcad..529bfb865c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.160.0) stable; urgency=medium + + * New synapse release 1.160.0. + + -- Synapse Packaging team Wed, 02 Sep 2026 21:22:51 +0000 + matrix-synapse-py3 (1.160.0~rc2) stable; urgency=medium * New synapse release 1.160.0rc2. diff --git a/pyproject.toml b/pyproject.toml index 69a493ee09..600660bd93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.160.0rc2" +version = "1.160.0" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [