From 7da440cb05d93f0b93e783bf670caaa5be518d99 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Mon, 22 Dec 2025 14:11:22 +0000 Subject: [PATCH 01/28] Add gather_optional_coroutines/7 overload --- synapse/util/async_helpers.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 1c3cd48a9c0..20a1d3e6a94 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -388,6 +388,7 @@ async def yieldable_gather_results_delaying_cancellation( T4 = TypeVar("T4") T5 = TypeVar("T5") T6 = TypeVar("T6") +T7 = TypeVar("T7") @overload @@ -517,6 +518,30 @@ async def gather_optional_coroutines( ) -> tuple[T1 | None, T2 | None, T3 | None, T4 | None, T5 | None, T6 | None]: ... +@overload +async def gather_optional_coroutines( + *coroutines: Unpack[ + tuple[ + Coroutine[Any, Any, T1] | None, + Coroutine[Any, Any, T2] | None, + Coroutine[Any, Any, T3] | None, + Coroutine[Any, Any, T4] | None, + Coroutine[Any, Any, T5] | None, + Coroutine[Any, Any, T6] | None, + Coroutine[Any, Any, T7] | None, + ] + ], +) -> tuple[ + T1 | None, + T2 | None, + T3 | None, + T4 | None, + T5 | None, + T6 | None, + T7 | None, +]: ... + + async def gather_optional_coroutines( *coroutines: Unpack[tuple[Coroutine[Any, Any, T1] | None, ...]], ) -> tuple[T1 | None, ...]: From f1e200e654ff7c5876298a2bf56b0a2f553fbb02 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 27 Feb 2026 17:13:51 +0000 Subject: [PATCH 02/28] Add explicit Absent utility type --- synapse/types/__init__.py | 65 +++++++++++++++++++++++++++++++ synapse/util/sentinel.py | 11 ++++++ tests/test_types.py | 81 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 156 insertions(+), 1 deletion(-) diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index 02889795bbd..6621145f25a 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -29,6 +29,7 @@ AbstractSet, Any, ClassVar, + Final, Literal, Mapping, Match, @@ -42,7 +43,10 @@ ) import attr +import pydantic_core.core_schema from immutabledict import immutabledict +from pydantic import GetCoreSchemaHandler +from pydantic_core import CoreSchema from signedjson.key import decode_verify_key_bytes from signedjson.types import VerifyKey from typing_extensions import Self @@ -109,6 +113,67 @@ StrSequence = tuple[str, ...] | list[str] +class AbsentType(Enum): + """ + Type of a sentinel to use as an alternative to `None` + for when we really mean 'absent' and not JSON null. + + For a Sentinel for internal (non-API-facing) use, instead consider + `Sentinel.UNSET_SENTINEL`. + + It is falsy (like None is), so shorthand forms like `x or 0` can be used. + """ + + # Making this an Enum member makes this compatible with type narrowing, + # meaning `x is not Absent` will narrow `x: int | AbsentType` to `x: int` etc. + _Absent = object() + + @classmethod + def __get_pydantic_core_schema__( + cls, source_type: object, handler: GetCoreSchemaHandler + ) -> CoreSchema: + return pydantic_core.core_schema.is_instance_schema(cls) + + def __copy__(self) -> "AbsentType": + """ + Copy implementation used by `copy.copy()`. + Always use the same instance. + + Without this and the deep version `__deepcopy__`, + `copy.copy(Absent)` on Python 3.10 (olddeps) + had a problem where it tried to construct a new Absent + as part of a deepcopy operation and resulted in: + ValueError: is not a valid AbsentType + """ + return self + + def __deepcopy__(self, memo: object) -> "AbsentType": + """ + Copy implementation used by `copy.deepcopy()`. + Always use the same instance. + """ + return self + + def __bool__(self) -> Literal[False]: + return False + + def __str__(self) -> str: + return "Absent" + + def __repr__(self) -> str: + return "Absent" + + +Absent: Final = AbsentType._Absent +""" +Sentinel to use as an alternative to `None` +for when we really mean 'absent' and not JSON null. + +For a Sentinel for internal (non-API-facing) use, instead consider +`Sentinel.UNSET_SENTINEL`. +""" + + # Note that this seems to require inheriting *directly* from Interface in order # for mypy-zope to realize it is an interface. class ISynapseThreadlessReactor( diff --git a/synapse/util/sentinel.py b/synapse/util/sentinel.py index c8434fc97a0..e885f81879a 100644 --- a/synapse/util/sentinel.py +++ b/synapse/util/sentinel.py @@ -16,6 +16,17 @@ class Sentinel(enum.Enum): + """ + Internal marker sentinel for distinguishing a default state from user-suppliable values. + Has no meaning on its own. + + Use this when you want to be absolutely sure that the marker came from Synapse code + and not from request body parsing. + + If you want a Pydantic-compatible Sentinel that is suitable for expressing + 'absent from some parsed JSON payload' or equivalent, see `Absent`. + """ + # defining a sentinel in this way allows mypy to correctly handle the # type of a dictionary lookup and subsequent type narrowing. UNSET_SENTINEL = object() diff --git a/tests/test_types.py b/tests/test_types.py index 1802f0fae3e..fb8735d8a4c 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -18,14 +18,17 @@ # [This file includes modifications made by New Vector Limited] # # - +import copy from unittest import skipUnless from immutabledict import immutabledict from parameterized import parameterized_class +from pydantic import BaseModel, PydanticInvalidForJsonSchema, ValidationError from synapse.api.errors import SynapseError from synapse.types import ( + Absent, + AbsentType, AbstractMultiWriterStreamToken, MultiWriterStreamToken, RoomAlias, @@ -199,3 +202,79 @@ def test_parse_bad_token(self) -> None: parsed_token = self.get_success(self.token_type.parse(store, "m5~")) self.assertEqual(parsed_token, self.token_type(stream=5)) + + +class AbsentTestCase(unittest.TestCase): + """ + Tests for the `Absent` utility, which is meant to be like `None` except + explicitly signalling absence rather than JSON null. + """ + + def test_cant_create_second_absent(self) -> None: + """ + Tests that we aren't allowed to instantiate a second `Absent`. + """ + with self.assertRaises(TypeError): + AbsentType() # type: ignore[call-arg] + + def test_is_falsy(self) -> None: + """ + Tests `Absent` is falsy and can therefore be used a bit like `None`. + """ + if Absent: + self.fail("Absent is truthy!") + + self.assertEqual(Absent or "something", "something") + + def test_pydantic_jsonschema(self) -> None: + """ + Tests that `Absent` can't be used to produce JSONSchema in Pydantic models. + + In the future, it may be useful to produce correct JSONSchema, but for now + I was mostly interested in making sure we don't produce weird/invalid JSONSchema. + """ + + class MyModel(BaseModel): + absent: AbsentType = Absent + + with self.assertRaises(PydanticInvalidForJsonSchema): + MyModel.model_json_schema() + + def test_pydantic_reject_null(self) -> None: + """ + Tests that `Absent` rejects `None` (JSON null) when used in Pydantic models. + """ + + class MyModel(BaseModel): + absent: AbsentType = Absent + + with self.assertRaises(ValidationError): + MyModel.model_validate({"absent": None}) + + with self.assertRaises(ValidationError): + MyModel.model_validate_json('{"absent": null}') + + def test_pydantic_accept_absence(self) -> None: + """ + Tests that `Absent` accepts the absence of a value when used in Pydantic models. + """ + + class MyModel(BaseModel): + absent: AbsentType = Absent + + self.assertEqual(MyModel.model_validate({}), MyModel(absent=Absent)) + self.assertEqual(MyModel.model_validate_json("{}"), MyModel(absent=Absent)) + + def test_copy(self) -> None: + """ + Tests that the `copy` module always uses the same instance of Absent. + """ + + class MyModel(BaseModel): + absent: AbsentType = Absent + + a = MyModel.model_validate({}) + b = copy.deepcopy(a) + + self.assertIs(copy.copy(Absent), Absent) + self.assertIs(a.absent, b.absent) From 46fab790290081a07aeb3361e983e7889fa0d94a Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 20 Mar 2026 15:36:59 +0000 Subject: [PATCH 03/28] Add NonNegativeStrictInt utility type --- synapse/types/__init__.py | 11 +++++- tests/test_types.py | 72 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index 6621145f25a..90d7c0ed724 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -27,6 +27,7 @@ from typing import ( TYPE_CHECKING, AbstractSet, + Annotated, Any, ClassVar, Final, @@ -42,10 +43,11 @@ overload, ) +import annotated_types import attr import pydantic_core.core_schema from immutabledict import immutabledict -from pydantic import GetCoreSchemaHandler +from pydantic import GetCoreSchemaHandler, StrictInt from pydantic_core import CoreSchema from signedjson.key import decode_verify_key_bytes from signedjson.types import VerifyKey @@ -174,6 +176,13 @@ def __repr__(self) -> str: """ +NonNegativeStrictInt = Annotated[StrictInt, annotated_types.Ge(0)] +"""A strict integer that must be greater than or equal to zero. + +Should be preferred in place of Pydantic's own (lax) NonNegativeInt. +""" + + # Note that this seems to require inheriting *directly* from Interface in order # for mypy-zope to realize it is an interface. class ISynapseThreadlessReactor( diff --git a/tests/test_types.py b/tests/test_types.py index fb8735d8a4c..43fd96d6f55 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -31,6 +31,7 @@ AbsentType, AbstractMultiWriterStreamToken, MultiWriterStreamToken, + NonNegativeStrictInt, RoomAlias, RoomStreamToken, UserID, @@ -278,3 +279,74 @@ class MyModel(BaseModel): self.assertIs(copy.copy(Absent), Absent) self.assertIs(a.absent, b.absent) + + +class NonNegativeStrictIntTestCase(unittest.TestCase): + """ + Tests for the `NonNegativeStrictInt` utility. + """ + + def test_pydantic_jsonschema(self) -> None: + """ + Tests that `NonNegativeStrictInt` produces sensible JSONSchema. + """ + + class MyModel(BaseModel): + limit: NonNegativeStrictInt = 100 + + self.assertEqual( + MyModel.model_json_schema(), + { + "properties": { + "limit": { + "default": 100, + "minimum": 0, + "title": "Limit", + "type": "integer", + } + }, + "title": "MyModel", + "type": "object", + }, + f"JSONSchema actually is:\n{MyModel.model_json_schema()!r}", + ) + + def test_pydantic_reject(self) -> None: + """ + Tests that `NonNegativeStrictInt` rejects negative numbers + and non-ints. + """ + + class MyModel(BaseModel): + limit: NonNegativeStrictInt = 100 + + with self.assertRaises(ValidationError): + MyModel.model_validate({"limit": -1}) + + with self.assertRaises(ValidationError): + MyModel.model_validate_json('{"limit": -1}') + + # StrictInt, so don't accept floats... + with self.assertRaises(ValidationError): + MyModel.model_validate({"limit": 1.5}) + + with self.assertRaises(ValidationError): + MyModel.model_validate_json('{"limit": 1.5}') + + # ...and don't accept stringy ints either. + with self.assertRaises(ValidationError): + MyModel.model_validate({"limit": "42"}) + + with self.assertRaises(ValidationError): + MyModel.model_validate_json('{"limit": "42"}') + + def test_pydantic_accept(self) -> None: + """ + Tests that `Absent` accepts the absence of a value when used in Pydantic models. + """ + + class MyModel(BaseModel): + limit: NonNegativeStrictInt = 100 + + self.assertEqual(MyModel.model_validate_json('{"limit": 0}'), MyModel(limit=0)) + self.assertEqual(MyModel.model_validate({"limit": 42}), MyModel(limit=42)) From 89009dfdacc9f58128ac6ee1224e5248e5472976 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Mon, 22 Dec 2025 14:11:55 +0000 Subject: [PATCH 04/28] Add fields for sticky events sliding sync extension --- synapse/types/handlers/sliding_sync.py | 22 +++++++- synapse/types/rest/client/__init__.py | 71 +++++++++++++++++++++++++- 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/synapse/types/handlers/sliding_sync.py b/synapse/types/handlers/sliding_sync.py index 694b3e1645e..63f09dca22c 100644 --- a/synapse/types/handlers/sliding_sync.py +++ b/synapse/types/handlers/sliding_sync.py @@ -47,7 +47,7 @@ ThreadSubscriptionsToken, UserID, ) -from synapse.types.rest.client import SlidingSyncBody +from synapse.types.rest.client import SlidingSyncBody, SlidingSyncStickyEventsToken from synapse.util.clock import Clock from synapse.util.duration import Duration @@ -396,12 +396,31 @@ def __bool__(self) -> bool: or bool(self.prev_batch) ) + @attr.s(slots=True, frozen=True, auto_attribs=True) + class StickyEventsExtension: + """The Sticky Events extension (MSC4354) + + Attributes: + room_id_to_sticky_events: map (room_id -> [unexpired_sticky_events]) + The events are ordered by the sticky events stream. + + The events haven't yet been deduplicated to remove + events that also appear in the timeline. + """ + + room_id_to_sticky_events: Mapping[str, list[EventBase]] + next_batch: SlidingSyncStickyEventsToken + + def __bool__(self) -> bool: + return bool(self.room_id_to_sticky_events) + to_device: ToDeviceExtension | None = None e2ee: E2eeExtension | None = None account_data: AccountDataExtension | None = None receipts: ReceiptsExtension | None = None typing: TypingExtension | None = None thread_subscriptions: ThreadSubscriptionsExtension | None = None + sticky_events: StickyEventsExtension | None = None def __bool__(self) -> bool: return bool( @@ -411,6 +430,7 @@ def __bool__(self) -> bool: or self.receipts or self.typing or self.thread_subscriptions + or self.sticky_events ) next_pos: SlidingSyncStreamToken diff --git a/synapse/types/rest/client/__init__.py b/synapse/types/rest/client/__init__.py index 49782b52348..1bb4724a694 100644 --- a/synapse/types/rest/client/__init__.py +++ b/synapse/types/rest/client/__init__.py @@ -18,9 +18,13 @@ # [This file includes modifications made by New Vector Limited] # # +import re + +import pydantic_core.core_schema from pydantic import ( ConfigDict, Field, + GetCoreSchemaHandler, StrictBool, StrictInt, StrictStr, @@ -28,9 +32,10 @@ field_validator, model_validator, ) -from pydantic_core import PydanticCustomError +from pydantic_core import CoreSchema, PydanticCustomError from typing_extensions import Annotated, Self +from synapse.types import Absent, AbsentType, NonNegativeStrictInt from synapse.types.rest import RequestBodyModel from synapse.util.threepids import validate_email @@ -107,6 +112,54 @@ class MsisdnRequestTokenBody(ThreepidRequestTokenBody): phone_number: StrictStr +class SlidingSyncStickyEventsToken: + """ + A token returned by `next_batch` of the MSC4354 Sticky Events extension to Sliding Sync + and then accepted as the `since` parameter in the requests of the same extension. + + Current format: + SlidingSyncStickyEventsToken ::= 'sticky_' DIGIT+ + DIGIT ::= '0'-'9' + + The `sticky_` prefix allows us to make sure it's not swapped for another token + or to evolve the type of token accepted with backwards compatibility in the future. + """ + + PATTERN = re.compile(r"^sticky_([0-9]+)$") + + def __init__(self, *, sticky_events_stream_id: int) -> None: + self.sticky_events_stream_id = sticky_events_stream_id + + @classmethod + def __get_pydantic_core_schema__( + cls, source_type: object, handler: GetCoreSchemaHandler + ) -> CoreSchema: + return pydantic_core.core_schema.no_info_plain_validator_function( + cls._validate, + serialization=pydantic_core.core_schema.plain_serializer_function_ser_schema( + cls.serialise, + info_arg=False, + ), + ) + + @classmethod + def _validate(cls, v: object) -> Self: + if isinstance(v, cls): + return v + if isinstance(v, str): + match = cls.PATTERN.match(v) + if match is None: + raise ValueError(f"Invalid SlidingSyncStickyEventsToken format: {v!r}") + return cls(sticky_events_stream_id=int(match.group(1))) + raise ValueError(f"Cannot parse SlidingSyncStickyEventsToken from {type(v)}") + + def serialise(self) -> str: + return f"sticky_{self.sticky_events_stream_id}" + + def __repr__(self) -> str: + return self.serialise() + + class SlidingSyncBody(RequestBodyModel): """ Sliding Sync API request body. @@ -383,6 +436,19 @@ class ThreadSubscriptionsExtension(RequestBodyModel): enabled: StrictBool | None = False limit: StrictInt = 100 + class StickyEventsExtension(RequestBodyModel): + """The Sticky Events extension (MSC4354) + + Attributes: + enabled + limit: maximum number of sticky events to return in the extension (default 100) + since: either a string with the Sticky Events since token or absent + """ + + enabled: StrictBool = False + limit: NonNegativeStrictInt = 100 + since: SlidingSyncStickyEventsToken | AbsentType = Absent + to_device: ToDeviceExtension | None = None e2ee: E2eeExtension | None = None account_data: AccountDataExtension | None = None @@ -391,6 +457,9 @@ class ThreadSubscriptionsExtension(RequestBodyModel): thread_subscriptions: ThreadSubscriptionsExtension | None = Field( None, alias="io.element.msc4308.thread_subscriptions" ) + sticky_events: StickyEventsExtension | AbsentType = Field( + Absent, alias="org.matrix.msc4354.sticky_events" + ) conn_id: StrictStr | None = None lists: ( From bb4b53e85a9bdc85387503afb36795219961321b Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Mon, 22 Dec 2025 14:21:18 +0000 Subject: [PATCH 05/28] Implement sliding sync extension for sticky events --- synapse/handlers/sliding_sync/__init__.py | 4 +- synapse/handlers/sliding_sync/extensions.py | 92 ++++++++++++++++++++- synapse/rest/client/sync.py | 89 +++++++++++++++++++- 3 files changed, 178 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index 6feb6c292e9..87cf180913c 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -259,7 +259,6 @@ async def current_sync_for_user( lists = interested_rooms.lists relevant_room_map = interested_rooms.relevant_room_map - all_rooms = interested_rooms.all_rooms room_membership_for_user_map = interested_rooms.room_membership_for_user_map relevant_rooms_to_send_map = interested_rooms.relevant_rooms_to_send_map @@ -306,6 +305,7 @@ async def handle_room(room_id: str) -> None: # extensions care about more than just normal events in the rooms (like # account data, read receipts, typing indicators, to-device messages, etc). actual_room_ids=set(relevant_room_map.keys()), + all_interested_room_ids=interested_rooms.all_rooms, actual_room_response_map=rooms, from_token=from_token, to_token=to_token, @@ -322,7 +322,7 @@ async def handle_room(room_id: str) -> None: if from_token: # The set of rooms that the client (may) care about, but aren't # in any list range (or subscribed to). - missing_rooms = all_rooms - relevant_room_map.keys() + missing_rooms = interested_rooms.all_rooms - relevant_room_map.keys() # We now just go and try fetching any events in the above rooms # to see if anything has happened since the `from_token`. diff --git a/synapse/handlers/sliding_sync/extensions.py b/synapse/handlers/sliding_sync/extensions.py index 9b7a01df142..ce00334d076 100644 --- a/synapse/handlers/sliding_sync/extensions.py +++ b/synapse/handlers/sliding_sync/extensions.py @@ -11,7 +11,6 @@ # See the GNU Affero General Public License for more details: # . # - import itertools import logging from collections import ChainMap @@ -26,11 +25,13 @@ from typing_extensions import TypeAlias, assert_never -from synapse.api.constants import AccountDataTypes, EduTypes +from synapse.api.constants import AccountDataTypes, EduTypes, StickyEvent +from synapse.events import EventBase from synapse.handlers.receipts import ReceiptEventSource from synapse.logging.opentracing import trace from synapse.storage.databases.main.receipts import ReceiptInRoom from synapse.types import ( + Absent, DeviceListUpdates, JsonMapping, MultiWriterStreamToken, @@ -47,10 +48,12 @@ SlidingSyncConfig, SlidingSyncResult, ) +from synapse.types.rest.client import SlidingSyncStickyEventsToken from synapse.util.async_helpers import ( concurrently_execute, gather_optional_coroutines, ) +from synapse.visibility import filter_and_transform_events_for_client _ThreadSubscription: TypeAlias = ( SlidingSyncResult.Extensions.ThreadSubscriptionsExtension.ThreadSubscription @@ -73,7 +76,10 @@ def __init__(self, hs: "HomeServer"): self.event_sources = hs.get_event_sources() self.device_handler = hs.get_device_handler() self.push_rules_handler = hs.get_push_rules_handler() + self.clock = hs.get_clock() + self._storage_controllers = hs.get_storage_controllers() self._enable_thread_subscriptions = hs.config.experimental.msc4306_enabled + self._enable_sticky_events = hs.config.experimental.msc4354_enabled @trace async def get_extensions_response( @@ -83,6 +89,7 @@ async def get_extensions_response( new_connection_state: "MutablePerConnectionState", actual_lists: Mapping[str, SlidingSyncResult.SlidingWindowList], actual_room_ids: set[str], + all_interested_room_ids: set[str], actual_room_response_map: Mapping[str, SlidingSyncResult.RoomResult], to_token: StreamToken, from_token: SlidingSyncStreamToken | None, @@ -97,6 +104,9 @@ async def get_extensions_response( actual_lists: Sliding window API. A map of list key to list results in the Sliding Sync response. actual_room_ids: The actual room IDs in the the Sliding Sync response. + all_interested_room_ids: The IDs of all rooms that the client is interested in, + even if they don't appear in the current limited window. + See `SlidingSyncInterestedRooms.all_rooms`. actual_room_response_map: A map of room ID to room results in the the Sliding Sync response. to_token: The latest point in the stream to sync up to. @@ -174,6 +184,19 @@ async def get_extensions_response( from_token=from_token, ) + sticky_events_coro = None + if ( + sync_config.extensions.sticky_events is not Absent + and self._enable_sticky_events + ): + sticky_events_coro = self.get_sticky_events_extension_response( + sync_config=sync_config, + sticky_events_request=sync_config.extensions.sticky_events, + all_interested_room_ids=all_interested_room_ids, + to_token=to_token, + from_token=from_token, + ) + ( to_device_response, e2ee_response, @@ -181,6 +204,7 @@ async def get_extensions_response( receipts_response, typing_response, thread_subs_response, + sticky_events_response, ) = await gather_optional_coroutines( to_device_coro, e2ee_coro, @@ -188,6 +212,7 @@ async def get_extensions_response( receipts_coro, typing_coro, thread_subs_coro, + sticky_events_coro, ) return SlidingSyncResult.Extensions( @@ -197,6 +222,7 @@ async def get_extensions_response( receipts=receipts_response, typing=typing_response, thread_subscriptions=thread_subs_response, + sticky_events=sticky_events_response, ) def find_relevant_room_ids_for_extension( @@ -967,3 +993,65 @@ async def get_thread_subscriptions_extension_response( unsubscribed=unsubscribed_threads, prev_batch=prev_batch, ) + + async def get_sticky_events_extension_response( + self, + sync_config: SlidingSyncConfig, + sticky_events_request: SlidingSyncConfig.Extensions.StickyEventsExtension, + all_interested_room_ids: set[str], + to_token: StreamToken, + from_token: SlidingSyncStreamToken | None, + ) -> SlidingSyncResult.Extensions.StickyEventsExtension | None: + if not sticky_events_request.enabled: + return None + now = self.clock.time_msec() + since_token = sticky_events_request.since or SlidingSyncStickyEventsToken( + sticky_events_stream_id=0 + ) + ( + sticky_events_to_id, + room_to_event_ids, + ) = await self.store.get_sticky_events_in_rooms( + all_interested_room_ids, + from_id=since_token.sticky_events_stream_id, + to_id=to_token.sticky_events_key, + now=now, + limit=min(sticky_events_request.limit, StickyEvent.MAX_EVENTS_IN_SYNC), + ) + # No need to preserve sticky event order here because we will + # reassemble it in the right order after. + all_sticky_event_ids = { + ev_id for evs in room_to_event_ids.values() for ev_id in evs + } + unfiltered_events = await self.store.get_events_as_list(all_sticky_event_ids) + filtered_events = await filter_and_transform_events_for_client( + self._storage_controllers, + sync_config.user.to_string(), + unfiltered_events, + # As per MSC4354: + # > History visibility checks MUST NOT be applied to sticky events. + # > Any joined user is authorised to see sticky events for the duration they remain sticky. + always_include_ids=frozenset(all_sticky_event_ids), + ) + filtered_event_map = {ev.event_id: ev for ev in filtered_events} + + room_id_to_sticky_events: dict[str, list[EventBase]] = {} + for room_id, sticky_event_ids in room_to_event_ids.items(): + filtered_events_for_room = [ + filtered_event_map[event_id] + # This reintroduces the correct order + # (by the sticky events stream) + for event_id in sticky_event_ids + if event_id in filtered_event_map + ] + if len(filtered_events_for_room) == 0: + continue + + room_id_to_sticky_events[room_id] = filtered_events_for_room + + return SlidingSyncResult.Extensions.StickyEventsExtension( + room_id_to_sticky_events=room_id_to_sticky_events, + next_batch=SlidingSyncStickyEventsToken( + sticky_events_stream_id=sticky_events_to_id + ), + ) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 710d097eab0..ab41353e8ef 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -21,7 +21,7 @@ import itertools import logging from collections import defaultdict -from typing import TYPE_CHECKING, Any, Mapping +from typing import TYPE_CHECKING, Any, Literal, Mapping import attr @@ -656,6 +656,7 @@ class SlidingSyncRestServlet(RestServlet): - receipts (MSC3960) - account data (MSC3959) - thread subscriptions (MSC4308) + - sticky events (MSC4354) Request query parameters: timeout: How long to wait for new events in milliseconds. @@ -879,7 +880,7 @@ async def encode_response( requester, sliding_sync_result.rooms ) response["extensions"] = await self.encode_extensions( - requester, sliding_sync_result.extensions + requester, sliding_sync_result.extensions, sliding_sync_result.rooms ) return response @@ -1029,8 +1030,18 @@ async def encode_rooms( @trace_with_opname("sliding_sync.encode_extensions") async def encode_extensions( - self, requester: Requester, extensions: SlidingSyncResult.Extensions + self, + requester: Requester, + extensions: SlidingSyncResult.Extensions, + ref_rooms_results: Mapping[str, SlidingSyncResult.RoomResult], ) -> JsonDict: + """ + Args: + ref_rooms_results: + Map of room ID -> RoomResult that was serialised as the `room` section + of the Sliding Sync response. + Will not be mutated, only used for reading. + """ serialized_extensions: JsonDict = {} if extensions.to_device is not None: @@ -1099,8 +1110,80 @@ async def encode_extensions( _serialise_thread_subscriptions(extensions.thread_subscriptions) ) + if extensions.sticky_events: + serialized_extensions[ + "org.matrix.msc4354.sticky_events" + ] = await self._serialise_sticky_events( + requester, extensions.sticky_events, ref_rooms_results + ) + return serialized_extensions + async def _serialise_sticky_events( + self, + requester: Requester, + sticky_events: SlidingSyncResult.Extensions.StickyEventsExtension, + ref_rooms_results: Mapping[str, SlidingSyncResult.RoomResult], + ) -> JsonDict: + """ + Serialise the sticky events extension response. + + This includes deduplicating by filtering out sticky events + from this extension that already appeared in the timeline + section. + + Args: + ref_rooms_results: + Map of room ID -> RoomResult that was serialised as the `room` section + of the Sliding Sync response. + Will not be mutated, only used for reading. + """ + + time_now = self.clock.time_msec() + # Same as SSS timelines. + # + serialize_options = SerializeEventConfig( + event_format=format_event_for_client_v2_without_room_id, + requester=requester, + ) + + rooms_out: dict[str, dict[Literal["events"], list[JsonDict]]] = {} + for ( + room_id, + possibly_duplicated_sticky_events, + ) in sticky_events.room_id_to_sticky_events.items(): + # As per MSC4354: + # Remove sticky events that are already in the timeline, else we will needlessly duplicate + # events. + # There is no purpose in including sticky events in the sticky section if they're already in + # the timeline, as either way the client becomes aware of them. + # This is particularly important given the risk of sticky events spam since + # anyone can send sticky events, so halving the bandwidth on average for each sticky + # event is helpful. + room_result = ref_rooms_results.get(room_id) + if room_result is None: + # Nothing to deduplicate + sticky_events_to_write = possibly_duplicated_sticky_events + else: + sent_event_ids_in_room_section = { + ev.event_id for ev in room_result.timeline_events + } + sticky_events_to_write = [ + ev + for ev in possibly_duplicated_sticky_events + if ev.event_id not in sent_event_ids_in_room_section + ] + rooms_out[room_id] = { + "events": await self.event_serializer.serialize_events( + sticky_events_to_write, time_now, config=serialize_options + ) + } + + return { + "rooms": rooms_out, + "next_batch": sticky_events.next_batch.serialise(), + } + def _serialise_thread_subscriptions( thread_subscriptions: SlidingSyncResult.Extensions.ThreadSubscriptionsExtension, From ff8f6b32f232cb322278c8bac25e5004999ae59d Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 27 Feb 2026 17:12:39 +0000 Subject: [PATCH 06/28] Add sliding sync extension test --- .../test_extension_sticky_events.py | 616 ++++++++++++++++++ 1 file changed, 616 insertions(+) create mode 100644 tests/rest/client/sliding_sync/test_extension_sticky_events.py diff --git a/tests/rest/client/sliding_sync/test_extension_sticky_events.py b/tests/rest/client/sliding_sync/test_extension_sticky_events.py new file mode 100644 index 00000000000..de4827a755c --- /dev/null +++ b/tests/rest/client/sliding_sync/test_extension_sticky_events.py @@ -0,0 +1,616 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +import logging +import sqlite3 + +from twisted.internet.testing import MemoryReactor + +import synapse.rest.admin +import synapse.rest.client.account_data +from synapse.api.constants import EventTypes, EventUnsignedContentFields +from synapse.rest.client import account_data, login, register, room, sync +from synapse.server import HomeServer +from synapse.types import JsonDict +from synapse.util.clock import Clock +from synapse.util.duration import Duration + +from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase +from tests.server import TimedOutException +from tests.utils import USE_POSTGRES_FOR_TESTS + +logger = logging.getLogger(__name__) + + +DUMMY_LISTS = { + "main": { + # Don't include any rooms in the top-N window + "ranges": [[0, 0]], + "required_state": [], + "timeline_limit": 0, + } +} +""" +Subscription lists that can be used in the Sliding Sync request `lists` field, +which sets up a subscription that is interested in all rooms but does not let any rooms into the window, +thus does not return any timelines. + +Sufficient to get sticky event updates as per MSC4354: + +> The server MUST include sticky events across all rooms that would be matched by at least one subscription list +> (i.e. all rooms that the client is interested in), even if the room does not appear in top-N window for that +> subscription list at this time. +> Rooms that would not be matched by a list are not included, as this means the client is not interested +> in those rooms. +> +> — https://github.com/matrix-org/matrix-spec-proposals/pull/4354/changes#diff-d76bc1a1d612c6da37d024f5b57f7b8352939b8db8a7ee9c6b71c1a848359afdR213-R217 +""" + + +class SlidingSyncStickyEventsExtensionTestCase(SlidingSyncBase): + """Tests for the sticky events sliding sync extension""" + + if not USE_POSTGRES_FOR_TESTS and sqlite3.sqlite_version_info < (3, 40, 0): + # We need the JSON functionality in SQLite + skip = f"SQLite version is too old to support sticky events: {sqlite3.sqlite_version_info} (See https://github.com/element-hq/synapse/issues/19428)" + + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + register.register_servlets, + room.register_servlets, + sync.register_servlets, + account_data.register_servlets, + ] + + def default_config(self) -> JsonDict: + config = super().default_config() + # Enable sliding sync and sticky events MSCs + config["experimental_features"] = { + "msc3575_enabled": True, + "msc4354_enabled": True, + } + return config + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + super().prepare(reactor, clock, hs) + + def _assert_sticky_events_response( + self, + response_body: JsonDict, + expected_events_by_room: dict[str, list[str]] | None, + ) -> str | None: + """Assert the sliding sync response was successful and has the expected + sticky events. + + Args: + response_body: Sliding Sync response body + expected_events_by_room: + map of room ID to list of event IDs to expect (in the order we expect them), + or None if we expect an empty sticky events extension response + + Returns the next_batch token from the sticky events section, + unless we're expecting an empty response. + """ + extensions = response_body["extensions"] + sticky_events = extensions.get("org.matrix.msc4354.sticky_events") + + # If there are no expected events, we shouldn't get anything in the response + if expected_events_by_room is None: + self.assertIsNone(sticky_events) + return None + + self.assertIsNotNone(sticky_events) + self.assertIsInstance(sticky_events["next_batch"], str) + + actual_rooms = sticky_events["rooms"] + # Check that we have the expected rooms + self.assertEqual(set(actual_rooms.keys()), set(expected_events_by_room.keys())) + + # Check the events in each room + for room_id, expected_events in expected_events_by_room.items(): + actual_events = actual_rooms[room_id]["events"] + actual_event_ids = [e["event_id"] for e in actual_events] + self.assertEqual(actual_event_ids, expected_events) + for actual_event in actual_events: + # Check the sticky TTL is sent + self.assertIn("unsigned", actual_event) + ttl = actual_event["unsigned"][EventUnsignedContentFields.STICKY_TTL] + self.assertIsInstance(ttl, int) + + self.assertIn("next_batch", sticky_events) + return sticky_events["next_batch"] + + def test_empty_sync(self) -> None: + """Test that enabling sticky events extension works on initial and incremental sync, + even if there is no data. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + sync_body = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + # No sticky events in initial sync. + self._assert_sticky_events_response(response_body, None) + + # Incremental sync should also have no sticky events + response_body, _ = self.do_sync( + sync_body, since=response_body["pos"], tok=user1_tok + ) + self._assert_sticky_events_response(response_body, None) + + def test_initial_sync(self) -> None: + """Test that we get sticky events when we don't specify a since token + (initial sync). + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("u2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a room and join both users + room_id = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # Send a sticky event from user2 + sticky_event_id: str = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(minutes=5), + content={"body": "sticky message", "msgtype": "m.text"}, + tok=user2_tok, + )["event_id"] + + # Initial sync should return the sticky event + sync_body: JsonDict = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Assert the response and then get the next_batch for the next sliding sync request + next_batch = self._assert_sticky_events_response( + response_body, {room_id: [sticky_event_id]} + ) + assert next_batch is not None + + # Do an incremental sync immediately again + sync_body = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + "since": next_batch, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Check we don't get that event again + self._assert_sticky_events_response(response_body, None) + + # Send another sticky event + sticky_event_id2: str = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(minutes=5), + content={"body": "another sticky message", "msgtype": "m.text"}, + tok=user1_tok, + )["event_id"] + + # Now the incremental sync should give us that event + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + self._assert_sticky_events_response( + response_body, {room_id: [sticky_event_id2]} + ) + + def test_expired_events_not_returned(self) -> None: + """Test that expired sticky events are not returned.""" + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("u2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a room + room_id = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # Send a sticky event with a short duration + sticky_event_id = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(seconds=2), + content={"body": "sticky message", "msgtype": "m.text"}, + tok=user2_tok, + )["event_id"] + + # Initial sync should return the sticky event + sync_body = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # We should still get the event for now + self._assert_sticky_events_response(response_body, {room_id: [sticky_event_id]}) + + # Advance time past the sticky duration + self.reactor.advance(3) + + # A second initial sync should not return the expired sticky event + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + self._assert_sticky_events_response(response_body, None) + + def test_wait_for_new_data(self) -> None: + """Test that the sliding sync request waits for new sticky events to arrive. + (Only applies to incremental syncs with a `timeout` specified). + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("u2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a room + room_id = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # Initial sync with no sticky events + sync_body = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + } + }, + } + _, from_token = self.do_sync(sync_body, tok=user1_tok) + + # Make the sliding sync request with a timeout + channel = self.make_request( + "POST", + self.sync_endpoint + "?timeout=10000" + f"&pos={from_token}", + content=sync_body, + access_token=user1_tok, + await_result=False, + ) + + # Block for 5 seconds to make sure we are in `notifier.wait_for_events(...)` + with self.assertRaises(TimedOutException): + channel.await_result(timeout_ms=5000) + + # Send a sticky event to trigger new results + sticky_event_id = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(minutes=5), + content={"body": "sticky message", "msgtype": "m.text"}, + tok=user2_tok, + )["event_id"] + + # Should respond before the 10 second timeout + channel.await_result(timeout_ms=100) + self.assertEqual(channel.code, 200, channel.json_body) + + self._assert_sticky_events_response( + channel.json_body, + {room_id: [sticky_event_id]}, + ) + + def test_ignored_users_sticky_events(self) -> None: + """ + Test that sticky events from ignored users are not delivered to clients. + + > As with normal events, sticky events sent by ignored users MUST NOT be + > delivered to clients. + > — https://github.com/matrix-org/matrix-spec-proposals/blob/4340903c15e9eab1bfb2f6a31cfa08fd535f7e7c/proposals/4354-sticky-events.md#sync-api-changes + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a room + room_id = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # User1 ignores user2 + channel = self.make_request( + "PUT", + f"/_matrix/client/v3/user/{user1_id}/account_data/m.ignored_user_list", + {"ignored_users": {user2_id: {}}}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + # User2 sends a sticky event + sticky_event_id = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(minutes=5), + content={"body": "sticky from ignored user", "msgtype": "m.text"}, + tok=user2_tok, + )["event_id"] + + # Initial sync for user1 + sync_body = { + "lists": { + "main": { + "ranges": [[0, 10]], + "required_state": [], + # In this test we ask for 10 events of timeline. + "timeline_limit": 10, + } + }, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Timeline events should not include sticky event from ignored user + timeline_events = response_body["rooms"][room_id]["timeline"] + timeline_event_ids = [e["event_id"] for e in timeline_events] + + self.assertNotIn( + sticky_event_id, + timeline_event_ids, + "Sticky event from ignored user should not be in timeline", + ) + + # Sticky events section should also not include the event from ignored user + self._assert_sticky_events_response(response_body, None) + + def test_history_visibility_bypass_for_sticky_events(self) -> None: + """ + Test that joined users can see sticky events even when history visibility + is set to "joined" and they joined after the event was sent. + + > History visibility checks MUST NOT be applied to sticky events. + > Any joined user is authorised to see sticky events for the duration they remain sticky. + > — https://github.com/matrix-org/matrix-spec-proposals/blob/4340903c15e9eab1bfb2f6a31cfa08fd535f7e7c/proposals/4354-sticky-events.md#proposal + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + # Create a room with restrictive history visibility + room_id = self.helper.create_room_as( + user1_id, + tok=user1_tok, + extra_content={ + # Anyone can join + "preset": "public_chat", + # But you can't see history before you joined + "initial_state": [ + { + "type": EventTypes.RoomHistoryVisibility, + "state_key": "", + "content": {"history_visibility": "joined"}, + } + ], + }, + is_public=False, + ) + + # User1 sends a sticky event + sticky_event_id = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(minutes=5), + content={"body": "sticky message", "msgtype": "m.text"}, + tok=user1_tok, + )["event_id"] + + # User1 also sends a regular event, to verify our test setup + regular_event_id = self.helper.send( + room_id=room_id, + body="regular message", + tok=user1_tok, + )["event_id"] + + # Register and join a second user after the sticky event was sent + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + self.helper.join(room_id, user2_id, tok=user2_tok) + + # User2 syncs - they should see sticky event even though + # history visibility is "joined" and they joined after it was sent + sync_body = { + "lists": { + "main": { + "ranges": [[0, 10]], + "required_state": [], + # In this test, we ask for 10 events of timeline. + "timeline_limit": 10, + } + }, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user2_tok) + + # The sticky event is fully visible in its own right, + # but AFAICT the timeline only includes events since we join the room + # (regardless of history visibility), + # so this comes down in the sticky extension + self._assert_sticky_events_response(response_body, {room_id: [sticky_event_id]}) + + # Instead the sticky event is in the timeline + timeline_events = response_body["rooms"][room_id]["timeline"] + timeline_event_ids = [e["event_id"] for e in timeline_events] + self.assertNotIn( + regular_event_id, + timeline_event_ids, + f"Expecting to not see regular event ({regular_event_id}) before user1 joined.", + ) + + def test_sticky_event_pagination(self) -> None: + """ + Test that pagination works correctly when there are many sticky events. + Also check they are delivered in stream order. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a room + room_id = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # Send 4 sticky events (more than our limit of 2) + sticky_event_ids: list[str] = [] + for i in range(4): + event_id = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(minutes=5), + content={"body": f"sticky message {i}", "msgtype": "m.text"}, + tok=user2_tok, + )["event_id"] + sticky_event_ids.append(event_id) + + # Initial sync + sync_body = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": {"enabled": True, "limit": 2} + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # We expect to see the first 2 sticky events by stream order + # and they should be in that stream order + next_batch = self._assert_sticky_events_response( + response_body, {room_id: sticky_event_ids[0:2]} + ) + + # Incremental sync to get remaining sticky events + sync_body = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + # This makes it incremental + "since": next_batch, + "limit": 3, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Should get remaining events, in stream order again + self._assert_sticky_events_response( + response_body, {room_id: sticky_event_ids[2:4]} + ) + + def test_deduplication_with_timeline(self) -> None: + """ + Test that sticky events are not included in the sticky event extension of sliding sync + if they are included in the main timeline section. + + Send 3 events: + 1. sticky + 2. sticky + 3. regular + + We then will sync with a timeline limit of 2 and a sticky event limit of 2. + We should then see (2) and (3) included in the timeline + and (1) in the sticky event response (but not (2) because it's already + included in the timeline.) + + 1. sticky [in sticky section] + + ------------->>> Timeline section + 2. sticky + 3. regular + -------------<<< + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + room_id = self.helper.create_room_as(user1_id, tok=user1_tok) + + sticky_event_ids: list[str] = [] + for i in range(2): + event_id = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(minutes=5), + content={"body": f"sticky message {i}", "msgtype": "m.text"}, + tok=user1_tok, + )["event_id"] + sticky_event_ids.append(event_id) + + non_sticky_event_id = self.helper.send_event( + room_id, + EventTypes.Message, + content={"body": "regular message", "msgtype": "m.text"}, + tok=user1_tok, + )["event_id"] + + # Sync + sync_body = { + "lists": { + "main": { + "ranges": [[0, 10]], + "required_state": [], + # In this test, we want a timeline window of the 2 latest messages + "timeline_limit": 2, + } + }, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + "limit": 2, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + events_in_sticky_section = response_body["extensions"][ + "org.matrix.msc4354.sticky_events" + ]["rooms"][room_id]["events"] + event_ids_in_sticky_section = [e["event_id"] for e in events_in_sticky_section] + + events_in_timeline_section = response_body["rooms"][room_id]["timeline"] + event_ids_in_timeline_section = [ + e["event_id"] for e in events_in_timeline_section + ] + + self.assertEqual( + event_ids_in_sticky_section, + [sticky_event_ids[0]], + ) + self.assertEqual( + event_ids_in_timeline_section, [sticky_event_ids[1], non_sticky_event_id] + ) From 6b4cf724161b42a001e6adef2bd2b507001c4973 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 27 Feb 2026 19:14:23 +0000 Subject: [PATCH 07/28] drive-by docstring tweak on ordering --- synapse/visibility.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/visibility.py b/synapse/visibility.py index 5ba2a14a24a..ca892cf6bd0 100644 --- a/synapse/visibility.py +++ b/synapse/visibility.py @@ -104,6 +104,7 @@ async def filter_and_transform_events_for_client( Returns: The filtered events. The `unsigned` data is annotated with the membership state of `user_id` at each event. + The events are returned in the same order. """ # Filter out events that have been soft failed so that we don't relay them # to clients, unless they're a server admin and want that to happen. From d9a1c430a602f166bed2096bc59e99de65c14a67 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 20 Mar 2026 14:45:58 +0000 Subject: [PATCH 08/28] Newsfile Signed-off-by: Olivier 'reivilibre --- changelog.d/19591.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/19591.feature diff --git a/changelog.d/19591.feature b/changelog.d/19591.feature new file mode 100644 index 00000000000..f7800308070 --- /dev/null +++ b/changelog.d/19591.feature @@ -0,0 +1 @@ +Expose [MSC4354 Sticky Events](https://github.com/matrix-org/matrix-spec-proposals/pull/4354) over [MSC4186 (Simplified) Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186). \ No newline at end of file From 9f153a3a17873e1f289bf760bbacca14bc6ab662 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 20 Mar 2026 15:41:00 +0000 Subject: [PATCH 09/28] drive-by docstring typo fix --- synapse/handlers/sliding_sync/extensions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/sliding_sync/extensions.py b/synapse/handlers/sliding_sync/extensions.py index ce00334d076..47a1879b11e 100644 --- a/synapse/handlers/sliding_sync/extensions.py +++ b/synapse/handlers/sliding_sync/extensions.py @@ -99,7 +99,7 @@ async def get_extensions_response( Args: sync_config: Sync configuration new_connection_state: Snapshot of the current per-connection state - new_per_connection_state: A mutable copy of the per-connection + new_connection_state: A mutable copy of the per-connection state, used to record updates to the state during this request. actual_lists: Sliding window API. A map of list key to list results in the Sliding Sync response. From 11d2ac7044ea2d35e258ab2be00abfd8ecb41538 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 20 Mar 2026 16:49:24 +0000 Subject: [PATCH 10/28] Bump Pydantic to >= 2.10 Fixes builtins.NotImplementedError: Cannot check isinstance when validating from json, use a JsonOrPython validator instead. --- poetry.lock | 46 +++++++++++++++++++++++----------------------- pyproject.toml | 2 +- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/poetry.lock b/poetry.lock index 36d3035277e..eddb1c36caf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,7 +31,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" +markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" files = [ {file = "authlib-1.6.9-py2.py3-none-any.whl", hash = "sha256:f08b4c14e08f0861dc18a32357b33fbcfd2ea86cfe3fe149484b4d764c4a0ac3"}, {file = "authlib-1.6.9.tar.gz", hash = "sha256:d8f2421e7e5980cc1ddb4e32d3f5fa659cfaf60d8eaf3281ebed192e4ab74f04"}, @@ -531,7 +531,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -556,7 +556,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "elementpath-4.8.0-py3-none-any.whl", hash = "sha256:5393191f84969bcf8033b05ec4593ef940e58622ea13cefe60ecefbbf09d58d9"}, {file = "elementpath-4.8.0.tar.gz", hash = "sha256:5822a2560d99e2633d95f78694c7ff9646adaa187db520da200a8e9479dc46ae"}, @@ -606,7 +606,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "hiredis-3.3.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:9937d9b69321b393fbace69f55423480f098120bc55a3316e1ca3508c4dbbd6f"}, {file = "hiredis-3.3.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:50351b77f89ba6a22aff430b993653847f36b71d444509036baa0f2d79d1ebf4"}, @@ -930,7 +930,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1122,7 +1122,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1239,7 +1239,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"url-preview\"" +markers = "extra == \"url-preview\" or extra == \"all\"" files = [ {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e77dd455b9a16bbd2a5036a63ddbd479c19572af81b624e79ef422f929eef388"}, {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d444858b9f07cefff6455b983aea9a67f7462ba1f6cbe4a21e8bf6791bf2153"}, @@ -1553,7 +1553,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "matrix_synapse_ldap3-0.4.0-py3-none-any.whl", hash = "sha256:bf080037230d2af5fd3639cb87266de65c1cad7a68ea206278c5b4bf9c1a17f3"}, {file = "matrix_synapse_ldap3-0.4.0.tar.gz", hash = "sha256:cff52ba780170de5e6e8af42863d2648ee23f3bf0a9fea6db52372f9fc00be2b"}, @@ -1834,7 +1834,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -2032,7 +2032,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"postgres\"" +markers = "extra == \"postgres\" or extra == \"all\"" files = [ {file = "psycopg2-2.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:103e857f46bb76908768ead4e2d0ba1d1a130e7b8ed77d3ae91e8b33481813e8"}, {file = "psycopg2-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:210daed32e18f35e3140a1ebe059ac29209dd96468f2f7559aa59f75ee82a5cb"}, @@ -2050,7 +2050,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -2066,7 +2066,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -2348,7 +2348,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"cache-memory\"" +markers = "extra == \"cache-memory\" or extra == \"all\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2480,7 +2480,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2505,7 +2505,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -2533,7 +2533,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pytz-2026.1.post1-py2.py3-none-any.whl", hash = "sha256:f2fd16142fda348286a75e1a524be810bb05d444e5a081f37f7affc635035f7a"}, {file = "pytz-2026.1.post1.tar.gz", hash = "sha256:3378dde6a0c3d26719182142c56e60c7f9af7e968076f31aae569d72a0358ee1"}, @@ -2937,7 +2937,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"sentry\"" +markers = "extra == \"sentry\" or extra == \"all\"" files = [ {file = "sentry_sdk-2.54.0-py2.py3-none-any.whl", hash = "sha256:fd74e0e281dcda63afff095d23ebcd6e97006102cdc8e78a29f19ecdf796a0de"}, {file = "sentry_sdk-2.54.0.tar.gz", hash = "sha256:2620c2575128d009b11b20f7feb81e4e4e8ae08ec1d36cbc845705060b45cc1b"}, @@ -3136,7 +3136,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -3152,7 +3152,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "thrift-0.22.0.tar.gz", hash = "sha256:42e8276afbd5f54fe1d364858b6877bc5e5a4a5ed69f6a005b94ca4918fe1466"}, ] @@ -3227,7 +3227,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "tornado-6.5.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:487dc9cc380e29f58c7ab88f9e27cdeef04b2140862e5076a66fb6bb68bb1bfa"}, {file = "tornado-6.5.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:65a7f1d46d4bb41df1ac99f5fcb685fb25c7e61613742d5108b010975a9a6521"}, @@ -3359,7 +3359,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3620,7 +3620,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "xmlschema-2.5.1-py3-none-any.whl", hash = "sha256:ec2b2a15c8896c1fcd14dcee34ca30032b99456c3c43ce793fdb9dca2fb4b869"}, {file = "xmlschema-2.5.1.tar.gz", hash = "sha256:4f7497de6c8b6dc2c28ad7b9ed6e21d186f4afe248a5bea4f54eedab4da44083"}, @@ -3754,4 +3754,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = ">=3.10.0,<4.0.0" -content-hash = "ce9ac9da9e7ffaf24b3e1e7892342ba486e7af4ea25385f875d0f3a2d5c5d133" +content-hash = "ab21711922ec9bc72f506a7ab8300a73487d5cc66b75c819931a1ea5fd8d33ec" diff --git a/pyproject.toml b/pyproject.toml index adb9993aae8..c5fd38e93a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ dependencies = [ "matrix-common>=1.3.0,<2.0.0", # We need packaging.verison.Version(...).major added in 20.0. "packaging>=20.0", - "pydantic>=2.8;python_version < '3.14'", + "pydantic>=2.10;python_version < '3.14'", "pydantic>=2.12;python_version >= '3.14'", # This is for building the rust components during "poetry install", which From ea610b6512744c4d6c739639d41258448cb31439 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 31 Mar 2026 17:41:03 +0100 Subject: [PATCH 11/28] Revert "Bump Pydantic to >= 2.10" This reverts commit 6883dcdc43b9a8aa5537de26a7a44a8b8ded319e. EPEL 10 only had 2.9.2 so try to work around issue without updating --- poetry.lock | 46 +++++++++++++++++++++++----------------------- pyproject.toml | 2 +- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/poetry.lock b/poetry.lock index eddb1c36caf..36d3035277e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,7 +31,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" files = [ {file = "authlib-1.6.9-py2.py3-none-any.whl", hash = "sha256:f08b4c14e08f0861dc18a32357b33fbcfd2ea86cfe3fe149484b4d764c4a0ac3"}, {file = "authlib-1.6.9.tar.gz", hash = "sha256:d8f2421e7e5980cc1ddb4e32d3f5fa659cfaf60d8eaf3281ebed192e4ab74f04"}, @@ -531,7 +531,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -556,7 +556,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "elementpath-4.8.0-py3-none-any.whl", hash = "sha256:5393191f84969bcf8033b05ec4593ef940e58622ea13cefe60ecefbbf09d58d9"}, {file = "elementpath-4.8.0.tar.gz", hash = "sha256:5822a2560d99e2633d95f78694c7ff9646adaa187db520da200a8e9479dc46ae"}, @@ -606,7 +606,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "hiredis-3.3.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:9937d9b69321b393fbace69f55423480f098120bc55a3316e1ca3508c4dbbd6f"}, {file = "hiredis-3.3.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:50351b77f89ba6a22aff430b993653847f36b71d444509036baa0f2d79d1ebf4"}, @@ -930,7 +930,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1122,7 +1122,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1239,7 +1239,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"url-preview\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"url-preview\"" files = [ {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e77dd455b9a16bbd2a5036a63ddbd479c19572af81b624e79ef422f929eef388"}, {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d444858b9f07cefff6455b983aea9a67f7462ba1f6cbe4a21e8bf6791bf2153"}, @@ -1553,7 +1553,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "matrix_synapse_ldap3-0.4.0-py3-none-any.whl", hash = "sha256:bf080037230d2af5fd3639cb87266de65c1cad7a68ea206278c5b4bf9c1a17f3"}, {file = "matrix_synapse_ldap3-0.4.0.tar.gz", hash = "sha256:cff52ba780170de5e6e8af42863d2648ee23f3bf0a9fea6db52372f9fc00be2b"}, @@ -1834,7 +1834,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -2032,7 +2032,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"postgres\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"postgres\"" files = [ {file = "psycopg2-2.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:103e857f46bb76908768ead4e2d0ba1d1a130e7b8ed77d3ae91e8b33481813e8"}, {file = "psycopg2-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:210daed32e18f35e3140a1ebe059ac29209dd96468f2f7559aa59f75ee82a5cb"}, @@ -2050,7 +2050,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -2066,7 +2066,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -2348,7 +2348,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"cache-memory\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"cache-memory\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2480,7 +2480,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2505,7 +2505,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -2533,7 +2533,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pytz-2026.1.post1-py2.py3-none-any.whl", hash = "sha256:f2fd16142fda348286a75e1a524be810bb05d444e5a081f37f7affc635035f7a"}, {file = "pytz-2026.1.post1.tar.gz", hash = "sha256:3378dde6a0c3d26719182142c56e60c7f9af7e968076f31aae569d72a0358ee1"}, @@ -2937,7 +2937,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"sentry\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"sentry\"" files = [ {file = "sentry_sdk-2.54.0-py2.py3-none-any.whl", hash = "sha256:fd74e0e281dcda63afff095d23ebcd6e97006102cdc8e78a29f19ecdf796a0de"}, {file = "sentry_sdk-2.54.0.tar.gz", hash = "sha256:2620c2575128d009b11b20f7feb81e4e4e8ae08ec1d36cbc845705060b45cc1b"}, @@ -3136,7 +3136,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -3152,7 +3152,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "thrift-0.22.0.tar.gz", hash = "sha256:42e8276afbd5f54fe1d364858b6877bc5e5a4a5ed69f6a005b94ca4918fe1466"}, ] @@ -3227,7 +3227,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "tornado-6.5.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:487dc9cc380e29f58c7ab88f9e27cdeef04b2140862e5076a66fb6bb68bb1bfa"}, {file = "tornado-6.5.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:65a7f1d46d4bb41df1ac99f5fcb685fb25c7e61613742d5108b010975a9a6521"}, @@ -3359,7 +3359,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3620,7 +3620,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "xmlschema-2.5.1-py3-none-any.whl", hash = "sha256:ec2b2a15c8896c1fcd14dcee34ca30032b99456c3c43ce793fdb9dca2fb4b869"}, {file = "xmlschema-2.5.1.tar.gz", hash = "sha256:4f7497de6c8b6dc2c28ad7b9ed6e21d186f4afe248a5bea4f54eedab4da44083"}, @@ -3754,4 +3754,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = ">=3.10.0,<4.0.0" -content-hash = "ab21711922ec9bc72f506a7ab8300a73487d5cc66b75c819931a1ea5fd8d33ec" +content-hash = "ce9ac9da9e7ffaf24b3e1e7892342ba486e7af4ea25385f875d0f3a2d5c5d133" diff --git a/pyproject.toml b/pyproject.toml index c5fd38e93a0..adb9993aae8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ dependencies = [ "matrix-common>=1.3.0,<2.0.0", # We need packaging.verison.Version(...).major added in 20.0. "packaging>=20.0", - "pydantic>=2.10;python_version < '3.14'", + "pydantic>=2.8;python_version < '3.14'", "pydantic>=2.12;python_version >= '3.14'", # This is for building the rust components during "poetry install", which From 47c930f0b7d57e787f57db00b4f03d0862324949 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 31 Mar 2026 18:01:53 +0100 Subject: [PATCH 12/28] Work around Pydantic < 2.10 error builtins.NotImplementedError: Cannot check isinstance when validating from json, use a JsonOrPython validator instead. --- synapse/types/__init__.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index 90d7c0ed724..741e7462ce1 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -134,7 +134,22 @@ class AbsentType(Enum): def __get_pydantic_core_schema__( cls, source_type: object, handler: GetCoreSchemaHandler ) -> CoreSchema: - return pydantic_core.core_schema.is_instance_schema(cls) + def _reject_from_json(v: object) -> "AbsentType": + """ + Reject the JSON value, no matter what it is, since absent values + are meant to be ... absent, thus have nothing they can be deserialised + from. + """ + raise ValueError("AbsentType cannot be deserialized from JSON") + + # `json_or_python_schema` wrapper needed for Pydantic < 2.10 + # but can be replaced with just the `is_instance_schema` after that version. + return pydantic_core.core_schema.json_or_python_schema( + json_schema=pydantic_core.core_schema.no_info_plain_validator_function( + _reject_from_json + ), + python_schema=pydantic_core.core_schema.is_instance_schema(cls), + ) def __copy__(self) -> "AbsentType": """ From 37412f4c2085eefd76a96856c44c052b9e08dab2 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 7 Apr 2026 12:03:17 +0100 Subject: [PATCH 13/28] Move SlidingSyncInterestedRooms attributes docs to attributes --- synapse/handlers/sliding_sync/room_lists.py | 44 ++++++++++++++------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index 8969d915836..984f899ca96 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -112,31 +112,47 @@ class SlidingSyncInterestedRooms: sliding sync request. Returned by `compute_interested_rooms`. - - Attributes: - lists: A mapping from list name to the list result for the response - relevant_room_map: A map from rooms that match the sync request to - their room sync config. - relevant_rooms_to_send_map: Subset of `relevant_room_map` that - includes the rooms that *may* have relevant updates. Rooms not - in this map will definitely not have room updates (though - extensions may have updates in these rooms). - newly_joined_rooms: The set of rooms that were joined in the token range - and the user is still joined to at the end of this range. - newly_left_rooms: The set of rooms that we left in the token range - and are still "leave" at the end of this range. - dm_room_ids: The set of rooms the user consider as direct-message (DM) rooms """ lists: Mapping[str, SlidingSyncResult.SlidingWindowList] + """ + A mapping from list name to the list result for the response + """ + relevant_room_map: Mapping[str, RoomSyncConfig] + """ + A map from rooms that match the sync request to + their room sync config. + """ + relevant_rooms_to_send_map: Mapping[str, RoomSyncConfig] + """ + Subset of `relevant_room_map` that + includes the rooms that *may* have relevant updates. Rooms not + in this map will definitely not have room updates (though + extensions may have updates in these rooms). + """ + all_rooms: set[str] + room_membership_for_user_map: Mapping[str, RoomsForUserType] newly_joined_rooms: AbstractSet[str] + """ + The set of rooms that were joined in the token range + and the user is still joined to at the end of this range. + """ + newly_left_rooms: AbstractSet[str] + """ + The set of rooms that we left in the token range + and are still "leave" at the end of this range. + """ + dm_room_ids: AbstractSet[str] + """ + The set of rooms the user consider as direct-message (DM) rooms + """ @staticmethod def empty() -> "SlidingSyncInterestedRooms": From c5c1b016090083451771bb82c2f67f40cb8beba9 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 7 Apr 2026 12:27:57 +0100 Subject: [PATCH 14/28] Add docstring on SlidingSyncInterestedRooms.all_rooms --- synapse/handlers/sliding_sync/room_lists.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index 984f899ca96..3da2b63abb6 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -134,6 +134,14 @@ class SlidingSyncInterestedRooms: """ all_rooms: set[str] + """ + The set of room IDs of all rooms that could appear in any list. + This set includes rooms that are outside the list ranges. + In other words, this is the set of all rooms that the client is + _interested_ in (in a pure sense), + even if these rooms are omitted from the current window (which + is, in a sense, just a computational optimisation). + """ room_membership_for_user_map: Mapping[str, RoomsForUserType] From 3407bf8c1fb7f9e66d3558ab9bc300164fb1ade3 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 13 May 2026 15:56:41 +0100 Subject: [PATCH 15/28] fixup! drive-by docstring typo fix --- synapse/handlers/sliding_sync/extensions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/sliding_sync/extensions.py b/synapse/handlers/sliding_sync/extensions.py index 47a1879b11e..cacab909b7a 100644 --- a/synapse/handlers/sliding_sync/extensions.py +++ b/synapse/handlers/sliding_sync/extensions.py @@ -98,7 +98,7 @@ async def get_extensions_response( Args: sync_config: Sync configuration - new_connection_state: Snapshot of the current per-connection state + previous_connection_state: Snapshot of the current per-connection state new_connection_state: A mutable copy of the per-connection state, used to record updates to the state during this request. actual_lists: Sliding window API. A map of list key to list results in the From 2b42e54900e96fd9f846181100ee5940e54029f1 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 13 May 2026 16:08:18 +0100 Subject: [PATCH 16/28] Comment why we use .START (and refactor to make it a .START) --- synapse/handlers/sliding_sync/extensions.py | 6 +++--- synapse/types/rest/client/__init__.py | 8 ++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/sliding_sync/extensions.py b/synapse/handlers/sliding_sync/extensions.py index cacab909b7a..a66ffb12241 100644 --- a/synapse/handlers/sliding_sync/extensions.py +++ b/synapse/handlers/sliding_sync/extensions.py @@ -1005,9 +1005,9 @@ async def get_sticky_events_extension_response( if not sticky_events_request.enabled: return None now = self.clock.time_msec() - since_token = sticky_events_request.since or SlidingSyncStickyEventsToken( - sticky_events_stream_id=0 - ) + # If there is no `since` token specified, start from the beginning of the stream + # to make sure the client receives all visible (unexpired) sticky events + since_token = sticky_events_request.since or SlidingSyncStickyEventsToken.START ( sticky_events_to_id, room_to_event_ids, diff --git a/synapse/types/rest/client/__init__.py b/synapse/types/rest/client/__init__.py index 1bb4724a694..799ae746e2a 100644 --- a/synapse/types/rest/client/__init__.py +++ b/synapse/types/rest/client/__init__.py @@ -19,6 +19,7 @@ # # import re +from typing import ClassVar import pydantic_core.core_schema from pydantic import ( @@ -126,6 +127,7 @@ class SlidingSyncStickyEventsToken: """ PATTERN = re.compile(r"^sticky_([0-9]+)$") + START: ClassVar["SlidingSyncStickyEventsToken"] def __init__(self, *, sticky_events_stream_id: int) -> None: self.sticky_events_stream_id = sticky_events_stream_id @@ -160,6 +162,12 @@ def __repr__(self) -> str: return self.serialise() +# Starting reading a stream at 0 ensures all stream fact rows wlil be read +SlidingSyncStickyEventsToken.START = SlidingSyncStickyEventsToken( + sticky_events_stream_id=0 +) + + class SlidingSyncBody(RequestBodyModel): """ Sliding Sync API request body. From 12c8d8ae5fc44f1c931510e2ef5b75af5750d20e Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 13 May 2026 17:44:39 +0100 Subject: [PATCH 17/28] Comment that we should use MultiWriterStreamToken Links https://github.com/element-hq/synapse/issues/19661 --- synapse/types/rest/client/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/types/rest/client/__init__.py b/synapse/types/rest/client/__init__.py index 799ae746e2a..867c3385a42 100644 --- a/synapse/types/rest/client/__init__.py +++ b/synapse/types/rest/client/__init__.py @@ -130,6 +130,8 @@ class SlidingSyncStickyEventsToken: START: ClassVar["SlidingSyncStickyEventsToken"] def __init__(self, *, sticky_events_stream_id: int) -> None: + # FIXME: We should use MultiWriterStreamToken here + # Track: https://github.com/element-hq/synapse/issues/19661 self.sticky_events_stream_id = sticky_events_stream_id @classmethod From fd2a3e75673d775aa08a602604db35ce2fd6eb09 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 13 May 2026 17:52:44 +0100 Subject: [PATCH 18/28] Describe __get_pydantic_core_schema__ --- synapse/types/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index 741e7462ce1..62084ef119b 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -134,6 +134,20 @@ class AbsentType(Enum): def __get_pydantic_core_schema__( cls, source_type: object, handler: GetCoreSchemaHandler ) -> CoreSchema: + """ + This function is checked for and used by Pydantic when + attempting to deserialise/validate a field of this type. + + As the `Absent` type has no valid value when deserialising + from JSON (as that's the point; `Absent` is a marker representing + a lack of any JSON value), we always reject any value. + Instead of deserialising from this type, we rely on the struct class + we are in having field defaults that provide an `Absent`, which does not + go through the JSON validation. + + When validating Python, we accept the absent marker itself. + """ + def _reject_from_json(v: object) -> "AbsentType": """ Reject the JSON value, no matter what it is, since absent values From a2897861992602f2b6db2cd8b0d12bff80fdf06d Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 13 May 2026 17:54:48 +0100 Subject: [PATCH 19/28] Describe NonNegativeStrictInt reason for preference --- synapse/types/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index 62084ef119b..223a007a3b7 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -208,7 +208,10 @@ def __repr__(self) -> str: NonNegativeStrictInt = Annotated[StrictInt, annotated_types.Ge(0)] """A strict integer that must be greater than or equal to zero. -Should be preferred in place of Pydantic's own (lax) NonNegativeInt. +Should be preferred in place of Pydantic's own (lax) NonNegativeInt, +which will coerce strings to integers in a way that does not agree with +the Matrix specification (and would risk backing us into a backward compatibility +hole where we had to support input forms we didn't intend). """ From 83f1da015696e637c47d582e7949458795829301 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 13 May 2026 17:58:38 +0100 Subject: [PATCH 20/28] Describe methods on SlidingSyncStickyEventsToken --- synapse/types/rest/client/__init__.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/synapse/types/rest/client/__init__.py b/synapse/types/rest/client/__init__.py index 867c3385a42..c8a14fa13a0 100644 --- a/synapse/types/rest/client/__init__.py +++ b/synapse/types/rest/client/__init__.py @@ -138,6 +138,14 @@ def __init__(self, *, sticky_events_stream_id: int) -> None: def __get_pydantic_core_schema__( cls, source_type: object, handler: GetCoreSchemaHandler ) -> CoreSchema: + """ + This function is checked for and used by Pydantic when + attempting to deserialise/validate a field of this type. + + This returns a schema that will parse a string into an + instance of `SlidingSyncStickyEventsToken`. + """ + return pydantic_core.core_schema.no_info_plain_validator_function( cls._validate, serialization=pydantic_core.core_schema.plain_serializer_function_ser_schema( @@ -148,6 +156,11 @@ def __get_pydantic_core_schema__( @classmethod def _validate(cls, v: object) -> Self: + """ + Create an instance from serialised string form. + + The inverse of `serialise`. + """ if isinstance(v, cls): return v if isinstance(v, str): @@ -158,9 +171,15 @@ def _validate(cls, v: object) -> Self: raise ValueError(f"Cannot parse SlidingSyncStickyEventsToken from {type(v)}") def serialise(self) -> str: + """ + Convert this instance to string. + + The inverse of `_validate`. + """ return f"sticky_{self.sticky_events_stream_id}" def __repr__(self) -> str: + # Use the serialised form as debug output. return self.serialise() From 37c1351f863e4792cb2d92af62d58f40110448d5 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Mon, 15 Jun 2026 14:00:25 +0100 Subject: [PATCH 21/28] Integrate concurrent FilteredEvent changes --- synapse/handlers/sliding_sync/extensions.py | 6 +++--- synapse/rest/client/sync.py | 4 ++-- synapse/types/handlers/sliding_sync.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/sliding_sync/extensions.py b/synapse/handlers/sliding_sync/extensions.py index 4cfb49a1fd7..516bb3c753f 100644 --- a/synapse/handlers/sliding_sync/extensions.py +++ b/synapse/handlers/sliding_sync/extensions.py @@ -26,7 +26,7 @@ from typing_extensions import TypeAlias, assert_never from synapse.api.constants import AccountDataTypes, EduTypes, StickyEvent -from synapse.events import EventBase +from synapse.events.utils import FilteredEvent from synapse.handlers.receipts import ReceiptEventSource from synapse.logging.opentracing import trace from synapse.storage.databases.main.receipts import ReceiptInRoom @@ -1033,9 +1033,9 @@ async def get_sticky_events_extension_response( # > Any joined user is authorised to see sticky events for the duration they remain sticky. always_include_ids=frozenset(all_sticky_event_ids), ) - filtered_event_map = {ev.event_id: ev for ev in filtered_events} + filtered_event_map = {ev.event.event_id: ev for ev in filtered_events} - room_id_to_sticky_events: dict[str, list[EventBase]] = {} + room_id_to_sticky_events: dict[str, list[FilteredEvent]] = {} for room_id, sticky_event_ids in room_to_event_ids.items(): filtered_events_for_room = [ filtered_event_map[event_id] diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 08957262144..702ddcd6ca1 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -1182,12 +1182,12 @@ async def _serialise_sticky_events( sticky_events_to_write = possibly_duplicated_sticky_events else: sent_event_ids_in_room_section = { - ev.event_id for ev in room_result.timeline_events + ev.event.event_id for ev in room_result.timeline_events } sticky_events_to_write = [ ev for ev in possibly_duplicated_sticky_events - if ev.event_id not in sent_event_ids_in_room_section + if ev.event.event_id not in sent_event_ids_in_room_section ] rooms_out[room_id] = { "events": await self.event_serializer.serialize_events( diff --git a/synapse/types/handlers/sliding_sync.py b/synapse/types/handlers/sliding_sync.py index 186365aa86c..dd913250ba1 100644 --- a/synapse/types/handlers/sliding_sync.py +++ b/synapse/types/handlers/sliding_sync.py @@ -436,7 +436,7 @@ class StickyEventsExtension: events that also appear in the timeline. """ - room_id_to_sticky_events: Mapping[str, list[EventBase]] + room_id_to_sticky_events: Mapping[str, list[FilteredEvent]] next_batch: SlidingSyncStickyEventsToken def __bool__(self) -> bool: From 925d873e4e07ce9a557e2cca04276cda4b4ad2dc Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Mon, 15 Jun 2026 13:58:05 +0100 Subject: [PATCH 22/28] Replace Sentinel.UNSET_SENTINEL with Absent --- synapse/handlers/delayed_events.py | 8 ++---- synapse/handlers/sliding_sync/room_lists.py | 8 ++---- synapse/module_api/__init__.py | 7 +++-- synapse/types/__init__.py | 12 +++++--- synapse/util/sentinel.py | 32 --------------------- 5 files changed, 18 insertions(+), 49 deletions(-) delete mode 100644 synapse/util/sentinel.py diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index 4a9f646d4db..2514798a239 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -36,6 +36,7 @@ ) from synapse.storage.databases.main.state_deltas import StateDelta from synapse.types import ( + Absent, JsonDict, Requester, RoomID, @@ -45,7 +46,6 @@ from synapse.util.duration import Duration from synapse.util.events import generate_fake_event_id from synapse.util.metrics import Measure -from synapse.util.sentinel import Sentinel if TYPE_CHECKING: from synapse.server import HomeServer @@ -273,9 +273,7 @@ async def _handle_state_deltas(self, deltas: list[StateDelta]) -> None: ) continue - sender_str = event_id_and_sender_dict.get( - delta.event_id, Sentinel.UNSET_SENTINEL - ) + sender_str = event_id_and_sender_dict.get(delta.event_id, Absent) if sender_str is None: # An event exists, but the `sender` field was "null" and Synapse # incorrectly accepted the event. This is not expected. @@ -285,7 +283,7 @@ async def _handle_state_deltas(self, deltas: list[StateDelta]) -> None: delta.event_id, ) continue - if sender_str is Sentinel.UNSET_SENTINEL: + if sender_str is Absent: # We have an event ID, but the event was not found in the # datastore. This can happen if a room, or its history, is # purged. State deltas related to the room are left behind, but diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index 5a73b91fde5..836cee6c20f 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -52,6 +52,7 @@ RoomsForUserStateReset, ) from synapse.types import ( + Absent, MutableStateMap, RoomStreamToken, StateMap, @@ -71,7 +72,6 @@ from synapse.types.state import StateFilter from synapse.util import MutableOverlayMapping from synapse.util.duration import Duration -from synapse.util.sentinel import Sentinel if TYPE_CHECKING: from synapse.server import HomeServer @@ -1727,10 +1727,8 @@ async def _bulk_get_partial_current_state_content_for_rooms( # (applies to invite/knock rooms) rooms_ids_without_stripped_state: set[str] = set() for room_id in room_ids_without_results: - stripped_state_map = room_id_to_stripped_state_map.get( - room_id, Sentinel.UNSET_SENTINEL - ) - assert stripped_state_map is not Sentinel.UNSET_SENTINEL, ( + stripped_state_map = room_id_to_stripped_state_map.get(room_id, Absent) + assert stripped_state_map is not Absent, ( f"Stripped state left unset for room {room_id}. " + "Make sure you're calling `_bulk_get_stripped_state_for_rooms_from_sync_room_map(...)` " + "with that room_id. (this is a problem with Synapse itself)" diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 947be24d3e3..3341e49b856 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -142,6 +142,8 @@ from synapse.storage.database import DatabasePool, LoggingTransaction from synapse.storage.databases.main.roommember import ProfileInfo from synapse.types import ( + Absent, + AbsentType, DomainSpecificString, JsonDict, JsonMapping, @@ -160,7 +162,6 @@ from synapse.util.clock import Clock from synapse.util.duration import Duration from synapse.util.frozenutils import freeze -from synapse.util.sentinel import Sentinel if TYPE_CHECKING: # Old versions don't have `LiteralString` @@ -1990,7 +1991,7 @@ async def set_displayname( self, user_id: UserID, new_displayname: str, - deactivation: bool | Sentinel = Sentinel.UNSET_SENTINEL, + deactivation: bool | AbsentType = Absent, ) -> None: """Sets a user's display name. @@ -2020,7 +2021,7 @@ async def set_displayname( """ requester = create_requester(user_id) - if deactivation is not Sentinel.UNSET_SENTINEL: + if deactivation is not Absent: logger.error( "Deprecated `deactivation` parameter passed to `set_displayname` Module API (value: %r). This will break in 2027.", deactivation, diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index 4e2c6fe880b..a6fc806701d 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -120,8 +120,8 @@ class AbsentType(Enum): Type of a sentinel to use as an alternative to `None` for when we really mean 'absent' and not JSON null. - For a Sentinel for internal (non-API-facing) use, instead consider - `Sentinel.UNSET_SENTINEL`. + Generally suitable for distinguishing a default state from user-suppliable values. + Has no meaning on its own. It is falsy (like None is), so shorthand forms like `x or 0` can be used. """ @@ -200,8 +200,12 @@ def __repr__(self) -> str: Sentinel to use as an alternative to `None` for when we really mean 'absent' and not JSON null. -For a Sentinel for internal (non-API-facing) use, instead consider -`Sentinel.UNSET_SENTINEL`. +Generally suitable for distinguishing a default state from user-suppliable values. +Has no meaning on its own. + +It is falsy (like None is), so shorthand forms like `x or 0` can be used. + +(Previously known as `Sentinel.UNSET_SENTINEL`.) """ diff --git a/synapse/util/sentinel.py b/synapse/util/sentinel.py deleted file mode 100644 index e885f81879a..00000000000 --- a/synapse/util/sentinel.py +++ /dev/null @@ -1,32 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright (C) 2025 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# - -import enum - - -class Sentinel(enum.Enum): - """ - Internal marker sentinel for distinguishing a default state from user-suppliable values. - Has no meaning on its own. - - Use this when you want to be absolutely sure that the marker came from Synapse code - and not from request body parsing. - - If you want a Pydantic-compatible Sentinel that is suitable for expressing - 'absent from some parsed JSON payload' or equivalent, see `Absent`. - """ - - # defining a sentinel in this way allows mypy to correctly handle the - # type of a dictionary lookup and subsequent type narrowing. - UNSET_SENTINEL = object() From 230dabec81d1c6b907721be4e184c446664ca496 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Mon, 15 Jun 2026 14:05:25 +0100 Subject: [PATCH 23/28] drive-by: your -> you're --- tests/rest/client/sliding_sync/test_sliding_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/rest/client/sliding_sync/test_sliding_sync.py b/tests/rest/client/sliding_sync/test_sliding_sync.py index fc7d6a279c3..2fd18f0e545 100644 --- a/tests/rest/client/sliding_sync/test_sliding_sync.py +++ b/tests/rest/client/sliding_sync/test_sliding_sync.py @@ -374,7 +374,7 @@ def _bump_notifier_wait_for_events( user_id: The user ID to wake up the notifier for wake_stream_key: The stream key to wake up. This will create an actual new entity in that stream so it's best to choose one that won't affect the - Sliding Sync results you're testing for. In other words, if your testing + Sliding Sync results you're testing for. In other words, if you're testing account data, choose `StreamKeyType.PRESENCE` instead. We support two possible stream keys because you're probably testing one or the other so one is always a "safe" option. From aa20913c1f664afed775f3b3bbd10356154fca03 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Mon, 15 Jun 2026 14:05:45 +0100 Subject: [PATCH 24/28] Add test_wait_for_new_data_timeout test --- .../test_extension_sticky_events.py | 60 ++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/tests/rest/client/sliding_sync/test_extension_sticky_events.py b/tests/rest/client/sliding_sync/test_extension_sticky_events.py index de4827a755c..becbe679f66 100644 --- a/tests/rest/client/sliding_sync/test_extension_sticky_events.py +++ b/tests/rest/client/sliding_sync/test_extension_sticky_events.py @@ -21,7 +21,7 @@ from synapse.api.constants import EventTypes, EventUnsignedContentFields from synapse.rest.client import account_data, login, register, room, sync from synapse.server import HomeServer -from synapse.types import JsonDict +from synapse.types import JsonDict, StreamKeyType from synapse.util.clock import Clock from synapse.util.duration import Duration @@ -322,6 +322,64 @@ def test_wait_for_new_data(self) -> None: {room_id: [sticky_event_id]}, ) + def test_wait_for_new_data_timeout(self) -> None: + """ + Test that the sliding sync request waits for new sticky events to arrive + and times out when no data arrives before the deadline. + (Only applies to incremental syncs with a `timeout` specified). + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("u2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a room + room_id = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # Initial sync with no sticky events + sync_body = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + } + }, + } + _, from_token = self.do_sync(sync_body, tok=user1_tok) + + # Make the sliding sync request with a timeout + channel = self.make_request( + "POST", + self.sync_endpoint + "?timeout=10000" + f"&pos={from_token}", + content=sync_body, + access_token=user1_tok, + await_result=False, + ) + + # Block for 5 seconds to make sure we are `notifier.wait_for_events(...)` + with self.assertRaises(TimedOutException): + channel.await_result(timeout_ms=5000) + # Wake-up `notifier.wait_for_events(...)` that will cause us test + # `SlidingSyncResult.__bool__` for new results. + self._bump_notifier_wait_for_events( + # wake key is intentionally unrelated to sticky events + user1_id, + wake_stream_key=StreamKeyType.ACCOUNT_DATA, + ) + # Block for a little bit more to ensure we don't see any new results. + with self.assertRaises(TimedOutException): + channel.await_result(timeout_ms=4000) + # Wait for the sync to complete (wait for the rest of the 10 second timeout, + # 5000 + 4000 + 1200 > 10000) + channel.await_result(timeout_ms=1200) + self.assertEqual(channel.code, 200, channel.json_body) + + self._assert_sticky_events_response( + channel.json_body, + None, + ) + def test_ignored_users_sticky_events(self) -> None: """ Test that sticky events from ignored users are not delivered to clients. From 9dc80959bf0d2ce23b178e46139ad32423bb074f Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 19 Jun 2026 11:24:40 +0100 Subject: [PATCH 25/28] Typo --- synapse/types/rest/client/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/types/rest/client/__init__.py b/synapse/types/rest/client/__init__.py index c8a14fa13a0..a7cb4d8b08c 100644 --- a/synapse/types/rest/client/__init__.py +++ b/synapse/types/rest/client/__init__.py @@ -183,7 +183,7 @@ def __repr__(self) -> str: return self.serialise() -# Starting reading a stream at 0 ensures all stream fact rows wlil be read +# Starting reading a stream at 0 ensures all stream fact rows will be read SlidingSyncStickyEventsToken.START = SlidingSyncStickyEventsToken( sticky_events_stream_id=0 ) From 0b0a35d2d665a872e38331373d112eade3276200 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 19 Jun 2026 11:25:54 +0100 Subject: [PATCH 26/28] Reorder args --- synapse/handlers/sliding_sync/__init__.py | 2 +- synapse/handlers/sliding_sync/extensions.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index 0b50a178775..10ca3ddea07 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -327,13 +327,13 @@ async def handle_room(room_id: str) -> None: actual_lists=lists, previous_connection_state=previous_connection_state, new_connection_state=new_connection_state, + all_interested_room_ids=interested_rooms.all_rooms, # We're purposely using `relevant_room_map` instead of # `relevant_rooms_to_send_map` here. This needs to be all room_ids we could # send regardless of whether they have an event update or not. The # extensions care about more than just normal events in the rooms (like # account data, read receipts, typing indicators, to-device messages, etc). actual_room_ids=set(relevant_room_map.keys()), - all_interested_room_ids=interested_rooms.all_rooms, actual_room_response_map=rooms, from_token=from_token, to_token=to_token, diff --git a/synapse/handlers/sliding_sync/extensions.py b/synapse/handlers/sliding_sync/extensions.py index 516bb3c753f..b3342de7781 100644 --- a/synapse/handlers/sliding_sync/extensions.py +++ b/synapse/handlers/sliding_sync/extensions.py @@ -87,9 +87,9 @@ async def get_extensions_response( sync_config: SlidingSyncConfig, previous_connection_state: "PerConnectionState", new_connection_state: "MutablePerConnectionState", + all_interested_room_ids: set[str], actual_lists: Mapping[str, SlidingSyncResult.SlidingWindowList], actual_room_ids: set[str], - all_interested_room_ids: set[str], actual_room_response_map: Mapping[str, SlidingSyncResult.RoomResult], to_token: StreamToken, from_token: SlidingSyncStreamToken | None, @@ -101,12 +101,12 @@ async def get_extensions_response( previous_connection_state: Snapshot of the current per-connection state new_connection_state: A mutable copy of the per-connection state, used to record updates to the state during this request. - actual_lists: Sliding window API. A map of list key to list results in the - Sliding Sync response. - actual_room_ids: The actual room IDs in the the Sliding Sync response. all_interested_room_ids: The IDs of all rooms that the client is interested in, even if they don't appear in the current limited window. See `SlidingSyncInterestedRooms.all_rooms`. + actual_lists: Sliding window API. A map of list key to list results in the + Sliding Sync response. + actual_room_ids: The actual room IDs in the the Sliding Sync response. actual_room_response_map: A map of room ID to room results in the the Sliding Sync response. to_token: The latest point in the stream to sync up to. From 0a71a9a9311b89bb7332420c5bbd9139d0b473fa Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 19 Jun 2026 11:26:25 +0100 Subject: [PATCH 27/28] Update tests/rest/client/sliding_sync/test_extension_sticky_events.py Co-authored-by: Eric Eastwood --- tests/rest/client/sliding_sync/test_extension_sticky_events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/rest/client/sliding_sync/test_extension_sticky_events.py b/tests/rest/client/sliding_sync/test_extension_sticky_events.py index becbe679f66..d29f360d785 100644 --- a/tests/rest/client/sliding_sync/test_extension_sticky_events.py +++ b/tests/rest/client/sliding_sync/test_extension_sticky_events.py @@ -116,7 +116,7 @@ def _assert_sticky_events_response( actual_rooms = sticky_events["rooms"] # Check that we have the expected rooms - self.assertEqual(set(actual_rooms.keys()), set(expected_events_by_room.keys())) + self.assertIncludes(set(actual_rooms.keys()), set(expected_events_by_room.keys()), exact=True) # Check the events in each room for room_id, expected_events in expected_events_by_room.items(): From dfb9b82d9e6b89247d60dd85050c4f4342c5992f Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 19 Jun 2026 17:55:54 +0100 Subject: [PATCH 28/28] Lint --- .../rest/client/sliding_sync/test_extension_sticky_events.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/rest/client/sliding_sync/test_extension_sticky_events.py b/tests/rest/client/sliding_sync/test_extension_sticky_events.py index d29f360d785..aaea6d67935 100644 --- a/tests/rest/client/sliding_sync/test_extension_sticky_events.py +++ b/tests/rest/client/sliding_sync/test_extension_sticky_events.py @@ -116,7 +116,9 @@ def _assert_sticky_events_response( actual_rooms = sticky_events["rooms"] # Check that we have the expected rooms - self.assertIncludes(set(actual_rooms.keys()), set(expected_events_by_room.keys()), exact=True) + self.assertIncludes( + set(actual_rooms.keys()), set(expected_events_by_room.keys()), exact=True + ) # Check the events in each room for room_id, expected_events in expected_events_by_room.items():