feat: migrations

frontend/src/types/Manifest.generated.ts

@@ -7,6 +7,7 @@
 // This file was automatically generated by generate-ts-types.py.
 // DO NOT MODIFY IT BY HAND. Instead, modify the source Pydantic models, and regenerate this file.
 
+export type ShortName = string
 export type PackageType = 'LIBRARY' | 'QUESTIONTYPE' | 'QUESTION'
 export type EnvironmentVariableName = string
 
@@ -16,7 +17,7 @@ export type EnvironmentVariableName = string
  * Contains fields valid in a pre-built package manifest.
  */
 export interface Manifest {
-    short_name: string
+    short_name: ShortName
     namespace: string
     version: string
     api_version: string
@@ -38,6 +39,8 @@ export interface Manifest {
     requirements: string | string[] | null
     static_files: StaticFiles
     dependencies: DistDependencies
+    state_version: number
+    possible_side_migrations: PossibleSideMigrations
 }
 export interface Name {
     [k: string]: string
@@ -70,3 +73,8 @@ export interface DistStaticQPyDependency {
     dir_name: string
     hash: string
 }
+export interface PossibleSideMigrations {
+    [k: string]: {
+        [k: string]: number[]
+    }
+}

pyproject.toml

@@ -18,7 +18,7 @@ dependencies = [
     "aiohttp >=3.11.18, <4.0.0",
     "pydantic >=2.11.4, <3.0.0",
     "PyYAML >=6.0.2, <7.0.0",
-    "questionpy-server @ git+https://github.com/questionpy-org/questionpy-server.git@32833396b7547d7a616cfc6249f2174153181bbd",
+    "questionpy-server @ git+https://github.com/questionpy-org/questionpy-server.git@6087d3d0eacf5c3940092d88e08b0fd50720d1a4",
     "jinja2 >=3.1.6, <4.0.0",
     "aiohttp-jinja2 >=1.6, <2.0",
     "lxml[html-clean] >=5.4.0, <5.5.0",

questionpy/__init__.py

@@ -46,6 +46,7 @@
     NeedsManualScoringError,
     ResponseNotScorableError,
 )
+from ._migration import Migration, MigrationNotPossibleError, SideMigration, get_migrations
 from ._qtype import BaseQuestionState, Question
 from ._ui import create_jinja2_environment
 from ._wrappers import QuestionTypeWrapper, QuestionWrapper
@@ -67,6 +68,8 @@
     "FeedbackType",
     "InvalidResponseError",
     "Manifest",
+    "Migration",
+    "MigrationNotPossibleError",
     "NeedsManualScoringError",
     "NoEnvironmentError",
     "OnRequestCallback",
@@ -87,9 +90,11 @@
     "ScoreModel",
     "ScoringCode",
     "ScoringMethod",
+    "SideMigration",
     "SourceManifest",
     "SubquestionModel",
     "create_jinja2_environment",
+    "get_migrations",
     "get_qpy_environment",
     "i18n",
     "make_question_type_init",

questionpy/_migration/__init__.py

@@ -0,0 +1,46 @@
+#  This file is part of the QuestionPy SDK. (https://questionpy.org)
+#  The QuestionPy SDK is free software released under terms of the MIT license. See LICENSE.md.
+#  (c) Technische Universität Berlin, innoCampus <info@isis.tu-berlin.de>
+import importlib
+import pkgutil
+from collections import defaultdict
+from typing import NamedTuple
+
+from ._base import MigrationQuestionStateWithVersion
+from ._migration import MIGRATIONS_REGISTRY, Migration, MigrationsRegistry
+from ._side_migration import SIDE_MIGRATIONS_REGISTRY, SideMigration, SideMigrationsRegistry
+from .errors import MigrationDiscoveryError, MigrationNotPossibleError
+
+__all__ = [
+    "Migration",
+    "MigrationNotPossibleError",
+    "MigrationQuestionStateWithVersion",
+    "Migrations",
+    "SideMigration",
+    "get_migrations",
+]
+
+
+class Migrations(NamedTuple):
+    package: MigrationsRegistry
+    """Migrations from and to this package."""
+    side: SideMigrationsRegistry
+    """Migrations from other packages to this package."""
+
+
+def get_migrations(namespace: str, short_name: str) -> Migrations:
+    """The package and its dependencies must be importable."""
+    module_name = f"{namespace}.{short_name}.migrations"
+
+    try:
+        module = importlib.import_module(module_name)
+    except ModuleNotFoundError:
+        return Migrations([], defaultdict(defaultdict))
+
+    try:
+        for module_info in pkgutil.walk_packages(module.__path__, prefix=f"{module_name}."):
+            importlib.import_module(module_info.name)
+    except Exception as e:
+        raise MigrationDiscoveryError from e
+
+    return Migrations(MIGRATIONS_REGISTRY, SIDE_MIGRATIONS_REGISTRY)

questionpy/_migration/_base.py

@@ -0,0 +1,28 @@
+#  This file is part of the QuestionPy SDK. (https://questionpy.org)
+#  The QuestionPy SDK is free software released under terms of the MIT license. See LICENSE.md.
+#  (c) Technische Universität Berlin, innoCampus <info@isis.tu-berlin.de>
+from pydantic import BaseModel, JsonValue
+
+
+class MigrationQuestionStateWithVersion(BaseModel):
+    package_namespace: str
+    package_short_name: str
+    package_version: str
+    options: dict[str, JsonValue]
+    state: dict[str, JsonValue]
+    state_version: int
+
+
+class BaseMigration:
+    _state: MigrationQuestionStateWithVersion
+
+    def __init__(self, state: MigrationQuestionStateWithVersion):
+        self._state = state
+
+    @property
+    def state(self) -> dict[str, JsonValue]:
+        return self._state.state
+
+    @property
+    def options(self) -> dict[str, JsonValue]:
+        return self._state.options

questionpy/_migration/_migration.py

@@ -0,0 +1,42 @@
+#  This file is part of the QuestionPy SDK. (https://questionpy.org)
+#  The QuestionPy SDK is free software released under terms of the MIT license. See LICENSE.md.
+#  (c) Technische Universität Berlin, innoCampus <info@isis.tu-berlin.de>
+from abc import ABC, abstractmethod
+from bisect import insort
+
+from ._base import BaseMigration
+from .errors import MigrationNotPossibleError
+
+type MigrationsRegistry = list[type[Migration]]
+
+
+MIGRATIONS_REGISTRY: MigrationsRegistry = []
+
+
+def _migration_strategy(migration_cls: type["Migration"]) -> str:
+    """Migrations are sorted by their module and class name."""
+    return migration_cls.__module__ + "." + migration_cls.__qualname__
+
+
+class Migration(BaseMigration, ABC):
+    """The base class for migrations from and to the current package."""
+
+    def __init_subclass__(cls, **kwargs: object) -> None:
+        super().__init_subclass__(**kwargs)
+        insort(MIGRATIONS_REGISTRY, cls, key=_migration_strategy)
+
+    @abstractmethod
+    def upgrade(self) -> None:
+        """Upgrade the previous state to this version.
+
+        It is generally assumed, that upgrading is always possible, but if that is not the case the
+        `MigrationNotPossibleError` should be raised.
+        """
+
+    def downgrade(self) -> None:
+        """Downgrade this state to the previous version.
+
+        The `MigrationNotPossibleError` should be raised if downgrading is not possible. This is also the default
+        behaviour.
+        """
+        raise MigrationNotPossibleError

questionpy/_migration/_side_migration.py

@@ -0,0 +1,29 @@
+#  This file is part of the QuestionPy SDK. (https://questionpy.org)
+#  The QuestionPy SDK is free software released under terms of the MIT license. See LICENSE.md.
+#  (c) Technische Universität Berlin, innoCampus <info@isis.tu-berlin.de>
+from abc import ABC, abstractmethod
+from collections import defaultdict
+
+from ._base import BaseMigration
+
+type SideMigrationsRegistry = defaultdict[str, defaultdict[str, dict[int, type[SideMigration]]]]
+
+
+SIDE_MIGRATIONS_REGISTRY: SideMigrationsRegistry = defaultdict(lambda: defaultdict(dict))
+
+
+class SideMigration(BaseMigration, ABC):
+    """The base class for migrations from other packages to the current package."""
+
+    def __init_subclass__(
+        cls, /, for_namespace: str, for_short_name: str, for_state_version: int, **kwargs: object
+    ) -> None:
+        super().__init_subclass__(**kwargs)
+        SIDE_MIGRATIONS_REGISTRY[for_namespace][for_short_name][for_state_version] = cls
+
+    @abstractmethod
+    def sidegrade(self) -> None:
+        """Sidegrade the given state to this version.
+
+        If the migration is not possible, raise the `MigrationNotPossibleError`.
+        """

questionpy/_migration/errors.py

@@ -0,0 +1,71 @@
+#  This file is part of the QuestionPy SDK. (https://questionpy.org)
+#  The QuestionPy SDK is free software released under terms of the MIT license. See LICENSE.md.
+#  (c) Technische Universität Berlin, innoCampus <info@isis.tu-berlin.de>
+from questionpy_common.api.qtype import MigrationError, MigrationErrorKind
+from questionpy_common.environment import Package
+
+from ._base import MigrationQuestionStateWithVersion
+
+
+class MigrationNotImplementedError(MigrationError):
+    def __init__(self) -> None:
+        super().__init__(kind=MigrationErrorKind.NOT_IMPLEMENTED)
+
+
+class MigrationNotPossibleError(MigrationError):
+    def __init__(self, *args: object, reason: str | None = None, temporary: bool = False) -> None:
+        super().__init__(*args, kind=MigrationErrorKind.NOT_POSSIBLE, reason=reason, temporary=temporary)
+
+
+class SpecificMigrationFailedError(MigrationError):
+    def __init__(self, cause: Exception, from_version: int, to_version: int, step: int):
+        temporary = False
+        msg = f"The migration at step {step} from state version {from_version} to state version {to_version} "
+
+        if isinstance(cause, MigrationNotPossibleError):
+            kind = MigrationErrorKind.NOT_POSSIBLE
+
+            temporary = cause.temporary
+            reason = f": {cause}" if cause.reason else "."
+            msg += f"is not possible{reason}"
+        else:
+            kind = MigrationErrorKind.FAILED
+            msg += "failed."
+
+        super().__init__(kind=kind, reason=msg, temporary=temporary)
+
+
+class MigrationPackageMissmatchError(MigrationError):
+    def __init__(self, package: Package, state: MigrationQuestionStateWithVersion):
+        msg = (
+            f"The provided question state must origin from this package. "
+            f"Expected @{package.manifest.namespace}/{package.manifest.short_name}, "
+            f"got @{state.package_namespace}/{state.package_short_name}."
+        )
+
+        super().__init__(kind=MigrationErrorKind.PACKAGE_MISSMATCH, reason=msg)
+
+
+class MigrationPackageVersionMissmatchError(MigrationError):
+    def __init__(self, expected_state_version: int, actual_state_version: int):
+        msg = (
+            f"The provided question state must have the same state version used by this package. Expected "
+            f"'{expected_state_version}', got '{actual_state_version}."
+        )
+
+        super().__init__(kind=MigrationErrorKind.PACKAGE_MISSMATCH, reason=msg)
+
+
+class MigrationQuestionStateInvalidError(MigrationError):
+    def __init__(self) -> None:
+        super().__init__(kind=MigrationErrorKind.QUESTION_STATE_INVALID)
+
+
+class MigrationFailedError(MigrationError):
+    def __init__(self) -> None:
+        super().__init__(kind=MigrationErrorKind.FAILED)
+
+
+class MigrationDiscoveryError(MigrationError):
+    def __init__(self) -> None:
+        super().__init__(kind=MigrationErrorKind.DISCOVERY_ERROR)

questionpy/_qtype.py

@@ -19,10 +19,12 @@
 
 
 class QuestionStateWithVersion[F: FormModel, S: "BaseQuestionState"](BaseModel):
-    package_name: str
+    package_namespace: str
+    package_short_name: str
     package_version: str
     options: F
     state: S
+    state_version: int
 
 
 class BaseQuestionState(BaseModel):
@@ -70,10 +72,12 @@ def new_from_options(cls, form_data: dict[str, JsonValue]) -> Self:
 
         env = get_qpy_environment()
         new_qswv: QuestionStateWithVersion = QuestionStateWithVersion(
-            package_name=f"{env.main_package.manifest.namespace}.{env.main_package.manifest.short_name}",
+            package_namespace=env.main_package.manifest.namespace,
+            package_short_name=env.main_package.manifest.short_name,
             package_version=env.main_package.manifest.version,
             options=options,
             state=question_state,
+            state_version=env.main_package.manifest.state_version,
         )
 
         return cls(new_qswv)
@@ -141,7 +145,7 @@ def validate_options(cls, form_data: dict[str, JsonValue]) -> FormModel:
 
                 error_dict[".".join(map(str, error_details["loc"]))] = message
 
-            raise OptionsFormValidationError(error_dict) from e
+            raise OptionsFormValidationError(errors=error_dict) from e
 
     def get_options_form(self) -> tuple[OptionsFormDefinition, dict[str, JsonValue]]:
         """Return the options form and field values for viewing or editing this question."""