diff --git a/alchemiscale/interface/api.py b/alchemiscale/interface/api.py index 341069e9..9eab81f4 100644 --- a/alchemiscale/interface/api.py +++ b/alchemiscale/interface/api.py @@ -31,7 +31,7 @@ from ..settings import get_base_api_settings from ..storage.statestore import Neo4jStore from ..storage.objectstore import S3ObjectStore -from ..storage.models import TaskStatusEnum, StrategyState +from ..storage.models import NetworkStateEnum, TaskStatusEnum, StrategyState from ..models import Scope, ScopedKey from ..security.models import TokenData, CredentialedUserIdentity @@ -138,6 +138,101 @@ async def create_network( return an_sk +@router.post("/networks/merge", response_model=ScopedKey) +async def merge_networks( + *, + networks: list[str] = Body(embed=True), + name: str = Body(embed=True), + scope: dict = Body(embed=True), + state: str = Body(embed=True, default=NetworkStateEnum.active.value), + n4js: Neo4jStore = Depends(get_n4js_depends), + token: TokenData = Depends(get_token_data_depends), +): + # validate the destination scope first + try: + target_scope = Scope(**scope) + except (TypeError, ValidationError) as e: + raise HTTPException( + status_code=http_status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=str(e), + ) + validate_scopes(target_scope, token) + + # validate each source network's scope is accessible to the token + network_sks = [] + for network in networks: + try: + network_sk = ScopedKey.from_str(network) + except ValueError as e: + raise HTTPException( + status_code=http_status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=e.args[0], + ) + validate_scopes(network_sk.scope, token) + network_sks.append(network_sk) + + try: + an_sk = n4js.merge_networks( + network_scoped_keys=network_sks, + name=name, + scope=target_scope, + state=state, + ) + except ValueError as e: + raise HTTPException( + status_code=http_status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=e.args[0], + ) + + return an_sk + + +@router.post("/networks/{network_scoped_key}/copy", response_model=ScopedKey) +async def copy_network( + network_scoped_key, + *, + scope: dict = Body(embed=True), + name: str | None = Body(embed=True, default=None), + state: str = Body(embed=True, default=NetworkStateEnum.active.value), + n4js: Neo4jStore = Depends(get_n4js_depends), + token: TokenData = Depends(get_token_data_depends), +): + # validate the source ScopedKey + its scope + try: + source_sk = ScopedKey.from_str(network_scoped_key) + except ValueError as e: + raise HTTPException( + status_code=http_status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=e.args[0], + ) + validate_scopes(source_sk.scope, token) + + # validate the destination scope + try: + target_scope = Scope(**scope) + except (TypeError, ValidationError) as e: + raise HTTPException( + status_code=http_status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=str(e), + ) + validate_scopes(target_scope, token) + + try: + an_sk = n4js.copy_network( + network_scoped_key=source_sk, + scope=target_scope, + name=name, + state=state, + ) + except ValueError as e: + raise HTTPException( + status_code=http_status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=e.args[0], + ) + + return an_sk + + @router.post("/bulk/networks/state/set") def set_networks_state( *, diff --git a/alchemiscale/interface/client.py b/alchemiscale/interface/client.py index f4fb7e0d..d61e8b83 100644 --- a/alchemiscale/interface/client.py +++ b/alchemiscale/interface/client.py @@ -190,6 +190,278 @@ def post(): return ScopedKey.from_dict(scoped_key) + def merge_networks( + self, + networks: list[ScopedKey], + name: str, + scope: Scope, + state: NetworkStateEnum | str = NetworkStateEnum.active, + visualize: bool = True, + ) -> ScopedKey: + """Merge multiple existing AlchemicalNetworks into a new AlchemicalNetwork. + + The resulting AlchemicalNetwork contains the union of all + Transformations and NonTransformations from the source networks. + Only Tasks in ``complete`` state on those Transformations are + cloned into the new network's scope along with their + ProtocolDAGResultRefs, so previously-computed results do not need + to be re-run. + + Any ``Strategy`` (:meth:`set_network_strategy`) and + ``TaskRestartPattern`` s (:meth:`add_task_restart_patterns`) on + the source networks are not carried over; set them on the merged + network afterward if needed. + + Parameters + ---------- + networks + The ScopedKeys of the AlchemicalNetworks to merge. The source + networks may live in different Scopes; the caller must have access + to each. + name + The name of the new AlchemicalNetwork. + scope + The Scope in which to create the new AlchemicalNetwork. + This must be a *specific* Scope; it must not contain wildcards. + state + The starting state of the new AlchemicalNetwork. See + :meth:`AlchemiscaleClient.set_network_state` for valid states. + Defaults to ``"active"``. + visualize + If ``True``, show progress indicator. + + Returns + ------- + ScopedKey + The ScopedKey of the new, merged AlchemicalNetwork. + """ + if not scope.specific(): + raise ValueError( + f"`scope` '{scope}' contains wildcards ('*'); `scope` must be *specific*" + ) + + if not networks: + raise ValueError("`networks` must contain at least one ScopedKey") + + network_sks = [ + sk if isinstance(sk, ScopedKey) else ScopedKey.from_str(sk) + for sk in networks + ] + + for network_sk in network_sks: + if network_sk.qualname not in ("AlchemicalNetwork",): + raise ValueError( + f"ScopedKey '{network_sk}' does not refer to an AlchemicalNetwork" + ) + + state = NetworkStateEnum(state) + + data = dict( + networks=[str(sk) for sk in network_sks], + name=name, + scope=scope.to_dict(), + state=state.value, + ) + + def post(): + return self._post_resource("/networks/merge", data) + + if visualize: + from rich.progress import Progress + + with Progress(*self._rich_waiting_columns(), transient=False) as progress: + task = progress.add_task( + f"Merging [bold]{len(network_sks)}[/bold] networks into " + f"[bold]'{name}'[/bold] in scope [bold]'{scope}'[/bold]...", + total=None, + ) + + scoped_key = post() + progress.start_task(task) + progress.update(task, total=1, completed=1) + else: + scoped_key = post() + + return ScopedKey.from_dict(scoped_key) + + def copy_network( + self, + network: ScopedKey, + scope: Scope, + name: str | None = None, + state: NetworkStateEnum | str = NetworkStateEnum.active, + visualize: bool = True, + ) -> ScopedKey: + """Copy an AlchemicalNetwork to a new Scope, carrying over every + existing ``complete`` Task and its associated + ProtocolDAGResultRefs. + + Any ``Strategy`` (:meth:`set_network_strategy`) and + ``TaskRestartPattern`` s (:meth:`add_task_restart_patterns`) on + the source network are not carried over; set them on the copy + afterward if needed. + + Parameters + ---------- + network + The ScopedKey of the AlchemicalNetwork to copy. + scope + The destination Scope for the copy. This must be a *specific* + Scope; it must not contain wildcards. + name + Optional new name for the copy. If ``None``, the source + AlchemicalNetwork's name is preserved and the resulting copy + has the same gufe key as the source. If a different name is + given, the resulting copy gets a fresh gufe key derived from + the renamed content. + state + The starting state of the new AlchemicalNetwork. See + :meth:`AlchemiscaleClient.set_network_state` for valid states. + Defaults to ``"active"``. + visualize + If ``True``, show progress indicator. + + Returns + ------- + ScopedKey + The ScopedKey of the new, copied AlchemicalNetwork. + """ + if not scope.specific(): + raise ValueError( + f"`scope` '{scope}' contains wildcards ('*'); `scope` must be *specific*" + ) + + if not isinstance(network, ScopedKey): + network = ScopedKey.from_str(network) + if network.qualname != "AlchemicalNetwork": + raise ValueError( + f"ScopedKey '{network}' does not refer to an AlchemicalNetwork" + ) + + state = NetworkStateEnum(state) + + data = dict( + scope=scope.to_dict(), + name=name, + state=state.value, + ) + + def post(): + return self._post_resource(f"/networks/{network}/copy", data) + + if visualize: + from rich.progress import Progress + + with Progress(*self._rich_waiting_columns(), transient=False) as progress: + task = progress.add_task( + f"Copying [bold]'{network}'[/bold] into scope " + f"[bold]'{scope}'[/bold]...", + total=None, + ) + scoped_key = post() + progress.start_task(task) + progress.update(task, total=1, completed=1) + else: + scoped_key = post() + + return ScopedKey.from_dict(scoped_key) + + def merge_scopes( + self, + scopes: list[Scope], + target_scope: Scope, + visualize: bool = True, + ) -> list[ScopedKey]: + """Copy every AlchemicalNetwork in each of the given source Scopes + into ``target_scope``. + + Each source AlchemicalNetwork is copied via :meth:`copy_network`, + preserving its name, its ``complete`` Tasks, and its associated + ProtocolDAGResultRefs. Each copy also preserves its source + network's *state* (``active``, ``inactive``, ``invalid``, or + ``deleted``) so that ``merge_scopes`` is a faithful move of the + scope's contents rather than a silent reactivation. + + This method is not transactional: if a per-network + :meth:`copy_network` call fails partway through, earlier copies + are already committed and the failure propagates. Because + :meth:`copy_network` is idempotent -- re-copying an already-copied + network dedups onto the existing target node -- rerunning after a + failure is safe. + + Parameters + ---------- + scopes + The source Scopes to copy from. Each must be a *specific* + Scope (no wildcards). The caller must have access to each. + target_scope + The destination Scope. Must be a *specific* Scope. + visualize + If ``True``, show a progress bar tracking the number of + networks copied. + + Returns + ------- + list[ScopedKey] + The ScopedKeys of all newly-copied AlchemicalNetworks in + ``target_scope``, in the order they were copied. + """ + if not target_scope.specific(): + raise ValueError( + f"`target_scope` '{target_scope}' contains wildcards ('*'); " + "`target_scope` must be *specific*" + ) + if not scopes: + raise ValueError("`scopes` must contain at least one Scope") + for sc in scopes: + if not sc.specific(): + raise ValueError( + f"source scope '{sc}' contains wildcards ('*'); each source " + "scope must be *specific*" + ) + + # gather every AlchemicalNetwork across the source scopes up front, + # regardless of state, so the progress bar has an accurate total. + # Also fetch each source network's state so the copy preserves it + # rather than defaulting every copy to ``active``. + network_sks: list[ScopedKey] = [] + network_states: list[str] = [] + for sc in scopes: + sc_sks = self.query_networks(scope=sc, state=None) + if not sc_sks: + continue + sc_states = self.get_networks_state(sc_sks) + network_sks.extend(sc_sks) + # ``get_networks_state`` returns ``None`` for any network the + # server did not find; that should not happen for SKs we just + # queried, but be defensive and fall back to ``active``. + network_states.extend( + s if s is not None else NetworkStateEnum.active.value for s in sc_states + ) + + def _do_copy(sk: ScopedKey, state: str) -> ScopedKey: + return self.copy_network(sk, target_scope, state=state, visualize=False) + + if visualize: + from rich.progress import Progress + + with Progress(*self._rich_waiting_columns(), transient=False) as progress: + task = progress.add_task( + f"Copying [bold]{len(network_sks)}[/bold] networks into " + f"[bold]'{target_scope}'[/bold]...", + total=len(network_sks), + ) + new_sks: list[ScopedKey] = [] + for sk, state in zip(network_sks, network_states): + new_sks.append(_do_copy(sk, state)) + progress.update(task, advance=1) + else: + new_sks = [ + _do_copy(sk, state) for sk, state in zip(network_sks, network_states) + ] + + return new_sks + def set_network_state( self, network: ScopedKey, state: NetworkStateEnum | str ) -> ScopedKey | None: diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index 343bdcde..60e7792a 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -7,6 +7,7 @@ import abc import bisect import datetime +from dataclasses import dataclass, field from contextlib import contextmanager import json import re @@ -24,7 +25,12 @@ Protocol, ) from gufe.settings import SettingsBaseModel -from gufe.tokenization import GufeTokenizable, GufeKey, JSON_HANDLER, KeyedChain +from gufe.tokenization import ( + GufeTokenizable, + GufeKey, + JSON_HANDLER, + KeyedChain, +) from gufe.protocols import ProtocolUnitFailure from neo4j import Transaction, GraphDatabase, Driver, NotificationDisabledClassification @@ -78,6 +84,202 @@ def get_n4js(settings: Neo4jStoreSettings): class Neo4JStoreError(Exception): ... +def _is_transformation_keyed_dict(keyed_dict: dict) -> bool: + """Return ``True`` if ``keyed_dict`` represents a ``Transformation`` or ``NonTransformation``. + + Used as the predicate for ``KeyedChain.decode_subchains`` when walking + an ``AlchemicalNetwork``'s keyed chain in :meth:`Neo4jStore.merge_networks` + and :meth:`Neo4jStore.copy_network`. + """ + return keyed_dict.get("__qualname__") in ("Transformation", "NonTransformation") + + +@dataclass +class _TransformationData: + """Bookkeeping for one Transformation as it is reconstructed during + :meth:`Neo4jStore.merge_networks` or :meth:`Neo4jStore.copy_network`. + + Attributes + ---------- + transformation + The decoded ``Transformation`` (or ``NonTransformation``) + ``GufeTokenizable``. + task_tree + Flat list of Neo4j records, one per ``Task`` associated with this + ``Transformation`` in the source network(s). Each record carries: + + - ``tf_key``: this ``Transformation``'s decoded gufe key (string) + - ``task``: the full ``Task`` node + - ``extended_task``: optional ``Task`` node that this ``Task`` extends + - ``pdrrs``: list of ``ProtocolDAGResultRef`` nodes for this ``Task`` + known_scoped_keys + All ``ScopedKey``\\ s representing this ``Transformation`` across the + source network(s). Multiple ``ScopedKey``\\ s can map to a single + decoded ``Transformation`` if the same content was serialized under + different gufe versions. + """ + + transformation: Transformation + task_tree: list = field(default_factory=list) + known_scoped_keys: list = field(default_factory=list) + + def add_known_scoped_key(self, key, scope): + self.known_scoped_keys.append( + ScopedKey(gufe_key=GufeKey(key), **scope.to_dict()) + ) + + @staticmethod + def update_task_trees( + transformation_data: list, + statestore, + task_statuses: list[str] | None = None, + ): + """Given a list of ``_TransformationData``, extract all necessary + info from Neo4j and load the task trees. + + Parameters + ---------- + transformation_data + List of ``_TransformationData`` to populate. + statestore + The ``Neo4jStore`` to query. + task_statuses + If provided, only ``Task``\\ s whose ``status`` is in this list + are loaded. If ``None`` (the default), ``Task``\\ s of any + status are loaded. Both ``merge_networks`` and + ``copy_network`` pass ``["complete"]`` so only ``Task``\\ s + carrying successful results are preserved. + """ + key_to_data_map = {str(td.transformation.key): td for td in transformation_data} + # prepare for unwind clause, include transformation key + # for updating each entry of transformation_data for + # each scoped key + transformation_sk_pairs = [ + [str(td.transformation.key), str(sk)] + for td in transformation_data + for sk in td.known_scoped_keys + ] + # When task_statuses is provided, filter the Task match; otherwise + # include every Task that PERFORMs on one of the listed Transformations. + if task_statuses is None: + status_clause = "" + params = {"tf_sk_pairs": transformation_sk_pairs} + else: + status_clause = "WHERE task.status IN $task_statuses" + params = { + "tf_sk_pairs": transformation_sk_pairs, + "task_statuses": list(task_statuses), + } + query = f""" + UNWIND $tf_sk_pairs as pairs + WITH pairs[0] AS tf_key, pairs[1] AS tf_scoped_key + MATCH (task:Task)-[:PERFORMS]->(:Transformation|NonTransformation {{`_scoped_key`: tf_scoped_key}}) + {status_clause} + OPTIONAL MATCH (task)-[:EXTENDS]->(extended_task:Task) + OPTIONAL MATCH (task)-[:RESULTS_IN]->(pdrr:ProtocolDAGResultRef) + RETURN tf_key, task, extended_task as extended_task, collect(pdrr) as pdrrs + """ + results = statestore.execute_query(query, **params).records + + for record in results: + key_to_data_map[record["tf_key"]].task_tree.append(record) + + def to_subgraph(self, target_scope, tf_node): + """Create a subgraph of cloned Tasks + EXTENDS + PERFORMS + RESULTS_IN + relationships, anchored at ``tf_node``. + + Each cloned Task is wired back to the Transformation via a + ``PERFORMS`` edge so the standard + ``(:AlchemicalNetwork)-[:DEPENDS_ON]->(:Transformation)<-[:PERFORMS]-(:Task)`` + traversal keeps working on the merged network. + + Parameters + ---------- + target_scope + The destination ``Scope`` to stamp on every cloned Task / PDRR. + tf_node + The existing Transformation ``Node`` from the surrounding + merged-network subgraph. Reusing the **same Python Node object** + (rather than constructing a fresh one with a matching + ``_scoped_key``) is required: :func:`merge_subgraph` assigns a + Neo4j ``elementId`` to only the first Python Node it sees per + primary key, so any Relationship pointing at a duplicate Node + object would be silently dropped on commit. + """ + # if there are no tasks, return an empty subgraph + if not self.task_tree: + return Subgraph() + + subgraph = Subgraph() + + scope_props = { + "_org": target_scope.org, + "_campaign": target_scope.campaign, + "_project": target_scope.project, + } + + def record_to_node(record): + # create node from a neo4j record with updated scoped key + scoped_key = ScopedKey( + gufe_key=record["_gufe_key"], **target_scope.to_dict() + ) + return Node( + *record.labels, + **record._properties | scope_props | {"_scoped_key": str(scoped_key)}, + ) + + # Memoize Task Nodes by gufe key across iterations of ``task_tree``. + # If ``record[i]["task"]`` is the same Task as + # ``record[j]["extended_task"]``, we must reuse the same Python + # Node object -- otherwise ``merge_subgraph`` assigns a Neo4j + # ``elementId`` only to the first Python Node it sees per primary + # key and the EXTENDS Relationship's endpoint (pointing at the + # duplicate) is silently dropped on commit. Same bug pattern as + # the AN's Transformation Node vs. the to_subgraph anchor. + task_node_cache: dict[str, Node] = {} + + def get_or_create_task_node(record): + gufe_key = record["_gufe_key"] + node = task_node_cache.get(gufe_key) + if node is None: + node = record_to_node(record) + task_node_cache[gufe_key] = node + return node + + # process each task found. Each record represents a single task. + for record in self.task_tree: + # update task node to have new scoped key + task_node = get_or_create_task_node(record["task"]) + # wire the cloned Task back to its Transformation; without + # this edge the Task is unreachable from get_network_tasks + # and every other PERFORMS-based traversal + subgraph |= Relationship.type("PERFORMS")(task_node, tf_node, **scope_props) + # create the task node this task extends if it exists. The + # source query returns ``record["extended_task"]`` as the Task + # that ``record["task"]`` extends -- i.e. in the source graph, + # ``(task)-[:EXTENDS]->(extended_task)``. The relationship's + # direction must be preserved on the clone so downstream + # traversals that read ``(task)-[:EXTENDS]->(other_task)`` + # keep working. + etask_node = ( + None + if not record["extended_task"] + else get_or_create_task_node(record["extended_task"]) + ) + if etask_node: + subgraph |= Relationship.type("EXTENDS")( + task_node, etask_node, **scope_props + ) + # clone all result refs for the task + for pdrr_record in record["pdrrs"]: + pdrr_node = record_to_node(pdrr_record) + subgraph |= Relationship.type("RESULTS_IN")( + task_node, pdrr_node, **scope_props + ) + + return subgraph + + class AlchemiscaleStateStore(abc.ABC): ... @@ -876,6 +1078,266 @@ def delete_network( """ raise NotImplementedError + def merge_networks( + self, + network_scoped_keys: list[ScopedKey], + name: str, + scope: Scope, + state: NetworkStateEnum | str = NetworkStateEnum.active, + ) -> ScopedKey: + """Merge multiple ``AlchemicalNetwork`` nodes into a new ``AlchemicalNetwork``. + + Each ``Transformation`` / ``NonTransformation`` in the input + networks is included exactly once in the new network. Only + ``Task``\\ s on the source networks in ``complete`` state are + cloned into the new network's ``Scope``, along with their + ``ProtocolDAGResultRef``\\ s and ``EXTENDS`` relationships, + wired to their ``Transformation`` via ``PERFORMS``. + + Any ``Strategy`` (``PROGRESSES``) and ``TaskRestartPattern``\\ s + (``ENFORCES`` / ``APPLIES``) on the source networks are not + carried over; set them on the merged network via + :meth:`set_network_strategy` and :meth:`add_task_restart_patterns` + if needed. + + Parameters + ---------- + network_scoped_keys + List of ``AlchemicalNetwork`` ``ScopedKey`` objects to merge. + name + The name of the new ``AlchemicalNetwork``. + scope + The ``Scope`` of the new ``AlchemicalNetwork``. + state + The starting state for the new ``AlchemicalNetwork``'s + ``NetworkMark``. Defaults to ``NetworkStateEnum.active``. + + Returns + ------- + The ``ScopedKey`` of the new ``AlchemicalNetwork`` in the database. + """ + # reject non-AlchemicalNetwork ScopedKeys up front (both HTTP and + # Python callers land here; the client-side check alone would leave + # the HTTP endpoint open to Transformation/ChemicalSystem SKs) + non_networks = [ + sk for sk in network_scoped_keys if sk.qualname != "AlchemicalNetwork" + ] + if non_networks: + joined = ", ".join(str(sk) for sk in non_networks) + raise ValueError( + f"The following ScopedKey(s) do not refer to AlchemicalNetworks: {joined}" + ) + + # Collect keyed chain representation for all alchemical networks, + # gathering every missing ScopedKey up front so callers passing + # many SKs get a single, complete error. + network_keyed_chains: list[tuple[Scope, KeyedChain]] = [] + missing: list[ScopedKey] = [] + for network_scoped_key in network_scoped_keys: + try: + keyed_chain = self.get_keyed_chain(network_scoped_key) + except KeyError: + missing.append(network_scoped_key) + continue + network_keyed_chains.append((network_scoped_key.scope, keyed_chain)) + if missing: + joined = ", ".join(str(sk) for sk in missing) + raise ValueError( + f"The following ScopedKey(s) were not found in the database: {joined}" + ) + + # Map decoded Transformation / NonTransformation objects to all of + # their original database GufeKeys (potentially across multiple + # source networks, and including duplicates introduced by minor + # serialization-version drift). These original keys are needed to + # locate Tasks associated with each Transformation in the source + # networks. We key the dedup map on ``transformation.key`` (the + # decoded GufeKey) to avoid O(N^2) GufeTokenizable equality checks + # for large networks. + transformation_data: dict[GufeKey, _TransformationData] = {} + for network_scope, network_keyed_chain in network_keyed_chains: + # database keys for Transformations / NonTransformations in this + # source network's chain, in chain order + database_keys = [ + gufe_key + for gufe_key, keyed_dict in network_keyed_chain + if _is_transformation_keyed_dict(keyed_dict) + ] + # decoded Transformation / NonTransformation objects, yielded + # by decode_subchains in the same chain order as the predicate + # selects above (gufe contract); decode_subchains also shares + # a tokenizable_map across yields so common dependencies are + # decoded only once per source network. + transformations = network_keyed_chain.decode_subchains( + _is_transformation_keyed_dict + ) + for database_key, transformation in zip(database_keys, transformations): + data = transformation_data.get(transformation.key) + if data is None: + data = _TransformationData(transformation) + transformation_data[transformation.key] = data + data.add_known_scoped_key(database_key, network_scope) + + # Collect all transformation gufe objects and collect into a new set of edges. + # Only ``complete`` Tasks carry over -- the new network reflects the + # results the user has already computed; in-flight, errored, or + # tombstoned Tasks are not preserved. + _TransformationData.update_task_trees( + list(transformation_data.values()), + self, + task_statuses=["complete"], + ) + new_edges = [td.transformation for td in transformation_data.values()] + # Make new alchemiscale network with these edges + combined_alchemical_network = AlchemicalNetwork(edges=new_edges, name=name) + an_subgraph, an_node, an_sk = self._keyed_chain_to_subgraph( + KeyedChain.from_gufe(combined_alchemical_network), scope + ) + # create and fold in taskhub and network mark supporting nodes + an_subgraph |= ( + self.create_network_mark_subgraph(an_node, state=state)[0] + | self.create_taskhub_subgraph(an_node)[0] + ) + + # Build a lookup from gufe key to the existing Transformation Node + # already present in ``an_subgraph``. ``to_subgraph`` must reuse + # these Python Node objects rather than building fresh ones with + # the same ``_scoped_key`` -- ``merge_subgraph`` assigns a Neo4j + # ``elementId`` only to the first Python Node it sees per primary + # key, so Relationships pointing at a duplicate Node would be + # silently dropped on commit. + tf_node_lookup = { + node["_gufe_key"]: node + for node in an_subgraph.nodes + if "Transformation" in node.labels or "NonTransformation" in node.labels + } + + # create and fold in all task and results data + for td in transformation_data.values(): + tf_node = tf_node_lookup[str(td.transformation.key)] + an_subgraph |= td.to_subgraph(scope, tf_node) + + # merge the new network into neo4j + with self.transaction() as tx: + merge_subgraph(tx, an_subgraph, "GufeTokenizable", "_scoped_key") + return an_sk + + def copy_network( + self, + network_scoped_key: ScopedKey, + scope: Scope, + name: str | None = None, + state: NetworkStateEnum | str = NetworkStateEnum.active, + ) -> ScopedKey: + """Copy an ``AlchemicalNetwork`` to a new ``Scope``, carrying over + every ``complete`` ``Task`` and its associated + ``ProtocolDAGResultRef``\\ s, wired to their ``Transformation`` + via ``PERFORMS`` and to one another via ``EXTENDS`` where + applicable. + + Any ``Strategy`` (``PROGRESSES``) and ``TaskRestartPattern``\\ s + (``ENFORCES`` / ``APPLIES``) on the source network are not + carried over; set them on the copy via + :meth:`set_network_strategy` and :meth:`add_task_restart_patterns` + if needed. + + Parameters + ---------- + network_scoped_key + The ``ScopedKey`` of the source ``AlchemicalNetwork``. + scope + The destination ``Scope`` for the copy. + name + Optional new name for the copy. If ``None``, the source + ``AlchemicalNetwork``'s name is preserved; the resulting copy + has the same gufe key as the source. If a different name is + given, the resulting copy has a fresh gufe key derived from + the renamed content. + state + Starting ``NetworkStateEnum`` for the copy's ``NetworkMark``. + Defaults to ``NetworkStateEnum.active``. + + Returns + ------- + The ``ScopedKey`` of the new ``AlchemicalNetwork`` in the database. + """ + # reject non-AlchemicalNetwork ScopedKeys up front (both HTTP and + # Python callers land here; the client-side check alone would leave + # the HTTP endpoint open to Transformation/ChemicalSystem SKs) + if network_scoped_key.qualname != "AlchemicalNetwork": + raise ValueError( + f"ScopedKey ({network_scoped_key}) does not refer to an AlchemicalNetwork" + ) + + # decode the source network's keyed chain + try: + source_kc = self.get_keyed_chain(network_scoped_key) + except KeyError: + raise ValueError( + f"ScopedKey ({network_scoped_key}) not found in the database." + ) + source_an: AlchemicalNetwork = source_kc.to_gufe() + + # optionally rename; preserving the name preserves the gufe key + if name is not None and name != source_an.name: + target_an = source_an.copy_with_replacements(name=name) + else: + target_an = source_an + + # gather Transformations and their source-scope database keys, + # then load every Task that PERFORMs on each (no status filter) + source_scope = network_scoped_key.scope + transformation_data: dict[GufeKey, _TransformationData] = {} + database_keys = [ + gufe_key + for gufe_key, keyed_dict in source_kc + if _is_transformation_keyed_dict(keyed_dict) + ] + transformations = source_kc.decode_subchains(_is_transformation_keyed_dict) + for database_key, transformation in zip(database_keys, transformations): + data = transformation_data.get(transformation.key) + if data is None: + data = _TransformationData(transformation) + transformation_data[transformation.key] = data + data.add_known_scoped_key(database_key, source_scope) + + # Only ``complete`` Tasks carry over -- the copy reflects the + # results the user has already computed; in-flight, errored, or + # tombstoned Tasks are not preserved. + _TransformationData.update_task_trees( + list(transformation_data.values()), + self, + task_statuses=["complete"], + ) + + # build the new AlchemicalNetwork's subgraph + NetworkMark + TaskHub + an_subgraph, an_node, an_sk = self._keyed_chain_to_subgraph( + KeyedChain.from_gufe(target_an), scope + ) + an_subgraph |= ( + self.create_network_mark_subgraph(an_node, state=state)[0] + | self.create_taskhub_subgraph(an_node)[0] + ) + + # Look up the existing Transformation Nodes in ``an_subgraph`` so + # ``to_subgraph`` can reuse the same Python Node objects rather + # than building fresh duplicates; see :meth:`merge_networks` for + # why this matters. + tf_node_lookup = { + node["_gufe_key"]: node + for node in an_subgraph.nodes + if "Transformation" in node.labels or "NonTransformation" in node.labels + } + + # fold in cloned Tasks, EXTENDS, PERFORMS, and RESULTS_IN + for td in transformation_data.values(): + tf_node = tf_node_lookup[str(td.transformation.key)] + an_subgraph |= td.to_subgraph(scope, tf_node) + + with self.transaction() as tx: + merge_subgraph(tx, an_subgraph, "GufeTokenizable", "_scoped_key") + return an_sk + def get_network_state(self, networks: list[ScopedKey]) -> list[str | None]: """Get the states of a group of networks. @@ -2853,7 +3315,10 @@ def task_count(task_dict: dict): ## tasks - def _validate_extends_tasks(self, task_list) -> dict[str, tuple[Node, str]]: + def _validate_extends_tasks( + self, + task_list, + ) -> dict[str, tuple[Node, str]]: if not task_list: return {} @@ -2943,7 +3408,7 @@ def create_tasks( transformation_map[transformation.qualname][1].append(extends[i]) extends_nodes = self._validate_extends_tasks( - [_extends for _extends in extends if _extends is not None] + [_extends for _extends in extends if _extends is not None], ) subgraph = Subgraph() diff --git a/alchemiscale/tests/integration/interface/client/test_client.py b/alchemiscale/tests/integration/interface/client/test_client.py index 041deb15..2cffa331 100644 --- a/alchemiscale/tests/integration/interface/client/test_client.py +++ b/alchemiscale/tests/integration/interface/client/test_client.py @@ -2,6 +2,7 @@ import datetime from time import sleep import os +import uuid from pathlib import Path from itertools import chain import json @@ -200,6 +201,646 @@ def test_create_network( # common with an existing network # user_client.create_network( + def test_merge_networks( + self, + scope_test, + multiple_scopes, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + network_tyk2, + ): + # gather source AlchemicalNetwork ScopedKeys across all scopes + # n4js_preloaded creates `network_tyk2` and a trimmed copy named + # "incomplete" in each of `multiple_scopes` + source_sks = user_client.query_networks(state=None) + + # destination scope: reuse `scope_test`, which is authorized for + # the test identity. The merged network's name differs from the + # pre-loaded networks, so there is no _scoped_key collision. + merge_scope = scope_test + + merged_sk = user_client.merge_networks( + networks=source_sks, + name="merged_tyk2", + scope=merge_scope, + visualize=False, + ) + + assert isinstance(merged_sk, ScopedKey) + assert merged_sk.scope == merge_scope + assert merged_sk.qualname == "AlchemicalNetwork" + + # the merged network should exist + assert user_client.check_exists(merged_sk) + + # the merged network should appear in queries (defaults to active state) + all_active_sks = user_client.query_networks() + assert merged_sk in all_active_sks + + # the merged network should contain the union of all source edges; + # since `network_tyk2` is a superset of `incomplete`, the union equals + # `network_tyk2.edges` + merged_network = user_client.get_network(merged_sk) + assert merged_network.name == "merged_tyk2" + assert len(merged_network.edges) == len(network_tyk2.edges) + assert {t.key for t in merged_network.edges} == { + t.key for t in network_tyk2.edges + } + + @pytest.mark.parametrize("state", ["active", "inactive"]) + def test_merge_networks_respects_state( + self, + state, + scope_test, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + ): + """The state parameter must control the NetworkMark on the merged + AlchemicalNetwork the same way it does on create_network.""" + source_sks = user_client.query_networks(scope=scope_test, state=None) + assert source_sks + + # destination scope: reuse `scope_test`, which is authorized for + # the test identity; the unique network name keeps each parametrize + # case from colliding on _scoped_key. + merge_scope = scope_test + merged_sk = user_client.merge_networks( + networks=source_sks, + name=f"merged_state_{state}", + scope=merge_scope, + state=state, + visualize=False, + ) + + assert user_client.get_network_state(merged_sk) == state + + def test_merge_networks_rejects_wildcard_scope( + self, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + ): + source_sks = user_client.query_networks(state=None) + with pytest.raises(ValueError, match="wildcards"): + user_client.merge_networks( + networks=source_sks, + name="should_fail", + scope=Scope(org="test_org"), + visualize=False, + ) + + def test_merge_networks_rejects_empty_list( + self, + scope_test, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + ): + with pytest.raises(ValueError, match="at least one"): + user_client.merge_networks( + networks=[], + name="should_fail", + scope=scope_test, + visualize=False, + ) + + def test_merge_networks_rejects_non_network_scoped_key( + self, + scope_test, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + ): + # pass a Transformation ScopedKey rather than an AlchemicalNetwork one + tf_sks = user_client.query_transformations(scope=scope_test) + assert tf_sks + with pytest.raises(ValueError, match="does not refer to an AlchemicalNetwork"): + user_client.merge_networks( + networks=[tf_sks[0]], + name="should_fail", + scope=scope_test, + visualize=False, + ) + + def test_merge_networks_preserves_tasks_and_results( + self, + scope_test, + multiple_scopes, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + network_tyk2, + ): + """The merged network must carry over ``complete`` Tasks -- and only + ``complete`` Tasks -- along with their ProtocolDAGResultRefs, cloned + into the new scope. + + Tasks in any other status (``waiting``, ``running``, ``error``, + ``invalid``, ``deleted``) must not be cloned. + """ + # pick a source network in scope_test + source_sks = user_client.query_networks(scope=scope_test, state=None) + assert source_sks + source_sk = source_sks[0] + + # create Tasks on four of its Transformations directly through n4js + # so we can drive them to distinct statuses without actually + # executing protocols. The four statuses cover the "carry"/"drop" + # boundary: complete is preserved, error/waiting/running are dropped. + transformation_sks = n4js_preloaded.get_network_transformations(source_sk) + assert len(transformation_sks) >= 4 + + task_sks = n4js_preloaded.create_tasks(transformation_sks[:4]) + + # task 0: complete + ok PDRR -- carried over + n4js_preloaded.set_task_running(task_sks[:1]) + n4js_preloaded.set_task_complete(task_sks[:1]) + ok_pdrr = ProtocolDAGResultRef( + obj_key=f"ProtocolDAGResult-{uuid.uuid4()}", + scope=task_sks[0].scope, + ok=True, + ) + n4js_preloaded.set_task_result(task_sks[0], ok_pdrr) + + # task 1: error + failure PDRR -- dropped + n4js_preloaded.set_task_running(task_sks[1:2]) + n4js_preloaded.set_task_error(task_sks[1:2]) + err_pdrr = ProtocolDAGResultRef( + obj_key=f"ProtocolDAGResult-{uuid.uuid4()}", + scope=task_sks[1].scope, + ok=False, + ) + n4js_preloaded.set_task_result(task_sks[1], err_pdrr) + + # task 2: waiting -- dropped + # (default status after create_tasks) + + # task 3: running -- dropped + n4js_preloaded.set_task_running(task_sks[3:4]) + + # merge into a different authorized scope from where we set up the + # source Tasks, so the per-scope counts below remain clean (the + # destination scope has pre-loaded networks but no Tasks/PDRRs). + merge_scope = multiple_scopes[1] + assert merge_scope != scope_test + merged_sk = user_client.merge_networks( + networks=[source_sk], + name="merged_with_results", + scope=merge_scope, + visualize=False, + ) + assert user_client.check_exists(merged_sk) + + # only the complete Task should appear in the new scope + task_records = n4js_preloaded.execute_query( + """ + MATCH (t:Task {`_project`: $project}) + RETURN t.status AS status + """, + project=merge_scope.project, + ).records + assert [r["status"] for r in task_records] == ["complete"] + + # only the ok PDRR should appear in the new scope; the error PDRR + # is dropped along with its Task + pdrr_records = n4js_preloaded.execute_query( + """ + MATCH (pdrr:ProtocolDAGResultRef {`_project`: $project}) + RETURN pdrr.ok AS ok, pdrr.obj_key AS obj_key + """, + project=merge_scope.project, + ).records + assert len(pdrr_records) == 1 + assert pdrr_records[0]["ok"] is True + assert pdrr_records[0]["obj_key"] == ok_pdrr.obj_key + + # cloned PDRR must be wired to the cloned Task in the new scope + linked = n4js_preloaded.execute_query( + """ + MATCH (t:Task {`_project`: $project})-[:RESULTS_IN]-> + (pdrr:ProtocolDAGResultRef {`_project`: $project}) + RETURN t.status AS status, pdrr.ok AS ok + """, + project=merge_scope.project, + ).records + assert [(r["status"], r["ok"]) for r in linked] == [("complete", True)] + + # the cloned Task must be reachable from the merged AlchemicalNetwork + # via the standard PERFORMS traversal that the user-facing API uses; + # this catches any case where Tasks are written to the new scope but + # not wired back to their Transformations in the merged network + merged_task_sks = user_client.get_network_tasks(merged_sk) + assert len(merged_task_sks) == 1 + assert merged_task_sks[0].scope == merge_scope + assert user_client.get_tasks_status(merged_task_sks) == ["complete"] + + def test_copy_network( + self, + scope_test, + multiple_scopes, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + network_tyk2, + ): + """copy_network must duplicate the source AlchemicalNetwork into the + target scope and carry over only ``complete`` Tasks (with their + PDRRs), while dropping Tasks in any other status. EXTENDS + relationships and PERFORMS wiring must be intact for the Tasks + that do carry over. + """ + source_sks = user_client.query_networks(scope=scope_test, state=None) + assert source_sks + source_sk = source_sks[0] + source_an = user_client.get_network(source_sk) + + transformation_sks = n4js_preloaded.get_network_transformations(source_sk) + assert len(transformation_sks) >= 4 + + # Stage four tasks across four statuses so we can prove copy_network + # carries over only ``complete`` Tasks and drops the rest: + # task 0: complete + ok PDRR -- carried + # task 1: error + not-ok PDRR -- dropped + # task 2: waiting (no result) -- dropped + # task 3: running (no result) -- dropped + task_sks = n4js_preloaded.create_tasks(transformation_sks[:4]) + + n4js_preloaded.set_task_running(task_sks[:1]) + n4js_preloaded.set_task_complete(task_sks[:1]) + ok_pdrr = ProtocolDAGResultRef( + obj_key=f"ProtocolDAGResult-{uuid.uuid4()}", + scope=task_sks[0].scope, + ok=True, + ) + n4js_preloaded.set_task_result(task_sks[0], ok_pdrr) + + n4js_preloaded.set_task_running(task_sks[1:2]) + n4js_preloaded.set_task_error(task_sks[1:2]) + err_pdrr = ProtocolDAGResultRef( + obj_key=f"ProtocolDAGResult-{uuid.uuid4()}", + scope=task_sks[1].scope, + ok=False, + ) + n4js_preloaded.set_task_result(task_sks[1], err_pdrr) + + # task_sks[2] stays waiting (default after create_tasks) + n4js_preloaded.set_task_running(task_sks[3:4]) + + # copy into a scope distinct from the source, so per-_project counts + # are clean + target_scope = multiple_scopes[2] + assert target_scope != scope_test + copied_sk = user_client.copy_network( + network=source_sk, + scope=target_scope, + visualize=False, + ) + + # name is preserved → gufe key is preserved → only the scope changes + assert copied_sk.gufe_key == source_sk.gufe_key + assert copied_sk.scope == target_scope + assert user_client.check_exists(copied_sk) + assert user_client.get_network(copied_sk).name == source_an.name + + # the target scope is preloaded with this same AlchemicalNetwork, so + # the copy must dedup onto the existing node rather than create a + # duplicate; the copy is additive to the preexisting network + an_count = n4js_preloaded.execute_query( + "MATCH (an:AlchemicalNetwork {_scoped_key: $sk}) RETURN count(an) AS n", + sk=str(copied_sk), + ).records[0]["n"] + assert an_count == 1 + + # only the ``complete`` Task must be reachable from the new network + # via the standard PERFORMS traversal; the error / waiting / running + # Tasks are dropped + copied_task_sks = user_client.get_network_tasks(copied_sk) + assert len(copied_task_sks) == 1 + assert copied_task_sks[0].scope == target_scope + assert user_client.get_tasks_status(copied_task_sks) == ["complete"] + + # only the ok PDRR lands in the target scope; the error PDRR is + # dropped along with its Task + pdrr_records = n4js_preloaded.execute_query( + """ + MATCH (pdrr:ProtocolDAGResultRef {`_project`: $project}) + RETURN pdrr.ok AS ok, pdrr.obj_key AS obj_key + """, + project=target_scope.project, + ).records + assert len(pdrr_records) == 1 + assert pdrr_records[0]["ok"] is True + assert pdrr_records[0]["obj_key"] == ok_pdrr.obj_key + + def test_copy_network_with_rename( + self, + scope_test, + multiple_scopes, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + ): + """copy_network with a non-default `name` must produce a distinct + gufe key while still copying into the target scope.""" + source_sks = user_client.query_networks(scope=scope_test, state=None) + source_sk = source_sks[0] + source_an = user_client.get_network(source_sk) + + target_scope = multiple_scopes[2] + new_name = source_an.name + "_renamed_copy" + + copied_sk = user_client.copy_network( + network=source_sk, + scope=target_scope, + name=new_name, + visualize=False, + ) + + assert copied_sk.gufe_key != source_sk.gufe_key + assert copied_sk.scope == target_scope + copied_an = user_client.get_network(copied_sk) + assert copied_an.name == new_name + + def test_copy_network_preserves_extends_direction( + self, + scope_test, + multiple_scopes, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + ): + """The cloned EXTENDS edge must match the source direction: + ``(task)-[:EXTENDS]->(extended_task)``. Regressing this direction + would silently invert every EXTENDS chain on copied networks and + break the downstream traversals that read it forward (e.g. + ``get_task_extends``, ``get_task_results``).""" + source_sks = user_client.query_networks(scope=scope_test, state=None) + source_sk = source_sks[0] + + transformation_sks = n4js_preloaded.get_network_transformations(source_sk) + assert transformation_sks + + # base Task: complete with ok PDRR + base_task_sks = n4js_preloaded.create_tasks(transformation_sks[:1]) + n4js_preloaded.set_task_running(base_task_sks) + n4js_preloaded.set_task_complete(base_task_sks) + base_pdrr = ProtocolDAGResultRef( + obj_key=f"ProtocolDAGResult-{uuid.uuid4()}", + scope=base_task_sks[0].scope, + ok=True, + ) + n4js_preloaded.set_task_result(base_task_sks[0], base_pdrr) + + # extending Task: extends the base, also complete; the EXTENDS edge + # in the source graph is ``(extending)-[:EXTENDS]->(base)`` + extending_task_sks = n4js_preloaded.create_tasks( + transformation_sks[:1], extends=base_task_sks + ) + n4js_preloaded.set_task_running(extending_task_sks) + n4js_preloaded.set_task_complete(extending_task_sks) + ext_pdrr = ProtocolDAGResultRef( + obj_key=f"ProtocolDAGResult-{uuid.uuid4()}", + scope=extending_task_sks[0].scope, + ok=True, + ) + n4js_preloaded.set_task_result(extending_task_sks[0], ext_pdrr) + + target_scope = multiple_scopes[2] + user_client.copy_network(network=source_sk, scope=target_scope, visualize=False) + + # exactly one EXTENDS edge between the two cloned Tasks must exist + # in the target scope, with the same direction as the source: + # extending -> EXTENDS -> base + edges = n4js_preloaded.execute_query( + """ + MATCH (a:Task {`_project`: $project})-[:EXTENDS]->(b:Task {`_project`: $project}) + RETURN a._gufe_key AS extender, b._gufe_key AS extended + """, + project=target_scope.project, + ).records + assert len(edges) == 1 + assert edges[0]["extender"] == str(extending_task_sks[0].gufe_key) + assert edges[0]["extended"] == str(base_task_sks[0].gufe_key) + + def test_copy_network_rejects_wildcard_scope( + self, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + ): + source_sks = user_client.query_networks(state=None) + with pytest.raises(ValueError, match="wildcards"): + user_client.copy_network( + network=source_sks[0], + scope=Scope(org="test_org"), + visualize=False, + ) + + def test_copy_network_rejects_non_network_scoped_key( + self, + scope_test, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + ): + tf_sks = user_client.query_transformations(scope=scope_test) + assert tf_sks + with pytest.raises(ValueError, match="does not refer to an AlchemicalNetwork"): + user_client.copy_network( + network=tf_sks[0], + scope=scope_test, + visualize=False, + ) + + def test_merge_scopes( + self, + scope_test, + multiple_scopes, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + ): + """merge_scopes must copy every AlchemicalNetwork in the listed + source scopes into the target scope, preserving names (and + therefore gufe keys).""" + source_scopes = [scope_test, multiple_scopes[1]] + target_scope = multiple_scopes[2] + + # baseline: 2 networks pre-loaded in target_scope + baseline_target_sks = user_client.query_networks(scope=target_scope, state=None) + assert len(baseline_target_sks) == 2 + + expected_source_count = sum( + len(user_client.query_networks(scope=sc, state=None)) + for sc in source_scopes + ) + # 2 source networks per scope, 2 source scopes + assert expected_source_count == 4 + + new_sks = user_client.merge_scopes( + scopes=source_scopes, + target_scope=target_scope, + visualize=False, + ) + + # one returned ScopedKey per source network + assert len(new_sks) == expected_source_count + assert all(sk.scope == target_scope for sk in new_sks) + + # every copy preserves the source network's gufe key, so each copy's + # ScopedKey is just (source_gufe_key, target_scope). Since the source + # scopes contain the same set of networks (preloaded with the same + # names), the returned ScopedKeys collapse to the same 2 unique keys. + assert len(set(new_sks)) == 2 + + # target scope ends up with the union of its baseline networks and + # the copied networks; baseline and copies share the same gufe keys + # (assemble_network preloaded the same names everywhere), so the + # final count is unchanged from baseline + final_target_sks = user_client.query_networks(scope=target_scope, state=None) + assert set(baseline_target_sks) == set(final_target_sks) + + def test_merge_scopes_creates_new_networks_in_target( + self, + scope_test, + multiple_scopes, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + network_tyk2, + ): + """merge_scopes must create fresh AlchemicalNetwork nodes in the + target scope for source networks that don't already exist there -- + not just dedup onto preexisting ones. + + n4js_preloaded seeds the same 2 networks (network_tyk2 + an2) into + every scope in multiple_scopes, so we first add a *third* + AlchemicalNetwork to the source scope only. After merge_scopes, + the target scope must contain that third network as a new node, + alongside the 2 that dedup onto its preexisting entries. + """ + source_scope = scope_test + target_scope = multiple_scopes[2] + + # add a fresh AN to the source scope that does not exist in the + # target; a unique name → unique gufe key + fresh_an = AlchemicalNetwork( + edges=list(network_tyk2.edges)[:2], + name="fresh_source_only", + ) + fresh_source_sk = user_client.create_network(fresh_an, source_scope) + fresh_expected_target_sk = ScopedKey( + gufe_key=fresh_source_sk.gufe_key, + org=target_scope.org, + campaign=target_scope.campaign, + project=target_scope.project, + ) + + # sanity: fresh AN exists in source, does not exist in target + assert user_client.check_exists(fresh_source_sk) + assert not user_client.check_exists(fresh_expected_target_sk) + + baseline_target_sks = set( + user_client.query_networks(scope=target_scope, state=None) + ) + # baseline: network_tyk2 + an2 preloaded in target + assert len(baseline_target_sks) == 2 + assert fresh_expected_target_sk not in baseline_target_sks + + new_sks = user_client.merge_scopes( + scopes=[source_scope], + target_scope=target_scope, + visualize=False, + ) + + # source scope holds 3 networks now (2 preloaded + 1 fresh) → 3 + # returned ScopedKeys, all anchored to target_scope + assert len(new_sks) == 3 + assert all(sk.scope == target_scope for sk in new_sks) + assert fresh_expected_target_sk in new_sks + + # target scope now holds the baseline union the fresh AN as a + # genuinely new node + final_target_sks = set( + user_client.query_networks(scope=target_scope, state=None) + ) + assert final_target_sks == baseline_target_sks | {fresh_expected_target_sk} + + # exactly one AN node backs the fresh copy in the target scope + fresh_count = n4js_preloaded.execute_query( + "MATCH (an:AlchemicalNetwork {_scoped_key: $sk}) RETURN count(an) AS n", + sk=str(fresh_expected_target_sk), + ).records[0]["n"] + assert fresh_count == 1 + + def test_merge_scopes_preserves_source_state( + self, + scope_test, + multiple_scopes, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + network_tyk2, + ): + """merge_scopes must copy each source network into the target scope + with the source network's *state*, not silently default every + copy to ``active``. Regressing this would resurrect + inactive/invalid/deleted networks as active on the target. + """ + source_scope = scope_test + target_scope = multiple_scopes[2] + + # add three fresh networks to source, one per exercised state, so + # we can observe the state flowing through cleanly without + # interference from n4js_preloaded's baseline + fresh_networks = { + "active": AlchemicalNetwork( + edges=list(network_tyk2.edges)[:2], name="state_test_active" + ), + "inactive": AlchemicalNetwork( + edges=list(network_tyk2.edges)[:2], name="state_test_inactive" + ), + "deleted": AlchemicalNetwork( + edges=list(network_tyk2.edges)[:2], name="state_test_deleted" + ), + } + expected_target_sks: dict[str, ScopedKey] = {} + for state, an in fresh_networks.items(): + sk = user_client.create_network(an, source_scope, state=state) + expected_target_sks[state] = ScopedKey( + gufe_key=sk.gufe_key, + org=target_scope.org, + campaign=target_scope.campaign, + project=target_scope.project, + ) + # sanity: source got the right state + assert user_client.get_network_state(sk) == state + + user_client.merge_scopes( + scopes=[source_scope], + target_scope=target_scope, + visualize=False, + ) + + # each copy must land in target_scope with the source's state + for state, target_sk in expected_target_sks.items(): + assert user_client.check_exists(target_sk) + assert user_client.get_network_state(target_sk) == state + + def test_merge_scopes_rejects_empty( + self, + scope_test, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + ): + with pytest.raises(ValueError, match="at least one"): + user_client.merge_scopes( + scopes=[], + target_scope=scope_test, + visualize=False, + ) + + def test_merge_scopes_rejects_wildcard_target( + self, + scope_test, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + ): + with pytest.raises(ValueError, match="wildcards"): + user_client.merge_scopes( + scopes=[scope_test], + target_scope=Scope(org="test_org"), + visualize=False, + ) + def test_check_exists( self, scope_test, diff --git a/alchemiscale/tests/integration/interface/test_api.py b/alchemiscale/tests/integration/interface/test_api.py index 21322c02..549f4702 100644 --- a/alchemiscale/tests/integration/interface/test_api.py +++ b/alchemiscale/tests/integration/interface/test_api.py @@ -103,6 +103,206 @@ def test_create_network_bad_scope( assert str(bad_scope) in details assert str(scope_test) in details + def test_merge_networks( + self, n4js_preloaded, test_client, network_tyk2, scope_test + ): + n4js = n4js_preloaded + + # source networks in scope_test (pre-loaded by n4js_preloaded) + source_sks = n4js.query_networks(scope=scope_test) + assert len(source_sks) >= 2 + + # destination scope: a new project under the same org/campaign so the + # test_client's scope_test token has access + merge_scope_dict = { + "org": scope_test.org, + "campaign": scope_test.campaign, + "project": scope_test.project, + } + + headers = {"Content-type": "application/json"} + data = dict( + networks=[str(sk) for sk in source_sks], + name="api_merged", + scope=merge_scope_dict, + ) + jsondata = json.dumps(data, cls=JSON_HANDLER.encoder) + + response = test_client.post("/networks/merge", data=jsondata, headers=headers) + assert response.status_code == 200 + + merged_sk = ScopedKey(**response.json()) + assert merged_sk.scope == scope_test + assert merged_sk.gufe_key.startswith("AlchemicalNetwork-") + + # network should now be present in the database + assert n4js.check_existence(merged_sk) + + # merged network's union-of-edges equals network_tyk2's edge set, + # since one of the sources is network_tyk2 and the other is a strict subset + merged_network = n4js.get_gufe(merged_sk) + assert merged_network.name == "api_merged" + assert {t.key for t in merged_network.edges} == { + t.key for t in network_tyk2.edges + } + + def test_merge_networks_bad_scope( + self, n4js_preloaded, test_client, scope_test, multiple_scopes + ): + # destination scope the test_client's token does not have access to + bad_scope = multiple_scopes[1] + assert bad_scope != scope_test + + source_sks = n4js_preloaded.query_networks(scope=scope_test) + assert source_sks + + headers = {"Content-type": "application/json"} + data = dict( + networks=[str(sk) for sk in source_sks], + name="should_fail", + scope=bad_scope.to_dict(), + ) + jsondata = json.dumps(data, cls=JSON_HANDLER.encoder) + + response = test_client.post("/networks/merge", data=jsondata, headers=headers) + assert response.status_code == 401 + details = response.json() + assert "detail" in details + assert str(bad_scope) in details["detail"] + + def test_merge_networks_bad_source_scope( + self, n4js_preloaded, test_client, scope_test, multiple_scopes + ): + # source network in a scope the test_client's token does not authorize + unauth_scope = multiple_scopes[1] + assert unauth_scope != scope_test + + unauth_source_sks = n4js_preloaded.query_networks(scope=unauth_scope) + assert unauth_source_sks + + headers = {"Content-type": "application/json"} + data = dict( + networks=[str(sk) for sk in unauth_source_sks], + name="should_fail", + scope=scope_test.to_dict(), + ) + jsondata = json.dumps(data, cls=JSON_HANDLER.encoder) + + response = test_client.post("/networks/merge", data=jsondata, headers=headers) + assert response.status_code == 401 + details = response.json() + assert "detail" in details + assert str(unauth_scope) in details["detail"] + + def test_merge_networks_bad_qualname(self, n4js_preloaded, test_client, scope_test): + """POST /networks/merge with a non-AlchemicalNetwork ScopedKey in + ``networks`` must be rejected at the store layer (422).""" + # a Transformation ScopedKey masquerading as a network SK + tf_sks = n4js_preloaded.query_transformations(scope=scope_test) + assert tf_sks + + headers = {"Content-type": "application/json"} + data = dict( + networks=[str(tf_sks[0])], + name="should_fail", + scope=scope_test.to_dict(), + ) + jsondata = json.dumps(data, cls=JSON_HANDLER.encoder) + + response = test_client.post("/networks/merge", data=jsondata, headers=headers) + assert response.status_code == 422 + assert "AlchemicalNetwork" in response.json()["detail"] + + def test_copy_network(self, n4js_preloaded, test_client, network_tyk2, scope_test): + n4js = n4js_preloaded + + # pick a source network in scope_test (pre-loaded) + source_sks = n4js.query_networks(scope=scope_test) + assert source_sks + source_sk = source_sks[0] + + # destination scope: reuse scope_test since the test_client's token + # only authorizes it; the copy dedups onto the preexisting node + # (name preserved, so gufe_key preserved, so ScopedKey preserved) + headers = {"Content-type": "application/json"} + data = dict(scope=scope_test.to_dict()) + jsondata = json.dumps(data, cls=JSON_HANDLER.encoder) + + response = test_client.post( + f"/networks/{source_sk}/copy", data=jsondata, headers=headers + ) + assert response.status_code == 200 + + copied_sk = ScopedKey(**response.json()) + assert copied_sk.scope == scope_test + assert copied_sk.gufe_key == source_sk.gufe_key + assert n4js.check_existence(copied_sk) + + def test_copy_network_bad_target_scope( + self, n4js_preloaded, test_client, scope_test, multiple_scopes + ): + """POST /networks/{sk}/copy with a destination scope the token + does not authorize must be denied (401).""" + # source in the authorized scope, destination in an unauthorized one + source_sks = n4js_preloaded.query_networks(scope=scope_test) + assert source_sks + source_sk = source_sks[0] + + bad_scope = multiple_scopes[1] + assert bad_scope != scope_test + + headers = {"Content-type": "application/json"} + data = dict(scope=bad_scope.to_dict()) + jsondata = json.dumps(data, cls=JSON_HANDLER.encoder) + + response = test_client.post( + f"/networks/{source_sk}/copy", data=jsondata, headers=headers + ) + assert response.status_code == 401 + details = response.json() + assert "detail" in details + assert str(bad_scope) in details["detail"] + + def test_copy_network_bad_source_scope( + self, n4js_preloaded, test_client, scope_test, multiple_scopes + ): + """POST /networks/{sk}/copy with a source network in a scope the + token does not authorize must be denied (401).""" + unauth_scope = multiple_scopes[1] + assert unauth_scope != scope_test + + unauth_source_sks = n4js_preloaded.query_networks(scope=unauth_scope) + assert unauth_source_sks + unauth_source_sk = unauth_source_sks[0] + + headers = {"Content-type": "application/json"} + data = dict(scope=scope_test.to_dict()) + jsondata = json.dumps(data, cls=JSON_HANDLER.encoder) + + response = test_client.post( + f"/networks/{unauth_source_sk}/copy", data=jsondata, headers=headers + ) + assert response.status_code == 401 + details = response.json() + assert "detail" in details + assert str(unauth_scope) in details["detail"] + + def test_copy_network_bad_qualname(self, n4js_preloaded, test_client, scope_test): + """POST /networks/{sk}/copy with a non-AlchemicalNetwork ScopedKey + must be rejected at the store layer (422).""" + tf_sks = n4js_preloaded.query_transformations(scope=scope_test) + assert tf_sks + + headers = {"Content-type": "application/json"} + data = dict(scope=scope_test.to_dict()) + jsondata = json.dumps(data, cls=JSON_HANDLER.encoder) + + response = test_client.post( + f"/networks/{tf_sks[0]}/copy", data=jsondata, headers=headers + ) + assert response.status_code == 422 + assert "AlchemicalNetwork" in response.json()["detail"] + def test_get_network(self, prepared_network, test_client): network, scoped_key = prepared_network response = test_client.get(f"/networks/{scoped_key}") diff --git a/alchemiscale/tests/integration/storage/test_statestore.py b/alchemiscale/tests/integration/storage/test_statestore.py index fa35617b..232a3ec7 100644 --- a/alchemiscale/tests/integration/storage/test_statestore.py +++ b/alchemiscale/tests/integration/storage/test_statestore.py @@ -138,6 +138,330 @@ def test_create_overlapping_networks(self, n4js, network_tyk2, scope_test): def test_delete_network(self): raise NotImplementedError + def test_merge_networks(self, n4js, network_tyk2, scope_test): + network_sks = [] + + scope_args = scope_test.to_dict() + all_transformations = list(network_tyk2.edges) + all_transformations.sort(key=lambda x: x.key) + + def project_scope(iteration): + return Scope(**(scope_args | {"project": f"project{iteration}"})) + + transformations_common = all_transformations[:3] + + # note the nonlocal task_sks and transformation_sks to reduce + # clutter down below + def set_result(_slice, *, ok: bool): + nonlocal task_sks, transformation_sks + for task_sk in task_sks[_slice]: + pdrr = ProtocolDAGResultRef( + obj_key=f"ProtocolDAGResult-{uuid.uuid4()}", + scope=task_sk.scope, + ok=ok, + ) + n4js.set_task_result(task_sk, pdrr) + + # NETWORK 1---JUST TYK2 + # 5 tasks: [completed, completed, running, error, waiting] + # no extends + an = network_tyk2.copy_with_replacements( + name=network_tyk2.name + f"_test_network_clone_1" + ) + scope = project_scope(1) + sk, th_sk, _ = n4js.assemble_network(an, scope) + network_sks.append(sk) + # add two more of the transformations + transformation_sks = [ + n4js.get_scoped_key(transformation, scope) + for transformation in transformations_common + all_transformations[3:5] + ] + task_sks = n4js.create_tasks(transformation_sks) + n4js.set_task_running(task_sks[2:3]) + n4js.action_tasks(task_sks[2:3], th_sk) + + n4js.set_task_waiting(task_sks[5:]) + n4js.action_tasks(task_sks[5:], th_sk) + + n4js.set_task_running(task_sks[4:5]) + n4js.set_task_error(task_sks[4:5]) + n4js.action_tasks(task_sks[4:5], th_sk) + + n4js.set_task_running(task_sks[:2]) + n4js.set_task_complete(task_sks[:2]) + + set_result(slice(None, 2), ok=True) + set_result(slice(4, 5), ok=False) + + # NETWORK 2---TYK2 WITH TRIMMED EDGES + # 6 tasks: [complete, complete, complete] + # ^ ^ ^ + # | | | + # extends: [waiting, running, complete] + an = network_tyk2.copy_with_replacements( + name=network_tyk2.name + f"_test_network_clone_2_fewer_transformations", + edges=all_transformations[:-2], + ) + scope = project_scope(2) + sk, _, _ = n4js.assemble_network(an, scope) + network_sks.append(sk) + transformation_sks = [ + n4js.get_scoped_key(transformation, scope) + for transformation in transformations_common + ] + task_sks = n4js.create_tasks(transformation_sks) + n4js.set_task_running(task_sks) + n4js.set_task_complete(task_sks) + set_result(slice(None), ok=True) + + # create extending tasks + task_sks = n4js.create_tasks(transformation_sks, extends=task_sks) + + n4js.set_task_waiting(task_sks[0:1]) + n4js.set_task_running(task_sks[1:2]) + n4js.set_task_running(task_sks[2:3]) + n4js.set_task_complete(task_sks[2:3]) + + set_result(slice(2, 3), ok=True) + + # network 3---tyk2 + # 6 tasks: [waiting, waiting, waiting, waiting, waiting, complete] + # no extends + # last task is for a transformation missing from NETWORK 2 + an = network_tyk2.copy_with_replacements( + name=network_tyk2.name + f"_test_network_clone_2_name_only" + ) + scope = project_scope(3) + sk, _, _ = n4js.assemble_network(an, scope) + network_sks.append(sk) + + transformation_sks = [ + n4js.get_scoped_key(transformation, scope) + for transformation in transformations_common + all_transformations[-2:] + ] + + task_sks = n4js.create_tasks(transformation_sks) + n4js.set_task_waiting(task_sks[:-1]) + n4js.set_task_running(task_sks[-1:]) + n4js.set_task_complete(task_sks[-1:]) + set_result(slice(-1, None), ok=True) + + scope_dict = scope_test.to_dict() + scope_dict["project"] = "mergedproject" + new_scope = Scope(**scope_dict) + sk_merged = n4js.merge_networks( + network_sks, f"{network_tyk2.name}_combined", new_scope + ) + assert len(n4js.get_gufe(sk_merged).edges) == len( + n4js.get_gufe(network_sks[0]).edges + ) + assert len(n4js.get_gufe(sk_merged).edges) != len( + n4js.get_gufe(network_sks[1]).edges + ) + + # we expect 7 pdrrs from the completed tasks + results = n4js.execute_query( + """ + MATCH (pdrr: ProtocolDAGResultRef {`_project`: $project}) + WHERE pdrr.ok = True + RETURN pdrr + """, + project=new_scope.project, + ) + assert len(results.records) == 7 + + # errored Task is not cloned (complete-only policy), so its not-ok + # PDRR must not appear in the merged scope + results = n4js.execute_query( + """ + MATCH (pdrr: ProtocolDAGResultRef {`_project`: $project}) + WHERE pdrr.ok = False + RETURN pdrr + """, + project=new_scope.project, + ) + assert len(results.records) == 0 + + def test_copy_network_partial_preload_target(self, n4js, network_tyk2, scope_test): + """copy_network must correctly place cloned Tasks in a target + scope that has only a *partial* overlap with the source's + Transformations. + + Transformations already present in the target scope must + dedup onto the preexisting nodes; Transformations missing from + the target must be created fresh. Either way, cloned Tasks + must land wired to the correct target-scope Transformation via + PERFORMS, EXTENDS chains must be preserved, and + ProtocolDAGResultRefs must land in the target scope. + + This case is not reachable through the interface tests, which + always start from ``n4js_preloaded`` -- a fixture that seeds + the same networks (and therefore the same Transformations) + into every scope, so every copy is either pure dedup or pure + fresh-write, never a mix. + """ + all_edges = sorted(network_tyk2.edges, key=lambda e: e.key) + overlap_edges = all_edges[:2] + + # source: full network + source_scope = Scope(**(scope_test.to_dict() | {"project": "source"})) + source_sk, _, _ = n4js.assemble_network(network_tyk2, source_scope) + + # target: preload an AN carrying only the overlap subset of + # Transformations. This produces the partial-preload shape. + target_scope = Scope(**(scope_test.to_dict() | {"project": "target"})) + partial_an = AlchemicalNetwork(edges=overlap_edges, name="target_preexisting") + n4js.assemble_network(partial_an, target_scope) + + def target_tf_count(): + return n4js.execute_query( + """ + MATCH (tf) + WHERE (tf:Transformation OR tf:NonTransformation) + AND tf._project = $project + RETURN count(tf) AS n + """, + project=target_scope.project, + ).records[0]["n"] + + # baseline: target scope holds exactly the overlap Transformations + assert target_tf_count() == len(overlap_edges) + + # hang Tasks off one overlap Transformation and one non-overlap + # Transformation so both sides of the boundary get exercised. + # Under the complete-only policy, only complete Tasks carry over; + # a waiting Task on the overlap side lets us also verify that + # non-complete Tasks are dropped, not leaked into the target. + overlap_tf_sk = n4js.get_scoped_key(overlap_edges[0], source_scope) + fresh_tf_sk = n4js.get_scoped_key(all_edges[-1], source_scope) + + # overlap side: 2 tasks + # [0]: complete + ok PDRR -- carried + # [1]: waiting -- dropped + overlap_tasks = n4js.create_tasks([overlap_tf_sk, overlap_tf_sk]) + n4js.set_task_running(overlap_tasks[:1]) + n4js.set_task_complete(overlap_tasks[:1]) + overlap_pdrr = ProtocolDAGResultRef( + obj_key=f"ProtocolDAGResult-{uuid.uuid4()}", + scope=overlap_tasks[0].scope, + ok=True, + ) + n4js.set_task_result(overlap_tasks[0], overlap_pdrr) + + # fresh side: 1 complete base + 1 complete EXTENDS on top; both + # carried, so we can verify EXTENDS is preserved across the copy + fresh_base = n4js.create_tasks([fresh_tf_sk]) + n4js.set_task_running(fresh_base) + n4js.set_task_complete(fresh_base) + fresh_pdrr = ProtocolDAGResultRef( + obj_key=f"ProtocolDAGResult-{uuid.uuid4()}", + scope=fresh_base[0].scope, + ok=True, + ) + n4js.set_task_result(fresh_base[0], fresh_pdrr) + + fresh_ext = n4js.create_tasks([fresh_tf_sk], extends=fresh_base) + n4js.set_task_running(fresh_ext) + n4js.set_task_complete(fresh_ext) + fresh_ext_pdrr = ProtocolDAGResultRef( + obj_key=f"ProtocolDAGResult-{uuid.uuid4()}", + scope=fresh_ext[0].scope, + ok=True, + ) + n4js.set_task_result(fresh_ext[0], fresh_ext_pdrr) + + # --- copy --- + copied_sk = n4js.copy_network(source_sk, target_scope) + + def target_key(source_task_sk): + return ScopedKey( + gufe_key=source_task_sk.gufe_key, + org=target_scope.org, + campaign=target_scope.campaign, + project=target_scope.project, + ) + + # 1) exactly one AN node in target with the copied ScopedKey + an_count = n4js.execute_query( + "MATCH (an:AlchemicalNetwork {_scoped_key: $sk}) RETURN count(an) AS n", + sk=str(copied_sk), + ).records[0]["n"] + assert an_count == 1 + + # 2) Transformation node count in target rises to the full + # source count -- overlap subset stays deduplicated, non-overlap + # subset is freshly created + assert target_tf_count() == len(all_edges) + + # 3) every source Transformation appears exactly once in target + # (proves no duplication of overlap nodes, no missed fresh nodes) + for edge in all_edges: + cnt = n4js.execute_query( + """ + MATCH (tf) + WHERE (tf:Transformation OR tf:NonTransformation) + AND tf._project = $project + AND tf._gufe_key = $gufe_key + RETURN count(tf) AS n + """, + project=target_scope.project, + gufe_key=str(edge.key), + ).records[0]["n"] + assert cnt == 1 + + # 4) every carried (complete) source Task lands in target wired via + # PERFORMS to a target-scope Transformation + carried_tasks = [overlap_tasks[0], fresh_base[0], fresh_ext[0]] + for source_task_sk in carried_tasks: + target_sk_ = target_key(source_task_sk) + perf = n4js.execute_query( + """ + MATCH (t:Task {_scoped_key: $sk})-[:PERFORMS]->(tf) + WHERE (tf:Transformation OR tf:NonTransformation) + AND tf._project = $project + RETURN count(tf) AS n + """, + sk=str(target_sk_), + project=target_scope.project, + ).records[0]["n"] + assert perf == 1 + + # 5) the waiting overlap Task must NOT appear in the target scope + # under the complete-only policy + dropped_target_sk = target_key(overlap_tasks[1]) + dropped_count = n4js.execute_query( + "MATCH (t:Task {_scoped_key: $sk}) RETURN count(t) AS n", + sk=str(dropped_target_sk), + ).records[0]["n"] + assert dropped_count == 0 + + # 6) EXTENDS preserved for the fresh_ext -> fresh_base pair + ext_edge = n4js.execute_query( + """ + MATCH (a:Task {_scoped_key: $a})-[:EXTENDS]->(b:Task {_scoped_key: $b}) + RETURN count(*) AS n + """, + a=str(target_key(fresh_ext[0])), + b=str(target_key(fresh_base[0])), + ).records[0]["n"] + assert ext_edge == 1 + + # 7) exactly the three ok PDRRs land in target scope with obj_keys + # intact -- one per carried Task + pdrr_records = n4js.execute_query( + """ + MATCH (pdrr:ProtocolDAGResultRef {_project: $project}) + RETURN pdrr.ok AS ok, pdrr.obj_key AS obj_key + """, + project=target_scope.project, + ).records + assert len(pdrr_records) == 3 + assert {r["obj_key"] for r in pdrr_records} == { + overlap_pdrr.obj_key, + fresh_pdrr.obj_key, + fresh_ext_pdrr.obj_key, + } + def test_set_network_state(self, n4js, network_tyk2, scope_test): valid_states = [state.value for state in NetworkStateEnum] network_sks = [] diff --git a/docs/user_guide/index.rst b/docs/user_guide/index.rst index 2125c0ad..8c707022 100644 --- a/docs/user_guide/index.rst +++ b/docs/user_guide/index.rst @@ -11,5 +11,6 @@ If you are making use of an **alchemiscale** instance, this document will famili :maxdepth: 1 ./getting_started + ./merging_and_copying_networks ./handling_errors ./strategy_automation diff --git a/docs/user_guide/merging_and_copying_networks.rst b/docs/user_guide/merging_and_copying_networks.rst new file mode 100644 index 00000000..711cd811 --- /dev/null +++ b/docs/user_guide/merging_and_copying_networks.rst @@ -0,0 +1,90 @@ +.. _merging-and-copying-networks: + +###################################### +Merging and copying AlchemicalNetworks +###################################### + +Once you have :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s submitted with :py:class:`~alchemiscale.storage.models.Task`\s and results attached, you may want to combine several of them into a single :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`, or move an :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` (or a whole :py:class:`~alchemiscale.models.Scope` of them) into a different :py:class:`~alchemiscale.models.Scope`. +Three :py:class:`~alchemiscale.interface.client.AlchemiscaleClient` methods support these workflows: + +* :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_networks` — combine several :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s into a single new one. +* :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.copy_network` — copy one :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` into a different :py:class:`~alchemiscale.models.Scope`. +* :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_scopes` — copy every :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` from one or more source :py:class:`~alchemiscale.models.Scope`\s into a single target :py:class:`~alchemiscale.models.Scope`. + +The destination :py:class:`~alchemiscale.models.Scope` for each of these must be a *specific* :py:class:`~alchemiscale.models.Scope` (no wildcards), and your user must have permissions on the source and destination :py:class:`~alchemiscale.models.Scope`\s. + +All three methods share the same Task-retention policy: **only** :py:class:`~alchemiscale.storage.models.Task`\s in ``complete`` status carry over to the new :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`, together with their :py:class:`~alchemiscale.storage.models.ProtocolDAGResultRef`\s. +:py:class:`~alchemiscale.storage.models.Task`\s in any other status (``waiting``, ``running``, ``error``, ``invalid``, ``deleted``) are not carried over. +This keeps the semantics simple: **the results of computed work are preserved; nothing else is**. + +.. note:: + + Any ``Strategy`` (via :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.set_network_strategy`) or ``TaskRestartPattern``\s (via :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.add_task_restart_patterns`) on the source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s are not copied. + Set these on the new :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` yourself after the merge or copy if you need them. + + +*********************************** +Merging multiple AlchemicalNetworks +*********************************** + +Use :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_networks` to combine multiple :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s into a single new :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` in the target :py:class:`~alchemiscale.models.Scope`:: + + >>> merged_sk = asc.merge_networks( + ... networks=[an_sk_a, an_sk_b, an_sk_c], + ... name='combined_campaign', + ... scope=Scope('my_org', 'my_campaign', 'combined_project'), + ... ) + >>> merged_sk + + +The new :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` contains the union of the :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s and :external+gufe:py:class:`~gufe.transformations.transformation.NonTransformation`\s from every source network. +The source networks themselves are unchanged. + +The ``complete`` :py:class:`~alchemiscale.storage.models.Task`\s attached to those source :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s are cloned onto the merged network's :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s along with their :py:class:`~alchemiscale.storage.models.ProtocolDAGResultRef`\s, so previously-computed results do not need to be re-run. + +If you want to add fresh :py:class:`~alchemiscale.storage.models.Task`\s to the merged network -- either to retry a :external+gufe:py:class:`~gufe.transformations.transformation.Transformation` that previously errored, or to gather more repeats -- create new ones with :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.create_tasks` on the relevant :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s and then :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.action_tasks` them on the merged network. + + +********************************** +Copying a single AlchemicalNetwork +********************************** + +Use :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.copy_network` to copy one :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` into a different :py:class:`~alchemiscale.models.Scope`:: + + >>> copy_sk = asc.copy_network( + ... network=an_sk, + ... scope=Scope('my_org', 'my_campaign', 'shared_project'), + ... ) + +Like :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_networks`, only the source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\'s ``complete`` :py:class:`~alchemiscale.storage.models.Task`\s are carried over, together with their :py:class:`~alchemiscale.storage.models.ProtocolDAGResultRef`\s. + +If ``name`` is not given, the source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\'s name is preserved. +Pass ``name`` to rename the copy:: + + >>> renamed_copy_sk = asc.copy_network( + ... network=an_sk, + ... scope=Scope('my_org', 'my_campaign', 'shared_project'), + ... name='shared_snapshot', + ... ) + + +********************************************************* +Copying every AlchemicalNetwork from one Scope to another +********************************************************* + +Use :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_scopes` to copy every :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` in one or more source :py:class:`~alchemiscale.models.Scope`\s into a single target :py:class:`~alchemiscale.models.Scope`:: + + >>> new_sks = asc.merge_scopes( + ... scopes=[Scope('my_org', 'my_campaign', 'project_a'), + ... Scope('my_org', 'my_campaign', 'project_b')], + ... target_scope=Scope('my_org', 'my_campaign', 'consolidated'), + ... ) + >>> len(new_sks) + 17 + +This is a convenience method: each :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` is copied via :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.copy_network` in turn, with names *and* source states preserved. +Networks in ``inactive``, ``invalid``, or ``deleted`` state on a source :py:class:`~alchemiscale.models.Scope` land in the target :py:class:`~alchemiscale.models.Scope` in that same state, rather than being silently reactivated. +It is useful for consolidating :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s from multiple :py:class:`~alchemiscale.models.Scope`\s into a shared workspace, or for relocating a whole :py:class:`~alchemiscale.models.Scope`\'s worth of :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s. + +Note that :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_scopes` is not transactional: if a per-network copy fails partway through, the copies that already succeeded remain in the target :py:class:`~alchemiscale.models.Scope`. +Because :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.copy_network` is idempotent -- re-copying an :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` that already exists in the target :py:class:`~alchemiscale.models.Scope` dedups onto the existing node -- rerunning after a failure is safe. diff --git a/news/issue-221.rst b/news/issue-221.rst new file mode 100644 index 00000000..00d31811 --- /dev/null +++ b/news/issue-221.rst @@ -0,0 +1,28 @@ +**Added:** + +* ``AlchemiscaleClient.merge_networks`` for combining multiple existing ``AlchemicalNetwork``\s into a new ``AlchemicalNetwork`` in a target ``Scope``. Backed by ``Neo4jStore.merge_networks`` and a new ``POST /networks/merge`` endpoint on the user API. +* ``AlchemiscaleClient.copy_network`` for copying a single ``AlchemicalNetwork`` to a new ``Scope``. An optional ``name`` argument creates the copy under a new name (and therefore a new gufe key); when ``name`` is omitted, the source network's name and gufe key are preserved. Backed by ``Neo4jStore.copy_network`` and a new ``POST /networks/{network_scoped_key}/copy`` endpoint. +* ``AlchemiscaleClient.merge_scopes`` for copying every ``AlchemicalNetwork`` in a set of source ``Scope``\s into a single target ``Scope``, via per-network ``copy_network`` calls. Each copy preserves its source network's *state* (``active``, ``inactive``, ``invalid``, or ``deleted``), so this method is a faithful move rather than a silent reactivation of soft-deleted or invalidated networks. It is not transactional, but is safe to rerun after a partial failure since ``copy_network`` is idempotent. +* For all three methods, only ``Task``\s in ``complete`` state on the source networks are cloned into the new network -- along with their ``ProtocolDAGResultRef``\s -- so previously-computed results do not need to be re-run. ``Task``\s in any other status are not carried over. Cloned ``Task``\s are wired to their ``Transformation``\s via ``PERFORMS`` so the standard ``(:AlchemicalNetwork)-[:DEPENDS_ON]->(:Transformation)<-[:PERFORMS]-(:Task)`` traversal sees them on the new network. +* Any ``Strategy`` (``AlchemiscaleClient.set_network_strategy``) or ``TaskRestartPattern``\s (``AlchemiscaleClient.add_task_restart_patterns``) on the source networks are not carried over by any of the three new methods. Set them explicitly on the new network afterward if needed. + +**Changed:** + +* + +**Deprecated:** + +* + +**Removed:** + +* + +**Fixed:** + +* + +**Security:** + +* +