From 1484de46dd5e236602c1734e524c5a724b4c6b43 Mon Sep 17 00:00:00 2001 From: Ian Kenney Date: Tue, 16 Sep 2025 11:57:22 -0400 Subject: [PATCH 01/35] Add unimplemented merge_networks --- alchemiscale/storage/statestore.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index de397160..b5f089df 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -861,6 +861,9 @@ def delete_network( """ raise NotImplementedError + def merge_networks(self, networks: list[ScopedKey], scope: Scope) -> ScopedKey: + raise NotImplementedError + def get_network_state(self, networks: list[ScopedKey]) -> list[str | None]: """Get the states of a group of networks. From 40cb0ab6c35b54b0390b2bdebda6e7108e0beb47 Mon Sep 17 00:00:00 2001 From: Ian Kenney Date: Sat, 27 Sep 2025 10:34:07 -0400 Subject: [PATCH 02/35] Make assemble_network chainable --- alchemiscale/storage/statestore.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index b5f089df..b09e5b3c 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -766,11 +766,13 @@ def _topological_sort(graph_data: dict[GufeKey, tuple[Node, list[GufeKey]]]): graph_data.pop(rk) return L + @chainable def assemble_network( self, network: AlchemicalNetwork, scope: Scope, state: NetworkStateEnum | str = NetworkStateEnum.active, + tx=None, ) -> tuple[ScopedKey, ScopedKey, ScopedKey]: """Create all nodes and relationships needed for an AlchemicalNetwork represented in an alchemiscale state store. @@ -796,8 +798,7 @@ def assemble_network( subgraph = nw_subgraph | th_subgraph | nm_subgraph - with self.transaction() as tx: - merge_subgraph(tx, subgraph, "GufeTokenizable", "_scoped_key") + merge_subgraph(tx, subgraph, "GufeTokenizable", "_scoped_key") return nw_sk, th_sk, nm_sk From 30e7dd3682197b3a62167b513bace0c40e2e3488 Mon Sep 17 00:00:00 2001 From: Ian Kenney Date: Sat, 27 Sep 2025 11:57:18 -0400 Subject: [PATCH 03/35] Partial implementation of merge_networks Networks are merged into a new network under a new scope with a given name. Task and ProtocolDAGResultRef support is not implemented here. --- alchemiscale/storage/statestore.py | 115 ++++++++++++++++- .../integration/storage/test_statestore.py | 116 ++++++++++++++++++ 2 files changed, 229 insertions(+), 2 deletions(-) diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index b09e5b3c..6d8bd95e 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -862,8 +862,119 @@ def delete_network( """ raise NotImplementedError - def merge_networks(self, networks: list[ScopedKey], scope: Scope) -> ScopedKey: - raise NotImplementedError + @chainable + def merge_networks( + self, + network_scoped_keys: list[ScopedKey], + name: str, + scope: Scope, + clone_incomplete_tasks=False, + tx=None, + ) -> ScopedKey: + """Merge multiple ``AlchemicalNetwork`` nodes into a new ``AlchemicalNetwork``. + + Parameters + ---------- + network_scoped_keys + List of ``AlchemicalNetwork`` ``ScopedKey`` objects to merge. + name + The name of the new ``AlchemicalNetwork``. + scope + The ``Scope`` of the new ``AlchemicalNetwork``. + + Returns + ------- + The ``ScopedKey`` of the new ``AlchemicalNetwork`` in the database. + """ + + # - Collect keyed chain representation for all alchemiscale networks + try: + network_keyed_chains: list[tuple[Scope, KeyedChain]] = [] + for network_scoped_key in network_scoped_keys: + keyed_chain = self.get_keyed_chain(network_scoped_key) + network_keyed_chains.append((network_scoped_key.scope, keyed_chain)) + # could not find specified network by provided scoped key + except KeyError: + raise ValueError( + f"ScopedKey ({network_scoped_key}) not found in the database." + ) + + from dataclasses import dataclass, field + + @dataclass + class TransformationData: + transformation: Transformation + task_tree: list = field( + default_factory=list + ) # flat represenation of task tree with results + + def __eq__(self, other): + if isinstance(other, (Transformation, NonTransformation)): + return other is self.transformation + return other.transformation is self.transformation + + def add_known_scoped_key_results(self, key, scope): + known_scoped_key = ScopedKey(gufe_key=GufeKey(key), **scope.to_dict()) + self.update_task_tree(known_scoped_key) + + def update_task_tree(self, tf_scoped_key): + nonlocal tx + query = """ + MATCH (task:Task)-[:PERFORMS]->(:Transformation {`_scoped_key`: $tf_scoped_key}) + OPTIONAL MATCH (task)-[:EXTENDS]->(extended_task:Task) + OPTIONAL MATCH (task)<-[:RESULTS_IN]-(pdrr:ProtocolDAGResultRef) + RETURN task, extended_task._scoped_key as etask_sk, pdrr + """ + results = ( + tx.run(query, tf_scoped_key=str(tf_scoped_key)) + .to_eager_result() + .records + ) + + self.task_tree.extend(results) + + from gufe.tokenization import key_decode_dependencies + + # TODO: upstream to gufe + def kc_to_gufe(kc, gts): + for gufe_key, keyed_dict in kc: + if gt := gts.get(gufe_key): + continue + gt = key_decode_dependencies(keyed_dict, registry=gts) + gts[gufe_key] = gt + return gt + + # Map Transformation and NonTransformation objects to their + # potentially duplicated database gufe keys. This may happen + # due to minor version changes between serialization + # times. These keys are then used to get all results for Tasks + # associated with them. + transformation_data: list[TransformationData] = [] + for network_scope, network_keyed_chain in network_keyed_chains: + gts = {} + for index, (database_key, _) in enumerate(network_keyed_chain): + # only process Transformation or NonTransformation keys + if ( + "Transformation" in database_key + or "NonTransformation" in database_key + ): + # get the tokenizable at the index along with all previous data + subchain = KeyedChain(network_keyed_chain[: index + 1]) + transformation = kc_to_gufe(subchain, gts) + try: + index = transformation_data.index(transformation) + data = transformation_data[index] + except ValueError: + data = TransformationData(transformation) + transformation_data.append(data) + data.add_known_scoped_key_results(database_key, network_scope) + # - Collect all transformation gufe objects and collect into a new set of edges + new_edges = [td.transformation for td in transformation_data] + # - Make new alchemiscale network with these edges + combined_alchemical_network = AlchemicalNetwork(edges=new_edges, name=name) + # - assemble the network + an_sk, _, _ = self.assemble_network(combined_alchemical_network, scope, tx=tx) + return an_sk def get_network_state(self, networks: list[ScopedKey]) -> list[str | None]: """Get the states of a group of networks. diff --git a/alchemiscale/tests/integration/storage/test_statestore.py b/alchemiscale/tests/integration/storage/test_statestore.py index d15e168d..c42fa5ca 100644 --- a/alchemiscale/tests/integration/storage/test_statestore.py +++ b/alchemiscale/tests/integration/storage/test_statestore.py @@ -138,6 +138,122 @@ def test_create_overlapping_networks(self, n4js, network_tyk2, scope_test): def test_delete_network(self): raise NotImplementedError + def test_merge_networks(self, n4js, network_tyk2, scope_test): + network_sks = [] + + scope_args = scope_test.to_dict() + all_transformations = list(network_tyk2.edges) + all_transformations.sort(key=lambda x: x.key) + + def project_scope(iteration): + return Scope(**(scope_args | {"project": f"project{iteration}"})) + + transformations_common = all_transformations[:3] + + # note the nonlocal task_sks and transformation_sks to reduce + # clutter down below + def set_result(_slice, *, ok: bool): + nonlocal task_sks, transformation_sks + for task_sk, transformation_sk in zip( + task_sks[_slice], transformation_sks[_slice] + ): + pdrr = ProtocolDAGResultRef( + obj_key=transformation_sk.gufe_key, scope=task_sk.scope, ok=ok + ) + n4js.set_task_result(task_sk, pdrr) + + # NETWORK 1---JUST TYK2 + # 5 tasks: [completed, completed, running, error, waiting] + # no extends + an = network_tyk2.copy_with_replacements( + name=network_tyk2.name + f"_test_network_clone_1" + ) + scope = project_scope(1) + sk, th_sk, _ = n4js.assemble_network(an, scope) + network_sks.append(sk) + # add two more of the transformations + transformation_sks = [ + n4js.get_scoped_key(transformation, scope) + for transformation in transformations_common + all_transformations[3:5] + ] + task_sks = n4js.create_tasks(transformation_sks) + n4js.set_task_running(task_sks[2:3]) + n4js.action_tasks(task_sks[2:3], th_sk) + + n4js.set_task_waiting(task_sks[5:]) + n4js.action_tasks(task_sks[5:], th_sk) + + n4js.set_task_error(task_sks[4:5]) + n4js.action_tasks(task_sks[4:5], th_sk) + + n4js.set_task_complete(task_sks[:2]) + + set_result(slice(None, 2), ok=True) + set_result(slice(4, 5), ok=False) + + # NETWORK 2---TYK2 WITH TRIMMED EDGES + # 6 tasks: [complete, complete, complete] + # ^ ^ ^ + # | | | + # extends: [waiting, running, complete] + an = network_tyk2.copy_with_replacements( + name=network_tyk2.name + f"_test_network_clone_2_fewer_transformations", + edges=all_transformations[:-2], + ) + scope = project_scope(2) + sk, _, _ = n4js.assemble_network(an, scope) + network_sks.append(sk) + transformation_sks = [ + n4js.get_scoped_key(transformation, scope) + for transformation in transformations_common + ] + task_sks = n4js.create_tasks(transformation_sks) + n4js.set_task_complete(task_sks) + set_result(slice(None), ok=True) + + # create extending tasks + task_sks = n4js.create_tasks(transformation_sks, extends=task_sks) + + n4js.set_task_waiting(task_sks[0:1]) + n4js.set_task_running(task_sks[1:2]) + n4js.set_task_complete(task_sks[2:3]) + + set_result(slice(2, 3), ok=True) + + # network 3---tyk2 + # 6 tasks: [waiting, waiting, waiting, waiting, waiting, complete] + # no extends + # last task is for a transformation missing from NETWORK 2 + an = network_tyk2.copy_with_replacements( + name=network_tyk2.name + f"_test_network_clone_2_name_only" + ) + scope = project_scope(3) + sk, _, _ = n4js.assemble_network(an, scope) + network_sks.append(sk) + + transformation_sks = [ + n4js.get_scoped_key(transformation, scope) + for transformation in transformations_common + all_transformations[-2:] + ] + + task_sks = n4js.create_tasks(transformation_sks) + n4js.set_task_waiting(task_sks[:-1]) + n4js.set_task_complete(task_sks[-1:]) + set_result(slice(-1, None), ok=True) + + scope_dict = scope_test.to_dict() + scope_dict["project"] = "mergedproject" + new_scope = Scope(**scope_dict) + sk_merged = n4js.merge_networks( + network_sks, f"{network_tyk2.name}_combined", new_scope + ) + assert len(n4js.get_gufe(sk_merged).edges) == len( + n4js.get_gufe(network_sks[0]).edges + ) + assert len(n4js.get_gufe(sk_merged).edges) != len( + n4js.get_gufe(network_sks[1]).edges + ) + def test_set_network_state(self, n4js, network_tyk2, scope_test): valid_states = [state.value for state in NetworkStateEnum] network_sks = [] From 865a46f2483f78b539b87942b4181af1dcbfa91f Mon Sep 17 00:00:00 2001 From: Ian Kenney Date: Mon, 29 Sep 2025 15:23:18 -0400 Subject: [PATCH 04/35] Make _validate_extends_tasks and create_tasks chainable --- alchemiscale/storage/statestore.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index 5468e7b1..7dc9b48d 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -2895,7 +2895,10 @@ def task_count(task_dict: dict): ## tasks - def _validate_extends_tasks(self, task_list) -> dict[str, tuple[Node, str]]: + @chainable + def _validate_extends_tasks( + self, task_list, tx=None + ) -> dict[str, tuple[Node, str]]: if not task_list: return {} @@ -2906,7 +2909,7 @@ def _validate_extends_tasks(self, task_list) -> dict[str, tuple[Node, str]]: return t, tf._scoped_key as tf_sk """ - results = self.execute_query(q, task_list=list(map(str, task_list))) + results = tx.run(q, task_list=list(map(str, task_list))).to_eager_result() nodes = {} @@ -2926,11 +2929,13 @@ def _validate_extends_tasks(self, task_list) -> dict[str, tuple[Node, str]]: return nodes + @chainable def create_tasks( self, transformations: list[ScopedKey], extends: list[ScopedKey | None] | None = None, creator: str | None = None, + tx=None, ) -> list[ScopedKey]: """Create Tasks for the given Transformations. @@ -2985,7 +2990,8 @@ def create_tasks( transformation_map[transformation.qualname][1].append(extends[i]) extends_nodes = self._validate_extends_tasks( - [_extends for _extends in extends if _extends is not None] + [_extends for _extends in extends if _extends is not None], + tx=tx, ) subgraph = Subgraph() @@ -3006,9 +3012,9 @@ def create_tasks( RETURN n """ - results = self.execute_query( + results = tx.run( q, transformation_subset=list(map(str, transformation_subset)) - ) + ).to_eager_result() transformation_nodes = {} for record in results.records: @@ -3057,8 +3063,7 @@ def create_tasks( _project=scope.project, ) - with self.transaction() as tx: - merge_subgraph(tx, subgraph, "GufeTokenizable", "_scoped_key") + merge_subgraph(tx, subgraph, "GufeTokenizable", "_scoped_key") return sks From fb28a599f4f21ae86c36f9e2b4cdea011ae1094e Mon Sep 17 00:00:00 2001 From: Ian Kenney Date: Tue, 7 Oct 2025 08:22:49 -0400 Subject: [PATCH 05/35] Use Subgraphs for task (+PDRR) and network assembly --- alchemiscale/storage/statestore.py | 55 ++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index 7dc9b48d..a8b09af8 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -901,7 +901,6 @@ def merge_networks( ------- The ``ScopedKey`` of the new ``AlchemicalNetwork`` in the database. """ - # - Collect keyed chain representation for all alchemiscale networks try: network_keyed_chains: list[tuple[Scope, KeyedChain]] = [] @@ -928,17 +927,17 @@ def __eq__(self, other): return other is self.transformation return other.transformation is self.transformation - def add_known_scoped_key_results(self, key, scope): + def add_known_scoped_key_results(self, key, scope, tx): known_scoped_key = ScopedKey(gufe_key=GufeKey(key), **scope.to_dict()) - self.update_task_tree(known_scoped_key) + self.update_task_tree(known_scoped_key, tx) - def update_task_tree(self, tf_scoped_key): - nonlocal tx + def update_task_tree(self, tf_scoped_key, tx): + # TODO: filter out tasks that are not wanted before returning them query = """ - MATCH (task:Task)-[:PERFORMS]->(:Transformation {`_scoped_key`: $tf_scoped_key}) + MATCH (task:Task)-[:PERFORMS]->(:Transformation|NonTransformation {`_scoped_key`: $tf_scoped_key}) OPTIONAL MATCH (task)-[:EXTENDS]->(extended_task:Task) - OPTIONAL MATCH (task)<-[:RESULTS_IN]-(pdrr:ProtocolDAGResultRef) - RETURN task, extended_task._scoped_key as etask_sk, pdrr + OPTIONAL MATCH (task)-[:RESULTS_IN]->(pdrr:ProtocolDAGResultRef) + RETURN task, extended_task as extended_task, collect(pdrr) as pdrrs """ results = ( tx.run(query, tf_scoped_key=str(tf_scoped_key)) @@ -948,6 +947,37 @@ def update_task_tree(self, tf_scoped_key): self.task_tree.extend(results) + def to_subgraph(self, target_scope, statestore): + if not self.task_tree: + return Subgraph() + + _, tf_node, _ = statestore._keyed_chain_to_subgraph(KeyedChain.from_gufe(self.transformation), target_scope) + subgraph = Subgraph() | tf_node + + scope_props = { + "_org": target_scope.org, + "_campaign": target_scope.campaign, + "_project": target_scope.project, + } + + def record_to_node(record): + scoped_key = record["_scoped_key"] + scoped_key = ScopedKey(gufe_key=record["_gufe_key"], **scope.to_dict()) + return Node(*record.labels, **record._properties | scope_props | {"_scoped_key": str(scoped_key)}) + + for record in self.task_tree: + etask_record = record["extended_task"] + pdrr_records = record["pdrrs"] + task_node = record_to_node(record["task"]) + etask_node = None if not record["extended_task"] else record_to_node(record["extended_task"]) + if etask_node: + subgraph |= Relationship.type("EXTENDS")(etask_node, task_node, **scope_props) + for pdrr_record in record["pdrrs"]: + pdrr_node = record_to_node(pdrr_record) + subgraph |= Relationship.type("RESULTS_IN")(task_node, pdrr_node, **scope_props) + + return subgraph + from gufe.tokenization import key_decode_dependencies # TODO: upstream to gufe @@ -982,13 +1012,16 @@ def kc_to_gufe(kc, gts): except ValueError: data = TransformationData(transformation) transformation_data.append(data) - data.add_known_scoped_key_results(database_key, network_scope) + data.add_known_scoped_key_results(database_key, network_scope, tx) # - Collect all transformation gufe objects and collect into a new set of edges new_edges = [td.transformation for td in transformation_data] # - Make new alchemiscale network with these edges combined_alchemical_network = AlchemicalNetwork(edges=new_edges, name=name) - # - assemble the network - an_sk, _, _ = self.assemble_network(combined_alchemical_network, scope, tx=tx) + an_subgraph, an_node, an_sk = self._keyed_chain_to_subgraph(KeyedChain.from_gufe(combined_alchemical_network), scope) + an_subgraph |= self.create_network_mark_subgraph(an_node)[0] | self.create_taskhub_subgraph(an_node)[0] + for td in transformation_data: + an_subgraph |= td.to_subgraph(scope, self) + merge_subgraph(tx, an_subgraph, "GufeTokenizable", "_scoped_key") return an_sk def get_network_state(self, networks: list[ScopedKey]) -> list[str | None]: From 50322959e1b7c7374f2cbbb6e9df79de5f017346 Mon Sep 17 00:00:00 2001 From: Ian Kenney Date: Tue, 7 Oct 2025 10:07:13 -0400 Subject: [PATCH 06/35] Update task trees with one query and cache subchains for performance increase --- alchemiscale/storage/statestore.py | 41 ++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index a8b09af8..1e8163f6 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -921,37 +921,44 @@ class TransformationData: task_tree: list = field( default_factory=list ) # flat represenation of task tree with results + known_scoped_keys: list = field( + default_factory=list + ) def __eq__(self, other): if isinstance(other, (Transformation, NonTransformation)): return other is self.transformation return other.transformation is self.transformation - def add_known_scoped_key_results(self, key, scope, tx): - known_scoped_key = ScopedKey(gufe_key=GufeKey(key), **scope.to_dict()) - self.update_task_tree(known_scoped_key, tx) + def add_known_scoped_key(self, key, scope): + self.known_scoped_keys.append(ScopedKey(gufe_key=GufeKey(key), **scope.to_dict())) - def update_task_tree(self, tf_scoped_key, tx): + @staticmethod + def update_task_trees(transformation_data: list, tx): + key_to_data_map = {str(td.transformation.key): td for td in transformation_data} + transformation_sk_pairs = [[str(td.transformation.key), str(sk)] for td in transformation_data for sk in td.known_scoped_keys] # TODO: filter out tasks that are not wanted before returning them query = """ - MATCH (task:Task)-[:PERFORMS]->(:Transformation|NonTransformation {`_scoped_key`: $tf_scoped_key}) + UNWIND $tf_sk_pairs as pairs + WITH pairs[0] AS tf_key, pairs[1] AS tf_scoped_key + MATCH (task:Task)-[:PERFORMS]->(:Transformation|NonTransformation {`_scoped_key`: tf_scoped_key}) OPTIONAL MATCH (task)-[:EXTENDS]->(extended_task:Task) OPTIONAL MATCH (task)-[:RESULTS_IN]->(pdrr:ProtocolDAGResultRef) - RETURN task, extended_task as extended_task, collect(pdrr) as pdrrs + RETURN tf_key, task, extended_task as extended_task, collect(pdrr) as pdrrs """ results = ( - tx.run(query, tf_scoped_key=str(tf_scoped_key)) + tx.run(query, tf_sk_pairs=transformation_sk_pairs) .to_eager_result() .records ) - self.task_tree.extend(results) + for record in results: + key_to_data_map[record["tf_key"]].task_tree.append(record) - def to_subgraph(self, target_scope, statestore): + def to_subgraph(self, target_scope, statestore, subchain_cache): if not self.task_tree: return Subgraph() - - _, tf_node, _ = statestore._keyed_chain_to_subgraph(KeyedChain.from_gufe(self.transformation), target_scope) + _, tf_node, _ = statestore._keyed_chain_to_subgraph(subchain_cache[self.transformation], target_scope) subgraph = Subgraph() | tf_node scope_props = { @@ -994,7 +1001,11 @@ def kc_to_gufe(kc, gts): # due to minor version changes between serialization # times. These keys are then used to get all results for Tasks # associated with them. + # + # TODO: this is currently slow from all of the to_gufe calls + # on subchains that contain more information than is necessary transformation_data: list[TransformationData] = [] + subchain_cache = {} for network_scope, network_keyed_chain in network_keyed_chains: gts = {} for index, (database_key, _) in enumerate(network_keyed_chain): @@ -1006,21 +1017,25 @@ def kc_to_gufe(kc, gts): # get the tokenizable at the index along with all previous data subchain = KeyedChain(network_keyed_chain[: index + 1]) transformation = kc_to_gufe(subchain, gts) + subchain_cache[transformation] = subchain try: index = transformation_data.index(transformation) data = transformation_data[index] except ValueError: data = TransformationData(transformation) transformation_data.append(data) - data.add_known_scoped_key_results(database_key, network_scope, tx) + data.add_known_scoped_key(database_key, network_scope) # - Collect all transformation gufe objects and collect into a new set of edges + TransformationData.update_task_trees(transformation_data, tx) new_edges = [td.transformation for td in transformation_data] # - Make new alchemiscale network with these edges combined_alchemical_network = AlchemicalNetwork(edges=new_edges, name=name) an_subgraph, an_node, an_sk = self._keyed_chain_to_subgraph(KeyedChain.from_gufe(combined_alchemical_network), scope) an_subgraph |= self.create_network_mark_subgraph(an_node)[0] | self.create_taskhub_subgraph(an_node)[0] + task_subgraph = Subgraph() for td in transformation_data: - an_subgraph |= td.to_subgraph(scope, self) + task_subgraph |= td.to_subgraph(scope, self, subchain_cache) + an_subgraph |= task_subgraph merge_subgraph(tx, an_subgraph, "GufeTokenizable", "_scoped_key") return an_sk From d4f3dda6d9acc959bac518c619a681b2c99acb4b Mon Sep 17 00:00:00 2001 From: Ian Kenney Date: Tue, 7 Oct 2025 10:20:27 -0400 Subject: [PATCH 07/35] Remove need for methods to be chainable --- alchemiscale/storage/statestore.py | 94 ++++++++++++++++++------------ 1 file changed, 58 insertions(+), 36 deletions(-) diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index 1e8163f6..f7eb25b8 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -781,13 +781,11 @@ def _topological_sort(graph_data: dict[GufeKey, tuple[Node, list[GufeKey]]]): graph_data.pop(rk) return L - @chainable def assemble_network( self, network: AlchemicalNetwork, scope: Scope, state: NetworkStateEnum | str = NetworkStateEnum.active, - tx=None, ) -> tuple[ScopedKey, ScopedKey, ScopedKey]: """Create all nodes and relationships needed for an AlchemicalNetwork represented in an alchemiscale state store. @@ -813,7 +811,8 @@ def assemble_network( subgraph = nw_subgraph | th_subgraph | nm_subgraph - merge_subgraph(tx, subgraph, "GufeTokenizable", "_scoped_key") + with self.transaction() as tx: + merge_subgraph(tx, subgraph, "GufeTokenizable", "_scoped_key") return nw_sk, th_sk, nm_sk @@ -877,14 +876,12 @@ def delete_network( """ raise NotImplementedError - @chainable def merge_networks( self, network_scoped_keys: list[ScopedKey], name: str, scope: Scope, clone_incomplete_tasks=False, - tx=None, ) -> ScopedKey: """Merge multiple ``AlchemicalNetwork`` nodes into a new ``AlchemicalNetwork``. @@ -921,9 +918,7 @@ class TransformationData: task_tree: list = field( default_factory=list ) # flat represenation of task tree with results - known_scoped_keys: list = field( - default_factory=list - ) + known_scoped_keys: list = field(default_factory=list) def __eq__(self, other): if isinstance(other, (Transformation, NonTransformation)): @@ -931,12 +926,20 @@ def __eq__(self, other): return other.transformation is self.transformation def add_known_scoped_key(self, key, scope): - self.known_scoped_keys.append(ScopedKey(gufe_key=GufeKey(key), **scope.to_dict())) + self.known_scoped_keys.append( + ScopedKey(gufe_key=GufeKey(key), **scope.to_dict()) + ) @staticmethod - def update_task_trees(transformation_data: list, tx): - key_to_data_map = {str(td.transformation.key): td for td in transformation_data} - transformation_sk_pairs = [[str(td.transformation.key), str(sk)] for td in transformation_data for sk in td.known_scoped_keys] + def update_task_trees(transformation_data: list, statestore): + key_to_data_map = { + str(td.transformation.key): td for td in transformation_data + } + transformation_sk_pairs = [ + [str(td.transformation.key), str(sk)] + for td in transformation_data + for sk in td.known_scoped_keys + ] # TODO: filter out tasks that are not wanted before returning them query = """ UNWIND $tf_sk_pairs as pairs @@ -946,11 +949,9 @@ def update_task_trees(transformation_data: list, tx): OPTIONAL MATCH (task)-[:RESULTS_IN]->(pdrr:ProtocolDAGResultRef) RETURN tf_key, task, extended_task as extended_task, collect(pdrr) as pdrrs """ - results = ( - tx.run(query, tf_sk_pairs=transformation_sk_pairs) - .to_eager_result() - .records - ) + results = statestore.execute_query( + query, tf_sk_pairs=transformation_sk_pairs + ).records for record in results: key_to_data_map[record["tf_key"]].task_tree.append(record) @@ -958,7 +959,9 @@ def update_task_trees(transformation_data: list, tx): def to_subgraph(self, target_scope, statestore, subchain_cache): if not self.task_tree: return Subgraph() - _, tf_node, _ = statestore._keyed_chain_to_subgraph(subchain_cache[self.transformation], target_scope) + _, tf_node, _ = statestore._keyed_chain_to_subgraph( + subchain_cache[self.transformation], target_scope + ) subgraph = Subgraph() | tf_node scope_props = { @@ -969,19 +972,34 @@ def to_subgraph(self, target_scope, statestore, subchain_cache): def record_to_node(record): scoped_key = record["_scoped_key"] - scoped_key = ScopedKey(gufe_key=record["_gufe_key"], **scope.to_dict()) - return Node(*record.labels, **record._properties | scope_props | {"_scoped_key": str(scoped_key)}) + scoped_key = ScopedKey( + gufe_key=record["_gufe_key"], **scope.to_dict() + ) + return Node( + *record.labels, + **record._properties + | scope_props + | {"_scoped_key": str(scoped_key)}, + ) for record in self.task_tree: etask_record = record["extended_task"] pdrr_records = record["pdrrs"] task_node = record_to_node(record["task"]) - etask_node = None if not record["extended_task"] else record_to_node(record["extended_task"]) + etask_node = ( + None + if not record["extended_task"] + else record_to_node(record["extended_task"]) + ) if etask_node: - subgraph |= Relationship.type("EXTENDS")(etask_node, task_node, **scope_props) + subgraph |= Relationship.type("EXTENDS")( + etask_node, task_node, **scope_props + ) for pdrr_record in record["pdrrs"]: pdrr_node = record_to_node(pdrr_record) - subgraph |= Relationship.type("RESULTS_IN")(task_node, pdrr_node, **scope_props) + subgraph |= Relationship.type("RESULTS_IN")( + task_node, pdrr_node, **scope_props + ) return subgraph @@ -1026,17 +1044,23 @@ def kc_to_gufe(kc, gts): transformation_data.append(data) data.add_known_scoped_key(database_key, network_scope) # - Collect all transformation gufe objects and collect into a new set of edges - TransformationData.update_task_trees(transformation_data, tx) + TransformationData.update_task_trees(transformation_data, self) new_edges = [td.transformation for td in transformation_data] # - Make new alchemiscale network with these edges combined_alchemical_network = AlchemicalNetwork(edges=new_edges, name=name) - an_subgraph, an_node, an_sk = self._keyed_chain_to_subgraph(KeyedChain.from_gufe(combined_alchemical_network), scope) - an_subgraph |= self.create_network_mark_subgraph(an_node)[0] | self.create_taskhub_subgraph(an_node)[0] + an_subgraph, an_node, an_sk = self._keyed_chain_to_subgraph( + KeyedChain.from_gufe(combined_alchemical_network), scope + ) + an_subgraph |= ( + self.create_network_mark_subgraph(an_node)[0] + | self.create_taskhub_subgraph(an_node)[0] + ) task_subgraph = Subgraph() for td in transformation_data: task_subgraph |= td.to_subgraph(scope, self, subchain_cache) an_subgraph |= task_subgraph - merge_subgraph(tx, an_subgraph, "GufeTokenizable", "_scoped_key") + with self.transaction() as tx: + merge_subgraph(tx, an_subgraph, "GufeTokenizable", "_scoped_key") return an_sk def get_network_state(self, networks: list[ScopedKey]) -> list[str | None]: @@ -2943,9 +2967,9 @@ def task_count(task_dict: dict): ## tasks - @chainable def _validate_extends_tasks( - self, task_list, tx=None + self, + task_list, ) -> dict[str, tuple[Node, str]]: if not task_list: @@ -2957,7 +2981,7 @@ def _validate_extends_tasks( return t, tf._scoped_key as tf_sk """ - results = tx.run(q, task_list=list(map(str, task_list))).to_eager_result() + results = self.execute_query(q, task_list=list(map(str, task_list))) nodes = {} @@ -2977,13 +3001,11 @@ def _validate_extends_tasks( return nodes - @chainable def create_tasks( self, transformations: list[ScopedKey], extends: list[ScopedKey | None] | None = None, creator: str | None = None, - tx=None, ) -> list[ScopedKey]: """Create Tasks for the given Transformations. @@ -3039,7 +3061,6 @@ def create_tasks( extends_nodes = self._validate_extends_tasks( [_extends for _extends in extends if _extends is not None], - tx=tx, ) subgraph = Subgraph() @@ -3060,9 +3081,9 @@ def create_tasks( RETURN n """ - results = tx.run( + results = self.execute_query( q, transformation_subset=list(map(str, transformation_subset)) - ).to_eager_result() + ) transformation_nodes = {} for record in results.records: @@ -3111,7 +3132,8 @@ def create_tasks( _project=scope.project, ) - merge_subgraph(tx, subgraph, "GufeTokenizable", "_scoped_key") + with self.transaction() as tx: + merge_subgraph(tx, subgraph, "GufeTokenizable", "_scoped_key") return sks From b9a527e6172767637ee9a280c1d73ca9a93bf424 Mon Sep 17 00:00:00 2001 From: Ian Kenney Date: Tue, 7 Oct 2025 11:34:34 -0400 Subject: [PATCH 08/35] Change confusing, but beneign clashing names --- alchemiscale/storage/statestore.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index f7eb25b8..0a3ac8fa 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -1037,8 +1037,8 @@ def kc_to_gufe(kc, gts): transformation = kc_to_gufe(subchain, gts) subchain_cache[transformation] = subchain try: - index = transformation_data.index(transformation) - data = transformation_data[index] + idx = transformation_data.index(transformation) + data = transformation_data[idx] except ValueError: data = TransformationData(transformation) transformation_data.append(data) From 96b527eafa4239d74b00e38ff6e48ea2ee6c7b44 Mon Sep 17 00:00:00 2001 From: Ian Kenney Date: Tue, 7 Oct 2025 11:35:43 -0400 Subject: [PATCH 09/35] Merge directly into an_subgraph There is hardly any difference in performance. --- alchemiscale/storage/statestore.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index 0a3ac8fa..568be782 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -1055,10 +1055,8 @@ def kc_to_gufe(kc, gts): self.create_network_mark_subgraph(an_node)[0] | self.create_taskhub_subgraph(an_node)[0] ) - task_subgraph = Subgraph() for td in transformation_data: - task_subgraph |= td.to_subgraph(scope, self, subchain_cache) - an_subgraph |= task_subgraph + an_subgraph |= td.to_subgraph(scope, self, subchain_cache) with self.transaction() as tx: merge_subgraph(tx, an_subgraph, "GufeTokenizable", "_scoped_key") return an_sk From 8f3b17fa766fa9929c9c3918febb4182517b1e54 Mon Sep 17 00:00:00 2001 From: Ian Kenney Date: Thu, 16 Oct 2025 13:26:43 -0400 Subject: [PATCH 10/35] Only create tasks that were complete --- alchemiscale/storage/statestore.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index 568be782..e496e7d6 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -7,6 +7,7 @@ import abc import bisect import datetime +from dataclasses import dataclass, field from contextlib import contextmanager import json import re @@ -881,7 +882,6 @@ def merge_networks( network_scoped_keys: list[ScopedKey], name: str, scope: Scope, - clone_incomplete_tasks=False, ) -> ScopedKey: """Merge multiple ``AlchemicalNetwork`` nodes into a new ``AlchemicalNetwork``. @@ -910,8 +910,6 @@ def merge_networks( f"ScopedKey ({network_scoped_key}) not found in the database." ) - from dataclasses import dataclass, field - @dataclass class TransformationData: transformation: Transformation @@ -920,11 +918,6 @@ class TransformationData: ) # flat represenation of task tree with results known_scoped_keys: list = field(default_factory=list) - def __eq__(self, other): - if isinstance(other, (Transformation, NonTransformation)): - return other is self.transformation - return other.transformation is self.transformation - def add_known_scoped_key(self, key, scope): self.known_scoped_keys.append( ScopedKey(gufe_key=GufeKey(key), **scope.to_dict()) @@ -940,11 +933,10 @@ def update_task_trees(transformation_data: list, statestore): for td in transformation_data for sk in td.known_scoped_keys ] - # TODO: filter out tasks that are not wanted before returning them query = """ UNWIND $tf_sk_pairs as pairs WITH pairs[0] AS tf_key, pairs[1] AS tf_scoped_key - MATCH (task:Task)-[:PERFORMS]->(:Transformation|NonTransformation {`_scoped_key`: tf_scoped_key}) + MATCH (task:Task {status: "complete"})-[:PERFORMS]->(:Transformation|NonTransformation {`_scoped_key`: tf_scoped_key}) OPTIONAL MATCH (task)-[:EXTENDS]->(extended_task:Task) OPTIONAL MATCH (task)-[:RESULTS_IN]->(pdrr:ProtocolDAGResultRef) RETURN tf_key, task, extended_task as extended_task, collect(pdrr) as pdrrs @@ -959,6 +951,7 @@ def update_task_trees(transformation_data: list, statestore): def to_subgraph(self, target_scope, statestore, subchain_cache): if not self.task_tree: return Subgraph() + _, tf_node, _ = statestore._keyed_chain_to_subgraph( subchain_cache[self.transformation], target_scope ) From f566d4f1dfc1445a6404153973e59be108b892a2 Mon Sep 17 00:00:00 2001 From: Ian Kenney Date: Thu, 16 Oct 2025 14:37:59 -0400 Subject: [PATCH 11/35] Avoid soft failure for setting tasks to complete/error --- .../integration/storage/test_statestore.py | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/alchemiscale/tests/integration/storage/test_statestore.py b/alchemiscale/tests/integration/storage/test_statestore.py index 2469fc6e..315ab474 100644 --- a/alchemiscale/tests/integration/storage/test_statestore.py +++ b/alchemiscale/tests/integration/storage/test_statestore.py @@ -154,11 +154,11 @@ def project_scope(iteration): # clutter down below def set_result(_slice, *, ok: bool): nonlocal task_sks, transformation_sks - for task_sk, transformation_sk in zip( - task_sks[_slice], transformation_sks[_slice] - ): + for task_sk in task_sks[_slice]: pdrr = ProtocolDAGResultRef( - obj_key=transformation_sk.gufe_key, scope=task_sk.scope, ok=ok + obj_key=f"ProtocolDAGResult-{uuid.uuid4()}", + scope=task_sk.scope, + ok=ok, ) n4js.set_task_result(task_sk, pdrr) @@ -183,9 +183,11 @@ def set_result(_slice, *, ok: bool): n4js.set_task_waiting(task_sks[5:]) n4js.action_tasks(task_sks[5:], th_sk) + n4js.set_task_running(task_sks[4:5]) n4js.set_task_error(task_sks[4:5]) n4js.action_tasks(task_sks[4:5], th_sk) + n4js.set_task_running(task_sks[:2]) n4js.set_task_complete(task_sks[:2]) set_result(slice(None, 2), ok=True) @@ -208,6 +210,7 @@ def set_result(_slice, *, ok: bool): for transformation in transformations_common ] task_sks = n4js.create_tasks(transformation_sks) + n4js.set_task_running(task_sks) n4js.set_task_complete(task_sks) set_result(slice(None), ok=True) @@ -216,6 +219,7 @@ def set_result(_slice, *, ok: bool): n4js.set_task_waiting(task_sks[0:1]) n4js.set_task_running(task_sks[1:2]) + n4js.set_task_running(task_sks[2:3]) n4js.set_task_complete(task_sks[2:3]) set_result(slice(2, 3), ok=True) @@ -238,6 +242,7 @@ def set_result(_slice, *, ok: bool): task_sks = n4js.create_tasks(transformation_sks) n4js.set_task_waiting(task_sks[:-1]) + n4js.set_task_running(task_sks[-1:]) n4js.set_task_complete(task_sks[-1:]) set_result(slice(-1, None), ok=True) @@ -254,6 +259,16 @@ def set_result(_slice, *, ok: bool): n4js.get_gufe(network_sks[1]).edges ) + # we expect 7 pdrrs from the completed tasks + results = n4js.execute_query( + """ + MATCH (pdrr: ProtocolDAGResultRef {`_project`: $project}) + RETURN pdrr + """, + project=new_scope.project, + ) + assert len(results.records) == 7 + def test_set_network_state(self, n4js, network_tyk2, scope_test): valid_states = [state.value for state in NetworkStateEnum] network_sks = [] From 1d8dfde8d74e0243c53f8ffa335f86cd992ed1e2 Mon Sep 17 00:00:00 2001 From: Ian Kenney Date: Thu, 16 Oct 2025 14:47:37 -0400 Subject: [PATCH 12/35] Also include error task ProtocolDAGResultRefs --- alchemiscale/storage/statestore.py | 3 ++- .../tests/integration/storage/test_statestore.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index e496e7d6..0dae93fc 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -936,7 +936,8 @@ def update_task_trees(transformation_data: list, statestore): query = """ UNWIND $tf_sk_pairs as pairs WITH pairs[0] AS tf_key, pairs[1] AS tf_scoped_key - MATCH (task:Task {status: "complete"})-[:PERFORMS]->(:Transformation|NonTransformation {`_scoped_key`: tf_scoped_key}) + MATCH (task:Task)-[:PERFORMS]->(:Transformation|NonTransformation {`_scoped_key`: tf_scoped_key}) + WHERE task.status IN ["complete", "error"] OPTIONAL MATCH (task)-[:EXTENDS]->(extended_task:Task) OPTIONAL MATCH (task)-[:RESULTS_IN]->(pdrr:ProtocolDAGResultRef) RETURN tf_key, task, extended_task as extended_task, collect(pdrr) as pdrrs diff --git a/alchemiscale/tests/integration/storage/test_statestore.py b/alchemiscale/tests/integration/storage/test_statestore.py index 315ab474..f8f1c4c0 100644 --- a/alchemiscale/tests/integration/storage/test_statestore.py +++ b/alchemiscale/tests/integration/storage/test_statestore.py @@ -263,12 +263,24 @@ def set_result(_slice, *, ok: bool): results = n4js.execute_query( """ MATCH (pdrr: ProtocolDAGResultRef {`_project`: $project}) + WHERE pdrr.ok = True RETURN pdrr """, project=new_scope.project, ) assert len(results.records) == 7 + # we expect 1 pdrrs from the errored task + results = n4js.execute_query( + """ + MATCH (pdrr: ProtocolDAGResultRef {`_project`: $project}) + WHERE pdrr.ok = False + RETURN pdrr + """, + project=new_scope.project, + ) + assert len(results.records) == 1 + def test_set_network_state(self, n4js, network_tyk2, scope_test): valid_states = [state.value for state in NetworkStateEnum] network_sks = [] From f19985f9c5c939c5cdce41dfabf8761d037aa942 Mon Sep 17 00:00:00 2001 From: Ian Kenney Date: Mon, 20 Oct 2025 10:13:11 -0400 Subject: [PATCH 13/35] Add comments to merge_networks --- alchemiscale/storage/statestore.py | 69 +++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 10 deletions(-) diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index 0dae93fc..2845ef49 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -25,7 +25,13 @@ Protocol, ) from gufe.settings import SettingsBaseModel -from gufe.tokenization import GufeTokenizable, GufeKey, JSON_HANDLER, KeyedChain +from gufe.tokenization import ( + GufeTokenizable, + GufeKey, + JSON_HANDLER, + KeyedChain, + key_decode_dependencies, +) from gufe.protocols import ProtocolUnitFailure from neo4j import Transaction, GraphDatabase, Driver @@ -898,7 +904,7 @@ def merge_networks( ------- The ``ScopedKey`` of the new ``AlchemicalNetwork`` in the database. """ - # - Collect keyed chain representation for all alchemiscale networks + # Collect keyed chain representation for all alchemiscale networks try: network_keyed_chains: list[tuple[Scope, KeyedChain]] = [] for network_scoped_key in network_scoped_keys: @@ -910,8 +916,23 @@ def merge_networks( f"ScopedKey ({network_scoped_key}) not found in the database." ) + # Helper dataclass for managing transformations and tasks that + # perform them. @dataclass class TransformationData: + """ + transformation + + task_tree + flat representation of tasks associated with the transformation, each entry has: + - transformation gufe key + - full task neo4j node + - optional: full task neo4j node that this task extends + - list of ProtocolDAGResultRef nodes + known_scoped_keys + All scoped keys that represent the transformation across networks + """ + transformation: Transformation task_tree: list = field( default_factory=list @@ -925,9 +946,16 @@ def add_known_scoped_key(self, key, scope): @staticmethod def update_task_trees(transformation_data: list, statestore): + """Given a list of TransformationData, extract all + necessary info from Neo4j and load the task trees. + """ + key_to_data_map = { str(td.transformation.key): td for td in transformation_data } + # prepare for unwind claus, include transformation key + # for updating each entry of transformation_data for + # each scoped key transformation_sk_pairs = [ [str(td.transformation.key), str(sk)] for td in transformation_data @@ -950,9 +978,26 @@ def update_task_trees(transformation_data: list, statestore): key_to_data_map[record["tf_key"]].task_tree.append(record) def to_subgraph(self, target_scope, statestore, subchain_cache): + """Create a subgraph where the "central" node is the + transformation and iteratively add Task and PDRRs + nodes with their corresponding relationships. + + """ + # if there are no tasks, can return an subgraph, no + # need to make transformation node because it exists + # already outside of this method if not self.task_tree: return Subgraph() + # create the transformation node from its keyed chain, + # this allows later nodes to be easily connected to + # the subgraph outside of this method. subchain_cache + # is a nonlocal dict[Transformation, KeyedChain] that + # removes the need to find the transformation within a + # larger KeyedChain. This will likely be unnecessary + # with later versions of gufe when decode_subchains is + # added. + # (https://github.com/OpenFreeEnergy/gufe/pull/634) _, tf_node, _ = statestore._keyed_chain_to_subgraph( subchain_cache[self.transformation], target_scope ) @@ -965,7 +1010,7 @@ def to_subgraph(self, target_scope, statestore, subchain_cache): } def record_to_node(record): - scoped_key = record["_scoped_key"] + # create node from a neo4j record with updated scoped key scoped_key = ScopedKey( gufe_key=record["_gufe_key"], **scope.to_dict() ) @@ -976,10 +1021,14 @@ def record_to_node(record): | {"_scoped_key": str(scoped_key)}, ) + # process each task found. Each record represents a + # single task. for record in self.task_tree: etask_record = record["extended_task"] pdrr_records = record["pdrrs"] + # update task node to have new scoped key task_node = record_to_node(record["task"]) + # create the task node this task extends if it exists etask_node = ( None if not record["extended_task"] @@ -989,6 +1038,7 @@ def record_to_node(record): subgraph |= Relationship.type("EXTENDS")( etask_node, task_node, **scope_props ) + # clone all result refs for the task for pdrr_record in record["pdrrs"]: pdrr_node = record_to_node(pdrr_record) subgraph |= Relationship.type("RESULTS_IN")( @@ -997,8 +1047,6 @@ def record_to_node(record): return subgraph - from gufe.tokenization import key_decode_dependencies - # TODO: upstream to gufe def kc_to_gufe(kc, gts): for gufe_key, keyed_dict in kc: @@ -1013,9 +1061,6 @@ def kc_to_gufe(kc, gts): # due to minor version changes between serialization # times. These keys are then used to get all results for Tasks # associated with them. - # - # TODO: this is currently slow from all of the to_gufe calls - # on subchains that contain more information than is necessary transformation_data: list[TransformationData] = [] subchain_cache = {} for network_scope, network_keyed_chain in network_keyed_chains: @@ -1037,20 +1082,24 @@ def kc_to_gufe(kc, gts): data = TransformationData(transformation) transformation_data.append(data) data.add_known_scoped_key(database_key, network_scope) - # - Collect all transformation gufe objects and collect into a new set of edges + # Collect all transformation gufe objects and collect into a new set of edges TransformationData.update_task_trees(transformation_data, self) new_edges = [td.transformation for td in transformation_data] - # - Make new alchemiscale network with these edges + # Make new alchemiscale network with these edges combined_alchemical_network = AlchemicalNetwork(edges=new_edges, name=name) an_subgraph, an_node, an_sk = self._keyed_chain_to_subgraph( KeyedChain.from_gufe(combined_alchemical_network), scope ) + # create and fold in taskhub and network mark supporting nodes an_subgraph |= ( self.create_network_mark_subgraph(an_node)[0] | self.create_taskhub_subgraph(an_node)[0] ) + # create and fold in all task and results data for td in transformation_data: an_subgraph |= td.to_subgraph(scope, self, subchain_cache) + + # merge the new network into neo4j with self.transaction() as tx: merge_subgraph(tx, an_subgraph, "GufeTokenizable", "_scoped_key") return an_sk From 9374b9be90bbcc7867b56d1ce49a19e8d602833f Mon Sep 17 00:00:00 2001 From: David Dotson Date: Tue, 11 Nov 2025 21:38:10 -0700 Subject: [PATCH 14/35] Typo fix and formatting --- alchemiscale/storage/statestore.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index 2845ef49..0126f5cd 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -904,7 +904,7 @@ def merge_networks( ------- The ``ScopedKey`` of the new ``AlchemicalNetwork`` in the database. """ - # Collect keyed chain representation for all alchemiscale networks + # Collect keyed chain representation for all alchemical networks try: network_keyed_chains: list[tuple[Scope, KeyedChain]] = [] for network_scoped_key in network_scoped_keys: @@ -1082,6 +1082,7 @@ def kc_to_gufe(kc, gts): data = TransformationData(transformation) transformation_data.append(data) data.add_known_scoped_key(database_key, network_scope) + # Collect all transformation gufe objects and collect into a new set of edges TransformationData.update_task_trees(transformation_data, self) new_edges = [td.transformation for td in transformation_data] From 6859746212ec50e32c8c62a1b138ba747b047e45 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Mon, 8 Jun 2026 17:19:52 -0600 Subject: [PATCH 15/35] Expose merge_networks on the user-facing client and API Add the client + API surface for the merge_networks database operation introduced in this branch. Users can now combine multiple existing AlchemicalNetworks into a single new AlchemicalNetwork through AlchemiscaleClient, with completed and errored Tasks (and their ProtocolDAGResultRefs) carried over so previously-computed results do not need to be re-run. - POST /networks/merge endpoint validates the destination Scope and each source network's Scope against the caller's token, then defers to Neo4jStore.merge_networks. - AlchemiscaleClient.merge_networks wraps the endpoint and guards against wildcard Scopes, empty input, and non-AlchemicalNetwork ScopedKeys client-side. - Integration tests cover the happy path, scope authorization (both bad destination and bad source), and that Tasks + PDRRs in complete/error state are cloned into the new scope and reachable through get_network_tasks. Co-Authored-By: Claude Opus 4.7 (1M context) --- alchemiscale/interface/api.py | 47 +++++ alchemiscale/interface/client.py | 83 ++++++++ .../interface/client/test_client.py | 197 ++++++++++++++++++ .../tests/integration/interface/test_api.py | 91 ++++++++ news/issue-221.rst | 24 +++ 5 files changed, 442 insertions(+) create mode 100644 news/issue-221.rst diff --git a/alchemiscale/interface/api.py b/alchemiscale/interface/api.py index 341069e9..1777e6d4 100644 --- a/alchemiscale/interface/api.py +++ b/alchemiscale/interface/api.py @@ -138,6 +138,53 @@ async def create_network( return an_sk +@router.post("/networks/merge", response_model=ScopedKey) +def merge_networks( + *, + networks: list[str] = Body(embed=True), + name: str = Body(embed=True), + scope: dict = Body(embed=True), + n4js: Neo4jStore = Depends(get_n4js_depends), + token: TokenData = Depends(get_token_data_depends), +): + # validate the destination scope first + try: + target_scope = Scope(**scope) + except (TypeError, ValidationError) as e: + raise HTTPException( + status_code=http_status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=str(e), + ) + validate_scopes(target_scope, token) + + # validate each source network's scope is accessible to the token + network_sks = [] + for network in networks: + try: + network_sk = ScopedKey.from_str(network) + except ValueError as e: + raise HTTPException( + status_code=http_status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=e.args[0], + ) + validate_scopes(network_sk.scope, token) + network_sks.append(network_sk) + + try: + an_sk = n4js.merge_networks( + network_scoped_keys=network_sks, + name=name, + scope=target_scope, + ) + except ValueError as e: + raise HTTPException( + status_code=http_status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=e.args[0], + ) + + return an_sk + + @router.post("/bulk/networks/state/set") def set_networks_state( *, diff --git a/alchemiscale/interface/client.py b/alchemiscale/interface/client.py index f53859ba..d6215f43 100644 --- a/alchemiscale/interface/client.py +++ b/alchemiscale/interface/client.py @@ -187,6 +187,89 @@ def post(): return ScopedKey.from_dict(scoped_key) + def merge_networks( + self, + networks: list[ScopedKey], + name: str, + scope: Scope, + visualize: bool = True, + ) -> ScopedKey: + """Merge multiple existing AlchemicalNetworks into a new AlchemicalNetwork. + + The resulting AlchemicalNetwork contains the union of all + Transformations and NonTransformations from the source networks. + Existing Tasks for those transformations that are in ``complete`` or + ``error`` state are cloned into the new network's scope along with + their associated ProtocolDAGResultRefs, so previously-computed results + do not need to be re-run. + + The new AlchemicalNetwork is created in the active state. + + Parameters + ---------- + networks + The ScopedKeys of the AlchemicalNetworks to merge. The source + networks may live in different Scopes; the caller must have access + to each. + name + The name of the new AlchemicalNetwork. + scope + The Scope in which to create the new AlchemicalNetwork. + This must be a *specific* Scope; it must not contain wildcards. + visualize + If ``True``, show submission progress indicator. + + Returns + ------- + ScopedKey + The ScopedKey of the new, merged AlchemicalNetwork. + """ + if not scope.specific(): + raise ValueError( + f"`scope` '{scope}' contains wildcards ('*'); `scope` must be *specific*" + ) + + if not networks: + raise ValueError("`networks` must contain at least one ScopedKey") + + network_sks = [ + sk if isinstance(sk, ScopedKey) else ScopedKey.from_str(sk) + for sk in networks + ] + + for network_sk in network_sks: + if network_sk.qualname not in ("AlchemicalNetwork",): + raise ValueError( + f"ScopedKey '{network_sk}' does not refer to an AlchemicalNetwork" + ) + + data = dict( + networks=[str(sk) for sk in network_sks], + name=name, + scope=scope.to_dict(), + ) + + def post(): + return self._post_resource("/networks/merge", data) + + if visualize: + from rich.progress import Progress + + with Progress(*self._rich_waiting_columns(), transient=False) as progress: + task = progress.add_task( + f"Merging [bold]{len(network_sks)}[/bold] networks into " + f"[bold]'{name}'[/bold] in scope [bold]'{scope}'[/bold]...", + total=None, + ) + + scoped_key = post() + progress.start_task(task) + progress.update(task, total=1, completed=1) + else: + scoped_key = post() + + return ScopedKey.from_dict(scoped_key) + def set_network_state( self, network: ScopedKey, state: NetworkStateEnum | str ) -> ScopedKey | None: diff --git a/alchemiscale/tests/integration/interface/client/test_client.py b/alchemiscale/tests/integration/interface/client/test_client.py index 390acb71..b8dcf289 100644 --- a/alchemiscale/tests/integration/interface/client/test_client.py +++ b/alchemiscale/tests/integration/interface/client/test_client.py @@ -2,6 +2,7 @@ import datetime from time import sleep import os +import uuid from pathlib import Path from itertools import chain import json @@ -199,6 +200,202 @@ def test_create_network( # common with an existing network # user_client.create_network( + def test_merge_networks( + self, + scope_test, + multiple_scopes, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + network_tyk2, + ): + # gather source AlchemicalNetwork ScopedKeys across all scopes + # n4js_preloaded creates `network_tyk2` and a trimmed copy named + # "incomplete" in each of `multiple_scopes` + source_sks = user_client.query_networks(state=None) + + # destination scope: a new project under the existing org/campaign + merge_scope = Scope( + org=scope_test.org, + campaign=scope_test.campaign, + project="merged_project", + ) + + merged_sk = user_client.merge_networks( + networks=source_sks, + name="merged_tyk2", + scope=merge_scope, + visualize=False, + ) + + assert isinstance(merged_sk, ScopedKey) + assert merged_sk.scope == merge_scope + assert merged_sk.qualname == "AlchemicalNetwork" + + # the merged network should exist + assert user_client.check_exists(merged_sk) + + # the merged network should appear in queries (defaults to active state) + all_active_sks = user_client.query_networks() + assert merged_sk in all_active_sks + + # the merged network should contain the union of all source edges; + # since `network_tyk2` is a superset of `incomplete`, the union equals + # `network_tyk2.edges` + merged_network = user_client.get_network(merged_sk) + assert merged_network.name == "merged_tyk2" + assert len(merged_network.edges) == len(network_tyk2.edges) + assert {t.key for t in merged_network.edges} == { + t.key for t in network_tyk2.edges + } + + def test_merge_networks_rejects_wildcard_scope( + self, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + ): + source_sks = user_client.query_networks(state=None) + with pytest.raises(ValueError, match="wildcards"): + user_client.merge_networks( + networks=source_sks, + name="should_fail", + scope=Scope(org="test_org"), + visualize=False, + ) + + def test_merge_networks_rejects_empty_list( + self, + scope_test, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + ): + with pytest.raises(ValueError, match="at least one"): + user_client.merge_networks( + networks=[], + name="should_fail", + scope=scope_test, + visualize=False, + ) + + def test_merge_networks_rejects_non_network_scoped_key( + self, + scope_test, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + ): + # pass a Transformation ScopedKey rather than an AlchemicalNetwork one + tf_sks = user_client.query_transformations(scope=scope_test) + assert tf_sks + with pytest.raises(ValueError, match="does not refer to an AlchemicalNetwork"): + user_client.merge_networks( + networks=[tf_sks[0]], + name="should_fail", + scope=scope_test, + visualize=False, + ) + + def test_merge_networks_preserves_tasks_and_results( + self, + scope_test, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + network_tyk2, + ): + """The merged network must carry over Tasks in complete/error state + along with their ProtocolDAGResultRefs, cloned into the new scope.""" + # pick a source network in scope_test + source_sks = user_client.query_networks(scope=scope_test, state=None) + assert source_sks + source_sk = source_sks[0] + + # create Tasks on two of its Transformations directly through n4js so + # we can drive them to completed/errored states with PDRRs without + # actually executing protocols + transformation_sks = n4js_preloaded.get_network_transformations(source_sk) + assert len(transformation_sks) >= 2 + + task_sks = n4js_preloaded.create_tasks(transformation_sks[:2]) + + # task 0: complete, ok result + n4js_preloaded.set_task_running(task_sks[:1]) + n4js_preloaded.set_task_complete(task_sks[:1]) + ok_pdrr = ProtocolDAGResultRef( + obj_key=f"ProtocolDAGResult-{uuid.uuid4()}", + scope=task_sks[0].scope, + ok=True, + ) + n4js_preloaded.set_task_result(task_sks[0], ok_pdrr) + + # task 1: error, failure result + n4js_preloaded.set_task_running(task_sks[1:2]) + n4js_preloaded.set_task_error(task_sks[1:2]) + err_pdrr = ProtocolDAGResultRef( + obj_key=f"ProtocolDAGResult-{uuid.uuid4()}", + scope=task_sks[1].scope, + ok=False, + ) + n4js_preloaded.set_task_result(task_sks[1], err_pdrr) + + # merge into a fresh project under the same org/campaign + merge_scope = Scope( + org=scope_test.org, + campaign=scope_test.campaign, + project="merged_with_results", + ) + merged_sk = user_client.merge_networks( + networks=[source_sk], + name="merged_with_results", + scope=merge_scope, + visualize=False, + ) + assert user_client.check_exists(merged_sk) + + # both Tasks should appear in the new scope, with their original statuses + task_records = n4js_preloaded.execute_query( + """ + MATCH (t:Task {`_project`: $project}) + RETURN t.status AS status + """, + project=merge_scope.project, + ).records + assert sorted(r["status"] for r in task_records) == ["complete", "error"] + + # one ok and one not-ok PDRR should appear in the new scope, with their + # original object keys preserved + pdrr_records = n4js_preloaded.execute_query( + """ + MATCH (pdrr:ProtocolDAGResultRef {`_project`: $project}) + RETURN pdrr.ok AS ok, pdrr.obj_key AS obj_key + """, + project=merge_scope.project, + ).records + assert len(pdrr_records) == 2 + oks = sorted(r["ok"] for r in pdrr_records) + assert oks == [False, True] + obj_keys = {r["obj_key"] for r in pdrr_records} + assert obj_keys == {ok_pdrr.obj_key, err_pdrr.obj_key} + + # cloned PDRRs must be wired to the cloned Tasks in the new scope + linked = n4js_preloaded.execute_query( + """ + MATCH (t:Task {`_project`: $project})-[:RESULTS_IN]-> + (pdrr:ProtocolDAGResultRef {`_project`: $project}) + RETURN t.status AS status, pdrr.ok AS ok + """, + project=merge_scope.project, + ).records + pairs = sorted((r["status"], r["ok"]) for r in linked) + assert pairs == [("complete", True), ("error", False)] + + # the cloned Tasks must be reachable from the merged AlchemicalNetwork + # via the standard PERFORMS traversal that the user-facing API uses; + # this catches any case where Tasks are written to the new scope but + # not wired back to their Transformations in the merged network + merged_task_sks = user_client.get_network_tasks(merged_sk) + assert len(merged_task_sks) == 2 + assert all(sk.scope == merge_scope for sk in merged_task_sks) + statuses = sorted(user_client.get_tasks_status(merged_task_sks)) + assert statuses == ["complete", "error"] + def test_check_exists( self, scope_test, diff --git a/alchemiscale/tests/integration/interface/test_api.py b/alchemiscale/tests/integration/interface/test_api.py index 21322c02..24be0be1 100644 --- a/alchemiscale/tests/integration/interface/test_api.py +++ b/alchemiscale/tests/integration/interface/test_api.py @@ -103,6 +103,97 @@ def test_create_network_bad_scope( assert str(bad_scope) in details assert str(scope_test) in details + def test_merge_networks( + self, n4js_preloaded, test_client, network_tyk2, scope_test + ): + n4js = n4js_preloaded + + # source networks in scope_test (pre-loaded by n4js_preloaded) + source_sks = n4js.query_networks(scope=scope_test) + assert len(source_sks) >= 2 + + # destination scope: a new project under the same org/campaign so the + # test_client's scope_test token has access + merge_scope_dict = { + "org": scope_test.org, + "campaign": scope_test.campaign, + "project": scope_test.project, + } + + headers = {"Content-type": "application/json"} + data = dict( + networks=[str(sk) for sk in source_sks], + name="api_merged", + scope=merge_scope_dict, + ) + jsondata = json.dumps(data, cls=JSON_HANDLER.encoder) + + response = test_client.post("/networks/merge", data=jsondata, headers=headers) + assert response.status_code == 200 + + merged_sk = ScopedKey(**response.json()) + assert merged_sk.scope == scope_test + assert merged_sk.gufe_key.startswith("AlchemicalNetwork-") + + # network should now be present in the database + assert n4js.check_existence(merged_sk) + + # merged network's union-of-edges equals network_tyk2's edge set, + # since one of the sources is network_tyk2 and the other is a strict subset + merged_network = n4js.get_gufe(merged_sk) + assert merged_network.name == "api_merged" + assert {t.key for t in merged_network.edges} == { + t.key for t in network_tyk2.edges + } + + def test_merge_networks_bad_scope( + self, n4js_preloaded, test_client, network_tyk2, scope_test, multiple_scopes + ): + # destination scope the test_client's token does not have access to + bad_scope = multiple_scopes[1] + assert bad_scope != scope_test + + source_sks = n4js_preloaded.query_networks(scope=scope_test) + assert source_sks + + headers = {"Content-type": "application/json"} + data = dict( + networks=[str(sk) for sk in source_sks], + name="should_fail", + scope=bad_scope.to_dict(), + ) + jsondata = json.dumps(data, cls=JSON_HANDLER.encoder) + + response = test_client.post("/networks/merge", data=jsondata, headers=headers) + assert response.status_code == 401 + details = response.json() + assert "detail" in details + assert str(bad_scope) in details["detail"] + + def test_merge_networks_bad_source_scope( + self, n4js_preloaded, test_client, network_tyk2, scope_test, multiple_scopes + ): + # source network in a scope the test_client's token does not authorize + unauth_scope = multiple_scopes[1] + assert unauth_scope != scope_test + + unauth_source_sks = n4js_preloaded.query_networks(scope=unauth_scope) + assert unauth_source_sks + + headers = {"Content-type": "application/json"} + data = dict( + networks=[str(sk) for sk in unauth_source_sks], + name="should_fail", + scope=scope_test.to_dict(), + ) + jsondata = json.dumps(data, cls=JSON_HANDLER.encoder) + + response = test_client.post("/networks/merge", data=jsondata, headers=headers) + assert response.status_code == 401 + details = response.json() + assert "detail" in details + assert str(unauth_scope) in details["detail"] + def test_get_network(self, prepared_network, test_client): network, scoped_key = prepared_network response = test_client.get(f"/networks/{scoped_key}") diff --git a/news/issue-221.rst b/news/issue-221.rst new file mode 100644 index 00000000..75a23c91 --- /dev/null +++ b/news/issue-221.rst @@ -0,0 +1,24 @@ +**Added:** + +* ``AlchemiscaleClient.merge_networks`` for combining multiple existing ``AlchemicalNetwork``\s into a new ``AlchemicalNetwork``, preserving completed and errored ``Task`` results so they do not need to be re-run. Backed by ``Neo4jStore.merge_networks`` and a new ``POST /networks/merge`` endpoint on the user API. + +**Changed:** + +* + +**Deprecated:** + +* + +**Removed:** + +* + +**Fixed:** + +* + +**Security:** + +* + From 7e3f5d12abf41ed32ef6fb6aa0004adb404a6762 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Mon, 8 Jun 2026 17:20:03 -0600 Subject: [PATCH 16/35] Refactor merge_networks using KeyedChain.decode_subchains Replace the inline kc_to_gufe helper and manual chain-slicing loop with a call to KeyedChain.decode_subchains, paired with a list comprehension that captures the original database GufeKeys in the same order. The decode_subchains helper (gufe >=1.8.0) reuses a shared tokenizable_map across yields, so common dependencies in a source network are decoded only once. Lift the previously nested TransformationData dataclass to a private module-level _TransformationData so merge_networks reads top-to-bottom without chasing an inline class definition, and so the helper can be referenced by future tests in isolation. The dropped key_decode_dependencies import goes with kc_to_gufe. subchain_cache is preserved with the same shape but now populated via KeyedChain.from_gufe(transformation); whether the cache (and the unconnected tf_node it feeds) is needed at all is a separate question to revisit once the PERFORMS-wiring question is settled. Co-Authored-By: Claude Opus 4.7 (1M context) --- alchemiscale/storage/statestore.py | 333 ++++++++++++++--------------- 1 file changed, 166 insertions(+), 167 deletions(-) diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index ef2fc87c..7527a38a 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -30,7 +30,6 @@ GufeKey, JSON_HANDLER, KeyedChain, - key_decode_dependencies, ) from gufe.protocols import ProtocolUnitFailure @@ -85,6 +84,141 @@ def get_n4js(settings: Neo4jStoreSettings): class Neo4JStoreError(Exception): ... +def _is_transformation_keyed_dict(keyed_dict: dict) -> bool: + """Return ``True`` if ``keyed_dict`` represents a ``Transformation`` or ``NonTransformation``. + + Used as the predicate for ``KeyedChain.decode_subchains`` when walking + an ``AlchemicalNetwork``'s keyed chain inside :meth:`Neo4jStore.merge_networks`. + """ + return keyed_dict.get("__qualname__") in ("Transformation", "NonTransformation") + + +@dataclass +class _TransformationData: + """Bookkeeping for one Transformation as it is reconstructed during + :meth:`Neo4jStore.merge_networks`. + + Attributes + ---------- + transformation + The decoded ``Transformation`` (or ``NonTransformation``) + ``GufeTokenizable``. + task_tree + Flat list of Neo4j records, one per ``Task`` associated with this + ``Transformation`` in any of the source networks. Each record carries: + + - ``tf_key``: this ``Transformation``'s decoded gufe key (string) + - ``task``: the full ``Task`` node + - ``extended_task``: optional ``Task`` node that this ``Task`` extends + - ``pdrrs``: list of ``ProtocolDAGResultRef`` nodes for this ``Task`` + known_scoped_keys + All ``ScopedKey``\\ s representing this ``Transformation`` across the + source networks. Multiple ``ScopedKey``\\ s can map to a single + decoded ``Transformation`` if the same content was serialized under + different gufe versions. + """ + + transformation: Transformation + task_tree: list = field(default_factory=list) + known_scoped_keys: list = field(default_factory=list) + + def add_known_scoped_key(self, key, scope): + self.known_scoped_keys.append( + ScopedKey(gufe_key=GufeKey(key), **scope.to_dict()) + ) + + @staticmethod + def update_task_trees(transformation_data: list, statestore): + """Given a list of ``_TransformationData``, extract all necessary + info from Neo4j and load the task trees. + """ + key_to_data_map = {str(td.transformation.key): td for td in transformation_data} + # prepare for unwind clause, include transformation key + # for updating each entry of transformation_data for + # each scoped key + transformation_sk_pairs = [ + [str(td.transformation.key), str(sk)] + for td in transformation_data + for sk in td.known_scoped_keys + ] + query = """ + UNWIND $tf_sk_pairs as pairs + WITH pairs[0] AS tf_key, pairs[1] AS tf_scoped_key + MATCH (task:Task)-[:PERFORMS]->(:Transformation|NonTransformation {`_scoped_key`: tf_scoped_key}) + WHERE task.status IN ["complete", "error"] + OPTIONAL MATCH (task)-[:EXTENDS]->(extended_task:Task) + OPTIONAL MATCH (task)-[:RESULTS_IN]->(pdrr:ProtocolDAGResultRef) + RETURN tf_key, task, extended_task as extended_task, collect(pdrr) as pdrrs + """ + results = statestore.execute_query( + query, tf_sk_pairs=transformation_sk_pairs + ).records + + for record in results: + key_to_data_map[record["tf_key"]].task_tree.append(record) + + def to_subgraph(self, target_scope, statestore, subchain_cache): + """Create a subgraph where the "central" node is the + transformation and iteratively add Task and PDRR + nodes with their corresponding relationships. + """ + # if there are no tasks, can return an empty subgraph; no + # need to make a transformation node because it already exists + # in the surrounding merged-network subgraph + if not self.task_tree: + return Subgraph() + + # create the transformation node. With ``decode_subchains`` + # available upstream (gufe >=1.8.0), ``subchain_cache`` holds the + # re-serialized chain for the decoded ``Transformation``; the + # resulting Node's ``_scoped_key`` matches the one already produced + # by the combined AlchemicalNetwork's keyed chain and will dedupe + # against it during ``merge_subgraph``. + _, tf_node, _ = statestore._keyed_chain_to_subgraph( + subchain_cache[self.transformation], target_scope + ) + subgraph = Subgraph() | tf_node + + scope_props = { + "_org": target_scope.org, + "_campaign": target_scope.campaign, + "_project": target_scope.project, + } + + def record_to_node(record): + # create node from a neo4j record with updated scoped key + scoped_key = ScopedKey( + gufe_key=record["_gufe_key"], **target_scope.to_dict() + ) + return Node( + *record.labels, + **record._properties | scope_props | {"_scoped_key": str(scoped_key)}, + ) + + # process each task found. Each record represents a single task. + for record in self.task_tree: + # update task node to have new scoped key + task_node = record_to_node(record["task"]) + # create the task node this task extends if it exists + etask_node = ( + None + if not record["extended_task"] + else record_to_node(record["extended_task"]) + ) + if etask_node: + subgraph |= Relationship.type("EXTENDS")( + etask_node, task_node, **scope_props + ) + # clone all result refs for the task + for pdrr_record in record["pdrrs"]: + pdrr_node = record_to_node(pdrr_record) + subgraph |= Relationship.type("RESULTS_IN")( + task_node, pdrr_node, **scope_props + ) + + return subgraph + + class AlchemiscaleStateStore(abc.ABC): ... @@ -916,175 +1050,40 @@ def merge_networks( f"ScopedKey ({network_scoped_key}) not found in the database." ) - # Helper dataclass for managing transformations and tasks that - # perform them. - @dataclass - class TransformationData: - """ - transformation - - task_tree - flat representation of tasks associated with the transformation, each entry has: - - transformation gufe key - - full task neo4j node - - optional: full task neo4j node that this task extends - - list of ProtocolDAGResultRef nodes - known_scoped_keys - All scoped keys that represent the transformation across networks - """ - - transformation: Transformation - task_tree: list = field( - default_factory=list - ) # flat represenation of task tree with results - known_scoped_keys: list = field(default_factory=list) - - def add_known_scoped_key(self, key, scope): - self.known_scoped_keys.append( - ScopedKey(gufe_key=GufeKey(key), **scope.to_dict()) - ) - - @staticmethod - def update_task_trees(transformation_data: list, statestore): - """Given a list of TransformationData, extract all - necessary info from Neo4j and load the task trees. - """ - - key_to_data_map = { - str(td.transformation.key): td for td in transformation_data - } - # prepare for unwind claus, include transformation key - # for updating each entry of transformation_data for - # each scoped key - transformation_sk_pairs = [ - [str(td.transformation.key), str(sk)] - for td in transformation_data - for sk in td.known_scoped_keys - ] - query = """ - UNWIND $tf_sk_pairs as pairs - WITH pairs[0] AS tf_key, pairs[1] AS tf_scoped_key - MATCH (task:Task)-[:PERFORMS]->(:Transformation|NonTransformation {`_scoped_key`: tf_scoped_key}) - WHERE task.status IN ["complete", "error"] - OPTIONAL MATCH (task)-[:EXTENDS]->(extended_task:Task) - OPTIONAL MATCH (task)-[:RESULTS_IN]->(pdrr:ProtocolDAGResultRef) - RETURN tf_key, task, extended_task as extended_task, collect(pdrr) as pdrrs - """ - results = statestore.execute_query( - query, tf_sk_pairs=transformation_sk_pairs - ).records - - for record in results: - key_to_data_map[record["tf_key"]].task_tree.append(record) - - def to_subgraph(self, target_scope, statestore, subchain_cache): - """Create a subgraph where the "central" node is the - transformation and iteratively add Task and PDRRs - nodes with their corresponding relationships. - - """ - # if there are no tasks, can return an subgraph, no - # need to make transformation node because it exists - # already outside of this method - if not self.task_tree: - return Subgraph() - - # create the transformation node from its keyed chain, - # this allows later nodes to be easily connected to - # the subgraph outside of this method. subchain_cache - # is a nonlocal dict[Transformation, KeyedChain] that - # removes the need to find the transformation within a - # larger KeyedChain. This will likely be unnecessary - # with later versions of gufe when decode_subchains is - # added. - # (https://github.com/OpenFreeEnergy/gufe/pull/634) - _, tf_node, _ = statestore._keyed_chain_to_subgraph( - subchain_cache[self.transformation], target_scope - ) - subgraph = Subgraph() | tf_node - - scope_props = { - "_org": target_scope.org, - "_campaign": target_scope.campaign, - "_project": target_scope.project, - } - - def record_to_node(record): - # create node from a neo4j record with updated scoped key - scoped_key = ScopedKey( - gufe_key=record["_gufe_key"], **scope.to_dict() - ) - return Node( - *record.labels, - **record._properties - | scope_props - | {"_scoped_key": str(scoped_key)}, - ) - - # process each task found. Each record represents a - # single task. - for record in self.task_tree: - etask_record = record["extended_task"] - pdrr_records = record["pdrrs"] - # update task node to have new scoped key - task_node = record_to_node(record["task"]) - # create the task node this task extends if it exists - etask_node = ( - None - if not record["extended_task"] - else record_to_node(record["extended_task"]) - ) - if etask_node: - subgraph |= Relationship.type("EXTENDS")( - etask_node, task_node, **scope_props - ) - # clone all result refs for the task - for pdrr_record in record["pdrrs"]: - pdrr_node = record_to_node(pdrr_record) - subgraph |= Relationship.type("RESULTS_IN")( - task_node, pdrr_node, **scope_props - ) - - return subgraph - - # TODO: upstream to gufe - def kc_to_gufe(kc, gts): - for gufe_key, keyed_dict in kc: - if gt := gts.get(gufe_key): - continue - gt = key_decode_dependencies(keyed_dict, registry=gts) - gts[gufe_key] = gt - return gt - - # Map Transformation and NonTransformation objects to their - # potentially duplicated database gufe keys. This may happen - # due to minor version changes between serialization - # times. These keys are then used to get all results for Tasks - # associated with them. - transformation_data: list[TransformationData] = [] - subchain_cache = {} + # Map decoded Transformation / NonTransformation objects to all of + # their original database GufeKeys (potentially across multiple + # source networks, and including duplicates introduced by minor + # serialization-version drift). These original keys are needed to + # locate Tasks associated with each Transformation in the source + # networks. + transformation_data: list[_TransformationData] = [] + subchain_cache: dict[GufeTokenizable, KeyedChain] = {} for network_scope, network_keyed_chain in network_keyed_chains: - gts = {} - for index, (database_key, _) in enumerate(network_keyed_chain): - # only process Transformation or NonTransformation keys - if ( - "Transformation" in database_key - or "NonTransformation" in database_key - ): - # get the tokenizable at the index along with all previous data - subchain = KeyedChain(network_keyed_chain[: index + 1]) - transformation = kc_to_gufe(subchain, gts) - subchain_cache[transformation] = subchain - try: - idx = transformation_data.index(transformation) - data = transformation_data[idx] - except ValueError: - data = TransformationData(transformation) - transformation_data.append(data) - data.add_known_scoped_key(database_key, network_scope) + # database keys for Transformations / NonTransformations in this + # source network's chain, in chain order + database_keys = [ + gufe_key + for gufe_key, keyed_dict in network_keyed_chain + if _is_transformation_keyed_dict(keyed_dict) + ] + # decoded Transformation / NonTransformation objects in the same + # order; decode_subchains shares a tokenizable_map across yields + # so common dependencies are decoded only once per source network + transformations = network_keyed_chain.decode_subchains( + _is_transformation_keyed_dict + ) + for database_key, transformation in zip(database_keys, transformations): + subchain_cache[transformation] = KeyedChain.from_gufe(transformation) + try: + idx = transformation_data.index(transformation) + data = transformation_data[idx] + except ValueError: + data = _TransformationData(transformation) + transformation_data.append(data) + data.add_known_scoped_key(database_key, network_scope) # Collect all transformation gufe objects and collect into a new set of edges - TransformationData.update_task_trees(transformation_data, self) + _TransformationData.update_task_trees(transformation_data, self) new_edges = [td.transformation for td in transformation_data] # Make new alchemiscale network with these edges combined_alchemical_network = AlchemicalNetwork(edges=new_edges, name=name) From df3a1807f0a35b5b0f488f963c9a237e2256ab8c Mon Sep 17 00:00:00 2001 From: David Dotson Date: Mon, 8 Jun 2026 17:20:12 -0600 Subject: [PATCH 17/35] Bump gufe to >=1.8.0 in test and server env files merge_networks now uses KeyedChain.decode_subchains, which was added in gufe 1.8.0. Bump the pin in: - devtools/conda-envs/test.yml (was =1.7.1) so the integration tests exercise the new code path. - devtools/conda-envs/alchemiscale-server.yml (was =1.6.1) so server deployments running merge_networks pick up the required gufe API. Client and compute env files are left alone since neither imports the new code path. Co-Authored-By: Claude Opus 4.7 (1M context) --- devtools/conda-envs/alchemiscale-server.yml | 2 +- devtools/conda-envs/test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/devtools/conda-envs/alchemiscale-server.yml b/devtools/conda-envs/alchemiscale-server.yml index fa6f22b3..298e5861 100644 --- a/devtools/conda-envs/alchemiscale-server.yml +++ b/devtools/conda-envs/alchemiscale-server.yml @@ -8,7 +8,7 @@ dependencies: - cuda-version >=12 # alchemiscale dependencies - - gufe=1.6.1 + - gufe>=1.8.0 - openfe=1.6.1 - requests - click diff --git a/devtools/conda-envs/test.yml b/devtools/conda-envs/test.yml index d8591fcf..024256e7 100644 --- a/devtools/conda-envs/test.yml +++ b/devtools/conda-envs/test.yml @@ -7,7 +7,7 @@ dependencies: - cuda-version >=12 # alchemiscale dependencies - - gufe =1.7.1 + - gufe >=1.8.0 - openfe =1.8.0 - pydantic >2 - pydantic-settings From d0bf4ec7d6c0c719308ff10fffc92335ca019d38 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Mon, 8 Jun 2026 17:42:26 -0600 Subject: [PATCH 18/35] Address PR #507 review: PERFORMS wiring, state parameter, optimizations Code review revealed that cloned Tasks on the merged network were written but never wired back to their Transformations, leaving every PERFORMS-based traversal (get_network_tasks, get_task_transformation, set_tasks_status, etc.) blind to them. The new test_merge_networks_preserves_tasks_and_results check via get_network_tasks asserts this exact thing and would have failed in CI. Changes ------- - Add the PERFORMS edge in _TransformationData.to_subgraph so cloned Tasks are reachable from the standard (:AlchemicalNetwork)-[:DEPENDS_ON]->(:Transformation)<-[:PERFORMS]-(:Task) traversal. The pre-existing tf_node construction is now meaningful, resolving the open question about subchain_cache. - Add a `state` parameter to Neo4jStore.merge_networks, /networks/merge, and AlchemiscaleClient.merge_networks, bringing merge_networks to feature parity with assemble_network / create_network. Defaults to NetworkStateEnum.active. - Document the "cloned Tasks are not actioned to the merged TaskHub" semantics in the store-level and client docstrings, including the remediation path (call action_tasks after merging). - Replace the O(N^2) `transformation_data.index(transformation)` dedup with a `dict[GufeKey, _TransformationData]` keyed on transformation.key, avoiding repeated GufeTokenizable equality checks on large networks. - Improve the missing-ScopedKey error to collect and report all missing source SKs in a single ValueError, rather than only the one that first triggered KeyError. - Add a chain-order contract comment at the zip(database_keys, transformations) site pointing at the decode_subchains yield-order guarantee. - Make /networks/merge an `async def` endpoint, matching the style of the neighboring create_network endpoint. - Add a parametrized client test (state in {active, inactive}) that exercises the new state parameter end-to-end. - Drop the unused network_tyk2 fixture from the two API-level authorization tests. Co-Authored-By: Claude Opus 4.7 (1M context) --- alchemiscale/interface/api.py | 6 +- alchemiscale/interface/client.py | 16 ++- alchemiscale/storage/statestore.py | 106 ++++++++++++------ .../interface/client/test_client.py | 28 +++++ .../tests/integration/interface/test_api.py | 4 +- 5 files changed, 120 insertions(+), 40 deletions(-) diff --git a/alchemiscale/interface/api.py b/alchemiscale/interface/api.py index 1777e6d4..d5edf34f 100644 --- a/alchemiscale/interface/api.py +++ b/alchemiscale/interface/api.py @@ -31,7 +31,7 @@ from ..settings import get_base_api_settings from ..storage.statestore import Neo4jStore from ..storage.objectstore import S3ObjectStore -from ..storage.models import TaskStatusEnum, StrategyState +from ..storage.models import NetworkStateEnum, TaskStatusEnum, StrategyState from ..models import Scope, ScopedKey from ..security.models import TokenData, CredentialedUserIdentity @@ -139,11 +139,12 @@ async def create_network( @router.post("/networks/merge", response_model=ScopedKey) -def merge_networks( +async def merge_networks( *, networks: list[str] = Body(embed=True), name: str = Body(embed=True), scope: dict = Body(embed=True), + state: str = Body(embed=True, default=NetworkStateEnum.active.value), n4js: Neo4jStore = Depends(get_n4js_depends), token: TokenData = Depends(get_token_data_depends), ): @@ -175,6 +176,7 @@ def merge_networks( network_scoped_keys=network_sks, name=name, scope=target_scope, + state=state, ) except ValueError as e: raise HTTPException( diff --git a/alchemiscale/interface/client.py b/alchemiscale/interface/client.py index d6215f43..c0a0a0a3 100644 --- a/alchemiscale/interface/client.py +++ b/alchemiscale/interface/client.py @@ -192,6 +192,7 @@ def merge_networks( networks: list[ScopedKey], name: str, scope: Scope, + state: NetworkStateEnum | str = NetworkStateEnum.active, visualize: bool = True, ) -> ScopedKey: """Merge multiple existing AlchemicalNetworks into a new AlchemicalNetwork. @@ -203,7 +204,13 @@ def merge_networks( their associated ProtocolDAGResultRefs, so previously-computed results do not need to be re-run. - The new AlchemicalNetwork is created in the active state. + Cloned Tasks are wired to their Transformations via ``PERFORMS`` and + are reachable through standard network traversals + (``get_network_tasks``, ``get_network_results``, etc.). They are + intentionally **not** actioned to the new network's TaskHub; to + retry errored Tasks on the merged network, call + :meth:`action_tasks` with the merged network's ScopedKey after the + merge completes. Parameters ---------- @@ -216,6 +223,10 @@ def merge_networks( scope The Scope in which to create the new AlchemicalNetwork. This must be a *specific* Scope; it must not contain wildcards. + state + The starting state of the new AlchemicalNetwork in the database. + See :meth:`AlchemiscaleClient.set_network_state` for valid states. + Defaults to ``"active"``. visualize If ``True``, show submission progress indicator. @@ -243,10 +254,13 @@ def merge_networks( f"ScopedKey '{network_sk}' does not refer to an AlchemicalNetwork" ) + state = NetworkStateEnum(state) + data = dict( networks=[str(sk) for sk in network_sks], name=name, scope=scope.to_dict(), + state=state.value, ) def post(): diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index 7527a38a..5f94f83f 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -158,22 +158,29 @@ def update_task_trees(transformation_data: list, statestore): key_to_data_map[record["tf_key"]].task_tree.append(record) def to_subgraph(self, target_scope, statestore, subchain_cache): - """Create a subgraph where the "central" node is the - transformation and iteratively add Task and PDRR - nodes with their corresponding relationships. - """ - # if there are no tasks, can return an empty subgraph; no - # need to make a transformation node because it already exists - # in the surrounding merged-network subgraph + """Create a subgraph anchored at the Transformation node and + iteratively add Task and PDRR nodes with their corresponding + relationships. + + Each cloned Task is wired back to the Transformation via a + ``PERFORMS`` edge so the standard + ``(:AlchemicalNetwork)-[:DEPENDS_ON]->(:Transformation)<-[:PERFORMS]-(:Task)`` + traversal keeps working on the merged network. Note that cloned + Tasks are intentionally **not** actioned to the merged network's + TaskHub; see :meth:`Neo4jStore.merge_networks` for the rationale. + """ + # if there are no tasks, return an empty subgraph; the + # Transformation node already exists in the surrounding + # merged-network subgraph if not self.task_tree: return Subgraph() - # create the transformation node. With ``decode_subchains`` - # available upstream (gufe >=1.8.0), ``subchain_cache`` holds the - # re-serialized chain for the decoded ``Transformation``; the - # resulting Node's ``_scoped_key`` matches the one already produced - # by the combined AlchemicalNetwork's keyed chain and will dedupe - # against it during ``merge_subgraph``. + # build the Transformation node as the PERFORMS anchor for cloned + # Tasks. ``subchain_cache`` holds the re-serialized chain for the + # decoded ``Transformation``; the resulting Node's ``_scoped_key`` + # matches the one already produced by the combined + # AlchemicalNetwork's keyed chain and will dedupe against it + # during ``merge_subgraph``. _, tf_node, _ = statestore._keyed_chain_to_subgraph( subchain_cache[self.transformation], target_scope ) @@ -199,6 +206,10 @@ def record_to_node(record): for record in self.task_tree: # update task node to have new scoped key task_node = record_to_node(record["task"]) + # wire the cloned Task back to its Transformation; without + # this edge the Task is unreachable from get_network_tasks + # and every other PERFORMS-based traversal + subgraph |= Relationship.type("PERFORMS")(task_node, tf_node, **scope_props) # create the task node this task extends if it exists etask_node = ( None @@ -1022,9 +1033,23 @@ def merge_networks( network_scoped_keys: list[ScopedKey], name: str, scope: Scope, + state: NetworkStateEnum | str = NetworkStateEnum.active, ) -> ScopedKey: """Merge multiple ``AlchemicalNetwork`` nodes into a new ``AlchemicalNetwork``. + Each ``Transformation`` / ``NonTransformation`` in the input + networks is included exactly once in the new network. ``Task``\\ s + on the source networks that are in ``complete`` or ``error`` + state are cloned into the new network's ``Scope`` along with + their ``ProtocolDAGResultRef``\\ s and ``EXTENDS`` relationships, + and are wired to their ``Transformation`` via ``PERFORMS`` so + they are reachable from the standard network traversals. + + Cloned ``Task``\\ s are intentionally **not** actioned to the new + network's ``TaskHub``. Users wanting to retry errored tasks on + the merged network should call :meth:`action_tasks` themselves + with the merged network's ``TaskHub`` ``ScopedKey``. + Parameters ---------- network_scoped_keys @@ -1033,21 +1058,30 @@ def merge_networks( The name of the new ``AlchemicalNetwork``. scope The ``Scope`` of the new ``AlchemicalNetwork``. + state + The starting state for the new ``AlchemicalNetwork``'s + ``NetworkMark``. Defaults to ``NetworkStateEnum.active``. Returns ------- The ``ScopedKey`` of the new ``AlchemicalNetwork`` in the database. """ - # Collect keyed chain representation for all alchemical networks - try: - network_keyed_chains: list[tuple[Scope, KeyedChain]] = [] - for network_scoped_key in network_scoped_keys: + # Collect keyed chain representation for all alchemical networks, + # gathering every missing ScopedKey up front so callers passing + # many SKs get a single, complete error. + network_keyed_chains: list[tuple[Scope, KeyedChain]] = [] + missing: list[ScopedKey] = [] + for network_scoped_key in network_scoped_keys: + try: keyed_chain = self.get_keyed_chain(network_scoped_key) - network_keyed_chains.append((network_scoped_key.scope, keyed_chain)) - # could not find specified network by provided scoped key - except KeyError: + except KeyError: + missing.append(network_scoped_key) + continue + network_keyed_chains.append((network_scoped_key.scope, keyed_chain)) + if missing: + joined = ", ".join(str(sk) for sk in missing) raise ValueError( - f"ScopedKey ({network_scoped_key}) not found in the database." + f"The following ScopedKey(s) were not found in the database: {joined}" ) # Map decoded Transformation / NonTransformation objects to all of @@ -1055,8 +1089,10 @@ def merge_networks( # source networks, and including duplicates introduced by minor # serialization-version drift). These original keys are needed to # locate Tasks associated with each Transformation in the source - # networks. - transformation_data: list[_TransformationData] = [] + # networks. We key the dedup map on ``transformation.key`` (the + # decoded GufeKey) to avoid O(N^2) GufeTokenizable equality checks + # for large networks. + transformation_data: dict[GufeKey, _TransformationData] = {} subchain_cache: dict[GufeTokenizable, KeyedChain] = {} for network_scope, network_keyed_chain in network_keyed_chains: # database keys for Transformations / NonTransformations in this @@ -1066,25 +1102,25 @@ def merge_networks( for gufe_key, keyed_dict in network_keyed_chain if _is_transformation_keyed_dict(keyed_dict) ] - # decoded Transformation / NonTransformation objects in the same - # order; decode_subchains shares a tokenizable_map across yields - # so common dependencies are decoded only once per source network + # decoded Transformation / NonTransformation objects, yielded + # by decode_subchains in the same chain order as the predicate + # selects above (gufe contract); decode_subchains also shares + # a tokenizable_map across yields so common dependencies are + # decoded only once per source network. transformations = network_keyed_chain.decode_subchains( _is_transformation_keyed_dict ) for database_key, transformation in zip(database_keys, transformations): subchain_cache[transformation] = KeyedChain.from_gufe(transformation) - try: - idx = transformation_data.index(transformation) - data = transformation_data[idx] - except ValueError: + data = transformation_data.get(transformation.key) + if data is None: data = _TransformationData(transformation) - transformation_data.append(data) + transformation_data[transformation.key] = data data.add_known_scoped_key(database_key, network_scope) # Collect all transformation gufe objects and collect into a new set of edges - _TransformationData.update_task_trees(transformation_data, self) - new_edges = [td.transformation for td in transformation_data] + _TransformationData.update_task_trees(list(transformation_data.values()), self) + new_edges = [td.transformation for td in transformation_data.values()] # Make new alchemiscale network with these edges combined_alchemical_network = AlchemicalNetwork(edges=new_edges, name=name) an_subgraph, an_node, an_sk = self._keyed_chain_to_subgraph( @@ -1092,11 +1128,11 @@ def merge_networks( ) # create and fold in taskhub and network mark supporting nodes an_subgraph |= ( - self.create_network_mark_subgraph(an_node)[0] + self.create_network_mark_subgraph(an_node, state=state)[0] | self.create_taskhub_subgraph(an_node)[0] ) # create and fold in all task and results data - for td in transformation_data: + for td in transformation_data.values(): an_subgraph |= td.to_subgraph(scope, self, subchain_cache) # merge the new network into neo4j diff --git a/alchemiscale/tests/integration/interface/client/test_client.py b/alchemiscale/tests/integration/interface/client/test_client.py index b8dcf289..1fa56b1a 100644 --- a/alchemiscale/tests/integration/interface/client/test_client.py +++ b/alchemiscale/tests/integration/interface/client/test_client.py @@ -248,6 +248,34 @@ def test_merge_networks( t.key for t in network_tyk2.edges } + @pytest.mark.parametrize("state", ["active", "inactive"]) + def test_merge_networks_respects_state( + self, + state, + scope_test, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + ): + """The state parameter must control the NetworkMark on the merged + AlchemicalNetwork the same way it does on create_network.""" + source_sks = user_client.query_networks(scope=scope_test, state=None) + assert source_sks + + merge_scope = Scope( + org=scope_test.org, + campaign=scope_test.campaign, + project=f"merged_state_{state}", + ) + merged_sk = user_client.merge_networks( + networks=source_sks, + name=f"merged_state_{state}", + scope=merge_scope, + state=state, + visualize=False, + ) + + assert user_client.get_network_state(merged_sk) == state + def test_merge_networks_rejects_wildcard_scope( self, n4js_preloaded, diff --git a/alchemiscale/tests/integration/interface/test_api.py b/alchemiscale/tests/integration/interface/test_api.py index 24be0be1..f1427147 100644 --- a/alchemiscale/tests/integration/interface/test_api.py +++ b/alchemiscale/tests/integration/interface/test_api.py @@ -147,7 +147,7 @@ def test_merge_networks( } def test_merge_networks_bad_scope( - self, n4js_preloaded, test_client, network_tyk2, scope_test, multiple_scopes + self, n4js_preloaded, test_client, scope_test, multiple_scopes ): # destination scope the test_client's token does not have access to bad_scope = multiple_scopes[1] @@ -171,7 +171,7 @@ def test_merge_networks_bad_scope( assert str(bad_scope) in details["detail"] def test_merge_networks_bad_source_scope( - self, n4js_preloaded, test_client, network_tyk2, scope_test, multiple_scopes + self, n4js_preloaded, test_client, scope_test, multiple_scopes ): # source network in a scope the test_client's token does not authorize unauth_scope = multiple_scopes[1] From 5a92fbcb357a048539a1899f5ccd4f0495a7a148 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Mon, 8 Jun 2026 18:07:22 -0600 Subject: [PATCH 19/35] Pin gufe to =1.8.0 across all conda env files merge_networks now depends on KeyedChain.decode_subchains (gufe 1.8.0). Pin all five env files to an exact 1.8.0 release, matching the repo's convention of pinning gufe exactly rather than with a lower bound. Touches the test, server, compute, client, and docs envs. Co-Authored-By: Claude Opus 4.7 (1M context) --- devtools/conda-envs/alchemiscale-client.yml | 2 +- devtools/conda-envs/alchemiscale-compute.yml | 2 +- devtools/conda-envs/alchemiscale-server.yml | 2 +- devtools/conda-envs/docs.yml | 2 +- devtools/conda-envs/test.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/devtools/conda-envs/alchemiscale-client.yml b/devtools/conda-envs/alchemiscale-client.yml index 6ef6d801..8a536682 100644 --- a/devtools/conda-envs/alchemiscale-client.yml +++ b/devtools/conda-envs/alchemiscale-client.yml @@ -8,7 +8,7 @@ dependencies: - cuda-version >=12 # alchemiscale dependencies - - gufe=1.6.1 + - gufe=1.8.0 - openfe=1.6.1 - requests - click diff --git a/devtools/conda-envs/alchemiscale-compute.yml b/devtools/conda-envs/alchemiscale-compute.yml index 92b9d4be..6d8cd255 100644 --- a/devtools/conda-envs/alchemiscale-compute.yml +++ b/devtools/conda-envs/alchemiscale-compute.yml @@ -8,7 +8,7 @@ dependencies: - cuda-version >=12 # alchemiscale dependencies - - gufe=1.6.1 + - gufe=1.8.0 - openfe=1.6.1 - requests - click diff --git a/devtools/conda-envs/alchemiscale-server.yml b/devtools/conda-envs/alchemiscale-server.yml index 298e5861..6fea2592 100644 --- a/devtools/conda-envs/alchemiscale-server.yml +++ b/devtools/conda-envs/alchemiscale-server.yml @@ -8,7 +8,7 @@ dependencies: - cuda-version >=12 # alchemiscale dependencies - - gufe>=1.8.0 + - gufe=1.8.0 - openfe=1.6.1 - requests - click diff --git a/devtools/conda-envs/docs.yml b/devtools/conda-envs/docs.yml index f080a0af..69f4c6d2 100644 --- a/devtools/conda-envs/docs.yml +++ b/devtools/conda-envs/docs.yml @@ -10,6 +10,6 @@ dependencies: - myst-parser>=0.14 - docutils - sphinx-notfound-page - - gufe=1.3.0 + - gufe=1.8.0 - py2neo - stratocaster diff --git a/devtools/conda-envs/test.yml b/devtools/conda-envs/test.yml index eed7fb35..f4f6b5e7 100644 --- a/devtools/conda-envs/test.yml +++ b/devtools/conda-envs/test.yml @@ -7,7 +7,7 @@ dependencies: - cuda-version >=12 # alchemiscale dependencies - - gufe >=1.8.0 + - gufe =1.8.0 - openfe =1.8.0 - pydantic >2 - pydantic-settings From b63474e477c181aa6780067bbf78c3d1f0fe3035 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Fri, 26 Jun 2026 10:53:48 -0600 Subject: [PATCH 20/35] Bump gufe / openfe / feflow to align with feflow 0.2.1 release feflow 0.2.1 hit conda-forge and pins gufe >=1.9.0,<2 plus openfe >=1.10.0,<2, which means our previous gufe=1.8.0 + openfe=1.8.0 + feflow>=0.1.4 combination is unsolvable: feflow 0.2.0 caps gufe at <1.8, and feflow 0.2.1 forces gufe>=1.9.0. Bump the trio everywhere to the latest mutually-compatible exact pins: - gufe = 1.10.0 (latest stable; satisfies openfe 1.11.1's gufe >=1.10.0,<1.11) - openfe = 1.11.1 (latest stable; satisfies feflow 0.2.1's openfe >=1.10.0,<2) - feflow = 0.2.1 (latest stable) merge_networks only needs KeyedChain.decode_subchains (gufe >=1.8.0), so the bump is API-safe. Apply across test, server, compute, and client env files. docs.yml gets the gufe bump only (no openfe/feflow there) and gains an explicit `python >=3.11,<3.13` lower bound because gufe 1.10.0 requires python >=3.11; without the lower bound, ReadTheDocs's mambaforge-4.10 resolved to a python too old for gufe and the docs build failed. Co-Authored-By: Claude Opus 4.7 (1M context) --- devtools/conda-envs/alchemiscale-client.yml | 6 +++--- devtools/conda-envs/alchemiscale-compute.yml | 6 +++--- devtools/conda-envs/alchemiscale-server.yml | 6 +++--- devtools/conda-envs/docs.yml | 4 ++-- devtools/conda-envs/test.yml | 6 +++--- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/devtools/conda-envs/alchemiscale-client.yml b/devtools/conda-envs/alchemiscale-client.yml index 8a536682..84e942c6 100644 --- a/devtools/conda-envs/alchemiscale-client.yml +++ b/devtools/conda-envs/alchemiscale-client.yml @@ -8,8 +8,8 @@ dependencies: - cuda-version >=12 # alchemiscale dependencies - - gufe=1.8.0 - - openfe=1.6.1 + - gufe=1.10.0 + - openfe=1.11.1 - requests - click - httpx @@ -26,7 +26,7 @@ dependencies: - nest-asyncio # openmm protocols - - feflow=0.1.4 + - feflow=0.2.1 # additional pins - openmm=8.2.0 diff --git a/devtools/conda-envs/alchemiscale-compute.yml b/devtools/conda-envs/alchemiscale-compute.yml index 6d8cd255..7fb4680b 100644 --- a/devtools/conda-envs/alchemiscale-compute.yml +++ b/devtools/conda-envs/alchemiscale-compute.yml @@ -8,8 +8,8 @@ dependencies: - cuda-version >=12 # alchemiscale dependencies - - gufe=1.8.0 - - openfe=1.6.1 + - gufe=1.10.0 + - openfe=1.11.1 - requests - click - httpx @@ -22,7 +22,7 @@ dependencies: - stratocaster # openmm protocols - - feflow=0.1.4 + - feflow=0.2.1 # additional pins - openmm=8.2.0 diff --git a/devtools/conda-envs/alchemiscale-server.yml b/devtools/conda-envs/alchemiscale-server.yml index 6fea2592..ca881dd9 100644 --- a/devtools/conda-envs/alchemiscale-server.yml +++ b/devtools/conda-envs/alchemiscale-server.yml @@ -8,8 +8,8 @@ dependencies: - cuda-version >=12 # alchemiscale dependencies - - gufe=1.8.0 - - openfe=1.6.1 + - gufe=1.10.0 + - openfe=1.11.1 - requests - click - pydantic >2 @@ -39,7 +39,7 @@ dependencies: - cryptography # openmm protocols - - feflow=0.1.4 + - feflow=0.2.1 # additional pins - openmm=8.2.0 diff --git a/devtools/conda-envs/docs.yml b/devtools/conda-envs/docs.yml index 69f4c6d2..9a605c6a 100644 --- a/devtools/conda-envs/docs.yml +++ b/devtools/conda-envs/docs.yml @@ -3,13 +3,13 @@ channels: - conda-forge dependencies: - - python<3.13 + - python >=3.11,<3.13 - sphinx - furo - myst-nb - myst-parser>=0.14 - docutils - sphinx-notfound-page - - gufe=1.8.0 + - gufe=1.10.0 - py2neo - stratocaster diff --git a/devtools/conda-envs/test.yml b/devtools/conda-envs/test.yml index f4f6b5e7..943c00d2 100644 --- a/devtools/conda-envs/test.yml +++ b/devtools/conda-envs/test.yml @@ -7,8 +7,8 @@ dependencies: - cuda-version >=12 # alchemiscale dependencies - - gufe =1.8.0 - - openfe =1.8.0 + - gufe =1.10.0 + - openfe =1.11.1 - pydantic >2 - pydantic-settings - async-lru @@ -41,7 +41,7 @@ dependencies: - cryptography # openmm protocols - - feflow>=0.1.4 + - feflow =0.2.1 ## cli - click From 61a9f04d57d600ccb46b32d69880a41c568bc488 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Fri, 26 Jun 2026 11:21:56 -0600 Subject: [PATCH 21/35] Fix merge_networks tests' scope auth; revert docs.yml gufe bump CI ran the env solve and tests cleanly with the gufe/openfe/feflow trio bump (b63474e), revealing two real bugs: 1. The four merge_networks client tests merged into destination scopes the test identity does not have access to (e.g. `test_org-test_campaign-merged_project`). The test identity only has the three scopes in `multiple_scopes` (scope_test plus two others); creating a "new project" under the same org/campaign was not, in fact, authorized. Switch the destination scope to one the identity has: - test_merge_networks: scope_test (collision-free because the merged network's name differs from pre-loaded networks) - test_merge_networks_respects_state: scope_test (each parametrize case uses a unique network name) - test_merge_networks_preserves_tasks_and_results: multiple_scopes[1] (different from where the source Tasks were set up, so the per-`_project` Task/PDRR counts stay clean) 2. The docs.yml gufe bump from 1.3.0 to 1.10.0 breaks the RTD build, because sphinx 9's autodoc dynamic-importer trips on `gufe/settings/models.py`: ph: PositiveFloat | None = Field(None, ...) ~~~~~~~~~~~~~~^~~~~~ TypeError: unsupported operand type(s) for |: 'PositiveFloat' and 'NoneType' gufe 1.3.0 still allows pydantic v1 (where PositiveFloat is a class with `__or__`); gufe >=1.8.0 forces pydantic v2 (where PositiveFloat is `Annotated[float, ...]`, no `__or__`). docs.yml only needs gufe for intersphinx and type-hint resolution; neither benefits from the bump. Revert docs.yml to its pre-PR state. Also polish the client merge_networks docstring with a `set_tasks_status` hint alongside `action_tasks` for the errored-Task remediation path. Co-Authored-By: Claude Opus 4.7 (1M context) --- alchemiscale/interface/client.py | 17 +++++----- .../interface/client/test_client.py | 31 +++++++++---------- devtools/conda-envs/docs.yml | 4 +-- 3 files changed, 23 insertions(+), 29 deletions(-) diff --git a/alchemiscale/interface/client.py b/alchemiscale/interface/client.py index c0a0a0a3..113e545e 100644 --- a/alchemiscale/interface/client.py +++ b/alchemiscale/interface/client.py @@ -204,13 +204,10 @@ def merge_networks( their associated ProtocolDAGResultRefs, so previously-computed results do not need to be re-run. - Cloned Tasks are wired to their Transformations via ``PERFORMS`` and - are reachable through standard network traversals - (``get_network_tasks``, ``get_network_results``, etc.). They are - intentionally **not** actioned to the new network's TaskHub; to - retry errored Tasks on the merged network, call - :meth:`action_tasks` with the merged network's ScopedKey after the - merge completes. + Cloned Tasks are intentionally **not** actioned; to retry errored Tasks + on the merged network, call :meth:`action_tasks` with the merged + network's ScopedKey after the merge completes, and set the Tasks' + status to back to `waiting` with :meth:`set_tasks_status`. Parameters ---------- @@ -224,11 +221,11 @@ def merge_networks( The Scope in which to create the new AlchemicalNetwork. This must be a *specific* Scope; it must not contain wildcards. state - The starting state of the new AlchemicalNetwork in the database. - See :meth:`AlchemiscaleClient.set_network_state` for valid states. + The starting state of the new AlchemicalNetwork. See + :meth:`AlchemiscaleClient.set_network_state` for valid states. Defaults to ``"active"``. visualize - If ``True``, show submission progress indicator. + If ``True``, show progress indicator. Returns ------- diff --git a/alchemiscale/tests/integration/interface/client/test_client.py b/alchemiscale/tests/integration/interface/client/test_client.py index 1fa56b1a..49fba9ef 100644 --- a/alchemiscale/tests/integration/interface/client/test_client.py +++ b/alchemiscale/tests/integration/interface/client/test_client.py @@ -213,12 +213,10 @@ def test_merge_networks( # "incomplete" in each of `multiple_scopes` source_sks = user_client.query_networks(state=None) - # destination scope: a new project under the existing org/campaign - merge_scope = Scope( - org=scope_test.org, - campaign=scope_test.campaign, - project="merged_project", - ) + # destination scope: reuse `scope_test`, which is authorized for + # the test identity. The merged network's name differs from the + # pre-loaded networks, so there is no _scoped_key collision. + merge_scope = scope_test merged_sk = user_client.merge_networks( networks=source_sks, @@ -261,11 +259,10 @@ def test_merge_networks_respects_state( source_sks = user_client.query_networks(scope=scope_test, state=None) assert source_sks - merge_scope = Scope( - org=scope_test.org, - campaign=scope_test.campaign, - project=f"merged_state_{state}", - ) + # destination scope: reuse `scope_test`, which is authorized for + # the test identity; the unique network name keeps each parametrize + # case from colliding on _scoped_key. + merge_scope = scope_test merged_sk = user_client.merge_networks( networks=source_sks, name=f"merged_state_{state}", @@ -324,6 +321,7 @@ def test_merge_networks_rejects_non_network_scoped_key( def test_merge_networks_preserves_tasks_and_results( self, scope_test, + multiple_scopes, n4js_preloaded, user_client: client.AlchemiscaleClient, network_tyk2, @@ -363,12 +361,11 @@ def test_merge_networks_preserves_tasks_and_results( ) n4js_preloaded.set_task_result(task_sks[1], err_pdrr) - # merge into a fresh project under the same org/campaign - merge_scope = Scope( - org=scope_test.org, - campaign=scope_test.campaign, - project="merged_with_results", - ) + # merge into a different authorized scope from where we set up the + # source Tasks, so the per-scope counts below remain clean (the + # destination scope has pre-loaded networks but no Tasks/PDRRs). + merge_scope = multiple_scopes[1] + assert merge_scope != scope_test merged_sk = user_client.merge_networks( networks=[source_sk], name="merged_with_results", diff --git a/devtools/conda-envs/docs.yml b/devtools/conda-envs/docs.yml index 9a605c6a..f080a0af 100644 --- a/devtools/conda-envs/docs.yml +++ b/devtools/conda-envs/docs.yml @@ -3,13 +3,13 @@ channels: - conda-forge dependencies: - - python >=3.11,<3.13 + - python<3.13 - sphinx - furo - myst-nb - myst-parser>=0.14 - docutils - sphinx-notfound-page - - gufe=1.10.0 + - gufe=1.3.0 - py2neo - stratocaster From e6b7aa660915a815066d0eefd81136bc74ade317 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Fri, 26 Jun 2026 11:54:41 -0600 Subject: [PATCH 22/35] Add copy_network, merge_scopes; fix PERFORMS reachability bug Closes the remaining surface area in #221 (copy_network and merge_scopes) while fixing a latent bug in merge_networks that the previous CI run surfaced via test_merge_networks_preserves_tasks_and_results. Root cause of the PERFORMS bug ------------------------------ In `storage/subgraph.py` `merge_subgraph` (lines 90-108), when two Python Node objects in the same Subgraph share a primary key, the UNWIND-MERGE Cypher query returns one Neo4j elementId for both, but the result-iteration loop assigns that elementId only to the first Python Node. The duplicate keeps `identity=None`, so any Relationship referencing it gets `start/end_node.identity = None` and is silently dropped at commit. merge_networks hit this directly: 1. The AlchemicalNetwork's KeyedChain produced one Transformation Node per edge in an_subgraph. 2. `_TransformationData.to_subgraph` produced a *second* Python Transformation Node from `subchain_cache` -- same `_scoped_key`, different Python object -- and attached PERFORMS to it. 3. After `an_subgraph |= to_subgraph_result`, merge_subgraph kept the AN-chain Node identified but left the to_subgraph Node with no identity. Every PERFORMS edge was silently dropped on commit. Fix: `to_subgraph` now takes the *existing* tf_node from the surrounding subgraph as a parameter. merge_networks (and the new copy_network) build a `{gufe_key: node}` lookup from `an_subgraph` after constructing it, and pass the matching tf_node into each call. This eliminates the `subchain_cache` entirely, which had been left as an open question from the earlier review round; `tf_node`'s purpose finally has teeth as the genuine PERFORMS anchor. task_statuses knob on _TransformationData.update_task_trees ----------------------------------------------------------- The previous query hard-filtered `WHERE task.status IN ["complete", "error"]`. That filter is right for merge_networks ("carry over results, not in-flight work") but wrong for copy_network, which the issue specifies as carrying over *all* Tasks. Add an optional `task_statuses: list[str] | None = None` parameter to drop the WHERE clause when None. merge_networks passes `["complete", "error"]` explicitly; copy_network passes None. copy_network ------------ - `Neo4jStore.copy_network(network_scoped_key, scope, name=None, state=NetworkStateEnum.active)`: decodes the source AN, optionally renames (preserving the gufe key when name is unchanged), clones every Task with its PDRRs and EXTENDS edges, and writes the new network + NetworkMark + TaskHub into the target scope. Reuses `_TransformationData` machinery verbatim, just with the task_statuses filter dropped. - `POST /networks/{network_scoped_key}/copy` endpoint validates the source ScopedKey, source scope, and destination scope against the caller's token, then delegates to the store. - `AlchemiscaleClient.copy_network(network, scope, name=None, state, visualize)` mirrors create_network / merge_networks shape. Cloned Tasks are intentionally **not** actioned to the new network's TaskHub, matching merge_networks; the client docstrings spell out the action_tasks remediation path for callers who want errored or waiting Tasks back in the compute queue. merge_scopes ------------ - `AlchemiscaleClient.merge_scopes(scopes, target_scope, visualize)` is a pure client-side loop: query each source scope for every AlchemicalNetwork, then copy_network each into target_scope. No new endpoint required. Tests ----- - test_copy_network: stages 4 Tasks (complete + ok PDRR, error + not-ok PDRR, waiting, running) on three Transformations of a source network; copies into a different scope; asserts that all four Tasks are reachable via the standard `get_network_tasks` traversal with statuses preserved, both PDRRs land in the target scope with obj_keys intact, and gufe key is preserved when name is unchanged. - test_copy_network_with_rename: confirms passing a new name produces a distinct gufe key. - test_copy_network_rejects_wildcard_scope / test_copy_network_rejects_non_network_scoped_key: client-side guards. - test_merge_scopes: copies every network in two source scopes into a third; asserts per-source ScopedKey count, target-scope ScopedKey collapse (preloaded fixtures use the same network names across scopes, so copies dedupe), and gufe-key preservation. - test_merge_scopes_rejects_empty / _rejects_wildcard_target: client-side guards. news/issue-221.rst expanded to list all three new methods plus the merge_networks-only complete|error filter semantic and the no-ACTIONS policy. Co-Authored-By: Claude Opus 4.7 (1M context) --- alchemiscale/interface/api.py | 46 ++++ alchemiscale/interface/client.py | 156 +++++++++++++ alchemiscale/storage/statestore.py | 217 +++++++++++++++--- .../interface/client/test_client.py | 216 +++++++++++++++++ news/issue-221.rst | 5 +- 5 files changed, 608 insertions(+), 32 deletions(-) diff --git a/alchemiscale/interface/api.py b/alchemiscale/interface/api.py index d5edf34f..9eab81f4 100644 --- a/alchemiscale/interface/api.py +++ b/alchemiscale/interface/api.py @@ -187,6 +187,52 @@ async def merge_networks( return an_sk +@router.post("/networks/{network_scoped_key}/copy", response_model=ScopedKey) +async def copy_network( + network_scoped_key, + *, + scope: dict = Body(embed=True), + name: str | None = Body(embed=True, default=None), + state: str = Body(embed=True, default=NetworkStateEnum.active.value), + n4js: Neo4jStore = Depends(get_n4js_depends), + token: TokenData = Depends(get_token_data_depends), +): + # validate the source ScopedKey + its scope + try: + source_sk = ScopedKey.from_str(network_scoped_key) + except ValueError as e: + raise HTTPException( + status_code=http_status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=e.args[0], + ) + validate_scopes(source_sk.scope, token) + + # validate the destination scope + try: + target_scope = Scope(**scope) + except (TypeError, ValidationError) as e: + raise HTTPException( + status_code=http_status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=str(e), + ) + validate_scopes(target_scope, token) + + try: + an_sk = n4js.copy_network( + network_scoped_key=source_sk, + scope=target_scope, + name=name, + state=state, + ) + except ValueError as e: + raise HTTPException( + status_code=http_status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=e.args[0], + ) + + return an_sk + + @router.post("/bulk/networks/state/set") def set_networks_state( *, diff --git a/alchemiscale/interface/client.py b/alchemiscale/interface/client.py index 113e545e..007dbcf0 100644 --- a/alchemiscale/interface/client.py +++ b/alchemiscale/interface/client.py @@ -281,6 +281,162 @@ def post(): return ScopedKey.from_dict(scoped_key) + def copy_network( + self, + network: ScopedKey, + scope: Scope, + name: str | None = None, + state: NetworkStateEnum | str = NetworkStateEnum.active, + visualize: bool = True, + ) -> ScopedKey: + """Copy an AlchemicalNetwork to a new Scope, carrying over every + existing Task (regardless of status) and its associated + ProtocolDAGResultRefs. + + Cloned Tasks are wired to their Transformations via ``PERFORMS`` and + are reachable through standard network traversals + (``get_network_tasks``, ``get_network_results``, etc.). They are + intentionally **not** actioned to the new network's TaskHub; to + pick up errored or waiting Tasks for execution on the merged + network, call :meth:`action_tasks` with the new network's + ``ScopedKey`` after the copy completes. + + Parameters + ---------- + network + The ScopedKey of the AlchemicalNetwork to copy. + scope + The destination Scope for the copy. This must be a *specific* + Scope; it must not contain wildcards. + name + Optional new name for the copy. If ``None``, the source + AlchemicalNetwork's name is preserved and the resulting copy + has the same gufe key as the source. If a different name is + given, the resulting copy gets a fresh gufe key derived from + the renamed content. + state + The starting state of the new AlchemicalNetwork. See + :meth:`AlchemiscaleClient.set_network_state` for valid states. + Defaults to ``"active"``. + visualize + If ``True``, show progress indicator. + + Returns + ------- + ScopedKey + The ScopedKey of the new, copied AlchemicalNetwork. + """ + if not scope.specific(): + raise ValueError( + f"`scope` '{scope}' contains wildcards ('*'); `scope` must be *specific*" + ) + + if not isinstance(network, ScopedKey): + network = ScopedKey.from_str(network) + if network.qualname != "AlchemicalNetwork": + raise ValueError( + f"ScopedKey '{network}' does not refer to an AlchemicalNetwork" + ) + + state = NetworkStateEnum(state) + + data = dict( + scope=scope.to_dict(), + name=name, + state=state.value, + ) + + def post(): + return self._post_resource(f"/networks/{network}/copy", data) + + if visualize: + from rich.progress import Progress + + with Progress(*self._rich_waiting_columns(), transient=False) as progress: + task = progress.add_task( + f"Copying [bold]'{network}'[/bold] into scope " + f"[bold]'{scope}'[/bold]...", + total=None, + ) + scoped_key = post() + progress.start_task(task) + progress.update(task, total=1, completed=1) + else: + scoped_key = post() + + return ScopedKey.from_dict(scoped_key) + + def merge_scopes( + self, + scopes: list[Scope], + target_scope: Scope, + visualize: bool = True, + ) -> list[ScopedKey]: + """Copy every AlchemicalNetwork in each of the given source Scopes + into ``target_scope``. + + Each source AlchemicalNetwork is copied via :meth:`copy_network`, + preserving its name, Tasks, and ProtocolDAGResultRefs. Cloned + Tasks are not actioned to the target network's TaskHub. + + Parameters + ---------- + scopes + The source Scopes to copy from. Each must be a *specific* + Scope (no wildcards). The caller must have access to each. + target_scope + The destination Scope. Must be a *specific* Scope. + visualize + If ``True``, show a progress bar tracking the number of + networks copied. + + Returns + ------- + list[ScopedKey] + The ScopedKeys of all newly-copied AlchemicalNetworks in + ``target_scope``, in the order they were copied. + """ + if not target_scope.specific(): + raise ValueError( + f"`target_scope` '{target_scope}' contains wildcards ('*'); " + "`target_scope` must be *specific*" + ) + if not scopes: + raise ValueError("`scopes` must contain at least one Scope") + for sc in scopes: + if not sc.specific(): + raise ValueError( + f"source scope '{sc}' contains wildcards ('*'); each source " + "scope must be *specific*" + ) + + # gather every AlchemicalNetwork across the source scopes up front, + # regardless of state, so the progress bar has an accurate total + network_sks: list[ScopedKey] = [] + for sc in scopes: + network_sks.extend(self.query_networks(scope=sc, state=None)) + + if visualize: + from rich.progress import Progress + + with Progress(*self._rich_waiting_columns(), transient=False) as progress: + task = progress.add_task( + f"Copying [bold]{len(network_sks)}[/bold] networks into " + f"[bold]'{target_scope}'[/bold]...", + total=len(network_sks), + ) + new_sks: list[ScopedKey] = [] + for sk in network_sks: + new_sks.append(self.copy_network(sk, target_scope, visualize=False)) + progress.update(task, advance=1) + else: + new_sks = [ + self.copy_network(sk, target_scope, visualize=False) + for sk in network_sks + ] + + return new_sks + def set_network_state( self, network: ScopedKey, state: NetworkStateEnum | str ) -> ScopedKey | None: diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index 5f94f83f..d2110d37 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -128,9 +128,27 @@ def add_known_scoped_key(self, key, scope): ) @staticmethod - def update_task_trees(transformation_data: list, statestore): + def update_task_trees( + transformation_data: list, + statestore, + task_statuses: list[str] | None = None, + ): """Given a list of ``_TransformationData``, extract all necessary info from Neo4j and load the task trees. + + Parameters + ---------- + transformation_data + List of ``_TransformationData`` to populate. + statestore + The ``Neo4jStore`` to query. + task_statuses + If provided, only ``Task``\\ s whose ``status`` is in this list + are loaded. If ``None`` (the default), ``Task``\\ s of any + status are loaded. ``merge_networks`` passes + ``["complete", "error"]`` to clone only successfully-completed + or errored work; ``copy_network`` passes ``None`` so the + entire ``Task`` tree of the source network is preserved. """ key_to_data_map = {str(td.transformation.key): td for td in transformation_data} # prepare for unwind clause, include transformation key @@ -141,26 +159,34 @@ def update_task_trees(transformation_data: list, statestore): for td in transformation_data for sk in td.known_scoped_keys ] - query = """ + # When task_statuses is provided, filter the Task match; otherwise + # include every Task that PERFORMs on one of the listed Transformations. + if task_statuses is None: + status_clause = "" + params = {"tf_sk_pairs": transformation_sk_pairs} + else: + status_clause = "WHERE task.status IN $task_statuses" + params = { + "tf_sk_pairs": transformation_sk_pairs, + "task_statuses": list(task_statuses), + } + query = f""" UNWIND $tf_sk_pairs as pairs WITH pairs[0] AS tf_key, pairs[1] AS tf_scoped_key - MATCH (task:Task)-[:PERFORMS]->(:Transformation|NonTransformation {`_scoped_key`: tf_scoped_key}) - WHERE task.status IN ["complete", "error"] + MATCH (task:Task)-[:PERFORMS]->(:Transformation|NonTransformation {{`_scoped_key`: tf_scoped_key}}) + {status_clause} OPTIONAL MATCH (task)-[:EXTENDS]->(extended_task:Task) OPTIONAL MATCH (task)-[:RESULTS_IN]->(pdrr:ProtocolDAGResultRef) RETURN tf_key, task, extended_task as extended_task, collect(pdrr) as pdrrs """ - results = statestore.execute_query( - query, tf_sk_pairs=transformation_sk_pairs - ).records + results = statestore.execute_query(query, **params).records for record in results: key_to_data_map[record["tf_key"]].task_tree.append(record) - def to_subgraph(self, target_scope, statestore, subchain_cache): - """Create a subgraph anchored at the Transformation node and - iteratively add Task and PDRR nodes with their corresponding - relationships. + def to_subgraph(self, target_scope, tf_node): + """Create a subgraph of cloned Tasks + EXTENDS + PERFORMS + RESULTS_IN + relationships, anchored at ``tf_node``. Each cloned Task is wired back to the Transformation via a ``PERFORMS`` edge so the standard @@ -168,23 +194,25 @@ def to_subgraph(self, target_scope, statestore, subchain_cache): traversal keeps working on the merged network. Note that cloned Tasks are intentionally **not** actioned to the merged network's TaskHub; see :meth:`Neo4jStore.merge_networks` for the rationale. - """ - # if there are no tasks, return an empty subgraph; the - # Transformation node already exists in the surrounding - # merged-network subgraph + + Parameters + ---------- + target_scope + The destination ``Scope`` to stamp on every cloned Task / PDRR. + tf_node + The existing Transformation ``Node`` from the surrounding + merged-network subgraph. Reusing the **same Python Node object** + (rather than constructing a fresh one with a matching + ``_scoped_key``) is required: :func:`merge_subgraph` assigns a + Neo4j ``elementId`` to only the first Python Node it sees per + primary key, so any Relationship pointing at a duplicate Node + object would be silently dropped on commit. + """ + # if there are no tasks, return an empty subgraph if not self.task_tree: return Subgraph() - # build the Transformation node as the PERFORMS anchor for cloned - # Tasks. ``subchain_cache`` holds the re-serialized chain for the - # decoded ``Transformation``; the resulting Node's ``_scoped_key`` - # matches the one already produced by the combined - # AlchemicalNetwork's keyed chain and will dedupe against it - # during ``merge_subgraph``. - _, tf_node, _ = statestore._keyed_chain_to_subgraph( - subchain_cache[self.transformation], target_scope - ) - subgraph = Subgraph() | tf_node + subgraph = Subgraph() scope_props = { "_org": target_scope.org, @@ -1048,7 +1076,8 @@ def merge_networks( Cloned ``Task``\\ s are intentionally **not** actioned to the new network's ``TaskHub``. Users wanting to retry errored tasks on the merged network should call :meth:`action_tasks` themselves - with the merged network's ``TaskHub`` ``ScopedKey``. + with the merged network's ``TaskHub`` ``ScopedKey`` and reset + the ``Task`` statuses back to `waiting`. Parameters ---------- @@ -1093,7 +1122,6 @@ def merge_networks( # decoded GufeKey) to avoid O(N^2) GufeTokenizable equality checks # for large networks. transformation_data: dict[GufeKey, _TransformationData] = {} - subchain_cache: dict[GufeTokenizable, KeyedChain] = {} for network_scope, network_keyed_chain in network_keyed_chains: # database keys for Transformations / NonTransformations in this # source network's chain, in chain order @@ -1111,15 +1139,21 @@ def merge_networks( _is_transformation_keyed_dict ) for database_key, transformation in zip(database_keys, transformations): - subchain_cache[transformation] = KeyedChain.from_gufe(transformation) data = transformation_data.get(transformation.key) if data is None: data = _TransformationData(transformation) transformation_data[transformation.key] = data data.add_known_scoped_key(database_key, network_scope) - # Collect all transformation gufe objects and collect into a new set of edges - _TransformationData.update_task_trees(list(transformation_data.values()), self) + # Collect all transformation gufe objects and collect into a new set of edges. + # ``merge_networks`` carries over only completed or errored Tasks so the + # new network reflects "results to keep"; in-flight or invalid Tasks are + # not preserved. + _TransformationData.update_task_trees( + list(transformation_data.values()), + self, + task_statuses=["complete", "error"], + ) new_edges = [td.transformation for td in transformation_data.values()] # Make new alchemiscale network with these edges combined_alchemical_network = AlchemicalNetwork(edges=new_edges, name=name) @@ -1131,15 +1165,136 @@ def merge_networks( self.create_network_mark_subgraph(an_node, state=state)[0] | self.create_taskhub_subgraph(an_node)[0] ) + + # Build a lookup from gufe key to the existing Transformation Node + # already present in ``an_subgraph``. ``to_subgraph`` must reuse + # these Python Node objects rather than building fresh ones with + # the same ``_scoped_key`` -- ``merge_subgraph`` assigns a Neo4j + # ``elementId`` only to the first Python Node it sees per primary + # key, so Relationships pointing at a duplicate Node would be + # silently dropped on commit. + tf_node_lookup = { + node["_gufe_key"]: node + for node in an_subgraph.nodes + if "Transformation" in node.labels or "NonTransformation" in node.labels + } + # create and fold in all task and results data for td in transformation_data.values(): - an_subgraph |= td.to_subgraph(scope, self, subchain_cache) + tf_node = tf_node_lookup[str(td.transformation.key)] + an_subgraph |= td.to_subgraph(scope, tf_node) # merge the new network into neo4j with self.transaction() as tx: merge_subgraph(tx, an_subgraph, "GufeTokenizable", "_scoped_key") return an_sk + def copy_network( + self, + network_scoped_key: ScopedKey, + scope: Scope, + name: str | None = None, + state: NetworkStateEnum | str = NetworkStateEnum.active, + ) -> ScopedKey: + """Copy an ``AlchemicalNetwork`` to a new ``Scope``, carrying over + every ``Task`` (regardless of status) and its associated + ``ProtocolDAGResultRef``\\ s. + + ``Task`` clones are wired to their ``Transformation`` via + ``PERFORMS`` and to one another via ``EXTENDS`` where applicable. + They are **not** actioned to the new network's ``TaskHub`` + (consistent with :meth:`merge_networks`); callers wanting cloned + ``Task``\\ s to be picked up by compute services should call + :meth:`action_tasks` against the new network's ``TaskHub``. + + Parameters + ---------- + network_scoped_key + The ``ScopedKey`` of the source ``AlchemicalNetwork``. + scope + The destination ``Scope`` for the copy. + name + Optional new name for the copy. If ``None``, the source + ``AlchemicalNetwork``'s name is preserved; the resulting copy + has the same gufe key as the source. If a different name is + given, the resulting copy has a fresh gufe key derived from + the renamed content. + state + Starting ``NetworkStateEnum`` for the copy's ``NetworkMark``. + Defaults to ``NetworkStateEnum.active``. + + Returns + ------- + The ``ScopedKey`` of the new ``AlchemicalNetwork`` in the database. + """ + # decode the source network's keyed chain + try: + source_kc = self.get_keyed_chain(network_scoped_key) + except KeyError: + raise ValueError( + f"ScopedKey ({network_scoped_key}) not found in the database." + ) + source_an: AlchemicalNetwork = source_kc.to_gufe() + + # optionally rename; preserving the name preserves the gufe key + if name is not None and name != source_an.name: + target_an = source_an.copy_with_replacements(name=name) + else: + target_an = source_an + + # gather Transformations and their source-scope database keys, + # then load every Task that PERFORMs on each (no status filter) + source_scope = network_scoped_key.scope + transformation_data: dict[GufeKey, _TransformationData] = {} + database_keys = [ + gufe_key + for gufe_key, keyed_dict in source_kc + if _is_transformation_keyed_dict(keyed_dict) + ] + transformations = source_kc.decode_subchains(_is_transformation_keyed_dict) + for database_key, transformation in zip(database_keys, transformations): + data = transformation_data.get(transformation.key) + if data is None: + data = _TransformationData(transformation) + transformation_data[transformation.key] = data + data.add_known_scoped_key(database_key, source_scope) + + # ``copy_network`` preserves the entire Task tree of the source + # network; pass ``task_statuses=None`` so the status filter is dropped. + _TransformationData.update_task_trees( + list(transformation_data.values()), + self, + task_statuses=None, + ) + + # build the new AlchemicalNetwork's subgraph + NetworkMark + TaskHub + an_subgraph, an_node, an_sk = self._keyed_chain_to_subgraph( + KeyedChain.from_gufe(target_an), scope + ) + an_subgraph |= ( + self.create_network_mark_subgraph(an_node, state=state)[0] + | self.create_taskhub_subgraph(an_node)[0] + ) + + # Look up the existing Transformation Nodes in ``an_subgraph`` so + # ``to_subgraph`` can reuse the same Python Node objects rather + # than building fresh duplicates; see :meth:`merge_networks` for + # why this matters. + tf_node_lookup = { + node["_gufe_key"]: node + for node in an_subgraph.nodes + if "Transformation" in node.labels or "NonTransformation" in node.labels + } + + # fold in cloned Tasks, EXTENDS, PERFORMS, and RESULTS_IN + for td in transformation_data.values(): + tf_node = tf_node_lookup[str(td.transformation.key)] + an_subgraph |= td.to_subgraph(scope, tf_node) + + with self.transaction() as tx: + merge_subgraph(tx, an_subgraph, "GufeTokenizable", "_scoped_key") + return an_sk + def get_network_state(self, networks: list[ScopedKey]) -> list[str | None]: """Get the states of a group of networks. diff --git a/alchemiscale/tests/integration/interface/client/test_client.py b/alchemiscale/tests/integration/interface/client/test_client.py index 49fba9ef..f80eaaf8 100644 --- a/alchemiscale/tests/integration/interface/client/test_client.py +++ b/alchemiscale/tests/integration/interface/client/test_client.py @@ -421,6 +421,222 @@ def test_merge_networks_preserves_tasks_and_results( statuses = sorted(user_client.get_tasks_status(merged_task_sks)) assert statuses == ["complete", "error"] + def test_copy_network( + self, + scope_test, + multiple_scopes, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + network_tyk2, + ): + """copy_network must duplicate the source AlchemicalNetwork into the + target scope and carry over Tasks of *every* status, with their + PDRRs, EXTENDS relationships, and PERFORMS wiring intact.""" + source_sks = user_client.query_networks(scope=scope_test, state=None) + assert source_sks + source_sk = source_sks[0] + source_an = user_client.get_network(source_sk) + + transformation_sks = n4js_preloaded.get_network_transformations(source_sk) + assert len(transformation_sks) >= 3 + + # Stage four tasks across three statuses so we can prove copy_network + # is not silently filtering by status the way merge_networks does: + # task 0: complete + ok PDRR + # task 1: error + not-ok PDRR + # task 2: waiting (no result) + # task 3: running (no result) + task_sks = n4js_preloaded.create_tasks(transformation_sks[:4]) + + n4js_preloaded.set_task_running(task_sks[:1]) + n4js_preloaded.set_task_complete(task_sks[:1]) + ok_pdrr = ProtocolDAGResultRef( + obj_key=f"ProtocolDAGResult-{uuid.uuid4()}", + scope=task_sks[0].scope, + ok=True, + ) + n4js_preloaded.set_task_result(task_sks[0], ok_pdrr) + + n4js_preloaded.set_task_running(task_sks[1:2]) + n4js_preloaded.set_task_error(task_sks[1:2]) + err_pdrr = ProtocolDAGResultRef( + obj_key=f"ProtocolDAGResult-{uuid.uuid4()}", + scope=task_sks[1].scope, + ok=False, + ) + n4js_preloaded.set_task_result(task_sks[1], err_pdrr) + + # task_sks[2] stays waiting (default after create_tasks) + n4js_preloaded.set_task_running(task_sks[3:4]) + + # copy into a scope distinct from the source, so per-_project counts + # are clean + target_scope = multiple_scopes[2] + assert target_scope != scope_test + copied_sk = user_client.copy_network( + network=source_sk, + scope=target_scope, + visualize=False, + ) + + # name is preserved → gufe key is preserved → only the scope changes + assert copied_sk.gufe_key == source_sk.gufe_key + assert copied_sk.scope == target_scope + assert user_client.check_exists(copied_sk) + assert user_client.get_network(copied_sk).name == source_an.name + + # every cloned Task must be reachable via the standard PERFORMS + # traversal from the new network + copied_task_sks = user_client.get_network_tasks(copied_sk) + assert len(copied_task_sks) == 4 + assert all(sk.scope == target_scope for sk in copied_task_sks) + statuses = sorted(user_client.get_tasks_status(copied_task_sks)) + assert statuses == ["complete", "error", "running", "waiting"] + + # both PDRRs land in the target scope with their object keys intact + pdrr_records = n4js_preloaded.execute_query( + """ + MATCH (pdrr:ProtocolDAGResultRef {`_project`: $project}) + RETURN pdrr.ok AS ok, pdrr.obj_key AS obj_key + """, + project=target_scope.project, + ).records + assert len(pdrr_records) == 2 + assert sorted(r["ok"] for r in pdrr_records) == [False, True] + assert {r["obj_key"] for r in pdrr_records} == { + ok_pdrr.obj_key, + err_pdrr.obj_key, + } + + def test_copy_network_with_rename( + self, + scope_test, + multiple_scopes, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + ): + """copy_network with a non-default `name` must produce a distinct + gufe key while still copying into the target scope.""" + source_sks = user_client.query_networks(scope=scope_test, state=None) + source_sk = source_sks[0] + source_an = user_client.get_network(source_sk) + + target_scope = multiple_scopes[2] + new_name = source_an.name + "_renamed_copy" + + copied_sk = user_client.copy_network( + network=source_sk, + scope=target_scope, + name=new_name, + visualize=False, + ) + + assert copied_sk.gufe_key != source_sk.gufe_key + assert copied_sk.scope == target_scope + copied_an = user_client.get_network(copied_sk) + assert copied_an.name == new_name + + def test_copy_network_rejects_wildcard_scope( + self, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + ): + source_sks = user_client.query_networks(state=None) + with pytest.raises(ValueError, match="wildcards"): + user_client.copy_network( + network=source_sks[0], + scope=Scope(org="test_org"), + visualize=False, + ) + + def test_copy_network_rejects_non_network_scoped_key( + self, + scope_test, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + ): + tf_sks = user_client.query_transformations(scope=scope_test) + assert tf_sks + with pytest.raises(ValueError, match="does not refer to an AlchemicalNetwork"): + user_client.copy_network( + network=tf_sks[0], + scope=scope_test, + visualize=False, + ) + + def test_merge_scopes( + self, + scope_test, + multiple_scopes, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + ): + """merge_scopes must copy every AlchemicalNetwork in the listed + source scopes into the target scope, preserving names (and + therefore gufe keys).""" + source_scopes = [scope_test, multiple_scopes[1]] + target_scope = multiple_scopes[2] + + # baseline: 2 networks pre-loaded in target_scope + baseline_target_sks = user_client.query_networks(scope=target_scope, state=None) + assert len(baseline_target_sks) == 2 + + expected_source_count = sum( + len(user_client.query_networks(scope=sc, state=None)) + for sc in source_scopes + ) + # 2 source networks per scope, 2 source scopes + assert expected_source_count == 4 + + new_sks = user_client.merge_scopes( + scopes=source_scopes, + target_scope=target_scope, + visualize=False, + ) + + # one returned ScopedKey per source network + assert len(new_sks) == expected_source_count + assert all(sk.scope == target_scope for sk in new_sks) + + # every copy preserves the source network's gufe key, so each copy's + # ScopedKey is just (source_gufe_key, target_scope). Since the source + # scopes contain the same set of networks (preloaded with the same + # names), the returned ScopedKeys collapse to the same 2 unique keys. + assert len(set(new_sks)) == 2 + + # target scope ends up with the union of its baseline networks and + # the copied networks; baseline and copies share the same gufe keys + # (assemble_network preloaded the same names everywhere), so the + # final count is unchanged from baseline + final_target_sks = user_client.query_networks(scope=target_scope, state=None) + assert set(baseline_target_sks) == set(final_target_sks) + + def test_merge_scopes_rejects_empty( + self, + scope_test, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + ): + with pytest.raises(ValueError, match="at least one"): + user_client.merge_scopes( + scopes=[], + target_scope=scope_test, + visualize=False, + ) + + def test_merge_scopes_rejects_wildcard_target( + self, + scope_test, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + ): + with pytest.raises(ValueError, match="wildcards"): + user_client.merge_scopes( + scopes=[scope_test], + target_scope=Scope(org="test_org"), + visualize=False, + ) + def test_check_exists( self, scope_test, diff --git a/news/issue-221.rst b/news/issue-221.rst index 75a23c91..55259f4a 100644 --- a/news/issue-221.rst +++ b/news/issue-221.rst @@ -1,6 +1,9 @@ **Added:** -* ``AlchemiscaleClient.merge_networks`` for combining multiple existing ``AlchemicalNetwork``\s into a new ``AlchemicalNetwork``, preserving completed and errored ``Task`` results so they do not need to be re-run. Backed by ``Neo4jStore.merge_networks`` and a new ``POST /networks/merge`` endpoint on the user API. +* ``AlchemiscaleClient.merge_networks`` for combining multiple existing ``AlchemicalNetwork``\s into a new ``AlchemicalNetwork`` in a target ``Scope``. ``Task``\s in ``complete`` or ``error`` state on the source networks are cloned into the new network along with their ``ProtocolDAGResultRef``\s so previously-computed results do not need to be re-run; ``Task``\s in other statuses (``waiting``, ``running``, ``invalid``, ``deleted``) are not carried over. Backed by ``Neo4jStore.merge_networks`` and a new ``POST /networks/merge`` endpoint on the user API. +* ``AlchemiscaleClient.copy_network`` for copying a single ``AlchemicalNetwork`` to a new ``Scope``, with all of its ``Task``\s (regardless of status) and their ``ProtocolDAGResultRef``\s carried over. An optional ``name`` argument creates the copy under a new name (and therefore a new gufe key); when ``name`` is omitted, the source network's name and gufe key are preserved. Backed by ``Neo4jStore.copy_network`` and a new ``POST /networks/{network_scoped_key}/copy`` endpoint. +* ``AlchemiscaleClient.merge_scopes`` for copying every ``AlchemicalNetwork`` in a set of source ``Scope``\s into a single target ``Scope``, via per-network ``copy_network`` calls. +* For all three methods, cloned ``Task``\s are wired to their ``Transformation``\s via ``PERFORMS`` so the standard ``(:AlchemicalNetwork)-[:DEPENDS_ON]->(:Transformation)<-[:PERFORMS]-(:Task)`` traversal sees them on the new network. Cloned ``Task``\s are intentionally **not** actioned to the new network's ``TaskHub``; callers wanting cloned ``Task``\s picked up by compute services should call ``action_tasks`` against the new network's ``TaskHub`` after the copy completes. **Changed:** From 6192d0889311521818d55dadd9aec75ac6d1d9c6 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Fri, 26 Jun 2026 18:55:52 -0600 Subject: [PATCH 23/35] Fix inverted EXTENDS direction in _TransformationData.to_subgraph The previous to_subgraph emitted Relationship.type("EXTENDS")(etask_node, task_node, ...) which creates ``(etask)-[:EXTENDS]->(task)``, i.e. "extended_task extends task" -- the opposite of what the source graph encoded: OPTIONAL MATCH (task)-[:EXTENDS]->(extended_task:Task) and what every other place in the store writes (``Neo4jStore.create_tasks`` at line 3424: ``Relationship.type("EXTENDS")(task_node, extends_task_node, ...)``) and reads (lines 2392, 3152, 3547, 3634, 4219, all traversing ``(task)-[:EXTENDS]->(other_task)``). No existing test exercised the cloned EXTENDS direction (the store- level test_merge_networks only counted PDRRs, and my own client tests used independent Tasks with no EXTENDS chain), so the bug shipped silently as part of the original PR's inline to_subgraph. Swap the argument order and add a focused regression test: - test_copy_network_preserves_extends_direction stages a base complete Task plus an extending complete Task on the same Transformation (using ``create_tasks(transformation_sks, extends=base_task_sks)``), copies the source network into a different scope, then asserts via raw Cypher that exactly one EXTENDS edge exists in the target scope with ``extender == extending_task.gufe_key`` and ``extended == base_task.gufe_key`` -- i.e. the direction matches the source. Co-Authored-By: Claude Opus 4.7 (1M context) --- alchemiscale/interface/client.py | 14 ++--- alchemiscale/storage/statestore.py | 10 +++- .../interface/client/test_client.py | 60 +++++++++++++++++++ 3 files changed, 74 insertions(+), 10 deletions(-) diff --git a/alchemiscale/interface/client.py b/alchemiscale/interface/client.py index 007dbcf0..0467a231 100644 --- a/alchemiscale/interface/client.py +++ b/alchemiscale/interface/client.py @@ -293,13 +293,11 @@ def copy_network( existing Task (regardless of status) and its associated ProtocolDAGResultRefs. - Cloned Tasks are wired to their Transformations via ``PERFORMS`` and - are reachable through standard network traversals - (``get_network_tasks``, ``get_network_results``, etc.). They are - intentionally **not** actioned to the new network's TaskHub; to - pick up errored or waiting Tasks for execution on the merged - network, call :meth:`action_tasks` with the new network's - ``ScopedKey`` after the copy completes. + Cloned Tasks are intentionally **not** actioned; to pick up errored or + waiting Tasks for execution on the merged network, call + :meth:`action_tasks` with the new network's ``ScopedKey`` after the + copy completes, and set the Tasks' status to back to `waiting` with + :meth:`set_tasks_status`. Parameters ---------- @@ -377,7 +375,7 @@ def merge_scopes( Each source AlchemicalNetwork is copied via :meth:`copy_network`, preserving its name, Tasks, and ProtocolDAGResultRefs. Cloned - Tasks are not actioned to the target network's TaskHub. + Tasks are not actioned. Parameters ---------- diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index d2110d37..473a48f8 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -238,7 +238,13 @@ def record_to_node(record): # this edge the Task is unreachable from get_network_tasks # and every other PERFORMS-based traversal subgraph |= Relationship.type("PERFORMS")(task_node, tf_node, **scope_props) - # create the task node this task extends if it exists + # create the task node this task extends if it exists. The + # source query returns ``record["extended_task"]`` as the Task + # that ``record["task"]`` extends -- i.e. in the source graph, + # ``(task)-[:EXTENDS]->(extended_task)``. The relationship's + # direction must be preserved on the clone so the standard + # downstream traversals (e.g. lines 2392, 3152, 3547, 3634: + # ``(task)-[:EXTENDS]->(other_task)``) keep working. etask_node = ( None if not record["extended_task"] @@ -246,7 +252,7 @@ def record_to_node(record): ) if etask_node: subgraph |= Relationship.type("EXTENDS")( - etask_node, task_node, **scope_props + task_node, etask_node, **scope_props ) # clone all result refs for the task for pdrr_record in record["pdrrs"]: diff --git a/alchemiscale/tests/integration/interface/client/test_client.py b/alchemiscale/tests/integration/interface/client/test_client.py index f80eaaf8..8d46d9bb 100644 --- a/alchemiscale/tests/integration/interface/client/test_client.py +++ b/alchemiscale/tests/integration/interface/client/test_client.py @@ -536,6 +536,66 @@ def test_copy_network_with_rename( copied_an = user_client.get_network(copied_sk) assert copied_an.name == new_name + def test_copy_network_preserves_extends_direction( + self, + scope_test, + multiple_scopes, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + ): + """The cloned EXTENDS edge must match the source direction: + ``(task)-[:EXTENDS]->(extended_task)``. Regressing this direction + would silently invert every EXTENDS chain on copied networks and + break the downstream traversals that read it forward (e.g. + ``get_task_extends``, ``get_task_results``).""" + source_sks = user_client.query_networks(scope=scope_test, state=None) + source_sk = source_sks[0] + + transformation_sks = n4js_preloaded.get_network_transformations(source_sk) + assert transformation_sks + + # base Task: complete with ok PDRR + base_task_sks = n4js_preloaded.create_tasks(transformation_sks[:1]) + n4js_preloaded.set_task_running(base_task_sks) + n4js_preloaded.set_task_complete(base_task_sks) + base_pdrr = ProtocolDAGResultRef( + obj_key=f"ProtocolDAGResult-{uuid.uuid4()}", + scope=base_task_sks[0].scope, + ok=True, + ) + n4js_preloaded.set_task_result(base_task_sks[0], base_pdrr) + + # extending Task: extends the base, also complete; the EXTENDS edge + # in the source graph is ``(extending)-[:EXTENDS]->(base)`` + extending_task_sks = n4js_preloaded.create_tasks( + transformation_sks[:1], extends=base_task_sks + ) + n4js_preloaded.set_task_running(extending_task_sks) + n4js_preloaded.set_task_complete(extending_task_sks) + ext_pdrr = ProtocolDAGResultRef( + obj_key=f"ProtocolDAGResult-{uuid.uuid4()}", + scope=extending_task_sks[0].scope, + ok=True, + ) + n4js_preloaded.set_task_result(extending_task_sks[0], ext_pdrr) + + target_scope = multiple_scopes[2] + user_client.copy_network(network=source_sk, scope=target_scope, visualize=False) + + # exactly one EXTENDS edge between the two cloned Tasks must exist + # in the target scope, with the same direction as the source: + # extending -> EXTENDS -> base + edges = n4js_preloaded.execute_query( + """ + MATCH (a:Task {`_project`: $project})-[:EXTENDS]->(b:Task {`_project`: $project}) + RETURN a._gufe_key AS extender, b._gufe_key AS extended + """, + project=target_scope.project, + ).records + assert len(edges) == 1 + assert edges[0]["extender"] == str(extending_task_sks[0].gufe_key) + assert edges[0]["extended"] == str(base_task_sks[0].gufe_key) + def test_copy_network_rejects_wildcard_scope( self, n4js_preloaded, From f69f36f9bc462575e8b19dc820de495449339b28 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Fri, 26 Jun 2026 19:17:50 -0600 Subject: [PATCH 24/35] Memoize Task Nodes in to_subgraph so EXTENDS endpoints share identity The previous fix landed the correct EXTENDS direction but the test_copy_network_preserves_extends_direction regression test still failed (assert 0 == 1) -- the cypher MATCH returned no EXTENDS edges at all on the clone. Same Node-deduplication trap as the AN-chain PERFORMS bug, this time intra-to_subgraph. When ``record[i]["task"]`` is the same Task as ``record[j]["extended_task"]`` (the common case: an extending Task's target also appears as its own complete/error record), the previous code created two separate Python ``Node`` objects from ``record_to_node``: one when processing ``record[i]`` (named ``task_node``), one when processing ``record[j]`` (named ``etask_node``). Both have the same ``_scoped_key``. ``merge_subgraph`` assigns a Neo4j ``elementId`` only to the first Python Node it sees per primary key, so the EXTENDS Relationship's ``end_node.identity`` was ``None`` and the edge was silently dropped on commit. Memoize Task Nodes by their gufe key inside to_subgraph via ``task_node_cache: dict[str, Node]`` and a small ``get_or_create_task_node`` helper; every reference to the same Task across iterations now reuses the same Python Node and lands a real Neo4j identity. The cache is local to a single ``to_subgraph`` call so it does not leak across Transformations. Co-Authored-By: Claude Opus 4.7 (1M context) --- alchemiscale/storage/statestore.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index 473a48f8..40caafef 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -230,10 +230,28 @@ def record_to_node(record): **record._properties | scope_props | {"_scoped_key": str(scoped_key)}, ) + # Memoize Task Nodes by gufe key across iterations of ``task_tree``. + # If ``record[i]["task"]`` is the same Task as + # ``record[j]["extended_task"]``, we must reuse the same Python + # Node object -- otherwise ``merge_subgraph`` assigns a Neo4j + # ``elementId`` only to the first Python Node it sees per primary + # key and the EXTENDS Relationship's endpoint (pointing at the + # duplicate) is silently dropped on commit. Same bug pattern as + # the AN's Transformation Node vs. the to_subgraph anchor. + task_node_cache: dict[str, Node] = {} + + def get_or_create_task_node(record): + gufe_key = record["_gufe_key"] + node = task_node_cache.get(gufe_key) + if node is None: + node = record_to_node(record) + task_node_cache[gufe_key] = node + return node + # process each task found. Each record represents a single task. for record in self.task_tree: # update task node to have new scoped key - task_node = record_to_node(record["task"]) + task_node = get_or_create_task_node(record["task"]) # wire the cloned Task back to its Transformation; without # this edge the Task is unreachable from get_network_tasks # and every other PERFORMS-based traversal @@ -248,7 +266,7 @@ def record_to_node(record): etask_node = ( None if not record["extended_task"] - else record_to_node(record["extended_task"]) + else get_or_create_task_node(record["extended_task"]) ) if etask_node: subgraph |= Relationship.type("EXTENDS")( From 128ef9ed5da4c1083e5bd3831a6f02b06c5f83a1 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Thu, 2 Jul 2026 16:31:33 -0600 Subject: [PATCH 25/35] Document Strategy and TaskRestartPattern non-preservation Neither merge_networks nor copy_network carries over the source network's execution-orchestration state -- ``Strategy`` (via the ``PROGRESSES`` relationship on the source AN) and ``TaskRestartPattern``\s (via ``ENFORCES`` on the source TaskHub and ``APPLIES`` on individual Tasks). Both create a fresh empty TaskHub via ``create_taskhub_subgraph`` and stop there. This is intentional: Strategy and TRPs govern *how* Tasks run (compute planning, retry policy) rather than the results themselves, which are what merge_networks and copy_network are designed to preserve. Merge_networks in particular has no natural resolution across N sources' Strategies or TRPs; copy_network is skipping them by design for consistency. Callers wanting either on the new network should set them explicitly after the merge/copy via ``AlchemiscaleClient.set_network_strategy`` and ``AlchemiscaleClient.add_task_restart_patterns``. Adds the policy statement to: - ``Neo4jStore.merge_networks`` docstring - ``Neo4jStore.copy_network`` docstring - ``AlchemiscaleClient.merge_networks`` docstring - ``AlchemiscaleClient.copy_network`` docstring - news/issue-221.rst (as an additional bullet under Added) No code change; no tests added, since the behavior is "does not do X" and is already exercised implicitly by the existing tests (which would fail if PROGRESSES/ENFORCES/APPLIES were being silently duplicated across scopes). Co-Authored-By: Claude Opus 4.7 (1M context) --- alchemiscale/interface/client.py | 24 +++++++++++++++++++++++- alchemiscale/storage/statestore.py | 24 ++++++++++++++++++++++++ news/issue-221.rst | 1 + 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/alchemiscale/interface/client.py b/alchemiscale/interface/client.py index 0467a231..a7edfd19 100644 --- a/alchemiscale/interface/client.py +++ b/alchemiscale/interface/client.py @@ -209,6 +209,17 @@ def merge_networks( network's ScopedKey after the merge completes, and set the Tasks' status to back to `waiting` with :meth:`set_tasks_status`. + The source networks' execution-orchestration state is intentionally + **not** carried over -- these govern how Tasks run rather than the + results themselves. Specifically: + + - Any ``Strategy`` set on a source network via + :meth:`set_network_strategy` is not copied. + - Any ``TaskRestartPattern`` s added to a source network's TaskHub + via :meth:`add_task_restart_patterns` are not copied. + + Set these on the merged network yourself after the merge if needed. + Parameters ---------- networks @@ -296,9 +307,20 @@ def copy_network( Cloned Tasks are intentionally **not** actioned; to pick up errored or waiting Tasks for execution on the merged network, call :meth:`action_tasks` with the new network's ``ScopedKey`` after the - copy completes, and set the Tasks' status to back to `waiting` with + copy completes, and set errored Tasks' status to back to `waiting` with :meth:`set_tasks_status`. + The source network's execution-orchestration state is intentionally + **not** carried over -- these govern how Tasks run rather than the + results themselves. Specifically: + + - Any ``Strategy`` set on the source network via + :meth:`set_network_strategy` is not copied. + - Any ``TaskRestartPattern`` s added to the source network's TaskHub + via :meth:`add_task_restart_patterns` are not copied. + + Set these on the copy yourself after :meth:`copy_network` if needed. + Parameters ---------- network diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index 40caafef..c64a3f7d 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -1103,6 +1103,18 @@ def merge_networks( with the merged network's ``TaskHub`` ``ScopedKey`` and reset the ``Task`` statuses back to `waiting`. + Execution-orchestration state on the source networks is + intentionally **not** carried over -- these govern *how* Tasks + run rather than the results themselves: + + - ``Strategy`` (``PROGRESSES`` relationship on the source AN) + - ``TaskRestartPattern``\\ s (``ENFORCES`` on the source + ``TaskHub``, ``APPLIES`` on individual ``Task``\\ s) + + Callers wanting either on the merged network should set them + explicitly after the merge via :meth:`set_network_strategy` and + :meth:`add_task_restart_patterns`. + Parameters ---------- network_scoped_keys @@ -1231,6 +1243,18 @@ def copy_network( ``Task``\\ s to be picked up by compute services should call :meth:`action_tasks` against the new network's ``TaskHub``. + Execution-orchestration state on the source network is + intentionally **not** carried over -- these govern *how* Tasks + run rather than the results themselves: + + - ``Strategy`` (``PROGRESSES`` relationship on the source AN) + - ``TaskRestartPattern``\\ s (``ENFORCES`` on the source + ``TaskHub``, ``APPLIES`` on individual ``Task``\\ s) + + Callers wanting either on the copy should set them explicitly + after :meth:`copy_network` via :meth:`set_network_strategy` and + :meth:`add_task_restart_patterns`. + Parameters ---------- network_scoped_key diff --git a/news/issue-221.rst b/news/issue-221.rst index 55259f4a..4f0e855c 100644 --- a/news/issue-221.rst +++ b/news/issue-221.rst @@ -4,6 +4,7 @@ * ``AlchemiscaleClient.copy_network`` for copying a single ``AlchemicalNetwork`` to a new ``Scope``, with all of its ``Task``\s (regardless of status) and their ``ProtocolDAGResultRef``\s carried over. An optional ``name`` argument creates the copy under a new name (and therefore a new gufe key); when ``name`` is omitted, the source network's name and gufe key are preserved. Backed by ``Neo4jStore.copy_network`` and a new ``POST /networks/{network_scoped_key}/copy`` endpoint. * ``AlchemiscaleClient.merge_scopes`` for copying every ``AlchemicalNetwork`` in a set of source ``Scope``\s into a single target ``Scope``, via per-network ``copy_network`` calls. * For all three methods, cloned ``Task``\s are wired to their ``Transformation``\s via ``PERFORMS`` so the standard ``(:AlchemicalNetwork)-[:DEPENDS_ON]->(:Transformation)<-[:PERFORMS]-(:Task)`` traversal sees them on the new network. Cloned ``Task``\s are intentionally **not** actioned to the new network's ``TaskHub``; callers wanting cloned ``Task``\s picked up by compute services should call ``action_tasks`` against the new network's ``TaskHub`` after the copy completes. +* The source networks' execution-orchestration state -- any attached ``Strategy`` (``AlchemiscaleClient.set_network_strategy``) and any ``TaskRestartPattern``\s on the ``TaskHub`` (``AlchemiscaleClient.add_task_restart_patterns``) -- is intentionally **not** carried over by any of the three new methods, since these govern how ``Task``\s run rather than the results themselves. Set them explicitly on the new network afterward if needed. **Changed:** From a9b89df5ff9cce1c11c634a2d9b055f3da83574f Mon Sep 17 00:00:00 2001 From: David Dotson Date: Sun, 5 Jul 2026 15:59:35 -0600 Subject: [PATCH 26/35] Document merge_networks, copy_network, merge_scopes in user guide Adds a "Merging and copying AlchemicalNetworks" section to getting_started.rst covering the three new client methods, their Task carry-over semantics, and the non-preservation policy for actioning, Strategy, and TaskRestartPatterns. Co-Authored-By: Claude Opus 4.7 --- docs/user_guide/getting_started.rst | 88 +++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/docs/user_guide/getting_started.rst b/docs/user_guide/getting_started.rst index f4460887..fae08884 100644 --- a/docs/user_guide/getting_started.rst +++ b/docs/user_guide/getting_started.rst @@ -167,6 +167,94 @@ You can list all your accessible ``AlchemicalNetworks`` on the ``alchemiscale`` and you can use these with :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.get_network` above to pull them down as desired. +Merging and copying AlchemicalNetworks +====================================== + +Once you have :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s submitted, you may want to combine several of them into a single :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`, or move an :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` (or a whole :py:class:`~alchemiscale.models.Scope` of them) into a different :py:class:`~alchemiscale.models.Scope`. +Three :py:class:`~alchemiscale.interface.client.AlchemiscaleClient` methods support these workflows: + +* :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_networks` — combine several :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s into a single new one. +* :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.copy_network` — copy one :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` into a different :py:class:`~alchemiscale.models.Scope`. +* :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_scopes` — copy every :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` from one or more source :py:class:`~alchemiscale.models.Scope`\s into a single target :py:class:`~alchemiscale.models.Scope`. + +The destination :py:class:`~alchemiscale.models.Scope` for each of these must be a *specific* :py:class:`~alchemiscale.models.Scope` (no wildcards), and your user must have permissions on the source and destination :py:class:`~alchemiscale.models.Scope`\s. + +.. note:: + + For all three methods, the following execution-orchestration state is intentionally **not** carried over from the source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s to the new one, since these govern *how* :py:class:`~alchemiscale.storage.models.Task`\s run rather than the results themselves: + + * Any :py:class:`~alchemiscale.storage.models.Task`\s that were previously *actioned* on a source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` are not automatically actioned on the new one; call :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.action_tasks` on the new network afterward for any :py:class:`~alchemiscale.storage.models.Task`\s you want compute services to pick up. + * Any ``Strategy`` set via :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.set_network_strategy` is not copied. + * Any ``TaskRestartPattern``\s added via :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.add_task_restart_patterns` are not copied. + + Set these on the new :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` yourself after the merge or copy if you need them. + + +Merging multiple AlchemicalNetworks +----------------------------------- + +Use :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_networks` to combine multiple :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s into a single new :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` in the target :py:class:`~alchemiscale.models.Scope`:: + + >>> merged_sk = asc.merge_networks( + ... networks=[an_sk_a, an_sk_b, an_sk_c], + ... name='combined_campaign', + ... scope=Scope('my_org', 'my_campaign', 'combined_project'), + ... ) + >>> merged_sk + + +The new :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` contains the union of the :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s and ``NonTransformation``\s from every source network. +The source networks themselves are unchanged. + +Existing :py:class:`~alchemiscale.storage.models.Task`\s attached to those source :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s that are in ``complete`` or ``error`` state are cloned onto the merged network's :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s along with their :py:class:`~alchemiscale.storage.models.ProtocolDAGResultRef`\s, so previously-computed results do not need to be re-run. +:py:class:`~alchemiscale.storage.models.Task`\s in other states (``waiting``, ``running``, ``invalid``, ``deleted``) are **not** carried over. + +To retry the cloned ``error`` :py:class:`~alchemiscale.storage.models.Task`\s on the merged network, first set their status back to ``waiting`` with :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.set_tasks_status`, then :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.action_tasks` them:: + + >>> errored = asc.get_network_tasks(merged_sk, status='error') + >>> asc.set_tasks_status(errored, 'waiting') + >>> asc.action_tasks(errored, merged_sk) + + +Copying a single AlchemicalNetwork +---------------------------------- + +Use :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.copy_network` to copy one :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` into a different :py:class:`~alchemiscale.models.Scope`:: + + >>> copy_sk = asc.copy_network( + ... network=an_sk, + ... scope=Scope('my_org', 'my_campaign', 'shared_project'), + ... ) + +Unlike :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_networks`, :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.copy_network` carries over **every** existing :py:class:`~alchemiscale.storage.models.Task` for the source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\'s :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s, regardless of status, together with their :py:class:`~alchemiscale.storage.models.ProtocolDAGResultRef`\s. + +If ``name`` is not given, the source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\'s name is preserved and the copy has the same ``gufe.key`` as the source (so its :py:class:`~alchemiscale.models.ScopedKey` differs from the source's only in :py:class:`~alchemiscale.models.Scope`). +Pass ``name`` to rename the copy; this yields a fresh ``gufe.key`` derived from the renamed content:: + + >>> renamed_copy_sk = asc.copy_network( + ... network=an_sk, + ... scope=Scope('my_org', 'my_campaign', 'shared_project'), + ... name='shared_snapshot', + ... ) + + +Copying every AlchemicalNetwork from one Scope to another +--------------------------------------------------------- + +Use :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_scopes` to copy every :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` in one or more source :py:class:`~alchemiscale.models.Scope`\s into a single target :py:class:`~alchemiscale.models.Scope`:: + + >>> new_sks = asc.merge_scopes( + ... scopes=[Scope('my_org', 'my_campaign', 'project_a'), + ... Scope('my_org', 'my_campaign', 'project_b')], + ... target_scope=Scope('my_org', 'my_campaign', 'consolidated'), + ... ) + >>> len(new_sks) + 17 + +This is a convenience method: each :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` is copied via :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.copy_network` in turn, with names preserved. +It is useful for consolidating :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s from multiple :py:class:`~alchemiscale.models.Scope`\s into a shared workspace, or for relocating a whole :py:class:`~alchemiscale.models.Scope`\'s worth of :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s. + + .. _user-guide-create-tasks: **************************** From 564ce40ca21f350a35e395090130126a5683f06b Mon Sep 17 00:00:00 2001 From: David Dotson Date: Sun, 5 Jul 2026 16:01:58 -0600 Subject: [PATCH 27/35] Assert AlchemicalNetwork dedup in test_copy_network Explicitly verify that copying an AlchemicalNetwork into a scope where the same AlchemicalNetwork already exists collapses onto the existing node rather than creating a duplicate. The additive-attach semantics this test exercises depend on that dedup. Co-Authored-By: Claude Opus 4.7 --- .../tests/integration/interface/client/test_client.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/alchemiscale/tests/integration/interface/client/test_client.py b/alchemiscale/tests/integration/interface/client/test_client.py index 8d46d9bb..06820ec4 100644 --- a/alchemiscale/tests/integration/interface/client/test_client.py +++ b/alchemiscale/tests/integration/interface/client/test_client.py @@ -485,6 +485,15 @@ def test_copy_network( assert user_client.check_exists(copied_sk) assert user_client.get_network(copied_sk).name == source_an.name + # the target scope is preloaded with this same AlchemicalNetwork, so + # the copy must dedup onto the existing node rather than create a + # duplicate; the copy is additive to the preexisting network + an_count = n4js_preloaded.execute_query( + "MATCH (an:AlchemicalNetwork {_scoped_key: $sk}) RETURN count(an) AS n", + sk=str(copied_sk), + ).records[0]["n"] + assert an_count == 1 + # every cloned Task must be reachable via the standard PERFORMS # traversal from the new network copied_task_sks = user_client.get_network_tasks(copied_sk) From e8f5c3714a7296154a83e5adb8b05e9a38ace865 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Sun, 5 Jul 2026 16:27:24 -0600 Subject: [PATCH 28/35] Cover fan-in case in test_merge_scopes_creates_new_networks_in_target Existing merge_scopes coverage only exercised the pure-dedup case, where every source AlchemicalNetwork was already preloaded into the target scope with the same gufe key. The new test adds a third AlchemicalNetwork to the source scope only, then asserts that after merge_scopes the target scope holds it as a genuinely new node (alongside the 2 that dedup onto preexisting entries). Co-Authored-By: Claude Opus 4.7 --- .../interface/client/test_client.py | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/alchemiscale/tests/integration/interface/client/test_client.py b/alchemiscale/tests/integration/interface/client/test_client.py index 06820ec4..07817854 100644 --- a/alchemiscale/tests/integration/interface/client/test_client.py +++ b/alchemiscale/tests/integration/interface/client/test_client.py @@ -680,6 +680,78 @@ def test_merge_scopes( final_target_sks = user_client.query_networks(scope=target_scope, state=None) assert set(baseline_target_sks) == set(final_target_sks) + def test_merge_scopes_creates_new_networks_in_target( + self, + scope_test, + multiple_scopes, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + network_tyk2, + ): + """merge_scopes must create fresh AlchemicalNetwork nodes in the + target scope for source networks that don't already exist there -- + not just dedup onto preexisting ones. + + n4js_preloaded seeds the same 2 networks (network_tyk2 + an2) into + every scope in multiple_scopes, so we first add a *third* + AlchemicalNetwork to the source scope only. After merge_scopes, + the target scope must contain that third network as a new node, + alongside the 2 that dedup onto its preexisting entries. + """ + source_scope = scope_test + target_scope = multiple_scopes[2] + + # add a fresh AN to the source scope that does not exist in the + # target; a unique name → unique gufe key + fresh_an = AlchemicalNetwork( + edges=list(network_tyk2.edges)[:2], + name="fresh_source_only", + ) + fresh_source_sk = user_client.create_network(fresh_an, source_scope) + fresh_expected_target_sk = ScopedKey( + gufe_key=fresh_source_sk.gufe_key, + org=target_scope.org, + campaign=target_scope.campaign, + project=target_scope.project, + ) + + # sanity: fresh AN exists in source, does not exist in target + assert user_client.check_exists(fresh_source_sk) + assert not user_client.check_exists(fresh_expected_target_sk) + + baseline_target_sks = set( + user_client.query_networks(scope=target_scope, state=None) + ) + # baseline: network_tyk2 + an2 preloaded in target + assert len(baseline_target_sks) == 2 + assert fresh_expected_target_sk not in baseline_target_sks + + new_sks = user_client.merge_scopes( + scopes=[source_scope], + target_scope=target_scope, + visualize=False, + ) + + # source scope holds 3 networks now (2 preloaded + 1 fresh) → 3 + # returned ScopedKeys, all anchored to target_scope + assert len(new_sks) == 3 + assert all(sk.scope == target_scope for sk in new_sks) + assert fresh_expected_target_sk in new_sks + + # target scope now holds the baseline union the fresh AN as a + # genuinely new node + final_target_sks = set( + user_client.query_networks(scope=target_scope, state=None) + ) + assert final_target_sks == baseline_target_sks | {fresh_expected_target_sk} + + # exactly one AN node backs the fresh copy in the target scope + fresh_count = n4js_preloaded.execute_query( + "MATCH (an:AlchemicalNetwork {_scoped_key: $sk}) RETURN count(an) AS n", + sk=str(fresh_expected_target_sk), + ).records[0]["n"] + assert fresh_count == 1 + def test_merge_scopes_rejects_empty( self, scope_test, From c5bc737075195b24f8ec802a82398b109c5f4424 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Sun, 5 Jul 2026 16:42:17 -0600 Subject: [PATCH 29/35] Add store-level test_copy_network with partial-preload target The interface-level test_copy_network* tests all start from n4js_preloaded, which seeds the same networks (and therefore the same Transformations) into every scope. Every copy under that fixture is either pure dedup or pure fresh-write, never a mix. The new store-level test seeds a target scope with only a *subset* of the source's Transformations, then copies the full source into it. Post-copy assertions verify: - the target ends up with exactly one AN node for the copied ScopedKey; - the target's Transformation count equals the source's (overlap subset dedups, non-overlap subset is created fresh); - every source Transformation appears exactly once in the target; - every cloned Task is wired via PERFORMS to a target-scope Transformation regardless of which side of the overlap it lives on; - EXTENDS chains are preserved across the copy; - both source PDRRs land in the target scope with obj_keys intact. Co-Authored-By: Claude Opus 4.7 --- .../integration/storage/test_statestore.py | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) diff --git a/alchemiscale/tests/integration/storage/test_statestore.py b/alchemiscale/tests/integration/storage/test_statestore.py index 4af8addd..66cd8779 100644 --- a/alchemiscale/tests/integration/storage/test_statestore.py +++ b/alchemiscale/tests/integration/storage/test_statestore.py @@ -281,6 +281,161 @@ def set_result(_slice, *, ok: bool): ) assert len(results.records) == 1 + def test_copy_network_partial_preload_target(self, n4js, network_tyk2, scope_test): + """copy_network must correctly place cloned Tasks in a target + scope that has only a *partial* overlap with the source's + Transformations. + + Transformations already present in the target scope must + dedup onto the preexisting nodes; Transformations missing from + the target must be created fresh. Either way, cloned Tasks + must land wired to the correct target-scope Transformation via + PERFORMS, EXTENDS chains must be preserved, and + ProtocolDAGResultRefs must land in the target scope. + + This case is not reachable through the interface tests, which + always start from ``n4js_preloaded`` -- a fixture that seeds + the same networks (and therefore the same Transformations) + into every scope, so every copy is either pure dedup or pure + fresh-write, never a mix. + """ + all_edges = sorted(network_tyk2.edges, key=lambda e: e.key) + overlap_edges = all_edges[:2] + + # source: full network + source_scope = Scope(**(scope_test.to_dict() | {"project": "source"})) + source_sk, _, _ = n4js.assemble_network(network_tyk2, source_scope) + + # target: preload an AN carrying only the overlap subset of + # Transformations. This produces the partial-preload shape. + target_scope = Scope(**(scope_test.to_dict() | {"project": "target"})) + partial_an = AlchemicalNetwork(edges=overlap_edges, name="target_preexisting") + n4js.assemble_network(partial_an, target_scope) + + def target_tf_count(): + return n4js.execute_query( + """ + MATCH (tf) + WHERE (tf:Transformation OR tf:NonTransformation) + AND tf._project = $project + RETURN count(tf) AS n + """, + project=target_scope.project, + ).records[0]["n"] + + # baseline: target scope holds exactly the overlap Transformations + assert target_tf_count() == len(overlap_edges) + + # hang Tasks off one overlap Transformation and one non-overlap + # Transformation so both sides of the boundary get exercised + overlap_tf_sk = n4js.get_scoped_key(overlap_edges[0], source_scope) + fresh_tf_sk = n4js.get_scoped_key(all_edges[-1], source_scope) + + # overlap side: 2 tasks -- one complete+ok PDRR, one waiting + overlap_tasks = n4js.create_tasks([overlap_tf_sk, overlap_tf_sk]) + n4js.set_task_running(overlap_tasks[:1]) + n4js.set_task_complete(overlap_tasks[:1]) + overlap_pdrr = ProtocolDAGResultRef( + obj_key=f"ProtocolDAGResult-{uuid.uuid4()}", + scope=overlap_tasks[0].scope, + ok=True, + ) + n4js.set_task_result(overlap_tasks[0], overlap_pdrr) + + # fresh side: 1 complete base + 1 running EXTENDS on top + fresh_base = n4js.create_tasks([fresh_tf_sk]) + n4js.set_task_running(fresh_base) + n4js.set_task_complete(fresh_base) + fresh_pdrr = ProtocolDAGResultRef( + obj_key=f"ProtocolDAGResult-{uuid.uuid4()}", + scope=fresh_base[0].scope, + ok=True, + ) + n4js.set_task_result(fresh_base[0], fresh_pdrr) + + fresh_ext = n4js.create_tasks([fresh_tf_sk], extends=fresh_base) + n4js.set_task_running(fresh_ext) + + # --- copy --- + copied_sk = n4js.copy_network(source_sk, target_scope) + + # 1) exactly one AN node in target with the copied ScopedKey + an_count = n4js.execute_query( + "MATCH (an:AlchemicalNetwork {_scoped_key: $sk}) RETURN count(an) AS n", + sk=str(copied_sk), + ).records[0]["n"] + assert an_count == 1 + + # 2) Transformation node count in target rises to the full + # source count -- overlap subset stays deduplicated, non-overlap + # subset is freshly created + assert target_tf_count() == len(all_edges) + + # 3) every source Transformation appears exactly once in target + # (proves no duplication of overlap nodes, no missed fresh nodes) + for edge in all_edges: + cnt = n4js.execute_query( + """ + MATCH (tf) + WHERE (tf:Transformation OR tf:NonTransformation) + AND tf._project = $project + AND tf._gufe_key = $gufe_key + RETURN count(tf) AS n + """, + project=target_scope.project, + gufe_key=str(edge.key), + ).records[0]["n"] + assert cnt == 1 + + # 4) every source Task lands in target wired via PERFORMS to a + # target-scope Transformation + def target_key(source_task_sk): + return ScopedKey( + gufe_key=source_task_sk.gufe_key, + org=target_scope.org, + campaign=target_scope.campaign, + project=target_scope.project, + ) + + for source_task_sk in overlap_tasks + fresh_base + fresh_ext: + target_sk_ = target_key(source_task_sk) + perf = n4js.execute_query( + """ + MATCH (t:Task {_scoped_key: $sk})-[:PERFORMS]->(tf) + WHERE (tf:Transformation OR tf:NonTransformation) + AND tf._project = $project + RETURN count(tf) AS n + """, + sk=str(target_sk_), + project=target_scope.project, + ).records[0]["n"] + assert perf == 1 + + # 5) EXTENDS preserved for the fresh_ext → fresh_base pair + ext_edge = n4js.execute_query( + """ + MATCH (a:Task {_scoped_key: $a})-[:EXTENDS]->(b:Task {_scoped_key: $b}) + RETURN count(*) AS n + """, + a=str(target_key(fresh_ext[0])), + b=str(target_key(fresh_base[0])), + ).records[0]["n"] + assert ext_edge == 1 + + # 6) both PDRRs land in target scope with their obj_keys intact + pdrr_records = n4js.execute_query( + """ + MATCH (pdrr:ProtocolDAGResultRef {_project: $project}) + RETURN pdrr.ok AS ok, pdrr.obj_key AS obj_key + """, + project=target_scope.project, + ).records + assert len(pdrr_records) == 2 + assert {r["obj_key"] for r in pdrr_records} == { + overlap_pdrr.obj_key, + fresh_pdrr.obj_key, + } + def test_set_network_state(self, n4js, network_tyk2, scope_test): valid_states = [state.value for state in NetworkStateEnum] network_sks = [] From dce5b1697a3fa17a9201342758ff4b010bc48c39 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Mon, 6 Jul 2026 15:56:00 -0600 Subject: [PATCH 30/35] Split merge/copy docs into own user guide page The Merging and Copying AlchemicalNetworks section covers an advanced workflow that only makes sense after there are results attached to networks, so it doesn't belong in Getting Started. Move it to its own page in the user guide toctree, between getting_started and handling_errors. Co-Authored-By: Claude Opus 4.7 --- docs/user_guide/getting_started.rst | 88 ------------------ docs/user_guide/index.rst | 1 + .../merging_and_copying_networks.rst | 92 +++++++++++++++++++ 3 files changed, 93 insertions(+), 88 deletions(-) create mode 100644 docs/user_guide/merging_and_copying_networks.rst diff --git a/docs/user_guide/getting_started.rst b/docs/user_guide/getting_started.rst index fae08884..f4460887 100644 --- a/docs/user_guide/getting_started.rst +++ b/docs/user_guide/getting_started.rst @@ -167,94 +167,6 @@ You can list all your accessible ``AlchemicalNetworks`` on the ``alchemiscale`` and you can use these with :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.get_network` above to pull them down as desired. -Merging and copying AlchemicalNetworks -====================================== - -Once you have :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s submitted, you may want to combine several of them into a single :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`, or move an :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` (or a whole :py:class:`~alchemiscale.models.Scope` of them) into a different :py:class:`~alchemiscale.models.Scope`. -Three :py:class:`~alchemiscale.interface.client.AlchemiscaleClient` methods support these workflows: - -* :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_networks` — combine several :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s into a single new one. -* :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.copy_network` — copy one :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` into a different :py:class:`~alchemiscale.models.Scope`. -* :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_scopes` — copy every :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` from one or more source :py:class:`~alchemiscale.models.Scope`\s into a single target :py:class:`~alchemiscale.models.Scope`. - -The destination :py:class:`~alchemiscale.models.Scope` for each of these must be a *specific* :py:class:`~alchemiscale.models.Scope` (no wildcards), and your user must have permissions on the source and destination :py:class:`~alchemiscale.models.Scope`\s. - -.. note:: - - For all three methods, the following execution-orchestration state is intentionally **not** carried over from the source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s to the new one, since these govern *how* :py:class:`~alchemiscale.storage.models.Task`\s run rather than the results themselves: - - * Any :py:class:`~alchemiscale.storage.models.Task`\s that were previously *actioned* on a source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` are not automatically actioned on the new one; call :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.action_tasks` on the new network afterward for any :py:class:`~alchemiscale.storage.models.Task`\s you want compute services to pick up. - * Any ``Strategy`` set via :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.set_network_strategy` is not copied. - * Any ``TaskRestartPattern``\s added via :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.add_task_restart_patterns` are not copied. - - Set these on the new :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` yourself after the merge or copy if you need them. - - -Merging multiple AlchemicalNetworks ------------------------------------ - -Use :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_networks` to combine multiple :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s into a single new :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` in the target :py:class:`~alchemiscale.models.Scope`:: - - >>> merged_sk = asc.merge_networks( - ... networks=[an_sk_a, an_sk_b, an_sk_c], - ... name='combined_campaign', - ... scope=Scope('my_org', 'my_campaign', 'combined_project'), - ... ) - >>> merged_sk - - -The new :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` contains the union of the :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s and ``NonTransformation``\s from every source network. -The source networks themselves are unchanged. - -Existing :py:class:`~alchemiscale.storage.models.Task`\s attached to those source :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s that are in ``complete`` or ``error`` state are cloned onto the merged network's :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s along with their :py:class:`~alchemiscale.storage.models.ProtocolDAGResultRef`\s, so previously-computed results do not need to be re-run. -:py:class:`~alchemiscale.storage.models.Task`\s in other states (``waiting``, ``running``, ``invalid``, ``deleted``) are **not** carried over. - -To retry the cloned ``error`` :py:class:`~alchemiscale.storage.models.Task`\s on the merged network, first set their status back to ``waiting`` with :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.set_tasks_status`, then :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.action_tasks` them:: - - >>> errored = asc.get_network_tasks(merged_sk, status='error') - >>> asc.set_tasks_status(errored, 'waiting') - >>> asc.action_tasks(errored, merged_sk) - - -Copying a single AlchemicalNetwork ----------------------------------- - -Use :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.copy_network` to copy one :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` into a different :py:class:`~alchemiscale.models.Scope`:: - - >>> copy_sk = asc.copy_network( - ... network=an_sk, - ... scope=Scope('my_org', 'my_campaign', 'shared_project'), - ... ) - -Unlike :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_networks`, :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.copy_network` carries over **every** existing :py:class:`~alchemiscale.storage.models.Task` for the source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\'s :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s, regardless of status, together with their :py:class:`~alchemiscale.storage.models.ProtocolDAGResultRef`\s. - -If ``name`` is not given, the source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\'s name is preserved and the copy has the same ``gufe.key`` as the source (so its :py:class:`~alchemiscale.models.ScopedKey` differs from the source's only in :py:class:`~alchemiscale.models.Scope`). -Pass ``name`` to rename the copy; this yields a fresh ``gufe.key`` derived from the renamed content:: - - >>> renamed_copy_sk = asc.copy_network( - ... network=an_sk, - ... scope=Scope('my_org', 'my_campaign', 'shared_project'), - ... name='shared_snapshot', - ... ) - - -Copying every AlchemicalNetwork from one Scope to another ---------------------------------------------------------- - -Use :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_scopes` to copy every :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` in one or more source :py:class:`~alchemiscale.models.Scope`\s into a single target :py:class:`~alchemiscale.models.Scope`:: - - >>> new_sks = asc.merge_scopes( - ... scopes=[Scope('my_org', 'my_campaign', 'project_a'), - ... Scope('my_org', 'my_campaign', 'project_b')], - ... target_scope=Scope('my_org', 'my_campaign', 'consolidated'), - ... ) - >>> len(new_sks) - 17 - -This is a convenience method: each :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` is copied via :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.copy_network` in turn, with names preserved. -It is useful for consolidating :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s from multiple :py:class:`~alchemiscale.models.Scope`\s into a shared workspace, or for relocating a whole :py:class:`~alchemiscale.models.Scope`\'s worth of :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s. - - .. _user-guide-create-tasks: **************************** diff --git a/docs/user_guide/index.rst b/docs/user_guide/index.rst index 2125c0ad..8c707022 100644 --- a/docs/user_guide/index.rst +++ b/docs/user_guide/index.rst @@ -11,5 +11,6 @@ If you are making use of an **alchemiscale** instance, this document will famili :maxdepth: 1 ./getting_started + ./merging_and_copying_networks ./handling_errors ./strategy_automation diff --git a/docs/user_guide/merging_and_copying_networks.rst b/docs/user_guide/merging_and_copying_networks.rst new file mode 100644 index 00000000..f9fb044f --- /dev/null +++ b/docs/user_guide/merging_and_copying_networks.rst @@ -0,0 +1,92 @@ +.. _merging-and-copying-networks: + +###################################### +Merging and copying AlchemicalNetworks +###################################### + +Once you have :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s submitted with :py:class:`~alchemiscale.storage.models.Task`\s and results attached, you may want to combine several of them into a single :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`, or move an :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` (or a whole :py:class:`~alchemiscale.models.Scope` of them) into a different :py:class:`~alchemiscale.models.Scope`. +Three :py:class:`~alchemiscale.interface.client.AlchemiscaleClient` methods support these workflows: + +* :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_networks` — combine several :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s into a single new one. +* :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.copy_network` — copy one :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` into a different :py:class:`~alchemiscale.models.Scope`. +* :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_scopes` — copy every :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` from one or more source :py:class:`~alchemiscale.models.Scope`\s into a single target :py:class:`~alchemiscale.models.Scope`. + +The destination :py:class:`~alchemiscale.models.Scope` for each of these must be a *specific* :py:class:`~alchemiscale.models.Scope` (no wildcards), and your user must have permissions on the source and destination :py:class:`~alchemiscale.models.Scope`\s. + +.. note:: + + For all three methods, the following execution-orchestration state is intentionally **not** carried over from the source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s to the new one, since these govern *how* :py:class:`~alchemiscale.storage.models.Task`\s run rather than the results themselves: + + * Any :py:class:`~alchemiscale.storage.models.Task`\s that were previously *actioned* on a source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` are not automatically actioned on the new one; call :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.action_tasks` on the new network afterward for any :py:class:`~alchemiscale.storage.models.Task`\s you want compute services to pick up. + * Any ``Strategy`` set via :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.set_network_strategy` is not copied. + * Any ``TaskRestartPattern``\s added via :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.add_task_restart_patterns` are not copied. + + Set these on the new :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` yourself after the merge or copy if you need them. + + +*********************************** +Merging multiple AlchemicalNetworks +*********************************** + +Use :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_networks` to combine multiple :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s into a single new :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` in the target :py:class:`~alchemiscale.models.Scope`:: + + >>> merged_sk = asc.merge_networks( + ... networks=[an_sk_a, an_sk_b, an_sk_c], + ... name='combined_campaign', + ... scope=Scope('my_org', 'my_campaign', 'combined_project'), + ... ) + >>> merged_sk + + +The new :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` contains the union of the :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s and ``NonTransformation``\s from every source network. +The source networks themselves are unchanged. + +Existing :py:class:`~alchemiscale.storage.models.Task`\s attached to those source :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s that are in ``complete`` or ``error`` state are cloned onto the merged network's :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s along with their :py:class:`~alchemiscale.storage.models.ProtocolDAGResultRef`\s, so previously-computed results do not need to be re-run. +:py:class:`~alchemiscale.storage.models.Task`\s in other states (``waiting``, ``running``, ``invalid``, ``deleted``) are **not** carried over. + +To retry the cloned ``error`` :py:class:`~alchemiscale.storage.models.Task`\s on the merged network, first set their status back to ``waiting`` with :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.set_tasks_status`, then :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.action_tasks` them:: + + >>> errored = asc.get_network_tasks(merged_sk, status='error') + >>> asc.set_tasks_status(errored, 'waiting') + >>> asc.action_tasks(errored, merged_sk) + + +********************************** +Copying a single AlchemicalNetwork +********************************** + +Use :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.copy_network` to copy one :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` into a different :py:class:`~alchemiscale.models.Scope`:: + + >>> copy_sk = asc.copy_network( + ... network=an_sk, + ... scope=Scope('my_org', 'my_campaign', 'shared_project'), + ... ) + +Unlike :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_networks`, :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.copy_network` carries over **every** existing :py:class:`~alchemiscale.storage.models.Task` for the source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\'s :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s, regardless of status, together with their :py:class:`~alchemiscale.storage.models.ProtocolDAGResultRef`\s. + +If ``name`` is not given, the source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\'s name is preserved and the copy has the same ``gufe.key`` as the source (so its :py:class:`~alchemiscale.models.ScopedKey` differs from the source's only in :py:class:`~alchemiscale.models.Scope`). +Pass ``name`` to rename the copy; this yields a fresh ``gufe.key`` derived from the renamed content:: + + >>> renamed_copy_sk = asc.copy_network( + ... network=an_sk, + ... scope=Scope('my_org', 'my_campaign', 'shared_project'), + ... name='shared_snapshot', + ... ) + + +********************************************************* +Copying every AlchemicalNetwork from one Scope to another +********************************************************* + +Use :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_scopes` to copy every :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` in one or more source :py:class:`~alchemiscale.models.Scope`\s into a single target :py:class:`~alchemiscale.models.Scope`:: + + >>> new_sks = asc.merge_scopes( + ... scopes=[Scope('my_org', 'my_campaign', 'project_a'), + ... Scope('my_org', 'my_campaign', 'project_b')], + ... target_scope=Scope('my_org', 'my_campaign', 'consolidated'), + ... ) + >>> len(new_sks) + 17 + +This is a convenience method: each :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` is copied via :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.copy_network` in turn, with names preserved. +It is useful for consolidating :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s from multiple :py:class:`~alchemiscale.models.Scope`\s into a shared workspace, or for relocating a whole :py:class:`~alchemiscale.models.Scope`\'s worth of :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s. From ef443349bb4afb106a82009d3e6bc3e7923610a3 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Mon, 6 Jul 2026 16:19:13 -0600 Subject: [PATCH 31/35] Restrict merge/copy Task carry-over to complete status Previously the three network-transfer methods had three different policies for which Tasks carry over from source networks: - merge_networks: complete + error - copy_network: any status (regardless) - merge_scopes: any status (via copy_network) Collapse to a single rule across all three: only Tasks in ``complete`` state carry over, together with their ProtocolDAGResultRefs. Tasks in any other status (waiting, running, error, invalid, deleted) are not carried over. Rationale: consistency wins, and nothing valuable is lost. Waiting Tasks are trivially recreatable; running Tasks are held by a compute service in the source scope and cloning would create a phantom; invalid/deleted are tombstoned; and error Tasks can still be inspected on the source network via get_transformation_failures before the merge/copy. Users who want to retry an errored Transformation on the new network can simply create_tasks on it. Updates: store-level statestore.py, client-level docstrings, the merging_and_copying_networks user-guide page, the news entry, and the merge_networks/copy_network integration tests to reflect the tighter policy. test_copy_network_partial_preload_target and test_merge_networks_preserves_tasks_and_results are extended to assert non-complete Tasks are dropped, not leaked, into the target scope. Co-Authored-By: Claude Opus 4.7 --- alchemiscale/interface/client.py | 36 +++--- alchemiscale/storage/statestore.py | 47 ++++---- .../interface/client/test_client.py | 107 ++++++++++-------- .../integration/storage/test_statestore.py | 64 +++++++---- .../merging_and_copying_networks.rst | 23 ++-- news/issue-221.rst | 6 +- 6 files changed, 161 insertions(+), 122 deletions(-) diff --git a/alchemiscale/interface/client.py b/alchemiscale/interface/client.py index a7edfd19..53393f87 100644 --- a/alchemiscale/interface/client.py +++ b/alchemiscale/interface/client.py @@ -199,15 +199,16 @@ def merge_networks( The resulting AlchemicalNetwork contains the union of all Transformations and NonTransformations from the source networks. - Existing Tasks for those transformations that are in ``complete`` or - ``error`` state are cloned into the new network's scope along with - their associated ProtocolDAGResultRefs, so previously-computed results - do not need to be re-run. + Existing Tasks for those Transformations that are in ``complete`` + state are cloned into the new network's scope along with their + associated ProtocolDAGResultRefs, so previously-computed results do + not need to be re-run. Tasks in any other status (``waiting``, + ``running``, ``error``, ``invalid``, ``deleted``) are not carried + over. - Cloned Tasks are intentionally **not** actioned; to retry errored Tasks - on the merged network, call :meth:`action_tasks` with the merged - network's ScopedKey after the merge completes, and set the Tasks' - status to back to `waiting` with :meth:`set_tasks_status`. + Cloned Tasks are intentionally **not** actioned; call + :meth:`action_tasks` with the merged network's ScopedKey after the + merge completes if you want them picked up by compute services. The source networks' execution-orchestration state is intentionally **not** carried over -- these govern how Tasks run rather than the @@ -301,14 +302,14 @@ def copy_network( visualize: bool = True, ) -> ScopedKey: """Copy an AlchemicalNetwork to a new Scope, carrying over every - existing Task (regardless of status) and its associated - ProtocolDAGResultRefs. + existing ``complete`` Task and its associated + ProtocolDAGResultRefs. Tasks in any other status (``waiting``, + ``running``, ``error``, ``invalid``, ``deleted``) are not carried + over. - Cloned Tasks are intentionally **not** actioned; to pick up errored or - waiting Tasks for execution on the merged network, call - :meth:`action_tasks` with the new network's ``ScopedKey`` after the - copy completes, and set errored Tasks' status to back to `waiting` with - :meth:`set_tasks_status`. + Cloned Tasks are intentionally **not** actioned; call + :meth:`action_tasks` with the new network's ``ScopedKey`` after + the copy completes if you want them picked up by compute services. The source network's execution-orchestration state is intentionally **not** carried over -- these govern how Tasks run rather than the @@ -396,8 +397,9 @@ def merge_scopes( into ``target_scope``. Each source AlchemicalNetwork is copied via :meth:`copy_network`, - preserving its name, Tasks, and ProtocolDAGResultRefs. Cloned - Tasks are not actioned. + preserving its name, its ``complete`` Tasks, and its associated + ProtocolDAGResultRefs. Tasks in any other status are not carried + over, and cloned Tasks are not actioned. Parameters ---------- diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index c64a3f7d..d07b6c57 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -145,10 +145,9 @@ def update_task_trees( task_statuses If provided, only ``Task``\\ s whose ``status`` is in this list are loaded. If ``None`` (the default), ``Task``\\ s of any - status are loaded. ``merge_networks`` passes - ``["complete", "error"]`` to clone only successfully-completed - or errored work; ``copy_network`` passes ``None`` so the - entire ``Task`` tree of the source network is preserved. + status are loaded. Both ``merge_networks`` and + ``copy_network`` pass ``["complete"]`` so only ``Task``\\ s + carrying successful results are preserved. """ key_to_data_map = {str(td.transformation.key): td for td in transformation_data} # prepare for unwind clause, include transformation key @@ -1091,17 +1090,18 @@ def merge_networks( Each ``Transformation`` / ``NonTransformation`` in the input networks is included exactly once in the new network. ``Task``\\ s - on the source networks that are in ``complete`` or ``error`` - state are cloned into the new network's ``Scope`` along with - their ``ProtocolDAGResultRef``\\ s and ``EXTENDS`` relationships, - and are wired to their ``Transformation`` via ``PERFORMS`` so - they are reachable from the standard network traversals. + on the source networks that are in ``complete`` state are cloned + into the new network's ``Scope`` along with their + ``ProtocolDAGResultRef``\\ s and ``EXTENDS`` relationships, and + are wired to their ``Transformation`` via ``PERFORMS`` so they + are reachable from the standard network traversals. ``Task``\\ s + in any other status (``waiting``, ``running``, ``error``, + ``invalid``, ``deleted``) are not carried over. Cloned ``Task``\\ s are intentionally **not** actioned to the new - network's ``TaskHub``. Users wanting to retry errored tasks on - the merged network should call :meth:`action_tasks` themselves - with the merged network's ``TaskHub`` ``ScopedKey`` and reset - the ``Task`` statuses back to `waiting`. + network's ``TaskHub``. Users wanting the cloned ``Task``\\ s + picked up by compute services should call :meth:`action_tasks` + against the new network's ``TaskHub`` after the merge. Execution-orchestration state on the source networks is intentionally **not** carried over -- these govern *how* Tasks @@ -1182,13 +1182,13 @@ def merge_networks( data.add_known_scoped_key(database_key, network_scope) # Collect all transformation gufe objects and collect into a new set of edges. - # ``merge_networks`` carries over only completed or errored Tasks so the - # new network reflects "results to keep"; in-flight or invalid Tasks are - # not preserved. + # Only ``complete`` Tasks carry over -- the new network reflects the + # results the user has already computed; in-flight, errored, or + # tombstoned Tasks are not preserved. _TransformationData.update_task_trees( list(transformation_data.values()), self, - task_statuses=["complete", "error"], + task_statuses=["complete"], ) new_edges = [td.transformation for td in transformation_data.values()] # Make new alchemiscale network with these edges @@ -1233,8 +1233,10 @@ def copy_network( state: NetworkStateEnum | str = NetworkStateEnum.active, ) -> ScopedKey: """Copy an ``AlchemicalNetwork`` to a new ``Scope``, carrying over - every ``Task`` (regardless of status) and its associated - ``ProtocolDAGResultRef``\\ s. + every ``complete`` ``Task`` and its associated + ``ProtocolDAGResultRef``\\ s. ``Task``\\ s in any other status + (``waiting``, ``running``, ``error``, ``invalid``, ``deleted``) + are not carried over. ``Task`` clones are wired to their ``Transformation`` via ``PERFORMS`` and to one another via ``EXTENDS`` where applicable. @@ -1307,12 +1309,13 @@ def copy_network( transformation_data[transformation.key] = data data.add_known_scoped_key(database_key, source_scope) - # ``copy_network`` preserves the entire Task tree of the source - # network; pass ``task_statuses=None`` so the status filter is dropped. + # Only ``complete`` Tasks carry over -- the copy reflects the + # results the user has already computed; in-flight, errored, or + # tombstoned Tasks are not preserved. _TransformationData.update_task_trees( list(transformation_data.values()), self, - task_statuses=None, + task_statuses=["complete"], ) # build the new AlchemicalNetwork's subgraph + NetworkMark + TaskHub diff --git a/alchemiscale/tests/integration/interface/client/test_client.py b/alchemiscale/tests/integration/interface/client/test_client.py index 07817854..56b2ef29 100644 --- a/alchemiscale/tests/integration/interface/client/test_client.py +++ b/alchemiscale/tests/integration/interface/client/test_client.py @@ -326,22 +326,28 @@ def test_merge_networks_preserves_tasks_and_results( user_client: client.AlchemiscaleClient, network_tyk2, ): - """The merged network must carry over Tasks in complete/error state - along with their ProtocolDAGResultRefs, cloned into the new scope.""" + """The merged network must carry over ``complete`` Tasks -- and only + ``complete`` Tasks -- along with their ProtocolDAGResultRefs, cloned + into the new scope. + + Tasks in any other status (``waiting``, ``running``, ``error``, + ``invalid``, ``deleted``) must not be cloned. + """ # pick a source network in scope_test source_sks = user_client.query_networks(scope=scope_test, state=None) assert source_sks source_sk = source_sks[0] - # create Tasks on two of its Transformations directly through n4js so - # we can drive them to completed/errored states with PDRRs without - # actually executing protocols + # create Tasks on four of its Transformations directly through n4js + # so we can drive them to distinct statuses without actually + # executing protocols. The four statuses cover the "carry"/"drop" + # boundary: complete is preserved, error/waiting/running are dropped. transformation_sks = n4js_preloaded.get_network_transformations(source_sk) - assert len(transformation_sks) >= 2 + assert len(transformation_sks) >= 4 - task_sks = n4js_preloaded.create_tasks(transformation_sks[:2]) + task_sks = n4js_preloaded.create_tasks(transformation_sks[:4]) - # task 0: complete, ok result + # task 0: complete + ok PDRR -- carried over n4js_preloaded.set_task_running(task_sks[:1]) n4js_preloaded.set_task_complete(task_sks[:1]) ok_pdrr = ProtocolDAGResultRef( @@ -351,7 +357,7 @@ def test_merge_networks_preserves_tasks_and_results( ) n4js_preloaded.set_task_result(task_sks[0], ok_pdrr) - # task 1: error, failure result + # task 1: error + failure PDRR -- dropped n4js_preloaded.set_task_running(task_sks[1:2]) n4js_preloaded.set_task_error(task_sks[1:2]) err_pdrr = ProtocolDAGResultRef( @@ -361,6 +367,12 @@ def test_merge_networks_preserves_tasks_and_results( ) n4js_preloaded.set_task_result(task_sks[1], err_pdrr) + # task 2: waiting -- dropped + # (default status after create_tasks) + + # task 3: running -- dropped + n4js_preloaded.set_task_running(task_sks[3:4]) + # merge into a different authorized scope from where we set up the # source Tasks, so the per-scope counts below remain clean (the # destination scope has pre-loaded networks but no Tasks/PDRRs). @@ -374,7 +386,7 @@ def test_merge_networks_preserves_tasks_and_results( ) assert user_client.check_exists(merged_sk) - # both Tasks should appear in the new scope, with their original statuses + # only the complete Task should appear in the new scope task_records = n4js_preloaded.execute_query( """ MATCH (t:Task {`_project`: $project}) @@ -382,10 +394,10 @@ def test_merge_networks_preserves_tasks_and_results( """, project=merge_scope.project, ).records - assert sorted(r["status"] for r in task_records) == ["complete", "error"] + assert [r["status"] for r in task_records] == ["complete"] - # one ok and one not-ok PDRR should appear in the new scope, with their - # original object keys preserved + # only the ok PDRR should appear in the new scope; the error PDRR + # is dropped along with its Task pdrr_records = n4js_preloaded.execute_query( """ MATCH (pdrr:ProtocolDAGResultRef {`_project`: $project}) @@ -393,13 +405,11 @@ def test_merge_networks_preserves_tasks_and_results( """, project=merge_scope.project, ).records - assert len(pdrr_records) == 2 - oks = sorted(r["ok"] for r in pdrr_records) - assert oks == [False, True] - obj_keys = {r["obj_key"] for r in pdrr_records} - assert obj_keys == {ok_pdrr.obj_key, err_pdrr.obj_key} + assert len(pdrr_records) == 1 + assert pdrr_records[0]["ok"] is True + assert pdrr_records[0]["obj_key"] == ok_pdrr.obj_key - # cloned PDRRs must be wired to the cloned Tasks in the new scope + # cloned PDRR must be wired to the cloned Task in the new scope linked = n4js_preloaded.execute_query( """ MATCH (t:Task {`_project`: $project})-[:RESULTS_IN]-> @@ -408,18 +418,16 @@ def test_merge_networks_preserves_tasks_and_results( """, project=merge_scope.project, ).records - pairs = sorted((r["status"], r["ok"]) for r in linked) - assert pairs == [("complete", True), ("error", False)] + assert [(r["status"], r["ok"]) for r in linked] == [("complete", True)] - # the cloned Tasks must be reachable from the merged AlchemicalNetwork + # the cloned Task must be reachable from the merged AlchemicalNetwork # via the standard PERFORMS traversal that the user-facing API uses; # this catches any case where Tasks are written to the new scope but # not wired back to their Transformations in the merged network merged_task_sks = user_client.get_network_tasks(merged_sk) - assert len(merged_task_sks) == 2 - assert all(sk.scope == merge_scope for sk in merged_task_sks) - statuses = sorted(user_client.get_tasks_status(merged_task_sks)) - assert statuses == ["complete", "error"] + assert len(merged_task_sks) == 1 + assert merged_task_sks[0].scope == merge_scope + assert user_client.get_tasks_status(merged_task_sks) == ["complete"] def test_copy_network( self, @@ -430,22 +438,25 @@ def test_copy_network( network_tyk2, ): """copy_network must duplicate the source AlchemicalNetwork into the - target scope and carry over Tasks of *every* status, with their - PDRRs, EXTENDS relationships, and PERFORMS wiring intact.""" + target scope and carry over only ``complete`` Tasks (with their + PDRRs), while dropping Tasks in any other status. EXTENDS + relationships and PERFORMS wiring must be intact for the Tasks + that do carry over. + """ source_sks = user_client.query_networks(scope=scope_test, state=None) assert source_sks source_sk = source_sks[0] source_an = user_client.get_network(source_sk) transformation_sks = n4js_preloaded.get_network_transformations(source_sk) - assert len(transformation_sks) >= 3 - - # Stage four tasks across three statuses so we can prove copy_network - # is not silently filtering by status the way merge_networks does: - # task 0: complete + ok PDRR - # task 1: error + not-ok PDRR - # task 2: waiting (no result) - # task 3: running (no result) + assert len(transformation_sks) >= 4 + + # Stage four tasks across four statuses so we can prove copy_network + # carries over only ``complete`` Tasks and drops the rest: + # task 0: complete + ok PDRR -- carried + # task 1: error + not-ok PDRR -- dropped + # task 2: waiting (no result) -- dropped + # task 3: running (no result) -- dropped task_sks = n4js_preloaded.create_tasks(transformation_sks[:4]) n4js_preloaded.set_task_running(task_sks[:1]) @@ -494,15 +505,16 @@ def test_copy_network( ).records[0]["n"] assert an_count == 1 - # every cloned Task must be reachable via the standard PERFORMS - # traversal from the new network + # only the ``complete`` Task must be reachable from the new network + # via the standard PERFORMS traversal; the error / waiting / running + # Tasks are dropped copied_task_sks = user_client.get_network_tasks(copied_sk) - assert len(copied_task_sks) == 4 - assert all(sk.scope == target_scope for sk in copied_task_sks) - statuses = sorted(user_client.get_tasks_status(copied_task_sks)) - assert statuses == ["complete", "error", "running", "waiting"] + assert len(copied_task_sks) == 1 + assert copied_task_sks[0].scope == target_scope + assert user_client.get_tasks_status(copied_task_sks) == ["complete"] - # both PDRRs land in the target scope with their object keys intact + # only the ok PDRR lands in the target scope; the error PDRR is + # dropped along with its Task pdrr_records = n4js_preloaded.execute_query( """ MATCH (pdrr:ProtocolDAGResultRef {`_project`: $project}) @@ -510,12 +522,9 @@ def test_copy_network( """, project=target_scope.project, ).records - assert len(pdrr_records) == 2 - assert sorted(r["ok"] for r in pdrr_records) == [False, True] - assert {r["obj_key"] for r in pdrr_records} == { - ok_pdrr.obj_key, - err_pdrr.obj_key, - } + assert len(pdrr_records) == 1 + assert pdrr_records[0]["ok"] is True + assert pdrr_records[0]["obj_key"] == ok_pdrr.obj_key def test_copy_network_with_rename( self, diff --git a/alchemiscale/tests/integration/storage/test_statestore.py b/alchemiscale/tests/integration/storage/test_statestore.py index 66cd8779..232a3ec7 100644 --- a/alchemiscale/tests/integration/storage/test_statestore.py +++ b/alchemiscale/tests/integration/storage/test_statestore.py @@ -270,7 +270,8 @@ def set_result(_slice, *, ok: bool): ) assert len(results.records) == 7 - # we expect 1 pdrrs from the errored task + # errored Task is not cloned (complete-only policy), so its not-ok + # PDRR must not appear in the merged scope results = n4js.execute_query( """ MATCH (pdrr: ProtocolDAGResultRef {`_project`: $project}) @@ -279,7 +280,7 @@ def set_result(_slice, *, ok: bool): """, project=new_scope.project, ) - assert len(results.records) == 1 + assert len(results.records) == 0 def test_copy_network_partial_preload_target(self, n4js, network_tyk2, scope_test): """copy_network must correctly place cloned Tasks in a target @@ -327,11 +328,16 @@ def target_tf_count(): assert target_tf_count() == len(overlap_edges) # hang Tasks off one overlap Transformation and one non-overlap - # Transformation so both sides of the boundary get exercised + # Transformation so both sides of the boundary get exercised. + # Under the complete-only policy, only complete Tasks carry over; + # a waiting Task on the overlap side lets us also verify that + # non-complete Tasks are dropped, not leaked into the target. overlap_tf_sk = n4js.get_scoped_key(overlap_edges[0], source_scope) fresh_tf_sk = n4js.get_scoped_key(all_edges[-1], source_scope) - # overlap side: 2 tasks -- one complete+ok PDRR, one waiting + # overlap side: 2 tasks + # [0]: complete + ok PDRR -- carried + # [1]: waiting -- dropped overlap_tasks = n4js.create_tasks([overlap_tf_sk, overlap_tf_sk]) n4js.set_task_running(overlap_tasks[:1]) n4js.set_task_complete(overlap_tasks[:1]) @@ -342,7 +348,8 @@ def target_tf_count(): ) n4js.set_task_result(overlap_tasks[0], overlap_pdrr) - # fresh side: 1 complete base + 1 running EXTENDS on top + # fresh side: 1 complete base + 1 complete EXTENDS on top; both + # carried, so we can verify EXTENDS is preserved across the copy fresh_base = n4js.create_tasks([fresh_tf_sk]) n4js.set_task_running(fresh_base) n4js.set_task_complete(fresh_base) @@ -355,10 +362,25 @@ def target_tf_count(): fresh_ext = n4js.create_tasks([fresh_tf_sk], extends=fresh_base) n4js.set_task_running(fresh_ext) + n4js.set_task_complete(fresh_ext) + fresh_ext_pdrr = ProtocolDAGResultRef( + obj_key=f"ProtocolDAGResult-{uuid.uuid4()}", + scope=fresh_ext[0].scope, + ok=True, + ) + n4js.set_task_result(fresh_ext[0], fresh_ext_pdrr) # --- copy --- copied_sk = n4js.copy_network(source_sk, target_scope) + def target_key(source_task_sk): + return ScopedKey( + gufe_key=source_task_sk.gufe_key, + org=target_scope.org, + campaign=target_scope.campaign, + project=target_scope.project, + ) + # 1) exactly one AN node in target with the copied ScopedKey an_count = n4js.execute_query( "MATCH (an:AlchemicalNetwork {_scoped_key: $sk}) RETURN count(an) AS n", @@ -387,17 +409,10 @@ def target_tf_count(): ).records[0]["n"] assert cnt == 1 - # 4) every source Task lands in target wired via PERFORMS to a - # target-scope Transformation - def target_key(source_task_sk): - return ScopedKey( - gufe_key=source_task_sk.gufe_key, - org=target_scope.org, - campaign=target_scope.campaign, - project=target_scope.project, - ) - - for source_task_sk in overlap_tasks + fresh_base + fresh_ext: + # 4) every carried (complete) source Task lands in target wired via + # PERFORMS to a target-scope Transformation + carried_tasks = [overlap_tasks[0], fresh_base[0], fresh_ext[0]] + for source_task_sk in carried_tasks: target_sk_ = target_key(source_task_sk) perf = n4js.execute_query( """ @@ -411,7 +426,16 @@ def target_key(source_task_sk): ).records[0]["n"] assert perf == 1 - # 5) EXTENDS preserved for the fresh_ext → fresh_base pair + # 5) the waiting overlap Task must NOT appear in the target scope + # under the complete-only policy + dropped_target_sk = target_key(overlap_tasks[1]) + dropped_count = n4js.execute_query( + "MATCH (t:Task {_scoped_key: $sk}) RETURN count(t) AS n", + sk=str(dropped_target_sk), + ).records[0]["n"] + assert dropped_count == 0 + + # 6) EXTENDS preserved for the fresh_ext -> fresh_base pair ext_edge = n4js.execute_query( """ MATCH (a:Task {_scoped_key: $a})-[:EXTENDS]->(b:Task {_scoped_key: $b}) @@ -422,7 +446,8 @@ def target_key(source_task_sk): ).records[0]["n"] assert ext_edge == 1 - # 6) both PDRRs land in target scope with their obj_keys intact + # 7) exactly the three ok PDRRs land in target scope with obj_keys + # intact -- one per carried Task pdrr_records = n4js.execute_query( """ MATCH (pdrr:ProtocolDAGResultRef {_project: $project}) @@ -430,10 +455,11 @@ def target_key(source_task_sk): """, project=target_scope.project, ).records - assert len(pdrr_records) == 2 + assert len(pdrr_records) == 3 assert {r["obj_key"] for r in pdrr_records} == { overlap_pdrr.obj_key, fresh_pdrr.obj_key, + fresh_ext_pdrr.obj_key, } def test_set_network_state(self, n4js, network_tyk2, scope_test): diff --git a/docs/user_guide/merging_and_copying_networks.rst b/docs/user_guide/merging_and_copying_networks.rst index f9fb044f..0b98376c 100644 --- a/docs/user_guide/merging_and_copying_networks.rst +++ b/docs/user_guide/merging_and_copying_networks.rst @@ -13,13 +13,17 @@ Three :py:class:`~alchemiscale.interface.client.AlchemiscaleClient` methods supp The destination :py:class:`~alchemiscale.models.Scope` for each of these must be a *specific* :py:class:`~alchemiscale.models.Scope` (no wildcards), and your user must have permissions on the source and destination :py:class:`~alchemiscale.models.Scope`\s. +All three methods share the same Task-retention policy: **only** :py:class:`~alchemiscale.storage.models.Task`\s in ``complete`` status carry over to the new :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`, together with their :py:class:`~alchemiscale.storage.models.ProtocolDAGResultRef`\s. +:py:class:`~alchemiscale.storage.models.Task`\s in any other status (``waiting``, ``running``, ``error``, ``invalid``, ``deleted``) are not carried over. +This keeps the semantics simple: **the results of computed work are preserved; nothing else is**. + .. note:: - For all three methods, the following execution-orchestration state is intentionally **not** carried over from the source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s to the new one, since these govern *how* :py:class:`~alchemiscale.storage.models.Task`\s run rather than the results themselves: + None of the source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s' execution-orchestration state is carried over, since it governs *how* :py:class:`~alchemiscale.storage.models.Task`\s run rather than the results themselves: - * Any :py:class:`~alchemiscale.storage.models.Task`\s that were previously *actioned* on a source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` are not automatically actioned on the new one; call :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.action_tasks` on the new network afterward for any :py:class:`~alchemiscale.storage.models.Task`\s you want compute services to pick up. - * Any ``Strategy`` set via :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.set_network_strategy` is not copied. - * Any ``TaskRestartPattern``\s added via :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.add_task_restart_patterns` are not copied. + * Cloned :py:class:`~alchemiscale.storage.models.Task`\s are not *actioned* on the new network; call :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.action_tasks` on the new network afterward for any you want compute services to pick up. + * Any ``Strategy`` set on a source network via :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.set_network_strategy` is not copied. + * Any ``TaskRestartPattern``\s added to a source network via :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.add_task_restart_patterns` are not copied. Set these on the new :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` yourself after the merge or copy if you need them. @@ -41,14 +45,9 @@ Use :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_networks` The new :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` contains the union of the :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s and ``NonTransformation``\s from every source network. The source networks themselves are unchanged. -Existing :py:class:`~alchemiscale.storage.models.Task`\s attached to those source :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s that are in ``complete`` or ``error`` state are cloned onto the merged network's :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s along with their :py:class:`~alchemiscale.storage.models.ProtocolDAGResultRef`\s, so previously-computed results do not need to be re-run. -:py:class:`~alchemiscale.storage.models.Task`\s in other states (``waiting``, ``running``, ``invalid``, ``deleted``) are **not** carried over. - -To retry the cloned ``error`` :py:class:`~alchemiscale.storage.models.Task`\s on the merged network, first set their status back to ``waiting`` with :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.set_tasks_status`, then :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.action_tasks` them:: +The ``complete`` :py:class:`~alchemiscale.storage.models.Task`\s attached to those source :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s are cloned onto the merged network's :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s along with their :py:class:`~alchemiscale.storage.models.ProtocolDAGResultRef`\s, so previously-computed results do not need to be re-run. - >>> errored = asc.get_network_tasks(merged_sk, status='error') - >>> asc.set_tasks_status(errored, 'waiting') - >>> asc.action_tasks(errored, merged_sk) +If you want to add fresh :py:class:`~alchemiscale.storage.models.Task`\s to the merged network -- either to retry a :external+gufe:py:class:`~gufe.transformations.transformation.Transformation` that previously errored, or to gather more repeats -- create new ones with :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.create_tasks` on the relevant :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s and then :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.action_tasks` them on the merged network. ********************************** @@ -62,7 +61,7 @@ Use :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.copy_network` to ... scope=Scope('my_org', 'my_campaign', 'shared_project'), ... ) -Unlike :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_networks`, :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.copy_network` carries over **every** existing :py:class:`~alchemiscale.storage.models.Task` for the source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\'s :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s, regardless of status, together with their :py:class:`~alchemiscale.storage.models.ProtocolDAGResultRef`\s. +Like :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_networks`, only the source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\'s ``complete`` :py:class:`~alchemiscale.storage.models.Task`\s are carried over, together with their :py:class:`~alchemiscale.storage.models.ProtocolDAGResultRef`\s. If ``name`` is not given, the source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\'s name is preserved and the copy has the same ``gufe.key`` as the source (so its :py:class:`~alchemiscale.models.ScopedKey` differs from the source's only in :py:class:`~alchemiscale.models.Scope`). Pass ``name`` to rename the copy; this yields a fresh ``gufe.key`` derived from the renamed content:: diff --git a/news/issue-221.rst b/news/issue-221.rst index 4f0e855c..e0c046d7 100644 --- a/news/issue-221.rst +++ b/news/issue-221.rst @@ -1,9 +1,9 @@ **Added:** -* ``AlchemiscaleClient.merge_networks`` for combining multiple existing ``AlchemicalNetwork``\s into a new ``AlchemicalNetwork`` in a target ``Scope``. ``Task``\s in ``complete`` or ``error`` state on the source networks are cloned into the new network along with their ``ProtocolDAGResultRef``\s so previously-computed results do not need to be re-run; ``Task``\s in other statuses (``waiting``, ``running``, ``invalid``, ``deleted``) are not carried over. Backed by ``Neo4jStore.merge_networks`` and a new ``POST /networks/merge`` endpoint on the user API. -* ``AlchemiscaleClient.copy_network`` for copying a single ``AlchemicalNetwork`` to a new ``Scope``, with all of its ``Task``\s (regardless of status) and their ``ProtocolDAGResultRef``\s carried over. An optional ``name`` argument creates the copy under a new name (and therefore a new gufe key); when ``name`` is omitted, the source network's name and gufe key are preserved. Backed by ``Neo4jStore.copy_network`` and a new ``POST /networks/{network_scoped_key}/copy`` endpoint. +* ``AlchemiscaleClient.merge_networks`` for combining multiple existing ``AlchemicalNetwork``\s into a new ``AlchemicalNetwork`` in a target ``Scope``. Backed by ``Neo4jStore.merge_networks`` and a new ``POST /networks/merge`` endpoint on the user API. +* ``AlchemiscaleClient.copy_network`` for copying a single ``AlchemicalNetwork`` to a new ``Scope``. An optional ``name`` argument creates the copy under a new name (and therefore a new gufe key); when ``name`` is omitted, the source network's name and gufe key are preserved. Backed by ``Neo4jStore.copy_network`` and a new ``POST /networks/{network_scoped_key}/copy`` endpoint. * ``AlchemiscaleClient.merge_scopes`` for copying every ``AlchemicalNetwork`` in a set of source ``Scope``\s into a single target ``Scope``, via per-network ``copy_network`` calls. -* For all three methods, cloned ``Task``\s are wired to their ``Transformation``\s via ``PERFORMS`` so the standard ``(:AlchemicalNetwork)-[:DEPENDS_ON]->(:Transformation)<-[:PERFORMS]-(:Task)`` traversal sees them on the new network. Cloned ``Task``\s are intentionally **not** actioned to the new network's ``TaskHub``; callers wanting cloned ``Task``\s picked up by compute services should call ``action_tasks`` against the new network's ``TaskHub`` after the copy completes. +* For all three methods, only ``Task``\s in ``complete`` state on the source networks are cloned into the new network -- along with their ``ProtocolDAGResultRef``\s -- so previously-computed results do not need to be re-run. ``Task``\s in any other status (``waiting``, ``running``, ``error``, ``invalid``, ``deleted``) are not carried over. Cloned ``Task``\s are wired to their ``Transformation``\s via ``PERFORMS`` so the standard ``(:AlchemicalNetwork)-[:DEPENDS_ON]->(:Transformation)<-[:PERFORMS]-(:Task)`` traversal sees them on the new network. Cloned ``Task``\s are intentionally **not** actioned to the new network's ``TaskHub``; callers wanting them picked up by compute services should call ``action_tasks`` against the new network's ``TaskHub`` after the copy completes. * The source networks' execution-orchestration state -- any attached ``Strategy`` (``AlchemiscaleClient.set_network_strategy``) and any ``TaskRestartPattern``\s on the ``TaskHub`` (``AlchemiscaleClient.add_task_restart_patterns``) -- is intentionally **not** carried over by any of the three new methods, since these govern how ``Task``\s run rather than the results themselves. Set them explicitly on the new network afterward if needed. **Changed:** From 1e210f486ca567ec02fd525e917d00fca4a571a9 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Mon, 6 Jul 2026 16:32:13 -0600 Subject: [PATCH 32/35] Trim stale actioning caveats from merge/copy docstrings Under the complete-only Task carry-over policy, cloned Tasks are already terminal -- actioning them would be meaningless, since compute services only claim ``waiting`` Tasks. The "cloned Tasks are intentionally not actioned" language in the docstrings and user guide was carried over from the earlier design where non-complete Tasks could carry over. Drop the actioning caveat everywhere. Keep the Strategy / TaskRestartPatterns non-carry note (still meaningful, since users may want them re-established on the new network). Co-Authored-By: Claude Opus 4.7 --- alchemiscale/interface/client.py | 53 ++++---------- alchemiscale/storage/statestore.py | 71 ++++++------------- .../merging_and_copying_networks.rst | 7 +- news/issue-221.rst | 4 +- 4 files changed, 38 insertions(+), 97 deletions(-) diff --git a/alchemiscale/interface/client.py b/alchemiscale/interface/client.py index 53393f87..b4dd52eb 100644 --- a/alchemiscale/interface/client.py +++ b/alchemiscale/interface/client.py @@ -199,27 +199,15 @@ def merge_networks( The resulting AlchemicalNetwork contains the union of all Transformations and NonTransformations from the source networks. - Existing Tasks for those Transformations that are in ``complete`` - state are cloned into the new network's scope along with their - associated ProtocolDAGResultRefs, so previously-computed results do - not need to be re-run. Tasks in any other status (``waiting``, - ``running``, ``error``, ``invalid``, ``deleted``) are not carried - over. + Only Tasks in ``complete`` state on those Transformations are + cloned into the new network's scope along with their + ProtocolDAGResultRefs, so previously-computed results do not need + to be re-run. - Cloned Tasks are intentionally **not** actioned; call - :meth:`action_tasks` with the merged network's ScopedKey after the - merge completes if you want them picked up by compute services. - - The source networks' execution-orchestration state is intentionally - **not** carried over -- these govern how Tasks run rather than the - results themselves. Specifically: - - - Any ``Strategy`` set on a source network via - :meth:`set_network_strategy` is not copied. - - Any ``TaskRestartPattern`` s added to a source network's TaskHub - via :meth:`add_task_restart_patterns` are not copied. - - Set these on the merged network yourself after the merge if needed. + Any ``Strategy`` (:meth:`set_network_strategy`) and + ``TaskRestartPattern`` s (:meth:`add_task_restart_patterns`) on + the source networks are not carried over; set them on the merged + network afterward if needed. Parameters ---------- @@ -303,24 +291,12 @@ def copy_network( ) -> ScopedKey: """Copy an AlchemicalNetwork to a new Scope, carrying over every existing ``complete`` Task and its associated - ProtocolDAGResultRefs. Tasks in any other status (``waiting``, - ``running``, ``error``, ``invalid``, ``deleted``) are not carried - over. - - Cloned Tasks are intentionally **not** actioned; call - :meth:`action_tasks` with the new network's ``ScopedKey`` after - the copy completes if you want them picked up by compute services. - - The source network's execution-orchestration state is intentionally - **not** carried over -- these govern how Tasks run rather than the - results themselves. Specifically: - - - Any ``Strategy`` set on the source network via - :meth:`set_network_strategy` is not copied. - - Any ``TaskRestartPattern`` s added to the source network's TaskHub - via :meth:`add_task_restart_patterns` are not copied. + ProtocolDAGResultRefs. - Set these on the copy yourself after :meth:`copy_network` if needed. + Any ``Strategy`` (:meth:`set_network_strategy`) and + ``TaskRestartPattern`` s (:meth:`add_task_restart_patterns`) on + the source network are not carried over; set them on the copy + afterward if needed. Parameters ---------- @@ -398,8 +374,7 @@ def merge_scopes( Each source AlchemicalNetwork is copied via :meth:`copy_network`, preserving its name, its ``complete`` Tasks, and its associated - ProtocolDAGResultRefs. Tasks in any other status are not carried - over, and cloned Tasks are not actioned. + ProtocolDAGResultRefs. Parameters ---------- diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index d07b6c57..5330009e 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -190,9 +190,7 @@ def to_subgraph(self, target_scope, tf_node): Each cloned Task is wired back to the Transformation via a ``PERFORMS`` edge so the standard ``(:AlchemicalNetwork)-[:DEPENDS_ON]->(:Transformation)<-[:PERFORMS]-(:Task)`` - traversal keeps working on the merged network. Note that cloned - Tasks are intentionally **not** actioned to the merged network's - TaskHub; see :meth:`Neo4jStore.merge_networks` for the rationale. + traversal keeps working on the merged network. Parameters ---------- @@ -1089,31 +1087,17 @@ def merge_networks( """Merge multiple ``AlchemicalNetwork`` nodes into a new ``AlchemicalNetwork``. Each ``Transformation`` / ``NonTransformation`` in the input - networks is included exactly once in the new network. ``Task``\\ s - on the source networks that are in ``complete`` state are cloned - into the new network's ``Scope`` along with their - ``ProtocolDAGResultRef``\\ s and ``EXTENDS`` relationships, and - are wired to their ``Transformation`` via ``PERFORMS`` so they - are reachable from the standard network traversals. ``Task``\\ s - in any other status (``waiting``, ``running``, ``error``, - ``invalid``, ``deleted``) are not carried over. - - Cloned ``Task``\\ s are intentionally **not** actioned to the new - network's ``TaskHub``. Users wanting the cloned ``Task``\\ s - picked up by compute services should call :meth:`action_tasks` - against the new network's ``TaskHub`` after the merge. - - Execution-orchestration state on the source networks is - intentionally **not** carried over -- these govern *how* Tasks - run rather than the results themselves: - - - ``Strategy`` (``PROGRESSES`` relationship on the source AN) - - ``TaskRestartPattern``\\ s (``ENFORCES`` on the source - ``TaskHub``, ``APPLIES`` on individual ``Task``\\ s) - - Callers wanting either on the merged network should set them - explicitly after the merge via :meth:`set_network_strategy` and - :meth:`add_task_restart_patterns`. + networks is included exactly once in the new network. Only + ``Task``\\ s on the source networks in ``complete`` state are + cloned into the new network's ``Scope``, along with their + ``ProtocolDAGResultRef``\\ s and ``EXTENDS`` relationships, + wired to their ``Transformation`` via ``PERFORMS``. + + Any ``Strategy`` (``PROGRESSES``) and ``TaskRestartPattern``\\ s + (``ENFORCES`` / ``APPLIES``) on the source networks are not + carried over; set them on the merged network via + :meth:`set_network_strategy` and :meth:`add_task_restart_patterns` + if needed. Parameters ---------- @@ -1234,28 +1218,15 @@ def copy_network( ) -> ScopedKey: """Copy an ``AlchemicalNetwork`` to a new ``Scope``, carrying over every ``complete`` ``Task`` and its associated - ``ProtocolDAGResultRef``\\ s. ``Task``\\ s in any other status - (``waiting``, ``running``, ``error``, ``invalid``, ``deleted``) - are not carried over. - - ``Task`` clones are wired to their ``Transformation`` via - ``PERFORMS`` and to one another via ``EXTENDS`` where applicable. - They are **not** actioned to the new network's ``TaskHub`` - (consistent with :meth:`merge_networks`); callers wanting cloned - ``Task``\\ s to be picked up by compute services should call - :meth:`action_tasks` against the new network's ``TaskHub``. - - Execution-orchestration state on the source network is - intentionally **not** carried over -- these govern *how* Tasks - run rather than the results themselves: - - - ``Strategy`` (``PROGRESSES`` relationship on the source AN) - - ``TaskRestartPattern``\\ s (``ENFORCES`` on the source - ``TaskHub``, ``APPLIES`` on individual ``Task``\\ s) - - Callers wanting either on the copy should set them explicitly - after :meth:`copy_network` via :meth:`set_network_strategy` and - :meth:`add_task_restart_patterns`. + ``ProtocolDAGResultRef``\\ s, wired to their ``Transformation`` + via ``PERFORMS`` and to one another via ``EXTENDS`` where + applicable. + + Any ``Strategy`` (``PROGRESSES``) and ``TaskRestartPattern``\\ s + (``ENFORCES`` / ``APPLIES``) on the source network are not + carried over; set them on the copy via + :meth:`set_network_strategy` and :meth:`add_task_restart_patterns` + if needed. Parameters ---------- diff --git a/docs/user_guide/merging_and_copying_networks.rst b/docs/user_guide/merging_and_copying_networks.rst index 0b98376c..8b7884fe 100644 --- a/docs/user_guide/merging_and_copying_networks.rst +++ b/docs/user_guide/merging_and_copying_networks.rst @@ -19,12 +19,7 @@ This keeps the semantics simple: **the results of computed work are preserved; n .. note:: - None of the source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s' execution-orchestration state is carried over, since it governs *how* :py:class:`~alchemiscale.storage.models.Task`\s run rather than the results themselves: - - * Cloned :py:class:`~alchemiscale.storage.models.Task`\s are not *actioned* on the new network; call :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.action_tasks` on the new network afterward for any you want compute services to pick up. - * Any ``Strategy`` set on a source network via :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.set_network_strategy` is not copied. - * Any ``TaskRestartPattern``\s added to a source network via :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.add_task_restart_patterns` are not copied. - + Any ``Strategy`` (via :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.set_network_strategy`) or ``TaskRestartPattern``\s (via :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.add_task_restart_patterns`) on the source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s are not copied. Set these on the new :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` yourself after the merge or copy if you need them. diff --git a/news/issue-221.rst b/news/issue-221.rst index e0c046d7..35fb1855 100644 --- a/news/issue-221.rst +++ b/news/issue-221.rst @@ -3,8 +3,8 @@ * ``AlchemiscaleClient.merge_networks`` for combining multiple existing ``AlchemicalNetwork``\s into a new ``AlchemicalNetwork`` in a target ``Scope``. Backed by ``Neo4jStore.merge_networks`` and a new ``POST /networks/merge`` endpoint on the user API. * ``AlchemiscaleClient.copy_network`` for copying a single ``AlchemicalNetwork`` to a new ``Scope``. An optional ``name`` argument creates the copy under a new name (and therefore a new gufe key); when ``name`` is omitted, the source network's name and gufe key are preserved. Backed by ``Neo4jStore.copy_network`` and a new ``POST /networks/{network_scoped_key}/copy`` endpoint. * ``AlchemiscaleClient.merge_scopes`` for copying every ``AlchemicalNetwork`` in a set of source ``Scope``\s into a single target ``Scope``, via per-network ``copy_network`` calls. -* For all three methods, only ``Task``\s in ``complete`` state on the source networks are cloned into the new network -- along with their ``ProtocolDAGResultRef``\s -- so previously-computed results do not need to be re-run. ``Task``\s in any other status (``waiting``, ``running``, ``error``, ``invalid``, ``deleted``) are not carried over. Cloned ``Task``\s are wired to their ``Transformation``\s via ``PERFORMS`` so the standard ``(:AlchemicalNetwork)-[:DEPENDS_ON]->(:Transformation)<-[:PERFORMS]-(:Task)`` traversal sees them on the new network. Cloned ``Task``\s are intentionally **not** actioned to the new network's ``TaskHub``; callers wanting them picked up by compute services should call ``action_tasks`` against the new network's ``TaskHub`` after the copy completes. -* The source networks' execution-orchestration state -- any attached ``Strategy`` (``AlchemiscaleClient.set_network_strategy``) and any ``TaskRestartPattern``\s on the ``TaskHub`` (``AlchemiscaleClient.add_task_restart_patterns``) -- is intentionally **not** carried over by any of the three new methods, since these govern how ``Task``\s run rather than the results themselves. Set them explicitly on the new network afterward if needed. +* For all three methods, only ``Task``\s in ``complete`` state on the source networks are cloned into the new network -- along with their ``ProtocolDAGResultRef``\s -- so previously-computed results do not need to be re-run. ``Task``\s in any other status are not carried over. Cloned ``Task``\s are wired to their ``Transformation``\s via ``PERFORMS`` so the standard ``(:AlchemicalNetwork)-[:DEPENDS_ON]->(:Transformation)<-[:PERFORMS]-(:Task)`` traversal sees them on the new network. +* Any ``Strategy`` (``AlchemiscaleClient.set_network_strategy``) or ``TaskRestartPattern``\s (``AlchemiscaleClient.add_task_restart_patterns``) on the source networks are not carried over by any of the three new methods. Set them explicitly on the new network afterward if needed. **Changed:** From ee1318ad39d2d5e45986328387ac470780f9f961 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Mon, 6 Jul 2026 16:48:57 -0600 Subject: [PATCH 33/35] Link NonTransformation in merge/copy user guide Replace the bare ``NonTransformation`` reference in merging_and_copying_networks.rst with an intersphinx link to gufe.transformations.transformation.NonTransformation, matching the style used for Transformation elsewhere in the same paragraph. Co-Authored-By: Claude Opus 4.7 --- docs/user_guide/merging_and_copying_networks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user_guide/merging_and_copying_networks.rst b/docs/user_guide/merging_and_copying_networks.rst index 8b7884fe..ccd5eb53 100644 --- a/docs/user_guide/merging_and_copying_networks.rst +++ b/docs/user_guide/merging_and_copying_networks.rst @@ -37,7 +37,7 @@ Use :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_networks` >>> merged_sk -The new :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` contains the union of the :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s and ``NonTransformation``\s from every source network. +The new :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` contains the union of the :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s and :external+gufe:py:class:`~gufe.transformations.transformation.NonTransformation`\s from every source network. The source networks themselves are unchanged. The ``complete`` :py:class:`~alchemiscale.storage.models.Task`\s attached to those source :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s are cloned onto the merged network's :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s along with their :py:class:`~alchemiscale.storage.models.ProtocolDAGResultRef`\s, so previously-computed results do not need to be re-run. From ea94afae81e93a311652a02039c992917929b5c9 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Mon, 6 Jul 2026 17:05:50 -0600 Subject: [PATCH 34/35] Docs simplification. --- docs/user_guide/merging_and_copying_networks.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user_guide/merging_and_copying_networks.rst b/docs/user_guide/merging_and_copying_networks.rst index ccd5eb53..f0e3861b 100644 --- a/docs/user_guide/merging_and_copying_networks.rst +++ b/docs/user_guide/merging_and_copying_networks.rst @@ -58,8 +58,8 @@ Use :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.copy_network` to Like :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_networks`, only the source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\'s ``complete`` :py:class:`~alchemiscale.storage.models.Task`\s are carried over, together with their :py:class:`~alchemiscale.storage.models.ProtocolDAGResultRef`\s. -If ``name`` is not given, the source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\'s name is preserved and the copy has the same ``gufe.key`` as the source (so its :py:class:`~alchemiscale.models.ScopedKey` differs from the source's only in :py:class:`~alchemiscale.models.Scope`). -Pass ``name`` to rename the copy; this yields a fresh ``gufe.key`` derived from the renamed content:: +If ``name`` is not given, the source :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\'s name is preserved. +Pass ``name`` to rename the copy:: >>> renamed_copy_sk = asc.copy_network( ... network=an_sk, From a31c8e5fcfccc07ae98c1b2b0903c2358789750b Mon Sep 17 00:00:00 2001 From: David Dotson Date: Tue, 14 Jul 2026 10:32:01 -0600 Subject: [PATCH 35/35] Address final review: qualname guards, state preservation, hygiene Addresses six findings from the final PR review: 1. Add missing API-level tests for POST /networks/{sk}/copy: happy path, bad-target-scope, bad-source-scope. The source-scope auth path validate_scopes(source_sk.scope, token) at api.py:208 had no negative-path coverage; the test identity in test_client authorizes every scope, so client tests couldn't exercise it. 2. Push a qualname guard into Neo4jStore.merge_networks and Neo4jStore.copy_network so both HTTP and Python callers get the check. Previously only the client-side validated qualname; a caller hitting the HTTP endpoint directly with a Transformation ScopedKey passed validate_scopes and let the store build a "network" out of a Transformation. Add API-level test coverage for this via test_merge_networks_bad_qualname and test_copy_network_bad_qualname (both expect 422). 3. AlchemiscaleClient.merge_scopes now preserves each source network's state (active/inactive/invalid/deleted) instead of defaulting every copy to active. Previously a "move a whole scope" call silently resurrected soft-deleted or invalidated networks as active on the target. Add test_merge_scopes_preserves_source_state to lock in the behavior across all three exercised states. 4. Document merge_scopes' non-atomicity + safe-to-rerun property (via copy_network idempotence) in the docstring and user guide. 5. Drop stale line-number references in _TransformationData.to_subgraph comment; keep the pattern description. 6. Update _is_transformation_keyed_dict and _TransformationData docstrings to mention both merge_networks and copy_network as callers. Co-Authored-By: Claude Opus 4.7 --- alchemiscale/interface/client.py | 39 +++++-- alchemiscale/storage/statestore.py | 35 ++++-- .../interface/client/test_client.py | 53 +++++++++ .../tests/integration/interface/test_api.py | 109 ++++++++++++++++++ .../merging_and_copying_networks.rst | 6 +- news/issue-221.rst | 2 +- 6 files changed, 228 insertions(+), 16 deletions(-) diff --git a/alchemiscale/interface/client.py b/alchemiscale/interface/client.py index 2276b142..d61e8b83 100644 --- a/alchemiscale/interface/client.py +++ b/alchemiscale/interface/client.py @@ -377,7 +377,17 @@ def merge_scopes( Each source AlchemicalNetwork is copied via :meth:`copy_network`, preserving its name, its ``complete`` Tasks, and its associated - ProtocolDAGResultRefs. + ProtocolDAGResultRefs. Each copy also preserves its source + network's *state* (``active``, ``inactive``, ``invalid``, or + ``deleted``) so that ``merge_scopes`` is a faithful move of the + scope's contents rather than a silent reactivation. + + This method is not transactional: if a per-network + :meth:`copy_network` call fails partway through, earlier copies + are already committed and the failure propagates. Because + :meth:`copy_network` is idempotent -- re-copying an already-copied + network dedups onto the existing target node -- rerunning after a + failure is safe. Parameters ---------- @@ -411,10 +421,26 @@ def merge_scopes( ) # gather every AlchemicalNetwork across the source scopes up front, - # regardless of state, so the progress bar has an accurate total + # regardless of state, so the progress bar has an accurate total. + # Also fetch each source network's state so the copy preserves it + # rather than defaulting every copy to ``active``. network_sks: list[ScopedKey] = [] + network_states: list[str] = [] for sc in scopes: - network_sks.extend(self.query_networks(scope=sc, state=None)) + sc_sks = self.query_networks(scope=sc, state=None) + if not sc_sks: + continue + sc_states = self.get_networks_state(sc_sks) + network_sks.extend(sc_sks) + # ``get_networks_state`` returns ``None`` for any network the + # server did not find; that should not happen for SKs we just + # queried, but be defensive and fall back to ``active``. + network_states.extend( + s if s is not None else NetworkStateEnum.active.value for s in sc_states + ) + + def _do_copy(sk: ScopedKey, state: str) -> ScopedKey: + return self.copy_network(sk, target_scope, state=state, visualize=False) if visualize: from rich.progress import Progress @@ -426,13 +452,12 @@ def merge_scopes( total=len(network_sks), ) new_sks: list[ScopedKey] = [] - for sk in network_sks: - new_sks.append(self.copy_network(sk, target_scope, visualize=False)) + for sk, state in zip(network_sks, network_states): + new_sks.append(_do_copy(sk, state)) progress.update(task, advance=1) else: new_sks = [ - self.copy_network(sk, target_scope, visualize=False) - for sk in network_sks + _do_copy(sk, state) for sk, state in zip(network_sks, network_states) ] return new_sks diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index 5330009e..60e7792a 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -88,7 +88,8 @@ def _is_transformation_keyed_dict(keyed_dict: dict) -> bool: """Return ``True`` if ``keyed_dict`` represents a ``Transformation`` or ``NonTransformation``. Used as the predicate for ``KeyedChain.decode_subchains`` when walking - an ``AlchemicalNetwork``'s keyed chain inside :meth:`Neo4jStore.merge_networks`. + an ``AlchemicalNetwork``'s keyed chain in :meth:`Neo4jStore.merge_networks` + and :meth:`Neo4jStore.copy_network`. """ return keyed_dict.get("__qualname__") in ("Transformation", "NonTransformation") @@ -96,7 +97,7 @@ def _is_transformation_keyed_dict(keyed_dict: dict) -> bool: @dataclass class _TransformationData: """Bookkeeping for one Transformation as it is reconstructed during - :meth:`Neo4jStore.merge_networks`. + :meth:`Neo4jStore.merge_networks` or :meth:`Neo4jStore.copy_network`. Attributes ---------- @@ -105,7 +106,7 @@ class _TransformationData: ``GufeTokenizable``. task_tree Flat list of Neo4j records, one per ``Task`` associated with this - ``Transformation`` in any of the source networks. Each record carries: + ``Transformation`` in the source network(s). Each record carries: - ``tf_key``: this ``Transformation``'s decoded gufe key (string) - ``task``: the full ``Task`` node @@ -113,7 +114,7 @@ class _TransformationData: - ``pdrrs``: list of ``ProtocolDAGResultRef`` nodes for this ``Task`` known_scoped_keys All ``ScopedKey``\\ s representing this ``Transformation`` across the - source networks. Multiple ``ScopedKey``\\ s can map to a single + source network(s). Multiple ``ScopedKey``\\ s can map to a single decoded ``Transformation`` if the same content was serialized under different gufe versions. """ @@ -257,9 +258,9 @@ def get_or_create_task_node(record): # source query returns ``record["extended_task"]`` as the Task # that ``record["task"]`` extends -- i.e. in the source graph, # ``(task)-[:EXTENDS]->(extended_task)``. The relationship's - # direction must be preserved on the clone so the standard - # downstream traversals (e.g. lines 2392, 3152, 3547, 3634: - # ``(task)-[:EXTENDS]->(other_task)``) keep working. + # direction must be preserved on the clone so downstream + # traversals that read ``(task)-[:EXTENDS]->(other_task)`` + # keep working. etask_node = ( None if not record["extended_task"] @@ -1115,6 +1116,18 @@ def merge_networks( ------- The ``ScopedKey`` of the new ``AlchemicalNetwork`` in the database. """ + # reject non-AlchemicalNetwork ScopedKeys up front (both HTTP and + # Python callers land here; the client-side check alone would leave + # the HTTP endpoint open to Transformation/ChemicalSystem SKs) + non_networks = [ + sk for sk in network_scoped_keys if sk.qualname != "AlchemicalNetwork" + ] + if non_networks: + joined = ", ".join(str(sk) for sk in non_networks) + raise ValueError( + f"The following ScopedKey(s) do not refer to AlchemicalNetworks: {joined}" + ) + # Collect keyed chain representation for all alchemical networks, # gathering every missing ScopedKey up front so callers passing # many SKs get a single, complete error. @@ -1248,6 +1261,14 @@ def copy_network( ------- The ``ScopedKey`` of the new ``AlchemicalNetwork`` in the database. """ + # reject non-AlchemicalNetwork ScopedKeys up front (both HTTP and + # Python callers land here; the client-side check alone would leave + # the HTTP endpoint open to Transformation/ChemicalSystem SKs) + if network_scoped_key.qualname != "AlchemicalNetwork": + raise ValueError( + f"ScopedKey ({network_scoped_key}) does not refer to an AlchemicalNetwork" + ) + # decode the source network's keyed chain try: source_kc = self.get_keyed_chain(network_scoped_key) diff --git a/alchemiscale/tests/integration/interface/client/test_client.py b/alchemiscale/tests/integration/interface/client/test_client.py index 3505b2f6..2cffa331 100644 --- a/alchemiscale/tests/integration/interface/client/test_client.py +++ b/alchemiscale/tests/integration/interface/client/test_client.py @@ -762,6 +762,59 @@ def test_merge_scopes_creates_new_networks_in_target( ).records[0]["n"] assert fresh_count == 1 + def test_merge_scopes_preserves_source_state( + self, + scope_test, + multiple_scopes, + n4js_preloaded, + user_client: client.AlchemiscaleClient, + network_tyk2, + ): + """merge_scopes must copy each source network into the target scope + with the source network's *state*, not silently default every + copy to ``active``. Regressing this would resurrect + inactive/invalid/deleted networks as active on the target. + """ + source_scope = scope_test + target_scope = multiple_scopes[2] + + # add three fresh networks to source, one per exercised state, so + # we can observe the state flowing through cleanly without + # interference from n4js_preloaded's baseline + fresh_networks = { + "active": AlchemicalNetwork( + edges=list(network_tyk2.edges)[:2], name="state_test_active" + ), + "inactive": AlchemicalNetwork( + edges=list(network_tyk2.edges)[:2], name="state_test_inactive" + ), + "deleted": AlchemicalNetwork( + edges=list(network_tyk2.edges)[:2], name="state_test_deleted" + ), + } + expected_target_sks: dict[str, ScopedKey] = {} + for state, an in fresh_networks.items(): + sk = user_client.create_network(an, source_scope, state=state) + expected_target_sks[state] = ScopedKey( + gufe_key=sk.gufe_key, + org=target_scope.org, + campaign=target_scope.campaign, + project=target_scope.project, + ) + # sanity: source got the right state + assert user_client.get_network_state(sk) == state + + user_client.merge_scopes( + scopes=[source_scope], + target_scope=target_scope, + visualize=False, + ) + + # each copy must land in target_scope with the source's state + for state, target_sk in expected_target_sks.items(): + assert user_client.check_exists(target_sk) + assert user_client.get_network_state(target_sk) == state + def test_merge_scopes_rejects_empty( self, scope_test, diff --git a/alchemiscale/tests/integration/interface/test_api.py b/alchemiscale/tests/integration/interface/test_api.py index f1427147..549f4702 100644 --- a/alchemiscale/tests/integration/interface/test_api.py +++ b/alchemiscale/tests/integration/interface/test_api.py @@ -194,6 +194,115 @@ def test_merge_networks_bad_source_scope( assert "detail" in details assert str(unauth_scope) in details["detail"] + def test_merge_networks_bad_qualname(self, n4js_preloaded, test_client, scope_test): + """POST /networks/merge with a non-AlchemicalNetwork ScopedKey in + ``networks`` must be rejected at the store layer (422).""" + # a Transformation ScopedKey masquerading as a network SK + tf_sks = n4js_preloaded.query_transformations(scope=scope_test) + assert tf_sks + + headers = {"Content-type": "application/json"} + data = dict( + networks=[str(tf_sks[0])], + name="should_fail", + scope=scope_test.to_dict(), + ) + jsondata = json.dumps(data, cls=JSON_HANDLER.encoder) + + response = test_client.post("/networks/merge", data=jsondata, headers=headers) + assert response.status_code == 422 + assert "AlchemicalNetwork" in response.json()["detail"] + + def test_copy_network(self, n4js_preloaded, test_client, network_tyk2, scope_test): + n4js = n4js_preloaded + + # pick a source network in scope_test (pre-loaded) + source_sks = n4js.query_networks(scope=scope_test) + assert source_sks + source_sk = source_sks[0] + + # destination scope: reuse scope_test since the test_client's token + # only authorizes it; the copy dedups onto the preexisting node + # (name preserved, so gufe_key preserved, so ScopedKey preserved) + headers = {"Content-type": "application/json"} + data = dict(scope=scope_test.to_dict()) + jsondata = json.dumps(data, cls=JSON_HANDLER.encoder) + + response = test_client.post( + f"/networks/{source_sk}/copy", data=jsondata, headers=headers + ) + assert response.status_code == 200 + + copied_sk = ScopedKey(**response.json()) + assert copied_sk.scope == scope_test + assert copied_sk.gufe_key == source_sk.gufe_key + assert n4js.check_existence(copied_sk) + + def test_copy_network_bad_target_scope( + self, n4js_preloaded, test_client, scope_test, multiple_scopes + ): + """POST /networks/{sk}/copy with a destination scope the token + does not authorize must be denied (401).""" + # source in the authorized scope, destination in an unauthorized one + source_sks = n4js_preloaded.query_networks(scope=scope_test) + assert source_sks + source_sk = source_sks[0] + + bad_scope = multiple_scopes[1] + assert bad_scope != scope_test + + headers = {"Content-type": "application/json"} + data = dict(scope=bad_scope.to_dict()) + jsondata = json.dumps(data, cls=JSON_HANDLER.encoder) + + response = test_client.post( + f"/networks/{source_sk}/copy", data=jsondata, headers=headers + ) + assert response.status_code == 401 + details = response.json() + assert "detail" in details + assert str(bad_scope) in details["detail"] + + def test_copy_network_bad_source_scope( + self, n4js_preloaded, test_client, scope_test, multiple_scopes + ): + """POST /networks/{sk}/copy with a source network in a scope the + token does not authorize must be denied (401).""" + unauth_scope = multiple_scopes[1] + assert unauth_scope != scope_test + + unauth_source_sks = n4js_preloaded.query_networks(scope=unauth_scope) + assert unauth_source_sks + unauth_source_sk = unauth_source_sks[0] + + headers = {"Content-type": "application/json"} + data = dict(scope=scope_test.to_dict()) + jsondata = json.dumps(data, cls=JSON_HANDLER.encoder) + + response = test_client.post( + f"/networks/{unauth_source_sk}/copy", data=jsondata, headers=headers + ) + assert response.status_code == 401 + details = response.json() + assert "detail" in details + assert str(unauth_scope) in details["detail"] + + def test_copy_network_bad_qualname(self, n4js_preloaded, test_client, scope_test): + """POST /networks/{sk}/copy with a non-AlchemicalNetwork ScopedKey + must be rejected at the store layer (422).""" + tf_sks = n4js_preloaded.query_transformations(scope=scope_test) + assert tf_sks + + headers = {"Content-type": "application/json"} + data = dict(scope=scope_test.to_dict()) + jsondata = json.dumps(data, cls=JSON_HANDLER.encoder) + + response = test_client.post( + f"/networks/{tf_sks[0]}/copy", data=jsondata, headers=headers + ) + assert response.status_code == 422 + assert "AlchemicalNetwork" in response.json()["detail"] + def test_get_network(self, prepared_network, test_client): network, scoped_key = prepared_network response = test_client.get(f"/networks/{scoped_key}") diff --git a/docs/user_guide/merging_and_copying_networks.rst b/docs/user_guide/merging_and_copying_networks.rst index f0e3861b..711cd811 100644 --- a/docs/user_guide/merging_and_copying_networks.rst +++ b/docs/user_guide/merging_and_copying_networks.rst @@ -82,5 +82,9 @@ Use :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_scopes` to >>> len(new_sks) 17 -This is a convenience method: each :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` is copied via :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.copy_network` in turn, with names preserved. +This is a convenience method: each :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` is copied via :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.copy_network` in turn, with names *and* source states preserved. +Networks in ``inactive``, ``invalid``, or ``deleted`` state on a source :py:class:`~alchemiscale.models.Scope` land in the target :py:class:`~alchemiscale.models.Scope` in that same state, rather than being silently reactivated. It is useful for consolidating :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s from multiple :py:class:`~alchemiscale.models.Scope`\s into a shared workspace, or for relocating a whole :py:class:`~alchemiscale.models.Scope`\'s worth of :external+gufe:py:class:`~gufe.network.AlchemicalNetwork`\s. + +Note that :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.merge_scopes` is not transactional: if a per-network copy fails partway through, the copies that already succeeded remain in the target :py:class:`~alchemiscale.models.Scope`. +Because :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.copy_network` is idempotent -- re-copying an :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` that already exists in the target :py:class:`~alchemiscale.models.Scope` dedups onto the existing node -- rerunning after a failure is safe. diff --git a/news/issue-221.rst b/news/issue-221.rst index 35fb1855..00d31811 100644 --- a/news/issue-221.rst +++ b/news/issue-221.rst @@ -2,7 +2,7 @@ * ``AlchemiscaleClient.merge_networks`` for combining multiple existing ``AlchemicalNetwork``\s into a new ``AlchemicalNetwork`` in a target ``Scope``. Backed by ``Neo4jStore.merge_networks`` and a new ``POST /networks/merge`` endpoint on the user API. * ``AlchemiscaleClient.copy_network`` for copying a single ``AlchemicalNetwork`` to a new ``Scope``. An optional ``name`` argument creates the copy under a new name (and therefore a new gufe key); when ``name`` is omitted, the source network's name and gufe key are preserved. Backed by ``Neo4jStore.copy_network`` and a new ``POST /networks/{network_scoped_key}/copy`` endpoint. -* ``AlchemiscaleClient.merge_scopes`` for copying every ``AlchemicalNetwork`` in a set of source ``Scope``\s into a single target ``Scope``, via per-network ``copy_network`` calls. +* ``AlchemiscaleClient.merge_scopes`` for copying every ``AlchemicalNetwork`` in a set of source ``Scope``\s into a single target ``Scope``, via per-network ``copy_network`` calls. Each copy preserves its source network's *state* (``active``, ``inactive``, ``invalid``, or ``deleted``), so this method is a faithful move rather than a silent reactivation of soft-deleted or invalidated networks. It is not transactional, but is safe to rerun after a partial failure since ``copy_network`` is idempotent. * For all three methods, only ``Task``\s in ``complete`` state on the source networks are cloned into the new network -- along with their ``ProtocolDAGResultRef``\s -- so previously-computed results do not need to be re-run. ``Task``\s in any other status are not carried over. Cloned ``Task``\s are wired to their ``Transformation``\s via ``PERFORMS`` so the standard ``(:AlchemicalNetwork)-[:DEPENDS_ON]->(:Transformation)<-[:PERFORMS]-(:Task)`` traversal sees them on the new network. * Any ``Strategy`` (``AlchemiscaleClient.set_network_strategy``) or ``TaskRestartPattern``\s (``AlchemiscaleClient.add_task_restart_patterns``) on the source networks are not carried over by any of the three new methods. Set them explicitly on the new network afterward if needed.