Merge 'release-1.11' to 'develop' with resolved conflicts.

.github/workflows/repository-dispatch.yml

@@ -37,6 +37,7 @@ jobs:
         repo:
           - "opsmill/infrahub-demo-dc"
           - "opsmill/infrahub-demo-sp"
+          - "opsmill/infrahub-solution-ai-dc"
           - "INFRAHUB_PACKER_REPOSITORY"
           - "INFRAHUB_ENTERPRISE_REPOSITORY"
           - "INFRAHUB_CUSTOMER1_REPOSITORY"

.vale/styles/spelling-exceptions.txt

@@ -153,13 +153,15 @@ menu_placement
 metadata
 microservices
 middleware
+misconfiguration
 misconfigured
 mixin
 modeline
 modularization
 namespace
 namespaces
 nats
+netmask
 Nautobot
 Neo4j
 netmask
@@ -208,6 +210,8 @@ rebase
 Rebase
 rebased
 Rebased
+rebases
+Rebases
 rebasing
 Rebasing
 redis

backend/AGENTS.md

@@ -122,6 +122,7 @@ Each entry says *when* to load it — open the doc before working in that area.
 - `dev/guides/backend/creating-events.md` - Creating new events
 - `dev/guides/backend/creating-async-tasks.md` - How to create an async task, with a pre-submit checklist. Load when adding a `@task`/`@flow`.
 - `dev/guides/backend/creating-messages.md` - Creating message bus messages
+- `dev/guides/backend/creating-migrations.md` - Choosing a migration base class, `GRAPH_VERSION` bookkeeping, batching, and transaction retry. Load when adding a graph or schema migration.
 
 ### ADRs (Why we decided)
 

backend/infrahub/cli/db_commands/check_inheritance.py

@@ -252,20 +252,15 @@ async def check_inheritance(db: InfrahubDatabase, fix: bool = False) -> bool:
             migration_query = await NodeDuplicateQuery.init(
                 db=db,
                 branch=branches_by_name[branch_name],
-                previous_node=SchemaNodeInfo(
-                    name=node_schema.name,
-                    namespace=node_schema.namespace,
-                    branch_support=node_schema.branch.value,
-                    labels=list(kind_label_count.labels) + [kind_label_count.kind, InfrahubKind.NODE],
-                    kind=kind_label_count.kind,
-                ),
-                new_node=SchemaNodeInfo(
-                    name=node_schema.name,
-                    namespace=node_schema.namespace,
-                    branch_support=node_schema.branch.value,
-                    labels=list(node_schema.inherit_from) + [kind_label_count.kind, InfrahubKind.NODE],
-                    kind=kind_label_count.kind,
-                ),
+                kind_updates_map={
+                    kind_label_count.kind: SchemaNodeInfo(
+                        name=node_schema.name,
+                        namespace=node_schema.namespace,
+                        branch_support=node_schema.branch.value,
+                        labels=list(node_schema.inherit_from) + [kind_label_count.kind, InfrahubKind.NODE],
+                        kind=kind_label_count.kind,
+                    )
+                },
             )
             await migration_query.execute(db=db)
             rprint("done")

backend/infrahub/cli/tasks.py

@@ -16,6 +16,10 @@
 
 app = AsyncTyper()
 
+# States a flow run can be stuck in: RUNNING for a worker that died mid-flow, PENDING for one
+# that was picked up but never got to start.
+STALE_FLOW_RUN_STATES = [StateType.RUNNING, StateType.PENDING]
+
 
 @app.command()
 async def init(
@@ -89,7 +93,7 @@ async def stale_runs(
     days_to_keep: int = 2,
     batch_size: int = 100,
 ) -> None:
-    """Flush stale task runs."""
+    """Flush stale task runs, marking long RUNNING or PENDING flow runs as crashed."""
     logging.getLogger("infrahub").setLevel(logging.WARNING)
     logging.getLogger("neo4j").setLevel(logging.ERROR)
     logging.getLogger("prefect").setLevel(logging.ERROR)
@@ -98,5 +102,5 @@ async def stale_runs(
 
     async with get_client(sync_client=False) as client:
         await FlowRunRetention(client=PrefectClientAdapter(client)).purge(
-            states=[StateType.RUNNING], delete=False, days_to_keep=days_to_keep, batch_size=batch_size
+            states=STALE_FLOW_RUN_STATES, delete=False, days_to_keep=days_to_keep, batch_size=batch_size
         )

backend/infrahub/core/constants/__init__.py

@@ -407,6 +407,9 @@ class MetadataOptions(Flag):
     USER_TIMESTAMPS = TIMESTAMPS | USERS
 
 
+PROFILE_NAMESPACE = "Profile"
+TEMPLATE_NAMESPACE = "Template"
+
 RESTRICTED_NAMESPACES: list[str] = [
     "Account",
     "Branch",
@@ -418,8 +421,8 @@ class MetadataOptions(Flag):
     "Internal",
     "Lineage",
     "Schema",
-    "Profile",
-    "Template",
+    PROFILE_NAMESPACE,
+    TEMPLATE_NAMESPACE,
 ]
 
 NODE_NAME_REGEX = r"^[A-Z][a-zA-Z0-9]+$"

backend/infrahub/core/migrations/graph/m012_convert_account_generic.py

@@ -113,14 +113,15 @@ def __init__(self, **kwargs: Any) -> None:
 
         kwargs.pop("branch", None)
 
-        super().__init__(new_node=new_node, previous_node=previous_node, branch=branch, **kwargs)
+        super().__init__(kind_updates_map={previous_node.kind: new_node}, branch=branch, **kwargs)
 
     def render_match(self) -> str:
-        return f"""
+        return """
         // Find all the active nodes
-        MATCH (node:{self.previous_node.kind})
+        MATCH (node:%(previous_kinds)s)
         WHERE NOT "CoreGenericAccount" IN LABELS(node)
-        """
+        WITH node, $kind_map[node.kind] AS target
+        """ % {"previous_kinds": self.previous_kinds}
 
 
 class Migration012RenameTypeAttributeSchema(SchemaAttributeUpdateQuery):

backend/infrahub/core/migrations/query/attribute_rename.py

@@ -1,17 +1,13 @@
 from __future__ import annotations
 
-from itertools import starmap
 from typing import TYPE_CHECKING, Any
 
 from pydantic import BaseModel
 
 from infrahub.core.constants import BranchSupportType, RelationshipStatus
-from infrahub.core.graph.schema import GraphAttributeRelationships
 from infrahub.core.query import Query
 
 if TYPE_CHECKING:
-    from pydantic.fields import FieldInfo
-
     from infrahub.database import InfrahubDatabase
 
 
@@ -36,56 +32,15 @@ def __init__(
         super().__init__(**kwargs)
 
     def render_match(self) -> str:
+        """Select the nodes to rename the attribute on. Subclasses narrow this to add their own guards."""
         return """
-        // Find all the active nodes
-        CALL () {
-            MATCH (node:%(node_kind)s)
-            WHERE exists((node)-[:HAS_ATTRIBUTE]-(:Attribute { name: $prev_attr.name }))
-            RETURN node
-            UNION
-            MATCH (node:Profile%(node_kind)s)
-            WHERE exists((node)-[:HAS_ATTRIBUTE]-(:Attribute { name: $prev_attr.name }))
-            RETURN node
-            UNION
-            MATCH (node:Template%(node_kind)s)
-            WHERE exists((node)-[:HAS_ATTRIBUTE]-(:Attribute { name: $prev_attr.name }))
-            RETURN node
-        }
-        WITH node
+        // --------------
+        // Find all possible nodes
+        // --------------
+        MATCH (node:%(node_kind)s|Profile%(node_kind)s|Template%(node_kind)s)
+        WHERE exists((node)-[:HAS_ATTRIBUTE]-(:Attribute { name: $prev_attr.name }))
         """ % {"node_kind": self.previous_attr.node_kind}
 
-    @staticmethod
-    def _render_sub_query_per_rel_type_update_active(rel_type: str, rel_def: FieldInfo) -> str:
-        subquery = [
-            "WITH peer_node, rb, active_attr",
-            f'WHERE type(rb) = "{rel_type}"',
-        ]
-        if rel_def.default.direction.value == "outbound":
-            subquery.append(f"CREATE (active_attr)-[:{rel_type} $rel_props_delete ]->(peer_node)")
-        elif rel_def.default.direction.value == "inbound":
-            subquery.append(f"CREATE (active_attr)<-[:{rel_type} $rel_props_delete ]-(peer_node)")
-        else:
-            subquery.append(f"CREATE (active_attr)-[:{rel_type} $rel_props_delete ]-(peer_node)")
-
-        subquery.append("RETURN peer_node as p2")
-        return "\n".join(subquery)
-
-    @staticmethod
-    def _render_sub_query_per_rel_type_create_new(rel_type: str, rel_def: FieldInfo) -> str:
-        subquery = [
-            "WITH peer_node, rb, active_attr, new_attr",
-            f'WHERE type(rb) = "{rel_type}"',
-        ]
-        if rel_def.default.direction.value == "outbound":
-            subquery.append(f"CREATE (new_attr)-[:{rel_type} $rel_props_create ]->(peer_node)")
-        elif rel_def.default.direction.value == "inbound":
-            subquery.append(f"CREATE (new_attr)<-[:{rel_type} $rel_props_create ]-(peer_node)")
-        else:
-            subquery.append(f"CREATE (new_attr)-[:{rel_type} $rel_props_create ]-(peer_node)")
-
-        subquery.append("RETURN peer_node as p2")
-        return "\n".join(subquery)
-
     async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None:  # noqa: ARG002
         branch_filter, branch_params = self.branch.get_query_filter_path(at=self.at.to_string())
         self.params.update(branch_params)
@@ -117,76 +72,102 @@ async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None:  # noqa
         # Set metadata for vertex properties on default/global branch
         self.params["set_metadata"] = self.branch.is_default or self.branch.is_global
 
-        sub_queries_create = list(
-            starmap(self._render_sub_query_per_rel_type_create_new, GraphAttributeRelationships.model_fields.items())
-        )
-        sub_query_create_all = "\nUNION\n".join(sub_queries_create)
-
-        sub_queries_update = list(
-            starmap(self._render_sub_query_per_rel_type_update_active, GraphAttributeRelationships.model_fields.items())
-        )
-        sub_query_update_all = "\nUNION\n".join(sub_queries_update)
-
         self.add_to_query(self.render_match())
 
         add_uuid = db.render_uuid_generation(node_label="new_attr", node_attr="uuid")
         query = """
+        // --------------
+        // Filter to just the active nodes
+        // --------------
         CALL (node) {
             MATCH (root:Root)<-[r:IS_PART_OF]-(node)
             WHERE %(branch_filter)s
-            RETURN node as n1, r as r1
+            RETURN r
             ORDER BY r.branch_level DESC, r.from DESC
             LIMIT 1
         }
-        WITH n1 as active_node, r1 as rb
-        WHERE rb.status = "active"
+        WITH node as active_node
+        WHERE r.status = "active"
+        // --------------
         // Find all the attributes that need to be updated
+        // --------------
         CALL (active_node) {
-            MATCH (active_node)-[r:HAS_ATTRIBUTE]-(attr:Attribute { name: $prev_attr.name })
+            MATCH (active_node)-[r:HAS_ATTRIBUTE]-(active_attr:Attribute { name: $prev_attr.name })
             WHERE %(branch_filter)s
-            RETURN active_node as n1, r as r1, attr as attr1
+            RETURN r, active_attr
             ORDER BY r.branch_level DESC, r.from DESC
             LIMIT 1
         }
-        WITH n1 as active_node, r1 as rb, attr1 as active_attr
-        WHERE rb.status = "active"
+        WITH active_node, active_attr
+        WHERE r.status = "active"
+        // --------------
+        // Create the new attribute vertexes
+        // --------------
         CREATE (new_attr:Attribute { name: $new_attr.name, branch_support: $new_attr.branch_support })
         %(add_uuid)s
         WITH active_node, active_attr, new_attr
-        MATCH (active_attr)-[]-(peer)
-        WITH DISTINCT active_node, active_attr, new_attr, peer
-        CALL (active_attr, peer) {
-            MATCH (active_attr)-[r]-(peer)
+        MATCH (active_attr)-[]-(peer_node)
+        WITH DISTINCT active_node, active_attr, new_attr, peer_node
+        CALL (active_attr, peer_node) {
+            MATCH (active_attr)-[r]-(peer_node)
             WHERE %(branch_filter)s
-            RETURN active_attr as a1, r as r1, peer as p1
+            RETURN r
             ORDER BY r.branch_level DESC, r.from DESC
             LIMIT 1
         }
-        WITH active_node, a1 as active_attr, r1 as rb, p1 as peer_node, new_attr
-        WHERE rb.status = "active"
-        CALL (peer_node, rb, active_attr, new_attr){
-            %(sub_query_create_all)s
+        WITH active_node, active_attr, r as r, peer_node, new_attr
+        WHERE r.status = "active"
+        // --------------
+        // Copy every edge of the old attribute onto the new one, preserving its direction
+        // --------------
+        CALL (peer_node, r, new_attr) {
+            WITH peer_node, r, new_attr
+            WHERE startNode(r) = peer_node
+            CREATE (new_attr)<-[:$(type(r)) $rel_props_create ]-(peer_node)
+        }
+        CALL (peer_node, r, new_attr) {
+            WITH peer_node, r, new_attr
+            WHERE endNode(r) = peer_node
+            CREATE (new_attr)-[:$(type(r)) $rel_props_create ]->(peer_node)
         }
-        WITH p2 as peer_node, rb, new_attr, active_attr, active_node
-        """ % {"branch_filter": branch_filter, "add_uuid": add_uuid, "sub_query_create_all": sub_query_create_all}
+        """ % {"branch_filter": branch_filter, "add_uuid": add_uuid}
         self.add_to_query(query)
 
         if not (self.branch.is_default or self.branch.is_global):
             query = """
-            CALL (peer_node, rb, active_attr) {
-                %(sub_query_update_all)s
+            // --------------
+            // An edge owned by another branch cannot be modified from here, so the old attribute is
+            // ended by shadowing it with a deleted edge; the ones this branch owns are closed below
+            // --------------
+            CALL (peer_node, r, active_attr) {
+                WITH peer_node, r, active_attr
+                WHERE r.branch <> $branch_name AND startNode(r) = peer_node
+                CREATE (active_attr)<-[:$(type(r)) $rel_props_delete ]-(peer_node)
+            }
+            CALL (peer_node, r, active_attr) {
+                WITH peer_node, r, active_attr
+                WHERE r.branch <> $branch_name AND endNode(r) = peer_node
+                CREATE (active_attr)-[:$(type(r)) $rel_props_delete ]->(peer_node)
+            }
+            CALL (r) {
+                WITH r
+                WHERE r.branch = $branch_name
+                SET r.to = $current_time, r.to_user_id = $user_id
             }
-            WITH p2 as peer_node, rb, new_attr
             RETURN DISTINCT new_attr
-            """ % {"sub_query_update_all": sub_query_update_all}
+            """
             self.add_to_query(query)
         else:
             query = """
-            FOREACH (i in CASE WHEN rb.branch = $branch_name THEN [1] ELSE [] END |
-                SET rb.to = $current_time, rb.to_user_id = $user_id
-            )
+            CALL (r) {
+                WITH r
+                WHERE r.branch = $branch_name
+                SET r.to = $current_time, r.to_user_id = $user_id
+            }
             WITH new_attr, active_node
+            // --------------
             // Set metadata on new Attribute and Node vertices if on default/global branch
+            // --------------
             CALL (new_attr, active_node) {
                 WITH new_attr, active_node
                 WHERE $set_metadata