Delete functional test resources in dependency order to fix LRT

.github/workflows/long-running-azure.yaml

@@ -410,16 +410,18 @@ jobs:
           GH_TOKEN: ${{ steps.get_installation_token.outputs.token }}
           GIT_HTTP_PASSWORD: ${{ env.GIT_HTTP_PASSWORD }}
           FUNCTEST_PREPROVISIONED_RESOURCE_JSON: ${{ secrets.FUNCTEST_PREPROVISIONED_RESOURCE_JSON }}
-          # The released v0.59 test code predates two incompatibilities with main:
+          # The released test code predates fixes and compatibility updates on main:
           #   - Test_ACI: tracked in #12044 (cleanup timeout flake).
           #   - Test_RecipePack, Test_RadiusCoreEnv: the v0.59 recipe-pack fixture
           #     uses the old recipeKind/recipeLocation properties that were renamed
           #     to kind/source in PR #12104 (merged 2026-06-22), four days after the
           #     v0.59.0 release (2026-06-18). The old names produce a BCP037 compile
           #     error against the mutable latest Bicep extension.
+          #   - Test_RadiusCore_AzureMySql_PortalLink: v0.60 deletes the environment
+          #     before its application, so recipe cleanup fails. Tracked in #12701.
           # Skip these exact tests until a release that includes the fixes is cut.
-          # See #12044, #12104, #12163, #12247, and #12392.
-          GOTEST_OPTS: "-skip='^(Test_ACI|Test_RecipePack|Test_RadiusCoreEnv)$$'"
+          # See #12044, #12104, #12163, #12247, #12392, and #12701.
+          GOTEST_OPTS: "-skip='^(Test_ACI|Test_RecipePack|Test_RadiusCoreEnv|Test_RadiusCore_AzureMySql_PortalLink)$$'"
         working-directory: ${{ steps.checkout-release-codebase.outputs.release-dir }}
         run: |
           set -euo pipefail

docs/contributing/contributing-code/contributing-code-tests/running-functional-tests.md

@@ -89,7 +89,7 @@ The Make targets accept these environment variables:
 Functional tests support two cleanup modes, selected with the `RADIUS_TEST_FAST_CLEANUP` environment variable:
 
 - **Standard cleanup** (default for local development): waits for each resource to be fully deleted before proceeding, logs the deletion process, and shows retries for resources stuck in "Updating". Best for debugging cleanup issues.
-- **Fast cleanup** (default for CI): initiates deletions in the background without waiting, which avoids deletion timeouts and dramatically reduces run time. It **skips post-delete verification**, so it is only safe for non-cloud tests where Kubernetes cluster cleanup handles orphaned resources. CI enables it with `RADIUS_TEST_FAST_CLEANUP=true`.
+- **Fast cleanup** (default for CI, including the cloud suites): initiates deletions in the background without waiting, which avoids deletion timeouts and dramatically reduces run time. It **skips post-delete verification and ignores deletion errors**, and relies on the per-run cluster and Azure resource group being deleted afterward. CI enables it with `RADIUS_TEST_FAST_CLEANUP=true`.
 
 ```bash
 # Enable fast cleanup (useful for local testing with unique resource names)
@@ -101,7 +101,13 @@ export RADIUS_TEST_FAST_CLEANUP=false
 go test ./test/functional-portable/corerp/noncloud/resources
 ```
 
-> ⚠️ **Important**: Fast cleanup is only safe for non-cloud tests. Cloud tests always use standard cleanup to ensure proper deletion of cloud resources that incur costs.
+> ⚠️ **Important**: Fast cleanup discards deletion errors, so a teardown bug stays invisible in any suite that runs with it enabled. The long-running test runs against a persistent cluster with standard cleanup, which is where such bugs surface.
+
+### Resource deletion order
+
+Tests declare the resources they expect in `RPResources`, and the framework deletes them in dependency order rather than declaration order: applications first, then application-scoped resources, then environments, and finally recipe packs. Declaration order is preserved within each of those groups.
+
+This ordering matters because deleting an application cascades into the resources it owns, and a recipe-backed resource loads its environment configuration to run the recipe's delete. Removing the environment or recipe pack first makes that delete fail with an `Internal` error wrapping a `NotFound` on the environment.
 
 ### See log output in VS Code
 

test/functional-portable/corerp/cloud/resources/radiuscore_azure_mysql_portallink_test.go

@@ -130,10 +130,10 @@ func Test_RadiusCore_AzureMySql_PortalLink(t *testing.T) {
 			).WithRetry(5*time.Minute, 60*time.Second, isTransientCloudDeployError),
 			RPResources: &validation.RPResourceSet{
 				Resources: []validation.RPResource{
-					{Name: recipePackName, Type: validation.CoreRecipePacksResource},
-					{Name: envName, Type: validation.CoreEnvironmentsResource},
 					{Name: appName, Type: validation.CoreApplicationsResource, App: appName},
 					{Name: mysqlName, Type: validation.DataMySQLDatabasesResource, App: appName},
+					{Name: envName, Type: validation.CoreEnvironmentsResource},
+					{Name: recipePackName, Type: validation.CoreRecipePacksResource},
 				},
 			},
 			SkipKubernetesOutputResourceValidation: true,

test/rp/rptest.go

@@ -608,7 +608,7 @@ func (ct RPTest) Test(t *testing.T) {
 			continue
 		}
 
-		for _, resource := range step.RPResources.Resources {
+		for _, resource := range validation.ResourcesInDeletionOrder(step.RPResources) {
 			t.Logf("deleting %s", resource.Name)
 
 			if ct.FastCleanup {

test/validation/shared.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"sort"
 	"strings"
 	"testing"
 
@@ -84,6 +85,63 @@ type RPResourceSet struct {
 	Resources []RPResource
 }
 
+// Deletion ranks order teardown so a resource is never removed before the
+// resources that depend on it. Lower ranks are deleted first.
+const (
+	// Applications are deleted first because deleting an application cascades
+	// into the resources it owns, and those resources still need their
+	// environment and recipe pack to run their recipe's delete.
+	deletionRankApplication = iota
+
+	// Application-scoped resources (containers, databases, extenders, ...) are
+	// deleted next. Most are already gone via the application cascade above.
+	deletionRankResource
+
+	// Environments are deleted after everything scoped to them, because a
+	// recipe-backed resource's delete loads its environment configuration.
+	deletionRankEnvironment
+
+	// Recipe packs are deleted last because environments reference them.
+	deletionRankRecipePack
+)
+
+// deletionRank returns the teardown rank for a resource type.
+func deletionRank(resourceType string) int {
+	switch {
+	case strings.EqualFold(resourceType, ApplicationsResource),
+		strings.EqualFold(resourceType, CoreApplicationsResource):
+		return deletionRankApplication
+	case strings.EqualFold(resourceType, EnvironmentsResource),
+		strings.EqualFold(resourceType, CoreEnvironmentsResource):
+		return deletionRankEnvironment
+	case strings.EqualFold(resourceType, CoreRecipePacksResource):
+		return deletionRankRecipePack
+	default:
+		return deletionRankResource
+	}
+}
+
+// ResourcesInDeletionOrder returns the resources of the set ordered so that teardown removes
+// applications first, then application-scoped resources, then environments, and finally recipe
+// packs. Declaration order is preserved within each group, and a nil set yields no resources.
+//
+// Ordering is applied here rather than left to each test's declaration order because deleting an
+// environment or recipe pack before the application that uses it makes the application's delete
+// fail: a recipe-backed resource loads its environment configuration to run the recipe's delete.
+func ResourcesInDeletionOrder(set *RPResourceSet) []RPResource {
+	if set == nil || len(set.Resources) == 0 {
+		return nil
+	}
+
+	ordered := make([]RPResource, len(set.Resources))
+	copy(ordered, set.Resources)
+	sort.SliceStable(ordered, func(i, j int) bool {
+		return deletionRank(ordered[i].Type) < deletionRank(ordered[j].Type)
+	})
+
+	return ordered
+}
+
 // DeleteRPResource deletes an environment or application resource depending on the type of the resource passed in, and
 // returns an error if one occurs.
 func DeleteRPResource(ctx context.Context, t *testing.T, cli *radcli.CLI, client clients.ApplicationsManagementClient, resource RPResource) error {

test/validation/shared_test.go

@@ -0,0 +1,145 @@
+/*
+Copyright 2023 The Radius Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package validation
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_ResourcesInDeletionOrder(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name     string
+		set      *RPResourceSet
+		expected []string
+	}{
+		{
+			name:     "nil set yields no resources",
+			set:      nil,
+			expected: nil,
+		},
+		{
+			name:     "empty set yields no resources",
+			set:      &RPResourceSet{},
+			expected: nil,
+		},
+		{
+			name: "dependencies are deleted after their dependents",
+			set: &RPResourceSet{
+				Resources: []RPResource{
+					{Name: "pack", Type: CoreRecipePacksResource},
+					{Name: "env", Type: CoreEnvironmentsResource},
+					{Name: "app", Type: CoreApplicationsResource},
+					{Name: "db", Type: DataMySQLDatabasesResource},
+				},
+			},
+			expected: []string{"app", "db", "env", "pack"},
+		},
+		{
+			name: "already ordered set is unchanged",
+			set: &RPResourceSet{
+				Resources: []RPResource{
+					{Name: "app", Type: CoreApplicationsResource},
+					{Name: "db", Type: DataMySQLDatabasesResource},
+					{Name: "env", Type: CoreEnvironmentsResource},
+					{Name: "pack", Type: CoreRecipePacksResource},
+				},
+			},
+			expected: []string{"app", "db", "env", "pack"},
+		},
+		{
+			name: "applications.core types are ordered like their radius.core equivalents",
+			set: &RPResourceSet{
+				Resources: []RPResource{
+					{Name: "env", Type: EnvironmentsResource},
+					{Name: "app", Type: ApplicationsResource},
+					{Name: "container", Type: ContainersResource},
+				},
+			},
+			expected: []string{"app", "container", "env"},
+		},
+		{
+			name: "declaration order is preserved within a group",
+			set: &RPResourceSet{
+				Resources: []RPResource{
+					{Name: "env", Type: CoreEnvironmentsResource},
+					{Name: "second", Type: ComputeContainersResource},
+					{Name: "first", Type: DataMySQLDatabasesResource},
+				},
+			},
+			expected: []string{"second", "first", "env"},
+		},
+		{
+			name: "resource types are matched case-insensitively",
+			set: &RPResourceSet{
+				Resources: []RPResource{
+					{Name: "env", Type: "Radius.Core/environments"},
+					{Name: "app", Type: "Radius.Core/applications"},
+				},
+			},
+			expected: []string{"app", "env"},
+		},
+		{
+			name: "unknown types are treated as application-scoped resources",
+			set: &RPResourceSet{
+				Resources: []RPResource{
+					{Name: "env", Type: CoreEnvironmentsResource},
+					{Name: "usertype", Type: "Test.Resources/userTypeAlphas"},
+					{Name: "app", Type: CoreApplicationsResource},
+				},
+			},
+			expected: []string{"app", "usertype", "env"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			ordered := ResourcesInDeletionOrder(tt.set)
+
+			names := make([]string, 0, len(ordered))
+			for _, resource := range ordered {
+				names = append(names, resource.Name)
+			}
+
+			if tt.expected == nil {
+				require.Empty(t, names)
+			} else {
+				require.Equal(t, tt.expected, names)
+			}
+		})
+	}
+}
+
+func Test_ResourcesInDeletionOrder_DoesNotMutateInput(t *testing.T) {
+	t.Parallel()
+
+	set := &RPResourceSet{
+		Resources: []RPResource{
+			{Name: "pack", Type: CoreRecipePacksResource},
+			{Name: "app", Type: CoreApplicationsResource},
+		},
+	}
+
+	_ = ResourcesInDeletionOrder(set)
+
+	require.Equal(t, "pack", set.Resources[0].Name)
+	require.Equal(t, "app", set.Resources[1].Name)
+}