refac: make admin network errors recoverable when cached data exists

web-admin/src/client/utils.ts

@@ -4,7 +4,9 @@ import type { AxiosError } from "axios";
 import { derived, type Readable } from "svelte/store";
 
 export function isAdminServerQuery(query: Query): boolean {
-  const [apiPath] = query.queryKey as string[];
+  const [apiPath] = query.queryKey as unknown[];
+  if (typeof apiPath !== "string") return false;
+
   const adminApiEndpoints = [
     "/v1/deployments",
     "/v1/github",

web-admin/src/components/errors/admin-network-errors.spec.ts

@@ -0,0 +1,129 @@
+import { isAdminServerQuery } from "@rilldata/web-admin/client/utils";
+import {
+  clearAdminNetworkErrorState,
+  handleAdminServerNetworkError,
+  handleAdminServerQuerySuccess,
+  recoverFromAdminNetworkError,
+} from "@rilldata/web-admin/components/errors/admin-network-errors";
+import { errorStore } from "@rilldata/web-admin/components/errors/error-store";
+import { eventBus } from "@rilldata/web-common/lib/event-bus/event-bus";
+import type { Query, QueryClient } from "@tanstack/svelte-query";
+import { get } from "svelte/store";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+type TestQueryClient = Pick<QueryClient, "refetchQueries">;
+
+function makeQuery(queryKey: unknown[], data?: unknown): Query {
+  return {
+    queryKey,
+    state: { data },
+  } as unknown as Query;
+}
+
+function makeQueryClient(): TestQueryClient {
+  return {
+    refetchQueries: vi.fn(() => Promise.resolve()),
+  } as unknown as TestQueryClient;
+}
+
+describe("admin network errors", () => {
+  beforeEach(() => {
+    errorStore.reset();
+    vi.spyOn(eventBus, "emit");
+  });
+
+  afterEach(() => {
+    clearAdminNetworkErrorState();
+    vi.restoreAllMocks();
+  });
+
+  it("keeps cached data visible and shows a banner for admin network errors", () => {
+    const queryClient = makeQueryClient();
+    const handled = handleAdminServerNetworkError(
+      new Error("Network Error"),
+      makeQuery(["/v1/projects/org/proj"], { project: { name: "proj" } }),
+      queryClient,
+    );
+
+    expect(handled).toBe(true);
+    expect(get(errorStore).header).toBe("");
+    expect(eventBus.emit).toHaveBeenCalledWith(
+      "add-banner",
+      expect.objectContaining({
+        id: "admin-network",
+        message: expect.objectContaining({
+          message: expect.stringContaining("Showing cached data"),
+          type: "warning",
+        }),
+      }),
+    );
+  });
+
+  it("uses the full-page error only when there is no cached data", () => {
+    const handled = handleAdminServerNetworkError(
+      new Error("Network Error"),
+      makeQuery(["/v1/projects/org/proj"]),
+      makeQueryClient(),
+    );
+
+    expect(handled).toBe(true);
+    expect(get(errorStore).header).toBe("Network Error");
+    expect(eventBus.emit).not.toHaveBeenCalledWith(
+      "add-banner",
+      expect.anything(),
+    );
+  });
+
+  it("ignores non-admin query network errors", () => {
+    const handled = handleAdminServerNetworkError(
+      new Error("Network Error"),
+      makeQuery(["/v1/instances/runtime/api/query"], { rows: [] }),
+      makeQueryClient(),
+    );
+
+    expect(handled).toBe(false);
+    expect(get(errorStore).header).toBe("");
+    expect(eventBus.emit).not.toHaveBeenCalledWith(
+      "add-banner",
+      expect.anything(),
+    );
+  });
+
+  it("clears the banner after a successful admin query", () => {
+    handleAdminServerNetworkError(
+      new Error("Network Error"),
+      makeQuery(["/v1/projects/org/proj"], { project: { name: "proj" } }),
+      makeQueryClient(),
+    );
+    vi.mocked(eventBus.emit).mockClear();
+
+    handleAdminServerQuerySuccess(makeQuery(["/v1/users/me"], { user: {} }));
+
+    expect(eventBus.emit).toHaveBeenCalledWith(
+      "remove-banner",
+      "admin-network",
+    );
+  });
+
+  it("refetches active admin queries during recovery", async () => {
+    const queryClient = makeQueryClient();
+    handleAdminServerNetworkError(
+      new Error("Network Error"),
+      makeQuery(["/v1/projects/org/proj"], { project: { name: "proj" } }),
+      queryClient,
+    );
+
+    await recoverFromAdminNetworkError(queryClient);
+
+    expect(queryClient.refetchQueries).toHaveBeenCalledWith({
+      type: "active",
+      predicate: isAdminServerQuery,
+    });
+  });
+
+  it("treats non-string query keys as non-admin queries", () => {
+    expect(isAdminServerQuery(makeQuery([{ path: "/v1/projects" }]))).toBe(
+      false,
+    );
+  });
+});

web-admin/src/components/errors/admin-network-errors.ts

@@ -0,0 +1,110 @@
+import { isAdminServerQuery } from "@rilldata/web-admin/client/utils";
+import { errorStore } from "@rilldata/web-admin/components/errors/error-store";
+import { createUserFacingError } from "@rilldata/web-admin/components/errors/user-facing-errors";
+import { eventBus } from "@rilldata/web-common/lib/event-bus/event-bus";
+import type { Query, QueryClient } from "@tanstack/svelte-query";
+
+export const AdminNetworkErrorMessage = "Network Error";
+
+const AdminNetworkBannerID = "admin-network";
+const AdminNetworkBannerPriority = 0;
+
+type QueryRefetcher = Pick<QueryClient, "refetchQueries">;
+
+let adminNetworkErrorActive = false;
+
+export function isNetworkError(error: unknown): boolean {
+  return error instanceof Error && error.message === AdminNetworkErrorMessage;
+}
+
+export function handleAdminServerNetworkError(
+  error: unknown,
+  query: Query,
+  queryClient: QueryRefetcher,
+): boolean {
+  if (!isNetworkError(error) || !isAdminServerQuery(query)) return false;
+
+  adminNetworkErrorActive = true;
+
+  if (queryHasCachedData(query)) {
+    showAdminNetworkBanner(queryClient);
+  } else {
+    showAdminNetworkErrorPage();
+  }
+
+  return true;
+}
+
+export function handleAdminServerQuerySuccess(query: Query): void {
+  if (!adminNetworkErrorActive || !isAdminServerQuery(query)) return;
+  clearAdminNetworkErrorState();
+}
+
+export function recoverFromAdminNetworkError(
+  queryClient: QueryRefetcher,
+): Promise<unknown> {
+  if (!adminNetworkErrorActive) return Promise.resolve();
+
+  clearAdminNetworkErrorState();
+  return queryClient.refetchQueries({
+    type: "active",
+    predicate: isAdminServerQuery,
+  });
+}
+
+export function registerAdminNetworkRecoveryListeners(
+  queryClient: QueryRefetcher,
+): () => void {
+  if (typeof window === "undefined") return () => {};
+
+  const recover = () => {
+    void recoverFromAdminNetworkError(queryClient);
+  };
+  const recoverWhenVisible = () => {
+    if (document.visibilityState !== "hidden") recover();
+  };
+
+  window.addEventListener("online", recover);
+  window.addEventListener("focus", recoverWhenVisible);
+  document.addEventListener("visibilitychange", recoverWhenVisible);
+
+  return () => {
+    window.removeEventListener("online", recover);
+    window.removeEventListener("focus", recoverWhenVisible);
+    document.removeEventListener("visibilitychange", recoverWhenVisible);
+  };
+}
+
+export function clearAdminNetworkErrorState(): void {
+  adminNetworkErrorActive = false;
+  errorStore.reset();
+  eventBus.emit("remove-banner", AdminNetworkBannerID);
+}
+
+function queryHasCachedData(query: Query): boolean {
+  return query.state.data !== undefined;
+}
+
+function showAdminNetworkErrorPage(): void {
+  errorStore.set(createUserFacingError(null, AdminNetworkErrorMessage));
+}
+
+function showAdminNetworkBanner(queryClient: QueryRefetcher): void {
+  eventBus.emit("add-banner", {
+    id: AdminNetworkBannerID,
+    priority: AdminNetworkBannerPriority,
+    message: {
+      type: "warning",
+      iconType: "alert",
+      message:
+        "Connection to Rill Cloud was interrupted. Showing cached data while we reconnect.",
+      cta: {
+        type: "button",
+        text: "Retry now",
+        async onClick() {
+          await recoverFromAdminNetworkError(queryClient);
+        },
+      },
+    },
+  });
+}

web-admin/src/routes/+layout.svelte

@@ -1,8 +1,10 @@
 <script lang="ts">
   import { page } from "$app/stores";
-  import { isAdminServerQuery } from "@rilldata/web-admin/client/utils";
-  import { errorStore } from "@rilldata/web-admin/components/errors/error-store";
-  import { createUserFacingError } from "@rilldata/web-admin/components/errors/user-facing-errors";
+  import {
+    handleAdminServerNetworkError,
+    handleAdminServerQuerySuccess,
+    registerAdminNetworkRecoveryListeners,
+  } from "@rilldata/web-admin/components/errors/admin-network-errors";
   import { dynamicHeight } from "@rilldata/web-common/layout/layout-settings.ts";
   import BillingBannerManager from "@rilldata/web-admin/features/billing/banner/BillingBannerManager.svelte";
   import {
@@ -63,13 +65,14 @@
     // Add TanStack Query errors to telemetry
     errorEventHandler?.requestErrorEventHandler(error, query);
 
-    // Handle network errors
-    // Note: ideally, we'd throw this in the root `+layout.ts` file, but we're blocked by
-    // https://github.com/sveltejs/kit/issues/10201
-    const errorMessage = error instanceof Error ? error.message : String(error);
-    if (isAdminServerQuery(query) && errorMessage === "Network Error") {
-      errorStore.set(createUserFacingError(null, errorMessage));
-    }
+    handleAdminServerNetworkError(error, query, queryClient);
+  };
+
+  queryClient.getQueryCache().config.onSuccess = (
+    _data: unknown,
+    query: Query,
+  ) => {
+    handleAdminServerQuerySuccess(query);
   };
 
   // The admin server enables some dashboard features like scheduled reports and alerts
@@ -87,7 +90,13 @@
   initPylonWidget();
 
   onMount(() => {
-    return () => removeJavascriptListeners?.();
+    const removeNetworkRecoveryListeners =
+      registerAdminNetworkRecoveryListeners(queryClient);
+
+    return () => {
+      removeJavascriptListeners?.();
+      removeNetworkRecoveryListeners();
+    };
   });
 
   $: isEmbed = isEmbedPage($page);

web-common/src/lib/svelte-query/globalQueryClient.ts

@@ -1,13 +1,21 @@
 import { QueryClient } from "@tanstack/svelte-query";
 
+const MaxNetworkErrorRetries = 2;
+
+function isNetworkError(error: unknown): boolean {
+  return error instanceof Error && error.message === "Network Error";
+}
+
 export function createQueryClient() {
   return new QueryClient({
     defaultOptions: {
       queries: {
         refetchOnMount: false,
         refetchOnReconnect: false,
         refetchOnWindowFocus: false,
-        retry: false,
+        retry: (failureCount, error) =>
+          isNetworkError(error) && failureCount < MaxNetworkErrorRetries,
+        retryDelay: (failureCount) => Math.min(1000 * 2 ** failureCount, 4000),
         networkMode: "always",
       },
       mutations: {