fix: cache readToolkitData lookups and align category sources

Deduplicate production readToolkitData calls so generateMetadata and Page
share one in-flight read, recover failed cache entries, and drop stale
"others" references from broken-link-check and sidebar sync docs.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: teallarson

app/_lib/toolkit-data.ts

@@ -128,8 +128,23 @@ type ToolkitDataMap = {
  * unbounded process-global cache.
  */
 const loadsByDataDir = new Map<string, Promise<ToolkitDataMap>>();
+/**
+ * One in-flight/resolved read per (dataDir, normalized lookup key) in
+ * production. `readToolkitData`'s direct-file fast path re-reads and
+ * re-parses on every call without this — `loadAllToolkitData` only caches
+ * the directory scan fallback, so generateMetadata + Page for the same
+ * toolkit would each pay for a full file read during static generation.
+ *
+ * Skipped in development (generator can refresh JSON while `next dev` runs)
+ * and when callers pass an explicit `dataDir` (tests use scratch fixtures
+ * and expect fresh reads after mutating files).
+ */
+const readsByLookupKey = new Map<string, Promise<ToolkitData | null>>();
 const DEFAULT_DATA_DIR = resolveToolkitDataDir();
 
+const readLookupCacheKey = (dataDir: string, toolkitId: string): string =>
+  `${dataDir}\0${normalizeToolkitId(toolkitId)}`;
+
 const loadAllToolkitDataUncached = async (
   dataDir: string
 ): Promise<ToolkitDataMap> => {
@@ -206,19 +221,13 @@ export const loadAllToolkitData = cache(
   }
 );
 
-export const readToolkitData = async (
+const readToolkitDataUncached = async (
   toolkitId: string,
-  options?: ToolkitDataOptions
+  dataDir: string
 ): Promise<ToolkitData | null> => {
   // Normalize the toolkit ID to lowercase alphanumeric
   const normalizedId = normalizeToolkitId(toolkitId);
 
-  // Guard against empty normalized ID (e.g., input was only special characters)
-  if (!normalizedId) {
-    return null;
-  }
-
-  const dataDir = resolveDataDir(options);
   // The API route normally receives the normalized toolkit id. Keep that
   // common path O(1), especially on a cold serverless instance: eagerly
   // loading every toolkit JSON file just to serve one toolkit adds tens of
@@ -238,6 +247,41 @@ export const readToolkitData = async (
   );
 };
 
+export function readToolkitData(
+  toolkitId: string,
+  options?: ToolkitDataOptions
+): Promise<ToolkitData | null> {
+  const normalizedId = normalizeToolkitId(toolkitId);
+
+  // Guard against empty normalized ID (e.g., input was only special characters)
+  if (!normalizedId) {
+    return Promise.resolve(null);
+  }
+
+  const dataDir = resolveDataDir(options);
+
+  if (
+    process.env.NODE_ENV === "development" ||
+    options?.dataDir !== undefined
+  ) {
+    return readToolkitDataUncached(toolkitId, dataDir);
+  }
+
+  const key = readLookupCacheKey(dataDir, toolkitId);
+  let promise = readsByLookupKey.get(key);
+  if (!promise) {
+    promise = readToolkitDataUncached(toolkitId, dataDir);
+    readsByLookupKey.set(key, promise);
+
+    promise.catch(() => {
+      if (readsByLookupKey.get(key) === promise) {
+        readsByLookupKey.delete(key);
+      }
+    });
+  }
+  return promise;
+}
+
 export const readToolkitIndex = async (
   options?: ToolkitDataOptions
 ): Promise<ToolkitIndex | null> => {

app/en/resources/integrations/_lib/toolkit-docs-page.tsx

@@ -13,11 +13,6 @@ type ToolkitDocsParams = {
 };
 
 export function createToolkitDocsPage(category: IntegrationCategory) {
-  // readToolkitData is itself backed by a shared, process-wide cache (see
-  // loadAllToolkitData in app/_lib/toolkit-data.ts), so generateMetadata and
-  // Page below calling it separately for the same toolkitId costs one map
-  // lookup each rather than a second file read — no per-factory cache needed
-  // here.
   const getToolkitData = (toolkitId: string) => readToolkitData(toolkitId);
 
   const generateStaticParams = async () =>

tests/broken-link-check.test.ts

@@ -3,6 +3,10 @@ import { join } from "node:path";
 import fg from "fast-glob";
 import { scanURLs, validateFiles } from "next-validate-link";
 import { expect, test } from "vitest";
+import {
+  INTEGRATION_CATEGORIES,
+  normalizeToolkitId,
+} from "@/toolkit-docs-generator/src/shared/toolkit-primitives";
 
 const TIMEOUT = 30_000;
 
@@ -14,9 +18,8 @@ const toolkitDataDir = join(
   "data",
   "toolkits"
 );
-const TOOLKIT_ID_NORMALIZER = /[^a-z0-9]+/g;
 const normalizeToolkitSlug = (value: string): string =>
-  value.toLowerCase().replace(TOOLKIT_ID_NORMALIZER, "");
+  normalizeToolkitId(value);
 
 function getDocsLinkSlug(docsLink?: string | null): string | null {
   if (!docsLink) {
@@ -113,18 +116,6 @@ function toToolAnchorId(value: string): string {
 }
 
 const SUPPORTED_LOCALES = ["en", "es", "pt-BR"] as const;
-const SUPPORTED_INTEGRATION_CATEGORIES = [
-  "productivity",
-  "development",
-  "social",
-  "databases",
-  "search",
-  "sales",
-  "payments",
-  "customer-support",
-  "entertainment",
-  "others",
-] as const;
 
 function validateToolkitIntegrationRoute(
   urlPath: string,
@@ -155,9 +146,7 @@ function validateToolkitIntegrationRoute(
     return false;
   }
 
-  if (
-    !(SUPPORTED_INTEGRATION_CATEGORIES as readonly string[]).includes(category)
-  ) {
+  if (!(INTEGRATION_CATEGORIES as readonly string[]).includes(category)) {
     return false;
   }
 

tests/toolkit-data-cache.test.ts

@@ -1,7 +1,7 @@
 import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { afterAll, describe, expect, test } from "vitest";
+import { afterAll, describe, expect, test, vi } from "vitest";
 import { readToolkitData, readToolkitIndex } from "@/app/_lib/toolkit-data";
 
 /**
@@ -131,6 +131,57 @@ describe("readToolkitData direct-file fast path", () => {
   });
 });
 
+describe("readToolkitData production lookup cache", () => {
+  test("repeated lookups for the same toolkit share one in-flight promise", async () => {
+    const dataDir = makeFixtureDir();
+    dirsToClean.push(dataDir);
+
+    vi.stubEnv("NODE_ENV", "production");
+    vi.stubEnv("TOOLKIT_DATA_DIR", dataDir);
+
+    try {
+      const firstPromise = readToolkitData("ValidToolkitOne");
+      const secondPromise = readToolkitData("ValidToolkitOne");
+
+      expect(firstPromise).toBe(secondPromise);
+
+      const [first, second] = await Promise.all([firstPromise, secondPromise]);
+      expect(first?.id).toBe("ValidToolkitOne");
+      expect(second?.id).toBe("ValidToolkitOne");
+    } finally {
+      vi.unstubAllEnvs();
+    }
+  });
+
+  test("a transient direct read failure can recover after the file is repaired", async () => {
+    const dataDir = makeFixtureDir();
+    dirsToClean.push(dataDir);
+    writeFileSync(
+      join(dataDir, "corrupttoolkit.json"),
+      "{ this is not valid json"
+    );
+
+    vi.stubEnv("NODE_ENV", "production");
+    vi.stubEnv("TOOLKIT_DATA_DIR", dataDir);
+
+    try {
+      await expect(readToolkitData("CorruptToolkit")).rejects.toThrow(
+        join(dataDir, "corrupttoolkit.json")
+      );
+
+      writeFileSync(
+        join(dataDir, "corrupttoolkit.json"),
+        validToolkitJson("RecoveredToolkit", "recovered-toolkit")
+      );
+
+      const recovered = await readToolkitData("RecoveredToolkit");
+      expect(recovered?.id).toBe("RecoveredToolkit");
+    } finally {
+      vi.unstubAllEnvs();
+    }
+  });
+});
+
 describe("readToolkitIndex schema validation", () => {
   const dataDir = mkdtempSync(join(tmpdir(), "toolkit-index-schema-test-"));
   dirsToClean.push(dataDir);

toolkit-docs-generator/scripts/README-sync-toolkit-sidebar.md

@@ -6,24 +6,24 @@ This script synchronizes the sidebar navigation with available toolkit JSON data
 
 ```bash
 # Run the sync (updates sidebar navigation)
-npx tsx .github/scripts/sync-toolkit-sidebar.ts
+npx tsx toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts
 
 # Dry run (shows what would change without making changes)
-npx tsx .github/scripts/sync-toolkit-sidebar.ts --dry-run
+npx tsx toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts --dry-run
 
 # Verbose output
-npx tsx .github/scripts/sync-toolkit-sidebar.ts --verbose
+npx tsx toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts --verbose
 
 # Both flags
-npx tsx .github/scripts/sync-toolkit-sidebar.ts --dry-run --verbose
+npx tsx toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts --dry-run --verbose
 ```
 
 ## What it does
 
 1. Reads toolkit JSON files from `toolkit-docs-generator/data/toolkits/`.
 2. Maps toolkits to categories using the design system catalog.
 3. Creates or updates `_meta.tsx` files for each category folder.
-4. Handles the "Others" category for toolkits not in the design system.
+4. Skips toolkits without a recognized integration category.
 5. Updates the main integrations `_meta.tsx`.
 
 ## When to run
@@ -37,7 +37,8 @@ Run this script when:
 
 ## Category mapping
 
-Toolkits are mapped to categories based on `@arcadeai/design-system`:
+Toolkits are mapped to categories based on `@arcadeai/design-system` and
+`INTEGRATION_CATEGORIES` in `toolkit-docs-generator/src/shared/toolkit-primitives.ts`:
 
 | Category | Display name |
 | --- | --- |
@@ -50,18 +51,18 @@ Toolkits are mapped to categories based on `@arcadeai/design-system`:
 | sales | Sales |
 | entertainment | Entertainment |
 | payments | Payments & Finance |
-| others | Others |
 
-Toolkits not found in the design system are placed in the "Others" category.
+Toolkits with an unrecognized category fail loudly instead of being routed to a
+catch-all bucket.
 
 ## Testing
 
 ```bash
 # Run tests
-npx vitest run .github/scripts/__tests__/sync-toolkit-sidebar.test.ts
+pnpm vitest run toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts
 
 # Watch mode
-npx vitest watch .github/scripts/__tests__/sync-toolkit-sidebar.test.ts
+pnpm vitest watch toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts
 ```
 
 ## Output
@@ -74,16 +75,16 @@ The script prints a summary of changes:
 Total toolkits: 96
 
 Categories created (1):
-  + others
+  + sales
 
 Categories updated (7):
   ~ productivity
   ~ development
   ~ customer-support
   ~ search
-  ~ sales
   ~ social
   ~ payments
+  ~ entertainment
 
 ====================================
 ```

toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts

@@ -418,14 +418,14 @@ describe("groupByCategory", () => {
     expect(result.size).toBe(0);
   });
 
-  it("should handle 'others' category", () => {
+  it("should group toolkits by category", () => {
     const toolkits: ToolkitInfo[] = [
-      { id: "custom", slug: "custom", label: "Custom", category: "others" },
+      { id: "custom", slug: "custom", label: "Custom", category: "payments" },
     ];
 
     const result = groupByCategory(toolkits);
-    expect(result.has("others")).toBe(true);
-    expect(result.get("others")).toHaveLength(1);
+    expect(result.has("payments")).toBe(true);
+    expect(result.get("payments")).toHaveLength(1);
   });
 });
 
@@ -521,11 +521,11 @@ describe("generateCategoryMeta", () => {
         id: "test",
         slug: "test",
         label: 'Test "Quoted" Label',
-        category: "others",
+        category: "productivity",
       },
     ];
 
-    const result = generateCategoryMeta(toolkits, "others", "/preview");
+    const result = generateCategoryMeta(toolkits, "productivity", "/preview");
 
     expect(result).toContain('title: "Test \\"Quoted\\" Label"');
   });