From 12c6e3c80554fb97f998b0b4550c1fc778beb4b0 Mon Sep 17 00:00:00 2001 From: Luan Gjokaj Date: Thu, 30 Jul 2026 20:59:59 +0200 Subject: [PATCH 1/3] refactor: split the generator engine into focused modules src/mdx-to-nextjs-generator.ts shrinks from ~2760 lines to a thin orchestrator that wires together eleven new modules under src/generator/: - secure-source-fs.ts: SecureSourceFs, a hardened fs layer that scopes reads to the documentation root and project root, guarding against path traversal and symlink escapes. - app-scaffolder.ts: AppScaffolder writes the initial Next.js app structure (appStructure/obsoleteFiles/startingDocsStructure) plus the robots, next.config, pnpm-workspace, and proxy templates. - project-config-repository.ts: ProjectConfigRepository reads, validates, and writes the JSON config files (navigation, sections, fonts, analytics) through SecureSourceFs. - section-resolver.ts: validateSectionsConfig, discoverSections, addApiReferenceSection, and determineSectionRoute resolve an MDX file's section and slug. - page-catalog.ts: parseMdxPageMeta, validateRouteCollisions, and mergePages build and merge the page-metadata catalog from MDX frontmatter. - page-renderer.ts: renderMdxPage, renderHomepage, and renderSectionPage turn parsed MDX into page content and RSS route state. - generated-route-manager.ts: GeneratedRouteManager reconciles routes owned by generated MDX pages via the GeneratedArtifacts registry. - api-reference-generator.ts: ApiReferenceGenerator produces the OpenAPI-driven synthetic reference pages and routes. - public-asset-manager.ts: PublicAssetManager mirrors the public/ directory and manages the llms.txt/llms-full.txt/skill.md/ .well-known/mcp.json aggregate files. - site-artifacts.ts: buildSitemapEntries and related helpers generate sitemap.xml, robots.txt, and the llms files from the page catalog. - watch-coordinator.ts: WatchCoordinator owns the chokidar watchers for MDX, config, font, analytics, doccupine.json, public/, and OpenAPI spec sources. No behavioral changes are intended; this is a pure extraction to keep each concern independently readable and testable. --- src/generator/api-reference-generator.ts | 154 ++ src/generator/app-scaffolder.ts | 82 + src/generator/generated-route-manager.ts | 116 + src/generator/page-catalog.ts | 105 + src/generator/page-renderer.ts | 316 +++ src/generator/project-config-repository.ts | 198 ++ src/generator/public-asset-manager.ts | 293 +++ src/generator/section-resolver.ts | 207 ++ src/generator/secure-source-fs.ts | 542 ++++ src/generator/site-artifacts.ts | 263 ++ src/generator/watch-coordinator.ts | 644 +++++ src/mdx-to-nextjs-generator.ts | 2761 ++------------------ 12 files changed, 3161 insertions(+), 2520 deletions(-) create mode 100644 src/generator/api-reference-generator.ts create mode 100644 src/generator/app-scaffolder.ts create mode 100644 src/generator/generated-route-manager.ts create mode 100644 src/generator/page-catalog.ts create mode 100644 src/generator/page-renderer.ts create mode 100644 src/generator/project-config-repository.ts create mode 100644 src/generator/public-asset-manager.ts create mode 100644 src/generator/section-resolver.ts create mode 100644 src/generator/secure-source-fs.ts create mode 100644 src/generator/site-artifacts.ts create mode 100644 src/generator/watch-coordinator.ts diff --git a/src/generator/api-reference-generator.ts b/src/generator/api-reference-generator.ts new file mode 100644 index 0000000..1ca3560 --- /dev/null +++ b/src/generator/api-reference-generator.ts @@ -0,0 +1,154 @@ +import chalk from "chalk"; +import fs from "fs-extra"; +import path from "node:path"; + +import { GeneratedArtifacts } from "../lib/generated-artifacts.js"; +import { buildEndpointDoc, OpenApiRegistry } from "../lib/openapi.js"; +import type { OperationDescriptor } from "../lib/openapi-types.js"; +import { resolveOutputPath } from "../lib/output-safety.js"; +import type { MDXFile, PageMeta } from "../lib/types.js"; +import { writeFileAtomic } from "../lib/utils.js"; +import { mergePages as mergePageCatalog } from "./page-catalog.js"; + +type GeneratePage = ( + mdxFile: MDXFile, + options?: { apiOperation?: OperationDescriptor }, +) => Promise; + +type RemoveOwnedRoute = (slug: string) => Promise; + +type WriteAllowlist = () => Promise; + +type CleanupStalePages = ( + nextRoutes: Map, + realSlugs: Set, +) => Promise; + +export class ApiReferenceGenerator { + constructor( + private readonly outputDir: string, + private readonly artifacts: GeneratedArtifacts, + ) {} + + mergePages(registry: OpenApiRegistry, realPages: PageMeta[]): PageMeta[] { + return mergePageCatalog( + realPages, + registry.isEmpty ? [] : registry.syntheticPages(), + ); + } + + async writePages( + registry: OpenApiRegistry, + apiBaseSlug: string, + realPages: PageMeta[], + generatePage: GeneratePage, + writeAllowlist: WriteAllowlist, + cleanupStalePages: CleanupStalePages, + ): Promise { + const realSlugs = new Set(realPages.map((page) => page.slug)); + const nextRoutes = new Map(); + + const indexPage = registry + .syntheticPages() + .find((page) => page.slug === apiBaseSlug); + if (indexPage && !realSlugs.has(indexPage.slug)) { + try { + await generatePage({ + path: indexPage.path, + content: registry.bodyForSlug(indexPage.slug) ?? "", + frontmatter: { + title: indexPage.title, + description: indexPage.description, + }, + slug: indexPage.slug, + }); + nextRoutes.set(`@openapi/${indexPage.slug}`, indexPage.slug); + } catch (error) { + console.error( + chalk.red(`❌ Error generating API index ${indexPage.slug}:`), + error, + ); + } + } + + for (const op of registry.all) { + const methodUpper = op.method.toUpperCase(); + const mdxFile: MDXFile = { + path: `@openapi/${op.specName}/${op.method}${op.path}`, + content: buildEndpointDoc(op), + frontmatter: { + title: op.summary ?? `${methodUpper} ${op.path}`, + description: op.summary ?? "", + }, + slug: op.slug, + }; + if (realSlugs.has(op.slug)) { + console.log( + chalk.yellow( + `⚠️ API page ${op.slug} is shadowed by a hand-written page; skipping`, + ), + ); + continue; + } + try { + await generatePage(mdxFile, { apiOperation: op }); + nextRoutes.set(`@openapi/${op.slug}`, op.slug); + } catch (error) { + console.error( + chalk.red(`❌ Error generating API page ${op.slug}:`), + error, + ); + } + } + + await writeAllowlist(); + await cleanupStalePages(nextRoutes, realSlugs); + + if (registry.all.length > 0) { + console.log( + chalk.green(`🧩 Generated ${nextRoutes.size} API reference page(s)`), + ); + } + } + + async writeAllowlist(registry: OpenApiRegistry): Promise { + const target = resolveOutputPath( + this.outputDir, + "services", + "openapi", + "playground-allowlist.json", + ); + await fs.ensureDir(path.dirname(target)); + await writeFileAtomic( + target, + `${JSON.stringify(registry.allowlist(), null, 2)}\n`, + ); + } + + async cleanupStalePages( + nextRoutes: Map, + realSlugs: Set, + removeOwnedRoute: RemoveOwnedRoute, + ): Promise { + const nextSlugs = new Set(nextRoutes.values()); + for (const previous of this.artifacts.routesFor("openapi")) { + if (nextRoutes.has(previous.source) || nextSlugs.has(previous.slug)) { + continue; + } + // A hand-written page may have taken ownership of this route since the + // previous OpenAPI pass. Never remove an output now claimed by MDX. + if (realSlugs.has(previous.slug)) continue; + try { + await removeOwnedRoute(previous.slug); + } catch { + // ignore + } + } + + this.artifacts.replaceRoutes( + "openapi", + [...nextRoutes].map(([source, slug]) => ({ source, slug })), + ); + await this.artifacts.save(); + } +} diff --git a/src/generator/app-scaffolder.ts b/src/generator/app-scaffolder.ts new file mode 100644 index 0000000..45aa094 --- /dev/null +++ b/src/generator/app-scaffolder.ts @@ -0,0 +1,82 @@ +import fs from "fs-extra"; +import path from "node:path"; + +import { + appStructure, + obsoleteFiles, + startingDocsStructure, +} from "../lib/structures.js"; +import { resolveOutputPath } from "../lib/output-safety.js"; +import type { AnalyticsConfig } from "../lib/types.js"; +import { writeFileAtomic } from "../lib/utils.js"; +import { robotsTemplate } from "../templates/app/robots.js"; +import { nextConfigTemplate } from "../templates/next.config.js"; +import { pnpmWorkspaceTemplate } from "../templates/pnpmWorkspace.js"; +import { proxyTemplate } from "../templates/proxy.js"; + +interface AppStructureCallbacks { + generateRootLayout(): Promise; + generateSiteLayout(): Promise; + updateSitemap(): Promise; + updateLlmsFiles(): Promise; +} + +interface StarterDocumentCallbacks { + getAllMdxFiles(): Promise; + ensureSafeStarterPath(relativePath: string): Promise; +} + +export class AppScaffolder { + constructor(private readonly outputDir: string) {} + + private outputPath(...segments: string[]): string { + return resolveOutputPath(this.outputDir, ...segments); + } + + async createNextJsStructure( + analyticsConfig: AnalyticsConfig | null, + callbacks: AppStructureCallbacks, + ): Promise { + // Everything under app/ is generated, so clear stale routes before writing + // the current structure. Other generated directories remain untouched. + await fs.remove(this.outputPath("app")); + + await Promise.all( + obsoleteFiles.map((file) => fs.remove(this.outputPath(file))), + ); + + const structure: Record> = { + ...appStructure, + "next.config.ts": nextConfigTemplate(analyticsConfig), + "pnpm-workspace.yaml": pnpmWorkspaceTemplate, + "proxy.ts": proxyTemplate(analyticsConfig), + "analytics.json": `{}\n`, + "config.json": `{}\n`, + "links.json": `[]\n`, + "navigation.json": `[]\n`, + "sections.json": `[]\n`, + "theme.json": `{}\n`, + "app/robots.ts": robotsTemplate, + "app/layout.tsx": callbacks.generateRootLayout(), + "app/(site)/layout.tsx": callbacks.generateSiteLayout(), + }; + + for (const [filePath, content] of Object.entries(structure)) { + const fullPath = this.outputPath(filePath); + await fs.ensureDir(path.dirname(fullPath)); + await writeFileAtomic(fullPath, String(await content)); + } + + await callbacks.updateSitemap(); + await callbacks.updateLlmsFiles(); + } + + async createStartingDocs(callbacks: StarterDocumentCallbacks): Promise { + if ((await callbacks.getAllMdxFiles()).length > 0) return; + + for (const [filePath, content] of Object.entries(startingDocsStructure)) { + const fullPath = await callbacks.ensureSafeStarterPath(filePath); + await writeFileAtomic(fullPath, String(content)); + } + } +} diff --git a/src/generator/generated-route-manager.ts b/src/generator/generated-route-manager.ts new file mode 100644 index 0000000..c5e867c --- /dev/null +++ b/src/generator/generated-route-manager.ts @@ -0,0 +1,116 @@ +import chalk from "chalk"; +import fs from "fs-extra"; +import path from "node:path"; + +import { GeneratedArtifacts } from "../lib/generated-artifacts.js"; +import { resolveOutputPath } from "../lib/output-safety.js"; +import type { PageMeta } from "../lib/types.js"; + +export class GeneratedRouteManager { + private generatedSectionIndexSlugs = new Set(); + + constructor( + private readonly outputDir: string, + private readonly artifacts: GeneratedArtifacts, + ) {} + + private outputPath(...segments: string[]): string { + return resolveOutputPath(this.outputDir, ...segments); + } + + routeForMdxSource(source: string): string | undefined { + return this.artifacts.routeFor("mdx", source); + } + + async replaceMdxRoutes(realPages: PageMeta[]): Promise { + this.artifacts.replaceRoutes( + "mdx", + realPages + .filter((page) => page.slug !== "") + .map((page) => ({ source: page.path, slug: page.slug })), + ); + await this.artifacts.save(); + } + + async removeMdxRoute(source: string): Promise { + this.artifacts.removeRoute("mdx", source); + await this.artifacts.save(); + } + + async removeOwnedRoute(slug: string): Promise { + if (!slug) return; + const siteDir = this.outputPath("app", "(site)"); + const routeDir = resolveOutputPath(siteDir, slug); + await Promise.all([ + fs.remove(resolveOutputPath(siteDir, slug, "page.tsx")), + fs.remove(resolveOutputPath(siteDir, slug, "rss.xml")), + ]); + await this.removeEmptyDirsUpTo(routeDir, siteDir); + } + + async removeStaleMdxRoutes( + realPages: PageMeta[], + removeOwnedRoute: (slug: string) => Promise, + ): Promise { + const nextBySource = new Map( + realPages + .filter((page) => page.slug !== "") + .map((page) => [page.path.replace(/\\/g, "/"), page.slug]), + ); + const nextSlugs = new Set(nextBySource.values()); + + for (const previous of this.artifacts.routesFor("mdx")) { + if (nextBySource.get(previous.source) === previous.slug) continue; + if (nextSlugs.has(previous.slug)) continue; + await removeOwnedRoute(previous.slug); + } + } + + async cleanupStaleSectionIndexPages( + nextSlugs: Set, + removeEmptyDirs: (dir: string, stopDir: string) => Promise, + ): Promise { + for (const stale of this.generatedSectionIndexSlugs) { + if (nextSlugs.has(stale)) continue; + const pagePath = resolveOutputPath( + this.outputDir, + "app", + "(site)", + stale, + "page.tsx", + ); + try { + if (!(await fs.pathExists(pagePath))) continue; + const content = await fs.readFile(pagePath, "utf8"); + if (!content.includes("function SectionIndex()")) continue; + await fs.remove(pagePath); + await removeEmptyDirs( + path.dirname(pagePath), + this.outputPath("app", "(site)"), + ); + console.log( + chalk.blue(`🧹 Removed stale section index redirect: /${stale}`), + ); + } catch { + // ignore + } + } + this.generatedSectionIndexSlugs = nextSlugs; + } + + /** Best-effort removal of now-empty directories up to (not incl.) stopDir. */ + async removeEmptyDirsUpTo(dir: string, stopDir: string): Promise { + const stop = path.resolve(stopDir); + let current = path.resolve(dir); + while (current !== stop && current.startsWith(stop + path.sep)) { + try { + const entries = await fs.readdir(current); + if (entries.length > 0) return; + await fs.remove(current); + } catch { + return; + } + current = path.dirname(current); + } + } +} diff --git a/src/generator/page-catalog.ts b/src/generator/page-catalog.ts new file mode 100644 index 0000000..2491356 --- /dev/null +++ b/src/generator/page-catalog.ts @@ -0,0 +1,105 @@ +import chalk from "chalk"; + +import type { PageMeta } from "../lib/types.js"; +import { getFullSlug, safeMatter } from "../lib/utils.js"; + +type ReadMdxSource = ( + filePath: string, +) => Promise<{ content: string; stat: { mtime: Date } }>; + +type ResolveSectionRoute = ( + filePath: string, + frontmatter: Record, +) => { sectionSlug: string; pageSlug: string }; + +type ResolveHttpMethod = (reference: string) => string | undefined; + +export async function parseMdxPageMeta( + filePath: string, + readMdxSource: ReadMdxSource, + resolveSectionRoute: ResolveSectionRoute, + resolveHttpMethod: ResolveHttpMethod, +): Promise { + const { content, stat } = await readMdxSource(filePath); + const { data: frontmatter } = safeMatter(content, filePath); + const { sectionSlug, pageSlug } = resolveSectionRoute(filePath, frontmatter); + const fullSlug = getFullSlug(pageSlug, sectionSlug); + + let lastModified: string | undefined; + const authoredLastModified = frontmatter.updated ?? frontmatter.date; + if (authoredLastModified) { + const parsed = new Date(authoredLastModified); + if (!Number.isNaN(parsed.getTime())) { + lastModified = parsed.toISOString(); + } + } + if (!lastModified) { + lastModified = stat.mtime.toISOString(); + } + + // Hand-written OpenAPI pages receive the same method badge as generated ones. + const httpMethod = frontmatter.openapi + ? resolveHttpMethod(String(frontmatter.openapi))?.toUpperCase() + : undefined; + + return { + slug: fullSlug, + title: frontmatter.title || "Untitled", + description: frontmatter.description || "", + date: frontmatter.date || null, + category: frontmatter.category || "", + path: filePath, + categoryOrder: frontmatter.categoryOrder || 0, + order: frontmatter.order || 0, + section: sectionSlug, + ...(frontmatter.navIcon ? { navIcon: String(frontmatter.navIcon) } : {}), + ...(frontmatter.categoryIcon + ? { categoryIcon: String(frontmatter.categoryIcon) } + : {}), + ...(httpMethod ? { httpMethod } : {}), + lastModified, + }; +} + +export function validateRouteCollisions(pages: PageMeta[]): void { + const bySlug = new Map(); + for (const page of pages) { + const existing = bySlug.get(page.slug); + if (existing) { + throw new Error( + `Route collision at "/${page.slug}": both "${existing}" and "${page.path}" generate the same page.`, + ); + } + bySlug.set(page.slug, page.path); + } +} + +export async function buildRealPagesMeta( + files: string[], + parsePage: (filePath: string) => Promise, +): Promise { + const pages = await Promise.all(files.map((file) => parsePage(file))); + validateRouteCollisions(pages); + return pages; +} + +export function mergePages( + realPages: PageMeta[], + syntheticPages: PageMeta[], +): PageMeta[] { + if (syntheticPages.length === 0) return realPages; + + const realSlugs = new Set(realPages.map((page) => page.slug)); + const unshadowedSyntheticPages = syntheticPages.filter((page) => { + if (realSlugs.has(page.slug)) { + console.log( + chalk.yellow( + `⚠️ API page ${page.slug} is shadowed by a hand-written page; skipping`, + ), + ); + return false; + } + return true; + }); + return [...realPages, ...unshadowedSyntheticPages]; +} diff --git a/src/generator/page-renderer.ts b/src/generator/page-renderer.ts new file mode 100644 index 0000000..a349823 --- /dev/null +++ b/src/generator/page-renderer.ts @@ -0,0 +1,316 @@ +import { + generateJsonLdScript, + generateMetadataBlock, + generateRuntimeOnlyMetadataBlock, +} from "../lib/metadata.js"; +import type { OperationDescriptor } from "../lib/openapi-types.js"; +import { parseUpdateBlocks } from "../lib/rss.js"; +import type { MDXFile } from "../lib/types.js"; +import { escapeTemplateContent, toJsStringLiteral } from "../lib/utils.js"; +import { rssRouteTemplate } from "../templates/app/rssRoute.js"; + +export type RssRouteState = + | { action: "write"; content: string } + | { action: "remove" } + | { action: "preserve" }; + +export interface RenderedPage { + pageContent: string; + rssRoute: RssRouteState; +} + +export interface HomepageSource { + content: string; + title: string; + description: string; + icon?: string; + image?: string; + name?: string; + date?: string; + updated?: string; + openapi?: string; + rss?: boolean; +} + +export function renderMdxPage( + mdxFile: MDXFile, + options?: { apiOperation?: OperationDescriptor }, +): RenderedPage { + const fm = mdxFile.frontmatter; + const apiOperation = options?.apiOperation; + const isSynthetic = mdxFile.path.startsWith("@openapi/"); + const updates = isSynthetic ? [] : parseUpdateBlocks(mdxFile.content); + const hasFeed = updates.length > 0; + const feedPath = `/${mdxFile.slug}/rss.xml`; + + const metadataBlock = generateMetadataBlock({ + title: fm.title, + titleFallback: "Generated with Doccupine", + name: fm.name, + titleOrder: "page-first", + description: fm.description, + icon: fm.icon, + image: fm.image, + canonicalPath: mdxFile.slug, + rssPath: hasFeed ? feedPath : undefined, + }); + + const jsonLd = generateJsonLdScript({ + kind: "article", + canonicalPath: mdxFile.slug, + title: fm.title, + description: fm.description, + date: typeof fm.date === "string" ? fm.date : undefined, + updated: + typeof fm.updated === "string" + ? fm.updated + : typeof fm.date === "string" + ? fm.date + : undefined, + image: fm.image, + }); + + const apiImport = apiOperation + ? `\nimport { ApiPlayground } from "@/components/layout/ApiPlayground";` + : ""; + const apiConst = apiOperation + ? (() => { + const arg = toJsStringLiteral(JSON.stringify(apiOperation)); + const inline = `const operation = JSON.parse(${arg});`; + const decl = + inline.length <= 80 + ? inline + : `const operation = JSON.parse(\n ${arg},\n);`; + return `\n${decl}\n`; + })() + : ""; + const sourcePathLiteral = JSON.stringify(mdxFile.path); + const showRssButton = hasFeed && fm.rss === true && !apiOperation; + const docsAttrs = [ + `content={content}`, + `sourcePath={${sourcePathLiteral}}`, + ...(showRssButton ? [`rssHref={${JSON.stringify(feedPath)}}`] : []), + ]; + const inlineDocs = ``; + const docsElement = apiOperation + ? ` + + ` + : inlineDocs.length + 6 <= 80 + ? inlineDocs + : ` ` ${attr}`).join("\n")}\n />`; + + const pageContent = `import { Metadata } from "next"; +import { Docs } from "@/components/Docs"; +import { config } from "@/utils/config";${apiImport} + +const content = \`${escapeTemplateContent(mdxFile.content)}\`; +${apiConst} +${metadataBlock} + +// Doc pages have no per-request data: theme resolves client-side via the +// "dark" class on (set before paint by the theme-init blocking +// script). Static rendering lets every response come from the edge cache. +export const dynamic = "force-static"; +export const revalidate = false; + +export default function Page() { + ${jsonLd.declarations} + + return ( + <> + ${jsonLd.element} + ${docsElement} + + ); +} +`; + + const rssRoute: RssRouteState = isSynthetic + ? { action: "preserve" } + : hasFeed + ? { + action: "write", + content: rssRouteTemplate({ + pagePath: mdxFile.slug, + title: typeof fm.title === "string" ? fm.title : null, + description: + typeof fm.description === "string" ? fm.description : null, + items: updates.map((update) => ({ + title: update.label, + anchor: update.anchor, + description: update.description, + })), + }), + } + : { action: "remove" }; + + return { pageContent, rssRoute }; +} + +export function renderHomepage( + indexMDX: HomepageSource | null, + apiOperation?: OperationDescriptor, +): RenderedPage { + const updates = indexMDX ? parseUpdateBlocks(indexMDX.content) : []; + const hasFeed = updates.length > 0; + const feedPath = "/rss.xml"; + + const metadataBlock = indexMDX + ? generateMetadataBlock({ + title: indexMDX.title, + titleFallback: "Welcome", + name: indexMDX.name, + titleOrder: "name-first", + description: indexMDX.description || undefined, + icon: indexMDX.icon, + image: indexMDX.image, + canonicalPath: "", + rssPath: hasFeed ? feedPath : undefined, + }) + : generateRuntimeOnlyMetadataBlock(); + + const homeJsonLd = generateJsonLdScript({ + kind: "homepage", + canonicalPath: "", + title: indexMDX?.title, + description: indexMDX?.description || undefined, + date: indexMDX?.date, + updated: indexMDX?.updated ?? indexMDX?.date, + image: indexMDX?.image, + }); + + const apiImport = apiOperation + ? `\nimport { ApiPlayground } from "@/components/layout/ApiPlayground";` + : ""; + const apiConst = apiOperation + ? `\nconst operation = JSON.parse(${JSON.stringify( + JSON.stringify(apiOperation), + )});\n` + : ""; + const showRssButton = hasFeed && indexMDX?.rss === true && !apiOperation; + const docsElement = apiOperation + ? ` + + ` + : showRssButton + ? `` + : ``; + + const pageContent = `import { Metadata } from "next"; +import { Docs } from "@/components/Docs"; +import { config } from "@/utils/config";${apiImport} + +${indexMDX ? `const content = \`${escapeTemplateContent(indexMDX.content)}\`;` : `const content = null;`} +${apiConst} +${metadataBlock} + +export const dynamic = "force-static"; +export const revalidate = false; + +export default function Home() { + ${homeJsonLd.declarations} + + return ( + <> + ${homeJsonLd.element} + ${docsElement} + + ); +} +`; + + const rssRoute: RssRouteState = + hasFeed && indexMDX + ? { + action: "write", + content: rssRouteTemplate({ + pagePath: "", + title: indexMDX.title, + description: indexMDX.description || null, + items: updates.map((update) => ({ + title: update.label, + anchor: update.anchor, + description: update.description, + })), + }), + } + : { action: "remove" }; + + return { pageContent, rssRoute }; +} + +export function renderSectionPage( + sectionSlug: string, + frontmatter: Record, + mdxContent: string, + sourcePath?: string, +): RenderedPage { + const updates = parseUpdateBlocks(mdxContent); + const hasFeed = updates.length > 0; + const feedPath = `/${sectionSlug}/rss.xml`; + const showRssButton = hasFeed && frontmatter.rss === true; + + const metadataBlock = generateMetadataBlock({ + title: frontmatter.title, + titleFallback: "Section", + name: frontmatter.name, + titleOrder: "name-first", + description: frontmatter.description || undefined, + icon: frontmatter.icon, + image: frontmatter.image, + canonicalPath: sectionSlug, + rssPath: hasFeed ? feedPath : undefined, + }); + + const sectionJsonLd = generateJsonLdScript({ + kind: "article", + canonicalPath: sectionSlug, + title: frontmatter.title, + description: frontmatter.description, + date: typeof frontmatter.date === "string" ? frontmatter.date : undefined, + updated: + typeof frontmatter.updated === "string" + ? frontmatter.updated + : typeof frontmatter.date === "string" + ? frontmatter.date + : undefined, + image: frontmatter.image, + }); + + const docsAttrs = [ + `content={content}`, + `sourcePath={${JSON.stringify(sourcePath ?? `${sectionSlug}/index.mdx`)}}`, + ...(showRssButton ? [`rssHref={${JSON.stringify(feedPath)}}`] : []), + ]; + const inlineDocs = ``; + const docsElement = + inlineDocs.length + 6 <= 80 + ? inlineDocs + : ` ` ${attr}`).join("\n")}\n />`; + + const pageContent = `import { Metadata } from "next"; +import { Docs } from "@/components/Docs"; +import { config } from "@/utils/config"; + +const content = \`${escapeTemplateContent(mdxContent)}\`; + +${metadataBlock} + +export const dynamic = "force-static"; +export const revalidate = false; + +export default function Page() { + ${sectionJsonLd.declarations} + + return ( + <> + ${sectionJsonLd.element} + ${docsElement} + + ); +} +`; + + return { pageContent, rssRoute: { action: "preserve" } }; +} diff --git a/src/generator/project-config-repository.ts b/src/generator/project-config-repository.ts new file mode 100644 index 0000000..939c2db --- /dev/null +++ b/src/generator/project-config-repository.ts @@ -0,0 +1,198 @@ +import chalk from "chalk"; +import fs from "fs-extra"; +import path from "node:path"; + +import { resolveOutputPath } from "../lib/output-safety.js"; +import { + validateAnalyticsConfig, + validateFontConfig, +} from "../lib/project-config.js"; +import type { + AnalyticsConfig, + FontConfig, + SectionConfig, +} from "../lib/types.js"; +import { writeFileAtomic } from "../lib/utils.js"; +import { SecureSourceFs } from "./secure-source-fs.js"; +import { validateSectionsConfig } from "./section-resolver.js"; + +const ARRAY_CONFIG_DEFAULTS = new Set([ + "links.json", + "navigation.json", + "sections.json", +]); + +export class ProjectConfigRepository { + constructor( + private readonly rootDir: string, + private readonly outputDir: string, + private readonly sourceFs: SecureSourceFs, + private readonly configFiles: readonly string[], + private readonly fontConfigFile: string, + private readonly analyticsConfigFile: string, + ) {} + + private outputPath(...segments: string[]): string { + return resolveOutputPath(this.outputDir, ...segments); + } + + private async copyRootSourceFile( + sourcePath: string, + destPath: string, + label: string, + ): Promise { + const { data } = await this.sourceFs.readProjectSourceFile( + sourcePath, + label, + ); + await writeFileAtomic(destPath, data); + } + + async copyConfigFile(configFile: string): Promise { + await this.copyRootSourceFile( + path.join(this.rootDir, configFile), + this.outputPath(configFile), + "config source", + ); + } + + async resetConfigFile(configFile: string): Promise { + await writeFileAtomic( + this.outputPath(configFile), + ARRAY_CONFIG_DEFAULTS.has(configFile) ? `[]\n` : `{}\n`, + ); + } + + async copyCustomConfigFiles(): Promise { + console.log(chalk.blue(`πŸ” Checking for config files in: ${this.rootDir}`)); + + for (const configFile of this.configFiles) { + const sourcePath = path.join(this.rootDir, configFile); + + console.log(chalk.gray(` Checking ${configFile}...`)); + + if (await fs.pathExists(sourcePath)) { + await this.copyConfigFile(configFile); + console.log(chalk.green(` βœ“ Copied ${configFile} to Next.js app`)); + } else { + console.log(chalk.gray(` βœ— ${configFile} not found, skipping`)); + } + } + } + + async copyFontConfigFile(): Promise { + await this.copyRootSourceFile( + path.join(this.rootDir, this.fontConfigFile), + this.outputPath(this.fontConfigFile), + "font source", + ); + } + + async copyFontConfig(): Promise { + console.log(chalk.blue(`πŸ” Checking for font configuration...`)); + + const sourcePath = path.join(this.rootDir, this.fontConfigFile); + if (await fs.pathExists(sourcePath)) { + await this.copyFontConfigFile(); + console.log( + chalk.green(` βœ“ Copied ${this.fontConfigFile} to Next.js app`), + ); + } else { + console.log(chalk.gray(` βœ— ${this.fontConfigFile} not found, skipping`)); + } + } + + async removeFontConfig(): Promise { + const destPath = this.outputPath(this.fontConfigFile); + if (!(await fs.pathExists(destPath))) return false; + await fs.remove(destPath); + return true; + } + + async loadFontConfig(): Promise { + const fontPath = path.join(this.rootDir, this.fontConfigFile); + + try { + if (await fs.pathExists(fontPath)) { + const { data } = await this.sourceFs.readProjectSourceFile( + fontPath, + "font source", + ); + return validateFontConfig(JSON.parse(data.toString("utf8"))); + } + } catch (error) { + console.warn( + chalk.yellow(`⚠️ Error reading ${this.fontConfigFile}`), + error, + ); + } + + return null; + } + + async loadAnalyticsConfig(): Promise { + const analyticsPath = path.join(this.rootDir, this.analyticsConfigFile); + + try { + if (await fs.pathExists(analyticsPath)) { + const { data } = await this.sourceFs.readProjectSourceFile( + analyticsPath, + "analytics source", + ); + return validateAnalyticsConfig(JSON.parse(data.toString("utf8"))); + } + } catch (error) { + console.warn( + chalk.yellow(`⚠️ Error reading ${this.analyticsConfigFile}`), + error, + ); + } + + return null; + } + + async writeAnalyticsConfig(config: AnalyticsConfig | null): Promise { + await writeFileAtomic( + this.outputPath(this.analyticsConfigFile), + config ? `${JSON.stringify(config, null, 2)}\n` : `{}\n`, + ); + } + + async copyAnalyticsConfig( + loadConfig: () => Promise, + ): Promise { + console.log(chalk.blue(`πŸ” Checking for analytics configuration...`)); + + const sourcePath = path.join(this.rootDir, this.analyticsConfigFile); + if (await fs.pathExists(sourcePath)) { + await this.writeAnalyticsConfig(await loadConfig()); + console.log( + chalk.green(` βœ“ Copied ${this.analyticsConfigFile} to Next.js app`), + ); + } else { + console.log( + chalk.gray(` βœ— ${this.analyticsConfigFile} not found, skipping`), + ); + } + } + + async resetAnalyticsConfig(): Promise { + await this.writeAnalyticsConfig(null); + } + + async loadSectionsConfig(): Promise { + const sectionsPath = path.join(this.rootDir, "sections.json"); + + try { + if (await fs.pathExists(sectionsPath)) { + const content = await fs.readFile(sectionsPath, "utf8"); + const parsed = JSON.parse(content) as unknown; + return validateSectionsConfig(parsed); + } + } catch (error) { + console.warn(chalk.yellow("⚠️ Error reading sections.json"), error); + } + + return null; + } +} diff --git a/src/generator/public-asset-manager.ts b/src/generator/public-asset-manager.ts new file mode 100644 index 0000000..8f36788 --- /dev/null +++ b/src/generator/public-asset-manager.ts @@ -0,0 +1,293 @@ +import chalk from "chalk"; +import fs from "fs-extra"; +import path from "node:path"; + +import { GeneratedArtifacts } from "../lib/generated-artifacts.js"; +import { resolveOutputPath, resolveWithin } from "../lib/output-safety.js"; +import { writeFileAtomic } from "../lib/utils.js"; +import type { PageWithBody } from "../templates/llms/llmsFull.js"; +import { llmsPageTemplate } from "../templates/llms/llmsPage.js"; +import { siteDocsSlug } from "../templates/llms/skillMd.js"; +import { SecureSourceFs } from "./secure-source-fs.js"; + +const PUBLIC_AGGREGATE_PATHS = new Set([ + "llms.txt", + "llms-full.txt", + "skill.md", + ".well-known/mcp.json", +]); + +function normalizePublicArtifactPath(relativePath: string): string { + return relativePath.replace(/\\/g, "/").toLowerCase(); +} + +function isPublicAggregate(relativePath: string): boolean { + return PUBLIC_AGGREGATE_PATHS.has(normalizePublicArtifactPath(relativePath)); +} + +function isManagedPublicArtifact(relativePath: string): boolean { + const normalized = normalizePublicArtifactPath(relativePath); + return PUBLIC_AGGREGATE_PATHS.has(normalized) || normalized.endsWith(".md"); +} + +function publicDestinationRelativePath(relativePath: string): string { + return isPublicAggregate(relativePath) + ? normalizePublicArtifactPath(relativePath) + : relativePath.replace(/\\/g, "/"); +} + +export class PublicAssetManager { + constructor( + private readonly rootDir: string, + private readonly outputDir: string, + private readonly artifacts: GeneratedArtifacts, + private readonly sourceFs: SecureSourceFs, + ) {} + + private outputPath(...segments: string[]): string { + return resolveOutputPath(this.outputDir, ...segments); + } + + publicOutputFilePath(relativePath: string): string { + const parent = path.dirname(relativePath); + const outputParent = + parent === "." + ? this.outputPath("public") + : this.outputPath("public", parent); + return resolveWithin(outputParent, path.basename(relativePath)); + } + + async copyRegularPublicFile( + sourcePath: string, + destPath: string, + ): Promise { + const { data } = await this.sourceFs.readPublicSourceFile(sourcePath); + await writeFileAtomic(destPath, data); + } + + async copyPublicFiles(): Promise { + const publicDir = path.join(this.rootDir, "public"); + const previousFiles = this.artifacts.publicFiles(); + + console.log(chalk.blue(`πŸ” Checking for public directory...`)); + + const files = await this.sourceFs.scanPublicFiles(); + if (files === null) { + console.log(chalk.gray(` βœ— public directory not found, skipping`)); + for (const stale of previousFiles) { + await fs.remove(this.publicOutputFilePath(stale)); + } + this.artifacts.replacePublicFiles([]); + await this.artifacts.save(); + return; + } + const nextFiles = new Set(); + const nextByFoldedPath = new Map(); + for (const relativePath of files) { + const destRelativePath = publicDestinationRelativePath(relativePath); + nextFiles.add(destRelativePath); + nextByFoldedPath.set(destRelativePath.toLowerCase(), destRelativePath); + } + const removedBeforeCopy = new Set(); + for (const stale of previousFiles) { + const replacement = nextByFoldedPath.get(stale.toLowerCase()); + if ( + replacement && + replacement !== stale && + (await fs.pathExists(this.publicOutputFilePath(replacement))) + ) { + await fs.remove(this.publicOutputFilePath(stale)); + removedBeforeCopy.add(stale); + } + } + for (const relativePath of files) { + const destRelativePath = publicDestinationRelativePath(relativePath); + await this.copyRegularPublicFile( + path.join(publicDir, relativePath), + this.publicOutputFilePath(destRelativePath), + ); + } + for (const stale of previousFiles) { + if (!nextFiles.has(stale) && !removedBeforeCopy.has(stale)) { + await fs.remove(this.publicOutputFilePath(stale)); + } + } + this.artifacts.replacePublicFiles(nextFiles); + await this.artifacts.save(); + console.log(chalk.green(` βœ“ Copied public directory to Next.js app`)); + } + + async handlePublicFileChange( + filePath: string, + restoreGeneratedArtifacts: () => Promise, + ): Promise { + const publicDir = path.join(this.rootDir, "public"); + const relativePath = path.relative(publicDir, filePath); + const destRelativePath = publicDestinationRelativePath(relativePath); + const destPath = this.publicOutputFilePath(destRelativePath); + + try { + await this.copyRegularPublicFile(filePath, destPath); + const publicFiles = this.artifacts.publicFiles(); + publicFiles.add(destRelativePath); + this.artifacts.replacePublicFiles(publicFiles); + await this.artifacts.save(); + console.log( + chalk.green(`πŸ“‹ Updated public/${relativePath} in Next.js app`), + ); + if (isManagedPublicArtifact(relativePath)) { + await restoreGeneratedArtifacts(); + } + } catch (error) { + console.error( + chalk.red(`❌ Error copying public/${relativePath}:`), + error, + ); + throw error; + } + } + + async handlePublicFileDelete( + filePath: string, + copyCurrentSource: () => Promise, + restoreGeneratedArtifacts: () => Promise, + ): Promise { + const publicDir = path.join(this.rootDir, "public"); + const relativePath = path.relative(publicDir, filePath); + const destRelativePath = publicDestinationRelativePath(relativePath); + const destPath = this.outputPath("public", destRelativePath); + + try { + // A rapid replace can queue an unlink after the replacement already + // exists. Copy the current source instead of deleting its fresh mirror. + if (await fs.pathExists(filePath)) { + await copyCurrentSource(); + return; + } + if (await fs.pathExists(destPath)) { + await fs.remove(destPath); + console.log( + chalk.yellow(`πŸ—‘οΈ Removed public/${relativePath} from Next.js app`), + ); + } + const publicFiles = this.artifacts.publicFiles(); + publicFiles.delete(destRelativePath); + this.artifacts.replacePublicFiles(publicFiles); + await this.artifacts.save(); + if (isManagedPublicArtifact(relativePath)) { + await restoreGeneratedArtifacts(); + } + } catch (error) { + console.error( + chalk.red(`❌ Error removing public/${relativePath}:`), + error, + ); + } + } + + async findSourcePublicAsset(relativePath: string): Promise { + return this.sourceFs.findPublicAsset(relativePath); + } + + async writePublicAggregate( + relativePath: string, + content: string, + ): Promise { + const sourcePath = await this.findSourcePublicAsset(relativePath); + const targetPath = this.publicOutputFilePath(relativePath); + if (sourcePath) { + console.warn( + chalk.yellow( + `⚠️ Skipping generated public/${relativePath}; a project public asset owns that path`, + ), + ); + await this.copyRegularPublicFile(sourcePath, targetPath); + return; + } + + await fs.ensureDir(path.dirname(targetPath)); + await writeFileAtomic(targetPath, content); + } + + async syncMcpManifest( + baseUrl: string | null, + siteName: string, + ): Promise { + const relativePath = ".well-known/mcp.json"; + const outputPath = this.publicOutputFilePath(relativePath); + const sourcePath = await this.findSourcePublicAsset(relativePath); + if (sourcePath) { + console.warn( + chalk.yellow( + `⚠️ Skipping generated public/${relativePath}; a project public asset owns that path`, + ), + ); + await this.copyRegularPublicFile(sourcePath, outputPath); + } else if (baseUrl) { + const content = + JSON.stringify( + { + mcpServers: { + [siteDocsSlug(siteName)]: { + url: `${baseUrl}/api/mcp`, + transport: "streamable-http", + }, + }, + }, + null, + 2, + ) + "\n"; + await fs.ensureDir(path.dirname(outputPath)); + await writeFileAtomic(outputPath, content); + } else if (await fs.pathExists(outputPath)) { + await fs.remove(outputPath); + } + } + + async syncLlmsPageFiles( + pages: PageWithBody[], + baseUrl: string | null, + ): Promise { + const publicDir = this.outputPath("public"); + const nextRelativePaths = new Set(); + await Promise.all( + pages.map(async (page) => { + const relativePath = page.slug === "" ? "index.md" : `${page.slug}.md`; + if (isPublicAggregate(relativePath)) return; + const sourcePath = await this.findSourcePublicAsset(relativePath); + if (sourcePath) { + console.warn( + chalk.yellow( + `⚠️ Skipping generated public/${relativePath}; a project public asset owns that path`, + ), + ); + const targetPath = this.publicOutputFilePath(relativePath); + await this.copyRegularPublicFile(sourcePath, targetPath); + return; + } + const targetPath = this.publicOutputFilePath(relativePath); + await fs.ensureDir(path.dirname(targetPath)); + await writeFileAtomic(targetPath, llmsPageTemplate(page, baseUrl)); + nextRelativePaths.add(relativePath); + }), + ); + + const previousRelativePaths = this.artifacts.llmsPageFiles(); + for (const stale of previousRelativePaths) { + if (!nextRelativePaths.has(stale)) { + try { + if (isPublicAggregate(stale)) continue; + if (await this.findSourcePublicAsset(stale)) continue; + const stalePath = resolveOutputPath(publicDir, stale); + if (await fs.pathExists(stalePath)) { + await fs.remove(stalePath); + } + } catch { + // ignore + } + } + } + this.artifacts.replaceLlmsPageFiles(nextRelativePaths); + await this.artifacts.save(); + } +} diff --git a/src/generator/section-resolver.ts b/src/generator/section-resolver.ts new file mode 100644 index 0000000..9523c7b --- /dev/null +++ b/src/generator/section-resolver.ts @@ -0,0 +1,207 @@ +import { generateSlug } from "../lib/utils.js"; +import { slugifySegment } from "../lib/openapi.js"; +import type { SectionConfig } from "../lib/types.js"; + +export interface SectionDocument { + filePath: string; + frontmatter: Record; +} + +export interface SectionRoute { + sectionSlug: string; + pageSlug: string; +} + +export function validateSectionsConfig( + parsed: unknown, +): SectionConfig[] | null { + if (!Array.isArray(parsed) || parsed.length === 0) return null; + + const seenLabels = new Set(); + const seenSlugs = new Set(); + return parsed.map((entry, index) => { + if (!entry || typeof entry !== "object") { + throw new Error(`sections.json entry ${index + 1} must be an object`); + } + const candidate = entry as Record; + const label = + typeof candidate.label === "string" ? candidate.label.trim() : ""; + const slug = + typeof candidate.slug === "string" ? candidate.slug.trim() : ""; + if (!label) { + throw new Error( + `sections.json entry ${index + 1} needs a non-empty label`, + ); + } + if ( + slug !== "" && + (slug !== slugifySegment(slug) || + slug.includes("/") || + slug === "." || + slug === "..") + ) { + throw new Error( + `Unsafe section slug "${slug}"; use a lowercase URL segment such as "${slugifySegment(slug)}"`, + ); + } + if (seenLabels.has(label) || seenSlugs.has(slug)) { + throw new Error(`Duplicate section label or slug at entry ${index + 1}`); + } + seenLabels.add(label); + seenSlugs.add(slug); + + let directory: string | undefined; + if (candidate.directory !== undefined) { + if (typeof candidate.directory !== "string") { + throw new Error( + `sections.json directory at entry ${index + 1} must be a string`, + ); + } + directory = candidate.directory + .replace(/\\/g, "/") + .replace(/^\/+|\/+$/g, ""); + const parts = directory.split("/"); + if ( + !directory || + parts.some( + (part) => + part === "." || part === ".." || part !== slugifySegment(part), + ) + ) { + throw new Error(`Unsafe section directory "${candidate.directory}"`); + } + } + + return { label, slug, ...(directory ? { directory } : {}) }; + }); +} + +export function discoverSections( + documents: SectionDocument[], +): SectionConfig[] | null { + const sectionMap = new Map(); + let hasUnsectionedPages = false; + let defaultSectionLabel = "Docs"; + + for (const { filePath, frontmatter } of documents) { + if (typeof frontmatter.section === "string" && frontmatter.section.trim()) { + const label = frontmatter.section.trim(); + const order = + typeof frontmatter.sectionOrder === "number" + ? frontmatter.sectionOrder + : 0; + const existing = sectionMap.get(label); + if (!existing || order < existing.order) { + sectionMap.set(label, { label, order }); + } + } else { + hasUnsectionedPages = true; + } + + if ( + (filePath === "index.mdx" || filePath === "./index.mdx") && + typeof frontmatter.sectionLabel === "string" && + frontmatter.sectionLabel.trim() + ) { + defaultSectionLabel = frontmatter.sectionLabel.trim(); + } + } + + if (sectionMap.size === 0) return null; + + const sorted = [...sectionMap.values()].sort((a, b) => a.order - b.order); + const sections: SectionConfig[] = []; + + if (hasUnsectionedPages) { + sections.push({ label: defaultSectionLabel, slug: "" }); + } + + const usedSlugs = new Set(sections.map((section) => section.slug)); + for (const section of sorted) { + const slug = slugifySegment(section.label); + if (usedSlugs.has(slug)) { + throw new Error( + `Section labels resolve to the same slug "${slug}". Rename one section or define sections.json explicitly.`, + ); + } + usedSlugs.add(slug); + sections.push({ label: section.label, slug }); + } + + return sections; +} + +export function addApiReferenceSection( + sections: SectionConfig[] | null, + hasApiReference: boolean, + apiBaseSlug: string, +): SectionConfig[] | null { + if (!hasApiReference) return sections; + const apiSection: SectionConfig = { + label: "API Reference", + slug: apiBaseSlug, + }; + if (!sections || sections.length === 0) { + return [{ label: "Documentation", slug: "" }, apiSection]; + } + if (sections.some((section) => section.slug === apiBaseSlug)) return sections; + return [...sections, apiSection]; +} + +export function determineSectionRoute( + filePath: string, + frontmatter: Record, + sections: SectionConfig[] | null, +): SectionRoute { + if (!sections || sections.length === 0) { + return { sectionSlug: "", pageSlug: generateSlug(filePath) }; + } + + const normalizedPath = filePath.replace(/\\/g, "/"); + const firstDir = normalizedPath.includes("/") + ? normalizedPath.split("/")[0] + : ""; + + for (const section of sections) { + if (!section.directory) continue; + const dirPrefix = section.directory + "/"; + if (normalizedPath.startsWith(dirPrefix)) { + return { + sectionSlug: section.slug, + pageSlug: generateSlug(normalizedPath.slice(dirPrefix.length)), + }; + } + } + + if (firstDir) { + const match = sections.find((section) => section.slug === firstDir); + if (match) { + const pathForSlug = normalizedPath.slice(firstDir.length + 1); + return { + sectionSlug: match.slug, + pageSlug: generateSlug(pathForSlug), + }; + } + } + + if (frontmatter.section) { + const label = frontmatter.section as string; + const match = sections.find((section) => section.label === label); + if (match) { + let pathForSlug = filePath; + if (firstDir && firstDir === match.slug) { + pathForSlug = normalizedPath.slice(firstDir.length + 1); + } + + return { + sectionSlug: match.slug, + pageSlug: generateSlug(pathForSlug), + }; + } + } + + return { + sectionSlug: "", + pageSlug: generateSlug(filePath), + }; +} diff --git a/src/generator/secure-source-fs.ts b/src/generator/secure-source-fs.ts new file mode 100644 index 0000000..c9a23ce --- /dev/null +++ b/src/generator/secure-source-fs.ts @@ -0,0 +1,542 @@ +import fs from "fs-extra"; +import { createHash } from "node:crypto"; +import { constants } from "node:fs"; +import { open, type FileHandle } from "node:fs/promises"; +import path from "node:path"; + +import { isPathInside, resolveWithin } from "../lib/output-safety.js"; + +function errorCode(error: unknown): string | undefined { + return error && typeof error === "object" && "code" in error + ? String(error.code) + : undefined; +} + +function sameFileIdentity(left: fs.Stats, right: fs.Stats): boolean { + return left.dev === right.dev && left.ino === right.ino; +} + +export class SecureSourceFs { + private documentationRoot: string; + private projectRoot: string; + + constructor(documentationRoot: string, projectRoot: string) { + this.documentationRoot = path.resolve(documentationRoot); + this.projectRoot = path.resolve(projectRoot); + } + + private sourcePathError(label: string, sourcePath: string, detail: string) { + return new Error( + `Refusing to use ${label} at ${sourcePath}: ${detail}. Replace it with a real file or directory inside the source root.`, + ); + } + + private async realSourceRoot( + root: string, + label: string, + rejectRootSymlink: boolean, + ): Promise { + let rootStat: fs.Stats; + try { + rootStat = await fs.lstat(root); + } catch (error) { + if (errorCode(error) === "ENOENT") { + throw this.sourcePathError( + label, + root, + "the source root does not exist", + ); + } + throw error; + } + if (rootStat.isSymbolicLink() && rejectRootSymlink) { + throw this.sourcePathError( + label, + root, + "the source root is a symbolic link", + ); + } + + const realRoot = await fs.realpath(root); + if (!(await fs.stat(realRoot)).isDirectory()) { + throw this.sourcePathError( + label, + root, + "the source root is not a directory", + ); + } + return realRoot; + } + + private async readSafeSourceFile( + root: string, + sourcePath: string, + label: string, + rejectRootSymlink: boolean, + ): Promise<{ data: Buffer; stat: fs.Stats }> { + const resolvedRoot = path.resolve(root); + const resolvedSource = path.resolve(sourcePath); + if (!isPathInside(resolvedRoot, resolvedSource)) { + throw this.sourcePathError( + label, + resolvedSource, + `the path is outside ${resolvedRoot}`, + ); + } + + const realRoot = await this.realSourceRoot( + resolvedRoot, + label, + rejectRootSymlink, + ); + const relativePath = path.relative(resolvedRoot, resolvedSource); + const components = relativePath.split(path.sep).filter(Boolean); + let currentPath = resolvedRoot; + for (const [index, component] of components.entries()) { + currentPath = path.join(currentPath, component); + let stat: fs.Stats; + try { + stat = await fs.lstat(currentPath); + } catch (error) { + if (errorCode(error) === "ENOENT") { + throw this.sourcePathError( + label, + currentPath, + "the path does not exist", + ); + } + throw error; + } + if (stat.isSymbolicLink()) { + throw this.sourcePathError( + label, + currentPath, + "the path is a symbolic link", + ); + } + if (index < components.length - 1 && !stat.isDirectory()) { + throw this.sourcePathError( + label, + currentPath, + "a path component is not a directory", + ); + } + } + + const sourceStat = await fs.lstat(resolvedSource); + if (!sourceStat.isFile()) { + throw this.sourcePathError( + label, + resolvedSource, + "expected a regular file", + ); + } + const realSource = await fs.realpath(resolvedSource); + if (!isPathInside(realRoot, realSource)) { + throw this.sourcePathError( + label, + resolvedSource, + `the real path ${realSource} is outside ${realRoot}`, + ); + } + + const noFollow = + typeof constants.O_NOFOLLOW === "number" ? constants.O_NOFOLLOW : 0; + let handle: FileHandle; + try { + handle = await open(resolvedSource, constants.O_RDONLY | noFollow); + } catch (error) { + if (errorCode(error) === "ELOOP") { + throw this.sourcePathError( + label, + resolvedSource, + "the final path became a symbolic link while it was being opened", + ); + } + throw error; + } + + try { + const openedStat = await handle.stat(); + if (!openedStat.isFile()) { + throw this.sourcePathError( + label, + resolvedSource, + "the opened source is not a regular file", + ); + } + + let currentStat: fs.Stats; + let currentRealSource: string; + let currentRealStat: fs.Stats; + try { + currentStat = await fs.lstat(resolvedSource); + currentRealSource = await fs.realpath(resolvedSource); + currentRealStat = await fs.lstat(currentRealSource); + } catch { + throw this.sourcePathError( + label, + resolvedSource, + "the path changed while it was being opened", + ); + } + + if (!isPathInside(realRoot, currentRealSource)) { + throw this.sourcePathError( + label, + resolvedSource, + `the real path ${currentRealSource} is outside ${realRoot}`, + ); + } + if ( + currentStat.isSymbolicLink() || + !currentStat.isFile() || + !currentRealStat.isFile() || + !sameFileIdentity(sourceStat, openedStat) || + !sameFileIdentity(openedStat, currentStat) || + !sameFileIdentity(openedStat, currentRealStat) + ) { + throw this.sourcePathError( + label, + resolvedSource, + "the source identity changed while it was being opened", + ); + } + + return { data: await handle.readFile(), stat: openedStat }; + } finally { + await handle.close(); + } + } + + async readMdxSourceFile( + filePath: string, + ): Promise<{ content: string; stat: fs.Stats }> { + if (!filePath.toLowerCase().endsWith(".mdx")) { + throw this.sourcePathError( + "documentation source", + filePath, + "expected an .mdx file", + ); + } + const { data, stat } = await this.readSafeSourceFile( + this.documentationRoot, + path.resolve(this.documentationRoot, filePath), + "documentation source", + false, + ); + return { content: data.toString("utf8"), stat }; + } + + async readProjectSourceFile( + sourcePath: string, + label: string, + ): Promise<{ data: Buffer; stat: fs.Stats }> { + return this.readSafeSourceFile(this.projectRoot, sourcePath, label, false); + } + + async readPublicSourceFile( + sourcePath: string, + ): Promise<{ data: Buffer; stat: fs.Stats }> { + return this.readSafeSourceFile( + path.join(this.projectRoot, "public"), + sourcePath, + "public source", + true, + ); + } + + async ensureSafeStarterPath(relativePath: string): Promise { + const resolvedRoot = path.resolve(this.documentationRoot); + const targetPath = path.resolve(resolvedRoot, relativePath); + if (!isPathInside(resolvedRoot, targetPath)) { + throw this.sourcePathError( + "documentation source", + targetPath, + `the starter path is outside ${resolvedRoot}`, + ); + } + + const realRoot = await this.realSourceRoot( + resolvedRoot, + "documentation source", + false, + ); + const parentRelativePath = path.relative( + resolvedRoot, + path.dirname(targetPath), + ); + let currentPath = resolvedRoot; + for (const component of parentRelativePath + .split(path.sep) + .filter(Boolean)) { + currentPath = path.join(currentPath, component); + try { + await fs.mkdir(currentPath); + } catch (error) { + if (errorCode(error) !== "EEXIST") throw error; + } + const stat = await fs.lstat(currentPath); + if (stat.isSymbolicLink() || !stat.isDirectory()) { + throw this.sourcePathError( + "documentation source", + currentPath, + "a starter directory component is not a real directory", + ); + } + const realPath = await fs.realpath(currentPath); + if (!isPathInside(realRoot, realPath)) { + throw this.sourcePathError( + "documentation source", + currentPath, + `the real path ${realPath} is outside ${realRoot}`, + ); + } + } + + try { + const stat = await fs.lstat(targetPath); + const detail = stat.isSymbolicLink() + ? "the starter file is a symbolic link" + : "the starter file appeared after the empty-source check"; + throw this.sourcePathError("documentation source", targetPath, detail); + } catch (error) { + if (errorCode(error) !== "ENOENT") throw error; + } + return targetPath; + } + + async pathState(filePath: string, hashContents = false): Promise { + try { + const stat = await fs.lstat(filePath); + const kind = stat.isDirectory() + ? "directory" + : stat.isFile() + ? "file" + : stat.isSymbolicLink() + ? "symlink" + : "other"; + const hash = + hashContents && stat.isFile() + ? createHash("sha256") + .update(await fs.readFile(filePath)) + .digest("hex") + : ""; + return `${kind}:${stat.size}:${stat.mtimeMs}:${stat.dev}:${stat.ino}:${hash}`; + } catch (error) { + if (errorCode(error) === "ENOENT") return "missing"; + throw error; + } + } + + async treeState( + root: string, + includeFile: (relativePath: string) => boolean, + hashContents = false, + ): Promise { + if (!(await fs.pathExists(root))) return "missing"; + const entries: string[] = []; + const scan = async (directory: string, relativePath = "") => { + const children = await fs.readdir(directory, { withFileTypes: true }); + children.sort((left, right) => left.name.localeCompare(right.name)); + for (const child of children) { + const childRelativePath = path.join(relativePath, child.name); + const childPath = path.join(directory, child.name); + const state = await this.pathState(childPath, hashContents); + if (state === "missing") continue; + if (child.isDirectory()) { + entries.push(`${childRelativePath.replace(/\\/g, "/")}:${state}`); + await scan(childPath, childRelativePath); + } else if (includeFile(childRelativePath)) { + entries.push(`${childRelativePath.replace(/\\/g, "/")}:${state}`); + } + } + }; + await scan(root); + return entries.join("\n"); + } + + async scanPublicFiles(): Promise { + const publicDir = path.join(this.projectRoot, "public"); + let publicStat: fs.Stats; + try { + publicStat = await fs.lstat(publicDir); + } catch (error) { + if (errorCode(error) === "ENOENT") return null; + throw error; + } + if (publicStat.isSymbolicLink() || !publicStat.isDirectory()) { + throw this.sourcePathError( + "public source", + publicDir, + "the public source root must be a real directory", + ); + } + + const realPublicDir = await this.realSourceRoot( + publicDir, + "public source", + true, + ); + const files: string[] = []; + const scanDir = async (directory: string, relativePath = "") => { + const entries = await fs.readdir(directory, { withFileTypes: true }); + for (const entry of entries) { + const sourcePath = path.join(directory, entry.name); + const entryRelativePath = path.join(relativePath, entry.name); + const stat = await fs.lstat(sourcePath); + if (stat.isSymbolicLink()) { + throw this.sourcePathError( + "public source", + sourcePath, + "the path is a symbolic link", + ); + } + const realPath = await fs.realpath(sourcePath); + if (!isPathInside(realPublicDir, realPath)) { + throw this.sourcePathError( + "public source", + sourcePath, + `the real path ${realPath} is outside ${realPublicDir}`, + ); + } + if (stat.isDirectory()) { + await scanDir(sourcePath, entryRelativePath); + } else if (stat.isFile()) { + files.push(entryRelativePath); + } else { + throw this.sourcePathError( + "public source", + sourcePath, + "expected a regular file or directory", + ); + } + } + }; + + await scanDir(publicDir); + return files; + } + + async getAllMdxFiles(): Promise { + const files: string[] = []; + const realWatchDir = await this.realSourceRoot( + this.documentationRoot, + "documentation source", + false, + ); + + const scanDir = async (dir: string, relativePath = "") => { + const entries = await fs.readdir(dir, { withFileTypes: true }); + + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + const relPath = path.join(relativePath, entry.name); + const stat = await fs.lstat(fullPath); + + if (stat.isSymbolicLink()) { + let linksToDirectory = false; + try { + linksToDirectory = (await fs.stat(fullPath)).isDirectory(); + } catch (error) { + if (errorCode(error) !== "ENOENT") throw error; + } + if (entry.name.endsWith(".mdx") || linksToDirectory) { + throw this.sourcePathError( + "documentation source", + fullPath, + "the path is a symbolic link", + ); + } + continue; + } + const realPath = await fs.realpath(fullPath); + if (!isPathInside(realWatchDir, realPath)) { + throw this.sourcePathError( + "documentation source", + fullPath, + `the real path ${realPath} is outside ${realWatchDir}`, + ); + } + + if (stat.isDirectory()) { + await scanDir(fullPath, relPath); + } else if (stat.isFile() && entry.name.endsWith(".mdx")) { + files.push(relPath); + } else if (!stat.isFile() && entry.name.endsWith(".mdx")) { + throw this.sourcePathError( + "documentation source", + fullPath, + "expected a regular .mdx file", + ); + } + } + }; + + await scanDir(this.documentationRoot); + return files; + } + + async findPublicAsset(relativePath: string): Promise { + const sourcePublicDir = path.join(this.projectRoot, "public"); + const normalized = relativePath.replace(/\\/g, "/"); + resolveWithin(sourcePublicDir, normalized); + + let rootStat: fs.Stats; + try { + rootStat = await fs.lstat(sourcePublicDir); + } catch (error) { + if (errorCode(error) === "ENOENT") return null; + throw error; + } + if (rootStat.isSymbolicLink() || !rootStat.isDirectory()) { + throw this.sourcePathError( + "public source", + sourcePublicDir, + "the public source root must be a real directory", + ); + } + const realPublicDir = await this.realSourceRoot( + sourcePublicDir, + "public source", + true, + ); + + let currentPath = sourcePublicDir; + const parts = normalized.split("/").filter(Boolean); + for (const [index, part] of parts.entries()) { + let entries: string[]; + try { + entries = await fs.readdir(currentPath); + } catch { + return null; + } + const actualName = + entries.find((entry) => entry === part) ?? + entries.find((entry) => entry.toLowerCase() === part.toLowerCase()); + if (!actualName) return null; + currentPath = path.join(currentPath, actualName); + const stat = await fs.lstat(currentPath); + if (stat.isSymbolicLink()) { + throw this.sourcePathError( + "public source", + currentPath, + "the path is a symbolic link", + ); + } + if (index < parts.length - 1 && !stat.isDirectory()) return null; + } + + const stat = await fs.lstat(currentPath); + if (!stat.isFile()) return null; + const realPath = await fs.realpath(currentPath); + if (!isPathInside(realPublicDir, realPath)) { + throw this.sourcePathError( + "public source", + currentPath, + `the real path ${realPath} is outside ${realPublicDir}`, + ); + } + return currentPath; + } +} diff --git a/src/generator/site-artifacts.ts b/src/generator/site-artifacts.ts new file mode 100644 index 0000000..c1daa5b --- /dev/null +++ b/src/generator/site-artifacts.ts @@ -0,0 +1,263 @@ +import chalk from "chalk"; +import fs from "fs-extra"; +import path from "node:path"; + +import { resolveOutputPath } from "../lib/output-safety.js"; +import type { PageMeta, SectionConfig } from "../lib/types.js"; +import { safeMatter, writeFileAtomic } from "../lib/utils.js"; +import { robotsTemplate } from "../templates/app/robots.js"; +import { + sitemapTemplate, + type SitemapEntry, +} from "../templates/app/sitemap.js"; +import { + llmsFullTemplate, + type PageWithBody, +} from "../templates/llms/llmsFull.js"; +import { llmsIndexTemplate } from "../templates/llms/llmsIndex.js"; +import { skillMdTemplate } from "../templates/llms/skillMd.js"; +import type { PublicAssetManager } from "./public-asset-manager.js"; + +export interface SiteMetadata { + url: string | null; + name: string; + description: string; +} + +type ResolvePages = () => Promise; +type ReadMdxSource = (filePath: string) => Promise<{ content: string }>; +type ReadOpenApiBody = (slug: string) => string | undefined; + +export async function loadSiteUrl(rootDir: string): Promise { + const fromEnv = process.env.NEXT_PUBLIC_SITE_URL?.trim(); + if (fromEnv) return fromEnv.replace(/\/$/, ""); + + const configPath = path.join(rootDir, "config.json"); + + try { + if (await fs.pathExists(configPath)) { + const content = await fs.readFile(configPath, "utf8"); + const parsed = JSON.parse(content) as { url?: unknown }; + if (typeof parsed.url === "string" && parsed.url.trim() !== "") { + return parsed.url.trim().replace(/\/$/, ""); + } + } + } catch (error) { + console.warn(chalk.yellow("⚠️ Error reading config.json"), error); + } + + return null; +} + +export async function loadSiteMetadata(rootDir: string): Promise { + const configPath = path.join(rootDir, "config.json"); + let url: string | null = null; + let name = "Documentation"; + let description = ""; + + try { + const fromEnv = process.env.NEXT_PUBLIC_SITE_URL?.trim(); + if (fromEnv) url = fromEnv.replace(/\/$/, ""); + if (await fs.pathExists(configPath)) { + const content = await fs.readFile(configPath, "utf8"); + const parsed = JSON.parse(content) as { + url?: unknown; + name?: unknown; + title?: unknown; + description?: unknown; + }; + if (!url && typeof parsed.url === "string" && parsed.url.trim() !== "") { + url = parsed.url.trim().replace(/\/$/, ""); + } + if (typeof parsed.name === "string" && parsed.name.trim() !== "") { + name = parsed.name.trim(); + } else if ( + typeof parsed.title === "string" && + parsed.title.trim() !== "" + ) { + name = parsed.title.trim(); + } + if ( + typeof parsed.description === "string" && + parsed.description.trim() !== "" + ) { + description = parsed.description.trim(); + } + } + } catch (error) { + console.warn( + chalk.yellow("⚠️ Error reading config.json for llms metadata"), + error, + ); + } + + return { url, name, description }; +} + +export function buildSitemapEntries( + pages: PageMeta[], + sectionsConfig: SectionConfig[] | null, +): SitemapEntry[] { + const sectionSlugs = new Set( + (sectionsConfig || []) + .map((section) => section.slug) + .filter( + (slug): slug is string => typeof slug === "string" && slug !== "", + ), + ); + + const entries: SitemapEntry[] = pages.map((page) => { + let priority = 0.5; + if (page.slug === "") { + priority = 1.0; + } else if (sectionSlugs.has(page.slug)) { + priority = 0.8; + } + return { + slug: page.slug, + lastModified: page.lastModified, + changeFrequency: "weekly", + priority, + }; + }); + + if (!entries.some((entry) => entry.slug === "")) { + entries.unshift({ + slug: "", + changeFrequency: "weekly", + priority: 1.0, + }); + } + + return entries; +} + +export async function writeSitemap( + outputDir: string, + sectionsConfig: SectionConfig[] | null, + resolvePages: ResolvePages, + resolveSiteUrl: () => Promise, +): Promise { + const siteUrl = await resolveSiteUrl(); + const entries = buildSitemapEntries(await resolvePages(), sectionsConfig); + await writeFileAtomic( + resolveOutputPath(outputDir, "app", "sitemap.ts"), + sitemapTemplate(entries), + ); + console.log( + chalk.green( + `πŸ—ΊοΈ Generated sitemap.ts with ${entries.length} page(s)${ + siteUrl ? ` using ${siteUrl}` : " (waiting for a deployment URL)" + }`, + ), + ); +} + +export async function writeRobots( + outputDir: string, + resolveSiteUrl: () => Promise, +): Promise { + const siteUrl = await resolveSiteUrl(); + await writeFileAtomic( + resolveOutputPath(outputDir, "app", "robots.ts"), + robotsTemplate, + ); + console.log( + chalk.green( + siteUrl + ? `πŸ€– Regenerated robots.ts with sitemap link` + : `πŸ€– Regenerated robots.ts (no sitemap link)`, + ), + ); +} + +export async function collectPageBodies( + pages: PageMeta[], + readMdxSource: ReadMdxSource, + readOpenApiBody: ReadOpenApiBody, +): Promise { + return Promise.all( + pages.map(async (page) => { + if (!page.path.endsWith(".mdx")) { + return { ...page, body: readOpenApiBody(page.slug) ?? "" }; + } + const { content: raw } = await readMdxSource(page.path); + const { content: body } = safeMatter(raw, page.path); + return { ...page, body }; + }), + ); +} + +export async function writeLlmsFiles( + outputDir: string, + sectionsConfig: SectionConfig[] | null, + resolvePages: ResolvePages, + readMdxSource: ReadMdxSource, + readOpenApiBody: ReadOpenApiBody, + resolveSiteMetadata: () => Promise, + publicAssetManager: PublicAssetManager, +): Promise { + await fs.ensureDir(resolveOutputPath(outputDir, "public")); + + const { url: baseUrl, name, description } = await resolveSiteMetadata(); + const resolvedPages = await resolvePages(); + const pagesWithBodies = await collectPageBodies( + resolvedPages, + readMdxSource, + readOpenApiBody, + ); + const docsContent = pagesWithBodies.map((page) => { + const route = page.slug.replace(/^\/+|\/+$/g, ""); + const pagePath = route + ? `app/(site)/${route}/page.tsx` + : "app/(site)/page.tsx"; + return { + uri: `docs://${route || "/"}`, + name: page.title, + path: pagePath, + content: page.body, + }; + }); + await writeFileAtomic( + resolveOutputPath(outputDir, "services", "mcp", "docs-content.json"), + JSON.stringify(docsContent, null, 2) + "\n", + ); + + const indexContent = llmsIndexTemplate({ + siteName: name, + siteDescription: description, + baseUrl, + pages: resolvedPages, + sectionsConfig, + }); + const fullContent = llmsFullTemplate({ + siteName: name, + siteDescription: description, + baseUrl, + pages: pagesWithBodies, + sectionsConfig, + }); + + await publicAssetManager.writePublicAggregate("llms.txt", indexContent); + await publicAssetManager.writePublicAggregate("llms-full.txt", fullContent); + + const skillContent = skillMdTemplate({ + siteName: name, + siteDescription: description, + baseUrl, + pages: resolvedPages, + sectionsConfig, + }); + await publicAssetManager.writePublicAggregate("skill.md", skillContent); + + await publicAssetManager.syncMcpManifest(baseUrl, name); + await publicAssetManager.syncLlmsPageFiles(pagesWithBodies, baseUrl); + + console.log( + chalk.green( + `πŸ€– Generated llms.txt and llms-full.txt with ${resolvedPages.length} page(s)${ + baseUrl ? ` using ${baseUrl}` : " (relative URLs)" + }`, + ), + ); +} diff --git a/src/generator/watch-coordinator.ts b/src/generator/watch-coordinator.ts new file mode 100644 index 0000000..7441a5d --- /dev/null +++ b/src/generator/watch-coordinator.ts @@ -0,0 +1,644 @@ +import chalk from "chalk"; +import chokidar, { type FSWatcher } from "chokidar"; +import fs from "fs-extra"; +import path from "node:path"; + +import { isPathInside } from "../lib/output-safety.js"; +import type { NormalizedOpenApiSpec } from "../lib/types.js"; +import type { SecureSourceFs } from "./secure-source-fs.js"; + +interface WatchSourceSnapshot { + mdx: string; + configs: Record; + font: string; + analytics: string; + doccupine: string; + public: string; + openapi: string; +} + +interface WatchCoordinatorCallbacks { + syncOpenApiSpecWatcher(): Promise; + handleFileChange(action: string, filePath: string): Promise; + handleFileDelete(filePath: string): Promise; + handleConfigFileChange(filePath: string): Promise; + handleConfigFileDelete(filePath: string): Promise; + handleFontConfigChange(): Promise; + handleFontConfigDelete(): Promise; + handleAnalyticsConfigChange(): Promise; + handleAnalyticsConfigDelete(): Promise; + handleDoccupineConfigChange(): Promise; + handleOpenApiChange(): Promise; + copyPublicFiles(): Promise; + handlePublicFileChange(filePath: string): Promise; + handlePublicFileDelete(filePath: string): Promise; + processAllMDXFiles(): Promise; +} + +interface WatchCoordinatorOptions { + watchDir: string; + rootDir: string; + configFiles: readonly string[]; + fontConfigFile: string; + analyticsConfigFile: string; + doccupineConfigFile: string; + sourceFs: SecureSourceFs; + getOpenApiSpecs(): readonly NormalizedOpenApiSpec[]; + callbacks: WatchCoordinatorCallbacks; +} + +export class WatchCoordinator { + private watcher: FSWatcher | null = null; + private configWatcher: FSWatcher | null = null; + private fontWatcher: FSWatcher | null = null; + private publicWatcher: FSWatcher | null = null; + private rootDirWatcher: FSWatcher | null = null; + private analyticsWatcher: FSWatcher | null = null; + private openApiWatcher: FSWatcher | null = null; + private doccupineConfigWatcher: FSWatcher | null = null; + private mutationQueue: Promise = Promise.resolve(); + private stopping = false; + private readyCancellations = new Set<() => void>(); + private sourceSnapshot: WatchSourceSnapshot | null = null; + private publicWatcherStarting = false; + + constructor(private readonly options: WatchCoordinatorOptions) {} + + private enqueueMutation(label: string, task: () => Promise): void { + this.mutationQueue = this.mutationQueue.then(task).catch((error) => { + console.error(chalk.red(`❌ ${label}:`), error); + }); + } + + private waitForWatcherReady(watcher: FSWatcher): Promise { + if (this.stopping) return Promise.resolve(); + + return new Promise((resolve, reject) => { + const cleanup = () => { + watcher.removeListener("ready", onReady); + watcher.removeListener("error", onError); + this.readyCancellations.delete(onStop); + }; + const onReady = () => { + cleanup(); + resolve(); + }; + const onError = (error: unknown) => { + cleanup(); + reject(error); + }; + const onStop = () => { + cleanup(); + resolve(); + }; + this.readyCancellations.add(onStop); + watcher.once("ready", onReady); + watcher.once("error", onError); + }); + } + + private async captureSourceSnapshot(): Promise { + const { + analyticsConfigFile, + configFiles, + doccupineConfigFile, + fontConfigFile, + getOpenApiSpecs, + rootDir, + sourceFs, + watchDir, + } = this.options; + const configs = Object.fromEntries( + await Promise.all( + configFiles.map(async (fileName) => [ + fileName, + await sourceFs.pathState(path.join(rootDir, fileName), true), + ]), + ), + ); + const openapi = ( + await Promise.all( + getOpenApiSpecs().map(async (spec) => { + const specPath = path.resolve(rootDir, spec.file); + return `${specPath}:${await sourceFs.pathState(specPath, true)}`; + }), + ) + ).join("\n"); + + return { + mdx: await sourceFs.treeState( + watchDir, + (relativePath) => relativePath.toLowerCase().endsWith(".mdx"), + true, + ), + configs, + font: await sourceFs.pathState(path.join(rootDir, fontConfigFile), true), + analytics: await sourceFs.pathState( + path.join(rootDir, analyticsConfigFile), + true, + ), + doccupine: await sourceFs.pathState( + path.join(rootDir, doccupineConfigFile), + true, + ), + public: await sourceFs.treeState( + path.join(rootDir, "public"), + () => true, + ), + openapi, + }; + } + + async establishSourceSnapshot(): Promise { + this.sourceSnapshot = await this.captureSourceSnapshot(); + } + + private async reconcileWatchedSources( + previous: WatchSourceSnapshot | null, + current: WatchSourceSnapshot, + ): Promise { + const { + analyticsConfigFile, + callbacks, + configFiles, + doccupineConfigFile, + fontConfigFile, + getOpenApiSpecs, + rootDir, + } = this.options; + const doccupineConfigPath = path.join(rootDir, doccupineConfigFile); + const doccupineChanged = + previous === null || previous.doccupine !== current.doccupine; + if (doccupineChanged && (await fs.pathExists(doccupineConfigPath))) { + await callbacks.handleDoccupineConfigChange(); + } + + let sectionsChanged = false; + for (const configFile of configFiles) { + if ( + previous !== null && + previous.configs[configFile] === current.configs[configFile] + ) { + continue; + } + const sourcePath = path.join(rootDir, configFile); + if (await fs.pathExists(sourcePath)) { + await callbacks.handleConfigFileChange(sourcePath); + } else { + await callbacks.handleConfigFileDelete(sourcePath); + } + if (configFile === "sections.json") sectionsChanged = true; + } + + const fontPath = path.join(rootDir, fontConfigFile); + if (previous === null || previous.font !== current.font) { + if (await fs.pathExists(fontPath)) { + await callbacks.handleFontConfigChange(); + } else { + await callbacks.handleFontConfigDelete(); + } + } + + const analyticsPath = path.join(rootDir, analyticsConfigFile); + if (previous === null || previous.analytics !== current.analytics) { + if (await fs.pathExists(analyticsPath)) { + await callbacks.handleAnalyticsConfigChange(); + } else { + await callbacks.handleAnalyticsConfigDelete(); + } + } + + const publicDir = path.join(rootDir, "public"); + if (previous === null || previous.public !== current.public) { + if ( + !this.stopping && + !this.publicWatcher && + (await fs.pathExists(publicDir)) + ) { + await this.waitForWatcherReady(this.setupPublicWatcher()); + if (this.stopping) return; + } + await callbacks.copyPublicFiles(); + } + + if ( + !doccupineChanged && + getOpenApiSpecs().length > 0 && + (previous === null || previous.openapi !== current.openapi) + ) { + await callbacks.handleOpenApiChange(); + } + if ( + !sectionsChanged && + (previous === null || previous.mdx !== current.mdx) + ) { + await callbacks.processAllMDXFiles(); + } + } + + async startWatching(): Promise { + const { + analyticsConfigFile, + callbacks, + configFiles, + doccupineConfigFile, + fontConfigFile, + rootDir, + watchDir, + } = this.options; + this.stopping = false; + console.log(chalk.yellow(`πŸ‘€ Watching for changes in: ${watchDir}`)); + const ready: Promise[] = []; + + this.watcher = chokidar.watch(watchDir, { + persistent: true, + ignoreInitial: true, + followSymlinks: false, + ignored: (filePath: string, stats?: fs.Stats) => { + const isFile = stats?.isFile() ?? path.extname(filePath) !== ""; + const fileName = path.basename(filePath); + + if (configFiles.includes(fileName)) { + return true; + } + + if (isFile && !filePath.endsWith(".mdx")) { + return true; + } + return false; + }, + }); + ready.push(this.waitForWatcherReady(this.watcher)); + + this.watcher + .on("add", (filePath: string) => { + const relativePath = path.relative(watchDir, filePath); + this.enqueueMutation("Error processing added MDX file", () => + callbacks.handleFileChange("added", relativePath), + ); + }) + .on("change", (filePath: string) => { + const relativePath = path.relative(watchDir, filePath); + this.enqueueMutation("Error processing changed MDX file", () => + callbacks.handleFileChange("changed", relativePath), + ); + }) + .on("unlink", (filePath: string) => { + const relativePath = path.relative(watchDir, filePath); + this.enqueueMutation("Error processing deleted MDX file", () => + callbacks.handleFileDelete(relativePath), + ); + }) + .on("ready", () => { + console.log( + chalk.green("πŸ“ Initial scan complete. Ready for changes..."), + ); + }) + .on("error", (error: unknown) => { + console.error(chalk.red("❌ Watcher error:"), error); + }); + + const configPaths = configFiles.map((file) => path.join(rootDir, file)); + this.configWatcher = chokidar.watch(configPaths, { + persistent: true, + ignoreInitial: true, + }); + ready.push(this.waitForWatcherReady(this.configWatcher)); + + this.configWatcher + .on("add", (filePath: string) => { + console.log( + chalk.cyan(`πŸ“ Config file added: ${path.basename(filePath)}`), + ); + this.enqueueMutation("Error applying config file", () => + callbacks.handleConfigFileChange(filePath), + ); + }) + .on("change", (filePath: string) => { + console.log( + chalk.cyan(`πŸ“ Config file changed: ${path.basename(filePath)}`), + ); + this.enqueueMutation("Error applying config file", () => + callbacks.handleConfigFileChange(filePath), + ); + }) + .on("unlink", (filePath: string) => { + console.log( + chalk.red(`πŸ—‘οΈ Config file deleted: ${path.basename(filePath)}`), + ); + this.enqueueMutation("Error deleting config file", () => + callbacks.handleConfigFileDelete(filePath), + ); + }) + .on("error", (error: unknown) => { + console.error(chalk.red("❌ Config watcher error:"), error); + }); + + const fontPath = path.join(rootDir, fontConfigFile); + this.fontWatcher = chokidar.watch(fontPath, { + persistent: true, + ignoreInitial: true, + }); + ready.push(this.waitForWatcherReady(this.fontWatcher)); + + this.fontWatcher + .on("add", () => { + console.log(chalk.cyan(`πŸ”€ Font configuration added`)); + this.enqueueMutation("Error applying font configuration", () => + callbacks.handleFontConfigChange(), + ); + }) + .on("change", () => { + this.enqueueMutation("Error applying font configuration", () => + callbacks.handleFontConfigChange(), + ); + }) + .on("unlink", () => { + this.enqueueMutation("Error deleting font configuration", () => + callbacks.handleFontConfigDelete(), + ); + }) + .on("error", (error: unknown) => { + console.error(chalk.red("❌ Font watcher error:"), error); + }); + + const analyticsPath = path.join(rootDir, analyticsConfigFile); + this.analyticsWatcher = chokidar.watch(analyticsPath, { + persistent: true, + ignoreInitial: true, + }); + ready.push(this.waitForWatcherReady(this.analyticsWatcher)); + + this.analyticsWatcher + .on("add", () => { + console.log(chalk.cyan(`πŸ“Š Analytics configuration added`)); + this.enqueueMutation("Error applying analytics configuration", () => + callbacks.handleAnalyticsConfigChange(), + ); + }) + .on("change", () => { + this.enqueueMutation("Error applying analytics configuration", () => + callbacks.handleAnalyticsConfigChange(), + ); + }) + .on("unlink", () => { + this.enqueueMutation("Error deleting analytics configuration", () => + callbacks.handleAnalyticsConfigDelete(), + ); + }) + .on("error", (error: unknown) => { + console.error(chalk.red("❌ Analytics watcher error:"), error); + }); + + await callbacks.syncOpenApiSpecWatcher(); + if (this.stopping) return; + + const doccupineConfigPath = path.join(rootDir, doccupineConfigFile); + this.doccupineConfigWatcher = chokidar.watch(doccupineConfigPath, { + persistent: true, + ignoreInitial: true, + }); + ready.push(this.waitForWatcherReady(this.doccupineConfigWatcher)); + + this.doccupineConfigWatcher + .on("add", () => + this.enqueueMutation("Error applying doccupine.json", () => + callbacks.handleDoccupineConfigChange(), + ), + ) + .on("change", () => + this.enqueueMutation("Error applying doccupine.json", () => + callbacks.handleDoccupineConfigChange(), + ), + ) + .on("error", (error: unknown) => { + console.error(chalk.red("❌ doccupine.json watcher error:"), error); + }); + + const publicDir = path.join(rootDir, "public"); + if (await fs.pathExists(publicDir)) { + if (this.stopping) return; + ready.push(this.waitForWatcherReady(this.setupPublicWatcher())); + } + if (this.stopping) return; + + this.rootDirWatcher = chokidar.watch(rootDir, { + persistent: true, + ignoreInitial: true, + depth: 1, + }); + ready.push(this.waitForWatcherReady(this.rootDirWatcher)); + + const queuePublicWatcherStart = () => { + if (this.stopping || this.publicWatcher || this.publicWatcherStarting) { + return; + } + this.publicWatcherStarting = true; + this.enqueueMutation("Error initializing public directory", async () => { + try { + if (this.stopping) return; + console.log(chalk.cyan("πŸ“ Public directory created")); + await this.waitForWatcherReady(this.setupPublicWatcher()); + if (this.stopping) return; + await callbacks.copyPublicFiles(); + } finally { + this.publicWatcherStarting = false; + } + }); + }; + + this.rootDirWatcher + .on("addDir", (dirPath: string) => { + if ( + path.basename(dirPath) === "public" && + path.dirname(dirPath) === rootDir + ) { + queuePublicWatcherStart(); + } + }) + .on("add", (filePath: string) => { + if (isPathInside(publicDir, filePath)) queuePublicWatcherStart(); + }) + .on("unlinkDir", (dirPath: string) => { + if ( + path.basename(dirPath) !== "public" || + path.dirname(dirPath) !== rootDir + ) { + return; + } + const watcher = this.publicWatcher; + this.publicWatcher = null; + void watcher?.close(); + this.enqueueMutation("Error removing public directory", () => + callbacks.copyPublicFiles(), + ); + }) + .on("error", (error: unknown) => { + console.error(chalk.red("❌ Root dir watcher error:"), error); + }); + + await Promise.all(ready); + if (this.stopping) return; + this.enqueueMutation("Error reconciling watched sources", async () => { + const current = await this.captureSourceSnapshot(); + await this.reconcileWatchedSources(this.sourceSnapshot, current); + this.sourceSnapshot = await this.captureSourceSnapshot(); + }); + await this.mutationQueue; + } + + private setupPublicWatcher(): FSWatcher { + if (this.publicWatcher) { + return this.publicWatcher; + } + if (this.stopping) { + throw new Error("Cannot start a public watcher while stopping"); + } + + const { callbacks, rootDir } = this.options; + const publicDir = path.join(rootDir, "public"); + this.publicWatcher = chokidar.watch(publicDir, { + persistent: true, + ignoreInitial: true, + followSymlinks: false, + }); + + this.publicWatcher + .on("add", (filePath: string) => { + console.log( + chalk.cyan( + `πŸ“ Public file added: ${path.relative(publicDir, filePath)}`, + ), + ); + this.enqueueMutation("Error copying public file", () => + callbacks.handlePublicFileChange(filePath), + ); + }) + .on("change", (filePath: string) => { + console.log( + chalk.cyan( + `πŸ“ Public file changed: ${path.relative(publicDir, filePath)}`, + ), + ); + this.enqueueMutation("Error copying public file", () => + callbacks.handlePublicFileChange(filePath), + ); + }) + .on("unlink", (filePath: string) => { + console.log( + chalk.red( + `πŸ—‘οΈ Public file deleted: ${path.relative(publicDir, filePath)}`, + ), + ); + this.enqueueMutation("Error deleting public file", () => + callbacks.handlePublicFileDelete(filePath), + ); + }) + .on("unlinkDir", (dirPath: string) => { + if (path.resolve(dirPath) !== path.resolve(publicDir)) return; + const watcher = this.publicWatcher; + this.publicWatcher = null; + void watcher?.close(); + this.enqueueMutation("Error removing public directory", () => + callbacks.copyPublicFiles(), + ); + }) + .on("error", (error: unknown) => { + console.error(chalk.red("❌ Public watcher error:"), error); + }); + return this.publicWatcher; + } + + async syncOpenApiSpecWatcher( + specs: readonly NormalizedOpenApiSpec[], + ): Promise { + const previousWatcher = this.openApiWatcher; + this.openApiWatcher = null; + if (previousWatcher) await previousWatcher.close(); + if (this.stopping || specs.length === 0) return; + + const { callbacks, rootDir } = this.options; + const specPaths = specs.map((spec) => path.resolve(rootDir, spec.file)); + const watcher = chokidar.watch(specPaths, { + persistent: true, + ignoreInitial: true, + }); + this.openApiWatcher = watcher; + + watcher + .on("add", () => + this.enqueueMutation("Error rebuilding API reference", () => + callbacks.handleOpenApiChange(), + ), + ) + .on("change", () => + this.enqueueMutation("Error rebuilding API reference", () => + callbacks.handleOpenApiChange(), + ), + ) + .on("unlink", () => + this.enqueueMutation("Error rebuilding API reference", () => + callbacks.handleOpenApiChange(), + ), + ) + .on("error", (error: unknown) => { + console.error(chalk.red("❌ OpenAPI watcher error:"), error); + }); + await this.waitForWatcherReady(watcher); + if (this.stopping && this.openApiWatcher === watcher) { + await watcher.close(); + this.openApiWatcher = null; + } + } + + async stop(): Promise { + this.stopping = true; + for (const cancel of [...this.readyCancellations]) cancel(); + if (this.watcher) { + await this.watcher.close(); + console.log(chalk.yellow("πŸ‘‹ Stopped watching for MDX changes")); + } + if (this.configWatcher) { + await this.configWatcher.close(); + console.log(chalk.yellow("πŸ‘‹ Stopped watching for config changes")); + } + if (this.fontWatcher) { + await this.fontWatcher.close(); + console.log(chalk.yellow("πŸ‘‹ Stopped watching for font config changes")); + } + if (this.analyticsWatcher) { + await this.analyticsWatcher.close(); + console.log( + chalk.yellow("πŸ‘‹ Stopped watching for analytics config changes"), + ); + } + if (this.openApiWatcher) { + await this.openApiWatcher.close(); + console.log(chalk.yellow("πŸ‘‹ Stopped watching for OpenAPI spec changes")); + } + if (this.doccupineConfigWatcher) { + await this.doccupineConfigWatcher.close(); + console.log( + chalk.yellow("πŸ‘‹ Stopped watching for doccupine.json changes"), + ); + } + if (this.publicWatcher) { + await this.publicWatcher.close(); + console.log( + chalk.yellow("πŸ‘‹ Stopped watching for public directory changes"), + ); + } + if (this.rootDirWatcher) { + await this.rootDirWatcher.close(); + } + await this.mutationQueue; + if (this.openApiWatcher) { + await this.openApiWatcher.close(); + this.openApiWatcher = null; + } + if (this.publicWatcher) { + await this.publicWatcher.close(); + this.publicWatcher = null; + } + } +} diff --git a/src/mdx-to-nextjs-generator.ts b/src/mdx-to-nextjs-generator.ts index 98732b3..9cb312d 100644 --- a/src/mdx-to-nextjs-generator.ts +++ b/src/mdx-to-nextjs-generator.ts @@ -1,66 +1,22 @@ -import chokidar, { FSWatcher } from "chokidar"; import fs from "fs-extra"; -import { createHash } from "node:crypto"; -import { constants } from "node:fs"; -import { open, type FileHandle } from "node:fs/promises"; import path from "path"; import chalk from "chalk"; -import { - appStructure, - obsoleteFiles, - startingDocsStructure, -} from "./lib/structures.js"; import { rootLayoutTemplate, siteLayoutTemplate } from "./lib/layout.js"; import { normalizeOpenApiConfig, validateConfig, } from "./lib/config-manager.js"; import { GeneratedArtifacts } from "./lib/generated-artifacts.js"; -import { - validateAnalyticsConfig, - validateFontConfig, -} from "./lib/project-config.js"; import { claimOutputDirectory, - isPathInside, resolveOutputPath, - resolveWithin, } from "./lib/output-safety.js"; -import { - OpenApiRegistry, - DEFAULT_API_BASE_SLUG, - buildEndpointDoc, - slugifySegment, -} from "./lib/openapi.js"; -import { - generateSlug, - getFullSlug, - escapeTemplateContent, - toJsStringLiteral, - safeMatter, - writeFileAtomic, -} from "./lib/utils.js"; -import { - generateMetadataBlock, - generateRuntimeOnlyMetadataBlock, - generateJsonLdScript, -} from "./lib/metadata.js"; -import { parseUpdateBlocks } from "./lib/rss.js"; +import { OpenApiRegistry, DEFAULT_API_BASE_SLUG } from "./lib/openapi.js"; +import { getFullSlug, safeMatter, writeFileAtomic } from "./lib/utils.js"; import { nextConfigTemplate } from "./templates/next.config.js"; -import { pnpmWorkspaceTemplate } from "./templates/pnpmWorkspace.js"; import { proxyTemplate } from "./templates/proxy.js"; -import { robotsTemplate } from "./templates/app/robots.js"; -import { rssRouteTemplate } from "./templates/app/rssRoute.js"; -import { sitemapTemplate, type SitemapEntry } from "./templates/app/sitemap.js"; -import { llmsIndexTemplate } from "./templates/llms/llmsIndex.js"; -import { - llmsFullTemplate, - type PageWithBody, -} from "./templates/llms/llmsFull.js"; -import { llmsPageTemplate } from "./templates/llms/llmsPage.js"; -import { siteDocsSlug, skillMdTemplate } from "./templates/llms/skillMd.js"; import type { DoccupineConfig, MDXFile, @@ -71,65 +27,41 @@ import type { NormalizedOpenApiSpec, } from "./lib/types.js"; import type { OperationDescriptor } from "./lib/openapi-types.js"; - -const PUBLIC_AGGREGATE_PATHS = new Set([ - "llms.txt", - "llms-full.txt", - "skill.md", - ".well-known/mcp.json", -]); - -function normalizePublicArtifactPath(relativePath: string): string { - return relativePath.replace(/\\/g, "/").toLowerCase(); -} - -function isPublicAggregate(relativePath: string): boolean { - return PUBLIC_AGGREGATE_PATHS.has(normalizePublicArtifactPath(relativePath)); -} - -function isManagedPublicArtifact(relativePath: string): boolean { - const normalized = normalizePublicArtifactPath(relativePath); - return PUBLIC_AGGREGATE_PATHS.has(normalized) || normalized.endsWith(".md"); -} - -function publicDestinationRelativePath(relativePath: string): string { - return isPublicAggregate(relativePath) - ? normalizePublicArtifactPath(relativePath) - : relativePath.replace(/\\/g, "/"); -} - -function errorCode(error: unknown): string | undefined { - return error && typeof error === "object" && "code" in error - ? String(error.code) - : undefined; -} - -function sameFileIdentity(left: fs.Stats, right: fs.Stats): boolean { - return left.dev === right.dev && left.ino === right.ino; -} - -interface WatchSourceSnapshot { - mdx: string; - configs: Record; - font: string; - analytics: string; - doccupine: string; - public: string; - openapi: string; -} +import { SecureSourceFs } from "./generator/secure-source-fs.js"; +import { AppScaffolder } from "./generator/app-scaffolder.js"; +import { ApiReferenceGenerator } from "./generator/api-reference-generator.js"; +import { GeneratedRouteManager } from "./generator/generated-route-manager.js"; +import { ProjectConfigRepository } from "./generator/project-config-repository.js"; +import { PublicAssetManager } from "./generator/public-asset-manager.js"; +import { WatchCoordinator } from "./generator/watch-coordinator.js"; +import { + addApiReferenceSection, + determineSectionRoute, + discoverSections, +} from "./generator/section-resolver.js"; +import { + renderHomepage, + renderMdxPage, + renderSectionPage, + type HomepageSource, +} from "./generator/page-renderer.js"; +import { + buildRealPagesMeta as buildRealPageCatalog, + mergePages as mergePageCatalog, + parseMdxPageMeta, +} from "./generator/page-catalog.js"; +import { + loadSiteMetadata, + loadSiteUrl as loadSiteUrlArtifact, + writeLlmsFiles, + writeRobots, + writeSitemap, +} from "./generator/site-artifacts.js"; export class MDXToNextJSGenerator { private watchDir: string; private outputDir: string; private rootDir: string; - private watcher: FSWatcher | null = null; - private configWatcher: FSWatcher | null = null; - private fontWatcher: FSWatcher | null = null; - private publicWatcher: FSWatcher | null = null; - private rootDirWatcher: FSWatcher | null = null; - private analyticsWatcher: FSWatcher | null = null; - private openApiWatcher: FSWatcher | null = null; - private doccupineConfigWatcher: FSWatcher | null = null; private doccupineConfigFile = "doccupine.json"; private configFiles = [ "theme.json", @@ -149,14 +81,13 @@ export class MDXToNextJSGenerator { private apiBaseSlug = DEFAULT_API_BASE_SLUG; private apiRegistry = new OpenApiRegistry(); private artifacts: GeneratedArtifacts; - /** Section slugs whose index redirect we wrote this session, for cleanup. */ - private generatedSectionIndexSlugs = new Set(); - /** Serializes watcher mutations so aggregate files never race one another. */ - private mutationQueue: Promise = Promise.resolve(); - private stopping = false; - private readyCancellations = new Set<() => void>(); - private watchSourceSnapshot: WatchSourceSnapshot | null = null; - private publicWatcherStarting = false; + private sourceFs: SecureSourceFs; + private appScaffolder: AppScaffolder; + private apiReferenceGenerator: ApiReferenceGenerator; + private generatedRouteManager: GeneratedRouteManager; + private projectConfigRepository: ProjectConfigRepository; + private publicAssetManager: PublicAssetManager; + private watchCoordinator: WatchCoordinator; constructor( watchDir: string, @@ -169,516 +100,76 @@ export class MDXToNextJSGenerator { this.rootDir = path.resolve(rootDir); this.openApiSpecs = openApiSpecs; this.artifacts = new GeneratedArtifacts(this.outputDir); - } - - private outputPath(...segments: string[]): string { - return resolveOutputPath(this.outputDir, ...segments); - } - - private publicOutputFilePath(relativePath: string): string { - const parent = path.dirname(relativePath); - const outputParent = - parent === "." - ? this.outputPath("public") - : this.outputPath("public", parent); - return resolveWithin(outputParent, path.basename(relativePath)); - } - - private sourcePathError(label: string, sourcePath: string, detail: string) { - return new Error( - `Refusing to use ${label} at ${sourcePath}: ${detail}. Replace it with a real file or directory inside the source root.`, - ); - } - - private async realSourceRoot( - root: string, - label: string, - rejectRootSymlink: boolean, - ): Promise { - let rootStat: fs.Stats; - try { - rootStat = await fs.lstat(root); - } catch (error) { - if (errorCode(error) === "ENOENT") { - throw this.sourcePathError( - label, - root, - "the source root does not exist", - ); - } - throw error; - } - if (rootStat.isSymbolicLink() && rejectRootSymlink) { - throw this.sourcePathError( - label, - root, - "the source root is a symbolic link", - ); - } - - const realRoot = await fs.realpath(root); - if (!(await fs.stat(realRoot)).isDirectory()) { - throw this.sourcePathError( - label, - root, - "the source root is not a directory", - ); - } - return realRoot; - } - - private async readSafeSourceFile( - root: string, - sourcePath: string, - label: string, - rejectRootSymlink: boolean, - ): Promise<{ data: Buffer; stat: fs.Stats }> { - const resolvedRoot = path.resolve(root); - const resolvedSource = path.resolve(sourcePath); - if (!isPathInside(resolvedRoot, resolvedSource)) { - throw this.sourcePathError( - label, - resolvedSource, - `the path is outside ${resolvedRoot}`, - ); - } - - const realRoot = await this.realSourceRoot( - resolvedRoot, - label, - rejectRootSymlink, - ); - const relativePath = path.relative(resolvedRoot, resolvedSource); - const components = relativePath.split(path.sep).filter(Boolean); - let currentPath = resolvedRoot; - for (const [index, component] of components.entries()) { - currentPath = path.join(currentPath, component); - let stat: fs.Stats; - try { - stat = await fs.lstat(currentPath); - } catch (error) { - if (errorCode(error) === "ENOENT") { - throw this.sourcePathError( - label, - currentPath, - "the path does not exist", - ); - } - throw error; - } - if (stat.isSymbolicLink()) { - throw this.sourcePathError( - label, - currentPath, - "the path is a symbolic link", - ); - } - if (index < components.length - 1 && !stat.isDirectory()) { - throw this.sourcePathError( - label, - currentPath, - "a path component is not a directory", - ); - } - } - - const sourceStat = await fs.lstat(resolvedSource); - if (!sourceStat.isFile()) { - throw this.sourcePathError( - label, - resolvedSource, - "expected a regular file", - ); - } - const realSource = await fs.realpath(resolvedSource); - if (!isPathInside(realRoot, realSource)) { - throw this.sourcePathError( - label, - resolvedSource, - `the real path ${realSource} is outside ${realRoot}`, - ); - } - - const noFollow = - typeof constants.O_NOFOLLOW === "number" ? constants.O_NOFOLLOW : 0; - let handle: FileHandle; - try { - handle = await open(resolvedSource, constants.O_RDONLY | noFollow); - } catch (error) { - if (errorCode(error) === "ELOOP") { - throw this.sourcePathError( - label, - resolvedSource, - "the final path became a symbolic link while it was being opened", - ); - } - throw error; - } - - try { - const openedStat = await handle.stat(); - if (!openedStat.isFile()) { - throw this.sourcePathError( - label, - resolvedSource, - "the opened source is not a regular file", - ); - } - - let currentStat: fs.Stats; - let currentRealSource: string; - let currentRealStat: fs.Stats; - try { - currentStat = await fs.lstat(resolvedSource); - currentRealSource = await fs.realpath(resolvedSource); - currentRealStat = await fs.lstat(currentRealSource); - } catch { - throw this.sourcePathError( - label, - resolvedSource, - "the path changed while it was being opened", - ); - } - - if (!isPathInside(realRoot, currentRealSource)) { - throw this.sourcePathError( - label, - resolvedSource, - `the real path ${currentRealSource} is outside ${realRoot}`, - ); - } - if ( - currentStat.isSymbolicLink() || - !currentStat.isFile() || - !currentRealStat.isFile() || - !sameFileIdentity(sourceStat, openedStat) || - !sameFileIdentity(openedStat, currentStat) || - !sameFileIdentity(openedStat, currentRealStat) - ) { - throw this.sourcePathError( - label, - resolvedSource, - "the source identity changed while it was being opened", - ); - } - - return { data: await handle.readFile(), stat: openedStat }; - } finally { - await handle.close(); - } - } - - private async readMdxSourceFile( - filePath: string, - ): Promise<{ content: string; stat: fs.Stats }> { - if (!filePath.toLowerCase().endsWith(".mdx")) { - throw this.sourcePathError( - "documentation source", - filePath, - "expected an .mdx file", - ); - } - const { data, stat } = await this.readSafeSourceFile( - this.watchDir, - path.resolve(this.watchDir, filePath), - "documentation source", - false, - ); - return { content: data.toString("utf8"), stat }; - } - - private async ensureSafeStarterPath(relativePath: string): Promise { - const resolvedRoot = path.resolve(this.watchDir); - const targetPath = path.resolve(resolvedRoot, relativePath); - if (!isPathInside(resolvedRoot, targetPath)) { - throw this.sourcePathError( - "documentation source", - targetPath, - `the starter path is outside ${resolvedRoot}`, - ); - } - - const realRoot = await this.realSourceRoot( - resolvedRoot, - "documentation source", - false, + this.sourceFs = new SecureSourceFs(this.watchDir, this.rootDir); + this.appScaffolder = new AppScaffolder(this.outputDir); + this.apiReferenceGenerator = new ApiReferenceGenerator( + this.outputDir, + this.artifacts, ); - const parentRelativePath = path.relative( - resolvedRoot, - path.dirname(targetPath), + this.generatedRouteManager = new GeneratedRouteManager( + this.outputDir, + this.artifacts, ); - let currentPath = resolvedRoot; - for (const component of parentRelativePath - .split(path.sep) - .filter(Boolean)) { - currentPath = path.join(currentPath, component); - try { - await fs.mkdir(currentPath); - } catch (error) { - if (errorCode(error) !== "EEXIST") throw error; - } - const stat = await fs.lstat(currentPath); - if (stat.isSymbolicLink() || !stat.isDirectory()) { - throw this.sourcePathError( - "documentation source", - currentPath, - "a starter directory component is not a real directory", - ); - } - const realPath = await fs.realpath(currentPath); - if (!isPathInside(realRoot, realPath)) { - throw this.sourcePathError( - "documentation source", - currentPath, - `the real path ${realPath} is outside ${realRoot}`, - ); - } - } - - try { - const stat = await fs.lstat(targetPath); - const detail = stat.isSymbolicLink() - ? "the starter file is a symbolic link" - : "the starter file appeared after the empty-source check"; - throw this.sourcePathError("documentation source", targetPath, detail); - } catch (error) { - if (errorCode(error) !== "ENOENT") throw error; - } - return targetPath; - } - - private async copyRegularPublicFile( - publicDir: string, - sourcePath: string, - destPath: string, - ): Promise { - const { data } = await this.readSafeSourceFile( - publicDir, - sourcePath, - "public source", - true, + this.projectConfigRepository = new ProjectConfigRepository( + this.rootDir, + this.outputDir, + this.sourceFs, + this.configFiles, + this.fontConfigFile, + this.analyticsConfigFile, ); - await writeFileAtomic(destPath, data); - } - - private async copyRootSourceFile( - sourcePath: string, - destPath: string, - label: string, - ): Promise { - const { data } = await this.readSafeSourceFile( + this.publicAssetManager = new PublicAssetManager( this.rootDir, - sourcePath, - label, - false, + this.outputDir, + this.artifacts, + this.sourceFs, ); - await writeFileAtomic(destPath, data); - } - - private enqueueMutation(label: string, task: () => Promise): void { - this.mutationQueue = this.mutationQueue.then(task).catch((error) => { - console.error(chalk.red(`❌ ${label}:`), error); + this.watchCoordinator = new WatchCoordinator({ + watchDir: this.watchDir, + rootDir: this.rootDir, + configFiles: this.configFiles, + fontConfigFile: this.fontConfigFile, + analyticsConfigFile: this.analyticsConfigFile, + doccupineConfigFile: this.doccupineConfigFile, + sourceFs: this.sourceFs, + getOpenApiSpecs: () => this.openApiSpecs, + callbacks: { + syncOpenApiSpecWatcher: () => this.syncOpenApiSpecWatcher(), + handleFileChange: (action, filePath) => + this.handleFileChange(action, filePath), + handleFileDelete: (filePath) => this.handleFileDelete(filePath), + handleConfigFileChange: (filePath) => + this.handleConfigFileChange(filePath), + handleConfigFileDelete: (filePath) => + this.handleConfigFileDelete(filePath), + handleFontConfigChange: () => this.handleFontConfigChange(), + handleFontConfigDelete: () => this.handleFontConfigDelete(), + handleAnalyticsConfigChange: () => this.handleAnalyticsConfigChange(), + handleAnalyticsConfigDelete: () => this.handleAnalyticsConfigDelete(), + handleDoccupineConfigChange: () => this.handleDoccupineConfigChange(), + handleOpenApiChange: () => this.handleOpenApiChange(), + copyPublicFiles: () => this.copyPublicFiles(), + handlePublicFileChange: (filePath) => + this.handlePublicFileChange(filePath), + handlePublicFileDelete: (filePath) => + this.handlePublicFileDelete(filePath), + processAllMDXFiles: () => this.processAllMDXFiles(), + }, }); } - private waitForWatcherReady(watcher: FSWatcher): Promise { - return new Promise((resolve, reject) => { - const cleanup = () => { - watcher.removeListener("ready", onReady); - watcher.removeListener("error", onError); - this.readyCancellations.delete(onStop); - }; - const onReady = () => { - cleanup(); - resolve(); - }; - const onError = (error: unknown) => { - cleanup(); - reject(error); - }; - const onStop = () => { - cleanup(); - resolve(); - }; - this.readyCancellations.add(onStop); - watcher.once("ready", onReady); - watcher.once("error", onError); - }); + private outputPath(...segments: string[]): string { + return resolveOutputPath(this.outputDir, ...segments); } - private async pathState( + private async readMdxSourceFile( filePath: string, - hashContents = false, - ): Promise { - try { - const stat = await fs.lstat(filePath); - const kind = stat.isDirectory() - ? "directory" - : stat.isFile() - ? "file" - : stat.isSymbolicLink() - ? "symlink" - : "other"; - const hash = - hashContents && stat.isFile() - ? createHash("sha256") - .update(await fs.readFile(filePath)) - .digest("hex") - : ""; - return `${kind}:${stat.size}:${stat.mtimeMs}:${stat.dev}:${stat.ino}:${hash}`; - } catch (error) { - if (errorCode(error) === "ENOENT") return "missing"; - throw error; - } - } - - private async treeState( - root: string, - includeFile: (relativePath: string) => boolean, - hashContents = false, - ): Promise { - if (!(await fs.pathExists(root))) return "missing"; - const entries: string[] = []; - const scan = async (directory: string, relativePath = "") => { - const children = await fs.readdir(directory, { withFileTypes: true }); - children.sort((left, right) => left.name.localeCompare(right.name)); - for (const child of children) { - const childRelativePath = path.join(relativePath, child.name); - const childPath = path.join(directory, child.name); - const state = await this.pathState(childPath, hashContents); - if (state === "missing") continue; - if (child.isDirectory()) { - entries.push(`${childRelativePath.replace(/\\/g, "/")}:${state}`); - await scan(childPath, childRelativePath); - } else if (includeFile(childRelativePath)) { - entries.push(`${childRelativePath.replace(/\\/g, "/")}:${state}`); - } - } - }; - await scan(root); - return entries.join("\n"); - } - - private async captureWatchSourceSnapshot(): Promise { - const configs = Object.fromEntries( - await Promise.all( - this.configFiles.map(async (fileName) => [ - fileName, - await this.pathState(path.join(this.rootDir, fileName), true), - ]), - ), - ); - const openapi = ( - await Promise.all( - this.openApiSpecs.map(async (spec) => { - const specPath = path.resolve(this.rootDir, spec.file); - return `${specPath}:${await this.pathState(specPath, true)}`; - }), - ) - ).join("\n"); - - return { - mdx: await this.treeState( - this.watchDir, - (relativePath) => relativePath.toLowerCase().endsWith(".mdx"), - true, - ), - configs, - font: await this.pathState( - path.join(this.rootDir, this.fontConfigFile), - true, - ), - analytics: await this.pathState( - path.join(this.rootDir, this.analyticsConfigFile), - true, - ), - doccupine: await this.pathState( - path.join(this.rootDir, this.doccupineConfigFile), - true, - ), - public: await this.treeState( - path.join(this.rootDir, "public"), - () => true, - ), - openapi, - }; + ): Promise<{ content: string; stat: fs.Stats }> { + return this.sourceFs.readMdxSourceFile(filePath); } - private async reconcileWatchedSources( - previous: WatchSourceSnapshot | null, - current: WatchSourceSnapshot, - ): Promise { - const doccupineConfigPath = path.join( - this.rootDir, - this.doccupineConfigFile, - ); - const doccupineChanged = - previous === null || previous.doccupine !== current.doccupine; - if (doccupineChanged && (await fs.pathExists(doccupineConfigPath))) { - await this.handleDoccupineConfigChange(); - } - - let sectionsChanged = false; - for (const configFile of this.configFiles) { - if ( - previous !== null && - previous.configs[configFile] === current.configs[configFile] - ) { - continue; - } - const sourcePath = path.join(this.rootDir, configFile); - if (await fs.pathExists(sourcePath)) { - await this.handleConfigFileChange(sourcePath); - } else { - await this.handleConfigFileDelete(sourcePath); - } - if (configFile === "sections.json") sectionsChanged = true; - } - - const fontPath = path.join(this.rootDir, this.fontConfigFile); - if (previous === null || previous.font !== current.font) { - if (await fs.pathExists(fontPath)) { - await this.handleFontConfigChange(); - } else { - await this.handleFontConfigDelete(); - } - } - - const analyticsPath = path.join(this.rootDir, this.analyticsConfigFile); - if (previous === null || previous.analytics !== current.analytics) { - if (await fs.pathExists(analyticsPath)) { - await this.handleAnalyticsConfigChange(); - } else { - await this.handleAnalyticsConfigDelete(); - } - } - - const publicDir = path.join(this.rootDir, "public"); - if (previous === null || previous.public !== current.public) { - if ( - !this.stopping && - !this.publicWatcher && - (await fs.pathExists(publicDir)) - ) { - await this.waitForWatcherReady(this.setupPublicWatcher()); - if (this.stopping) return; - } - await this.copyPublicFiles(); - } - - if ( - !doccupineChanged && - this.openApiSpecs.length > 0 && - (previous === null || previous.openapi !== current.openapi) - ) { - await this.handleOpenApiChange(); - } - if ( - !sectionsChanged && - (previous === null || previous.mdx !== current.mdx) - ) { - await this.processAllMDXFiles(); - } + private async ensureSafeStarterPath(relativePath: string): Promise { + return this.sourceFs.ensureSafeStarterPath(relativePath); } async init() { @@ -729,7 +220,7 @@ export class MDXToNextJSGenerator { await this.processAllMDXFiles(); - this.watchSourceSnapshot = await this.captureWatchSourceSnapshot(); + await this.watchCoordinator.establishSourceSnapshot(); console.log(chalk.green("βœ… Initial setup complete!")); console.log(chalk.cyan("πŸ’‘ To start the Next.js dev server:")); @@ -742,304 +233,56 @@ export class MDXToNextJSGenerator { } async createNextJSStructure() { - // Clear the generated app/ directory first so a fresh run never inherits - // stale routes from a previous version (e.g. pages left at their old paths - // after a route-group move would collide with the newly generated ones). - // Everything under app/ is regenerated below and by processAllMDXFiles / - // generateSectionIndexPages, so nothing here is user-authored. Config JSONs - // and other generated dirs live outside app/ and are untouched. - await fs.remove(this.outputPath("app")); - - // Drop files that earlier CLI versions generated but no longer exist in - // the template set, so upgraded projects don't keep stale copies. - await Promise.all( - obsoleteFiles.map((file) => fs.remove(this.outputPath(file))), - ); - - const structure: Record> = { - ...appStructure, - "next.config.ts": nextConfigTemplate(this.analyticsConfig), - "pnpm-workspace.yaml": pnpmWorkspaceTemplate, - "proxy.ts": proxyTemplate(this.analyticsConfig), - "analytics.json": `{}\n`, - "config.json": `{}\n`, - "links.json": `[]\n`, - "navigation.json": `[]\n`, - "sections.json": `[]\n`, - "theme.json": `{}\n`, - "app/robots.ts": robotsTemplate, - "app/layout.tsx": this.generateRootLayout(), - "app/(site)/layout.tsx": this.generateSiteLayout(), - }; - - for (const [filePath, content] of Object.entries(structure)) { - const fullPath = this.outputPath(filePath); - await fs.ensureDir(path.dirname(fullPath)); - await writeFileAtomic(fullPath, String(await content)); - } - - await this.updateSitemap(); - await this.updateLlmsFiles(); + return this.appScaffolder.createNextJsStructure(this.analyticsConfig, { + generateRootLayout: () => this.generateRootLayout(), + generateSiteLayout: () => this.generateSiteLayout(), + updateSitemap: () => this.updateSitemap(), + updateLlmsFiles: () => this.updateLlmsFiles(), + }); } async createStartingDocs() { - // Seed only a genuinely empty documentation source. Checking index.mdx - // alone could overwrite an existing components.mdx or nested page. - if ((await this.getAllMDXFiles()).length > 0) return; - - for (const [filePath, content] of Object.entries(startingDocsStructure)) { - const fullPath = await this.ensureSafeStarterPath(filePath); - await writeFileAtomic(fullPath, String(content)); - } + return this.appScaffolder.createStartingDocs({ + getAllMdxFiles: () => this.getAllMDXFiles(), + ensureSafeStarterPath: (filePath) => this.ensureSafeStarterPath(filePath), + }); } async copyCustomConfigFiles() { - console.log(chalk.blue(`πŸ” Checking for config files in: ${this.rootDir}`)); - - for (const configFile of this.configFiles) { - const sourcePath = path.join(this.rootDir, configFile); - const destPath = this.outputPath(configFile); - - console.log(chalk.gray(` Checking ${configFile}...`)); - - if (await fs.pathExists(sourcePath)) { - await this.copyRootSourceFile(sourcePath, destPath, "config source"); - console.log(chalk.green(` βœ“ Copied ${configFile} to Next.js app`)); - } else { - console.log(chalk.gray(` βœ— ${configFile} not found, skipping`)); - } - } + return this.projectConfigRepository.copyCustomConfigFiles(); } async copyFontConfig() { - console.log(chalk.blue(`πŸ” Checking for font configuration...`)); - - const sourcePath = path.join(this.rootDir, this.fontConfigFile); - const destPath = this.outputPath(this.fontConfigFile); - - if (await fs.pathExists(sourcePath)) { - await this.copyRootSourceFile(sourcePath, destPath, "font source"); - console.log( - chalk.green(` βœ“ Copied ${this.fontConfigFile} to Next.js app`), - ); - } else { - console.log(chalk.gray(` βœ— ${this.fontConfigFile} not found, skipping`)); - } + return this.projectConfigRepository.copyFontConfig(); } async loadFontConfig(): Promise { - const fontPath = path.join(this.rootDir, this.fontConfigFile); - - try { - if (await fs.pathExists(fontPath)) { - const { data } = await this.readSafeSourceFile( - this.rootDir, - fontPath, - "font source", - false, - ); - return validateFontConfig(JSON.parse(data.toString("utf8"))); - } - } catch (error) { - console.warn( - chalk.yellow(`⚠️ Error reading ${this.fontConfigFile}`), - error, - ); - } - - return null; + return this.projectConfigRepository.loadFontConfig(); } async loadAnalyticsConfig(): Promise { - const analyticsPath = path.join(this.rootDir, this.analyticsConfigFile); - - try { - if (await fs.pathExists(analyticsPath)) { - const { data } = await this.readSafeSourceFile( - this.rootDir, - analyticsPath, - "analytics source", - false, - ); - return validateAnalyticsConfig(JSON.parse(data.toString("utf8"))); - } - } catch (error) { - console.warn( - chalk.yellow(`⚠️ Error reading ${this.analyticsConfigFile}`), - error, - ); - } - - return null; + return this.projectConfigRepository.loadAnalyticsConfig(); } async copyAnalyticsConfig() { - console.log(chalk.blue(`πŸ” Checking for analytics configuration...`)); - - const sourcePath = path.join(this.rootDir, this.analyticsConfigFile); - const destPath = this.outputPath(this.analyticsConfigFile); - - if (await fs.pathExists(sourcePath)) { - const config = await this.loadAnalyticsConfig(); - await writeFileAtomic( - destPath, - config ? `${JSON.stringify(config, null, 2)}\n` : `{}\n`, - ); - console.log( - chalk.green(` βœ“ Copied ${this.analyticsConfigFile} to Next.js app`), - ); - } else { - console.log( - chalk.gray(` βœ— ${this.analyticsConfigFile} not found, skipping`), - ); - } + return this.projectConfigRepository.copyAnalyticsConfig(() => + this.loadAnalyticsConfig(), + ); } async loadSectionsConfig(): Promise { - const sectionsPath = path.join(this.rootDir, "sections.json"); - - try { - if (await fs.pathExists(sectionsPath)) { - const content = await fs.readFile(sectionsPath, "utf8"); - const parsed = JSON.parse(content) as unknown; - if (Array.isArray(parsed) && parsed.length > 0) { - const seenLabels = new Set(); - const seenSlugs = new Set(); - return parsed.map((entry, index) => { - if (!entry || typeof entry !== "object") { - throw new Error( - `sections.json entry ${index + 1} must be an object`, - ); - } - const candidate = entry as Record; - const label = - typeof candidate.label === "string" ? candidate.label.trim() : ""; - const slug = - typeof candidate.slug === "string" ? candidate.slug.trim() : ""; - if (!label) { - throw new Error( - `sections.json entry ${index + 1} needs a non-empty label`, - ); - } - if ( - slug !== "" && - (slug !== slugifySegment(slug) || - slug.includes("/") || - slug === "." || - slug === "..") - ) { - throw new Error( - `Unsafe section slug "${slug}"; use a lowercase URL segment such as "${slugifySegment(slug)}"`, - ); - } - if (seenLabels.has(label) || seenSlugs.has(slug)) { - throw new Error( - `Duplicate section label or slug at entry ${index + 1}`, - ); - } - seenLabels.add(label); - seenSlugs.add(slug); - - let directory: string | undefined; - if (candidate.directory !== undefined) { - if (typeof candidate.directory !== "string") { - throw new Error( - `sections.json directory at entry ${index + 1} must be a string`, - ); - } - directory = candidate.directory - .replace(/\\/g, "/") - .replace(/^\/+|\/+$/g, ""); - const parts = directory.split("/"); - if ( - !directory || - parts.some( - (part) => - part === "." || - part === ".." || - part !== slugifySegment(part), - ) - ) { - throw new Error( - `Unsafe section directory "${candidate.directory}"`, - ); - } - } - - return { label, slug, ...(directory ? { directory } : {}) }; - }); - } - } - } catch (error) { - console.warn(chalk.yellow("⚠️ Error reading sections.json"), error); - } - - return null; + return this.projectConfigRepository.loadSectionsConfig(); } async discoverSectionsFromFrontmatter(): Promise { const files = await this.getAllMDXFiles(); - const sectionMap = new Map(); - let hasUnsectionedPages = false; - let defaultSectionLabel = "Docs"; - - for (const file of files) { - const { content } = await this.readMdxSourceFile(file); - const { data: frontmatter } = safeMatter(content, file); - - if ( - typeof frontmatter.section === "string" && - frontmatter.section.trim() - ) { - const label = frontmatter.section.trim(); - const order = - typeof frontmatter.sectionOrder === "number" - ? frontmatter.sectionOrder - : 0; - const existing = sectionMap.get(label); - if (!existing || order < existing.order) { - sectionMap.set(label, { label, order }); - } - } else { - hasUnsectionedPages = true; - } - - if ( - (file === "index.mdx" || file === "./index.mdx") && - typeof frontmatter.sectionLabel === "string" && - frontmatter.sectionLabel.trim() - ) { - defaultSectionLabel = frontmatter.sectionLabel.trim(); - } - } - - if (sectionMap.size === 0) return null; - - const sorted = [...sectionMap.values()].sort((a, b) => a.order - b.order); - - const sections: SectionConfig[] = []; - - // Implicit root entry for pages without a section field - if (hasUnsectionedPages) { - sections.push({ label: defaultSectionLabel, slug: "" }); + const documents = []; + for (const filePath of files) { + const { content } = await this.readMdxSourceFile(filePath); + const { data: frontmatter } = safeMatter(content, filePath); + documents.push({ filePath, frontmatter }); } - - const usedSlugs = new Set(sections.map((section) => section.slug)); - for (const s of sorted) { - const slug = slugifySegment(s.label); - if (usedSlugs.has(slug)) { - throw new Error( - `Section labels resolve to the same slug "${slug}". Rename one section or define sections.json explicitly.`, - ); - } - usedSlugs.add(slug); - sections.push({ - label: s.label, - slug, - }); - } - - return sections; + return discoverSections(documents); } async resolveSections(): Promise { @@ -1057,16 +300,11 @@ export class MDXToNextJSGenerator { private withApiReferenceSection( sections: SectionConfig[] | null, ): SectionConfig[] | null { - if (this.apiRegistry.isEmpty) return sections; - const apiSection: SectionConfig = { - label: "API Reference", - slug: this.apiBaseSlug, - }; - if (!sections || sections.length === 0) { - return [{ label: "Documentation", slug: "" }, apiSection]; - } - if (sections.some((s) => s.slug === this.apiBaseSlug)) return sections; - return [...sections, apiSection]; + return addApiReferenceSection( + sections, + !this.apiRegistry.isEmpty, + this.apiBaseSlug, + ); } private async reloadSections(): Promise { @@ -1112,74 +350,15 @@ export class MDXToNextJSGenerator { filePath: string, frontmatter: Record, ): { sectionSlug: string; pageSlug: string } { - if (!this.sectionsConfig || this.sectionsConfig.length === 0) { - return { sectionSlug: "", pageSlug: generateSlug(filePath) }; - } - - const normalizedPath = filePath.replace(/\\/g, "/"); - - const firstDir = normalizedPath.includes("/") - ? normalizedPath.split("/")[0] - : ""; - - // Explicit directory matching (entries with a directory field) - for (const section of this.sectionsConfig) { - if (!section.directory) continue; - const dirPrefix = section.directory + "/"; - if (normalizedPath.startsWith(dirPrefix)) { - return { - sectionSlug: section.slug, - pageSlug: generateSlug(normalizedPath.slice(dirPrefix.length)), - }; - } - } - - // Directory matches section slug (auto-detect) - if (firstDir) { - const match = this.sectionsConfig.find((s) => s.slug === firstDir); - if (match) { - const pathForSlug = normalizedPath.slice(firstDir.length + 1); - return { - sectionSlug: match.slug, - pageSlug: generateSlug(pathForSlug), - }; - } - } - - // Frontmatter section field - if (frontmatter.section) { - const label = frontmatter.section as string; - const match = this.sectionsConfig.find((s) => s.label === label); - if (match) { - // Strip the directory if it matches the section slug - let pathForSlug = filePath; - if (firstDir && firstDir === match.slug) { - pathForSlug = normalizedPath.slice(firstDir.length + 1); - } - - return { - sectionSlug: match.slug, - pageSlug: generateSlug(pathForSlug), - }; - } - } - - // No section match - page stays at root - return { - sectionSlug: "", - pageSlug: generateSlug(filePath), - }; + return determineSectionRoute(filePath, frontmatter, this.sectionsConfig); } async handleConfigFileChange(filePath: string) { const fileName = path.basename(filePath); if (this.configFiles.includes(fileName)) { - const sourcePath = path.join(this.rootDir, fileName); - const destPath = this.outputPath(fileName); - try { - await this.copyRootSourceFile(sourcePath, destPath, "config source"); + await this.projectConfigRepository.copyConfigFile(fileName); console.log(chalk.green(`πŸ“‹ Updated ${fileName} in Next.js app`)); if (fileName === "sections.json") { @@ -1201,18 +380,8 @@ export class MDXToNextJSGenerator { const fileName = path.basename(filePath); if (this.configFiles.includes(fileName)) { - const destPath = this.outputPath(fileName); - try { - const arrayDefaults = new Set([ - "links.json", - "navigation.json", - "sections.json", - ]); - await writeFileAtomic( - destPath, - arrayDefaults.has(fileName) ? `[]\n` : `{}\n`, - ); + await this.projectConfigRepository.resetConfigFile(fileName); console.log( chalk.yellow(`πŸ—‘οΈ Reset ${fileName} to its generated default`), ); @@ -1235,11 +404,8 @@ export class MDXToNextJSGenerator { async handleFontConfigChange() { console.log(chalk.cyan(`πŸ”€ Font configuration changed`)); - const sourcePath = path.join(this.rootDir, this.fontConfigFile); - const destPath = this.outputPath(this.fontConfigFile); - try { - await this.copyRootSourceFile(sourcePath, destPath, "font source"); + await this.projectConfigRepository.copyFontConfigFile(); console.log( chalk.green(`πŸ“‹ Updated ${this.fontConfigFile} in Next.js app`), ); @@ -1254,11 +420,8 @@ export class MDXToNextJSGenerator { async handleFontConfigDelete() { console.log(chalk.red(`πŸ—‘οΈ Font configuration deleted`)); - const destPath = this.outputPath(this.fontConfigFile); - try { - if (await fs.pathExists(destPath)) { - await fs.remove(destPath); + if (await this.projectConfigRepository.removeFontConfig()) { console.log( chalk.yellow(`πŸ—‘οΈ Removed ${this.fontConfigFile} from Next.js app`), ); @@ -1276,15 +439,10 @@ export class MDXToNextJSGenerator { async handleAnalyticsConfigChange() { console.log(chalk.cyan(`πŸ“Š Analytics configuration changed`)); - const destPath = this.outputPath(this.analyticsConfigFile); - try { this.analyticsConfig = await this.loadAnalyticsConfig(); - await writeFileAtomic( - destPath, - this.analyticsConfig - ? `${JSON.stringify(this.analyticsConfig, null, 2)}\n` - : `{}\n`, + await this.projectConfigRepository.writeAnalyticsConfig( + this.analyticsConfig, ); console.log( chalk.green(`πŸ“‹ Updated ${this.analyticsConfigFile} in Next.js app`), @@ -1321,623 +479,82 @@ export class MDXToNextJSGenerator { async handleAnalyticsConfigDelete() { console.log(chalk.red(`πŸ—‘οΈ Analytics configuration deleted`)); - const destPath = this.outputPath(this.analyticsConfigFile); - try { // Write empty analytics.json so runtime imports don't break - await writeFileAtomic(destPath, `{}\n`); + await this.projectConfigRepository.resetAnalyticsConfig(); this.analyticsConfig = null; // Regenerate dynamic templates without analytics await writeFileAtomic( - this.outputPath("next.config.ts"), - nextConfigTemplate(null), - ); - await writeFileAtomic(this.outputPath("proxy.ts"), proxyTemplate(null)); - await this.updateRootLayout(); - - console.log(chalk.green(`βœ… Analytics removed from Next.js app`)); - } catch (error) { - console.error( - chalk.red(`❌ Error removing analytics configuration:`), - error, - ); - } - } - - async copyPublicFiles() { - const publicDir = path.join(this.rootDir, "public"); - const previousFiles = this.artifacts.publicFiles(); - - console.log(chalk.blue(`πŸ” Checking for public directory...`)); - - let publicStat: fs.Stats; - try { - publicStat = await fs.lstat(publicDir); - } catch (error) { - if (errorCode(error) !== "ENOENT") throw error; - console.log(chalk.gray(` βœ— public directory not found, skipping`)); - for (const stale of previousFiles) { - await fs.remove(this.publicOutputFilePath(stale)); - } - this.artifacts.replacePublicFiles([]); - await this.artifacts.save(); - return; - } - if (publicStat.isSymbolicLink() || !publicStat.isDirectory()) { - throw this.sourcePathError( - "public source", - publicDir, - "the public source root must be a real directory", - ); - } - - const realPublicDir = await this.realSourceRoot( - publicDir, - "public source", - true, - ); - const files: string[] = []; - const scanDir = async (directory: string, relativePath = "") => { - const entries = await fs.readdir(directory, { withFileTypes: true }); - for (const entry of entries) { - const sourcePath = path.join(directory, entry.name); - const entryRelativePath = path.join(relativePath, entry.name); - const stat = await fs.lstat(sourcePath); - if (stat.isSymbolicLink()) { - throw this.sourcePathError( - "public source", - sourcePath, - "the path is a symbolic link", - ); - } - const realPath = await fs.realpath(sourcePath); - if (!isPathInside(realPublicDir, realPath)) { - throw this.sourcePathError( - "public source", - sourcePath, - `the real path ${realPath} is outside ${realPublicDir}`, - ); - } - if (stat.isDirectory()) { - await scanDir(sourcePath, entryRelativePath); - } else if (stat.isFile()) { - files.push(entryRelativePath); - } else { - throw this.sourcePathError( - "public source", - sourcePath, - "expected a regular file or directory", - ); - } - } - }; - - await scanDir(publicDir); - const nextFiles = new Set(); - const nextByFoldedPath = new Map(); - for (const relativePath of files) { - const destRelativePath = publicDestinationRelativePath(relativePath); - nextFiles.add(destRelativePath); - nextByFoldedPath.set(destRelativePath.toLowerCase(), destRelativePath); - } - const removedBeforeCopy = new Set(); - for (const stale of previousFiles) { - const replacement = nextByFoldedPath.get(stale.toLowerCase()); - if ( - replacement && - replacement !== stale && - (await fs.pathExists(this.publicOutputFilePath(replacement))) - ) { - await fs.remove(this.publicOutputFilePath(stale)); - removedBeforeCopy.add(stale); - } - } - for (const relativePath of files) { - const destRelativePath = publicDestinationRelativePath(relativePath); - await this.copyRegularPublicFile( - publicDir, - path.join(publicDir, relativePath), - this.publicOutputFilePath(destRelativePath), - ); - } - for (const stale of previousFiles) { - if (!nextFiles.has(stale) && !removedBeforeCopy.has(stale)) { - await fs.remove(this.publicOutputFilePath(stale)); - } - } - this.artifacts.replacePublicFiles(nextFiles); - await this.artifacts.save(); - console.log(chalk.green(` βœ“ Copied public directory to Next.js app`)); - } - - async handlePublicFileChange(filePath: string) { - const publicDir = path.join(this.rootDir, "public"); - const relativePath = path.relative(publicDir, filePath); - const destRelativePath = publicDestinationRelativePath(relativePath); - const destPath = this.publicOutputFilePath(destRelativePath); - - try { - await this.copyRegularPublicFile(publicDir, filePath, destPath); - const publicFiles = this.artifacts.publicFiles(); - publicFiles.add(destRelativePath); - this.artifacts.replacePublicFiles(publicFiles); - await this.artifacts.save(); - console.log( - chalk.green(`πŸ“‹ Updated public/${relativePath} in Next.js app`), - ); - if (isManagedPublicArtifact(relativePath)) { - await this.updateLlmsFiles(); - } - } catch (error) { - console.error( - chalk.red(`❌ Error copying public/${relativePath}:`), - error, - ); - throw error; - } - } - - async handlePublicFileDelete(filePath: string) { - const publicDir = path.join(this.rootDir, "public"); - const relativePath = path.relative(publicDir, filePath); - const destRelativePath = publicDestinationRelativePath(relativePath); - const destPath = this.outputPath("public", destRelativePath); - - try { - // A rapid replace can queue an unlink after the replacement already - // exists. Copy the current source instead of deleting its fresh mirror. - if (await fs.pathExists(filePath)) { - await this.handlePublicFileChange(filePath); - return; - } - if (await fs.pathExists(destPath)) { - await fs.remove(destPath); - console.log( - chalk.yellow(`πŸ—‘οΈ Removed public/${relativePath} from Next.js app`), - ); - } - const publicFiles = this.artifacts.publicFiles(); - publicFiles.delete(destRelativePath); - this.artifacts.replacePublicFiles(publicFiles); - await this.artifacts.save(); - if (isManagedPublicArtifact(relativePath)) { - await this.updateLlmsFiles(); - } - } catch (error) { - console.error( - chalk.red(`❌ Error removing public/${relativePath}:`), - error, - ); - } - } - - async startWatching() { - this.stopping = false; - console.log(chalk.yellow(`πŸ‘€ Watching for changes in: ${this.watchDir}`)); - const ready: Promise[] = []; - - this.watcher = chokidar.watch(this.watchDir, { - persistent: true, - ignoreInitial: true, - followSymlinks: false, - ignored: (filePath: string, stats?: fs.Stats) => { - const isFile = stats?.isFile() ?? path.extname(filePath) !== ""; - const fileName = path.basename(filePath); - - if (this.configFiles.includes(fileName)) { - return true; - } - - if (isFile && !filePath.endsWith(".mdx")) { - return true; - } - return false; - }, - }); - ready.push(this.waitForWatcherReady(this.watcher)); - - this.watcher - .on("add", (filePath: string) => { - const relativePath = path.relative(this.watchDir, filePath); - this.enqueueMutation("Error processing added MDX file", () => - this.handleFileChange("added", relativePath), - ); - }) - .on("change", (filePath: string) => { - const relativePath = path.relative(this.watchDir, filePath); - this.enqueueMutation("Error processing changed MDX file", () => - this.handleFileChange("changed", relativePath), - ); - }) - .on("unlink", (filePath: string) => { - const relativePath = path.relative(this.watchDir, filePath); - this.enqueueMutation("Error processing deleted MDX file", () => - this.handleFileDelete(relativePath), - ); - }) - .on("ready", () => { - console.log( - chalk.green("πŸ“ Initial scan complete. Ready for changes..."), - ); - }) - .on("error", (error: unknown) => { - console.error(chalk.red("❌ Watcher error:"), error); - }); - - const configPaths = this.configFiles.map((f) => path.join(this.rootDir, f)); - - this.configWatcher = chokidar.watch(configPaths, { - persistent: true, - ignoreInitial: true, - }); - ready.push(this.waitForWatcherReady(this.configWatcher)); - - this.configWatcher - .on("add", (filePath: string) => { - console.log( - chalk.cyan(`πŸ“ Config file added: ${path.basename(filePath)}`), - ); - this.enqueueMutation("Error applying config file", () => - this.handleConfigFileChange(filePath), - ); - }) - .on("change", (filePath: string) => { - console.log( - chalk.cyan(`πŸ“ Config file changed: ${path.basename(filePath)}`), - ); - this.enqueueMutation("Error applying config file", () => - this.handleConfigFileChange(filePath), - ); - }) - .on("unlink", (filePath: string) => { - console.log( - chalk.red(`πŸ—‘οΈ Config file deleted: ${path.basename(filePath)}`), - ); - this.enqueueMutation("Error deleting config file", () => - this.handleConfigFileDelete(filePath), - ); - }) - .on("error", (error: unknown) => { - console.error(chalk.red("❌ Config watcher error:"), error); - }); - - const fontPath = path.join(this.rootDir, this.fontConfigFile); - - this.fontWatcher = chokidar.watch(fontPath, { - persistent: true, - ignoreInitial: true, - }); - ready.push(this.waitForWatcherReady(this.fontWatcher)); - - this.fontWatcher - .on("add", () => { - console.log(chalk.cyan(`πŸ”€ Font configuration added`)); - this.enqueueMutation("Error applying font configuration", () => - this.handleFontConfigChange(), - ); - }) - .on("change", () => { - this.enqueueMutation("Error applying font configuration", () => - this.handleFontConfigChange(), - ); - }) - .on("unlink", () => { - this.enqueueMutation("Error deleting font configuration", () => - this.handleFontConfigDelete(), - ); - }) - .on("error", (error: unknown) => { - console.error(chalk.red("❌ Font watcher error:"), error); - }); - - const analyticsPath = path.join(this.rootDir, this.analyticsConfigFile); - - this.analyticsWatcher = chokidar.watch(analyticsPath, { - persistent: true, - ignoreInitial: true, - }); - ready.push(this.waitForWatcherReady(this.analyticsWatcher)); - - this.analyticsWatcher - .on("add", () => { - console.log(chalk.cyan(`πŸ“Š Analytics configuration added`)); - this.enqueueMutation("Error applying analytics configuration", () => - this.handleAnalyticsConfigChange(), - ); - }) - .on("change", () => { - this.enqueueMutation("Error applying analytics configuration", () => - this.handleAnalyticsConfigChange(), - ); - }) - .on("unlink", () => { - this.enqueueMutation("Error deleting analytics configuration", () => - this.handleAnalyticsConfigDelete(), - ); - }) - .on("error", (error: unknown) => { - console.error(chalk.red("❌ Analytics watcher error:"), error); - }); - - await this.syncOpenApiSpecWatcher(); - - const doccupineConfigPath = path.join( - this.rootDir, - this.doccupineConfigFile, - ); - - this.doccupineConfigWatcher = chokidar.watch(doccupineConfigPath, { - persistent: true, - ignoreInitial: true, - }); - ready.push(this.waitForWatcherReady(this.doccupineConfigWatcher)); - - this.doccupineConfigWatcher - .on("add", () => - this.enqueueMutation("Error applying doccupine.json", () => - this.handleDoccupineConfigChange(), - ), - ) - .on("change", () => - this.enqueueMutation("Error applying doccupine.json", () => - this.handleDoccupineConfigChange(), - ), - ) - .on("error", (error: unknown) => { - console.error(chalk.red("❌ doccupine.json watcher error:"), error); - }); - - const publicDir = path.join(this.rootDir, "public"); - - if (await fs.pathExists(publicDir)) { - ready.push(this.waitForWatcherReady(this.setupPublicWatcher())); - } - - // Watch rootDir for public directory creation - this.rootDirWatcher = chokidar.watch(this.rootDir, { - persistent: true, - ignoreInitial: true, - depth: 1, - }); - ready.push(this.waitForWatcherReady(this.rootDirWatcher)); - - const queuePublicWatcherStart = () => { - if (this.stopping || this.publicWatcher || this.publicWatcherStarting) { - return; - } - this.publicWatcherStarting = true; - this.enqueueMutation("Error initializing public directory", async () => { - try { - console.log(chalk.cyan("πŸ“ Public directory created")); - await this.waitForWatcherReady(this.setupPublicWatcher()); - if (this.stopping) return; - await this.copyPublicFiles(); - } finally { - this.publicWatcherStarting = false; - } - }); - }; - - this.rootDirWatcher - .on("addDir", (dirPath: string) => { - if ( - path.basename(dirPath) === "public" && - path.dirname(dirPath) === this.rootDir - ) { - queuePublicWatcherStart(); - } - }) - .on("add", (filePath: string) => { - if (isPathInside(publicDir, filePath)) queuePublicWatcherStart(); - }) - .on("unlinkDir", (dirPath: string) => { - if ( - path.basename(dirPath) !== "public" || - path.dirname(dirPath) !== this.rootDir - ) { - return; - } - const watcher = this.publicWatcher; - this.publicWatcher = null; - void watcher?.close(); - this.enqueueMutation("Error removing public directory", () => - this.copyPublicFiles(), - ); - }) - .on("error", (error: unknown) => { - console.error(chalk.red("❌ Root dir watcher error:"), error); - }); - - await Promise.all(ready); - if (this.stopping) return; - this.enqueueMutation("Error reconciling watched sources", async () => { - const current = await this.captureWatchSourceSnapshot(); - await this.reconcileWatchedSources(this.watchSourceSnapshot, current); - this.watchSourceSnapshot = await this.captureWatchSourceSnapshot(); - }); - await this.mutationQueue; - } - - private setupPublicWatcher(): FSWatcher { - if (this.publicWatcher) { - return this.publicWatcher; - } - if (this.stopping) { - throw new Error("Cannot start a public watcher while stopping"); - } - - const publicDir = path.join(this.rootDir, "public"); - - this.publicWatcher = chokidar.watch(publicDir, { - persistent: true, - ignoreInitial: true, - followSymlinks: false, - }); - - this.publicWatcher - .on("add", (filePath: string) => { - console.log( - chalk.cyan( - `πŸ“ Public file added: ${path.relative(publicDir, filePath)}`, - ), - ); - this.enqueueMutation("Error copying public file", () => - this.handlePublicFileChange(filePath), - ); - }) - .on("change", (filePath: string) => { - console.log( - chalk.cyan( - `πŸ“ Public file changed: ${path.relative(publicDir, filePath)}`, - ), - ); - this.enqueueMutation("Error copying public file", () => - this.handlePublicFileChange(filePath), - ); - }) - .on("unlink", (filePath: string) => { - console.log( - chalk.red( - `πŸ—‘οΈ Public file deleted: ${path.relative(publicDir, filePath)}`, - ), - ); - this.enqueueMutation("Error deleting public file", () => - this.handlePublicFileDelete(filePath), - ); - }) - .on("unlinkDir", (dirPath: string) => { - if (path.resolve(dirPath) !== path.resolve(publicDir)) return; - const watcher = this.publicWatcher; - this.publicWatcher = null; - void watcher?.close(); - this.enqueueMutation("Error removing public directory", () => - this.copyPublicFiles(), - ); - }) - .on("error", (error: unknown) => { - console.error(chalk.red("❌ Public watcher error:"), error); - }); - return this.publicWatcher; + this.outputPath("next.config.ts"), + nextConfigTemplate(null), + ); + await writeFileAtomic(this.outputPath("proxy.ts"), proxyTemplate(null)); + await this.updateRootLayout(); + + console.log(chalk.green(`βœ… Analytics removed from Next.js app`)); + } catch (error) { + console.error( + chalk.red(`❌ Error removing analytics configuration:`), + error, + ); + } } - private async parseMDXFile(file: string): Promise { - const { content, stat } = await this.readMdxSourceFile(file); - const { data: frontmatter } = safeMatter(content, file); + async copyPublicFiles() { + return this.publicAssetManager.copyPublicFiles(); + } - const { sectionSlug, pageSlug } = this.determineSectionForFile( - file, - frontmatter, + async handlePublicFileChange(filePath: string) { + return this.publicAssetManager.handlePublicFileChange(filePath, () => + this.updateLlmsFiles(), ); - const fullSlug = getFullSlug(pageSlug, sectionSlug); + } - let lastModified: string | undefined; - const authoredLastModified = frontmatter.updated ?? frontmatter.date; - if (authoredLastModified) { - const parsed = new Date(authoredLastModified); - if (!Number.isNaN(parsed.getTime())) { - lastModified = parsed.toISOString(); - } - } - if (!lastModified) { - lastModified = stat.mtime.toISOString(); - } + async handlePublicFileDelete(filePath: string) { + return this.publicAssetManager.handlePublicFileDelete( + filePath, + () => this.handlePublicFileChange(filePath), + () => this.updateLlmsFiles(), + ); + } - // A hand-written page that embeds an endpoint via `openapi:` frontmatter - // gets the same method badge in the sidebar as a generated endpoint page. - let httpMethod: string | undefined; - if (frontmatter.openapi) { - const op = this.apiRegistry.lookup(String(frontmatter.openapi)); - if (op) httpMethod = op.method.toUpperCase(); - } + async startWatching() { + return this.watchCoordinator.startWatching(); + } - return { - slug: fullSlug, - title: frontmatter.title || "Untitled", - description: frontmatter.description || "", - date: frontmatter.date || null, - category: frontmatter.category || "", - path: file, - categoryOrder: frontmatter.categoryOrder || 0, - order: frontmatter.order || 0, - section: sectionSlug, - // Sidebar icons (Lucide names). Kept separate from `icon`, which is - // reserved for the favicon/OG metadata. Only emitted when set so the - // generated page literal stays lean. - ...(frontmatter.navIcon ? { navIcon: String(frontmatter.navIcon) } : {}), - ...(frontmatter.categoryIcon - ? { categoryIcon: String(frontmatter.categoryIcon) } - : {}), - ...(httpMethod ? { httpMethod } : {}), - lastModified, - }; + private async parseMDXFile(file: string): Promise { + return parseMdxPageMeta( + file, + (filePath) => this.readMdxSourceFile(filePath), + (filePath, frontmatter) => + this.determineSectionForFile(filePath, frontmatter), + (reference) => this.apiRegistry.lookup(reference)?.method, + ); } private async buildRealPagesMeta(): Promise { const files = await this.getAllMDXFiles(); - const real = await Promise.all( - files.map((file) => this.parseMDXFile(file)), - ); - const bySlug = new Map(); - for (const page of real) { - const existing = bySlug.get(page.slug); - if (existing) { - throw new Error( - `Route collision at "/${page.slug}": both "${existing}" and "${page.path}" generate the same page.`, - ); - } - bySlug.set(page.slug, page.path); - } - return real; + return buildRealPageCatalog(files, (file) => this.parseMDXFile(file)); } private async buildAllPagesMeta(): Promise { const real = await this.buildRealPagesMeta(); - if (this.apiRegistry.isEmpty) return real; - - // Inject synthetic OpenAPI endpoint pages here - the single funnel every - // aggregate (nav, sitemap, llms) flows through - so they cannot be dropped - // by the .mdx-only disk scan. Hand-written pages win on any slug collision. - const realSlugs = new Set(real.map((page) => page.slug)); - const synthetic = this.apiRegistry.syntheticPages().filter((page) => { - if (realSlugs.has(page.slug)) { - console.log( - chalk.yellow( - `⚠️ API page ${page.slug} is shadowed by a hand-written page; skipping`, - ), - ); - return false; - } - return true; - }); - return [...real, ...synthetic]; + return mergePageCatalog( + real, + this.apiRegistry.isEmpty ? [] : this.apiRegistry.syntheticPages(), + ); } private async removeOwnedRoute(slug: string): Promise { - if (!slug) return; - const siteDir = this.outputPath("app", "(site)"); - const routeDir = resolveOutputPath(siteDir, slug); - await Promise.all([ - fs.remove(resolveOutputPath(siteDir, slug, "page.tsx")), - fs.remove(resolveOutputPath(siteDir, slug, "rss.xml")), - ]); - await this.removeEmptyDirsUpTo(routeDir, siteDir); + return this.generatedRouteManager.removeOwnedRoute(slug); } private async removeStaleMdxRoutes(realPages: PageMeta[]): Promise { - const nextBySource = new Map( - realPages - .filter((page) => page.slug !== "") - .map((page) => [page.path.replace(/\\/g, "/"), page.slug]), + return this.generatedRouteManager.removeStaleMdxRoutes(realPages, (slug) => + this.removeOwnedRoute(slug), ); - const nextSlugs = new Set(nextBySource.values()); - - for (const previous of this.artifacts.routesFor("mdx")) { - if (nextBySource.get(previous.source) === previous.slug) continue; - if (nextSlugs.has(previous.slug)) continue; - await this.removeOwnedRoute(previous.slug); - } } /** @@ -2033,7 +650,8 @@ export class MDXToNextJSGenerator { // Validate the complete route set before writing this page so a collision // cannot transiently overwrite another route during watch mode. const normalizedSource = filePath.replace(/\\/g, "/"); - const previousSlug = this.artifacts.routeFor("mdx", normalizedSource); + const previousSlug = + this.generatedRouteManager.routeForMdxSource(normalizedSource); const realPages = await this.buildRealPagesMeta(); const currentPage = realPages.find( (page) => page.path.replace(/\\/g, "/") === normalizedSource, @@ -2042,13 +660,7 @@ export class MDXToNextJSGenerator { await this.removeStaleMdxRoutes(realPages); await this.writePageForFile(filePath); - this.artifacts.replaceRoutes( - "mdx", - realPages - .filter((page) => page.slug !== "") - .map((page) => ({ source: page.path, slug: page.slug })), - ); - await this.artifacts.save(); + await this.generatedRouteManager.replaceMdxRoutes(realPages); if (!this.apiRegistry.isEmpty) await this.writeApiPages(); const pages = await this.buildAllPagesMeta(); @@ -2070,10 +682,10 @@ export class MDXToNextJSGenerator { console.log(chalk.blue("🏠 Updating homepage - index.mdx deleted")); } else { const normalizedSource = filePath.replace(/\\/g, "/"); - const ownedSlug = this.artifacts.routeFor("mdx", normalizedSource); + const ownedSlug = + this.generatedRouteManager.routeForMdxSource(normalizedSource); if (ownedSlug) await this.removeOwnedRoute(ownedSlug); - this.artifacts.removeRoute("mdx", normalizedSource); - await this.artifacts.save(); + await this.generatedRouteManager.removeMdxRoute(normalizedSource); } if (!this.apiRegistry.isEmpty) await this.writeApiPages(); @@ -2110,75 +722,14 @@ export class MDXToNextJSGenerator { } } - this.artifacts.replaceRoutes( - "mdx", - realPages - .filter((page) => page.slug !== "") - .map((page) => ({ source: page.path, slug: page.slug })), - ); - await this.artifacts.save(); + await this.generatedRouteManager.replaceMdxRoutes(realPages); const pages = await this.buildAllPagesMeta(); await this.refreshSiteAggregates(pages); } async getAllMDXFiles(): Promise { - const files: string[] = []; - const realWatchDir = await this.realSourceRoot( - this.watchDir, - "documentation source", - false, - ); - - const scanDir = async (dir: string, relativePath = "") => { - const entries = await fs.readdir(dir, { withFileTypes: true }); - - for (const entry of entries) { - const fullPath = path.join(dir, entry.name); - const relPath = path.join(relativePath, entry.name); - const stat = await fs.lstat(fullPath); - - if (stat.isSymbolicLink()) { - let linksToDirectory = false; - try { - linksToDirectory = (await fs.stat(fullPath)).isDirectory(); - } catch (error) { - if (errorCode(error) !== "ENOENT") throw error; - } - if (entry.name.endsWith(".mdx") || linksToDirectory) { - throw this.sourcePathError( - "documentation source", - fullPath, - "the path is a symbolic link", - ); - } - continue; - } - const realPath = await fs.realpath(fullPath); - if (!isPathInside(realWatchDir, realPath)) { - throw this.sourcePathError( - "documentation source", - fullPath, - `the real path ${realPath} is outside ${realWatchDir}`, - ); - } - - if (stat.isDirectory()) { - await scanDir(fullPath, relPath); - } else if (stat.isFile() && entry.name.endsWith(".mdx")) { - files.push(relPath); - } else if (!stat.isFile() && entry.name.endsWith(".mdx")) { - throw this.sourcePathError( - "documentation source", - fullPath, - "expected a regular .mdx file", - ); - } - } - }; - - await scanDir(this.watchDir); - return files; + return this.sourceFs.getAllMdxFiles(); } async generateRootLayout(): Promise { @@ -2256,32 +807,10 @@ export default function SectionIndex() { private async cleanupStaleSectionIndexPages( nextSlugs: Set, ): Promise { - for (const stale of this.generatedSectionIndexSlugs) { - if (nextSlugs.has(stale)) continue; - const pagePath = resolveOutputPath( - this.outputDir, - "app", - "(site)", - stale, - "page.tsx", - ); - try { - if (!(await fs.pathExists(pagePath))) continue; - const content = await fs.readFile(pagePath, "utf8"); - if (!content.includes("function SectionIndex()")) continue; - await fs.remove(pagePath); - await this.removeEmptyDirsUpTo( - path.dirname(pagePath), - this.outputPath("app", "(site)"), - ); - console.log( - chalk.blue(`🧹 Removed stale section index redirect: /${stale}`), - ); - } catch { - // ignore - } - } - this.generatedSectionIndexSlugs = nextSlugs; + return this.generatedRouteManager.cleanupStaleSectionIndexPages( + nextSlugs, + (dir, stopDir) => this.removeEmptyDirsUpTo(dir, stopDir), + ); } /** Best-effort removal of now-empty directories up to (not incl.) stopDir. */ @@ -2289,138 +818,14 @@ export default function SectionIndex() { dir: string, stopDir: string, ): Promise { - const stop = path.resolve(stopDir); - let current = path.resolve(dir); - while (current !== stop && current.startsWith(stop + path.sep)) { - try { - const entries = await fs.readdir(current); - if (entries.length > 0) return; - await fs.remove(current); - } catch { - return; - } - current = path.dirname(current); - } + return this.generatedRouteManager.removeEmptyDirsUpTo(dir, stopDir); } async generatePageFromMDX( mdxFile: MDXFile, options?: { apiOperation?: OperationDescriptor }, ) { - const fm = mdxFile.frontmatter; - const apiOperation = options?.apiOperation; - - // Pages containing blocks publish a subscribable changelog: an - // RSS feed at {page-url}/rss.xml. Synthetic OpenAPI pages never contain - // Update blocks, so skip the parse for them. - const isSynthetic = mdxFile.path.startsWith("@openapi/"); - const updates = isSynthetic ? [] : parseUpdateBlocks(mdxFile.content); - const hasFeed = updates.length > 0; - const feedPath = `/${mdxFile.slug}/rss.xml`; - - const metadataBlock = generateMetadataBlock({ - title: fm.title, - titleFallback: "Generated with Doccupine", - name: fm.name, - titleOrder: "page-first", - description: fm.description, - icon: fm.icon, - image: fm.image, - canonicalPath: mdxFile.slug, - rssPath: hasFeed ? feedPath : undefined, - }); - - const jsonLd = generateJsonLdScript({ - kind: "article", - canonicalPath: mdxFile.slug, - title: fm.title, - description: fm.description, - date: typeof fm.date === "string" ? fm.date : undefined, - updated: - typeof fm.updated === "string" - ? fm.updated - : typeof fm.date === "string" - ? fm.date - : undefined, - image: fm.image, - }); - - // For an OpenAPI-backed page, embed the operation descriptor as a JS string - // literal parsed at load. Serializing to JSON then re-`JSON.parse`ing is - // total escaping for arbitrary JSON - unlike `escapeTemplateContent`, which - // only guards backticks/`${`/backslashes for the MDX prose literal. - const apiImport = apiOperation - ? `\nimport { ApiPlayground } from "@/components/layout/ApiPlayground";` - : ""; - // The descriptor JSON always exceeds the 80-col print width, so emit the - // call pre-wrapped in the exact shape Prettier produces (argument on its own - // line with a trailing comma, single-quoted so the JSON's own double quotes - // need no escaping). Keeps generated endpoint pages Prettier-stable without - // running a formatter at build time. - const apiConst = apiOperation - ? (() => { - const arg = toJsStringLiteral(JSON.stringify(apiOperation)); - const inline = `const operation = JSON.parse(${arg});`; - const decl = - inline.length <= 80 - ? inline - : `const operation = JSON.parse(\n ${arg},\n);`; - return `\n${decl}\n`; - })() - : ""; - // The playground renders as a child of so it sits inside the docs - // content column (a sibling would escape the layout and overlap the nav). - // Synthetic endpoint pages pass no `sourcePath`: it only namespaces Mermaid - // diagrams (which endpoint docs never contain), and its long `@openapi/...` - // value would push the opening tag past 80 cols and make Prettier rewrap it. - const sourcePathLiteral = JSON.stringify(mdxFile.path); - // `rss: true` frontmatter opts the page into an RSS button in the action - // bar (only when a feed actually exists). The playground branch keeps its - // fixed JSX shape - a feed on an inline-playground page stays reachable - // via autodiscovery. The buttoned form usually exceeds the 80-col print - // width, so pre-wrap it in the shape Prettier produces (attributes on - // their own lines) relative to its 6-space insertion indent. - const showRssButton = hasFeed && fm.rss === true && !apiOperation; - const docsAttrs = [ - `content={content}`, - `sourcePath={${sourcePathLiteral}}`, - ...(showRssButton ? [`rssHref={${JSON.stringify(feedPath)}}`] : []), - ]; - const inlineDocs = ``; - const docsElement = apiOperation - ? ` - - ` - : inlineDocs.length + 6 <= 80 - ? inlineDocs - : ` ` ${attr}`).join("\n")}\n />`; - - const pageContent = `import { Metadata } from "next"; -import { Docs } from "@/components/Docs"; -import { config } from "@/utils/config";${apiImport} - -const content = \`${escapeTemplateContent(mdxFile.content)}\`; -${apiConst} -${metadataBlock} - -// Doc pages have no per-request data: theme resolves client-side via the -// "dark" class on (set before paint by the theme-init blocking -// script). Static rendering lets every response come from the edge cache. -export const dynamic = "force-static"; -export const revalidate = false; - -export default function Page() { - ${jsonLd.declarations} - - return ( - <> - ${jsonLd.element} - ${docsElement} - - ); -} -`; - + const rendered = renderMdxPage(mdxFile, options); const pagePath = resolveOutputPath( this.outputDir, "app", @@ -2429,30 +834,20 @@ export default function Page() { "page.tsx", ); await fs.ensureDir(path.dirname(pagePath)); - await writeFileAtomic(pagePath, pageContent); + await writeFileAtomic(pagePath, rendered.pageContent); // The feed route lives inside the page's directory, so a deleted page // takes its feed along (handleFileDelete removes the whole dir) and the // else-branch prunes the route when a regenerated page no longer has // Update blocks. Cross-run staleness is covered by the app/ wipe in // createNextJSStructure. - if (!isSynthetic) { + if (rendered.rssRoute.action !== "preserve") { const rssDir = resolveOutputPath(path.dirname(pagePath), "rss.xml"); - if (hasFeed) { + if (rendered.rssRoute.action === "write") { await fs.ensureDir(rssDir); await writeFileAtomic( resolveOutputPath(path.dirname(pagePath), "rss.xml", "route.ts"), - rssRouteTemplate({ - pagePath: mdxFile.slug, - title: typeof fm.title === "string" ? fm.title : null, - description: - typeof fm.description === "string" ? fm.description : null, - items: updates.map((update) => ({ - title: update.label, - anchor: update.anchor, - description: update.description, - })), - }), + rendered.rssRoute.content, ); } else { await fs.remove(rssDir); @@ -2485,86 +880,21 @@ export default function Page() { * pages (e.g. after the `openapi` config is removed). */ private async writeApiPages(): Promise { - const realSlugs = new Set( - (await this.buildRealPagesMeta()).map((page) => page.slug), + const realPages = await this.buildRealPagesMeta(); + return this.apiReferenceGenerator.writePages( + this.apiRegistry, + this.apiBaseSlug, + realPages, + (mdxFile, options) => this.generatePageFromMDX(mdxFile, options), + () => this.writeApiAllowlist(), + (nextRoutes, realSlugs) => + this.cleanupStaleApiPages(nextRoutes, realSlugs), ); - const nextRoutes = new Map(); - - const indexPage = this.apiRegistry - .syntheticPages() - .find((page) => page.slug === this.apiBaseSlug); - if (indexPage && !realSlugs.has(indexPage.slug)) { - try { - await this.generatePageFromMDX({ - path: indexPage.path, - content: this.apiRegistry.bodyForSlug(indexPage.slug) ?? "", - frontmatter: { - title: indexPage.title, - description: indexPage.description, - }, - slug: indexPage.slug, - }); - nextRoutes.set(`@openapi/${indexPage.slug}`, indexPage.slug); - } catch (error) { - console.error( - chalk.red(`❌ Error generating API index ${indexPage.slug}:`), - error, - ); - } - } - - for (const op of this.apiRegistry.all) { - const methodUpper = op.method.toUpperCase(); - const mdxFile: MDXFile = { - path: `@openapi/${op.specName}/${op.method}${op.path}`, - content: buildEndpointDoc(op), - frontmatter: { - title: op.summary ?? `${methodUpper} ${op.path}`, - description: op.summary ?? "", - }, - slug: op.slug, - }; - if (realSlugs.has(op.slug)) { - console.log( - chalk.yellow( - `⚠️ API page ${op.slug} is shadowed by a hand-written page; skipping`, - ), - ); - continue; - } - try { - await this.generatePageFromMDX(mdxFile, { apiOperation: op }); - nextRoutes.set(`@openapi/${op.slug}`, op.slug); - } catch (error) { - console.error( - chalk.red(`❌ Error generating API page ${op.slug}:`), - error, - ); - } - } - - await this.writeApiAllowlist(); - await this.cleanupStaleApiPages(nextRoutes, realSlugs); - - if (this.apiRegistry.all.length > 0) { - console.log( - chalk.green(`🧩 Generated ${nextRoutes.size} API reference page(s)`), - ); - } } /** Writes the request-execution allowlist (overwrites the shipped stub). */ private async writeApiAllowlist(): Promise { - const target = this.outputPath( - "services", - "openapi", - "playground-allowlist.json", - ); - await fs.ensureDir(path.dirname(target)); - await writeFileAtomic( - target, - `${JSON.stringify(this.apiRegistry.allowlist(), null, 2)}\n`, - ); + return this.apiReferenceGenerator.writeAllowlist(this.apiRegistry); } /** Removes endpoint page directories that are no longer in the spec. */ @@ -2572,25 +902,11 @@ export default function Page() { nextRoutes: Map, realSlugs: Set, ): Promise { - const nextSlugs = new Set(nextRoutes.values()); - for (const previous of this.artifacts.routesFor("openapi")) { - if (nextRoutes.has(previous.source) || nextSlugs.has(previous.slug)) - continue; - // A hand-written page may have taken ownership of this route since the - // previous OpenAPI pass. Never remove an output now claimed by MDX. - if (realSlugs.has(previous.slug)) continue; - try { - await this.removeOwnedRoute(previous.slug); - } catch { - // ignore - } - } - - this.artifacts.replaceRoutes( - "openapi", - [...nextRoutes].map(([source, slug]) => ({ source, slug })), + return this.apiReferenceGenerator.cleanupStalePages( + nextRoutes, + realSlugs, + (slug) => this.removeOwnedRoute(slug), ); - await this.artifacts.save(); } /** @@ -2599,45 +915,7 @@ export default function Page() { * so specs added mid-session are watched without a restart. */ private async syncOpenApiSpecWatcher(): Promise { - if (this.openApiWatcher) { - await this.openApiWatcher.close(); - this.openApiWatcher = null; - } - if (this.stopping || this.openApiSpecs.length === 0) return; - - const specPaths = this.openApiSpecs.map((spec) => - path.resolve(this.rootDir, spec.file), - ); - - this.openApiWatcher = chokidar.watch(specPaths, { - persistent: true, - ignoreInitial: true, - }); - - this.openApiWatcher - .on("add", () => - this.enqueueMutation("Error rebuilding API reference", () => - this.handleOpenApiChange(), - ), - ) - .on("change", () => - this.enqueueMutation("Error rebuilding API reference", () => - this.handleOpenApiChange(), - ), - ) - .on("unlink", () => - this.enqueueMutation("Error rebuilding API reference", () => - this.handleOpenApiChange(), - ), - ) - .on("error", (error: unknown) => { - console.error(chalk.red("❌ OpenAPI watcher error:"), error); - }); - await this.waitForWatcherReady(this.openApiWatcher); - if (this.stopping && this.openApiWatcher) { - await this.openApiWatcher.close(); - this.openApiWatcher = null; - } + return this.watchCoordinator.syncOpenApiSpecWatcher(this.openApiSpecs); } /** @@ -2766,18 +1044,7 @@ export default function Page() { async updatePagesIndex() { const files = await this.getAllMDXFiles(); - let indexMDX: { - content: string; - title: string; - description: string; - icon?: string; - image?: string; - name?: string; - date?: string; - updated?: string; - openapi?: string; - rss?: boolean; - } | null = null; + let indexMDX: HomepageSource | null = null; for (const file of files) { if (file === "index.mdx" || file === "./index.mdx") { @@ -2810,37 +1077,6 @@ export default function Page() { } } - // The homepage publishes the same subscribable changelog as any other - // page (see generatePageFromMDX): blocks feed the site-root - // /rss.xml, and `rss: true` frontmatter opts into the RSS button. - const updates = indexMDX ? parseUpdateBlocks(indexMDX.content) : []; - const hasFeed = updates.length > 0; - const feedPath = "/rss.xml"; - - const metadataBlock = indexMDX - ? generateMetadataBlock({ - title: indexMDX.title, - titleFallback: "Welcome", - name: indexMDX.name, - titleOrder: "name-first", - description: indexMDX.description || undefined, - icon: indexMDX.icon, - image: indexMDX.image, - canonicalPath: "", - rssPath: hasFeed ? feedPath : undefined, - }) - : generateRuntimeOnlyMetadataBlock(); - - const homeJsonLd = generateJsonLdScript({ - kind: "homepage", - canonicalPath: "", - title: indexMDX?.title, - description: indexMDX?.description || undefined, - date: indexMDX?.date, - updated: indexMDX?.updated ?? indexMDX?.date, - image: indexMDX?.image, - }); - // The homepage supports the same `openapi: ` frontmatter as // any other page: look the operation up and embed its playground inline. let apiOperation: OperationDescriptor | undefined; @@ -2854,73 +1090,22 @@ export default function Page() { ); } } - const apiImport = apiOperation - ? `\nimport { ApiPlayground } from "@/components/layout/ApiPlayground";` - : ""; - const apiConst = apiOperation - ? `\nconst operation = JSON.parse(${JSON.stringify( - JSON.stringify(apiOperation), - )});\n` - : ""; - // Same gating as generatePageFromMDX: the playground branch keeps its - // fixed JSX shape, so a feed on a playground homepage stays reachable via - // autodiscovery. The buttoned inline form stays within the 80-col print - // width at its 6-space insertion indent, so it is Prettier-stable as is. - const showRssButton = hasFeed && indexMDX?.rss === true && !apiOperation; - const docsElement = apiOperation - ? ` - - ` - : showRssButton - ? `` - : ``; - - const indexContent = `import { Metadata } from "next"; -import { Docs } from "@/components/Docs"; -import { config } from "@/utils/config";${apiImport} - -${indexMDX ? `const content = \`${escapeTemplateContent(indexMDX.content)}\`;` : `const content = null;`} -${apiConst} -${metadataBlock} - -export const dynamic = "force-static"; -export const revalidate = false; - -export default function Home() { - ${homeJsonLd.declarations} - - return ( - <> - ${homeJsonLd.element} - ${docsElement} - - ); -} -`; + const rendered = renderHomepage(indexMDX, apiOperation); const homePath = this.outputPath("app", "(site)", "page.tsx"); await fs.ensureDir(path.dirname(homePath)); - await writeFileAtomic(homePath, indexContent); + await writeFileAtomic(homePath, rendered.pageContent); // Same lifecycle as the per-page feeds in generatePageFromMDX: write the // root feed route while the homepage has Update blocks, prune it when // they go away or index.mdx is deleted (this runs on every aggregate // refresh, including the delete path). const rssDir = this.outputPath("app", "(site)", "rss.xml"); - if (hasFeed && indexMDX) { + if (rendered.rssRoute.action === "write") { await fs.ensureDir(rssDir); await writeFileAtomic( this.outputPath("app", "(site)", "rss.xml", "route.ts"), - rssRouteTemplate({ - pagePath: "", - title: indexMDX.title, - description: indexMDX.description || null, - items: updates.map((update) => ({ - title: update.label, - anchor: update.anchor, - description: update.description, - })), - }), + rendered.rssRoute.content, ); } else { await fs.remove(rssDir); @@ -2933,79 +1118,12 @@ export default function Home() { mdxContent: string, sourcePath?: string, ) { - // This overwrites the page generatePageFromMDX just wrote for the same - // slug (section landings compose their metadata name-first), so the RSS - // state must be re-derived here or the overwrite silently drops the - // button and autodiscovery - the feed route survives either way since it - // lives in a sibling rss.xml/ dir. - const updates = parseUpdateBlocks(mdxContent); - const hasFeed = updates.length > 0; - const feedPath = `/${sectionSlug}/rss.xml`; - const showRssButton = hasFeed && frontmatter.rss === true; - - const metadataBlock = generateMetadataBlock({ - title: frontmatter.title, - titleFallback: "Section", - name: frontmatter.name, - titleOrder: "name-first", - description: frontmatter.description || undefined, - icon: frontmatter.icon, - image: frontmatter.image, - canonicalPath: sectionSlug, - rssPath: hasFeed ? feedPath : undefined, - }); - - const sectionJsonLd = generateJsonLdScript({ - kind: "article", - canonicalPath: sectionSlug, - title: frontmatter.title, - description: frontmatter.description, - date: typeof frontmatter.date === "string" ? frontmatter.date : undefined, - updated: - typeof frontmatter.updated === "string" - ? frontmatter.updated - : typeof frontmatter.date === "string" - ? frontmatter.date - : undefined, - image: frontmatter.image, - }); - - // Same Prettier pre-wrap contract as generatePageFromMDX: the buttoned - // form usually pushes the line past the 80-col print width, so emit it - // with attributes on their own lines relative to the 6-space indent. - const docsAttrs = [ - `content={content}`, - `sourcePath={${JSON.stringify(sourcePath ?? `${sectionSlug}/index.mdx`)}}`, - ...(showRssButton ? [`rssHref={${JSON.stringify(feedPath)}}`] : []), - ]; - const inlineDocs = ``; - const docsElement = - inlineDocs.length + 6 <= 80 - ? inlineDocs - : ` ` ${attr}`).join("\n")}\n />`; - - const indexContent = `import { Metadata } from "next"; -import { Docs } from "@/components/Docs"; -import { config } from "@/utils/config"; - -const content = \`${escapeTemplateContent(mdxContent)}\`; - -${metadataBlock} - -export const dynamic = "force-static"; -export const revalidate = false; - -export default function Page() { - ${sectionJsonLd.declarations} - - return ( - <> - ${sectionJsonLd.element} - ${docsElement} - - ); -} -`; + const rendered = renderSectionPage( + sectionSlug, + frontmatter, + mdxContent, + sourcePath, + ); const pagePath = resolveOutputPath( this.outputDir, @@ -3015,7 +1133,7 @@ export default function Page() { "page.tsx", ); await fs.ensureDir(path.dirname(pagePath)); - await writeFileAtomic(pagePath, indexContent); + await writeFileAtomic(pagePath, rendered.pageContent); } async updateRootLayout(pages?: PageMeta[]) { @@ -3029,432 +1147,35 @@ export default function Page() { } async loadSiteUrl(): Promise { - const fromEnv = process.env.NEXT_PUBLIC_SITE_URL?.trim(); - if (fromEnv) return fromEnv.replace(/\/$/, ""); - - const configPath = path.join(this.rootDir, "config.json"); - - try { - if (await fs.pathExists(configPath)) { - const content = await fs.readFile(configPath, "utf8"); - const parsed = JSON.parse(content) as { url?: unknown }; - if (typeof parsed.url === "string" && parsed.url.trim() !== "") { - return parsed.url.trim().replace(/\/$/, ""); - } - } - } catch (error) { - console.warn(chalk.yellow("⚠️ Error reading config.json"), error); - } - - return null; - } - - private buildSitemapEntries(pages: PageMeta[]): SitemapEntry[] { - const sectionSlugs = new Set( - (this.sectionsConfig || []) - .map((s) => s.slug) - .filter((s): s is string => typeof s === "string" && s !== ""), - ); - - const entries: SitemapEntry[] = pages.map((page) => { - let priority = 0.5; - if (page.slug === "") { - priority = 1.0; - } else if (sectionSlugs.has(page.slug)) { - priority = 0.8; - } - return { - slug: page.slug, - lastModified: page.lastModified, - changeFrequency: "weekly", - priority, - }; - }); - - if (!entries.some((entry) => entry.slug === "")) { - entries.unshift({ - slug: "", - changeFrequency: "weekly", - priority: 1.0, - }); - } - - return entries; + return loadSiteUrlArtifact(this.rootDir); } async updateSitemap(pages?: PageMeta[]) { - const sitemapPath = this.outputPath("app", "sitemap.ts"); - const siteUrl = await this.loadSiteUrl(); - - const resolvedPages = pages ?? (await this.buildAllPagesMeta()); - const entries = this.buildSitemapEntries(resolvedPages); - await writeFileAtomic(sitemapPath, sitemapTemplate(entries)); - console.log( - chalk.green( - `πŸ—ΊοΈ Generated sitemap.ts with ${entries.length} page(s)${ - siteUrl ? ` using ${siteUrl}` : " (waiting for a deployment URL)" - }`, - ), + return writeSitemap( + this.outputDir, + this.sectionsConfig, + async () => pages ?? (await this.buildAllPagesMeta()), + () => this.loadSiteUrl(), ); } async updateRobots() { - const siteUrl = await this.loadSiteUrl(); - await writeFileAtomic(this.outputPath("app", "robots.ts"), robotsTemplate); - console.log( - chalk.green( - siteUrl - ? `πŸ€– Regenerated robots.ts with sitemap link` - : `πŸ€– Regenerated robots.ts (no sitemap link)`, - ), - ); - } - - private async loadSiteMetadata(): Promise<{ - url: string | null; - name: string; - description: string; - }> { - const configPath = path.join(this.rootDir, "config.json"); - let url: string | null = null; - let name = "Documentation"; - let description = ""; - - try { - const fromEnv = process.env.NEXT_PUBLIC_SITE_URL?.trim(); - if (fromEnv) url = fromEnv.replace(/\/$/, ""); - if (await fs.pathExists(configPath)) { - const content = await fs.readFile(configPath, "utf8"); - const parsed = JSON.parse(content) as { - url?: unknown; - name?: unknown; - title?: unknown; - description?: unknown; - }; - if ( - !url && - typeof parsed.url === "string" && - parsed.url.trim() !== "" - ) { - url = parsed.url.trim().replace(/\/$/, ""); - } - if (typeof parsed.name === "string" && parsed.name.trim() !== "") { - name = parsed.name.trim(); - } else if ( - typeof parsed.title === "string" && - parsed.title.trim() !== "" - ) { - name = parsed.title.trim(); - } - if ( - typeof parsed.description === "string" && - parsed.description.trim() !== "" - ) { - description = parsed.description.trim(); - } - } - } catch (error) { - console.warn( - chalk.yellow("⚠️ Error reading config.json for llms metadata"), - error, - ); - } - - return { url, name, description }; - } - - private async readPageWithBody(page: PageMeta): Promise { - // Synthetic OpenAPI pages have no backing .mdx file; their markdown body - // comes from the registry instead of disk. - if (!page.path.endsWith(".mdx")) { - return { ...page, body: this.apiRegistry.bodyForSlug(page.slug) ?? "" }; - } - const { content: raw } = await this.readMdxSourceFile(page.path); - const { content: body } = safeMatter(raw, page.path); - return { ...page, body }; - } - - private async findSourcePublicAsset( - relativePath: string, - ): Promise { - const sourcePublicDir = path.join(this.rootDir, "public"); - const normalized = relativePath.replace(/\\/g, "/"); - resolveWithin(sourcePublicDir, normalized); - - let rootStat: fs.Stats; - try { - rootStat = await fs.lstat(sourcePublicDir); - } catch (error) { - if (errorCode(error) === "ENOENT") return null; - throw error; - } - if (rootStat.isSymbolicLink() || !rootStat.isDirectory()) { - throw this.sourcePathError( - "public source", - sourcePublicDir, - "the public source root must be a real directory", - ); - } - const realPublicDir = await this.realSourceRoot( - sourcePublicDir, - "public source", - true, - ); - - let currentPath = sourcePublicDir; - const parts = normalized.split("/").filter(Boolean); - for (const [index, part] of parts.entries()) { - let entries: string[]; - try { - entries = await fs.readdir(currentPath); - } catch { - return null; - } - const actualName = - entries.find((entry) => entry === part) ?? - entries.find((entry) => entry.toLowerCase() === part.toLowerCase()); - if (!actualName) return null; - currentPath = path.join(currentPath, actualName); - const stat = await fs.lstat(currentPath); - if (stat.isSymbolicLink()) { - throw this.sourcePathError( - "public source", - currentPath, - "the path is a symbolic link", - ); - } - if (index < parts.length - 1 && !stat.isDirectory()) return null; - } - - const stat = await fs.lstat(currentPath); - if (!stat.isFile()) return null; - const realPath = await fs.realpath(currentPath); - if (!isPathInside(realPublicDir, realPath)) { - throw this.sourcePathError( - "public source", - currentPath, - `the real path ${realPath} is outside ${realPublicDir}`, - ); - } - return currentPath; - } - - private async writePublicAggregate( - relativePath: string, - content: string, - ): Promise { - const sourcePath = await this.findSourcePublicAsset(relativePath); - const targetPath = this.publicOutputFilePath(relativePath); - if (sourcePath) { - console.warn( - chalk.yellow( - `⚠️ Skipping generated public/${relativePath}; a project public asset owns that path`, - ), - ); - await this.copyRegularPublicFile( - path.join(this.rootDir, "public"), - sourcePath, - targetPath, - ); - return; - } - - await fs.ensureDir(path.dirname(targetPath)); - await writeFileAtomic(targetPath, content); + return writeRobots(this.outputDir, () => this.loadSiteUrl()); } async updateLlmsFiles(pages?: PageMeta[]) { - const publicDir = this.outputPath("public"); - await fs.ensureDir(publicDir); - - const { url: baseUrl, name, description } = await this.loadSiteMetadata(); - const resolvedPages = pages ?? (await this.buildAllPagesMeta()); - const pagesWithBodies = await Promise.all( - resolvedPages.map((page) => this.readPageWithBody(page)), - ); - const docsContent = pagesWithBodies.map((page) => { - const route = page.slug.replace(/^\/+|\/+$/g, ""); - const pagePath = route - ? `app/(site)/${route}/page.tsx` - : "app/(site)/page.tsx"; - return { - uri: `docs://${route || "/"}`, - name: page.title, - path: pagePath, - content: page.body, - }; - }); - await writeFileAtomic( - this.outputPath("services", "mcp", "docs-content.json"), - JSON.stringify(docsContent, null, 2) + "\n", - ); - - const indexContent = llmsIndexTemplate({ - siteName: name, - siteDescription: description, - baseUrl, - pages: resolvedPages, - sectionsConfig: this.sectionsConfig, - }); - const fullContent = llmsFullTemplate({ - siteName: name, - siteDescription: description, - baseUrl, - pages: pagesWithBodies, - sectionsConfig: this.sectionsConfig, - }); - - await this.writePublicAggregate("llms.txt", indexContent); - await this.writePublicAggregate("llms-full.txt", fullContent); - - const skillContent = skillMdTemplate({ - siteName: name, - siteDescription: description, - baseUrl, - pages: resolvedPages, - sectionsConfig: this.sectionsConfig, - }); - await this.writePublicAggregate("skill.md", skillContent); - - // MCP discovery manifest. Needs an absolute URL, so it only exists when - // config.json declares the site url; it is pruned if the url is removed. - const mcpRelativePath = ".well-known/mcp.json"; - const mcpJsonPath = this.publicOutputFilePath(mcpRelativePath); - const sourceMcpJsonPath = await this.findSourcePublicAsset(mcpRelativePath); - if (sourceMcpJsonPath) { - console.warn( - chalk.yellow( - `⚠️ Skipping generated public/${mcpRelativePath}; a project public asset owns that path`, - ), - ); - await this.copyRegularPublicFile( - path.join(this.rootDir, "public"), - sourceMcpJsonPath, - mcpJsonPath, - ); - } else if (baseUrl) { - const mcpJson = - JSON.stringify( - { - mcpServers: { - [siteDocsSlug(name)]: { - url: `${baseUrl}/api/mcp`, - transport: "streamable-http", - }, - }, - }, - null, - 2, - ) + "\n"; - await fs.ensureDir(path.dirname(mcpJsonPath)); - await writeFileAtomic(mcpJsonPath, mcpJson); - } else if (await fs.pathExists(mcpJsonPath)) { - await fs.remove(mcpJsonPath); - } - - const nextRelativePaths = new Set(); - await Promise.all( - pagesWithBodies.map(async (page) => { - const relPath = page.slug === "" ? "index.md" : `${page.slug}.md`; - if (isPublicAggregate(relPath)) return; - const sourceAssetPath = await this.findSourcePublicAsset(relPath); - if (sourceAssetPath) { - console.warn( - chalk.yellow( - `⚠️ Skipping generated public/${relPath}; a project public asset owns that path`, - ), - ); - const targetPath = this.publicOutputFilePath(relPath); - await this.copyRegularPublicFile( - path.join(this.rootDir, "public"), - sourceAssetPath, - targetPath, - ); - return; - } - const targetPath = this.publicOutputFilePath(relPath); - await fs.ensureDir(path.dirname(targetPath)); - await writeFileAtomic(targetPath, llmsPageTemplate(page, baseUrl)); - nextRelativePaths.add(relPath); - }), - ); - - const previousRelativePaths = this.artifacts.llmsPageFiles(); - - for (const stale of previousRelativePaths) { - if (!nextRelativePaths.has(stale)) { - try { - if (isPublicAggregate(stale)) continue; - if (await this.findSourcePublicAsset(stale)) continue; - const stalePath = resolveOutputPath(publicDir, stale); - if (await fs.pathExists(stalePath)) { - await fs.remove(stalePath); - } - } catch { - // ignore - } - } - } - this.artifacts.replaceLlmsPageFiles(nextRelativePaths); - await this.artifacts.save(); - - console.log( - chalk.green( - `πŸ€– Generated llms.txt and llms-full.txt with ${resolvedPages.length} page(s)${ - baseUrl ? ` using ${baseUrl}` : " (relative URLs)" - }`, - ), + return writeLlmsFiles( + this.outputDir, + this.sectionsConfig, + async () => pages ?? (await this.buildAllPagesMeta()), + (filePath) => this.readMdxSourceFile(filePath), + (slug) => this.apiRegistry.bodyForSlug(slug), + () => loadSiteMetadata(this.rootDir), + this.publicAssetManager, ); } async stop() { - this.stopping = true; - for (const cancel of [...this.readyCancellations]) cancel(); - if (this.watcher) { - await this.watcher.close(); - console.log(chalk.yellow("πŸ‘‹ Stopped watching for MDX changes")); - } - if (this.configWatcher) { - await this.configWatcher.close(); - console.log(chalk.yellow("πŸ‘‹ Stopped watching for config changes")); - } - if (this.fontWatcher) { - await this.fontWatcher.close(); - console.log(chalk.yellow("πŸ‘‹ Stopped watching for font config changes")); - } - if (this.analyticsWatcher) { - await this.analyticsWatcher.close(); - console.log( - chalk.yellow("πŸ‘‹ Stopped watching for analytics config changes"), - ); - } - if (this.openApiWatcher) { - await this.openApiWatcher.close(); - console.log(chalk.yellow("πŸ‘‹ Stopped watching for OpenAPI spec changes")); - } - if (this.doccupineConfigWatcher) { - await this.doccupineConfigWatcher.close(); - console.log( - chalk.yellow("πŸ‘‹ Stopped watching for doccupine.json changes"), - ); - } - if (this.publicWatcher) { - await this.publicWatcher.close(); - console.log( - chalk.yellow("πŸ‘‹ Stopped watching for public directory changes"), - ); - } - if (this.rootDirWatcher) { - await this.rootDirWatcher.close(); - } - await this.mutationQueue; - if (this.openApiWatcher) { - await this.openApiWatcher.close(); - this.openApiWatcher = null; - } - if (this.publicWatcher) { - await this.publicWatcher.close(); - this.publicWatcher = null; - } + return this.watchCoordinator.stop(); } } From 9ed8739b937ed7ffcc785d034892a8dd050e3b2d Mon Sep 17 00:00:00 2001 From: Luan Gjokaj Date: Thu, 30 Jul 2026 20:59:59 +0200 Subject: [PATCH 2/3] fix(templates): drop the redundant Button wrapper from Accordion StyledAccordionTitle already reimplements a full manual button reset (appearance, border, background, font, cursor) inline, so wrapping cherry's Button and layering resetButton on top only fought that styling with Button's own defaults. Switch the styled-component base to a plain styled.button and drop the now-unused Button/resetButton import. --- src/templates/components/layout/Accordion.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/templates/components/layout/Accordion.ts b/src/templates/components/layout/Accordion.ts index 046acc0..1f14603 100644 --- a/src/templates/components/layout/Accordion.ts +++ b/src/templates/components/layout/Accordion.ts @@ -1,7 +1,7 @@ export const accordionTemplate = `"use client"; import { useId, useState } from "react"; import styled, { css } from "styled-components"; -import { Button, resetButton, styledText } from "cherry-styled-components"; +import { styledText } from "cherry-styled-components"; import { Theme } from "@/app/theme"; import { Icon } from "@/components/layout/Icon"; @@ -15,11 +15,10 @@ const StyledAccordion = styled.div<{ theme: Theme }>\` width: 100%; \`; -const StyledAccordionTitle = styled(Button)<{ +const StyledAccordionTitle = styled.button<{ theme: Theme; $isOpen: boolean; }>\` - \${resetButton}; appearance: none; display: block; width: 100%; From 39c4084518465e9e21b19c67eeacee068e3f745f Mon Sep 17 00:00:00 2001 From: Luan Gjokaj Date: Thu, 30 Jul 2026 21:04:10 +0200 Subject: [PATCH 3/3] chore: update dependencies --- src/templates/package.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/templates/package.ts b/src/templates/package.ts index 31691ac..b270ee3 100644 --- a/src/templates/package.ts +++ b/src/templates/package.ts @@ -26,8 +26,8 @@ export const packageJsonTemplate = minisearch: "^7.2.0", next: "16.2.12", "next-mdx-remote": "^6.0.0", - "posthog-js": "^1.408.3", - "posthog-node": "^5.46.1", + "posthog-js": "^1.409.0", + "posthog-node": "^5.47.0", react: "19.2.8", "react-dom": "19.2.8", "rehype-highlight": "^7.0.2",