feat: harden generator and generated site security

.github/workflows/ci.yml

@@ -14,14 +14,14 @@ jobs:
       - uses: actions/checkout@v7
 
       - uses: pnpm/action-setup@v6
-        with:
-          version: 10
 
       - uses: actions/setup-node@v7
         with:
           node-version: 22
           cache: pnpm
 
       - run: pnpm install --frozen-lockfile
-      - run: pnpm build
+      - run: pnpm format:check
       - run: pnpm test
+      - run: pnpm test:package
+      - run: pnpm smoke:generated

.gitignore

@@ -2,6 +2,7 @@
 
 # dependencies
 /node_modules
+/.pnpm-store
 /.pnp
 .pnp.js
 .yarn/install-state.gz

CONTRIBUTING.md

@@ -14,6 +14,9 @@ pnpm install
 pnpm dev            # Watch mode (recompiles on changes)
 pnpm build          # One-time compile
 pnpm test           # Run tests
+pnpm format:check   # Verify formatting
+pnpm test:package   # Verify the npm package contents
+pnpm smoke:generated # Generate, lint, type-check, and build a fixture site
 ```
 
 To test your changes locally:
@@ -26,13 +29,13 @@ node /path/to/cli/dist/index.js watch
 
 ## Project Structure
 
-All CLI logic lives in `src/index.ts`. Template files under `src/templates/` are string constants that get written into the generated Next.js app.
+The CLI entry point and generator orchestration live in `src/index.ts`. Reusable generator logic is under `src/lib/`, while template files under `src/templates/` are string constants written into the generated Next.js app. The output layout is registered centrally in `src/lib/structures.ts`.
 
 When adding a new template:
 
 1. Create the template file in the appropriate `src/templates/` subdirectory
 2. Export a named constant with the `Template` suffix (e.g., `export const myComponentTemplate = ...`)
-3. Import it in `src/index.ts` and add it to the `structure` object in `createNextJSStructure()`
+3. Import it in `src/lib/structures.ts` and add it to the `structure` object
 
 ## Code Conventions
 
@@ -43,7 +46,7 @@ When adding a new template:
 ## Pull Requests
 
 1. Fork the repo and create a branch from `main`
-2. Make your changes and ensure `pnpm build && pnpm test` passes
+2. Make your changes and ensure `pnpm build && pnpm test && pnpm format:check && pnpm test:package` passes
 3. Write a clear PR description explaining the change and why
 4. Keep PRs focused - one feature or fix per PR
 

README.md

@@ -35,11 +35,14 @@ Doccupine will prompt you for:
 
 It then scaffolds the app, installs dependencies, and starts the dev server. Open http://localhost:3000 to view your docs.
 
+The source and output directories must not overlap. To prevent accidental data loss, Doccupine only claims an empty output directory, one containing harmless local metadata such as `.DS_Store` or `.env.local`, or an existing Doccupine-generated app. Run `doccupine config --reset` if an older configuration no longer passes validation.
+
 ## CLI Commands
 
 ```bash
 doccupine watch [options]   # Default. Watch MDX files and start dev server
-doccupine build [options]   # One-time build without starting the server
+doccupine build [options]   # Generate the site without installing or serving it
+doccupine generate [options] # Alias for build
 doccupine config --show     # Show current configuration
 doccupine config --reset    # Re-prompt for configuration
 ```
@@ -53,6 +56,7 @@ doccupine config --reset    # Re-prompt for configuration
 | `--port <port>`            | Port for the dev server (default: `3000`). Auto-increments if taken.                                                                     |
 | `--verbose`                | Show all Next.js output including compilation details                                                                                    |
 | `--reset`                  | Re-prompt for watch/output directories                                                                                                   |
+| `--skip-install`           | Generate and serve without running the dependency installer (installs are skipped automatically when `package.json` is unchanged)        |
 | `--package-manager <name>` | Package manager for the generated app: `pnpm` or `npm` (default: auto-detect). Overrides the `packageManager` field in `doccupine.json`. |
 
 `build`:
@@ -149,7 +153,7 @@ Each entry has:
 
 ## API Reference
 
-Point Doccupine at an OpenAPI document (`.json`, `.yaml`, or `.yml`, OpenAPI 3.0/3.1) and it generates an interactive API reference: one page per operation, each with a live playground for sending requests. Set the `openapi` field in `doccupine.json`:
+Point Doccupine at an OpenAPI document (`.json`, `.yaml`, or `.yml`, OpenAPI 3.0/3.1) and it generates an interactive API reference: a directory at `/api-reference` linking every operation, plus one page per operation with a live playground for sending requests. Set the `openapi` field in `doccupine.json`:
 
 ```json
 {
@@ -209,7 +213,7 @@ Doccupine generates `robots.ts` automatically for every site. When you set a `ur
 }
 ```
 
-You can override the URL at deploy time by setting the `NEXT_PUBLIC_SITE_URL` environment variable. When no URL is configured (neither in `config.json` nor via env), the sitemap is skipped and `robots.txt` is emitted without a sitemap reference.
+You can override the URL at deploy time by setting the `NEXT_PUBLIC_SITE_URL` environment variable. When no URL is configured, `/sitemap.xml` is still served but stays empty, and `robots.txt` omits its sitemap reference until a public URL is available. The variable is baked in at build time, so changing it needs a redeploy.
 
 ## llms.txt
 
@@ -223,7 +227,7 @@ Doccupine generates [llms.txt](https://llmstxt.org) artifacts so AI agents can d
 
 The site name and description used in `llms.txt` come from `config.json` (`name`, `description`). Page URLs are absolute when `url` is set in `config.json` (or via `NEXT_PUBLIC_SITE_URL`), and root-relative otherwise.
 
-A `.doccupine-llms-manifest.json` file in the generated app tracks which per-page mirrors were emitted so renamed or deleted pages get cleaned up on the next regeneration. Don't commit this file — it's regenerated automatically.
+A `.doccupine-artifacts.json` file in the generated app tracks generated route ownership and per-page mirrors so renamed or deleted sources clean up only their own outputs. Don't commit this file; it is regenerated automatically.
 
 ## AI Chat Setup
 
@@ -250,10 +254,13 @@ LLM_EMBEDDING_MODEL=text-embedding-3-small  # Override the default embedding mod
 LLM_TEMPERATURE=0                           # Set temperature (0-1, default: 0)
 LLM_EMBEDDING_DIMS=512                       # Dimensions for the prebuilt search index (default: 512)
 RAG_RUNTIME_EMBED_MAX_CHUNKS=400             # Max chunks embedded on demand in production (default: 400; 0 requires a prebuilt index)
+# RAG_API_KEY=...                            # Optional bearer auth for direct /api/rag requests
 ```
 
 `LLM_EMBEDDING_DIMS` Matryoshka-truncates document vectors so the prebuilt search index stays small; lower values shrink the index at a slight cost to recall. `RAG_RUNTIME_EMBED_MAX_CHUNKS` caps how many chunks the chat will embed on demand in production before requiring a prebuilt index (it's unlimited under `next dev`).
 
+Public documentation leaves browser chat available by default. Setting `RAG_API_KEY` requires a bearer token for `/api/rag` and is intended for server-to-server use; the built-in browser cannot safely hold that secret. Use `SITE_PASSWORD` instead when authenticated browser visitors should retain chat access.
+
 Default models per provider:
 
 | Provider  | Chat model                   | Embedding model          |
@@ -270,7 +277,7 @@ The generated app exposes an MCP endpoint at `/api/mcp` with three tools:
 - `get_doc` - retrieve a specific document by path
 - `list_docs` - list all available documents
 
-This lets AI agents (Claude, ChatGPT, etc.) query your docs programmatically. Requires the AI chat setup above for embeddings.
+This lets AI agents (Claude, ChatGPT, etc.) query your docs programmatically. Semantic `search_docs` requires the AI setup above for embeddings; `get_doc` and `list_docs` work from the generated content manifest without an embedding provider.
 
 ## Password Protection
 
@@ -283,12 +290,12 @@ SITE_PASSWORD=choose-a-strong-shared-password
 When set, every visitor sees a login screen until they enter the password. Protection is enforced across three layers:
 
 - **Pages** are gated behind the login screen.
-- **Content APIs** (`/api/rag` chat and `/api/search`) return `401` without a valid session, so the docs can't be scraped around the login.
+- **Content APIs** (`/api/rag` chat, `/api/search`, and the `/api/playground` proxy) return `401` without a valid session, so the docs can't be scraped and the proxy can't relay anonymous requests around the login. Each route re-checks the session itself, not just the middleware.
 - **Search engines and crawlers** are blocked: `robots.txt` disallows everything, pages carry a `noindex, nofollow` tag, and responses include an `X-Robots-Tag` header.
 
 A successful login sets a signed, `httpOnly` cookie that lasts 30 days. The cookie stores an HMAC of the password, never the password itself. Leave `SITE_PASSWORD` unset (the default) to keep the site fully public. Documentation pages stay statically rendered either way - the gate is enforced in middleware.
 
-> **Note:** The [MCP endpoint](#mcp-server) uses its own `DOCS_API_KEY` bearer token and is not affected by `SITE_PASSWORD`.
+> **Note:** The [MCP endpoint](#mcp-server) uses `DOCS_API_KEY` bearer authentication when configured. Without an API key, a password-protected site requires the normal gate session for MCP as well.
 
 ## License
 

SECURITY.md

@@ -41,8 +41,9 @@ Please include:
 
 Doccupine is a CLI that generates a Next.js application you run and host yourself. A few points matter when assessing the security surface:
 
-- **API keys and secrets** - the generated app reads provider keys (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GOOGLE_API_KEY`), an optional `DOCS_API_KEY`, and an optional `SITE_PASSWORD` from environment variables. These belong in the generated app's `.env` file, which is git-ignored by default. Never commit real keys.
-- **MCP endpoint** - the `/api/mcp` route requires a bearer token only when `DOCS_API_KEY` is set. If `DOCS_API_KEY` is not set, the endpoint is publicly accessible with no authentication. Set `DOCS_API_KEY` before exposing the generated site publicly.
+- **API keys and secrets** - the generated app reads provider keys (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GOOGLE_API_KEY`), optional endpoint keys (`RAG_API_KEY`, `DOCS_API_KEY`), and an optional `SITE_PASSWORD` from environment variables. These belong in the generated app's `.env` file, which is git-ignored by default. Never commit real keys.
+- **Paid RAG endpoint** - public documentation leaves `/api/rag` available to the browser assistant by default. Set `RAG_API_KEY` to require a bearer token and prevent anonymous model spend on a public site. Because the browser cannot safely hold this secret, use `SITE_PASSWORD` instead when authenticated browser visitors need the built-in assistant.
+- **MCP endpoint** - the `/api/mcp` route requires a bearer token when `DOCS_API_KEY` is set. Without `DOCS_API_KEY`, it falls back to the site gate: a configured `SITE_PASSWORD` requires a valid gate session, while an intentionally public site leaves MCP public. Set `DOCS_API_KEY` for independent server-to-server authentication.
 - **Site password** - `SITE_PASSWORD` gates the whole site behind a single shared password. It is a lightweight access gate, not a substitute for per-user authentication.
 - **Generated output** - review generated code before deploying to production, especially if you customize templates.
 

package.json

@@ -7,13 +7,21 @@
     "doccupine": "dist/index.js"
   },
   "type": "module",
+  "engines": {
+    "node": ">=22"
+  },
+  "packageManager": "pnpm@11.18.0",
   "scripts": {
-    "build": "tsc",
+    "clean": "node scripts/clean.mjs",
+    "build": "pnpm clean && tsc",
     "dev": "tsc --watch",
     "start": "node dist/index.js",
-    "prepare": "tsc",
-    "test": "vitest run",
-    "format": "prettier --write ."
+    "prepare": "node scripts/clean.mjs && tsc",
+    "test": "pnpm build && vitest run",
+    "test:package": "pnpm build && node scripts/verify-package.mjs",
+    "smoke:generated": "node scripts/smoke-generated-site.mjs",
+    "format": "prettier --write .",
+    "format:check": "prettier --check ."
   },
   "keywords": [
     "doccupine",
@@ -40,10 +48,7 @@
     "commander": "^15.0.0",
     "fs-extra": "^11.4.0",
     "gray-matter": "^4.0.3",
-    "next": "^16.2.12",
-    "prompts": "^2.4.2",
-    "react": "^19.2.8",
-    "react-dom": "^19.2.8"
+    "prompts": "^2.4.2"
   },
   "devDependencies": {
     "@types/fs-extra": "^11.0.4",

scripts/clean.mjs

@@ -0,0 +1,9 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const scriptDir = path.dirname(fileURLToPath(import.meta.url));
+await fs.rm(path.resolve(scriptDir, "..", "dist"), {
+  recursive: true,
+  force: true,
+});

scripts/smoke-generated-site.mjs

@@ -0,0 +1,130 @@
+import { spawn } from "node:child_process";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const projectDir = await fs.mkdtemp(path.join(os.tmpdir(), "doccupine-smoke-"));
+const packageManager = process.platform === "win32" ? "pnpm.cmd" : "pnpm";
+
+function run(command, args, cwd) {
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, args, { cwd, stdio: "inherit" });
+    child.on("error", reject);
+    child.on("close", (code) => {
+      if (code === 0) resolve();
+      else
+        reject(new Error(`${command} ${args.join(" ")} exited with ${code}`));
+    });
+  });
+}
+
+try {
+  await fs.mkdir(path.join(projectDir, "docs"), { recursive: true });
+  await fs.writeFile(
+    path.join(projectDir, "doccupine.json"),
+    JSON.stringify(
+      {
+        watchDir: "docs",
+        outputDir: "site",
+        port: "3000",
+        openapi: "openapi.json",
+      },
+      null,
+      2,
+    ),
+  );
+  await fs.writeFile(
+    path.join(projectDir, "config.json"),
+    JSON.stringify({ name: "Smoke Docs", url: "https://docs.example.test" }),
+  );
+  await fs.writeFile(
+    path.join(projectDir, "docs", "index.mdx"),
+    [
+      "---",
+      'title: "Using `widgets` and ${safeText}"',
+      'description: "Quotes, `code`, and ${expressions} stay data."',
+      "---",
+      "",
+      "# Smoke test",
+      "",
+      "The generated site must compile.",
+      "",
+    ].join("\n"),
+  );
+  await fs.writeFile(
+    path.join(projectDir, "docs", "guide.mdx"),
+    [
+      "---",
+      'title: "Guide"',
+      'section: "Guides"',
+      "---",
+      "",
+      "# Guide",
+      "",
+      "A sectioned page.",
+      "",
+    ].join("\n"),
+  );
+  await fs.writeFile(
+    path.join(projectDir, "openapi.json"),
+    JSON.stringify({
+      openapi: "3.1.0",
+      info: { title: "Smoke API", version: "1.0.0" },
+      servers: [{ url: "https://api.example.test/v1" }],
+      paths: {
+        "/widgets/{quoted}": {
+          get: {
+            operationId: "getWidget",
+            summary: "Get a widget",
+            parameters: [
+              {
+                name: 'quoted"name',
+                in: "path",
+                required: true,
+                schema: { type: "string" },
+              },
+            ],
+            responses: { 200: { description: "OK" } },
+          },
+        },
+      },
+    }),
+  );
+
+  await run(
+    process.execPath,
+    [path.join(root, "dist", "index.js"), "build"],
+    projectDir,
+  );
+
+  const siteDir = path.join(projectDir, "site");
+  await run(packageManager, ["install", "--frozen-lockfile=false"], siteDir);
+  await run(packageManager, ["run", "type-check"], siteDir);
+  await run(packageManager, ["run", "lint", "--max-warnings=0"], siteDir);
+  await run(packageManager, ["run", "build"], siteDir);
+
+  for (const route of ["mcp", "rag"]) {
+    const tracePath = path.join(
+      siteDir,
+      ".next",
+      "server",
+      "app",
+      "api",
+      route,
+      "route.js.nft.json",
+    );
+    const trace = JSON.parse(await fs.readFile(tracePath, "utf8"));
+    if (
+      !Array.isArray(trace.files) ||
+      !trace.files.some((file) =>
+        file.endsWith("services/mcp/docs-content.json"),
+      )
+    ) {
+      throw new Error(`${route} route did not trace docs-content.json`);
+    }
+  }
+} finally {
+  await fs.rm(projectDir, { recursive: true, force: true });
+}