add sitemap.xml generation

.gitignore

@@ -23,6 +23,7 @@ tutorial-code/Cargo.lock
 
 # misc
 .DS_Store
+public/sitemap.xml
 
 # debug
 npm-debug.log*

package.json

@@ -4,8 +4,9 @@
   "private": true,
   "type": "module",
   "scripts": {
-    "dev": "next dev --turbopack",
-    "build": "next build && npm run build:rss",
+    "dev": "npm run build:sitemap && next dev --turbopack",
+    "build": "npm run build:sitemap && next build && npm run build:rss",
+    "build:sitemap": "node scripts/generate-sitemap.js",
     "build:rss": "tsc --outDir dist --noEmit false && sed -i.bak 's/markdown\"/markdown.js\"/g' dist/lib/api.js && rm dist/lib/api.js.bak && node scripts/generate-rss-feed.js",
     "fmt": "prettier --write --prose-wrap always '{components,content,pages,lib,styles}/**/*.{js,jsx,ts,tsx,scss}'",
     "fmt:check": "prettier --check --prose-wrap always '{components,content,pages,lib,styles}/**/*.{js,jsx,ts,tsx,scss}'"

public/robots.txt

@@ -0,0 +1,4 @@
+Sitemap: https://tokio.rs/sitemap.xml
+
+User-agent: *
+Allow: /

scripts/generate-sitemap.js

@@ -0,0 +1,65 @@
+import fs from "fs";
+import path from "path";
+
+const siteUrl = "https://tokio.rs";
+const outDir = path.join(process.cwd(), "public");
+const contentDir = path.join(process.cwd(), "content");
+
+const staticPages = [
+  { url: "/", changefreq: "weekly", priority: "1.0" },
+  { url: "/blog", changefreq: "weekly", priority: "0.8" },
+];
+
+const blogPosts = fs
+  .readdirSync(path.join(contentDir, "blog"))
+  .filter((f) => f.endsWith(".md"))
+  .map((f) => ({
+    url: `/blog/${f.replace(/\.md$/, "")}`,
+    changefreq: "monthly",
+    priority: "0.6",
+  }));
+
+function walkDir(dir) {
+  const entries = fs.readdirSync(dir, { withFileTypes: true });
+  let results = [];
+  for (const entry of entries) {
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      results = results.concat(walkDir(full));
+    } else if (entry.name.endsWith(".md")) {
+      results.push(full);
+    }
+  }
+  return results;
+}
+
+const docPages = walkDir(path.join(contentDir, "tokio"))
+  .map((f) => {
+    const rel = path.relative(contentDir, f).replace(/\.md$/, "").replace(/\/index$/, "");
+    return {
+      url: `/${rel}`,
+      changefreq: "monthly",
+      priority: "0.7",
+    };
+  })
+  .filter((p) => !p.url.includes("/api"));
+
+const allPages = [...staticPages, ...blogPosts, ...docPages];
+
+const sitemap = `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${allPages
+  .map(
+    (p) => `  <url>
+    <loc>${siteUrl}${p.url}</loc>
+    <changefreq>${p.changefreq}</changefreq>
+    <priority>${p.priority}</priority>
+  </url>`
+  )
+  .join("\n")}
+</urlset>
+`;
+
+fs.mkdirSync(outDir, { recursive: true });
+fs.writeFileSync(path.join(outDir, "sitemap.xml"), sitemap);
+console.log(`sitemap.xml generated with ${allPages.length} URLs`);