diff --git a/.gitignore b/.gitignore index 8fd1f43e..7f9bd12b 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ tutorial-code/Cargo.lock # misc .DS_Store +public/sitemap.xml # debug npm-debug.log* diff --git a/package.json b/package.json index a0ddaab1..06e5fc46 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,9 @@ "private": true, "type": "module", "scripts": { - "dev": "next dev --turbopack", - "build": "next build && npm run build:rss", + "dev": "npm run build:sitemap && next dev --turbopack", + "build": "npm run build:sitemap && next build && npm run build:rss", + "build:sitemap": "node scripts/generate-sitemap.js", "build:rss": "tsc --outDir dist --noEmit false && sed -i.bak 's/markdown\"/markdown.js\"/g' dist/lib/api.js && rm dist/lib/api.js.bak && node scripts/generate-rss-feed.js", "fmt": "prettier --write --prose-wrap always '{components,content,pages,lib,styles}/**/*.{js,jsx,ts,tsx,scss}'", "fmt:check": "prettier --check --prose-wrap always '{components,content,pages,lib,styles}/**/*.{js,jsx,ts,tsx,scss}'" diff --git a/public/robots.txt b/public/robots.txt new file mode 100644 index 00000000..d094f342 --- /dev/null +++ b/public/robots.txt @@ -0,0 +1,4 @@ +Sitemap: https://tokio.rs/sitemap.xml + +User-agent: * +Allow: / diff --git a/scripts/generate-sitemap.js b/scripts/generate-sitemap.js new file mode 100644 index 00000000..dacf14ae --- /dev/null +++ b/scripts/generate-sitemap.js @@ -0,0 +1,65 @@ +import fs from "fs"; +import path from "path"; + +const siteUrl = "https://tokio.rs"; +const outDir = path.join(process.cwd(), "public"); +const contentDir = path.join(process.cwd(), "content"); + +const staticPages = [ + { url: "/", changefreq: "weekly", priority: "1.0" }, + { url: "/blog", changefreq: "weekly", priority: "0.8" }, +]; + +const blogPosts = fs + .readdirSync(path.join(contentDir, "blog")) + .filter((f) => f.endsWith(".md")) + .map((f) => ({ + url: `/blog/${f.replace(/\.md$/, "")}`, + changefreq: "monthly", + priority: "0.6", + })); + +function walkDir(dir) { + const entries = fs.readdirSync(dir, { withFileTypes: true }); + let results = []; + for (const entry of entries) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + results = results.concat(walkDir(full)); + } else if (entry.name.endsWith(".md")) { + results.push(full); + } + } + return results; +} + +const docPages = walkDir(path.join(contentDir, "tokio")) + .map((f) => { + const rel = path.relative(contentDir, f).replace(/\.md$/, "").replace(/\/index$/, ""); + return { + url: `/${rel}`, + changefreq: "monthly", + priority: "0.7", + }; + }) + .filter((p) => !p.url.includes("/api")); + +const allPages = [...staticPages, ...blogPosts, ...docPages]; + +const sitemap = ` + +${allPages + .map( + (p) => ` + ${siteUrl}${p.url} + ${p.changefreq} + ${p.priority} + ` + ) + .join("\n")} + +`; + +fs.mkdirSync(outDir, { recursive: true }); +fs.writeFileSync(path.join(outDir, "sitemap.xml"), sitemap); +console.log(`sitemap.xml generated with ${allPages.length} URLs`);