diff --git a/.gitignore b/.gitignore index e7f818e6..badf9a29 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,9 @@ package-lock.json /public/llms/ /public/llms.txt +# Sitemap (generated by scripts/generate-sitemap.mjs) +/public/sitemap.xml + # Edit on GitHub index routes (generated by scripts/generate-github-routes.mjs) /src/lib/edit-on-github-routes.js diff --git a/CLAUDE.md b/CLAUDE.md index bcd60f2e..ea274d1c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,6 +20,7 @@ npm run gen # Regenerate API docs from NetBird OpenAPI spec npm run gen:llm # Regenerate LLM-friendly markdown (auto-runs with dev/build) npm run gen:edit-routes # Regenerate edit-on-GitHub routes (auto-runs with dev/build) npm run gen:last-updated # Regenerate per-page git last-modified dates (auto-runs with dev/build) +npm run gen:sitemap # Regenerate public/sitemap.xml (auto-runs with dev/build) ``` ## Security boundaries diff --git a/package.json b/package.json index ff9c06e8..0b7565fa 100644 --- a/package.json +++ b/package.json @@ -3,10 +3,11 @@ "version": "1.0.0", "private": true, "scripts": { - "dev": "npm run gen:llm && npm run gen:edit-routes && npm run gen:last-updated && next dev --webpack", - "build": "npm run gen:llm && npm run gen:edit-routes && npm run gen:last-updated && next build --webpack", + "dev": "npm run gen:llm && npm run gen:edit-routes && npm run gen:last-updated && npm run gen:sitemap && next dev --webpack", + "build": "npm run gen:llm && npm run gen:edit-routes && npm run gen:last-updated && npm run gen:sitemap && next build --webpack", "gen:edit-routes": "node scripts/generate-github-routes.mjs", "gen:last-updated": "node scripts/generate-last-updated.mjs", + "gen:sitemap": "node scripts/generate-sitemap.mjs", "gen": "swagger-codegen generate -i https://raw.githubusercontent.com/netbirdio/netbird/main/management/server/http/api/openapi.yml -l openapi -o generator/openapi && npx ts-node generator/index.ts gen --input generator/openapi/openapi.json --output src/pages/ipa/resources", "gen:llm": "node scripts/generate-llm-docs.mjs", "start": "next start", diff --git a/public/robots.txt b/public/robots.txt new file mode 100644 index 00000000..99b4ef3b --- /dev/null +++ b/public/robots.txt @@ -0,0 +1,4 @@ +User-agent: * +Allow: / + +Sitemap: https://docs.netbird.io/sitemap.xml diff --git a/scripts/generate-sitemap.mjs b/scripts/generate-sitemap.mjs new file mode 100644 index 00000000..daa81af0 --- /dev/null +++ b/scripts/generate-sitemap.mjs @@ -0,0 +1,89 @@ +#!/usr/bin/env node +/** + * Generates public/sitemap.xml by scanning src/pages for .mdx files. + * Routes under src/pages/ipa/ are emitted as /api/* to match the rewrite + * in next.config.mjs. Last-modified dates come from git history. + * + * Run automatically with dev and build. + */ + +import fs from 'fs' +import path from 'path' +import { fileURLToPath } from 'url' +import { getGitLastModified } from './git-dates.mjs' + +const __dirname = path.dirname(fileURLToPath(import.meta.url)) +const ROOT = path.join(__dirname, '..') +const PAGES_DIR = path.join(ROOT, 'src/pages') +const OUT_PATH = path.join(ROOT, 'public/sitemap.xml') + +const BASE_URL = 'https://docs.netbird.io' + +function findMdxFiles(dir, basePath = '') { + const entries = fs.readdirSync(dir, { withFileTypes: true }) + const results = [] + for (const e of entries) { + if (e.name.startsWith('_')) continue + const rel = basePath ? `${basePath}/${e.name}` : e.name + if (e.isDirectory()) { + results.push(...findMdxFiles(path.join(dir, e.name), rel)) + } else if (e.name.endsWith('.mdx')) { + const filePath = path.join(dir, e.name) + const route = + e.name === 'index.mdx' + ? basePath + ? `/${basePath}` + : '/' + : `/${rel.replace(/\.mdx$/, '')}` + results.push({ route, filePath }) + } + } + return results +} + +function toPublicUrl(route) { + if (route === '/ipa') return '/api' + if (route.startsWith('/ipa/')) return `/api/${route.slice(5)}` + return route +} + +function escapeXml(s) { + return s + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, ''') +} + +const entries = findMdxFiles(PAGES_DIR) + .map(({ route, filePath }) => ({ + url: toPublicUrl(route), + lastmod: getGitLastModified(filePath), + })) + .sort((a, b) => a.url.localeCompare(b.url)) + +const seen = new Set() +const unique = entries.filter((e) => { + if (seen.has(e.url)) return false + seen.add(e.url) + return true +}) + +const urls = unique + .map(({ url, lastmod }) => { + const loc = escapeXml(`${BASE_URL}${url}`) + const lastmodTag = lastmod ? `\n ${lastmod}` : '' + return ` \n ${loc}${lastmodTag}\n ` + }) + .join('\n') + +const xml = ` + +${urls} + +` + +fs.mkdirSync(path.dirname(OUT_PATH), { recursive: true }) +fs.writeFileSync(OUT_PATH, xml, 'utf8') +console.log('Generated', OUT_PATH, 'with', unique.length, 'URLs')