Add script to generate sitemap.xml and update related configurations (#742)

* Add script to generate sitemap.xml and update related configurations

* chore: add robots.txt referencing sitemap.xml

---------

Co-authored-by: Jack Carter <128555021+SunsetDrifter@users.noreply.github.com>

.gitignore

@@ -33,6 +33,9 @@ package-lock.json
 /public/llms/
 /public/llms.txt
 
+# Sitemap (generated by scripts/generate-sitemap.mjs)
+/public/sitemap.xml
+
 # Edit on GitHub index routes (generated by scripts/generate-github-routes.mjs)
 /src/lib/edit-on-github-routes.js
 

CLAUDE.md

@@ -20,6 +20,7 @@ npm run gen              # Regenerate API docs from NetBird OpenAPI spec
 npm run gen:llm          # Regenerate LLM-friendly markdown (auto-runs with dev/build)
 npm run gen:edit-routes  # Regenerate edit-on-GitHub routes (auto-runs with dev/build)
 npm run gen:last-updated # Regenerate per-page git last-modified dates (auto-runs with dev/build)
+npm run gen:sitemap      # Regenerate public/sitemap.xml (auto-runs with dev/build)
 ```
 
 ## Security boundaries

package.json

@@ -3,10 +3,11 @@
   "version": "1.0.0",
   "private": true,
   "scripts": {
-    "dev": "npm run gen:llm && npm run gen:edit-routes && npm run gen:last-updated && next dev --webpack",
-    "build": "npm run gen:llm && npm run gen:edit-routes && npm run gen:last-updated && next build --webpack",
+    "dev": "npm run gen:llm && npm run gen:edit-routes && npm run gen:last-updated && npm run gen:sitemap && next dev --webpack",
+    "build": "npm run gen:llm && npm run gen:edit-routes && npm run gen:last-updated && npm run gen:sitemap && next build --webpack",
     "gen:edit-routes": "node scripts/generate-github-routes.mjs",
     "gen:last-updated": "node scripts/generate-last-updated.mjs",
+    "gen:sitemap": "node scripts/generate-sitemap.mjs",
     "gen": "swagger-codegen generate -i https://raw.githubusercontent.com/netbirdio/netbird/main/management/server/http/api/openapi.yml -l openapi -o generator/openapi && npx ts-node generator/index.ts gen --input generator/openapi/openapi.json --output src/pages/ipa/resources",
     "gen:llm": "node scripts/generate-llm-docs.mjs",
     "start": "next start",

public/robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /
+
+Sitemap: https://docs.netbird.io/sitemap.xml

scripts/generate-sitemap.mjs

@@ -0,0 +1,89 @@
+#!/usr/bin/env node
+/**
+ * Generates public/sitemap.xml by scanning src/pages for .mdx files.
+ * Routes under src/pages/ipa/ are emitted as /api/* to match the rewrite
+ * in next.config.mjs. Last-modified dates come from git history.
+ *
+ * Run automatically with dev and build.
+ */
+
+import fs from 'fs'
+import path from 'path'
+import { fileURLToPath } from 'url'
+import { getGitLastModified } from './git-dates.mjs'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const ROOT = path.join(__dirname, '..')
+const PAGES_DIR = path.join(ROOT, 'src/pages')
+const OUT_PATH = path.join(ROOT, 'public/sitemap.xml')
+
+const BASE_URL = 'https://docs.netbird.io'
+
+function findMdxFiles(dir, basePath = '') {
+  const entries = fs.readdirSync(dir, { withFileTypes: true })
+  const results = []
+  for (const e of entries) {
+    if (e.name.startsWith('_')) continue
+    const rel = basePath ? `${basePath}/${e.name}` : e.name
+    if (e.isDirectory()) {
+      results.push(...findMdxFiles(path.join(dir, e.name), rel))
+    } else if (e.name.endsWith('.mdx')) {
+      const filePath = path.join(dir, e.name)
+      const route =
+        e.name === 'index.mdx'
+          ? basePath
+            ? `/${basePath}`
+            : '/'
+          : `/${rel.replace(/\.mdx$/, '')}`
+      results.push({ route, filePath })
+    }
+  }
+  return results
+}
+
+function toPublicUrl(route) {
+  if (route === '/ipa') return '/api'
+  if (route.startsWith('/ipa/')) return `/api/${route.slice(5)}`
+  return route
+}
+
+function escapeXml(s) {
+  return s
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&apos;')
+}
+
+const entries = findMdxFiles(PAGES_DIR)
+  .map(({ route, filePath }) => ({
+    url: toPublicUrl(route),
+    lastmod: getGitLastModified(filePath),
+  }))
+  .sort((a, b) => a.url.localeCompare(b.url))
+
+const seen = new Set()
+const unique = entries.filter((e) => {
+  if (seen.has(e.url)) return false
+  seen.add(e.url)
+  return true
+})
+
+const urls = unique
+  .map(({ url, lastmod }) => {
+    const loc = escapeXml(`${BASE_URL}${url}`)
+    const lastmodTag = lastmod ? `\n    <lastmod>${lastmod}</lastmod>` : ''
+    return `  <url>\n    <loc>${loc}</loc>${lastmodTag}\n  </url>`
+  })
+  .join('\n')
+
+const xml = `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${urls}
+</urlset>
+`
+
+fs.mkdirSync(path.dirname(OUT_PATH), { recursive: true })
+fs.writeFileSync(OUT_PATH, xml, 'utf8')
+console.log('Generated', OUT_PATH, 'with', unique.length, 'URLs')