Merge pull request #147 from fosrl/dev

Dev

Dockerfile

@@ -6,6 +6,8 @@ RUN npm ci
 
 FROM node:24-alpine AS build
 WORKDIR /app
+# sitemap lastmod comes from git history
+RUN apk add --no-cache git
 ENV NEXT_TELEMETRY_DISABLED=1
 ARG NEXT_PUBLIC_SITE_URL=https://docs.pangolin.net
 ENV NEXT_PUBLIC_SITE_URL=$NEXT_PUBLIC_SITE_URL

app/(docs)/[[...slug]]/page.tsx

@@ -5,8 +5,8 @@ import {
   DocsPage,
   DocsTitle,
   MarkdownCopyButton,
-  ViewOptionsPopover,
 } from 'fumadocs-ui/layouts/notebook/page';
+import { ViewOptionsPopover } from '@/components/view-options';
 import { notFound } from 'next/navigation';
 import { getMDXComponents } from '@/components/mdx';
 import type { Metadata } from 'next';
@@ -15,6 +15,7 @@ import { appName, docsDir, getPageMarkdownUrl, gitConfig, siteUrl } from '@/lib/
 import { AskAIAboutPage } from '@/components/ai/ask-page';
 import { SiteFooter } from '@/components/site-footer';
 import { AskBar } from '@/components/ai/ask-bar';
+import { pageJsonLd } from '@/lib/json-ld';
 
 export default async function Page(props: PageProps<'/[[...slug]]'>) {
   const params = await props.params;
@@ -25,20 +26,22 @@ export default async function Page(props: PageProps<'/[[...slug]]'>) {
   const markdownUrl = getPageMarkdownUrl(page).url;
 
   return (
-    <DocsPage
+    <>
-      toc={page.data.toc}
+      <script type="application/ld+json" dangerouslySetInnerHTML={{ __html: pageJsonLd(page) }} />
-      full={page.data.full}
+      <DocsPage
-      tableOfContent={{ style: 'clerk' }}
+        toc={page.data.toc}
-      // rendered after the previous / next page links
+        full={page.data.full}
-      footer={{
+        tableOfContent={{ style: 'clerk' }}
-        children: (
+        // rendered after the previous / next page links
-          <>
+        footer={{
-            <SiteFooter />
+          children: (
-            <AskBar />
+            <>
-          </>
+              <SiteFooter />
-        ),
+              <AskBar />
-      }}
+            </>
-    >
+          ),
+        }}
+      >
       <DocsTitle>{page.data.title}</DocsTitle>
       <DocsDescription className="mb-0">{page.data.description}</DocsDescription>
       <div className="flex flex-row flex-wrap gap-2 items-center border-b pb-6">
@@ -58,7 +61,8 @@ export default async function Page(props: PageProps<'/[[...slug]]'>) {
           })}
         />
       </DocsBody>
-    </DocsPage>
+      </DocsPage>
+    </>
   );
 }
 
@@ -77,7 +81,10 @@ export async function generateMetadata(props: PageProps<'/[[...slug]]'>): Promis
     description: page.data.description,
     alternates: {
       canonical: page.url,
-      types: { 'text/markdown': getPageMarkdownUrl(page).url },
+      types: {
+        'text/markdown': getPageMarkdownUrl(page).url,
+        'application/xml': '/sitemap.xml',
+      },
     },
     openGraph: {
       images: '/images/home-social-graph.png',

app/(docs)/not-found.tsx

@@ -0,0 +1,30 @@
+import type { Metadata } from 'next';
+import type { ReactNode } from 'react';
+import type * as PageTree from 'fumadocs-core/page-tree';
+import { NotFoundPage, type NotFoundLink } from '@/components/not-found';
+import { getNavigationTree } from '@/lib/navigation';
+
+export const metadata: Metadata = {
+  title: 'Page Not Found',
+};
+
+function label(name: ReactNode) {
+  return typeof name === 'string' ? name : null;
+}
+
+function flatten(nodes: PageTree.Node[]): NotFoundLink[] {
+  const links: NotFoundLink[] = [];
+  for (const node of nodes) {
+    if (node.type === 'page') {
+      const title = label(node.name);
+      if (title && node.url) links.push({ title, url: node.url });
+      continue;
+    }
+    if (node.type === 'folder') links.push(...flatten(node.children));
+  }
+  return links;
+}
+
+export default function NotFound() {
+  return <NotFoundPage pages={flatten(getNavigationTree().children)} />;
+}

app/global.css

@@ -323,7 +323,7 @@ body {
 .pg-ask-ai-btn svg {
   width: 1rem;
   height: 1rem;
-  color: var(--color-pg-orange);
+  color: currentColor;
 }
 
 /* logo: fixed height, width from the image's aspect ratio (never stretched) */

app/layout.tsx

@@ -15,6 +15,7 @@ export const metadata: Metadata = {
   title: { template: `%s | ${appName}`, default: appName },
   description: siteDescription,
   appleWebApp: { title: appName },
+  alternates: { types: { 'application/xml': '/sitemap.xml' } },
   openGraph: { siteName: appName, images: '/images/home-social-graph.png' },
   twitter: { card: 'summary_large_image' },
 };

app/llms.txt/route.ts

@@ -14,6 +14,7 @@ export async function GET() {
     `> ${siteDescription}`,
     '',
     `The full documentation as a single file is available at ${siteUrl}/llms-full.txt. Append \`.md\` to any page URL to get its Markdown source.`,
+    `An MCP server for searching and reading this documentation is available at ${siteUrl}/mcp.`,
     '',
   ];
   const listed = new Set<string>();

app/robots.ts

@@ -0,0 +1,21 @@
+import type { MetadataRoute } from 'next';
+import { siteUrl } from '@/lib/shared';
+
+/**
+ * Mintlify's generated robots.txt: AI crawlers may train, search, and answer
+ * from the docs, while Next.js build assets stay out of the index.
+ * https://www.mintlify.com/docs/optimize/seo
+ */
+export default function robots(): MetadataRoute.Robots {
+  return {
+    rules: {
+      userAgent: '*',
+      allow: '/_next/image',
+      disallow: ['/cdn-cgi/', '/_next/'],
+      other: {
+        'Content-Signal': 'ai-train=yes, search=yes, ai-input=yes',
+      },
+    },
+    sitemap: `${siteUrl}/sitemap.xml`,
+  };
+}

app/sitemap.ts

@@ -0,0 +1,18 @@
+import type { MetadataRoute } from 'next';
+import { pageLastModified } from '@/lib/last-modified';
+import { source } from '@/lib/source';
+import { siteUrl } from '@/lib/shared';
+
+/**
+ * Same shape as a Mintlify docs sitemap (`/docs/sitemap.xml` on Infisical):
+ * one `<loc>` and `<lastmod>` per published page.
+ */
+export default function sitemap(): MetadataRoute.Sitemap {
+  return source
+    .getPages()
+    .map((page) => ({
+      url: `${siteUrl}${page.url}`,
+      lastModified: pageLastModified(page),
+    }))
+    .sort((a, b) => a.url.localeCompare(b.url));
+}

components/not-found.tsx

@@ -0,0 +1,71 @@
+'use client';
+
+import Link from 'fumadocs-core/link';
+import { usePathname } from 'next/navigation';
+
+export type NotFoundLink = {
+  title: string;
+  url: string;
+};
+
+const fallbackUrls = ['/', '/about/how-pangolin-works', '/self-host/quick-install'];
+
+function segments(path: string) {
+  return path.split('/').filter(Boolean);
+}
+
+/** Pages that share the most of the requested path, same idea as Mintlify's suggestions. */
+function suggest(pages: NotFoundLink[], pathname: string) {
+  const wanted = segments(pathname);
+  const ranked = pages
+    .map((page, index) => {
+      const parts = segments(page.url);
+      let prefix = 0;
+      while (prefix < parts.length && prefix < wanted.length && parts[prefix] === wanted[prefix]) {
+        prefix += 1;
+      }
+      return {
+        page,
+        index,
+        score: prefix === 0 ? 0 : prefix * 100 - Math.abs(parts.length - wanted.length),
+      };
+    })
+    .filter((item) => item.score > 0)
+    .sort((a, b) => b.score - a.score || a.index - b.index);
+
+  const picks = ranked.slice(0, 3).map((item) => item.page);
+  if (picks.length > 0) return picks;
+
+  return fallbackUrls.flatMap((url) => {
+    const page = pages.find((item) => item.url === url);
+    return page ? [page] : [];
+  });
+}
+
+export function NotFoundPage({ pages }: { pages: NotFoundLink[] }) {
+  const suggestions = suggest(pages, usePathname() || '/');
+
+  return (
+    <div className="flex min-h-[calc(100dvh-var(--fd-docs-row-3,6.5rem))] flex-col items-center justify-center px-6 py-24 text-center [grid-area:main] lg:[grid-column:3/5]">
+      <p className="text-5xl font-medium text-pg-orange">404</p>
+      <h1 className="mt-4 text-2xl font-medium tracking-tight">Page Not Found</h1>
+      <p className="mt-4 max-w-sm text-sm leading-6 text-fd-muted-foreground">
+        We couldn&apos;t find the page. Maybe you were looking for one of these pages below?
+      </p>
+      {suggestions.length > 0 && (
+        <ul className="mt-8 flex flex-col items-center gap-3">
+          {suggestions.map((page) => (
+            <li key={page.url}>
+              <Link
+                href={page.url}
+                className="text-sm font-medium text-pg-orange hover:underline hover:underline-offset-4"
+              >
+                {page.title}
+              </Link>
+            </li>
+          ))}
+        </ul>
+      )}
+    </div>
+  );
+}

components/view-options.tsx

@@ -0,0 +1,129 @@
+'use client';
+
+import { buttonVariants } from 'fumadocs-ui/components/ui/button';
+import { Popover, PopoverContent, PopoverTrigger } from 'fumadocs-ui/components/ui/popover';
+import { ChevronDown, ExternalLink, Text } from 'lucide-react';
+import { useMemo, type ComponentProps, type ReactNode } from 'react';
+import { cn } from '@/lib/cn';
+import { GitHubIcon } from './icons';
+
+type ViewOptionsPopoverProps = ComponentProps<typeof PopoverTrigger> & {
+  markdownUrl?: string;
+  githubUrl?: string;
+  /** Canonical page URL the AI prompts ask to read. */
+  pageUrl: string;
+};
+
+type Option = {
+  title: string;
+  href: string;
+  icon: ReactNode;
+};
+
+export function ViewOptionsPopover({
+  markdownUrl,
+  githubUrl,
+  pageUrl,
+  children,
+  className,
+  ...props
+}: ViewOptionsPopoverProps) {
+  const items = useMemo(() => {
+    const prompt = `Read ${pageUrl}, I want to ask questions about it.`;
+    const options: Option[] = [];
+
+    if (githubUrl) {
+      options.push({
+        title: 'Open in GitHub',
+        href: githubUrl,
+        icon: <GitHubIcon />,
+      });
+    }
+
+    if (markdownUrl) {
+      options.push({
+        title: 'View as Markdown',
+        href: markdownUrl,
+        icon: <Text />,
+      });
+    }
+
+    options.push(
+      {
+        title: 'Open in ChatGPT',
+        href: `https://chatgpt.com/?${new URLSearchParams({ prompt, hints: 'search' })}`,
+        icon: <OpenAIIcon />,
+      },
+      {
+        title: 'Open in Claude',
+        href: `https://claude.ai/new?${new URLSearchParams({ q: prompt })}`,
+        icon: <ClaudeIcon />,
+      },
+      {
+        title: 'Open in Gemini',
+        // gemini.google.com ignores prompt query params; AI Mode is Gemini and reads `q`
+        href: `https://www.google.com/search?${new URLSearchParams({ udm: '50', q: prompt })}`,
+        icon: <GeminiIcon />,
+      },
+    );
+
+    return options;
+  }, [githubUrl, markdownUrl, pageUrl]);
+
+  return (
+    <Popover>
+      <PopoverTrigger
+        {...props}
+        className={(state) =>
+          cn(
+            buttonVariants({ variant: 'secondary', size: 'sm' }),
+            'gap-2 data-[popup-open]:bg-fd-accent data-[popup-open]:text-fd-accent-foreground',
+            typeof className === 'function' ? className(state) : className,
+          )
+        }
+      >
+        {children ?? 'Open'}
+        <ChevronDown className="size-3.5 text-fd-muted-foreground" />
+      </PopoverTrigger>
+      <PopoverContent className="flex flex-col">
+        {items.map((item) => (
+          <a
+            key={item.href}
+            href={item.href}
+            rel="noreferrer noopener"
+            target="_blank"
+            className="text-sm p-2 rounded-lg inline-flex items-center gap-2 hover:text-fd-accent-foreground hover:bg-fd-accent [&_svg]:size-4"
+          >
+            {item.icon}
+            {item.title}
+            <ExternalLink className="text-fd-muted-foreground size-3.5 ms-auto" />
+          </a>
+        ))}
+      </PopoverContent>
+    </Popover>
+  );
+}
+
+function OpenAIIcon() {
+  return (
+    <svg role="img" viewBox="0 0 24 24" fill="currentColor" aria-hidden>
+      <path d="M22.282 9.821a5.985 5.985 0 0 0-.516-4.91 6.046 6.046 0 0 0-6.51-2.9A6.065 6.065 0 0 0 4.981 4.18a5.985 5.985 0 0 0-3.998 2.9 6.046 6.046 0 0 0 .743 7.097 5.98 5.98 0 0 0 .51 4.911 6.051 6.051 0 0 0 6.515 2.9A5.985 5.985 0 0 0 13.26 24a6.056 6.056 0 0 0 5.772-4.206 5.99 5.99 0 0 0 3.997-2.9 6.056 6.056 0 0 0-.747-7.073zM13.26 22.43a4.476 4.476 0 0 1-2.876-1.04l.141-.081 4.779-2.758a.795.795 0 0 0 .392-.681v-6.737l2.02 1.168a.071.071 0 0 1 .038.052v5.583a4.504 4.504 0 0 1-4.494 4.494zM3.6 18.304a4.47 4.47 0 0 1-.535-3.014l.142.085 4.783 2.759a.771.771 0 0 0 .78 0l5.843-3.369v2.332a.08.08 0 0 1-.033.062L9.74 19.95a4.5 4.5 0 0 1-6.14-1.646zM2.34 7.896a4.485 4.485 0 0 1 2.366-1.973V11.6a.766.766 0 0 0 .388.676l5.815 3.355-2.02 1.168a.076.076 0 0 1-.071 0l-4.83-2.786A4.504 4.504 0 0 1 2.34 7.872zm16.597 3.855-5.833-3.387L15.119 7.2a.076.076 0 0 1 .071 0l4.83 2.791a4.494 4.494 0 0 1-.676 8.105v-5.678a.79.79 0 0 0-.407-.667zm2.01-3.023-.141-.085-4.774-2.782a.776.776 0 0 0-.785 0L9.409 9.23V6.897a.066.066 0 0 1 .028-.062l4.83-2.787a4.5 4.5 0 0 1 6.68 4.66zM8.307 12.863l-2.02-1.164a.08.08 0 0 1-.038-.057V6.074a4.5 4.5 0 0 1 7.375-3.453l-.142.08L8.704 5.46a.795.795 0 0 0-.393.681zm1.097-2.365 2.602-1.5 2.607 1.5v2.999l-2.597 1.5-2.607-1.5z" />
+    </svg>
+  );
+}
+
+function ClaudeIcon() {
+  return (
+    <svg fill="currentColor" role="img" viewBox="0 0 24 24" aria-hidden>
+      <path d="M17.304 3.541h-3.672l6.696 16.918H24zm-10.608 0L0 20.459h3.744l1.369-3.553h7.005l1.37 3.553h3.744L10.536 3.541zm-.372 10.223 2.292-5.946 2.291 5.946z" />
+    </svg>
+  );
+}
+
+function GeminiIcon() {
+  return (
+    <svg fill="currentColor" role="img" viewBox="0 0 24 24" aria-hidden>
+      <path d="M11.04 19.32Q12 21.51 12 24q0-2.49.93-4.68.96-2.19 2.58-3.81t3.81-2.55Q21.51 12 24 12q-2.49 0-4.68-.93a12.3 12.3 0 0 1-3.81-2.58 12.3 12.3 0 0 1-2.58-3.81Q12 2.49 12 0q0 2.49-.96 4.68-.93 2.19-2.55 3.81a12.3 12.3 0 0 1-3.81 2.58Q2.49 12 0 12q2.49 0 4.68.96 2.19.93 3.81 2.55t2.55 3.81" />
+    </svg>
+  );
+}

lib/json-ld.ts

@@ -0,0 +1,136 @@
+import navigation from '@/lib/navigation.json';
+import { pageLastModified } from '@/lib/last-modified';
+import type { Page } from '@/lib/source';
+import { appName, siteUrl } from '@/lib/shared';
+
+type NavItem = string | { group: string; pages: NavItem[] };
+type Crumb = { name: string; url: string };
+type IdRef = { '@id': string };
+
+/**
+ * Mintlify's per-page graph: Organization, WebSite, WebPage, BreadcrumbList,
+ * and TechArticle. `dateModified` is the same git date the sitemap uses.
+ */
+export function pageJsonLd(page: Page): string {
+  const url = absoluteUrl(page.url);
+  const dateModified = pageLastModified(page).toISOString();
+  const description = page.data.description || undefined;
+  const orgId = `${siteUrl}/#organization`;
+  const siteId = `${siteUrl}/#website`;
+  const pageId = `${url}#webpage`;
+  const crumbs = breadcrumb(page.slugs.length === 0 ? 'index' : page.slugs.join('/'), url, page.data.title);
+
+  const graph: Record<string, unknown>[] = [
+    {
+      '@type': 'Organization',
+      '@id': orgId,
+      name: appName,
+      url: siteUrl,
+      logo: { '@type': 'ImageObject', url: `${siteUrl}/logo/light.png` },
+    },
+    {
+      '@type': 'WebSite',
+      '@id': siteId,
+      name: appName,
+      url: siteUrl,
+      publisher: { '@id': orgId } satisfies IdRef,
+    },
+    {
+      '@type': 'WebPage',
+      '@id': pageId,
+      url,
+      name: page.data.title,
+      description,
+      dateModified,
+      isPartOf: { '@id': siteId } satisfies IdRef,
+      breadcrumb: crumbs ? { '@id': `${url}#breadcrumb` } : undefined,
+    },
+  ];
+
+  if (crumbs) {
+    graph.push({
+      '@type': 'BreadcrumbList',
+      '@id': `${url}#breadcrumb`,
+      itemListElement: crumbs.map((crumb, index) => ({
+        '@type': 'ListItem',
+        position: index + 1,
+        name: crumb.name,
+        item: crumb.url,
+      })),
+    });
+  }
+
+  graph.push({
+    '@type': ['Article', 'TechArticle'],
+    '@id': `${url}#article`,
+    headline: page.data.title,
+    name: page.data.title,
+    description,
+    url,
+    mainEntityOfPage: { '@id': pageId } satisfies IdRef,
+    dateModified,
+    publisher: { '@id': orgId } satisfies IdRef,
+    isPartOf: { '@id': siteId } satisfies IdRef,
+  });
+
+  return JSON.stringify({ '@context': 'https://schema.org', '@graph': graph }).replace(/</g, '\\u003c');
+}
+
+function absoluteUrl(url: string) {
+  return `${siteUrl}${url}`;
+}
+
+/**
+ * Two crumbs, same as Mintlify: the nearest nav group whose first page is not
+ * this one, then the page. The home page has no trail.
+ */
+function breadcrumb(navPath: string, url: string, title: string): Crumb[] | undefined {
+  const ancestors = findAncestors(navigation.groups as NavItem[], navPath) ?? [];
+
+  for (let i = ancestors.length - 1; i >= 0; i--) {
+    const group = ancestors[i];
+    if (!group) continue;
+    const leaf = firstLeaf(group.pages);
+    const parentUrl = leaf ? urlForNavPath(leaf) : undefined;
+    if (parentUrl && parentUrl !== url) {
+      return [
+        { name: group.group, url: parentUrl },
+        { name: title, url },
+      ];
+    }
+  }
+
+  if (url === absoluteUrl('/')) return undefined;
+  return [
+    { name: appName, url: absoluteUrl('/') },
+    { name: title, url },
+  ];
+}
+
+function urlForNavPath(navPath: string) {
+  const slugs = navPath === 'index' ? [] : navPath.split('/');
+  return absoluteUrl(slugs.length === 0 ? '/' : `/${slugs.join('/')}`);
+}
+
+function firstLeaf(items: NavItem[]): string | undefined {
+  for (const item of items) {
+    if (typeof item === 'string') return item;
+    const found = firstLeaf(item.pages);
+    if (found) return found;
+  }
+}
+
+function findAncestors(
+  items: NavItem[],
+  target: string,
+  trail: Extract<NavItem, { group: string }>[] = [],
+): Extract<NavItem, { group: string }>[] | undefined {
+  for (const item of items) {
+    if (typeof item === 'string') {
+      if (item === target) return trail;
+      continue;
+    }
+    const found = findAncestors(item.pages, target, [...trail, item]);
+    if (found) return found;
+  }
+}

lib/last-modified.ts

@@ -0,0 +1,48 @@
+import { execFileSync } from 'node:child_process';
+import { statSync } from 'node:fs';
+import path from 'node:path';
+import { docsDir } from './shared';
+
+type ContentFile = { path: string; absolutePath?: string };
+
+let cached: Map<string, Date> | undefined;
+
+/** Newest commit that touched each file under content/docs. Empty when git is unavailable. */
+function commitDates(): Map<string, Date> {
+  if (cached) return cached;
+
+  const dates = new Map<string, Date>();
+  try {
+    const out = execFileSync('git', ['log', '--format=COMMIT %cI', '--name-only', '--', docsDir], {
+      encoding: 'utf8',
+    });
+    let current: Date | undefined;
+    for (const line of out.split('\n')) {
+      if (line.startsWith('COMMIT ')) {
+        const date = new Date(line.slice('COMMIT '.length));
+        current = Number.isNaN(date.getTime()) ? undefined : date;
+      } else if (line && current && !dates.has(line)) {
+        dates.set(line, current);
+      }
+    }
+  } catch {
+    // Docker builds without git fall back to file mtime.
+  }
+
+  cached = dates;
+  return dates;
+}
+
+function mtime(file: string): Date {
+  try {
+    return statSync(file).mtime;
+  } catch {
+    return new Date();
+  }
+}
+
+/** Last time this page's MDX changed: newest commit, then file mtime. */
+export function pageLastModified(page: ContentFile): Date {
+  const file = `${docsDir}/${page.path}`;
+  return commitDates().get(file) ?? mtime(page.absolutePath ?? path.join(process.cwd(), file));
+}

proxy.ts

@@ -42,6 +42,6 @@ export default function proxy(request: NextRequest) {
 export const config = {
   // skip Next internals, API routes, generated files and static assets
   matcher: [
-    '/((?!_next/|api/|og/|llms|images/|logo/|favicon|icon|apple-icon|manifest|web-app-manifest).*)',
+    '/((?!_next/|api/|og/|llms|images/|logo/|favicon|icon|apple-icon|manifest|web-app-manifest|sitemap\\.xml|robots\\.txt).*)',
   ],
 };