Repository: promptfoo/promptfoo
Pull request #3600: feat(docs): Add llms.txt plugin and convert config to TypeScript
Author: login: mldangelo
Created: 2025-04-07T04:39:10Z | Merged: 2025-04-07T04:46:38Z

Adds a custom llms.txt plugin that generates machine-readable documentation files and converts Docusaurus config to TypeScript. The plugin creates both full content (llms-full.txt) and structured index (llms.txt) files that can be used by LLM tools for better documentation search and understanding.


Inspiration - Based on Prisma's implementation: https://github.com/facebook/docusaurus/issues/10899


Diff:
diff --git a/site/README.md b/site/README.md
index 8af373b2da4f..d8d50b10f3c3 100644
--- a/site/README.md
+++ b/site/README.md
@@ -23,3 +23,16 @@ npm run build
 ```
 
 This command generates static content into the `build` directory and can be served using any static content hosting service.
+
+## LLMs.txt Plugin
+
+This website includes a custom Docusaurus plugin that generates two files during the build process:
+
+1. `llms-full.txt` - A concatenated file containing all content from all markdown and MDX files in the docs directory, separated by dividers
+2. `llms.txt` - A structured index of all documentation pages with titles, paths, and descriptions
+
+These files are generated to support LLM-based tools that need to analyze or search through the documentation.
+
+### Usage
+
+No additional configuration is needed. The plugin runs automatically during the build process, and the files are created in the root of the build output directory.
diff --git a/site/docusaurus.config.js b/site/docusaurus.config.js
deleted file mode 100644
index 4ef9bc595ce4..000000000000
--- a/site/docusaurus.config.js
+++ /dev/null
@@ -1,385 +0,0 @@
-// @ts-check
-// Note: type annotations allow type checking and IDEs autocompletion
-
-const { themes } = require('prism-react-renderer');
-const lightCodeTheme = themes.github;
-const darkCodeTheme = themes.duotoneDark;
-
-/** @type {import('@docusaurus/types').Config} */
-const config = {
-  title: 'promptfoo',
-  tagline: 'Test your prompts',
-  favicon: '/favicon.ico',
-
-  // Set the production url of your site here
-  url: 'https://www.promptfoo.dev',
-  // Set the /<baseUrl>/ pathname under which your site is served
-  // For GitHub pages deployment, it is often '/<projectName>/'
-  baseUrl: '/',
-
-  trailingSlash: true,
-
-  // GitHub pages deployment config.
-  // If you aren't using GitHub pages, you don't need these.
-  organizationName: 'promptfoo', // Usually your GitHub org/user name.
-  projectName: 'promptfoo', // Usually your repo name.
-
-  onBrokenLinks: 'throw',
-  onBrokenMarkdownLinks: 'throw',
-  onBrokenAnchors: 'throw',
-  // Even if you don't use internalization, you can use this field to set useful
-  // metadata like html lang. For example, if your site is Chinese, you may want
-  // to replace "en" with "zh-Hans".
-  i18n: {
-    defaultLocale: 'en',
-    locales: ['en'],
-  },
-
-  headTags: [
-    {
-      tagName: 'link',
-      attributes: {
-        rel: 'preconnect',
-        href: 'https://fonts.googleapis.com',
-      },
-    },
-    {
-      tagName: 'link',
-      attributes: {
-        rel: 'preconnect',
-        href: 'https://fonts.gstatic.com',
-        crossorigin: 'true',
-      },
-    },
-    {
-      tagName: 'link',
-      attributes: {
-        rel: 'stylesheet',
-        href: 'https://fonts.googleapis.com/css2?family=Inter:wght@100..900&display=swap',
-      },
-    },
-  ],
-
-  scripts: [
-    {
-      src: '/js/scripts.js',
-      async: true,
-    },
-  ],
-
-  presets: [
-    [
-      'classic',
-      /** @type {import('@docusaurus/preset-classic').Options} */
-      ({
-        docs: {
-          sidebarPath: require.resolve('./sidebars.js'),
-          // Remove this to remove the "edit this page" links.
-          editUrl: 'https://github.com/promptfoo/promptfoo/tree/main/site',
-          sidebarCollapsed: false,
-        },
-        blog: {
-          showReadingTime: false,
-          blogSidebarCount: 0,
-          // Please change this to your repo.
-          // Remove this to remove the "edit this page" links.
-          //editUrl:
-          //  'https://github.com/facebook/docusaurus/tree/main/packages/create-docusaurus/templates/shared/',
-        },
-        theme: {
-          customCss: require.resolve('./src/css/custom.css'),
-        },
-        gtag:
-          process.env.NODE_ENV === 'development'
-            ? undefined
-            : {
-                trackingID: 'G-3TS8QLZQ93',
-                anonymizeIP: true,
-              },
-      }),
-    ],
-  ],
-
-  themeConfig:
-    /** @type {import('@docusaurus/preset-classic').ThemeConfig} */
-    ({
-      image: 'img/thumbnail.png',
-      navbar: {
-        title: 'promptfoo',
-        logo: {
-          alt: 'promptfoo logo',
-          src: 'img/logo-panda.svg',
-        },
-        items: [
-          /*
-          {
-            type: 'docSidebar',
-            sidebarId: 'tutorialSidebar',
-            position: 'left',
-            label: 'Tutorial',
-          },
-          */
-          {
-            type: 'dropdown',
-            label: 'Product',
-            position: 'left',
-            items: [
-              {
-                to: '/security/',
-                label: 'Gen AI Security',
-              },
-              {
-                to: '/llm-vulnerability-scanner/',
-                label: 'Vulnerability Scanner',
-              },
-              {
-                to: '/docs/getting-started/',
-                label: 'LLM Evaluations',
-              },
-            ],
-          },
-          {
-            type: 'dropdown',
-            label: 'Company',
-            position: 'left',
-            items: [
-              {
-                href: '/about/',
-                label: 'About',
-              },
-              {
-                href: '/blog/',
-                label: 'Blog',
-              },
-              {
-                href: '/press/',
-                label: 'Press',
-              },
-              {
-                href: '/contact/',
-                label: 'Contact',
-              },
-              {
-                href: '/careers/',
-                label: 'Careers',
-              },
-            ],
-          },
-          {
-            type: 'dropdown',
-            label: 'Resources',
-            position: 'left',
-            items: [
-              {
-                href: '/docs/intro/',
-                label: 'Docs',
-              },
-              {
-                to: 'https://www.promptfoo.dev/docs/api-reference/',
-                label: 'API Reference',
-              },
-              {
-                to: 'https://www.promptfoo.dev/models/',
-                label: 'Foundation Model Reports',
-              },
-              {
-                href: 'https://github.com/promptfoo/promptfoo',
-                label: 'GitHub',
-              },
-              {
-                href: 'https://discord.gg/promptfoo',
-                label: 'Discord',
-              },
-            ],
-          },
-          { to: '/pricing/', label: 'Enterprise', position: 'left' },
-          {
-            href: 'https://github.com/promptfoo/promptfoo',
-            position: 'right',
-            className: 'header-github-link',
-            'aria-label': 'GitHub repository',
-          },
-          {
-            href: 'https://discord.gg/promptfoo',
-            position: 'right',
-            className: 'header-discord-link',
-            'aria-label': 'Discord community',
-          },
-        ],
-      },
-      footer: {
-        style: 'dark',
-        links: [
-          {
-            title: 'Product',
-            items: [
-              {
-                label: 'Docs',
-                to: '/docs/intro/',
-              },
-              {
-                label: 'Command Line',
-                to: '/docs/getting-started/',
-              },
-              {
-                label: 'Node Package',
-                to: '/docs/usage/node-package/',
-              },
-              {
-                label: 'Enterprise',
-                href: '/pricing/',
-              },
-              {
-                label: 'Status',
-                href: 'https://status.promptfoo.dev',
-              },
-            ],
-          },
-          {
-            title: 'Resources',
-            items: [
-              {
-                label: 'LLM Red Teaming',
-                to: '/docs/red-team',
-              },
-              {
-                label: 'Foundation Model Reports',
-                to: 'https://www.promptfoo.dev/models/',
-              },
-              {
-                label: 'Running Benchmarks',
-                to: '/docs/guides/llama2-uncensored-benchmark-ollama',
-              },
-              {
-                label: 'Evaluating Factuality',
-                to: '/docs/guides/factuality-eval',
-              },
-              {
-                label: 'Evaluating RAGs',
-                to: '/docs/guides/evaluate-rag',
-              },
-              {
-                label: 'Minimizing Hallucinations',
-                to: '/docs/guides/prevent-llm-hallucations',
-              },
-              {
-                label: 'Config Validator',
-                to: '/validator',
-              },
-            ],
-          },
-          {
-            title: 'Company',
-            items: [
-              {
-                label: 'About',
-                to: '/about/',
-              },
-              {
-                label: 'Blog',
-                to: '/blog/',
-              },
-              {
-                label: 'Press',
-                to: '/press/',
-              },
-              {
-                label: 'Contact',
-                to: '/contact/',
-              },
-              {
-                label: 'Careers',
-                to: '/careers/',
-              },
-            ],
-          },
-          {
-            title: 'Legal & Social',
-            items: [
-              {
-                label: 'GitHub',
-                href: 'https://github.com/promptfoo/promptfoo',
-              },
-              {
-                label: 'Discord',
-                href: 'https://discord.gg/promptfoo',
-              },
-              {
-                label: 'LinkedIn',
-                href: 'https://www.linkedin.com/company/promptfoo/',
-              },
-              {
-                label: 'Privacy Policy',
-                to: '/privacy/',
-              },
-              {
-                label: 'Terms of Service',
-                to: '/terms-of-service/',
-              },
-              {
-                html: `
-                <div style="position: relative; margin-top:8px">
-                  <span style="position: absolute; left: 65px; top: 25px; font-size: 10px; font-weight: bold; background-color: #25842c; padding: 2px 4px; border-radius: 4px;">In Progress</span>
-                  <img loading="lazy" src="/img/badges/soc2.png" alt="SOC2 Compliance in progress" style="width:80px; height: auto"/>
-                </div>
-                `,
-              },
-            ],
-          },
-        ],
-        copyright: `© ${new Date().getFullYear()} promptfoo`,
-      },
-      prism: {
-        theme: lightCodeTheme,
-        darkTheme: darkCodeTheme,
-        additionalLanguages: [
-          'bash',
-          'javascript',
-          'typescript',
-          'python',
-          'json',
-          'yaml',
-          'markup-templating',
-          'liquid',
-        ],
-      },
-      zoom: {
-        selector: '.markdown :not(em) > img:not(.no-zoom)',
-      },
-      algolia: {
-        // The application ID provided by Algolia
-        appId: 'VPUDC1V4TA',
-
-        // Public API key: it is safe to commit it
-        apiKey: '0b4fcfd05976eb0aaf4b7c51ec4fcd23',
-
-        indexName: 'promptfoo',
-      },
-    }),
-  plugins: [
-    require.resolve('docusaurus-plugin-image-zoom'),
-    [
-      '@docusaurus/plugin-client-redirects',
-      {
-        redirects: [
-          {
-            from: '/docs/category/troubleshooting',
-            to: '/docs/usage/troubleshooting',
-          },
-          {
-            from: '/docs/providers/palm',
-            to: '/docs/providers/google',
-          },
-        ],
-      },
-    ],
-  ],
-
-  // Mermaid diagram support
-  markdown: {
-    mermaid: true,
-  },
-  themes: ['@docusaurus/theme-mermaid'],
-};
-
-module.exports = config;
diff --git a/site/docusaurus.config.ts b/site/docusaurus.config.ts
new file mode 100644
index 000000000000..73b17a92caf4
--- /dev/null
+++ b/site/docusaurus.config.ts
@@ -0,0 +1,445 @@
+import { themes } from 'prism-react-renderer';
+import type * as Preset from '@docusaurus/preset-classic';
+import type { Config } from '@docusaurus/types';
+import * as fs from 'fs';
+import * as path from 'path';
+
+const lightCodeTheme = themes.github;
+const darkCodeTheme = themes.duotoneDark;
+
+const config: Config = {
+  title: 'promptfoo',
+  tagline: 'Test your prompts',
+  favicon: '/favicon.ico',
+
+  // Set the production url of your site here
+  url: 'https://www.promptfoo.dev',
+  // Set the /<baseUrl>/ pathname under which your site is served
+  // For GitHub pages deployment, it is often '/<projectName>/'
+  baseUrl: '/',
+
+  trailingSlash: true,
+
+  // GitHub pages deployment config.
+  // If you aren't using GitHub pages, you don't need these.
+  organizationName: 'promptfoo', // Usually your GitHub org/user name.
+  projectName: 'promptfoo', // Usually your repo name.
+
+  onBrokenLinks: 'throw',
+  onBrokenMarkdownLinks: 'throw',
+  onBrokenAnchors: 'throw',
+  // Even if you don't use internalization, you can use this field to set useful
+  // metadata like html lang. For example, if your site is Chinese, you may want
+  // to replace "en" with "zh-Hans".
+  i18n: {
+    defaultLocale: 'en',
+    locales: ['en'],
+  },
+
+  headTags: [
+    {
+      tagName: 'link',
+      attributes: {
+        rel: 'preconnect',
+        href: 'https://fonts.googleapis.com',
+      },
+    },
+    {
+      tagName: 'link',
+      attributes: {
+        rel: 'preconnect',
+        href: 'https://fonts.gstatic.com',
+        crossorigin: 'true',
+      },
+    },
+    {
+      tagName: 'link',
+      attributes: {
+        rel: 'stylesheet',
+        href: 'https://fonts.googleapis.com/css2?family=Inter:wght@100..900&display=swap',
+      },
+    },
+  ],
+
+  scripts: [
+    {
+      src: '/js/scripts.js',
+      async: true,
+    },
+  ],
+
+  presets: [
+    [
+      'classic',
+      {
+        docs: {
+          sidebarPath: require.resolve('./sidebars.js'),
+          // Remove this to remove the "edit this page" links.
+          editUrl: 'https://github.com/promptfoo/promptfoo/tree/main/site',
+          sidebarCollapsed: false,
+        },
+        blog: {
+          showReadingTime: false,
+          blogSidebarCount: 0,
+          // Please change this to your repo.
+          // Remove this to remove the "edit this page" links.
+          //editUrl:
+          //  'https://github.com/facebook/docusaurus/tree/main/packages/create-docusaurus/templates/shared/',
+        },
+        theme: {
+          customCss: require.resolve('./src/css/custom.css'),
+        },
+        gtag:
+          process.env.NODE_ENV === 'development'
+            ? undefined
+            : {
+                trackingID: 'G-3TS8QLZQ93',
+                anonymizeIP: true,
+              },
+      } satisfies Preset.Options,
+    ],
+  ],
+
+  themeConfig: {
+    image: 'img/thumbnail.png',
+    navbar: {
+      title: 'promptfoo',
+      logo: {
+        alt: 'promptfoo logo',
+        src: 'img/logo-panda.svg',
+      },
+      items: [
+        {
+          type: 'dropdown',
+          label: 'Product',
+          position: 'left',
+          items: [
+            {
+              to: '/security/',
+              label: 'Gen AI Security',
+            },
+            {
+              to: '/llm-vulnerability-scanner/',
+              label: 'Vulnerability Scanner',
+            },
+            {
+              to: '/docs/getting-started/',
+              label: 'LLM Evaluations',
+            },
+          ],
+        },
+        {
+          type: 'dropdown',
+          label: 'Company',
+          position: 'left',
+          items: [
+            {
+              href: '/about/',
+              label: 'About',
+            },
+            {
+              href: '/blog/',
+              label: 'Blog',
+            },
+            {
+              href: '/press/',
+              label: 'Press',
+            },
+            {
+              href: '/contact/',
+              label: 'Contact',
+            },
+            {
+              href: '/careers/',
+              label: 'Careers',
+            },
+          ],
+        },
+        {
+          type: 'dropdown',
+          label: 'Resources',
+          position: 'left',
+          items: [
+            {
+              href: '/docs/intro/',
+              label: 'Docs',
+            },
+            {
+              to: 'https://www.promptfoo.dev/docs/api-reference/',
+              label: 'API Reference',
+            },
+            {
+              to: 'https://www.promptfoo.dev/models/',
+              label: 'Foundation Model Reports',
+            },
+            {
+              href: 'https://github.com/promptfoo/promptfoo',
+              label: 'GitHub',
+            },
+            {
+              href: 'https://discord.gg/promptfoo',
+              label: 'Discord',
+            },
+          ],
+        },
+        { to: '/pricing/', label: 'Enterprise', position: 'left' },
+        {
+          href: 'https://github.com/promptfoo/promptfoo',
+          position: 'right',
+          className: 'header-github-link',
+          'aria-label': 'GitHub repository',
+        },
+        {
+          href: 'https://discord.gg/promptfoo',
+          position: 'right',
+          className: 'header-discord-link',
+          'aria-label': 'Discord community',
+        },
+      ],
+    },
+    footer: {
+      style: 'dark',
+      links: [
+        {
+          title: 'Product',
+          items: [
+            {
+              label: 'Docs',
+              to: '/docs/intro/',
+            },
+            {
+              label: 'Command Line',
+              to: '/docs/getting-started/',
+            },
+            {
+              label: 'Node Package',
+              to: '/docs/usage/node-package/',
+            },
+            {
+              label: 'Enterprise',
+              href: '/pricing/',
+            },
+            {
+              label: 'Status',
+              href: 'https://status.promptfoo.dev',
+            },
+          ],
+        },
+        {
+          title: 'Resources',
+          items: [
+            {
+              label: 'LLM Red Teaming',
+              to: '/docs/red-team',
+            },
+            {
+              label: 'Foundation Model Reports',
+              to: 'https://www.promptfoo.dev/models/',
+            },
+            {
+              label: 'Running Benchmarks',
+              to: '/docs/guides/llama2-uncensored-benchmark-ollama',
+            },
+            {
+              label: 'Evaluating Factuality',
+              to: '/docs/guides/factuality-eval',
+            },
+            {
+              label: 'Evaluating RAGs',
+              to: '/docs/guides/evaluate-rag',
+            },
+            {
+              label: 'Minimizing Hallucinations',
+              to: '/docs/guides/prevent-llm-hallucations',
+            },
+            {
+              label: 'Config Validator',
+              to: '/validator',
+            },
+          ],
+        },
+        {
+          title: 'Company',
+          items: [
+            {
+              label: 'About',
+              to: '/about/',
+            },
+            {
+              label: 'Blog',
+              to: '/blog/',
+            },
+            {
+              label: 'Press',
+              to: '/press/',
+            },
+            {
+              label: 'Contact',
+              to: '/contact/',
+            },
+            {
+              label: 'Careers',
+              to: '/careers/',
+            },
+          ],
+        },
+        {
+          title: 'Legal & Social',
+          items: [
+            {
+              label: 'GitHub',
+              href: 'https://github.com/promptfoo/promptfoo',
+            },
+            {
+              label: 'Discord',
+              href: 'https://discord.gg/promptfoo',
+            },
+            {
+              label: 'LinkedIn',
+              href: 'https://www.linkedin.com/company/promptfoo/',
+            },
+            {
+              label: 'Privacy Policy',
+              to: '/privacy/',
+            },
+            {
+              label: 'Terms of Service',
+              to: '/terms-of-service/',
+            },
+            {
+              html: `
+                <div style="position: relative; margin-top:8px">
+                  <span style="position: absolute; left: 65px; top: 25px; font-size: 10px; font-weight: bold; background-color: #25842c; padding: 2px 4px; border-radius: 4px;">In Progress</span>
+                  <img loading="lazy" src="/img/badges/soc2.png" alt="SOC2 Compliance in progress" style="width:80px; height: auto"/>
+                </div>
+                `,
+            },
+          ],
+        },
+      ],
+      copyright: `© ${new Date().getFullYear()} promptfoo`,
+    },
+    prism: {
+      theme: lightCodeTheme,
+      darkTheme: darkCodeTheme,
+      additionalLanguages: [
+        'bash',
+        'javascript',
+        'typescript',
+        'python',
+        'json',
+        'yaml',
+        'markup-templating',
+        'liquid',
+      ],
+    },
+    zoom: {
+      selector: '.markdown :not(em) > img:not(.no-zoom)',
+    },
+    algolia: {
+      // The application ID provided by Algolia
+      appId: 'VPUDC1V4TA',
+
+      // Public API key: it is safe to commit it
+      apiKey: '0b4fcfd05976eb0aaf4b7c51ec4fcd23',
+
+      indexName: 'promptfoo',
+    },
+  } satisfies Preset.ThemeConfig,
+
+  plugins: [
+    require.resolve('docusaurus-plugin-image-zoom'),
+    [
+      '@docusaurus/plugin-client-redirects',
+      {
+        redirects: [
+          {
+            from: '/docs/category/troubleshooting',
+            to: '/docs/usage/troubleshooting',
+          },
+          {
+            from: '/docs/providers/palm',
+            to: '/docs/providers/google',
+          },
+        ],
+      },
+    ],
+    // Define the llms.txt plugin inline similar to the Prisma example
+    async function llmsTxtPlugin(context) {
+      return {
+        name: 'llms-txt-plugin',
+        loadContent: async () => {
+          const { siteDir } = context;
+          const docsDir = path.join(siteDir, 'docs');
+          const allMdx: string[] = [];
+
+          // Recursive function to get all mdx/md files
+          const getMdFiles = async (dir: string): Promise<void> => {
+            const entries = await fs.promises.readdir(dir, { withFileTypes: true });
+
+            for (const entry of entries) {
+              const fullPath = path.join(dir, entry.name);
+              if (entry.isDirectory()) {
+                await getMdFiles(fullPath);
+              } else if (entry.name.endsWith('.md') || entry.name.endsWith('.mdx')) {
+                const content = await fs.promises.readFile(fullPath, 'utf8');
+                allMdx.push(content);
+              }
+            }
+          };
+
+          await getMdFiles(docsDir);
+          return { allMdx };
+        },
+        postBuild: async ({ content, routesPaths, outDir }) => {
+          // Type assertion to handle TypeScript type checking
+          const pluginContent = content as { allMdx: string[] };
+          const { allMdx } = pluginContent;
+
+          // Write concatenated MDX content
+          const concatenatedPath = path.join(outDir, 'llms-full.txt');
+          await fs.promises.writeFile(concatenatedPath, allMdx.join('\n\n---\n\n'));
+
+          // Process routes - use routesPaths which is a string[] of all routes
+          const docsRoutes: string[] = [];
+
+          // Filter for docs paths and generate entries
+          for (const routePath of routesPaths) {
+            if (routePath.startsWith('/docs/')) {
+              // Extract a title from the route path as fallback
+              const pathParts = routePath.split('/').filter(Boolean);
+              const lastPart = pathParts[pathParts.length - 1];
+              const title = lastPart
+                .split('-')
+                .map((part) => part.charAt(0).toUpperCase() + part.slice(1))
+                .join(' ');
+
+              docsRoutes.push(`- [${title}](${routePath})`);
+            }
+          }
+
+          // Build up llms.txt file
+          const llmsTxt = `# ${context.siteConfig.title}\n\n## Docs\n\n${docsRoutes.join('\n')}`;
+
+          // Write llms.txt file
+          const llmsTxtPath = path.join(outDir, 'llms.txt');
+          try {
+            fs.writeFileSync(llmsTxtPath, llmsTxt);
+            console.log('Successfully created llms.txt and llms-full.txt files.');
+          } catch (err) {
+            console.error('Error writing llms.txt file:', err);
+            throw err;
+          }
+        },
+      };
+    },
+  ],
+
+  // Mermaid diagram support
+  markdown: {
+    mermaid: true,
+  },
+  themes: ['@docusaurus/theme-mermaid'],
+};
+
+export default config;


Reviews:
user: login: ellipsis-dev[bot]
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-07T04:41:34Z

Review comments:
diff_hunk: @@ -0,0 +1,445 @@
+import { themes } from 'prism-react-renderer';
+import type * as Preset from '@docusaurus/preset-classic';
+import type { Config } from '@docusaurus/types';
+import * as fs from 'fs';
+import * as path from 'path';
+
+const lightCodeTheme = themes.github;
+const darkCodeTheme = themes.duotoneDark;
+
+const config: Config = {
+  title: 'promptfoo',
+  tagline: 'Test your prompts',
+  favicon: '/favicon.ico',
+
+  // Set the production url of your site here
+  url: 'https://www.promptfoo.dev',
+  // Set the /<baseUrl>/ pathname under which your site is served
+  // For GitHub pages deployment, it is often '/<projectName>/'
+  baseUrl: '/',
+
+  trailingSlash: true,
+
+  // GitHub pages deployment config.
+  // If you aren't using GitHub pages, you don't need these.
+  organizationName: 'promptfoo', // Usually your GitHub org/user name.
+  projectName: 'promptfoo', // Usually your repo name.
+
+  onBrokenLinks: 'throw',
+  onBrokenMarkdownLinks: 'throw',
+  onBrokenAnchors: 'throw',
+  // Even if you don't use internalization, you can use this field to set useful
+  // metadata like html lang. For example, if your site is Chinese, you may want
+  // to replace "en" with "zh-Hans".
+  i18n: {
+    defaultLocale: 'en',
+    locales: ['en'],
+  },
+
+  headTags: [
+    {
+      tagName: 'link',
+      attributes: {
+        rel: 'preconnect',
+        href: 'https://fonts.googleapis.com',
+      },
+    },
+    {
+      tagName: 'link',
+      attributes: {
+        rel: 'preconnect',
+        href: 'https://fonts.gstatic.com',
+        crossorigin: 'true',
+      },
+    },
+    {
+      tagName: 'link',
+      attributes: {
+        rel: 'stylesheet',
+        href: 'https://fonts.googleapis.com/css2?family=Inter:wght@100..900&display=swap',
+      },
+    },
+  ],
+
+  scripts: [
+    {
+      src: '/js/scripts.js',
+      async: true,
+    },
+  ],
+
+  presets: [
+    [
+      'classic',
+      {
+        docs: {
+          sidebarPath: require.resolve('./sidebars.js'),
+          // Remove this to remove the "edit this page" links.
+          editUrl: 'https://github.com/promptfoo/promptfoo/tree/main/site',
+          sidebarCollapsed: false,
+        },
+        blog: {
+          showReadingTime: false,
+          blogSidebarCount: 0,
+          // Please change this to your repo.
+          // Remove this to remove the "edit this page" links.
+          //editUrl:
+          //  'https://github.com/facebook/docusaurus/tree/main/packages/create-docusaurus/templates/shared/',
+        },
+        theme: {
+          customCss: require.resolve('./src/css/custom.css'),
+        },
+        gtag:
+          process.env.NODE_ENV === 'development'
+            ? undefined
+            : {
+                trackingID: 'G-3TS8QLZQ93',
+                anonymizeIP: true,
+              },
+      } satisfies Preset.Options,
+    ],
+  ],
+
+  themeConfig: {
+    image: 'img/thumbnail.png',
+    navbar: {
+      title: 'promptfoo',
+      logo: {
+        alt: 'promptfoo logo',
+        src: 'img/logo-panda.svg',
+      },
+      items: [
+        {
+          type: 'dropdown',
+          label: 'Product',
+          position: 'left',
+          items: [
+            {
+              to: '/security/',
+              label: 'Gen AI Security',
+            },
+            {
+              to: '/llm-vulnerability-scanner/',
+              label: 'Vulnerability Scanner',
+            },
+            {
+              to: '/docs/getting-started/',
+              label: 'LLM Evaluations',
+            },
+          ],
+        },
+        {
+          type: 'dropdown',
+          label: 'Company',
+          position: 'left',
+          items: [
+            {
+              href: '/about/',
+              label: 'About',
+            },
+            {
+              href: '/blog/',
+              label: 'Blog',
+            },
+            {
+              href: '/press/',
+              label: 'Press',
+            },
+            {
+              href: '/contact/',
+              label: 'Contact',
+            },
+            {
+              href: '/careers/',
+              label: 'Careers',
+            },
+          ],
+        },
+        {
+          type: 'dropdown',
+          label: 'Resources',
+          position: 'left',
+          items: [
+            {
+              href: '/docs/intro/',
+              label: 'Docs',
+            },
+            {
+              to: 'https://www.promptfoo.dev/docs/api-reference/',
+              label: 'API Reference',
+            },
+            {
+              to: 'https://www.promptfoo.dev/models/',
+              label: 'Foundation Model Reports',
+            },
+            {
+              href: 'https://github.com/promptfoo/promptfoo',
+              label: 'GitHub',
+            },
+            {
+              href: 'https://discord.gg/promptfoo',
+              label: 'Discord',
+            },
+          ],
+        },
+        { to: '/pricing/', label: 'Enterprise', position: 'left' },
+        {
+          href: 'https://github.com/promptfoo/promptfoo',
+          position: 'right',
+          className: 'header-github-link',
+          'aria-label': 'GitHub repository',
+        },
+        {
+          href: 'https://discord.gg/promptfoo',
+          position: 'right',
+          className: 'header-discord-link',
+          'aria-label': 'Discord community',
+        },
+      ],
+    },
+    footer: {
+      style: 'dark',
+      links: [
+        {
+          title: 'Product',
+          items: [
+            {
+              label: 'Docs',
+              to: '/docs/intro/',
+            },
+            {
+              label: 'Command Line',
+              to: '/docs/getting-started/',
+            },
+            {
+              label: 'Node Package',
+              to: '/docs/usage/node-package/',
+            },
+            {
+              label: 'Enterprise',
+              href: '/pricing/',
+            },
+            {
+              label: 'Status',
+              href: 'https://status.promptfoo.dev',
+            },
+          ],
+        },
+        {
+          title: 'Resources',
+          items: [
+            {
+              label: 'LLM Red Teaming',
+              to: '/docs/red-team',
+            },
+            {
+              label: 'Foundation Model Reports',
+              to: 'https://www.promptfoo.dev/models/',
+            },
+            {
+              label: 'Running Benchmarks',
+              to: '/docs/guides/llama2-uncensored-benchmark-ollama',
+            },
+            {
+              label: 'Evaluating Factuality',
+              to: '/docs/guides/factuality-eval',
+            },
+            {
+              label: 'Evaluating RAGs',
+              to: '/docs/guides/evaluate-rag',
+            },
+            {
+              label: 'Minimizing Hallucinations',
+              to: '/docs/guides/prevent-llm-hallucations',
+            },
+            {
+              label: 'Config Validator',
+              to: '/validator',
+            },
+          ],
+        },
+        {
+          title: 'Company',
+          items: [
+            {
+              label: 'About',
+              to: '/about/',
+            },
+            {
+              label: 'Blog',
+              to: '/blog/',
+            },
+            {
+              label: 'Press',
+              to: '/press/',
+            },
+            {
+              label: 'Contact',
+              to: '/contact/',
+            },
+            {
+              label: 'Careers',
+              to: '/careers/',
+            },
+          ],
+        },
+        {
+          title: 'Legal & Social',
+          items: [
+            {
+              label: 'GitHub',
+              href: 'https://github.com/promptfoo/promptfoo',
+            },
+            {
+              label: 'Discord',
+              href: 'https://discord.gg/promptfoo',
+            },
+            {
+              label: 'LinkedIn',
+              href: 'https://www.linkedin.com/company/promptfoo/',
+            },
+            {
+              label: 'Privacy Policy',
+              to: '/privacy/',
+            },
+            {
+              label: 'Terms of Service',
+              to: '/terms-of-service/',
+            },
+            {
+              html: `
+                <div style="position: relative; margin-top:8px">
+                  <span style="position: absolute; left: 65px; top: 25px; font-size: 10px; font-weight: bold; background-color: #25842c; padding: 2px 4px; border-radius: 4px;">In Progress</span>
+                  <img loading="lazy" src="/img/badges/soc2.png" alt="SOC2 Compliance in progress" style="width:80px; height: auto"/>
+                </div>
+                `,
+            },
+          ],
+        },
+      ],
+      copyright: `© ${new Date().getFullYear()} promptfoo`,
+    },
+    prism: {
+      theme: lightCodeTheme,
+      darkTheme: darkCodeTheme,
+      additionalLanguages: [
+        'bash',
+        'javascript',
+        'typescript',
+        'python',
+        'json',
+        'yaml',
+        'markup-templating',
+        'liquid',
+      ],
+    },
+    zoom: {
+      selector: '.markdown :not(em) > img:not(.no-zoom)',
+    },
+    algolia: {
+      // The application ID provided by Algolia
+      appId: 'VPUDC1V4TA',
+
+      // Public API key: it is safe to commit it
+      apiKey: '0b4fcfd05976eb0aaf4b7c51ec4fcd23',
+
+      indexName: 'promptfoo',
+    },
+  } satisfies Preset.ThemeConfig,
+
+  plugins: [
+    require.resolve('docusaurus-plugin-image-zoom'),
+    [
+      '@docusaurus/plugin-client-redirects',
+      {
+        redirects: [
+          {
+            from: '/docs/category/troubleshooting',
+            to: '/docs/usage/troubleshooting',
+          },
+          {
+            from: '/docs/providers/palm',
+            to: '/docs/providers/google',
+          },
+        ],
+      },
+    ],
+    // Define the llms.txt plugin inline similar to the Prisma example
+    async function llmsTxtPlugin(context) {
+      return {
+        name: 'llms-txt-plugin',
+        loadContent: async () => {
+          const { siteDir } = context;
+          const docsDir = path.join(siteDir, 'docs');
+          const allMdx: string[] = [];
+
+          // Recursive function to get all mdx/md files
+          const getMdFiles = async (dir: string): Promise<void> => {
+            const entries = await fs.promises.readdir(dir, { withFileTypes: true });
+
+            for (const entry of entries) {
+              const fullPath = path.join(dir, entry.name);
+              if (entry.isDirectory()) {
+                await getMdFiles(fullPath);
+              } else if (entry.name.endsWith('.md') || entry.name.endsWith('.mdx')) {
+                const content = await fs.promises.readFile(fullPath, 'utf8');
+                allMdx.push(content);
+              }
+            }
+          };
+
+          await getMdFiles(docsDir);
+          return { allMdx };
+        },
+        postBuild: async ({ content, routesPaths, outDir }) => {
+          // Type assertion to handle TypeScript type checking
+          const pluginContent = content as { allMdx: string[] };
+          const { allMdx } = pluginContent;
+
+          // Write concatenated MDX content
+          const concatenatedPath = path.join(outDir, 'llms-full.txt');
+          await fs.promises.writeFile(concatenatedPath, allMdx.join('\n\n---\n\n'));
+
+          // Process routes - use routesPaths which is a string[] of all routes
+          const docsRoutes: string[] = [];
+
+          // Filter for docs paths and generate entries
+          for (const routePath of routesPaths) {
+            if (routePath.startsWith('/docs/')) {
+              // Extract a title from the route path as fallback
+              const pathParts = routePath.split('/').filter(Boolean);
+              const lastPart = pathParts[pathParts.length - 1];
+              const title = lastPart
+                .split('-')
+                .map((part) => part.charAt(0).toUpperCase() + part.slice(1))
+                .join(' ');
+
+              docsRoutes.push(`- [${title}](${routePath})`);
+            }
+          }
+
+          // Build up llms.txt file
+          const llmsTxt = `# ${context.siteConfig.title}\n\n## Docs\n\n${docsRoutes.join('\n')}`;
+
+          // Write llms.txt file
+          const llmsTxtPath = path.join(outDir, 'llms.txt');
+          try {
+            fs.writeFileSync(llmsTxtPath, llmsTxt);
path: site/docusaurus.config.ts
user: login: ellipsis-dev[bot]
body: Consider using `fs.promises.writeFile` (async) instead of `fs.writeFileSync` in `postBuild` to avoid blocking and maintain consistency with the async/await style.
```suggestion
            await fs.promises.writeFile(llmsTxtPath, llmsTxt);
```
created_at: 2025-04-07T04:41:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,445 @@
+import { themes } from 'prism-react-renderer';
+import type * as Preset from '@docusaurus/preset-classic';
+import type { Config } from '@docusaurus/types';
+import * as fs from 'fs';
+import * as path from 'path';
+
+const lightCodeTheme = themes.github;
+const darkCodeTheme = themes.duotoneDark;
+
+const config: Config = {
+  title: 'promptfoo',
+  tagline: 'Test your prompts',
+  favicon: '/favicon.ico',
+
+  // Set the production url of your site here
+  url: 'https://www.promptfoo.dev',
+  // Set the /<baseUrl>/ pathname under which your site is served
+  // For GitHub pages deployment, it is often '/<projectName>/'
+  baseUrl: '/',
+
+  trailingSlash: true,
+
+  // GitHub pages deployment config.
+  // If you aren't using GitHub pages, you don't need these.
+  organizationName: 'promptfoo', // Usually your GitHub org/user name.
+  projectName: 'promptfoo', // Usually your repo name.
+
+  onBrokenLinks: 'throw',
+  onBrokenMarkdownLinks: 'throw',
+  onBrokenAnchors: 'throw',
+  // Even if you don't use internalization, you can use this field to set useful
+  // metadata like html lang. For example, if your site is Chinese, you may want
+  // to replace "en" with "zh-Hans".
+  i18n: {
+    defaultLocale: 'en',
+    locales: ['en'],
+  },
+
+  headTags: [
+    {
+      tagName: 'link',
+      attributes: {
+        rel: 'preconnect',
+        href: 'https://fonts.googleapis.com',
+      },
+    },
+    {
+      tagName: 'link',
+      attributes: {
+        rel: 'preconnect',
+        href: 'https://fonts.gstatic.com',
+        crossorigin: 'true',
+      },
+    },
+    {
+      tagName: 'link',
+      attributes: {
+        rel: 'stylesheet',
+        href: 'https://fonts.googleapis.com/css2?family=Inter:wght@100..900&display=swap',
+      },
+    },
+  ],
+
+  scripts: [
+    {
+      src: '/js/scripts.js',
+      async: true,
+    },
+  ],
+
+  presets: [
+    [
+      'classic',
+      {
+        docs: {
+          sidebarPath: require.resolve('./sidebars.js'),
+          // Remove this to remove the "edit this page" links.
+          editUrl: 'https://github.com/promptfoo/promptfoo/tree/main/site',
+          sidebarCollapsed: false,
+        },
+        blog: {
+          showReadingTime: false,
+          blogSidebarCount: 0,
+          // Please change this to your repo.
+          // Remove this to remove the "edit this page" links.
+          //editUrl:
+          //  'https://github.com/facebook/docusaurus/tree/main/packages/create-docusaurus/templates/shared/',
+        },
+        theme: {
+          customCss: require.resolve('./src/css/custom.css'),
+        },
+        gtag:
+          process.env.NODE_ENV === 'development'
+            ? undefined
+            : {
+                trackingID: 'G-3TS8QLZQ93',
+                anonymizeIP: true,
+              },
+      } satisfies Preset.Options,
+    ],
+  ],
+
+  themeConfig: {
+    image: 'img/thumbnail.png',
+    navbar: {
+      title: 'promptfoo',
+      logo: {
+        alt: 'promptfoo logo',
+        src: 'img/logo-panda.svg',
+      },
+      items: [
+        {
+          type: 'dropdown',
+          label: 'Product',
+          position: 'left',
+          items: [
+            {
+              to: '/security/',
+              label: 'Gen AI Security',
+            },
+            {
+              to: '/llm-vulnerability-scanner/',
+              label: 'Vulnerability Scanner',
+            },
+            {
+              to: '/docs/getting-started/',
+              label: 'LLM Evaluations',
+            },
+          ],
+        },
+        {
+          type: 'dropdown',
+          label: 'Company',
+          position: 'left',
+          items: [
+            {
+              href: '/about/',
+              label: 'About',
+            },
+            {
+              href: '/blog/',
+              label: 'Blog',
+            },
+            {
+              href: '/press/',
+              label: 'Press',
+            },
+            {
+              href: '/contact/',
+              label: 'Contact',
+            },
+            {
+              href: '/careers/',
+              label: 'Careers',
+            },
+          ],
+        },
+        {
+          type: 'dropdown',
+          label: 'Resources',
+          position: 'left',
+          items: [
+            {
+              href: '/docs/intro/',
+              label: 'Docs',
+            },
+            {
+              to: 'https://www.promptfoo.dev/docs/api-reference/',
+              label: 'API Reference',
+            },
+            {
+              to: 'https://www.promptfoo.dev/models/',
+              label: 'Foundation Model Reports',
+            },
+            {
+              href: 'https://github.com/promptfoo/promptfoo',
+              label: 'GitHub',
+            },
+            {
+              href: 'https://discord.gg/promptfoo',
+              label: 'Discord',
+            },
+          ],
+        },
+        { to: '/pricing/', label: 'Enterprise', position: 'left' },
+        {
+          href: 'https://github.com/promptfoo/promptfoo',
+          position: 'right',
+          className: 'header-github-link',
+          'aria-label': 'GitHub repository',
+        },
+        {
+          href: 'https://discord.gg/promptfoo',
+          position: 'right',
+          className: 'header-discord-link',
+          'aria-label': 'Discord community',
+        },
+      ],
+    },
+    footer: {
+      style: 'dark',
+      links: [
+        {
+          title: 'Product',
+          items: [
+            {
+              label: 'Docs',
+              to: '/docs/intro/',
+            },
+            {
+              label: 'Command Line',
+              to: '/docs/getting-started/',
+            },
+            {
+              label: 'Node Package',
+              to: '/docs/usage/node-package/',
+            },
+            {
+              label: 'Enterprise',
+              href: '/pricing/',
+            },
+            {
+              label: 'Status',
+              href: 'https://status.promptfoo.dev',
+            },
+          ],
+        },
+        {
+          title: 'Resources',
+          items: [
+            {
+              label: 'LLM Red Teaming',
+              to: '/docs/red-team',
+            },
+            {
+              label: 'Foundation Model Reports',
+              to: 'https://www.promptfoo.dev/models/',
+            },
+            {
+              label: 'Running Benchmarks',
+              to: '/docs/guides/llama2-uncensored-benchmark-ollama',
+            },
+            {
+              label: 'Evaluating Factuality',
+              to: '/docs/guides/factuality-eval',
+            },
+            {
+              label: 'Evaluating RAGs',
+              to: '/docs/guides/evaluate-rag',
+            },
+            {
+              label: 'Minimizing Hallucinations',
+              to: '/docs/guides/prevent-llm-hallucations',
+            },
+            {
+              label: 'Config Validator',
+              to: '/validator',
+            },
+          ],
+        },
+        {
+          title: 'Company',
+          items: [
+            {
+              label: 'About',
+              to: '/about/',
+            },
+            {
+              label: 'Blog',
+              to: '/blog/',
+            },
+            {
+              label: 'Press',
+              to: '/press/',
+            },
+            {
+              label: 'Contact',
+              to: '/contact/',
+            },
+            {
+              label: 'Careers',
+              to: '/careers/',
+            },
+          ],
+        },
+        {
+          title: 'Legal & Social',
+          items: [
+            {
+              label: 'GitHub',
+              href: 'https://github.com/promptfoo/promptfoo',
+            },
+            {
+              label: 'Discord',
+              href: 'https://discord.gg/promptfoo',
+            },
+            {
+              label: 'LinkedIn',
+              href: 'https://www.linkedin.com/company/promptfoo/',
+            },
+            {
+              label: 'Privacy Policy',
+              to: '/privacy/',
+            },
+            {
+              label: 'Terms of Service',
+              to: '/terms-of-service/',
+            },
+            {
+              html: `
+                <div style="position: relative; margin-top:8px">
+                  <span style="position: absolute; left: 65px; top: 25px; font-size: 10px; font-weight: bold; background-color: #25842c; padding: 2px 4px; border-radius: 4px;">In Progress</span>
+                  <img loading="lazy" src="/img/badges/soc2.png" alt="SOC2 Compliance in progress" style="width:80px; height: auto"/>
+                </div>
+                `,
+            },
+          ],
+        },
+      ],
+      copyright: `© ${new Date().getFullYear()} promptfoo`,
+    },
+    prism: {
+      theme: lightCodeTheme,
+      darkTheme: darkCodeTheme,
+      additionalLanguages: [
+        'bash',
+        'javascript',
+        'typescript',
+        'python',
+        'json',
+        'yaml',
+        'markup-templating',
+        'liquid',
+      ],
+    },
+    zoom: {
+      selector: '.markdown :not(em) > img:not(.no-zoom)',
+    },
+    algolia: {
+      // The application ID provided by Algolia
+      appId: 'VPUDC1V4TA',
+
+      // Public API key: it is safe to commit it
+      apiKey: '0b4fcfd05976eb0aaf4b7c51ec4fcd23',
+
+      indexName: 'promptfoo',
+    },
+  } satisfies Preset.ThemeConfig,
+
+  plugins: [
+    require.resolve('docusaurus-plugin-image-zoom'),
+    [
+      '@docusaurus/plugin-client-redirects',
+      {
+        redirects: [
+          {
+            from: '/docs/category/troubleshooting',
+            to: '/docs/usage/troubleshooting',
+          },
+          {
+            from: '/docs/providers/palm',
+            to: '/docs/providers/google',
+          },
+        ],
+      },
+    ],
+    // Define the llms.txt plugin inline similar to the Prisma example
+    async function llmsTxtPlugin(context) {
+      return {
+        name: 'llms-txt-plugin',
+        loadContent: async () => {
+          const { siteDir } = context;
+          const docsDir = path.join(siteDir, 'docs');
+          const allMdx: string[] = [];
+
+          // Recursive function to get all mdx/md files
+          const getMdFiles = async (dir: string): Promise<void> => {
+            const entries = await fs.promises.readdir(dir, { withFileTypes: true });
+
+            for (const entry of entries) {
+              const fullPath = path.join(dir, entry.name);
+              if (entry.isDirectory()) {
+                await getMdFiles(fullPath);
+              } else if (entry.name.endsWith('.md') || entry.name.endsWith('.mdx')) {
+                const content = await fs.promises.readFile(fullPath, 'utf8');
+                allMdx.push(content);
+              }
+            }
+          };
+
+          await getMdFiles(docsDir);
+          return { allMdx };
+        },
+        postBuild: async ({ content, routesPaths, outDir }) => {
+          // Type assertion to handle TypeScript type checking
+          const pluginContent = content as { allMdx: string[] };
path: site/docusaurus.config.ts
user: login: ellipsis-dev[bot]
body: Instead of solely relying on a type assertion for `content`, consider adding a runtime invariant check to ensure `allMdx` exists before its usage.
```suggestion
          const pluginContent = (content && 'allMdx' in content) ? content as { allMdx: string[] } : { allMdx: [] };
```
created_at: 2025-04-07T04:41:34Z
author_association: CONTRIBUTOR

PR comments:
user: login: gru-agent[bot]
created_at: 2025-04-07T04:39:34Z
body: ## TestGru Assignment

### Summary

| Link | CommitId | Status | Reason |
| ---- | -------- | ------ | ------ |
| [Detail](https://gru.ai/:test/promptfoo@github/promptfoo/5251e68e-92fb-4d3c-a4ab-6a15584bbc33) | 29d015db588ce2e17ee9cbfdbcb942c7e63cfdac | 🚫 Skipped | No files need to be tested {"site/README.md":"target file(site/README.md) not in work scope \n include: src \n exclude: node_modules,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx,**/*.d.ts,**/*.test.js,**/*.spec.js,src/app/src","site/docusaurus.config.js":"target file not found: site/docusaurus.config.js in 29d015db588ce2e17ee9cbfdbcb942c7e63cfdac promptfoo/promptfoo","site/docusaurus.config.ts":"target file(site/docusaurus.config.ts) not in work scope \n include: src \n exclude: node_modules,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx,**/*.d.ts,**/*.test.js,**/*.spec.js,src/app/src"} |

> [!TIP]
> You can `@gru-agent` and leave your feedback. TestGru will make adjustments based on your input

author_association: CONTRIBUTOR
user: login: gru-agent[bot]
created_at: 2025-04-07T04:39:37Z
body: ## TestGru Assignment

### Summary

| Link | CommitId | Status | Reason |
| ---- | -------- | ------ | ------ |
| [Detail](https://gru.ai/:test/promptfoo@github/promptfoo/4ad15344-aa08-4a64-bf50-b9dd176cbc71) | 29d015db588ce2e17ee9cbfdbcb942c7e63cfdac | 🚫 Skipped | No files need to be tested {"site/README.md":"target file(site/README.md) not in work scope \n include: src/app/src/**/*.ts \n exclude: node_modules,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx,**/*.d.ts,**/*.test.js,**/*.spec.js","site/docusaurus.config.js":"target file not found: site/docusaurus.config.js in 29d015db588ce2e17ee9cbfdbcb942c7e63cfdac promptfoo/promptfoo","site/docusaurus.config.ts":"target file(site/docusaurus.config.ts) not in work scope \n include: src/app/src/**/*.ts \n exclude: node_modules,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx,**/*.d.ts,**/*.test.js,**/*.spec.js"} |

> [!TIP]
> You can `@gru-agent` and leave your feedback. TestGru will make adjustments based on your input

author_association: CONTRIBUTOR