Repository: arabold/docs-mcp-server (ts)

Issue:
Private Docs: Header Support for Authentication
Allowing arbitrary headers to be set for a source would enable internal documentation requiring authentication to be consumed.  For example, Confluence or something that needs an API token.

Allowing any header to be set would be most flexible, perhaps a JSON map of strings as an env variable would be the most flexible.

```json
{"Authorization": "Bearer ...."}
```

Pull request description:
feature/139 custom headers cli
## Custom HTTP Headers Support for Scraping (CLI, Web UI, Playwright)

This PR introduces robust, user-friendly support for specifying custom HTTP headers throughout the entire scraping pipeline, including:

- **CLI ( and  commands):**
  - Add  (repeatable) to both commands.
  - Headers are parsed and forwarded to all HTTP requests, including Playwright and the low-level fetcher.
- **Web UI:**
  - Users can dynamically add/remove custom HTTP headers in the scrape job form.
  - Headers are serialized, parsed on the backend, and propagated to the scraping pipeline.
- **Backend/Playwright:**
  - All custom headers are forwarded to Playwright requests, including for authentication and advanced scenarios.
  - Helper functions ensure robust merging and credential handling.
- **Testing:**
  - Unit and integration tests cover header propagation, merging, and Playwright middleware logic.

### Why?
- Enables scraping of authenticated docs, APIs, and sites requiring custom headers.
- Makes the system more flexible for enterprise and advanced use cases.
- Ensures consistent behavior between CLI and Web UI.

---

**Closes #139 and related issues.**

Let us know if you have feedback or want further UX improvements

Patch:
diff --git a/src/index.ts b/src/index.ts
index 70dbce3..ab95bc8 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -206,7 +206,6 @@ async function main() {
         await new Promise(() => {}); // Keep alive
       });
 
-    // ... (All other CLI command definitions: scrape, search, list, find-version, remove, fetch-url - remain unchanged) ...
     // --- Scrape Command ---
     program
       .command("scrape <library> <url>")
@@ -280,6 +279,12 @@ async function main() {
         (val: string, prev: string[] = []) => prev.concat([val]),
         [] as string[],
       )
+      .option(
+        "--header <name:value>",
+        "Custom HTTP header to send with each request (can be specified multiple times)",
+        (val: string, prev: string[] = []) => prev.concat([val]),
+        [] as string[],
+      )
       .action(async (library, url, options) => {
         commandExecuted = true; // Ensure this is set for CLI commands
         const docService = new DocumentManagementService();
@@ -289,6 +294,20 @@ async function main() {
           pipelineManager = new PipelineManager(docService);
           await pipelineManager.start();
           const scrapeTool = new ScrapeTool(docService, pipelineManager);
+
+          // Parse headers from CLI options
+          const headers: Record<string, string> = {};
+          if (Array.isArray(options.header)) {
+            for (const entry of options.header) {
+              const idx = entry.indexOf(":");
+              if (idx > 0) {
+                const name = entry.slice(0, idx).trim();
+                const value = entry.slice(idx + 1).trim();
+                if (name) headers[name] = value;
+              }
+            }
+          }
+
           const result = await scrapeTool.execute({
             url,
             library,
@@ -309,6 +328,7 @@ async function main() {
                 Array.isArray(options.excludePattern) && options.excludePattern.length > 0
                   ? options.excludePattern
                   : undefined,
+              headers: Object.keys(headers).length > 0 ? headers : undefined,
             },
           });
           if ("pagesScraped" in result)
@@ -449,14 +469,33 @@ async function main() {
         },
         ScrapeMode.Auto,
       )
+      .option(
+        "--header <name:value>",
+        "Custom HTTP header to send with the request (can be specified multiple times)",
+        (val: string, prev: string[] = []) => prev.concat([val]),
+        [] as string[],
+      )
       .action(async (url, options) => {
         commandExecuted = true; // Ensure this is set
+        // Parse headers from CLI options
+        const headers: Record<string, string> = {};
+        if (Array.isArray(options.header)) {
+          for (const entry of options.header) {
+            const idx = entry.indexOf(":");
+            if (idx > 0) {
+              const name = entry.slice(0, idx).trim();
+              const value = entry.slice(idx + 1).trim();
+              if (name) headers[name] = value;
+            }
+          }
+        }
         // FetchUrlTool does not require DocumentManagementService or PipelineManager
         const fetchUrlTool = new FetchUrlTool(new HttpFetcher(), new FileFetcher());
         const content = await fetchUrlTool.execute({
           url,
           followRedirects: options.followRedirects,
           scrapeMode: options.scrapeMode,
+          headers: Object.keys(headers).length > 0 ? headers : undefined,
         });
         console.log(content);
       });
diff --git a/src/scraper/middleware/HtmlPlaywrightMiddleware.ts b/src/scraper/middleware/HtmlPlaywrightMiddleware.ts
index 3c38741..7027c16 100644
--- a/src/scraper/middleware/HtmlPlaywrightMiddleware.ts
+++ b/src/scraper/middleware/HtmlPlaywrightMiddleware.ts
@@ -12,6 +12,8 @@ import type { ContentProcessorMiddleware, MiddlewareContext } from "./types";
  *
  * This middleware also supports URLs with embedded credentials (user:password@host) and ensures
  * credentials are used for all same-origin resource requests (not just the main page) via HTTP Basic Auth.
+ *
+ * Additionally, all custom headers from context.options?.headers are forwarded to Playwright requests.
  */
 export class HtmlPlaywrightMiddleware implements ContentProcessorMiddleware {
   private browser: Browser | null = null;
@@ -101,6 +103,7 @@ export class HtmlPlaywrightMiddleware implements ContentProcessorMiddleware {
    * - Parses credentials from the URL (if present).
    * - Uses browser.newContext({ httpCredentials }) for HTTP Basic Auth on the main page and subresources.
    * - Injects Authorization header for all same-origin requests if credentials are present and not already set.
+   * - Forwards all custom headers from context.options?.headers to Playwright requests.
    * - Waits for common loading indicators to disappear before extracting HTML.
    *
    * @param context The middleware context containing the HTML and source URL.
@@ -130,21 +133,11 @@ export class HtmlPlaywrightMiddleware implements ContentProcessorMiddleware {
     let browserContext: BrowserContext | null = null;
     let renderedHtml: string | null = null;
 
-    // --- Credential Extraction ---
-    let credentials: { username: string; password: string } | null = null;
-    let origin: string | null = null;
-    try {
-      const url = new URL(context.source);
-      origin = url.origin;
-      if (url.username && url.password) {
-        credentials = { username: url.username, password: url.password };
-        logger.debug(
-          `Playwright: Detected credentials for ${origin} (username: ${url.username})`,
-        );
-      }
-    } catch (e) {
-      logger.warn(`⚠️  Could not parse URL for credential extraction: ${context.source}`);
-    }
+    // Extract credentials and origin using helper
+    const { credentials, origin } = extractCredentialsAndOrigin(context.source);
+
+    // Extract custom headers (Record<string, string>)
+    const customHeaders: Record<string, string> = context.options?.headers ?? {};
 
     try {
       const browser = await this.ensureBrowser();
@@ -156,7 +149,7 @@ export class HtmlPlaywrightMiddleware implements ContentProcessorMiddleware {
       }
       logger.debug(`Playwright: Processing ${context.source}`);
 
-      // Block unnecessary resources and inject credentials for same-origin requests
+      // Block unnecessary resources and inject credentials and custom headers for same-origin requests
       await page.route("**/*", async (route) => {
         const reqUrl = route.request().url();
         const reqOrigin = (() => {
@@ -179,23 +172,15 @@ export class HtmlPlaywrightMiddleware implements ContentProcessorMiddleware {
         if (["image", "stylesheet", "font", "media"].includes(resourceType)) {
           return route.abort();
         }
-        // Inject Authorization header for same-origin requests if credentials are present
-        if (
-          credentials &&
-          origin &&
-          reqOrigin === origin &&
-          !route.request().headers().authorization
-        ) {
-          const basic = Buffer.from(
-            `${credentials.username}:${credentials.password}`,
-          ).toString("base64");
-          const headers = {
-            ...route.request().headers(),
-            Authorization: `Basic ${basic}`,
-          };
-          return route.continue({ headers });
-        }
-        return route.continue();
+        // Use helper to merge headers
+        const headers = mergePlaywrightHeaders(
+          route.request().headers(),
+          customHeaders,
+          credentials ?? undefined,
+          origin ?? undefined,
+          reqOrigin ?? undefined,
+        );
+        return route.continue({ headers });
       });
 
       // Load initial HTML content
@@ -238,3 +223,55 @@ export class HtmlPlaywrightMiddleware implements ContentProcessorMiddleware {
     await next();
   }
 }
+
+/**
+ * Extracts credentials and origin from a URL string.
+ * Returns { credentials, origin } where credentials is null if not present.
+ */
+export function extractCredentialsAndOrigin(urlString: string): {
+  credentials: { username: string; password: string } | null;
+  origin: string | null;
+} {
+  try {
+    const url = new URL(urlString);
+    const origin = url.origin;
+    if (url.username && url.password) {
+      return {
+        credentials: { username: url.username, password: url.password },
+        origin,
+      };
+    }
+    return { credentials: null, origin };
+  } catch {
+    return { credentials: null, origin: null };
+  }
+}
+
+/**
+ * Merges Playwright request headers, custom headers, and credentials.
+ * - Custom headers are merged in unless already present (except Authorization, see below).
+ * - If credentials are present and the request is same-origin, injects Authorization if not already set.
+ */
+export function mergePlaywrightHeaders(
+  requestHeaders: Record<string, string>,
+  customHeaders: Record<string, string>,
+  credentials?: { username: string; password: string },
+  origin?: string,
+  reqOrigin?: string,
+): Record<string, string> {
+  let headers = { ...requestHeaders };
+  for (const [key, value] of Object.entries(customHeaders)) {
+    if (key.toLowerCase() === "authorization" && headers.authorization) continue;
+    headers[key] = value;
+  }
+  if (credentials && origin && reqOrigin === origin && !headers.authorization) {
+    const basic = Buffer.from(`${credentials.username}:${credentials.password}`).toString(
+      "base64",
+    );
+    headers = {
+      ...headers,
+      Authorization: `Basic ${basic}`,
+    };
+  }
+  return headers;
+}
diff --git a/src/scraper/strategies/WebScraperStrategy.ts b/src/scraper/strategies/WebScraperStrategy.ts
index abd1ce1..472fe27 100644
--- a/src/scraper/strategies/WebScraperStrategy.ts
+++ b/src/scraper/strategies/WebScraperStrategy.ts
@@ -62,6 +62,14 @@ export class WebScraperStrategy extends BaseScraperStrategy {
     }
   }
 
+  /**
+   * Processes a single queue item by fetching its content and processing it through pipelines.
+   * @param item - The queue item to process.
+   * @param options - Scraper options including headers for HTTP requests.
+   * @param _progressCallback - Optional progress callback (not used here).
+   * @param signal - Optional abort signal for request cancellation.
+   * @returns An object containing the processed document and extracted links.
+   */
   protected override async processItem(
     item: QueueItem,
     options: ScraperOptions,
@@ -71,10 +79,11 @@ export class WebScraperStrategy extends BaseScraperStrategy {
     const { url } = item;
 
     try {
-      // Define fetch options, passing both signal and followRedirects
+      // Define fetch options, passing signal, followRedirects, and headers
       const fetchOptions = {
         signal,
         followRedirects: options.followRedirects,
+        headers: options.headers, // Forward custom headers
       };
 
       // Pass options to fetcher
diff --git a/src/scraper/types.ts b/src/scraper/types.ts
index 860b56e..870405e 100644
--- a/src/scraper/types.ts
+++ b/src/scraper/types.ts
@@ -65,6 +65,11 @@ export interface ScraperOptions {
    * Patterns for excluding URLs during scraping. Exclude takes precedence over include.
    */
   excludePatterns?: string[];
+  /**
+   * Custom HTTP headers to send with each HTTP request (e.g., for authentication).
+   * Keys are header names, values are header values.
+   */
+  headers?: Record<string, string>;
 }
 
 /**
diff --git a/src/tools/FetchUrlTool.ts b/src/tools/FetchUrlTool.ts
index 923612a..d6cf7bb 100644
--- a/src/tools/FetchUrlTool.ts
+++ b/src/tools/FetchUrlTool.ts
@@ -32,6 +32,12 @@ export interface FetchUrlToolOptions {
    * @default ScrapeMode.Auto
    */
   scrapeMode?: ScrapeMode;
+
+  /**
+   * Custom HTTP headers to send with the request (e.g., for authentication).
+   * Keys are header names, values are header values.
+   */
+  headers?: Record<string, string>;
 }
 
 /**
@@ -58,7 +64,7 @@ export class FetchUrlTool {
    * @throws {ToolError} If fetching or processing fails
    */
   async execute(options: FetchUrlToolOptions): Promise<string> {
-    const { url, scrapeMode = ScrapeMode.Auto } = options;
+    const { url, scrapeMode = ScrapeMode.Auto, headers } = options;
 
     const canFetchResults = this.fetchers.map((f) => f.canFetch(url));
     const fetcherIndex = canFetchResults.findIndex((result) => result === true);
@@ -79,6 +85,7 @@ export class FetchUrlTool {
       const rawContent: RawContent = await fetcher.fetch(url, {
         followRedirects: options.followRedirects ?? true,
         maxRetries: 3,
+        headers, // propagate custom headers
       });
 
       logger.info("🔄 Processing content...");
@@ -100,6 +107,7 @@ export class FetchUrlTool {
               excludeSelectors: undefined,
               ignoreErrors: false,
               scrapeMode,
+              headers, // propagate custom headers
             },
             fetcher,
           );
diff --git a/src/tools/ScrapeTool.ts b/src/tools/ScrapeTool.ts
index 43fada9..75003b6 100644
--- a/src/tools/ScrapeTool.ts
+++ b/src/tools/ScrapeTool.ts
@@ -50,6 +50,11 @@ export interface ScrapeToolOptions {
      * Regex patterns must be wrapped in slashes, e.g. /pattern/.
      */
     excludePatterns?: string[];
+    /**
+     * Custom HTTP headers to send with each request (e.g., for authentication).
+     * Keys are header names, values are header values.
+     */
+    headers?: Record<string, string>;
   };
   /** If false, returns jobId immediately without waiting. Defaults to true. */
   waitForCompletion?: boolean;
@@ -143,6 +148,7 @@ export class ScrapeTool {
       scrapeMode: scraperOptions?.scrapeMode ?? ScrapeMode.Auto, // Pass scrapeMode enum
       includePatterns: scraperOptions?.includePatterns,
       excludePatterns: scraperOptions?.excludePatterns,
+      headers: scraperOptions?.headers, // <-- propagate headers
     });
 
     // Conditionally wait for completion
diff --git a/src/web/components/ScrapeFormContent.tsx b/src/web/components/ScrapeFormContent.tsx
index b267d12..71732d1 100644
--- a/src/web/components/ScrapeFormContent.tsx
+++ b/src/web/components/ScrapeFormContent.tsx
@@ -19,6 +19,7 @@ const ScrapeFormContent = () => (
       x-data="{
         url: '',
         hasPath: false,
+        headers: [],
         checkUrlPath() {
           try {
             const url = new URL(this.url);
@@ -118,7 +119,7 @@ const ScrapeFormContent = () => (
         <summary class="cursor-pointer text-sm font-medium text-gray-600 dark:text-gray-400">
           Advanced Options
         </summary>
-        <div class="mt-2 space-y-2">
+        <div class="mt-2 space-y-2" x-data="{ headers: [] }">
           <div>
             <div class="flex items-center">
               <label
@@ -270,6 +271,55 @@ const ScrapeFormContent = () => (
               <option value={ScrapeMode.Playwright}>Playwright</option>
             </select>
           </div>
+          <div>
+            <div class="flex items-center mb-1">
+              <label class="block text-sm font-medium text-gray-700 dark:text-gray-300">
+                Custom HTTP Headers
+              </label>
+              <Tooltip text="Add custom HTTP headers (e.g., for authentication). These will be sent with every HTTP request." />
+            </div>
+            <div>
+              {/* AlpineJS dynamic header rows */}
+              <template x-for="(header, idx) in headers">
+                <div class="flex space-x-2 mb-1">
+                  <input
+                    type="text"
+                    class="w-1/3 px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-700 text-gray-900 dark:text-white text-xs"
+                    placeholder="Header Name"
+                    x-model="header.name"
+                    required
+                  />
+                  <span class="text-gray-500">:</span>
+                  <input
+                    type="text"
+                    class="w-1/2 px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-700 text-gray-900 dark:text-white text-xs"
+                    placeholder="Header Value"
+                    x-model="header.value"
+                    required
+                  />
+                  <button
+                    type="button"
+                    class="text-red-500 hover:text-red-700 text-xs"
+                    x-on:click="headers.splice(idx, 1)"
+                  >
+                    Remove
+                  </button>
+                  <input
+                    type="hidden"
+                    name="header[]"
+                    x-bind:value="header.name && header.value ? header.name + ':' + header.value : ''"
+                  />
+                </div>
+              </template>
+              <button
+                type="button"
+                class="mt-1 px-2 py-0.5 bg-indigo-100 dark:bg-indigo-900 text-indigo-700 dark:text-indigo-200 rounded text-xs"
+                x-on:click="headers.push({ name: '', value: '' })"
+              >
+                + Add Header
+              </button>
+            </div>
+          </div>
           <div class="flex items-center">
             <input
               id="followRedirects"
diff --git a/src/web/routes/jobs/new.tsx b/src/web/routes/jobs/new.tsx
index f985784..bc7257a 100644
--- a/src/web/routes/jobs/new.tsx
+++ b/src/web/routes/jobs/new.tsx
@@ -38,6 +38,7 @@ export function registerNewJobRoutes(
           ignoreErrors?: "on" | undefined;
           includePatterns?: string;
           excludePatterns?: string;
+          "header[]"?: string[] | string; // Added header field for custom headers
         };
       }>,
       reply
@@ -67,6 +68,24 @@ export function registerNewJobRoutes(
             .filter((s) => s.length > 0);
         }
 
+        // Parse custom headers from repeated header[] fields (format: name:value)
+        function parseHeaders(
+          input?: string[] | string
+        ): Record<string, string> | undefined {
+          if (!input) return undefined;
+          const arr = Array.isArray(input) ? input : [input];
+          const headers: Record<string, string> = {};
+          for (const entry of arr) {
+            const idx = entry.indexOf(":");
+            if (idx > 0) {
+              const name = entry.slice(0, idx).trim();
+              const value = entry.slice(idx + 1).trim();
+              if (name) headers[name] = value;
+            }
+          }
+          return Object.keys(headers).length > 0 ? headers : undefined;
+        }
+
         // Prepare options for ScrapeTool
         const scrapeOptions = {
           url: body.url,
@@ -87,6 +106,7 @@ export function registerNewJobRoutes(
             ignoreErrors: body.ignoreErrors === "on",
             includePatterns: parsePatterns(body.includePatterns),
             excludePatterns: parsePatterns(body.excludePatterns),
+            headers: parseHeaders(body["header[]"]), // <-- propagate custom headers from web UI
           },
         };
 


Test patch:
diff --git a/src/scraper/middleware/HtmlPlaywrightMiddleware.test.ts b/src/scraper/middleware/HtmlPlaywrightMiddleware.test.ts
index aeb556d..e6ed3a6 100644
--- a/src/scraper/middleware/HtmlPlaywrightMiddleware.test.ts
+++ b/src/scraper/middleware/HtmlPlaywrightMiddleware.test.ts
@@ -1,7 +1,11 @@
 import * as cheerio from "cheerio";
 import { type MockedObject, afterAll, afterEach, describe, expect, it, vi } from "vitest";
 import type { ScraperOptions } from "../types";
-import { HtmlPlaywrightMiddleware } from "./HtmlPlaywrightMiddleware";
+import {
+  HtmlPlaywrightMiddleware,
+  extractCredentialsAndOrigin,
+  mergePlaywrightHeaders,
+} from "./HtmlPlaywrightMiddleware";
 import type { MiddlewareContext } from "./types"; // Adjusted path
 
 // Suppress logger output during tests
@@ -256,3 +260,86 @@ describe("HtmlPlaywrightMiddleware", () => {
     expect(next).toHaveBeenCalled();
   });
 });
+
+describe("extractCredentialsAndOrigin", () => {
+  it("extracts credentials and origin from a URL with user:pass", () => {
+    const url = "https://user:pass@example.com/path";
+    const result = extractCredentialsAndOrigin(url);
+    expect(result).toEqual({
+      credentials: { username: "user", password: "pass" },
+      origin: "https://example.com",
+    });
+  });
+
+  it("returns null credentials if no user:pass", () => {
+    const url = "https://example.com/path";
+    const result = extractCredentialsAndOrigin(url);
+    expect(result).toEqual({
+      credentials: null,
+      origin: "https://example.com",
+    });
+  });
+
+  it("returns nulls for invalid URL", () => {
+    const url = "not a url";
+    const result = extractCredentialsAndOrigin(url);
+    expect(result).toEqual({ credentials: null, origin: null });
+  });
+});
+
+describe("mergePlaywrightHeaders", () => {
+  const baseHeaders = { foo: "bar", authorization: "existing" };
+  const customHeaders = {
+    foo: "baz",
+    custom: "value",
+    Authorization: "should-not-overwrite",
+  };
+  const credentials = { username: "user", password: "pass" };
+  const origin = "https://example.com";
+  const reqOrigin = "https://example.com";
+
+  it("merges custom headers, does not overwrite existing authorization", () => {
+    const result = mergePlaywrightHeaders(baseHeaders, customHeaders);
+    expect(result.foo).toBe("baz");
+    expect(result.custom).toBe("value");
+    expect(result.authorization).toBe("existing");
+  });
+
+  it("injects Authorization if credentials and same-origin and not already set", () => {
+    const result = mergePlaywrightHeaders(
+      { foo: "bar" },
+      {},
+      credentials,
+      origin,
+      reqOrigin,
+    );
+    expect(result.Authorization).toMatch(/^Basic /);
+  });
+
+  it("does not inject Authorization if origins differ", () => {
+    const result = mergePlaywrightHeaders(
+      { foo: "bar" },
+      {},
+      credentials,
+      origin,
+      "https://other.com",
+    );
+    expect(result.Authorization).toBeUndefined();
+  });
+
+  it("does not inject Authorization if already set", () => {
+    const result = mergePlaywrightHeaders(
+      { authorization: "existing" },
+      {},
+      credentials,
+      origin,
+      reqOrigin,
+    );
+    expect(result.authorization).toBe("existing");
+  });
+
+  it("works with no credentials and no custom headers", () => {
+    const result = mergePlaywrightHeaders({ foo: "bar" }, {});
+    expect(result.foo).toBe("bar");
+  });
+});
diff --git a/src/scraper/strategies/WebScraperStrategy.test.ts b/src/scraper/strategies/WebScraperStrategy.test.ts
index 8640cb5..d9427ec 100644
--- a/src/scraper/strategies/WebScraperStrategy.test.ts
+++ b/src/scraper/strategies/WebScraperStrategy.test.ts
@@ -492,4 +492,29 @@ describe("WebScraperStrategy", () => {
     expect(docCall![0].document.content).toContain(expectedMarkdown);
     expect(docCall![0].document.metadata.title).toBe(expectedTitle);
   }, 10000); // Set timeout to 10 seconds for Playwright test
+
+  it("should forward custom headers to HttpFetcher", async () => {
+    const progressCallback = vi.fn();
+    const testUrl = "https://example.com";
+    options.url = testUrl;
+    options.headers = {
+      Authorization: "Bearer test-token",
+      "X-Test-Header": "test-value",
+    };
+    mockFetchFn.mockResolvedValue({
+      content: "<html><body>Header Test</body></html>",
+      mimeType: "text/html",
+      source: testUrl,
+    });
+    await strategy.scrape(options, progressCallback);
+    expect(mockFetchFn).toHaveBeenCalledWith(
+      testUrl,
+      expect.objectContaining({
+        headers: {
+          Authorization: "Bearer test-token",
+          "X-Test-Header": "test-value",
+        },
+      }),
+    );
+  });
 });
diff --git a/src/tools/ScrapeTool.test.ts b/src/tools/ScrapeTool.test.ts
index 7068b5b..83def4e 100644
--- a/src/tools/ScrapeTool.test.ts
+++ b/src/tools/ScrapeTool.test.ts
@@ -182,5 +182,26 @@ describe("ScrapeTool", () => {
     expect(mockManagerInstance.enqueueJob).toHaveBeenCalledOnce(); // Job was still enqueued
   });
 
-  // --- Callback Tests --- (Removed as onProgress is deprecated and removed)
+  it("should pass custom headers to the pipeline manager", async () => {
+    const options: ScrapeToolOptions = {
+      ...getBaseOptions("2.0.0"),
+      options: {
+        headers: {
+          Authorization: "Bearer test-token",
+          "X-Custom-Header": "custom-value",
+        },
+      },
+    };
+    await scrapeTool.execute(options);
+    expect(mockManagerInstance.enqueueJob).toHaveBeenCalledWith(
+      "test-lib",
+      "2.0.0",
+      expect.objectContaining({
+        headers: {
+          Authorization: "Bearer test-token",
+          "X-Custom-Header": "custom-value",
+        },
+      }),
+    );
+  });
 });