Repository: arabold/docs-mcp-server
Pull request #70 (2025-04-29 15:49:33)

feat: remove empty anchor links in HtmlToMarkdownMiddleware
This PR adds a custom rule to remove empty and invalid anchor links during HTML to Markdown conversion and includes a test case to validate the behavior.


Patch:
diff --git a/src/scraper/middleware/components/HtmlToMarkdownMiddleware.ts b/src/scraper/middleware/components/HtmlToMarkdownMiddleware.ts
index 93c9f80..ea8adf1 100644
--- a/src/scraper/middleware/components/HtmlToMarkdownMiddleware.ts
+++ b/src/scraper/middleware/components/HtmlToMarkdownMiddleware.ts
@@ -53,17 +53,28 @@ export class HtmlToMarkdownMiddleware implements ContentProcessorMiddleware {
           }
         }
 
-        const brElements = element.querySelectorAll("br");
-        if (brElements.length > 0) {
-          for (const br of brElements) {
-            br.replaceWith("\n");
-          }
+        const brElements = Array.from(element.querySelectorAll("br"));
+        for (const br of brElements) {
+          br.replaceWith("\n");
         }
         const text = element.textContent || "";
 
         return `\n\`\`\`${language}\n${text.replace(/^\n+|\n+$/g, "")}\n\`\`\`\n`;
       },
     });
+    this.turndownService.addRule("anchor", {
+      filter: ["a"],
+      replacement: (content, node) => {
+        const href = (node as HTMLElement).getAttribute("href");
+        if (!content || content === "#") {
+          return ""; // Remove if content is # or empty
+        }
+        if (!href) {
+          return content; // Preserve content if href is missing or empty
+        }
+        return `[${content}](${href})`; // Standard link conversion
+      },
+    });
   }
 
   /**


Test patch:
diff --git a/src/scraper/middleware/components/HtmlToMarkdownMiddleware.test.ts b/src/scraper/middleware/components/HtmlToMarkdownMiddleware.test.ts
index 5954a91..14086b9 100644
--- a/src/scraper/middleware/components/HtmlToMarkdownMiddleware.test.ts
+++ b/src/scraper/middleware/components/HtmlToMarkdownMiddleware.test.ts
@@ -217,4 +217,38 @@ describe("HtmlToMarkdownMiddleware", () => {
     turndownSpy.mockRestore();
     // No close needed
   });
+
+  it("should apply custom anchor rule to remove empty or invalid links", async () => {
+    const middleware = new HtmlToMarkdownMiddleware();
+    const html = `
+      <html><body>
+        <p>A <a href="http://valid.com">Valid Link</a>.</p>
+        <p>An empty link: <a href="http://empty.com"></a>.</p>
+        <p>A hash link: <a href="http://hash.com">#</a>.</p>
+        <p>A link with no href: <a>No Href</a>.</p>
+        <p>A link with empty href: <a href="">Empty Href</a>.</p>
+        <p>Mixed: <a href="http://another.com">Another Valid</a> and <a href="http://bad.com"></a> bad one.</p>
+      </body></html>`;
+    const context = createMockContext("text/html", html);
+    const next = vi.fn().mockResolvedValue(undefined);
+
+    await middleware.process(context, next);
+
+    expect(next).toHaveBeenCalledOnce();
+    // Note: The content inside removed anchors ('No Href', 'Empty Href') remains as plain text.
+    const expectedMarkdown = `A [Valid Link](http://valid.com).
+
+An empty link: .
+
+A hash link: .
+
+A link with no href: No Href.
+
+A link with empty href: Empty Href.
+
+Mixed: [Another Valid](http://another.com) and bad one.`;
+    expect(context.content).toBe(expectedMarkdown);
+    expect(context.contentType).toBe("text/markdown");
+    expect(context.errors).toHaveLength(0);
+  });
 });