Repository: arabold/docs-mcp-server (ts)
Issue: Bug: HtmlProcessor fails to extract title when <title> tag has attributes
## Issue
The current implementation in `src/scraper/processor/HtmlProcessor.ts` uses a simple regex to extract page titles which fails when the `<title>` tag contains attributes.

## Current Implementation
```typescript
const titleMatch = htmlContent.match(/<title>([^<]+)</title>/i);
const title = titleMatch?.[1] || "Untitled";
```

## Problem
This regex pattern fails to extract titles from HTML where the `<title>` tag includes attributes. For example:
```html
<title lang="en">Example Page</title>
```

## Impact
When scraping documentation pages that use attributes in their `<title>` tags (e.g., for language specification), the processor fails to extract the proper title and falls back to "Untitled".

## Proposed Solution
Use the existing JSDOM instance's `document.title` property instead of regex matching. The JSDOM instance is already created in the `process` method:

```typescript
const window = new JSDOM(content.content, { url: content.source }).window;
// ...
const title = window.document.title || "Untitled";
```

This approach would be more robust as it properly handles HTML parsing and automatically accounts for any attributes in the title tag.
Pull request description: fix(scraper): use JSDOM title property for robust HTML title extraction
Replaces the fragile regex-based title extraction in HtmlProcessor with the more robust window.document.title property provided by JSDOM.
This correctly handles cases where the <title> tag contains attributes.

Adds a test case to verify title extraction with attributes.

Fixes #41

Patch:
diff --git a/src/scraper/processor/HtmlProcessor.ts b/src/scraper/processor/HtmlProcessor.ts
index 7220e11..b477135 100644
--- a/src/scraper/processor/HtmlProcessor.ts
+++ b/src/scraper/processor/HtmlProcessor.ts
@@ -162,11 +162,10 @@ export class HtmlProcessor implements ContentProcessor {
         ? content.content
         : content.content.toString((content.encoding as BufferEncoding) || "utf-8");
 
-    // Find title
-    const titleMatch = htmlContent.match(/<title>([^<]+)<\/title>/i);
-    const title = titleMatch?.[1] || "Untitled";
+    const window = new JSDOM(htmlContent, { url: content.source }).window;
 
-    const window = new JSDOM(content.content, { url: content.source }).window;
+    // Extract title using JSDOM
+    const title = window.document.title || "Untitled";
 
     const purify = createDOMPurify(window as unknown as WindowLike);
     const purifiedContent = purify.sanitize(htmlContent, {


Test patch:
diff --git a/src/scraper/processor/HtmlProcessor.test.ts b/src/scraper/processor/HtmlProcessor.test.ts
index dc796ca..3c663bd 100644
--- a/src/scraper/processor/HtmlProcessor.test.ts
+++ b/src/scraper/processor/HtmlProcessor.test.ts
@@ -23,6 +23,20 @@ describe("HtmlProcessor", () => {
     expect(result.links).toEqual([]); // No links in this example
   });
 
+  it("should process HTML with attributes in the title tag", async () => {
+    const processor = new HtmlProcessor();
+    const rawContent: RawContent = {
+      content:
+        '<html><head><title lang="en">Title With Attributes</title></head><body><h1>Hello</h1></body></html>',
+      mimeType: "text/html",
+      source: "https://example.com",
+    };
+
+    const result = await processor.process(rawContent);
+    expect(result.title).toBe("Title With Attributes");
+    expect(result.content).toContain("# Hello");
+  });
+
   it("should extract links", async () => {
     const processor = new HtmlProcessor();
     const rawContent: RawContent = {