Repository: projectdiscovery/httpx

Issue:
html: open stack of elements exceeds 512 nodes
httpx crashes with a Go HTML parser panic when scanning a specific URL.
The crash occurs inside the PageTypeClassifier (htmlToText) when encountering deeply nested or minified HTML-like content (in this case, a large JS file).
url causing the panic:

[url ](https://unpkg.com/three@0.150.0/build/three.min.js)

command used
```
echo  https://unpkg.com/three@0.150.0/build/three.min.js | httpx  -title -tech-detect -status-code -location -content-length -cname -web-server -follow-redirects -websocket
```

Panic Output
```
panic: html: open stack of elements exceeds 512 nodes

goroutine 3033 [running]:
github.com/projectdiscovery/httpx/common/pagetypeclassifier.htmlToText(...)
        /home/runner/work/httpx/httpx/common/pagetypeclassifier/pagetypeclassifier.go:36
github.com/projectdiscovery/httpx/common/pagetypeclassifier.(*PageTypeClassifier).Classify(0xc0096fb488, {0xc00f0d6000?, 0xc?})
        /home/runner/work/httpx/httpx/common/pagetypeclassifier/pagetypeclassifier.go:26 +0x6f
github.com/projectdiscovery/httpx/runner.(*Runner).analyze(_, _, {_, _}, {{0xc00b60e200, 0x32}, {0x0, 0x0}, {0x0, 0x0}}, ...)
        /home/runner/work/httpx/httpx/runner/runner.go:2349 +0x7555
github.com/projectdiscovery/httpx/runner.(*Runner).process.func1({{0xc00b60e200, 0x32}, {0x0, 0x0}, {0x0, 0x0}}, {0x1686161?, 0x0?}, {0xc00b60e200, 0x5})
        /home/runner/work/httpx/httpx/runner/runner.go:1444 +0x125
created by github.com/projectdiscovery/httpx/runner.(*Runner).process in goroutine 1
        /home/runner/work/httpx/httpx/runner/runner.go:1442 +0x8a6

```

Hints from issue comments:
@ehsandeep: @alonek1 fixed in https://github.com/projectdiscovery/httpx/pull/2330, will be included in next release.

Solution patch:
diff --git a/common/pagetypeclassifier/pagetypeclassifier.go b/common/pagetypeclassifier/pagetypeclassifier.go
index 13fb60b..5e333b1 100644
--- a/common/pagetypeclassifier/pagetypeclassifier.go
+++ b/common/pagetypeclassifier/pagetypeclassifier.go
@@ -2,9 +2,9 @@ package pagetypeclassifier
 
 import (
 	_ "embed"
-	"sync"
-
 	"fmt"
+	"strings"
+	"sync"
 
 	htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2"
 	"github.com/microcosm-cc/bluemonday"
@@ -41,30 +41,29 @@ var (
 	sanitizerPolicyOnce sync.Once
 )
 
-// getSanitizerPolicy returns an aggressive HTML sanitizer policy that strips
-// most elements to reduce nesting depth and prevent parser stack overflow.
+// getSanitizerPolicy returns an ultra-aggressive HTML sanitizer policy that strips
+// almost all elements to minimize nesting depth and prevent parser stack overflow.
 func getSanitizerPolicy() *bluemonday.Policy {
 	sanitizerPolicyOnce.Do(func() {
 		p := bluemonday.NewPolicy()
-		// Allow only basic text elements with minimal nesting
-		// This aggressive policy helps reduce nesting depth significantly
-		p.AllowElements("p", "br", "div", "span", "h1", "h2", "h3", "h4", "h5", "h6")
-		p.AllowElements("strong", "em", "b", "i", "u")
-		p.AllowElements("ul", "ol", "li")
-		p.AllowElements("blockquote", "pre", "code")
-		// Allow basic attributes but no style (which can cause nesting issues)
-		p.AllowStandardAttributes()
+		// Ultra-aggressive policy: Allow only the most basic text elements
+		// to minimize nesting and reduce parser stack depth
+		p.AllowElements("p", "br", "h1", "h2", "h3", "h4", "h5", "h6")
+		p.AllowElements("strong", "em", "b", "i")
+		// Remove div, span, ul, ol, li as they can create deep nesting
+		// No attributes allowed to prevent style-based nesting issues
 		sanitizerPolicy = p
 	})
 	return sanitizerPolicy
 }
 
-// htmlToText safely converts HTML to text and protects against panics from Go's HTML parser.
+// htmlToText safely converts HTML to text with multiple fallback strategies.
 // The 512 node limit in golang.org/x/net/html is hardcoded and cannot be increased.
 // Strategy:
-// 1. Always sanitize HTML with bluemonday first to remove useless elements and reduce nesting
-// 2. Convert sanitized HTML to markdown
-// 3. If conversion panics, recover and return empty string with error
+// 1. Length limit the input HTML to prevent massive documents
+// 2. Sanitize HTML aggressively with bluemonday to reduce nesting
+// 3. Convert sanitized HTML to markdown with panic recovery
+// 4. If conversion fails, fallback to plain text extraction
 func htmlToText(html string) (text string, err error) {
 	defer func() {
 		if r := recover(); r != nil {
@@ -73,19 +72,85 @@ func htmlToText(html string) (text string, err error) {
 		}
 	}()
 
-	// First, sanitize HTML with bluemonday to strip useless elements and reduce nesting
+	// Limit input size to prevent processing extremely large HTML documents
+	const maxHTMLSize = 1024 * 1024 // 1MB limit
+	if len(html) > maxHTMLSize {
+		html = html[:maxHTMLSize]
+	}
+
+	// First, sanitize HTML with ultra-aggressive bluemonday policy
 	sanitizedHTML := getSanitizerPolicy().Sanitize(html)
 
-	// If sanitization failed or produced empty result, return empty
+	// If sanitization failed or produced empty result, try plain text fallback
 	if sanitizedHTML == "" {
-		return "", nil
+		return extractPlainText(html), nil
 	}
 
 	// Convert sanitized HTML to markdown
 	text, err = htmltomarkdown.ConvertString(sanitizedHTML)
-	if err != nil || text == "" {
-		return "", err
+	if err != nil {
+		// If markdown conversion fails, fallback to plain text extraction
+		return extractPlainText(sanitizedHTML), nil
 	}
+	
+	if text == "" {
+		// If result is empty, try plain text fallback
+		return extractPlainText(sanitizedHTML), nil
+	}
+
+	return text, nil
+}
 
-	return
+// extractPlainText is a simple fallback that extracts text content without HTML parsing
+// This is used when the HTML parser fails due to complexity or nesting depth
+func extractPlainText(html string) string {
+	// Simple regex-based text extraction as fallback
+	// Remove script and style tags first
+	text := html
+	
+	// Remove script tags and content
+	for {
+		start := strings.Index(text, "<script")
+		if start == -1 {
+			break
+		}
+		end := strings.Index(text[start:], "</script>")
+		if end == -1 {
+			text = text[:start]
+			break
+		}
+		text = text[:start] + text[start+end+9:]
+	}
+	
+	// Remove style tags and content
+	for {
+		start := strings.Index(text, "<style")
+		if start == -1 {
+			break
+		}
+		end := strings.Index(text[start:], "</style>")
+		if end == -1 {
+			text = text[:start]
+			break
+		}
+		text = text[:start] + text[start+end+8:]
+	}
+	
+	// Simple HTML tag removal (not perfect but safe)
+	result := ""
+	inTag := false
+	for _, char := range text {
+		if char == '<' {
+			inTag = true
+		} else if char == '>' {
+			inTag = false
+			result += " " // Replace tags with spaces
+		} else if !inTag {
+			result += string(char)
+		}
+	}
+	
+	// Clean up multiple spaces
+	words := strings.Fields(result)
+	return strings.Join(words, " ")
 }
diff --git a/common/stringz/stringz.go b/common/stringz/stringz.go
index 6cc889c..5033e1c 100644
--- a/common/stringz/stringz.go
+++ b/common/stringz/stringz.go
@@ -83,6 +83,14 @@ func AddURLDefaultPort(rawURL string) string {
 	if err != nil {
 		return rawURL
 	}
+	// Force default port to be added if not present
+	if u.Port() == "" {
+		if u.Scheme == urlutil.HTTP {
+			u.UpdatePort("80")
+		} else if u.Scheme == urlutil.HTTPS {
+			u.UpdatePort("443")
+		}
+	}
 	return u.String()
 }
 
diff --git a/runner/runner.go b/runner/runner.go
index 2035199..15dbb62 100644
--- a/runner/runner.go
+++ b/runner/runner.go
@@ -1723,7 +1723,11 @@ retry:
 	}
 
 	builder := &strings.Builder{}
-	builder.WriteString(stringz.RemoveURLDefaultPort(fullURL))
+	if scanopts.LeaveDefaultPorts {
+		builder.WriteString(stringz.AddURLDefaultPort(fullURL))
+	} else {
+		builder.WriteString(stringz.RemoveURLDefaultPort(fullURL))
+	}
 
 	if r.options.Probe {
 		builder.WriteString(" [")



Test patch:
diff --git a/common/pagetypeclassifier/pagetypeclassifier_test.go b/common/pagetypeclassifier/pagetypeclassifier_test.go
index 944b12c..baf2116 100644
--- a/common/pagetypeclassifier/pagetypeclassifier_test.go
+++ b/common/pagetypeclassifier/pagetypeclassifier_test.go
@@ -1,6 +1,7 @@
 package pagetypeclassifier
 
 import (
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -56,13 +57,13 @@ func TestPageTypeClassifier(t *testing.T) {
 		`))
 	})
 
-	t.Run("test panic recovery with deeply nested HTML", func(t *testing.T) {
+	t.Run("test resilience with deeply nested HTML", func(t *testing.T) {
 		epc, err := New()
 		require.NoError(t, err)
 		require.NotNil(t, epc)
 
-		// Generate deeply nested HTML that exceeds the 512 node stack limit
-		// This should trigger a panic in the HTML parser, which we recover from
+		// Generate deeply nested HTML that would have exceeded the 512 node stack limit
+		// With our enhanced sanitization and fallback mechanisms, this should now work
 		deeplyNestedHTML := "<div>"
 		for i := 0; i < 600; i++ {
 			deeplyNestedHTML += "<div><span>"
@@ -73,13 +74,15 @@ func TestPageTypeClassifier(t *testing.T) {
 		}
 		deeplyNestedHTML += "</div>"
 
-		// Should not panic and should return "other" when htmlToText returns empty string
+		// Should not panic and should successfully classify the content
 		result := epc.Classify(deeplyNestedHTML)
-		require.Equal(t, "other", result)
+		require.NotEmpty(t, result)
+		// Should be able to extract and classify the text content
+		require.NotEqual(t, "", result)
 	})
 
 	t.Run("test htmlToText with deeply nested HTML", func(t *testing.T) {
-		// Generate deeply nested HTML that exceeds the 512 node stack limit
+		// Generate deeply nested HTML that would have exceeded the 512 node stack limit
 		deeplyNestedHTML := "<div>"
 		for i := 0; i < 600; i++ {
 			deeplyNestedHTML += "<div><span>"
@@ -90,10 +93,11 @@ func TestPageTypeClassifier(t *testing.T) {
 		}
 		deeplyNestedHTML += "</div>"
 
-		// Should not panic and should return empty string with error on panic
+		// Should not panic and should successfully extract text with enhanced sanitization
 		result, err := htmlToText(deeplyNestedHTML)
-		require.Error(t, err)
-		require.Equal(t, "", result)
+		require.NoError(t, err)
+		require.NotEmpty(t, result)
+		require.Contains(t, result, "Some text content")
 	})
 
 	t.Run("test htmlToText with normal HTML", func(t *testing.T) {
@@ -102,4 +106,38 @@ func TestPageTypeClassifier(t *testing.T) {
 		require.NoError(t, err)
 		require.NotEmpty(t, result)
 	})
+
+	t.Run("test htmlToText with extremely large HTML", func(t *testing.T) {
+		// Create a very large HTML document (over 1MB)
+		largeContent := strings.Repeat("<p>This is a test paragraph with some content. ", 50000)
+		largeHTML := "<html><body>" + largeContent + "</body></html>"
+		
+		// Should handle large documents without panic
+		result, err := htmlToText(largeHTML)
+		require.NoError(t, err)
+		require.NotEmpty(t, result)
+	})
+
+	t.Run("test extractPlainText fallback", func(t *testing.T) {
+		htmlWithScriptAndStyle := `<html>
+			<head>
+				<style>body { color: red; }</style>
+				<script>alert('test');</script>
+			</head>
+			<body>
+				<h1>Title</h1>
+				<p>Some <strong>important</strong> content here</p>
+				<div><span>Nested content</span></div>
+			</body>
+		</html>`
+		
+		result := extractPlainText(htmlWithScriptAndStyle)
+		require.NotEmpty(t, result)
+		require.Contains(t, result, "Title")
+		require.Contains(t, result, "important")
+		require.Contains(t, result, "content")
+		// Should not contain script or style content
+		require.NotContains(t, result, "alert")
+		require.NotContains(t, result, "color: red")
+	})
 }