Repository: projectdiscovery/httpx (go)

Issue:
Add feature to exclude the same http response 
while scanning wide scope there are a lot of urls have the same base code so they have the same http response so it's time to let httpx exclude these urls based on comparing every http response with others


Pull request:
filter out near duplicate responses
Closes #758

```console
$ go run . -u https://httpbin.org -path xml,json,html,html -fd -v

    __    __  __       _  __
   / /_  / /_/ /_____ | |/ /
  / __ \/ __/ __/ __ \|   /
 / / / / /_/ /_/ /_/ /   |
/_/ /_/\__/\__/ .___/_/|_|
             /_/

                projectdiscovery.io

[INF] Current httpx version v1.6.8 (latest)
[WRN] UI Dashboard is disabled, Use -dashboard option to enable
https://httpbin.org/html
[WRN] Skipping duplicate response with simhash 9850149872654300798
https://httpbin.org/xml
https://httpbin.org/json
```

Patch:
diff --git a/runner/options.go b/runner/options.go
index 866970e..c09944d 100644
--- a/runner/options.go
+++ b/runner/options.go
@@ -187,6 +187,7 @@ type Options struct {
 	OutputMatchContentLength  string
 	OutputFilterStatusCode    string
 	OutputFilterErrorPage     bool
+	FilterOutDuplicates       bool
 	OutputFilterContentLength string
 	InputRawRequest           string
 	rawRequest                string
@@ -409,6 +410,7 @@ func ParseOptions() *Options {
 	flagSet.CreateGroup("filters", "Filters",
 		flagSet.StringVarP(&options.OutputFilterStatusCode, "filter-code", "fc", "", "filter response with specified status code (-fc 403,401)"),
 		flagSet.BoolVarP(&options.OutputFilterErrorPage, "filter-error-page", "fep", false, "filter response with ML based error page detection"),
+		flagSet.BoolVarP(&options.FilterOutDuplicates, "filter-duplicates", "fd", false, "filter out near-duplicate responses (only first response is retained)"),
 		flagSet.StringVarP(&options.OutputFilterContentLength, "filter-length", "fl", "", "filter response with specified content length (-fl 23,33)"),
 		flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "filter response body with specified line count (-flc 423,532)"),
 		flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "filter response body with specified word count (-fwc 423,532)"),
diff --git a/runner/runner.go b/runner/runner.go
index 77c05be..9a2cc54 100644
--- a/runner/runner.go
+++ b/runner/runner.go
@@ -29,6 +29,7 @@ import (
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/corona10/goimagehash"
+	"github.com/mfonda/simhash"
 	asnmap "github.com/projectdiscovery/asnmap/libs"
 	"github.com/projectdiscovery/fastdialer/fastdialer"
 	"github.com/projectdiscovery/httpx/common/customextract"
@@ -65,6 +66,7 @@ import (
 	"github.com/projectdiscovery/httpx/common/stringz"
 	"github.com/projectdiscovery/mapcidr"
 	"github.com/projectdiscovery/rawhttp"
+	converstionutil "github.com/projectdiscovery/utils/conversion"
 	fileutil "github.com/projectdiscovery/utils/file"
 	pdhttputil "github.com/projectdiscovery/utils/http"
 	iputil "github.com/projectdiscovery/utils/ip"
@@ -84,8 +86,9 @@ type Runner struct {
 	ratelimiter        ratelimit.Limiter
 	HostErrorsCache    gcache.Cache[string, int]
 	browser            *Browser
-	pageTypeClassifier *pagetypeclassifier.PageTypeClassifier
+	pageTypeClassifier *pagetypeclassifier.PageTypeClassifier // Include this for general page classification
 	pHashClusters      []pHashCluster
+	simHashes          gcache.Cache[uint64, struct{}] // Include simHashes for efficient duplicate detection
 	httpApiEndpoint    *Server
 }
 
@@ -358,6 +361,7 @@ func New(options *Options) (*Runner, error) {
 		runner.HostErrorsCache = gc
 	}
 
+	runner.simHashes = gcache.New[uint64, struct{}](1000).ARC().Build()
 	runner.pageTypeClassifier = pagetypeclassifier.New()
 
 	if options.HttpApiEndpoint != "" {
@@ -438,7 +442,7 @@ func (r *Runner) prepareInput() {
 	// check if input target host(s) have been provided
 	if len(r.options.InputTargetHost) > 0 {
 		for _, target := range r.options.InputTargetHost {
-			expandedTarget := r.countTargetFromRawTarget(target)
+			expandedTarget, _ := r.countTargetFromRawTarget(target)
 			if expandedTarget > 0 {
 				numHosts += expandedTarget
 				r.hm.Set(target, nil) //nolint
@@ -514,6 +518,24 @@ func (r *Runner) seen(k string) bool {
 	return ok
 }
 
+func (r *Runner) duplicate(result *Result) bool {
+	respSimHash := simhash.Simhash(simhash.NewWordFeatureSet(converstionutil.Bytes(result.Raw)))
+	if r.simHashes.Has(respSimHash) {
+		gologger.Debug().Msgf("Skipping duplicate response with simhash %d for URL %s\n", respSimHash, result.URL)
+		return true
+	}
+
+	for simHash := range r.simHashes.GetALL(false) {
+		// lower threshold for increased precision
+		if simhash.Compare(simHash, respSimHash) <= 3 {
+			gologger.Debug().Msgf("Skipping near-duplicate response with simhash %d for URL %s\n", respSimHash, result.URL)
+			return true
+		}
+	}
+	_ = r.simHashes.Set(respSimHash, struct{}{})
+	return false
+}
+
 func (r *Runner) testAndSet(k string) bool {
 	// skip empty lines
 	k = strings.TrimSpace(k)
@@ -581,7 +603,7 @@ func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) {
 	for scanner.Scan() {
 		target := strings.TrimSpace(scanner.Text())
 		// Used just to get the exact number of targets
-		expandedTarget := r.countTargetFromRawTarget(target)
+		expandedTarget, _ := r.countTargetFromRawTarget(target)
 		if expandedTarget > 0 {
 			numTargets += expandedTarget
 			r.hm.Set(target, nil) //nolint
@@ -591,12 +613,12 @@ func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) {
 	return numTargets, err
 }
 
-func (r *Runner) countTargetFromRawTarget(rawTarget string) (numTargets int) {
+func (r *Runner) countTargetFromRawTarget(rawTarget string) (numTargets int, err error) {
 	if rawTarget == "" {
-		return 0
+		return 0, nil
 	}
 	if _, ok := r.hm.Get(rawTarget); ok {
-		return 0
+		return 0, nil
 	}
 
 	expandedTarget := 0
@@ -606,14 +628,17 @@ func (r *Runner) countTargetFromRawTarget(rawTarget string) (numTargets int) {
 			expandedTarget = int(ipsCount)
 		}
 	case asn.IsASN(rawTarget):
-		cidrs, _ := asn.GetCIDRsForASNNum(rawTarget)
+		cidrs, err := asn.GetCIDRsForASNNum(rawTarget)
+		if err != nil {
+			return 0, err
+		}
 		for _, cidr := range cidrs {
 			expandedTarget += int(mapcidr.AddressCountIpnet(cidr))
 		}
 	default:
 		expandedTarget = 1
 	}
-	return expandedTarget
+	return expandedTarget, nil
 }
 
 var (
@@ -884,6 +909,11 @@ func (r *Runner) RunEnumeration() {
 				logFilteredErrorPage(r.options.OutputFilterErrorPagePath, resp.URL)
 				continue
 			}
+
+			if r.options.FilterOutDuplicates && r.duplicate(&resp) {
+				continue
+			}
+
 			if len(r.options.filterStatusCode) > 0 && sliceutil.Contains(r.options.filterStatusCode, resp.StatusCode) {
 				continue
 			}


Test patch:
diff --git a/runner/runner_test.go b/runner/runner_test.go
index 08698b2..f456c1a 100644
--- a/runner/runner_test.go
+++ b/runner/runner_test.go
@@ -7,6 +7,8 @@ import (
 
 	_ "github.com/projectdiscovery/fdmax/autofdmax"
 	"github.com/projectdiscovery/httpx/common/httpx"
+	"github.com/projectdiscovery/mapcidr/asn"
+	stringsutil "github.com/projectdiscovery/utils/strings"
 	"github.com/stretchr/testify/require"
 )
 
@@ -106,11 +108,17 @@ func TestRunner_asn_targets(t *testing.T) {
 	for _, ip := range ips {
 		expected = append(expected, httpx.Target{Host: ip})
 	}
+
+	if _, err := asn.GetIPAddressesAsStream(input); err != nil && stringsutil.ContainsAnyI(err.Error(), "unauthorized: 401") {
+		t.Skip("skipping asn test due to missing/invalid api key")
+		return
+	}
+
 	got := []httpx.Target{}
 	for target := range r.targets(r.hp, input) {
 		got = append(got, target)
 	}
-	require.ElementsMatch(t, expected, got, "could not exepcted output")
+	require.ElementsMatch(t, expected, got, "could not get expected output")
 }
 
 func TestRunner_countTargetFromRawTarget(t *testing.T) {
@@ -120,32 +128,41 @@ func TestRunner_countTargetFromRawTarget(t *testing.T) {
 
 	input := "example.com"
 	expected := 1
-	got := r.countTargetFromRawTarget(input)
+	got, err := r.countTargetFromRawTarget(input)
+	require.Nil(t, err, "could not count targets")
 	require.Equal(t, expected, got, "got wrong output")
 
 	input = "example.com"
 	expected = 0
 	err = r.hm.Set(input, nil)
 	require.Nil(t, err, "could not set value to hm")
-	got = r.countTargetFromRawTarget(input)
+	got, err = r.countTargetFromRawTarget(input)
+	require.Nil(t, err, "could not count targets")
+	require.Equal(t, expected, got, "got wrong output")
+
+	input = "173.0.84.0/24"
+	expected = 256
+	got, err = r.countTargetFromRawTarget(input)
+	require.Nil(t, err, "could not count targets")
 	require.Equal(t, expected, got, "got wrong output")
 
 	input = ""
 	expected = 0
-	got = r.countTargetFromRawTarget(input)
+	got, err = r.countTargetFromRawTarget(input)
+	require.Nil(t, err, "could not count targets")
 	require.Equal(t, expected, got, "got wrong output")
 
 	if os.Getenv("PDCP_API_KEY") != "" {
 		input = "AS14421"
 		expected = 256
-		got = r.countTargetFromRawTarget(input)
+		got, err = r.countTargetFromRawTarget(input)
+		if err != nil && stringsutil.ContainsAnyI(err.Error(), "unauthorized: 401") {
+			t.Skip("skipping asn test due to missing/invalid api key")
+			return
+		}
+		require.Nil(t, err, "could not count targets")
 		require.Equal(t, expected, got, "got wrong output")
 	}
-
-	input = "173.0.84.0/24"
-	expected = 256
-	got = r.countTargetFromRawTarget(input)
-	require.Equal(t, expected, got, "got wrong output")
 }
 
 func TestRunner_urlWithComma_targets(t *testing.T) {