Repository: projectdiscovery/httpx (language: go)

Issue:
add icon md5 support #1787
<!--
1. Please make sure to provide a detailed description with all the relevant information that might be required to start working on this feature.
2. In case you are not sure about your request or whether the particular feature is already supported or not, please start a discussion instead.
3. GitHub Discussion: https://github.com/projectdiscovery/httpx/discussions/categories/ideas
4. Join our discord server at https://discord.gg/projectdiscovery to discuss the idea on the #httpx channel.
-->

### Please describe your feature request:
<!-- A clear and concise description of feature to implement -->
return icon_md5 value as well
### Describe the use case of this feature:
<!-- A clear and concise description of the feature request's motivation and the use-cases in which it could be useful. -->
some open source fingerprint db use icon_md5 instead of icon_hash (mmh3)...

So, hope to support md5 (or return raw favicon value?)


Discussion / hints:

Fix patch:
diff --git a/common/stringz/stringz.go b/common/stringz/stringz.go
index 12a4760b1..58d3ee40f 100644
--- a/common/stringz/stringz.go
+++ b/common/stringz/stringz.go
@@ -2,7 +2,9 @@ package stringz
 
 import (
 	"bytes"
+	"crypto/md5"
 	"encoding/base64"
+	"encoding/hex"
 	"errors"
 	"net/http"
 	"net/url"
@@ -121,12 +123,19 @@ func murmurhash(data []byte) int32 {
 	return int32(hasher.Sum32())
 }
 
-func FaviconHash(data []byte) (int32, error) {
+// md5Hash returns the md5 hash of the data
+func md5Hash(data []byte) string {
+	hasher := md5.New()
+	hasher.Write(data)
+	return hex.EncodeToString(hasher.Sum(nil))
+}
+
+func FaviconHash(data []byte) (int32, string, error) {
 	if isContentTypeImage(data) {
-		return murmurhash(data), nil
+		return murmurhash(data), md5Hash(data), nil
 	}
 
-	return 0, errors.New("content type is not image")
+	return 0, "", errors.New("content type is not image")
 }
 
 func InsertInto(s string, interval int, sep rune) string {
diff --git a/runner/runner.go b/runner/runner.go
index 17932d1da..a66663f27 100644
--- a/runner/runner.go
+++ b/runner/runner.go
@@ -1948,11 +1948,11 @@ retry:
 		builder.WriteRune(']')
 	}
 
-	var faviconMMH3, faviconPath, faviconURL string
+	var faviconMMH3, faviconMD5, faviconPath, faviconURL string
 	var faviconData []byte
 	if scanopts.Favicon {
 		var err error
-		faviconMMH3, faviconPath, faviconData, faviconURL, err = r.HandleFaviconHash(hp, req, resp.Data, true)
+		faviconMMH3, faviconMD5, faviconPath, faviconData, faviconURL, err = r.HandleFaviconHash(hp, req, resp.Data, true)
 		if err == nil {
 			builder.WriteString(" [")
 			if !scanopts.OutputWithNoColor {
@@ -2199,6 +2199,7 @@ retry:
 		Technologies:     technologies,
 		FinalURL:         finalURL,
 		FavIconMMH3:      faviconMMH3,
+		FavIconMD5:       faviconMD5,
 		FaviconPath:      faviconPath,
 		FaviconURL:       faviconURL,
 		Hashes:           hashesMap,
@@ -2257,11 +2258,11 @@ func calculatePerceptionHash(screenshotBytes []byte) (uint64, error) {
 	return pHash.GetHash(), nil
 }
 
-func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, currentResp []byte, defaultProbe bool) (string, string, []byte, string, error) {
+func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, currentResp []byte, defaultProbe bool) (string, string, string, []byte, string, error) {
 	// Check if current URI is ending with .ico => use current body without additional requests
 	if path.Ext(req.URL.Path) == ".ico" {
-		hash, err := r.calculateFaviconHashWithRaw(currentResp)
-		return hash, req.URL.Path, currentResp, "", err
+		MMH3Hash, MD5Hash, err := r.calculateFaviconHashWithRaw(currentResp)
+		return MMH3Hash, MD5Hash, req.URL.Path, currentResp, "", err
 	}
 
 	// search in the response of the requested path for element and rel shortcut/mask/apple-touch icon
@@ -2269,12 +2270,12 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request,
 	// if not, any of link from other icons can be requested
 	potentialURLs, err := extractPotentialFavIconsURLs(currentResp)
 	if err != nil {
-		return "", "", nil, "", err
+		return "", "", "", nil, "", err
 	}
 
 	clone := req.Clone(context.Background())
 
-	var faviconHash, faviconPath, faviconURL string
+	var faviconMMH3, faviconMD5, faviconPath, faviconURL string
 	var faviconData []byte
 	errCount := 0
 	if len(potentialURLs) == 0 && defaultProbe {
@@ -2309,25 +2310,26 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request,
 			errCount++
 			continue
 		}
-		hash, err := r.calculateFaviconHashWithRaw(resp.Data)
+		MMH3Hash, MD5Hash, err := r.calculateFaviconHashWithRaw(resp.Data)
 		if err != nil {
 			continue
 		}
 		faviconURL = clone.URL.String()
 		faviconPath = potentialURL
-		faviconHash = hash
+		faviconMMH3 = MMH3Hash
+		faviconMD5 = MD5Hash
 		faviconData = resp.Data
 		break
 	}
-	return faviconHash, faviconPath, faviconData, faviconURL, nil
+	return faviconMMH3, faviconMD5, faviconPath, faviconData, faviconURL, nil
 }
 
-func (r *Runner) calculateFaviconHashWithRaw(data []byte) (string, error) {
-	hashNum, err := stringz.FaviconHash(data)
+func (r *Runner) calculateFaviconHashWithRaw(data []byte) (string, string, error) {
+	hashNum, md5Hash, err := stringz.FaviconHash(data)
 	if err != nil {
-		return "", errorutil.NewWithTag("favicon", "could not calculate favicon hash").Wrap(err)
+		return "", "", errorutil.NewWithTag("favicon", "could not calculate favicon hash").Wrap(err)
 	}
-	return fmt.Sprintf("%d", hashNum), nil
+	return fmt.Sprintf("%d", hashNum), md5Hash, nil
 }
 
 func extractPotentialFavIconsURLs(resp []byte) ([]string, error) {
diff --git a/runner/types.go b/runner/types.go
index 87d5f791d..9c986ae17 100644
--- a/runner/types.go
+++ b/runner/types.go
@@ -59,6 +59,7 @@ type Result struct {
 	Host               string                 `json:"host,omitempty" csv:"host"`
 	Path               string                 `json:"path,omitempty" csv:"path"`
 	FavIconMMH3        string                 `json:"favicon,omitempty" csv:"favicon"`
+	FavIconMD5         string                 `json:"favicon_md5,omitempty" csv:"favicon"`
 	FaviconPath        string                 `json:"favicon_path,omitempty" csv:"favicon_path"`
 	FaviconURL         string                 `json:"favicon_url,omitempty" csv:"favicon_url"`
 	FinalURL           string                 `json:"final_url,omitempty" csv:"final_url"`


Test patch: