Repository: projectdiscovery/httpx (go)

Issue:
Issue with urls containing commas
### Discussed in https://github.com/projectdiscovery/httpx/discussions/1001

<div type='discussions-op-text'>

<sup>Originally posted by **molitona** February 25, 2023</sup>
hi comma in url broke httpx, if https://xx.xx?perms=X,Y,Z is used httpx will think X,Y,Z are paths but they're part of url as options to param perms.</div>

Pull request:
Improving split with URLs
Closes #1002 

Note: implemented only for URL from a file as the tool is widely adopted with comma-separated targets via stdin/CLI argument

```console
$ cat list.txt
http://192.168.1.2?a=b,c
$ httpx -v -l test.txt
...
[DBG] Failed 'http://192.168.1.2?a=b,c': GET http://192.168.1.2?a=b,c giving up after 1 attempts: Get "http://192.168.1.2?a=b,c": context deadline exceeded (Client.Timeout exceeded while awaiting headers)
```

Patch:
diff --git a/go.mod b/go.mod
index 9ecddf7..1aac4ae 100644
--- a/go.mod
+++ b/go.mod
@@ -16,16 +16,16 @@ require (
 	github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1
 	github.com/projectdiscovery/clistats v0.0.12
 	github.com/projectdiscovery/fdmax v0.0.4
-	github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c
+	github.com/projectdiscovery/goconfig v0.0.1
 	github.com/projectdiscovery/goflags v0.1.7
 	github.com/projectdiscovery/gologger v1.1.8
 	github.com/projectdiscovery/hmap v0.0.8
 	github.com/projectdiscovery/iputil v0.0.2 // indirect
 	github.com/projectdiscovery/mapcidr v1.1.0
-	github.com/projectdiscovery/rawhttp v0.1.9-0.20230208140210-81c468e769ed
-	github.com/projectdiscovery/retryablehttp-go v1.0.11-0.20230208144003-54f3176b2916
+	github.com/projectdiscovery/rawhttp v0.1.9
+	github.com/projectdiscovery/retryablehttp-go v1.0.12
 	github.com/projectdiscovery/stringsutil v0.0.2 // indirect
-	github.com/projectdiscovery/wappalyzergo v0.0.83
+	github.com/projectdiscovery/wappalyzergo v0.0.84
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.4.0
 	go.etcd.io/bbolt v1.3.7 // indirect
@@ -43,13 +43,13 @@ require (
 	github.com/hdm/jarm-go v0.0.7
 	github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6
 	github.com/mitchellh/mapstructure v1.5.0
-	github.com/projectdiscovery/asnmap v1.0.0
+	github.com/projectdiscovery/asnmap v1.0.1
 	github.com/projectdiscovery/dsl v0.0.3
 	github.com/projectdiscovery/fastdialer v0.0.24
 	github.com/projectdiscovery/ratelimit v0.0.6
 	github.com/projectdiscovery/tlsx v1.0.4
-	github.com/projectdiscovery/utils v0.0.8-0.20230208160155-47c6b52af9f0
-	github.com/stretchr/testify v1.8.1
+	github.com/projectdiscovery/utils v0.0.13
+	github.com/stretchr/testify v1.8.2
 	go.uber.org/multierr v1.9.0
 	golang.org/x/exp v0.0.0-20230206171751-46f607a40771
 )
@@ -90,7 +90,6 @@ require (
 	github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4 // indirect
 	github.com/projectdiscovery/freeport v0.0.4 // indirect
 	github.com/projectdiscovery/networkpolicy v0.0.4 // indirect
-	github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect
 	github.com/projectdiscovery/retryabledns v1.0.21 // indirect
 	github.com/prometheus/client_golang v1.14.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
diff --git a/go.sum b/go.sum
index c255439..4593aca 100644
--- a/go.sum
+++ b/go.sum
@@ -257,8 +257,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/projectdiscovery/asnmap v1.0.0 h1:h9aUEHT3gEWgeTxDCd0UMxBw1yPthDwL1ogqUWnkBXo=
-github.com/projectdiscovery/asnmap v1.0.0/go.mod h1:m+qedSAZERz7Ds942hWANjU9kg4ADAikFHfwSXN1Ey8=
+github.com/projectdiscovery/asnmap v1.0.1 h1:mRFrFulfI5ZjYx0uyWcWKyCPRExDXInmBKpZlwxwRgE=
+github.com/projectdiscovery/asnmap v1.0.1/go.mod h1:qEowdtLF9JQqQ4/tHTt1HGAY0RnanLfU9DcAvrLic0o=
 github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4 h1:EsrQ/zkotVodSJLOch3pV/UYt1vQcwyIs5HX0sm1ljE=
 github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4/go.mod h1:5tNGQP9kOfW+X5+40pZP8aqPYLHs45nJkFaSHLxdeH8=
 github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 h1:QtTPPx0uu42AsQJiXT86/wqdHS7/iVcgz1VM38tjv20=
@@ -273,8 +273,8 @@ github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUd
 github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I=
 github.com/projectdiscovery/freeport v0.0.4 h1:H4VrK/7hUcC1zbg46zv9iSMBACBDpUqcHkV+FUyXISw=
 github.com/projectdiscovery/freeport v0.0.4/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE=
-github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k=
-github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk=
+github.com/projectdiscovery/goconfig v0.0.1 h1:36m3QjohZvemqh9bkJAakaHsm9iEZ2AcQSS18+0QX/s=
+github.com/projectdiscovery/goconfig v0.0.1/go.mod h1:CPO25zR+mzTtyBrsygqsHse0sp/4vB/PjaHi9upXlDw=
 github.com/projectdiscovery/goflags v0.1.7 h1:KvMs1KnXMWRhJiJj6K+cWurTWpbDYYlo+1cyGvf1YCc=
 github.com/projectdiscovery/goflags v0.1.7/go.mod h1:yILgA7gbrHuTpIvMfikbivzoxkyxBD1Y5/PRHiGTIFk=
 github.com/projectdiscovery/gologger v1.1.8 h1:CFlCzGlqAhPqWIrAXBt1OVh5jkMs1qgoR/z4xhdzLNE=
@@ -289,22 +289,20 @@ github.com/projectdiscovery/networkpolicy v0.0.4 h1:zcGjEqZbyECZEdyCy1jVuwOS7Ww1
 github.com/projectdiscovery/networkpolicy v0.0.4/go.mod h1:DIXwKs3sQyfCoWHKRLQiRrEorSQW4Zrh4ftu7oDVK6w=
 github.com/projectdiscovery/ratelimit v0.0.6 h1:SAD2ArdT9F8NmbkAIZpl7DjNnbiXdUQLnMZt5dbVmZ0=
 github.com/projectdiscovery/ratelimit v0.0.6/go.mod h1:WFL6gIggPLTwYwDbxqQODuWrz/lcMP2E5ofKSAz3YwI=
-github.com/projectdiscovery/rawhttp v0.1.9-0.20230208140210-81c468e769ed h1:XbW1NY+l1B4qnDJyAqFKB7b7+hG+S6pDA4hkFsB5s20=
-github.com/projectdiscovery/rawhttp v0.1.9-0.20230208140210-81c468e769ed/go.mod h1:GoReErVAHUI24tw3AoEB/2C1xkvb5zHDI5qQwibJrik=
-github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f h1:HR3R/nhELwLXufUlO1ZkKVqrZl4lN1cWFBdN8RcMuLo=
-github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f/go.mod h1:3L0WfNIcVWXIDur8k+gKDLZLWY2F+rs0SQXtcn/3AYU=
+github.com/projectdiscovery/rawhttp v0.1.9 h1:hMMKIwsOcbUcgnVnHfp4LhuJamAq/Dom1oIwO+RljBs=
+github.com/projectdiscovery/rawhttp v0.1.9/go.mod h1:KZ2dPwu38NE6T5RIfrqN9r4vxA2GmWW7+NSV/qC+jOw=
 github.com/projectdiscovery/retryabledns v1.0.21 h1:vOpPQR1q8Z824uoA8JXCI/RyvDAssPeD68Onz9hP/ds=
 github.com/projectdiscovery/retryabledns v1.0.21/go.mod h1:6oTPKMRlKZ7lIIEzTH723K6RvNRjmm6fe9br4Dom3UI=
-github.com/projectdiscovery/retryablehttp-go v1.0.11-0.20230208144003-54f3176b2916 h1:9kdsj7NGuj/5treBvUbK+vQ8wQrUhi0LcIGvpfKrBpw=
-github.com/projectdiscovery/retryablehttp-go v1.0.11-0.20230208144003-54f3176b2916/go.mod h1:RWViUDjf9NTx1j8HatkstoSj2hE4xrrDIum1SsQqZfE=
+github.com/projectdiscovery/retryablehttp-go v1.0.12 h1:kjeXJ4V6ZwgoMeKV2dINERjxAsBPB/p5+NV2aSQbrrU=
+github.com/projectdiscovery/retryablehttp-go v1.0.12/go.mod h1:dnJK347etdVKz7ljhemUhBK6EpDnVf0U1O8dGj7MM+0=
 github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
 github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0=
 github.com/projectdiscovery/tlsx v1.0.4 h1:aVkoxl1Vq6Uh7Ietpv2X5TfXPRRBj3VGCtoelhj9BE0=
 github.com/projectdiscovery/tlsx v1.0.4/go.mod h1:N50JpF12eeLg+rcDRwIZ8r974MOvJZeV7M0rIxHWV2g=
-github.com/projectdiscovery/utils v0.0.8-0.20230208160155-47c6b52af9f0 h1:jGragVJHVeWtXcmxJGbDiVFrh3Q6Ud6krgwXuXHRwPs=
-github.com/projectdiscovery/utils v0.0.8-0.20230208160155-47c6b52af9f0/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ=
-github.com/projectdiscovery/wappalyzergo v0.0.83 h1:lD8piIbo7bbzsLDBhVpkBe0LVazg7QFdNjyECVMdWkw=
-github.com/projectdiscovery/wappalyzergo v0.0.83/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0=
+github.com/projectdiscovery/utils v0.0.13 h1:Bvguo7avXN+Zs9tHL+8+Qld65nwIE8Kr9qddErDDqmw=
+github.com/projectdiscovery/utils v0.0.13/go.mod h1:2CyxZXcx62NUiGJZZam23CpphqXy3kaomE9uvgHgkEo=
+github.com/projectdiscovery/wappalyzergo v0.0.84 h1:nf+yCMyRihinxYACycajSABW0wzLYAo09sS+nZGwOuU=
+github.com/projectdiscovery/wappalyzergo v0.0.84/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0=
 github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
 github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -353,8 +351,9 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI=
@@ -598,7 +597,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
 gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
 gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=
 gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
diff --git a/runner/runner.go b/runner/runner.go
index ee00135..d2159ba 100644
--- a/runner/runner.go
+++ b/runner/runner.go
@@ -1009,7 +1009,7 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target {
 			for _, ip := range ips {
 				results <- httpx.Target{Host: target, CustomIP: ip}
 			}
-		case strings.Index(target, ",") > 0:
+		case !stringsutil.HasPrefixAny(target, "http://", "https://") && stringsutil.ContainsAny(target, ","):
 			idxComma := strings.Index(target, ",")
 			results <- httpx.Target{Host: target[idxComma+1:], CustomHost: target[:idxComma]}
 		default:
@@ -1363,6 +1363,11 @@ retry:
 	} else {
 		// hp.Dialer.GetDialedIP would return only the last dialed one
 		ip = hp.Dialer.GetDialedIP(URL.Host)
+		if ip == "" {
+			if onlyHost, _, err := net.SplitHostPort(URL.Host); err == nil {
+				ip = hp.Dialer.GetDialedIP(onlyHost)
+			}
+		}
 	}
 
 	var asnResponse *AsnResponse


Test patch:
diff --git a/cmd/functional-test/testcases.txt b/cmd/functional-test/testcases.txt
index df81414..7a525c9 100644
--- a/cmd/functional-test/testcases.txt
+++ b/cmd/functional-test/testcases.txt
@@ -16,4 +16,5 @@ scanme.sh {{binary}} -silent -body 'a=b'
 scanme.sh {{binary}} -silent -exclude-cdn
 scanme.sh {{binary}} -silent -ports https:443
 scanme.sh {{binary}} -silent -ztls
-https://scanme.sh?a=1*1 {{binary}} -silent
\ No newline at end of file
+https://scanme.sh?a=1*1 {{binary}} -silent
+https://scanme.sh:443 {{binary}} -asn
\ No newline at end of file
diff --git a/runner/runner_test.go b/runner/runner_test.go
index 1b9abc7..f96d3f9 100644
--- a/runner/runner_test.go
+++ b/runner/runner_test.go
@@ -141,3 +141,20 @@ func TestRunner_countTargetFromRawTarget(t *testing.T) {
 	got = r.countTargetFromRawTarget(input)
 	require.Equal(t, expected, got, "got wrong output")
 }
+
+func TestRunner_urlWithComma_targets(t *testing.T) {
+	options := &Options{}
+	r, err := New(options)
+	require.Nil(t, err, "could not create httpx runner")
+	input := []string{"http://scanme.sh?a=1,2"}
+	expected := []httpx.Target{{
+		Host: "http://scanme.sh?a=1,2",
+	}}
+	got := []httpx.Target{}
+	for _, inp := range input {
+		for target := range r.targets(r.hp, inp) {
+			got = append(got, target)
+		}
+	}
+	require.ElementsMatch(t, expected, got, "could not exepcted output")
+}