Repository: projectdiscovery/httpx (language: go)

Issue:
out of memory error
<!-- 
1. Please search to see if an issue already exists for the bug you encountered.
2. For support requests, FAQs or "How to" questions, please use the GitHub Discussions section instead - https://github.com/projectdiscovery/httpx/discussions or
3. Join our discord server at https://discord.gg/projectdiscovery and post the question on the #httpx channel.
-->

<!-- ISSUES MISSING IMPORTANT INFORMATION MAY BE CLOSED WITHOUT INVESTIGATION. -->

### httpx version:
v1.2.3, also tested with the latest one v1.2.4

### Current Behavior:
I'm trying to scan working http/s servers from huge list and after some time "out of memory" error is thrown,
My server have 64gb of ram and it's not nearly using it fully while active scanning.

### Error

```
fatal error: runtime: out of memory

runtime stack:
runtime.throw({0xb7eae2?, 0x400000000?})
        /usr/local/go/src/runtime/panic.go:992 +0x71
runtime.sysMap(0xc4d5000000, 0x7f01cfffed38?, 0x7f01cfffeda0?)
        /usr/local/go/src/runtime/mem_linux.go:189 +0x11b
runtime.(*mheap).grow(0x11c4e80, 0x200000?)
        /usr/local/go/src/runtime/mheap.go:1404 +0x225
runtime.(*mheap).allocSpan(0x11c4e80, 0x200000, 0x0, 0x1)
        /usr/local/go/src/runtime/mheap.go:1170 +0x171
runtime.(*mheap).alloc.func1()
        /usr/local/go/src/runtime/mheap.go:912 +0x65
runtime.systemstack()
        /usr/local/go/src/runtime/asm_amd64.s:469 +0x49

goroutine 320007664 [running]:
runtime.systemstack_switch()
        /usr/local/go/src/runtime/asm_amd64.s:436 fp=0xc00e959df8 sp=0xc00e959df0 pc=0x4637a0
runtime.(*mheap).alloc(0x400000000?, 0x200000?, 0x80?)
        /usr/local/go/src/runtime/mheap.go:906 +0x65 fp=0xc00e959e40 sp=0xc00e959df8 pc=0x4281a5
runtime.(*mcache).allocLarge(0xc00c102600?, 0x3fffffe00, 0x1)
        /usr/local/go/src/runtime/mcache.go:213 +0x85 fp=0xc00e959e90 sp=0xc00e959e40 pc=0x4186a5
runtime.mallocgc(0x3fffffe00, 0xa72360, 0x1)
        /usr/local/go/src/runtime/malloc.go:1096 +0x5a5 fp=0xc00e959f08 sp=0xc00e959e90 pc=0x40ec65
runtime.makeslice(0xc48e16a000?, 0x7e00?, 0x7e00?)
        /usr/local/go/src/runtime/slice.go:103 +0x52 fp=0xc00e959f30 sp=0xc00e959f08 pc=0x44c9f2
bytes.makeSlice(0x3fffffe00)
        /usr/local/go/src/bytes/buffer.go:229 +0x65 fp=0xc00e959f80 sp=0xc00e959f30 pc=0x4f0345
bytes.(*Buffer).grow(0xc0014e67b0, 0x200)
        /usr/local/go/src/bytes/buffer.go:142 +0x11f fp=0xc00e959fc8 sp=0xc00e959f80 pc=0x4efc9f
bytes.(*Buffer).ReadFrom(0xc0014e67b0, {0x7f02101f1820, 0xc013a2e080})
        /usr/local/go/src/bytes/buffer.go:202 +0x45 fp=0xc00e95a020 sp=0xc00e959fc8 pc=0x4f0125
net/http/httputil.drainBody({0xccf830, 0xc013a2e080})
        /usr/local/go/src/net/http/httputil/dump.go:31 +0x7e fp=0xc00e95a088 sp=0xc00e95a020 pc=0x9d0f1e
net/http/httputil.DumpResponse(0xc00fcd3950, 0x1)
        /usr/local/go/src/net/http/httputil/dump.go:325 +0x95 fp=0xc00e95a0e0 sp=0xc00e95a088 pc=0x9d2bd5
github.com/projectdiscovery/httputil.DumpResponseHeadersAndRaw(0xc00fcd3950)
        /root/work/pkg/mod/github.com/projectdiscovery/httputil@v0.0.0-20210816170244-86fd46bc09f5/httputil.go:45 +0x10c fp=0xc00e95a1f8 sp=0xc00e95a0e0 pc=0x9d3c6c
github.com/projectdiscovery/httpx/common/httpx.(*HTTPX).Do(0xc0000ae370, 0xc0141e5530, {{0x0?, 0xb50268?}})
        /root/work/pkg/mod/github.com/projectdiscovery/httpx@v1.2.3/common/httpx/httpx.go:185 +0x23d fp=0xc00e95a3c8 sp=0xc00e95a1f8 pc=0x9e05fd
github.com/projectdiscovery/httpx/runner.(*Runner).analyze(_, _, {_, _}, {{0xc00003f7ef, 0x10}, {0x0, 0x0}, {0x0, 0x0}}, ...)
        /root/work/pkg/mod/github.com/projectdiscovery/httpx@v1.2.3/runner/runner.go:974 +0xc65 fp=0xc00e95b718 sp=0xc00e95a3c8 pc=0xa1a9c5
github.com/projectdiscovery/httpx/runner.(*Runner).process.func1({{0xc00003f7ef, 0x10}, {0x0, 0x0}, {0x0, 0x0}}, {0xb50268?, 0x3?}, {0xc01583f410, 0x4})
        /root/work/pkg/mod/github.com/projectdiscovery/httpx@v1.2.3/runner/runner.go:766 +0x13b fp=0xc00e95bf80 sp=0xc00e95b718 pc=0xa1923b
github.com/projectdiscovery/httpx/runner.(*Runner).process.func3()
        /root/work/pkg/mod/github.com/projectdiscovery/httpx@v1.2.3/runner/runner.go:792 +0x5c fp=0xc00e95bfe0 sp=0xc00e95bf80 pc=0xa190dc
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1571 +0x1 fp=0xc00e95bfe8 sp=0xc00e95bfe0 pc=0x4659c1
created by github.com/projectdiscovery/httpx/runner.(*Runner).process
        /root/work/pkg/mod/github.com/projectdiscovery/httpx@v1.2.3/runner/runner.go:764 +0xa3f
```


Discussion / hints:
Also I've cracked up ulimit limits,

![image](https://user-images.githubusercontent.com/110659900/183528197-7db40eec-bda2-4a9c-bb01-16b12e3b2e00.png)



Fix patch:
diff --git a/cmd/httpx/httpx.go b/cmd/httpx/httpx.go
index 8ddf3ce84..6890365fa 100644
--- a/cmd/httpx/httpx.go
+++ b/cmd/httpx/httpx.go
@@ -3,6 +3,8 @@ package main
 import (
 	"os"
 	"os/signal"
+	"runtime"
+	"runtime/pprof"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/httpx/runner"
@@ -12,6 +14,24 @@ func main() {
 	// Parse the command line flags and read config files
 	options := runner.ParseOptions()
 
+	// Profiling related code
+	if options.Memprofile != "" {
+		f, err := os.Create(options.Memprofile)
+		if err != nil {
+			gologger.Fatal().Msgf("profile: could not create memory profile %q: %v", options.Memprofile, err)
+		}
+		old := runtime.MemProfileRate
+		runtime.MemProfileRate = 4096
+		gologger.Print().Msgf("profile: memory profiling enabled (rate %d), %s", runtime.MemProfileRate, options.Memprofile)
+
+		defer func() {
+			_ = pprof.Lookup("heap").WriteTo(f, 0)
+			f.Close()
+			runtime.MemProfileRate = old
+			gologger.Print().Msgf("profile: memory profiling disabled, %s", options.Memprofile)
+		}()
+	}
+
 	httpxRunner, err := runner.New(options)
 	if err != nil {
 		gologger.Fatal().Msgf("Could not create runner: %s\n", err)
diff --git a/runner/options.go b/runner/options.go
index 920cf13e8..7f1641beb 100644
--- a/runner/options.go
+++ b/runner/options.go
@@ -231,6 +231,7 @@ type Options struct {
 	OutputMatchLinesCount     string
 	matchLinesCount           []int
 	OutputFilterLinesCount    string
+	Memprofile                string
 	filterLinesCount          []int
 	OutputWordsCount          bool
 	OutputMatchWordsCount     string
@@ -369,6 +370,7 @@ func ParseOptions() *Options {
 		flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "display response content in cli"),
 		flagSet.BoolVar(&options.Version, "version", false, "display httpx version"),
 		flagSet.BoolVar(&options.ShowStatistics, "stats", false, "display scan statistic"),
+		flagSet.StringVar(&options.Memprofile, "profile-mem", "", "optional httpx memory profile dump file"),
 		flagSet.BoolVar(&options.Silent, "silent", false, "silent mode"),
 		flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "verbose mode"),
 		flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 0, "number of seconds to wait between showing a statistics update (default: 5)"),


Test patch: