Repository: raviqqe/muffet
Pull request #234 (2022-07-28 16:25:10)

Add --max-response-body-size argument with default
Avoids overloading fasthttp and causing a panic.

I'm still trying to track down the specific URL that causes the problem so I can suggest a fix to fasthttp, but in the mean time it would probably be a good idea for muffet to avoid trying to read really huge pages.

Picked the default on the basis that the average page load these days is a few MB, and that includes resources such as images.

raviqqe: Thank you for the PR!

Patch:
diff --git a/arguments.go b/arguments.go
index 8a2fc14..41301fd 100644
--- a/arguments.go
+++ b/arguments.go
@@ -13,6 +13,7 @@ type arguments struct {
 	BufferSize            int      `short:"b" long:"buffer-size" value-name:"<size>" default:"4096" description:"HTTP response buffer size in bytes"`
 	MaxConnections        int      `short:"c" long:"max-connections" value-name:"<count>" default:"512" description:"Maximum number of HTTP connections"`
 	MaxConnectionsPerHost int      `long:"max-connections-per-host" value-name:"<count>" default:"512" description:"Maximum number of HTTP connections per host"`
+	MaxResponseBodySize   int      `long:"max-response-body-size" value-name:"<size>" default:"10000000" description:"Maximum web page body size to read"`
 	RawExcludedPatterns   []string `short:"e" long:"exclude" value-name:"<pattern>..." description:"Exclude URLs matched with given regular expressions"`
 	RawIncludedPatterns   []string `short:"i" long:"include" value-name:"<pattern>..." description:"Include URLs matched with given regular expressions"`
 	FollowRobotsTxt       bool     `long:"follow-robots-txt" description:"Follow robots.txt when scraping pages"`
diff --git a/command.go b/command.go
index ac6f967..35b25a9 100644
--- a/command.go
+++ b/command.go
@@ -50,6 +50,7 @@ func (c *command) runWithError(ss []string) (bool, error) {
 			c.httpClientFactory.Create(
 				httpClientOptions{
 					MaxConnectionsPerHost: args.MaxConnectionsPerHost,
+					MaxResponseBodySize:   args.MaxResponseBodySize,
 					BufferSize:            args.BufferSize,
 					Proxy:                 args.Proxy,
 					SkipTLSVerification:   args.SkipTLSVerification,
diff --git a/fasthttp_http_client_factory.go b/fasthttp_http_client_factory.go
index 3e32957..9012d0e 100644
--- a/fasthttp_http_client_factory.go
+++ b/fasthttp_http_client_factory.go
@@ -33,6 +33,7 @@ func (*fasthttpHttpClientFactory) Create(o httpClientOptions) httpClient {
 			},
 			Dial:                     d,
 			NoDefaultUserAgentHeader: true,
+			MaxResponseBodySize:      o.MaxResponseBodySize,
 		},
 		o.Timeout,
 		o.Headers,
diff --git a/go.mod b/go.mod
index d98349d..3356ec1 100644
--- a/go.mod
+++ b/go.mod
@@ -6,6 +6,7 @@ require (
 	github.com/andybalholm/brotli v1.0.4
 	github.com/bradleyjkemp/cupaloy v2.3.0+incompatible
 	github.com/jessevdk/go-flags v1.5.0
+	github.com/klauspost/compress v1.15.9 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/logrusorgru/aurora/v3 v3.0.0
 	github.com/mattn/go-colorable v0.1.12
@@ -17,6 +18,6 @@ require (
 	github.com/valyala/fasthttp v1.38.0
 	github.com/yhat/scrape v0.0.0-20161128144610-24b7890b0945
 	go.uber.org/ratelimit v0.2.0
-	golang.org/x/net v0.0.0-20220225172249-27dd8689420f
+	golang.org/x/net v0.0.0-20220728153142-1f511ac62c11
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 )
diff --git a/go.sum b/go.sum
index 2356308..016a0ec 100644
--- a/go.sum
+++ b/go.sum
@@ -12,6 +12,8 @@ github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LF
 github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
 github.com/klauspost/compress v1.15.0 h1:xqfchp4whNFxn5A4XFyyYtitiWI8Hy5EW59jEwcyL6U=
 github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
+github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -52,6 +54,8 @@ golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220728153142-1f511ac62c11 h1:BZ+7NKCw5UIzmPP4GFz9VcjTVpN9mswsWRDmJ2tWKAs=
+golang.org/x/net v0.0.0-20220728153142-1f511ac62c11/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -61,6 +65,8 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs=
 golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
diff --git a/http_client_factory.go b/http_client_factory.go
index 07125c5..0c2e403 100644
--- a/http_client_factory.go
+++ b/http_client_factory.go
@@ -4,6 +4,7 @@ import "time"
 
 type httpClientOptions struct {
 	MaxConnectionsPerHost,
+	MaxResponseBodySize,
 	BufferSize int
 	Proxy               string
 	SkipTLSVerification bool


Test patch:
diff --git a/.snapshots/TestHelp b/.snapshots/TestHelp
index 40295f0..e5f031b 100644
--- a/.snapshots/TestHelp
+++ b/.snapshots/TestHelp
@@ -8,6 +8,8 @@ Application Options:
                                             (default: 512)
       --max-connections-per-host=<count>    Maximum number of HTTP connections
                                             per host (default: 512)
+      --max-response-body-size=<size>       Maximum web page body size to read
+                                            (default: 10000000)
   -e, --exclude=<pattern>...                Exclude URLs matched with given
                                             regular expressions
   -i, --include=<pattern>...                Include URLs matched with given
diff --git a/arguments_test.go b/arguments_test.go
index 3f8007e..c2a7f1a 100644
--- a/arguments_test.go
+++ b/arguments_test.go
@@ -15,6 +15,7 @@ func TestGetArguments(t *testing.T) {
 		{"-c", "1", "https://foo.com"},
 		{"--max-connections", "1", "https://foo.com"},
 		{"--max-connections-per-host", "1", "https://foo.com"},
+		{"--max-response-body-size", "1", "https://foo.com"},
 		{"-e", "regex1", "-e", "regex2", "https://foo.com"},
 		{"--exclude", "regex1", "--exclude", "regex2", "https://foo.com"},
 		{"--header", "MyHeader: foo", "--header", "YourHeader: bar", "https://foo.com"},