Repository: raviqqe/muffet (go)

Issue:
Feature request: ability to limit request rate
I'd like to see a flag to specify a request rate limit. Out of the box I see a lot of 429's from Github Pages sites. I can reduce these by specifying `--max-connections`, but that isn't the same as limiting the number of requests per second.

As a comparison, [wget](https://www.gnu.org/software/wget/manual/wget.html) offers these flags:

> --wait=seconds
> Wait the specified number of seconds between the retrievals. Use of this option is recommended, as it lightens the server load by making the requests less frequent. Instead of in seconds, the time can be specified in minutes using the m suffix, in hours using h suffix, or in days using d suffix.
>
> --random-wait
> Some web sites may perform log analysis to identify retrieval programs such as Wget by looking for statistically significant similarities in the time between requests. This option causes the time between requests to vary between 0.5 and 1.5 * wait seconds, where wait was specified using the ‘--wait’ option, in order to mask Wget’s presence from such analysis.

Pull request:
Simple rate limiting
Close #145 

Patch:
diff --git a/arguments.go b/arguments.go
index 32d8787..4e10452 100644
--- a/arguments.go
+++ b/arguments.go
@@ -20,6 +20,7 @@ type arguments struct {
 	IgnoreFragments       bool     `short:"f" long:"ignore-fragments" description:"Ignore URL fragments"`
 	JSONOutput            bool     `long:"json" description:"Output results in JSON"`
 	MaxRedirections       int      `short:"r" long:"max-redirections" value-name:"<count>" default:"64" description:"Maximum number of redirections"`
+	RateLimit             int      `long:"rate-limit" value-name:"<rate>" description:"Max requests per second"`
 	Timeout               int      `short:"t" long:"timeout" value-name:"<seconds>" default:"10" description:"Timeout for HTTP requests in seconds"`
 	Verbose               bool     `short:"v" long:"verbose" description:"Show successful results too"`
 	Proxy                 string   `long:"proxy" value-name:"<host>" description:"HTTP proxy host"`
diff --git a/command.go b/command.go
index d841c19..1063efe 100644
--- a/command.go
+++ b/command.go
@@ -56,6 +56,7 @@ func (c *command) runWithError(ss []string) (bool, error) {
 				Timeout:               time.Duration(args.Timeout) * time.Second,
 			},
 		),
+		args.RateLimit,
 		args.MaxConnections,
 	)
 
diff --git a/go.mod b/go.mod
index 8d0a3ad..be83575 100644
--- a/go.mod
+++ b/go.mod
@@ -5,7 +5,6 @@ go 1.12
 require (
 	github.com/andybalholm/brotli v1.0.1
 	github.com/bradleyjkemp/cupaloy v2.3.0+incompatible
-	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/jessevdk/go-flags v1.4.0
 	github.com/klauspost/compress v1.11.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
@@ -18,6 +17,8 @@ require (
 	github.com/temoto/robotstxt v1.1.1
 	github.com/valyala/fasthttp v1.21.0
 	github.com/yhat/scrape v0.0.0-20161128144610-24b7890b0945
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/ratelimit v0.1.0
 	golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 )
diff --git a/go.sum b/go.sum
index 3b24317..1c603e7 100644
--- a/go.sum
+++ b/go.sum
@@ -46,6 +46,10 @@ github.com/valyala/fasthttp v1.21.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8erm
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
 github.com/yhat/scrape v0.0.0-20161128144610-24b7890b0945 h1:6Ju8pZBYFTN9FaV/JvNBiIHcsgEmP4z4laciqjfjY8E=
 github.com/yhat/scrape v0.0.0-20161128144610-24b7890b0945/go.mod h1:4vRFPPNYllgCacoj+0FoKOjTW68rUhEfqPLiEJaK2w8=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw=
+go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
diff --git a/throttled_http_client.go b/throttled_http_client.go
index 39e63f1..d5cc174 100644
--- a/throttled_http_client.go
+++ b/throttled_http_client.go
@@ -1,19 +1,30 @@
 package main
 
 import "net/url"
+import "go.uber.org/ratelimit"
 
+// TODO Throttle requests for each host.
 type throttledHTTPClient struct {
 	client    httpClient
+	limiter   ratelimit.Limiter
 	semaphore semaphore
 }
 
-func newThrottledHTTPClient(c httpClient, maxConnections int) httpClient {
-	return &throttledHTTPClient{c, newSemaphore(maxConnections)}
+func newThrottledHTTPClient(c httpClient, rps int, maxConnections int) httpClient {
+	l := ratelimit.NewUnlimited()
+
+	if rps > 0 {
+		l = ratelimit.New(rps)
+	}
+
+	return &throttledHTTPClient{c, l, newSemaphore(maxConnections)}
 }
 
 func (c *throttledHTTPClient) Get(u *url.URL, hs map[string]string) (httpResponse, error) {
 	c.semaphore.Request()
 	defer c.semaphore.Release()
 
+	c.limiter.Take()
+
 	return c.client.Get(u, hs)
 }


Test patch:
diff --git a/.snapshots/TestHelp b/.snapshots/TestHelp
index 669b017..55e127f 100644
--- a/.snapshots/TestHelp
+++ b/.snapshots/TestHelp
@@ -19,6 +19,7 @@ Application Options:
       --json                                Output results in JSON
   -r, --max-redirections=<count>            Maximum number of redirections
                                             (default: 64)
+      --rate-limit=<rate>                   Max requests per second
   -t, --timeout=<seconds>                   Timeout for HTTP requests in
                                             seconds (default: 10)
   -v, --verbose                             Show successful results too
diff --git a/arguments_test.go b/arguments_test.go
index 9062374..3f8007e 100644
--- a/arguments_test.go
+++ b/arguments_test.go
@@ -25,6 +25,7 @@ func TestGetArguments(t *testing.T) {
 		{"--follow-sitemap-xml", "https://foo.com"},
 		{"-t", "10", "https://foo.com"},
 		{"--timeout", "10", "https://foo.com"},
+		{"--rate-limit", "1", "https://foo.com"},
 		{"--proxy", "localhost:8080", "https://foo.com"},
 		{"--skip-tls-verification", "https://foo.com"},
 		{"-v", "https://foo.com"},