Repository: danielgtaylor/restish
Pull request #219 (2023-09-29 19:56:28)

feat: retries & timeouts
This adds support to Restish for automatic retries and timeouts. See the updated docs for examples. Here is what it looks like in practice:

```sh
$ restish api.rest.sh/status/429?x-retry-in=200ms
WARN: Got 429 Too Many Requests, retrying in 200ms
WARN: Got 429 Too Many Requests, retrying in 200ms
HTTP/2.0 429 Too Many Requests
... response headers truncated ...
```

This updates the core request handling functionality in Restish, so it equally applies to bulk resource management and any scripts you may have with loops or pagination making many requests over a short period of time.

Also includes some minor updates to the release scripts and the frontend syntax highlighter to make the examples pretty.


Patch:
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 1774db2..cdf4e81 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -21,6 +21,6 @@ jobs:
         uses: goreleaser/goreleaser-action@v2
         with:
           version: latest
-          args: release --rm-dist
+          args: release --clean
         env:
           GITHUB_TOKEN: ${{ secrets.GH_PAT }}
diff --git a/.goreleaser.yml b/.goreleaser.yml
index 292c180..9b122d3 100644
--- a/.goreleaser.yml
+++ b/.goreleaser.yml
@@ -19,7 +19,7 @@ archives:
       - goos: windows
         format: zip
 brews:
-  - tap:
+  - repository:
       owner: danielgtaylor
       name: homebrew-restish
     homepage: https://rest.sh/
diff --git a/README.md b/README.md
index 1546141..a3771b8 100644
--- a/README.md
+++ b/README.md
@@ -33,6 +33,7 @@ Features include:
   - MessagePack (<https://msgpack.org/>)
   - Amazon Ion (<http://amzn.github.io/ion-docs/>)
   - Gzip ([RFC 1952](https://tools.ietf.org/html/rfc1952)), Deflate ([RFC 1951](https://datatracker.ietf.org/doc/html/rfc1951)), and Brotli ([RFC 7932](https://tools.ietf.org/html/rfc7932)) content encoding
+- Automatic retries with support for [`Retry-After`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After) and `X-Retry-In` headers when APIs are rate-limited.
 - Standardized [hypermedia](https://smartbear.com/learn/api-design/what-is-hypermedia/) parsing into queryable/followable response links:
   - HTTP Link relation headers ([RFC 5988](https://tools.ietf.org/html/rfc5988#section-6.2.2))
   - [HAL](http://stateless.co/hal_specification.html)
diff --git a/cli/cli.go b/cli/cli.go
index e3defe3..38402a0 100644
--- a/cli/cli.go
+++ b/cli/cli.go
@@ -551,6 +551,8 @@ Not after (expires): %s (%s)
 	AddGlobalFlag("rsh-client-key", "", "Path to a PEM encoded private key", "", false)
 	AddGlobalFlag("rsh-ca-cert", "", "Path to a PEM encoded CA cert", "", false)
 	AddGlobalFlag("rsh-ignore-status-code", "", "Do not set exit code from HTTP status code", false, false)
+	AddGlobalFlag("rsh-retry", "", "Number of times to retry on certain failures", 2, false)
+	AddGlobalFlag("rsh-timeout", "t", "Timeout for HTTP requests", time.Duration(0), false)
 
 	Root.RegisterFlagCompletionFunc("rsh-output-format", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 		return []string{"auto", "json", "yaml"}, cobra.ShellCompDirectiveNoFileComp
@@ -770,6 +772,12 @@ func Run() (returnErr error) {
 	}
 	profile, _ := GlobalFlags.GetString("rsh-profile")
 	viper.Set("rsh-profile", profile)
+	if retries, _ := GlobalFlags.GetInt("rsh-retry"); retries > 0 {
+		viper.Set("rsh-retry", retries)
+	}
+	if timeout, _ := GlobalFlags.GetDuration("rsh-timeout"); timeout > 0 {
+		viper.Set("rsh-timeout", timeout)
+	}
 
 	// Now that global flags are parsed we can enable verbose mode if requested.
 	if viper.GetBool("rsh-verbose") {
diff --git a/cli/flag.go b/cli/flag.go
index ebbbdae..e336ab2 100644
--- a/cli/flag.go
+++ b/cli/flag.go
@@ -3,6 +3,7 @@ package cli
 import (
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/spf13/viper"
 )
@@ -21,6 +22,13 @@ func AddGlobalFlag(name, short, description string, defaultValue interface{}, mu
 			flags.BoolP(name, short, viper.GetBool(name), description)
 			GlobalFlags.BoolP(name, short, viper.GetBool(name), description)
 		}
+	case time.Duration:
+		if multi {
+			panic(fmt.Errorf("unsupported float slice param"))
+		} else {
+			flags.DurationP(name, short, viper.GetDuration(name), description)
+			GlobalFlags.DurationP(name, short, viper.GetDuration(name), description)
+		}
 	case int, int16, int32, int64, uint16, uint32, uint64:
 		if multi {
 			flags.IntSliceP(name, short, viper.Get(name).([]int), description)
diff --git a/cli/request.go b/cli/request.go
index 288fd40..32224a0 100644
--- a/cli/request.go
+++ b/cli/request.go
@@ -1,8 +1,11 @@
 package cli
 
 import (
+	"bytes"
+	"context"
 	"crypto/tls"
 	"crypto/x509"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -100,8 +103,6 @@ func IgnoreStatus() requestOption {
 // before sending it out on the wire. If verbose mode is enabled, it will
 // print out both the request and response.
 func MakeRequest(req *http.Request, options ...requestOption) (*http.Response, error) {
-	start := time.Now()
-
 	name, config := findAPI(req.URL.String())
 
 	if config == nil {
@@ -258,11 +259,7 @@ func MakeRequest(req *http.Request, options ...requestOption) (*http.Response, e
 		}
 	}
 
-	if log {
-		LogDebugRequest(req)
-	}
-
-	resp, err := client.Do(req)
+	resp, err := doRequestWithRetry(log, client, req)
 	if err != nil {
 		return nil, err
 	}
@@ -271,11 +268,109 @@ func MakeRequest(req *http.Request, options ...requestOption) (*http.Response, e
 		lastStatus = resp.StatusCode
 	}
 
-	if log {
-		LogDebugResponse(start, resp)
+	return resp, nil
+}
+
+// isRetryable returns true if a request should be retried.
+func isRetryable(code int) bool {
+	if code == /* 408 */ http.StatusRequestTimeout ||
+		code == /*  425 */ http.StatusTooEarly ||
+		code == /*  429 */ http.StatusTooManyRequests ||
+		code == /*  500 */ http.StatusInternalServerError ||
+		code == /*  502 */ http.StatusBadGateway ||
+		code == /*  503 */ http.StatusServiceUnavailable ||
+		code == /*  504 */ http.StatusGatewayTimeout {
+		return true
+	}
+	return false
+}
+
+// doRequestWithRetry logs and makes a request, retrying as needed (if
+// configured) and returning the last response.
+func doRequestWithRetry(log bool, client *http.Client, req *http.Request) (*http.Response, error) {
+	retries := viper.GetInt("rsh-retry")
+
+	if retries == 0 {
+		return client.Do(req)
 	}
 
-	return resp, nil
+	var bodyContents []byte
+	if req.Body != nil {
+		bodyContents, _ = io.ReadAll(req.Body)
+	}
+
+	var resp *http.Response
+	var err error
+	triesLeft := 1 + retries
+	for triesLeft > 0 {
+		triesLeft--
+
+		if len(bodyContents) > 0 {
+			// Reset the body reader for each retry.
+			req.Body = io.NopCloser(bytes.NewReader(bodyContents))
+		}
+
+		if log {
+			LogDebugRequest(req)
+		}
+
+		if timeout := viper.GetDuration("rsh-timeout"); timeout > 0 {
+			ctx, cancel := context.WithTimeout(req.Context(), timeout)
+			defer cancel()
+			req = req.WithContext(ctx)
+		}
+
+		start := time.Now()
+		resp, err = client.Do(req)
+		if err != nil {
+			if errors.Is(err, context.DeadlineExceeded) {
+				if triesLeft > 0 {
+					// Try again after letting the user know.
+					LogWarning("Got request timeout after %s, retrying", viper.GetDuration("rsh-timeout").Truncate(time.Millisecond))
+					continue
+				} else {
+					// Add a human-friendly error before the original (context deadline
+					// exceeded).
+					err = fmt.Errorf("Request timed out after %s: %w", viper.GetDuration("rsh-timeout"), err)
+				}
+			}
+			return resp, err
+		}
+
+		if log {
+			LogDebugResponse(start, resp)
+		}
+
+		if triesLeft > 0 && isRetryable(resp.StatusCode) {
+			// Attempt to parse when to retry! Default is 1 second.
+			retryAfter := 1 * time.Second
+
+			if v := resp.Header.Get("Retry-After"); v != "" {
+				// Could be either an integer number of seconds, or an HTTP date.
+				if d, err := strconv.ParseInt(v, 10, 64); err == nil {
+					retryAfter = time.Duration(d) * time.Second
+				}
+
+				if d, err := http.ParseTime(v); err == nil {
+					retryAfter = time.Until(d)
+				}
+			}
+
+			if v := resp.Header.Get("X-Retry-In"); v != "" {
+				if d, err := time.ParseDuration(v); err == nil {
+					retryAfter = d
+				}
+			}
+
+			LogWarning("Got %s, retrying in %s", resp.Status, retryAfter.Truncate(time.Millisecond))
+			time.Sleep(retryAfter)
+
+			continue
+		}
+		break
+	}
+
+	return resp, err
 }
 
 // Response describes a parsed HTTP response which can be marshalled to enable
diff --git a/docs/README.md b/docs/README.md
index c459a82..2eeb071 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -41,6 +41,7 @@ Start with the [guide](/guide.md) to learn how to install and configure Restish
   - MessagePack (<https://msgpack.org/>)
   - Amazon Ion (<http://amzn.github.io/ion-docs/>)
   - Gzip ([RFC 1952](https://tools.ietf.org/html/rfc1952)), Deflate ([RFC 1951](https://datatracker.ietf.org/doc/html/rfc1951)), and Brotli ([RFC 7932](https://tools.ietf.org/html/rfc7932)) content encoding
+- Automatic retries with support for [`Retry-After`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After) and `X-Retry-In` headers when APIs are rate-limited.
 - Standardized [hypermedia](https://smartbear.com/learn/api-design/what-is-hypermedia/) parsing into queryable/followable response links:
   - HTTP Link relation headers ([RFC 5988](https://tools.ietf.org/html/rfc5988#section-6.2.2))
   - [HAL](http://stateless.co/hal_specification.html)
diff --git a/docs/_sidebar.md b/docs/_sidebar.md
index 42e98c6..c2fac5f 100644
--- a/docs/_sidebar.md
+++ b/docs/_sidebar.md
@@ -6,5 +6,6 @@
 - [Input](input.md "Restish Input")
 - [CLI Shorthand](shorthand.md "CLI Shorthand")
 - [Output](output.md "Restish Output")
+- [Retries & Timeouts](retries.md "Retries & Timeouts")
 - [Hypermedia](hypermedia.md "Hypermedia Linking in Restish")
 - [Bulk Management](bulk.md "Bulk Resource Management")
diff --git a/docs/index.html b/docs/index.html
index ada4884..a708449 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -169,6 +169,10 @@
       .token.diffRemoved {
         color: #ff5f87;
       }
+
+      .token.warning {
+        color: #d78700;
+      }
     </style>
   </head>
   <body>
@@ -236,7 +240,7 @@
         comment: /^\s*#.*/m,
         redirect: /2>\/dev\/null/,
         response: {
-          pattern: /^(HTTP\/[12]|\{|\[)(.|\n)+(\]|\}(\n|$))/gm,
+          pattern: /^(HTTP\/[12].*$)|((\{|\[)(.|\n)+(\]|\}(\n|$)))/gm,
           greedy: false,
           inside: Prism.languages.readable
         },
@@ -259,6 +263,13 @@
           alias: "variable",
           pattern: /\$[A-Z0-9_]+/,
         },
+        log: {
+          pattern: /(INFO|WARN|ERROR): .*/,
+          inside: {
+            warning: /WARN:/,
+            error: /ERROR:/,
+          },
+        },
         header: {
           pattern: /-H [A-Z][a-zA-Z0-9-]+:\S+/,
           inside: {
diff --git a/docs/retries.md b/docs/retries.md
new file mode 100644
index 0000000..3c21b35
--- /dev/null
+++ b/docs/retries.md
@@ -0,0 +1,77 @@
+# Retries & Timeouts
+
+Restish has support for automatic retries for some types of requests, and also supports giving up after a certain amount of time. This is useful for APIs that are rate-limited or have intermittent issues.
+
+## Automatic Retries
+
+By default, Restish will retry the following responses two times:
+
+- `408 Request Timeout`
+- `425 Too Early`
+- `429 Too Many Requests`
+- `500 Internal Server Error`
+- `502 Bad Gateway`
+- `503 Service Unavailable`
+- `504 Gateway Timeout`
+
+This is configurable via the `--rsh-retry` parameter or `RSH_RETRY` environment variable, which should be a positive integer. Set to `0` to disable retries.
+
+Here is an example of the default behavior:
+
+```bash
+# Trigger retries by generating a 429 response.
+$ restish api.rest.sh/status/429
+WARN: Got 429 Too Many Requests, retrying in 1s
+WARN: Got 429 Too Many Requests, retrying in 1s
+HTTP/2.0 429 Too Many Requests
+Cache-Control: private
+Cf-Cache-Status: MISS
+Cf-Ray: 80e668787b3ec59c-SEA
+Content-Length: 0
+Date: Fri, 29 Sep 2023 18:49:47 GMT
+Server: cloudflare
+Vary: Accept-Encoding
+X-Do-App-Origin: 18871cde-e6ba-11ec-b1dc-0c42a19a82a7
+X-Do-Orig-Status: 429
+X-Varied-Accept-Encoding: deflate, gzip, br
+```
+
+By default, Restish will wait 1 second between retries. If the server responds with one of the following headers, it will be parsed and used to determine the retry delay:
+
+- `Retry-After` ([RFC 7231](https://tools.ietf.org/html/rfc7231#section-7.1.3))
+- `X-Retry-In` (as set by e.g. [Traefik](https://doc.traefik.io/traefik/middlewares/http/ratelimit/) [rate limiting](https://github.com/traefik/traefik/blob/v2.10/pkg/middlewares/ratelimiter/rate_limiter.go#L176-L177))
+
+For example:
+
+```bash
+# Trigger delayed retries with a 429 and Retry-After header.
+$ restish api.rest.sh/status/429?retry-after=3
+WARN: Got 429 Too Many Requests, retrying in 3s
+WARN: Got 429 Too Many Requests, retrying in 3s
+HTTP/2.0 429 Too Many Requests
+Cache-Control: private
+Cf-Cache-Status: MISS
+Cf-Ray: 80e669fbb95a283d-SEA
+Content-Length: 0
+Date: Fri, 29 Sep 2023 18:50:49 GMT
+Retry-After: 3
+Server: cloudflare
+Vary: Accept-Encoding
+X-Do-App-Origin: 18871cde-e6ba-11ec-b1dc-0c42a19a82a7
+X-Do-Orig-Status: 429
+X-Varied-Accept-Encoding: br, deflate, gzip
+```
+
+## Request Timeouts
+
+Restish has optional timeouts you can set on outgoing requests using the `--rsh-timeout` parameter or `RSH_TIMEOUT` environment variable. This should be a duration with suffix, e.g. `1s` or `500ms`. Set to `0` to disable timeouts (which is the default). Timeouts are retried since they are often due to intermittent network issues and subsequent requests may succeed.
+
+Here is an example of a timeout:
+
+```bash
+# Trigger a timeout with a ridiculously low value.
+$ restish api.rest.sh/ --rsh-timeout=10ms
+WARN: Got request timeout after 10ms, retrying
+WARN: Got request timeout after 10ms, retrying
+ERROR: Caught error: Request timed out after 10ms: Get "https://api.rest.sh/": context deadline exceeded
+```


Test patch:
diff --git a/cli/cli_test.go b/cli/cli_test.go
index 143dc28..3163ed0 100644
--- a/cli/cli_test.go
+++ b/cli/cli_test.go
@@ -23,6 +23,9 @@ func reset(color bool) {
 		viper.Set("nocolor", true)
 	}
 
+	// Most tests are easier to write without retries.
+	viper.Set("rsh-retry", 0)
+
 	Init("test", "1.0.0'")
 	Defaults()
 }
diff --git a/cli/request_test.go b/cli/request_test.go
index e92945f..148f808 100644
--- a/cli/request_test.go
+++ b/cli/request_test.go
@@ -1,10 +1,13 @@
 package cli
 
 import (
+	"bytes"
 	"errors"
 	"net/http"
 	"testing"
+	"time"
 
+	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
 	"gopkg.in/h2non/gock.v1"
 )
@@ -123,3 +126,83 @@ func TestIgnoreStatus(t *testing.T) {
 
 	assert.Equal(t, 0, GetLastStatus())
 }
+
+func TestRequestRetryIn(t *testing.T) {
+	defer gock.Off()
+
+	reset(false)
+	viper.Set("rsh-retry", 1)
+
+	// Duration string value (with units)
+	gock.New("http://example.com").
+		Get("/").
+		Times(1).
+		Reply(http.StatusTooManyRequests).
+		SetHeader("X-Retry-In", "1ms")
+
+	gock.New("http://example.com").
+		Get("/").
+		Times(1).
+		Reply(http.StatusOK)
+
+	req, _ := http.NewRequest(http.MethodGet, "http://example.com/", nil)
+	resp, err := MakeRequest(req)
+
+	assert.NoError(t, err)
+	assert.Equal(t, resp.StatusCode, http.StatusOK)
+}
+
+func TestRequestRetryAfter(t *testing.T) {
+	defer gock.Off()
+
+	reset(false)
+	viper.Set("rsh-retry", 2)
+
+	// Seconds value
+	gock.New("http://example.com").
+		Put("/").
+		Times(1).
+		Reply(http.StatusTooManyRequests).
+		SetHeader("Retry-After", "0")
+
+	// HTTP date value
+	gock.New("http://example.com").
+		Put("/").
+		Times(1).
+		Reply(http.StatusTooManyRequests).
+		SetHeader("Retry-After", time.Now().Format(http.TimeFormat))
+
+	gock.New("http://example.com").
+		Put("/").
+		Times(1).
+		Reply(http.StatusOK)
+
+	req, _ := http.NewRequest(http.MethodPut, "http://example.com/", bytes.NewReader([]byte("hello")))
+	resp, err := MakeRequest(req)
+
+	assert.NoError(t, err)
+	assert.Equal(t, resp.StatusCode, http.StatusOK)
+}
+
+func TestRequestRetryTimeout(t *testing.T) {
+	defer gock.Off()
+
+	reset(false)
+	viper.Set("rsh-retry", 1)
+	viper.Set("rsh-timeout", 1*time.Millisecond)
+
+	// Duration string value (with units)
+	gock.New("http://example.com").
+		Get("/").
+		Times(2).
+		Reply(http.StatusOK).
+		Delay(2 * time.Millisecond)
+		// Note: delay seems to have a bug where subsequent requests without the
+		// delay are still delayed... For now just have it reply twice.
+
+	req, _ := http.NewRequest(http.MethodGet, "http://example.com/", nil)
+	_, err := MakeRequest(req)
+
+	assert.Error(t, err)
+	assert.ErrorContains(t, err, "timed out")
+}