Repository: labstack/echo
Pull request #1687 (2020-11-20 04:54:40)

adding decompress gzipped request middleware
Adding decompress middleware that decompresses request if content-encoding is set to "gzip". We send in large amounts of data and we want to compress data on the network and decompress it in API. This is trivial for any API that ingests a large amount of payload. Also addresses one of the issues logged on echo https://github.com/labstack/echo/issues/986.

Added tests as well to cover most of the scenarios. Can you please review @lammel 

P.S Build was failing as github actions deprecated passing env variables, updated workflows/echo.yaml to fix this issue. More details here https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/

iambenkay: All the checks seem to be failing.
iambenkay: This is a really useful middleware
codecov[bot]: # [Codecov](https://codecov.io/gh/labstack/echo/pull/1687?src=pr&el=h1) Report
> Merging [#1687](https://codecov.io/gh/labstack/echo/pull/1687?src=pr&el=desc) (6b80d7e) into [master](https://codecov.io/gh/labstack/echo/commit/7a90304d2daa6cc94f6d383f8aaaaaa960a61d9a?el=desc) (7a90304) will **decrease** coverage by `0.00%`.
> The diff coverage is `84.21%`.

[![Impacted file tree graph](https://codecov.io/gh/labstack/echo/pull/1687/graphs/tree.svg?width=650&height=150&src=pr&token=vqnT76l79i)](https://codecov.io/gh/labstack/echo/pull/1687?src=pr&el=tree)

```diff
@@            Coverage Diff             @@
##           master    #1687      +/-   ##
==========================================
- Coverage   84.35%   84.35%   -0.01%     
==========================================
  Files          28       29       +1     
  Lines        1911     1930      +19     
==========================================
+ Hits         1612     1628      +16     
- Misses        189      191       +2     
- Partials      110      111       +1     
```


| [Impacted Files](https://codecov.io/gh/labstack/echo/pull/1687?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [middleware/decompress.go](https://codecov.io/gh/labstack/echo/pull/1687/diff?src=pr&el=tree#diff-bWlkZGxld2FyZS9kZWNvbXByZXNzLmdv) | `84.21% <84.21%> (ø)` | |

------

[Continue to review full report at Codecov](https://codecov.io/gh/labstack/echo/pull/1687?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/labstack/echo/pull/1687?src=pr&el=footer). Last update [7a90304...6b80d7e](https://codecov.io/gh/labstack/echo/pull/1687?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/.github/workflows/echo.yml b/.github/workflows/echo.yml
index 38596ab..c4fae77 100644
--- a/.github/workflows/echo.yml
+++ b/.github/workflows/echo.yml
@@ -18,10 +18,6 @@ on:
       - '_fixture/**'
       - '.github/**'
 
-env:
-  GO111MODULE: on
-  GOPROXY: https://proxy.golang.org
-
 jobs:
   test:
     strategy:
@@ -38,10 +34,15 @@ jobs:
 
       - name: Set GOPATH and PATH
         run: |
-          echo "::set-env name=GOPATH::$(dirname $GITHUB_WORKSPACE)"
-          echo "::add-path::$(dirname $GITHUB_WORKSPACE)/bin"
+          echo "GOPATH=$(dirname $GITHUB_WORKSPACE)" >> $GITHUB_ENV
+          echo "$(dirname $GITHUB_WORKSPACE)/bin" >> $GITHUB_PATH
         shell: bash
 
+      - name: Set build variables
+        run: |
+          echo "GOPROXY=https://proxy.golang.org" >> $GITHUB_ENV
+          echo "GO111MODULE=on" >> $GITHUB_ENV
+
       - name: Checkout Code
         uses: actions/checkout@v1
         with:
diff --git a/middleware/decompress.go b/middleware/decompress.go
new file mode 100644
index 0000000..99eaf06
--- /dev/null
+++ b/middleware/decompress.go
@@ -0,0 +1,58 @@
+package middleware
+
+import (
+	"bytes"
+	"compress/gzip"
+	"github.com/labstack/echo/v4"
+	"io"
+	"io/ioutil"
+)
+
+type (
+	// DecompressConfig defines the config for Decompress middleware.
+	DecompressConfig struct {
+		// Skipper defines a function to skip middleware.
+		Skipper Skipper
+	}
+)
+
+//GZIPEncoding content-encoding header if set to "gzip", decompress body contents.
+const GZIPEncoding string = "gzip"
+
+var (
+	//DefaultDecompressConfig defines the config for decompress middleware
+	DefaultDecompressConfig = DecompressConfig{Skipper: DefaultSkipper}
+)
+
+//Decompress decompresses request body based if content encoding type is set to "gzip" with default config
+func Decompress() echo.MiddlewareFunc {
+	return DecompressWithConfig(DefaultDecompressConfig)
+}
+
+//DecompressWithConfig decompresses request body based if content encoding type is set to "gzip" with config
+func DecompressWithConfig(config DecompressConfig) echo.MiddlewareFunc {
+	return func(next echo.HandlerFunc) echo.HandlerFunc {
+		return func(c echo.Context) error {
+			if config.Skipper(c) {
+				return next(c)
+			}
+			switch c.Request().Header.Get(echo.HeaderContentEncoding) {
+			case GZIPEncoding:
+				gr, err := gzip.NewReader(c.Request().Body)
+				if err != nil {
+					if err == io.EOF { //ignore if body is empty
+						return next(c)
+					}
+					return err
+				}
+				defer gr.Close()
+				var buf bytes.Buffer
+				io.Copy(&buf, gr)
+				r := ioutil.NopCloser(&buf)
+				defer r.Close()
+				c.Request().Body = r
+			}
+			return next(c)
+		}
+	}
+}


Test patch:
diff --git a/middleware/decompress_test.go b/middleware/decompress_test.go
new file mode 100644
index 0000000..772c14f
--- /dev/null
+++ b/middleware/decompress_test.go
@@ -0,0 +1,148 @@
+package middleware
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/labstack/echo/v4"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDecompress(t *testing.T) {
+	e := echo.New()
+	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("test"))
+	rec := httptest.NewRecorder()
+	c := e.NewContext(req, rec)
+
+	// Skip if no Content-Encoding header
+	h := Decompress()(func(c echo.Context) error {
+		c.Response().Write([]byte("test")) // For Content-Type sniffing
+		return nil
+	})
+	h(c)
+
+	assert := assert.New(t)
+	assert.Equal("test", rec.Body.String())
+
+	// Decompress
+	body := `{"name": "echo"}`
+	gz, _ := gzipString(body)
+	req = httptest.NewRequest(http.MethodPost, "/", strings.NewReader(string(gz)))
+	req.Header.Set(echo.HeaderContentEncoding, GZIPEncoding)
+	rec = httptest.NewRecorder()
+	c = e.NewContext(req, rec)
+	h(c)
+	assert.Equal(GZIPEncoding, req.Header.Get(echo.HeaderContentEncoding))
+	b, err := ioutil.ReadAll(req.Body)
+	assert.NoError(err)
+	assert.Equal(body, string(b))
+}
+
+func TestCompressRequestWithoutDecompressMiddleware(t *testing.T) {
+	e := echo.New()
+	body := `{"name":"echo"}`
+	gz, _ := gzipString(body)
+	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(string(gz)))
+	req.Header.Set(echo.HeaderContentEncoding, GZIPEncoding)
+	rec := httptest.NewRecorder()
+	e.NewContext(req, rec)
+	e.ServeHTTP(rec, req)
+	assert.Equal(t, GZIPEncoding, req.Header.Get(echo.HeaderContentEncoding))
+	b, err := ioutil.ReadAll(req.Body)
+	assert.NoError(t, err)
+	assert.NotEqual(t, b, body)
+	assert.Equal(t, b, gz)
+}
+
+func TestDecompressNoContent(t *testing.T) {
+	e := echo.New()
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
+	req.Header.Set(echo.HeaderContentEncoding, GZIPEncoding)
+	rec := httptest.NewRecorder()
+	c := e.NewContext(req, rec)
+	h := Decompress()(func(c echo.Context) error {
+		return c.NoContent(http.StatusNoContent)
+	})
+	if assert.NoError(t, h(c)) {
+		assert.Equal(t, GZIPEncoding, req.Header.Get(echo.HeaderContentEncoding))
+		assert.Empty(t, rec.Header().Get(echo.HeaderContentType))
+		assert.Equal(t, 0, len(rec.Body.Bytes()))
+	}
+}
+
+func TestDecompressErrorReturned(t *testing.T) {
+	e := echo.New()
+	e.Use(Decompress())
+	e.GET("/", func(c echo.Context) error {
+		return echo.ErrNotFound
+	})
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
+	req.Header.Set(echo.HeaderContentEncoding, GZIPEncoding)
+	rec := httptest.NewRecorder()
+	e.ServeHTTP(rec, req)
+	assert.Equal(t, http.StatusNotFound, rec.Code)
+	assert.Empty(t, rec.Header().Get(echo.HeaderContentEncoding))
+}
+
+func TestDecompressSkipper(t *testing.T) {
+	e := echo.New()
+	e.Use(DecompressWithConfig(DecompressConfig{
+		Skipper: func(c echo.Context) bool {
+			return c.Request().URL.Path == "/skip"
+		},
+	}))
+	body := `{"name": "echo"}`
+	req := httptest.NewRequest(http.MethodPost, "/skip", strings.NewReader(body))
+	req.Header.Set(echo.HeaderContentEncoding, GZIPEncoding)
+	rec := httptest.NewRecorder()
+	c := e.NewContext(req, rec)
+	e.ServeHTTP(rec, req)
+	assert.Equal(t, rec.Header().Get(echo.HeaderContentType), echo.MIMEApplicationJSONCharsetUTF8)
+	reqBody, err := ioutil.ReadAll(c.Request().Body)
+	assert.NoError(t, err)
+	assert.Equal(t, body, string(reqBody))
+}
+
+func BenchmarkDecompress(b *testing.B) {
+	e := echo.New()
+	body := `{"name": "echo"}`
+	gz, _ := gzipString(body)
+	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(string(gz)))
+	req.Header.Set(echo.HeaderContentEncoding, GZIPEncoding)
+
+	h := Decompress()(func(c echo.Context) error {
+		c.Response().Write([]byte(body)) // For Content-Type sniffing
+		return nil
+	})
+
+	b.ReportAllocs()
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		// Decompress
+		rec := httptest.NewRecorder()
+		c := e.NewContext(req, rec)
+		h(c)
+	}
+}
+
+func gzipString(body string) ([]byte, error) {
+	var buf bytes.Buffer
+	gz := gzip.NewWriter(&buf)
+
+	_, err := gz.Write([]byte(body))
+	if err != nil {
+		return nil, err
+	}
+
+	if err := gz.Close(); err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}