Repository: avenga/couper
Pull request #66 (2020-11-19 09:37:09)

Add gzip support


Patch:
diff --git a/handler/file.go b/handler/file.go
index 80687f45..25bf462f 100644
--- a/handler/file.go
+++ b/handler/file.go
@@ -82,7 +82,6 @@ func (f *File) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	// TODO: gzip, br?
 	http.ServeContent(rw, req, reqPath, info.ModTime(), file)
 }
 
@@ -102,7 +101,6 @@ func (f *File) serveDirectory(reqPath string, rw http.ResponseWriter, req *http.
 	}
 	defer file.Close()
 
-	// TODO: gzip, br?
 	http.ServeContent(rw, req, reqPath, info.ModTime(), file)
 }
 
diff --git a/handler/proxy.go b/handler/proxy.go
index f2457bbf..9bf64334 100644
--- a/handler/proxy.go
+++ b/handler/proxy.go
@@ -2,6 +2,7 @@ package handler
 
 import (
 	"bytes"
+	"compress/gzip"
 	"context"
 	"crypto/tls"
 	"fmt"
@@ -10,6 +11,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"regexp"
 	"strconv"
 	"strings"
 	"sync"
@@ -30,6 +32,14 @@ import (
 	"github.com/avenga/couper/utils"
 )
 
+const (
+	GzipName              = "gzip"
+	AcceptEncodingHeader  = "Accept-Encoding"
+	ContentEncodingHeader = "Content-Encoding"
+	ContentLengthHeader   = "Content-Length"
+	VaryHeader            = "Vary"
+)
+
 var (
 	_ http.Handler   = &Proxy{}
 	_ server.Context = &Proxy{}
@@ -39,6 +49,8 @@ var (
 	// headerBlacklist lists all header keys which will be removed after
 	// context variable evaluation to ensure to not pass them upstream.
 	headerBlacklist = []string{"Authorization", "Cookie"}
+
+	ReClientSupportsGZ = regexp.MustCompile(`(?i)\b` + GzipName + `\b`)
 )
 
 type Proxy struct {
@@ -131,6 +143,7 @@ func (p *Proxy) getTransport(scheme, origin, hostname string) *http.Transport {
 				}
 				return conn, nil
 			},
+			DisableCompression:    true,
 			ResponseHeaderTimeout: p.options.TTFBTimeout,
 			TLSClientConfig:       tlsConf,
 		}
@@ -222,6 +235,20 @@ func (p *Proxy) roundtrip(rw http.ResponseWriter, req *http.Request) {
 		return
 	}
 
+	if strings.ToLower(res.Header.Get(ContentEncodingHeader)) == GzipName {
+		var src io.Reader
+		var err error
+
+		res.Header.Del(ContentEncodingHeader)
+
+		src, err = gzip.NewReader(res.Body)
+		if err != nil {
+			src = res.Body
+		}
+
+		res.Body = eval.NewReadCloser(src, res.Body)
+	}
+
 	removeConnectionHeaders(res.Header)
 
 	for _, h := range hopHeaders {
@@ -307,6 +334,12 @@ func (p *Proxy) Director(req *http.Request) error {
 		req.Host = hostname
 	}
 
+	if ReClientSupportsGZ.MatchString(req.Header.Get(AcceptEncodingHeader)) {
+		req.Header.Set(AcceptEncodingHeader, GzipName)
+	} else {
+		req.Header.Del(AcceptEncodingHeader)
+	}
+
 	if pathMatch, ok := req.Context().
 		Value(request.Wildcard).(string); ok && strings.HasSuffix(path, "/**") {
 		if pathMatch == "" && req.URL.Path != "" { // wildcard "root" hit, take a look if the req has a trailing slash and apply
diff --git a/server/header_writer.go b/server/header_writer.go
deleted file mode 100644
index 7b1fde47..00000000
--- a/server/header_writer.go
+++ /dev/null
@@ -1,33 +0,0 @@
-package server
-
-import (
-	"net/http"
-)
-
-var _ http.ResponseWriter = &HeaderWriter{}
-
-// ServerHeaderWriter ensures the server header value to be couper.io.
-type HeaderWriter struct {
-	rw http.ResponseWriter
-}
-
-// NewHeaderWriter creates a new HeaderWriter object.
-func NewHeaderWriter(rw http.ResponseWriter) *HeaderWriter {
-	return &HeaderWriter{rw: rw}
-}
-
-// Header wraps the Header method of the ResponseWriter.
-func (sr *HeaderWriter) Header() http.Header {
-	return sr.rw.Header()
-}
-
-// Write wraps the Write method of the ResponseWriter.
-func (sr *HeaderWriter) Write(p []byte) (int, error) {
-	return sr.rw.Write(p)
-}
-
-// WriteHeader wraps the WriteHeader method of the ResponseWriter.
-func (sr *HeaderWriter) WriteHeader(statusCode int) {
-	sr.rw.Header().Set("Server", "couper.io")
-	sr.rw.WriteHeader(statusCode)
-}
diff --git a/server/http.go b/server/http.go
index 21d55fd6..7c76b9b0 100644
--- a/server/http.go
+++ b/server/http.go
@@ -169,7 +169,16 @@ func (s *HTTPServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 	req.Host = s.getHost(req)
 
 	h := s.mux.FindHandler(req)
-	s.accessLog.ServeHTTP(NewHeaderWriter(rw), req, h, startTime)
+	w := NewRWWrapper(rw,
+		handler.ReClientSupportsGZ.MatchString(
+			req.Header.Get(handler.AcceptEncodingHeader),
+		),
+	)
+	rw = w
+
+	s.accessLog.ServeHTTP(rw, req, h, startTime)
+
+	w.Close() // Closes the GZ writer.
 }
 
 // getHost configures the host from the incoming request host based on
diff --git a/server/rw_wrapper.go b/server/rw_wrapper.go
new file mode 100644
index 00000000..5feefd38
--- /dev/null
+++ b/server/rw_wrapper.go
@@ -0,0 +1,77 @@
+package server
+
+import (
+	"compress/gzip"
+	"net/http"
+
+	"github.com/avenga/couper/handler"
+)
+
+var (
+	_ http.ResponseWriter = &RWWrapper{}
+	_ http.Flusher        = &RWWrapper{}
+)
+
+// RWWrapper wraps the <http.ResponseWriter>.
+type RWWrapper struct {
+	rw http.ResponseWriter
+	gz *gzip.Writer
+}
+
+// NewRWWrapper creates a new RWWrapper object.
+func NewRWWrapper(rw http.ResponseWriter, useGZ bool) *RWWrapper {
+	w := &RWWrapper{
+		rw: rw,
+	}
+
+	if useGZ {
+		w.gz = gzip.NewWriter(rw)
+	}
+
+	return w
+}
+
+// Header wraps the Header method of the <http.ResponseWriter>.
+func (w *RWWrapper) Header() http.Header {
+	return w.rw.Header()
+}
+
+// Write wraps the Write method of the <http.ResponseWriter>.
+func (w *RWWrapper) Write(p []byte) (int, error) {
+	if w.gz != nil {
+		return w.gz.Write(p)
+	}
+
+	return w.rw.Write(p)
+}
+
+// Close closes the GZ writer.
+func (w *RWWrapper) Close() {
+	if w.gz != nil {
+		w.gz.Close()
+	}
+}
+
+// Flush implements the <http.Flusher> interface.
+func (w *RWWrapper) Flush() {
+	if w.gz != nil {
+		w.gz.Flush()
+	}
+
+	if rw, ok := w.rw.(http.Flusher); ok {
+		rw.Flush()
+	}
+}
+
+// WriteHeader wraps the WriteHeader method of the <http.ResponseWriter>.
+func (w *RWWrapper) WriteHeader(statusCode int) {
+	w.rw.Header().Set("Server", "couper.io")
+	w.rw.Header().Add(handler.VaryHeader, handler.AcceptEncodingHeader)
+
+	if w.gz != nil {
+		w.rw.Header().Del(handler.ContentLengthHeader)
+		w.rw.Header().Set(handler.ContentEncodingHeader, handler.GzipName)
+	}
+
+	w.rw.WriteHeader(statusCode)
+}


Test patch:
diff --git a/handler/proxy_test.go b/handler/proxy_test.go
index 408c485f..bea571b2 100644
--- a/handler/proxy_test.go
+++ b/handler/proxy_test.go
@@ -2,6 +2,7 @@ package handler_test
 
 import (
 	"bytes"
+	"compress/gzip"
 	"context"
 	"io"
 	"net/http"
@@ -13,6 +14,7 @@ import (
 
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/hclsimple"
+	"github.com/hashicorp/hcl/v2/hcltest"
 	"github.com/sirupsen/logrus"
 	logrustest "github.com/sirupsen/logrus/hooks/test"
 
@@ -721,3 +723,85 @@ func TestProxy_setRoundtripContext_Variables_json_body(t *testing.T) {
 		}
 	}
 }
+
+func TestServer_DisableCompression(t *testing.T) {
+	helper := test.New(t)
+
+	origin := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		if req.Header.Get("Accept-Encoding") != "" {
+			t.Error("Unexpected Accept-Encoding header")
+		}
+		rw.WriteHeader(http.StatusNoContent)
+	}))
+	defer origin.Close()
+
+	logger, _ := logrustest.NewNullLogger()
+
+	var hclBody []hcl.Body
+	u := seetie.GoToValue(origin.URL)
+	hclBody = append(hclBody, hcltest.MockBody(&hcl.BodyContent{
+		Attributes: hcltest.MockAttrs(map[string]hcl.Expression{
+			"origin": hcltest.MockExprLiteral(u),
+		}),
+	}))
+
+	p, err := handler.NewProxy(
+		&handler.ProxyOptions{Context: hclBody},
+		logger.WithContext(nil),
+		nil, eval.NewENVContext(nil),
+	)
+	helper.Must(err)
+
+	req := httptest.NewRequest(http.MethodOptions, "http://1.2.3.4/", nil)
+	p.ServeHTTP(httptest.NewRecorder(), req)
+}
+
+func TestServer_ModifyAcceptEncoding(t *testing.T) {
+	helper := test.New(t)
+
+	origin := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		if ae := req.Header.Get("Accept-Encoding"); ae != "gzip" {
+			t.Errorf("Unexpected Accept-Encoding header: %s", ae)
+		}
+
+		var b bytes.Buffer
+		w := gzip.NewWriter(&b)
+		for i := 1; i < 1000; i++ {
+			w.Write([]byte("<html/>"))
+		}
+		w.Close()
+
+		rw.Header().Set("Content-Encoding", "gzip")
+		rw.Write(b.Bytes())
+	}))
+	defer origin.Close()
+
+	logger, _ := logrustest.NewNullLogger()
+
+	var hclBody []hcl.Body
+	u := seetie.GoToValue(origin.URL)
+	hclBody = append(hclBody, hcltest.MockBody(&hcl.BodyContent{
+		Attributes: hcltest.MockAttrs(map[string]hcl.Expression{
+			"origin": hcltest.MockExprLiteral(u),
+		}),
+	}))
+
+	p, err := handler.NewProxy(
+		&handler.ProxyOptions{Context: hclBody},
+		logger.WithContext(nil),
+		nil, eval.NewENVContext(nil),
+	)
+	helper.Must(err)
+
+	req := httptest.NewRequest(http.MethodOptions, "http://1.2.3.4/", nil)
+	req.Header.Set("Accept-Encoding", "br, gzip")
+	res := httptest.NewRecorder()
+	p.ServeHTTP(res, req)
+
+	if l := res.Header().Get("Content-Length"); l != "60" {
+		t.Errorf("Unexpected C/L: %s", l)
+	}
+	if l := len(res.Body.Bytes()); l != 6993 {
+		t.Errorf("Unexpected body length: %d", l)
+	}
+}
diff --git a/internal/test/client.go b/internal/test/client.go
new file mode 100644
index 00000000..7e79c6d1
--- /dev/null
+++ b/internal/test/client.go
@@ -0,0 +1,13 @@
+package test
+
+import "net/http"
+
+// NewHTTPClient creates a new <http.Client> object.
+func NewHTTPClient() *http.Client {
+	client := &http.Client{
+		Transport: &http.Transport{
+			DisableCompression: true,
+		},
+	}
+	return client
+}
diff --git a/server/http_integration_test.go b/server/http_integration_test.go
index 7767ef8b..60704eac 100644
--- a/server/http_integration_test.go
+++ b/server/http_integration_test.go
@@ -2,9 +2,11 @@ package server_test
 
 import (
 	"bytes"
+	"compress/gzip"
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net"
 	"net/http"
@@ -110,7 +112,6 @@ func newClient() *http.Client {
 	dialer := &net.Dialer{}
 	return &http.Client{
 		Transport: &http.Transport{
-			DisableCompression: true,
 			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
 				_, port, _ := net.SplitHostPort(addr)
 				if port != "" {
@@ -118,6 +119,7 @@ func newClient() *http.Client {
 				}
 				return dialer.DialContext(ctx, "tcp4", "127.0.0.1")
 			},
+			DisableCompression: true,
 		},
 	}
 }
@@ -458,6 +460,73 @@ func TestHTTPServer_XFHHeader(t *testing.T) {
 	cleanup(shutdown, t)
 }
 
+func TestHTTPServer_Gzip(t *testing.T) {
+	client := newClient()
+
+	confPath := path.Join("testdata/integration", "files/03_gzip.hcl")
+	shutdown, _ := newCouper(confPath, test.New(t))
+
+	type testCase struct {
+		name                 string
+		headerAcceptEncoding string
+		path                 string
+		expectGzipResponse   bool
+	}
+
+	for _, tc := range []testCase{
+		{"with mixed header AE gzip", "br, gzip", "/index.html", true},
+		{"with header AE gzip", "gzip", "/index.html", true},
+		{"with header AE and without gzip", "deflate", "/index.html", false},
+		{"with header AE and space", " ", "/index.html", false},
+	} {
+		t.Run(tc.name, func(subT *testing.T) {
+			helper := test.New(subT)
+
+			req, err := http.NewRequest(http.MethodGet, "http://example.org:9898"+tc.path, nil)
+			helper.Must(err)
+
+			if tc.headerAcceptEncoding != "" {
+				req.Header.Set("Accept-Encoding", tc.headerAcceptEncoding)
+			}
+
+			res, err := client.Do(req)
+			helper.Must(err)
+
+			var body io.Reader
+			body = res.Body
+
+			if !tc.expectGzipResponse {
+				if val := res.Header.Get("Content-Encoding"); val != "" {
+					t.Errorf("Expected no header with key Content-Encoding, got value: %s", val)
+				}
+			} else {
+				if ce := res.Header.Get("Content-Encoding"); ce != "gzip" {
+					t.Errorf("Expected Content-Encoding header value: %q, got: %q", "gzip", ce)
+				}
+
+				body, err = gzip.NewReader(res.Body)
+				helper.Must(err)
+			}
+
+			if vr := res.Header.Get("Vary"); vr != "Accept-Encoding" {
+				t.Errorf("Expected Accept-Encoding header value %q, got: %q", "Vary", vr)
+			}
+
+			resBytes, err := ioutil.ReadAll(body)
+			helper.Must(err)
+
+			srcBytes, err := ioutil.ReadFile(filepath.Join(testWorkingDir, "testdata/integration/files/htdocs_c_gzip"+tc.path))
+			helper.Must(err)
+
+			if !bytes.Equal(resBytes, srcBytes) {
+				t.Errorf("Want:\n%s\nGot:\n%s", string(srcBytes), string(resBytes))
+			}
+		})
+	}
+
+	cleanup(shutdown, t)
+}
+
 func TestHTTPServer_Endpoint_Evaluation(t *testing.T) {
 	client := newClient()
 
diff --git a/server/http_test.go b/server/http_test.go
index 6f4b8d68..a4e541f1 100644
--- a/server/http_test.go
+++ b/server/http_test.go
@@ -78,6 +78,7 @@ func TestHTTPServer_ServeHTTP_Files(t *testing.T) {
 		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
 			return net.Dial("tcp4", gw.Addr())
 		},
+		DisableCompression: true,
 	}}
 
 	for i, testCase := range []struct {
@@ -94,7 +95,10 @@ func TestHTTPServer_ServeHTTP_Files(t *testing.T) {
 		{"/apps/shiny-product/api/", nil, http.StatusNoContent},
 		{"/apps/shiny-product/api/foo%20bar:%22baz%22", []byte(`{"code": 4001}`), 404},
 	} {
-		res, err := connectClient.Get(fmt.Sprintf("http://example.com:%s%s", port, testCase.path))
+		req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://example.com:%s%s", port, testCase.path), nil)
+		helper.Must(err)
+
+		res, err := connectClient.Do(req)
 		helper.Must(err)
 
 		if res.StatusCode != testCase.expectedStatus {
@@ -172,6 +176,7 @@ func TestHTTPServer_ServeHTTP_Files2(t *testing.T) {
 			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
 				return net.Dial("tcp4", couper.Addr())
 			},
+			DisableCompression: true,
 		},
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {
 			return http.ErrUseLastResponse
@@ -213,7 +218,10 @@ func TestHTTPServer_ServeHTTP_Files2(t *testing.T) {
 		//FIXME:
 		//{"/api", content500.Bytes(), 500},
 	} {
-		res, err := connectClient.Get(fmt.Sprintf("http://example.com:%s%s", port, testCase.path))
+		req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://example.com:%s%s", port, testCase.path), nil)
+		helper.Must(err)
+
+		res, err := connectClient.Do(req)
 		helper.Must(err)
 
 		if res.StatusCode != testCase.expectedStatus {
@@ -256,7 +264,7 @@ func TestHTTPServer_ServeHTTP_UUID_Option(t *testing.T) {
 
 			hook.Reset()
 
-			_, err := http.DefaultClient.Do(req)
+			_, err := test.NewHTTPClient().Do(req)
 			helper.Must(err)
 			time.Sleep(time.Millisecond * 10) // log hook needs some time?
 
diff --git a/server/testdata/integration/files/03_gzip.hcl b/server/testdata/integration/files/03_gzip.hcl
new file mode 100644
index 00000000..350cd5b4
--- /dev/null
+++ b/server/testdata/integration/files/03_gzip.hcl
@@ -0,0 +1,8 @@
+server "serve-gzip" {
+  hosts = ["example.org:9898"]
+  error_file = "./../server_error.html"
+
+  files {
+    document_root = "./htdocs_c_gzip"
+  }
+}
diff --git a/server/testdata/integration/files/htdocs_c_gzip/index.html b/server/testdata/integration/files/htdocs_c_gzip/index.html
new file mode 100644
index 00000000..babaf68b
--- /dev/null
+++ b/server/testdata/integration/files/htdocs_c_gzip/index.html
@@ -0,0 +1,22 @@
+<html>
+	<body>
+		<div>index C</div>
+		<div>index C</div>
+		<div>index C</div>
+		<div>index C</div>
+		<div>index C</div>
+		<div>index C</div>
+		<div>index C</div>
+		<div>index C</div>
+		<div>index C</div>
+		<div>index C</div>
+		<div>index C</div>
+		<div>index C</div>
+		<div>index C</div>
+		<div>index C</div>
+		<div>index C</div>
+		<div>index C</div>
+		<div>index C</div>
+		<div>index C</div>
+	</body>
+</html>