Repository: valyala/fasthttp
Pull request #1216 (2022-02-15 08:27:26)

feature: Keep the memory usage of the service at a stable level
add request and response body size limit, it prevents the large body from slowly stretching the memory of the entire service


Patch:
diff --git a/fasthttpproxy/http.go b/fasthttpproxy/http.go
index 6bc9c1e..2d91813 100644
--- a/fasthttpproxy/http.go
+++ b/fasthttpproxy/http.go
@@ -49,7 +49,7 @@ func FasthttpHTTPDialerTimeout(proxy string, timeout time.Duration) fasthttp.Dia
 			return nil, err
 		}
 
-		req := "CONNECT " + addr + " HTTP/1.1\r\n"
+		req := fmt.Sprintf("CONNECT %s HTTP/1.1\r\nHost: %s\r\n", addr, addr)
 		if auth != "" {
 			req += "Proxy-Authorization: Basic " + auth + "\r\n"
 		}
diff --git a/fs.go b/fs.go
index 257f066..72c832a 100644
--- a/fs.go
+++ b/fs.go
@@ -30,6 +30,10 @@ import (
 // with good compression ratio.
 //
 // See also RequestCtx.SendFileBytes.
+//
+// WARNING: do not pass any user supplied paths to this function!
+// WARNING: if path is based on user input users will be able to request
+// any file on your filesystem! Use fasthttp.FS with a sane Root instead.
 func ServeFileBytesUncompressed(ctx *RequestCtx, path []byte) {
 	ServeFileUncompressed(ctx, b2s(path))
 }
@@ -43,6 +47,10 @@ func ServeFileBytesUncompressed(ctx *RequestCtx, path []byte) {
 // with good compression ratio.
 //
 // See also RequestCtx.SendFile.
+//
+// WARNING: do not pass any user supplied paths to this function!
+// WARNING: if path is based on user input users will be able to request
+// any file on your filesystem! Use fasthttp.FS with a sane Root instead.
 func ServeFileUncompressed(ctx *RequestCtx, path string) {
 	ctx.Request.Header.DelBytes(strAcceptEncoding)
 	ServeFile(ctx, path)
@@ -62,6 +70,10 @@ func ServeFileUncompressed(ctx *RequestCtx, path string) {
 // file contents.
 //
 // See also RequestCtx.SendFileBytes.
+//
+// WARNING: do not pass any user supplied paths to this function!
+// WARNING: if path is based on user input users will be able to request
+// any file on your filesystem! Use fasthttp.FS with a sane Root instead.
 func ServeFileBytes(ctx *RequestCtx, path []byte) {
 	ServeFile(ctx, b2s(path))
 }
@@ -79,6 +91,10 @@ func ServeFileBytes(ctx *RequestCtx, path []byte) {
 // Use ServeFileUncompressed is you don't need serving compressed file contents.
 //
 // See also RequestCtx.SendFile.
+//
+// WARNING: do not pass any user supplied paths to this function!
+// WARNING: if path is based on user input users will be able to request
+// any file on your filesystem! Use fasthttp.FS with a sane Root instead.
 func ServeFile(ctx *RequestCtx, path string) {
 	rootFSOnce.Do(func() {
 		rootFSHandler = rootFS.NewRequestHandler()
diff --git a/go.mod b/go.mod
index 46acbb7..2219f83 100644
--- a/go.mod
+++ b/go.mod
@@ -4,10 +4,10 @@ go 1.12
 
 require (
 	github.com/andybalholm/brotli v1.0.4
-	github.com/klauspost/compress v1.14.2
+	github.com/klauspost/compress v1.14.4
 	github.com/valyala/bytebufferpool v1.0.0
 	github.com/valyala/tcplisten v1.0.0
-	golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838
-	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd
-	golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27
+	golang.org/x/crypto v0.0.0-20220214200702-86341886e292
+	golang.org/x/net v0.0.0-20220225172249-27dd8689420f
+	golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9
 )
diff --git a/go.sum b/go.sum
index 231e65c..2910ff1 100644
--- a/go.sum
+++ b/go.sum
@@ -1,22 +1,22 @@
 github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
 github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
-github.com/klauspost/compress v1.14.2 h1:S0OHlFk/Gbon/yauFJ4FfJJF5V0fc5HbBTJazi28pRw=
-github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/compress v1.14.4 h1:eijASRJcobkVtSt81Olfh7JX43osYLwy5krOJo6YEu4=
+github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
-golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838 h1:71vQrMauZZhcTVK6KdYM+rklehEEwb3E+ZhaE5jrPrE=
-golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE=
+golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
+golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 h1:XDXtA5hveEEV8JB2l7nhMTp3t3cHp9ZpwcdjqyEWLlo=
-golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs=
+golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
diff --git a/header.go b/header.go
index 123fe95..7b9df08 100644
--- a/header.go
+++ b/header.go
@@ -59,9 +59,10 @@ type ResponseHeader struct {
 type RequestHeader struct {
 	noCopy noCopy //nolint:unused,structcheck
 
-	disableNormalizing bool
-	noHTTP11           bool
-	connectionClose    bool
+	disableNormalizing   bool
+	noHTTP11             bool
+	connectionClose      bool
+	noDefaultContentType bool
 
 	// These two fields have been moved close to other bool fields
 	// for reducing RequestHeader object size.
@@ -943,9 +944,15 @@ func (h *ResponseHeader) resetSkipNormalize() {
 	h.trailer = h.trailer[:0]
 }
 
+// SetNoDefaultContentType allows you to control if a default Content-Type header will be set (false) or not (true).
+func (h *RequestHeader) SetNoDefaultContentType(noDefaultContentType bool) {
+	h.noDefaultContentType = noDefaultContentType
+}
+
 // Reset clears request header.
 func (h *RequestHeader) Reset() {
 	h.disableNormalizing = false
+	h.SetNoDefaultContentType(false)
 	h.resetSkipNormalize()
 }
 
@@ -2273,7 +2280,7 @@ func (h *RequestHeader) AppendBytes(dst []byte) []byte {
 	}
 
 	contentType := h.ContentType()
-	if len(contentType) == 0 && !h.ignoreBody() {
+	if !h.noDefaultContentType && len(contentType) == 0 && !h.ignoreBody() {
 		contentType = strDefaultContentType
 	}
 	if len(contentType) > 0 {
@@ -2435,6 +2442,10 @@ func (h *RequestHeader) parseTrailer(buf []byte) (int, error) {
 }
 
 func isBadTrailer(key []byte) bool {
+	if len(key) == 0 {
+		return true
+	}
+
 	switch key[0] | 0x20 {
 	case 'a':
 		return caseInsensitiveCompare(key, strAuthorization)
diff --git a/http.go b/http.go
index d61eb62..47431cd 100644
--- a/http.go
+++ b/http.go
@@ -17,6 +17,18 @@ import (
 	"github.com/valyala/bytebufferpool"
 )
 
+var (
+	requestBodyPoolSizeLimit  = -1
+	responseBodyPoolSizeLimit = -1
+)
+
+// SetBodySizePoolLimit set the max body size for bodies to be returned to the pool.
+// If the body size is larger it will be released instead of put back into the pool for reuse.
+func SetBodySizePoolLimit(reqBodyLimit, respBodyLimit int) {
+	requestBodyPoolSizeLimit = reqBodyLimit
+	responseBodyPoolSizeLimit = respBodyLimit
+}
+
 // Request represents HTTP request.
 //
 // It is forbidden copying Request instances. Create new instances
@@ -957,6 +969,9 @@ func readMultipartForm(r io.Reader, boundary string, size, maxInMemoryFileSize i
 
 // Reset clears request contents.
 func (req *Request) Reset() {
+	if requestBodyPoolSizeLimit >= 0 && req.body != nil {
+		req.ReleaseBody(requestBodyPoolSizeLimit)
+	}
 	req.Header.Reset()
 	req.resetSkipHeader()
 	req.timeout = 0
@@ -986,6 +1001,9 @@ func (req *Request) RemoveMultipartFormFiles() {
 
 // Reset clears response contents.
 func (resp *Response) Reset() {
+	if responseBodyPoolSizeLimit >= 0 && resp.body != nil {
+		resp.ReleaseBody(responseBodyPoolSizeLimit)
+	}
 	resp.Header.Reset()
 	resp.resetSkipHeader()
 	resp.SkipBody = false
diff --git a/server.go b/server.go
index c6a2923..1a8ba8c 100644
--- a/server.go
+++ b/server.go
@@ -1338,6 +1338,10 @@ func (ctx *RequestCtx) ResetBody() {
 // SendFile logs all the errors via ctx.Logger.
 //
 // See also ServeFile, FSHandler and FS.
+//
+// WARNING: do not pass any user supplied paths to this function!
+// WARNING: if path is based on user input users will be able to request
+// any file on your filesystem! Use fasthttp.FS with a sane Root instead.
 func (ctx *RequestCtx) SendFile(path string) {
 	ServeFile(ctx, path)
 }
@@ -1349,6 +1353,10 @@ func (ctx *RequestCtx) SendFile(path string) {
 // SendFileBytes logs all the errors via ctx.Logger.
 //
 // See also ServeFileBytes, FSHandler and FS.
+//
+// WARNING: do not pass any user supplied paths to this function!
+// WARNING: if path is based on user input users will be able to request
+// any file on your filesystem! Use fasthttp.FS with a sane Root instead.
 func (ctx *RequestCtx) SendFileBytes(path []byte) {
 	ServeFileBytes(ctx, path)
 }
@@ -2335,7 +2343,7 @@ func (s *Server) serveConn(c net.Conn) (err error) {
 			timeoutResponse.CopyTo(&ctx.Response)
 		}
 
-		if !ctx.IsGet() && ctx.IsHead() {
+		if ctx.IsHead() {
 			ctx.Response.SkipBody = true
 		}
 
diff --git a/strings.go b/strings.go
index e4fb08b..370e307 100644
--- a/strings.go
+++ b/strings.go
@@ -7,21 +7,25 @@ var (
 )
 
 var (
-	strSlash            = []byte("/")
-	strSlashSlash       = []byte("//")
-	strSlashDotDot      = []byte("/..")
-	strSlashDotSlash    = []byte("/./")
-	strSlashDotDotSlash = []byte("/../")
-	strCRLF             = []byte("\r\n")
-	strHTTP             = []byte("http")
-	strHTTPS            = []byte("https")
-	strHTTP10           = []byte("HTTP/1.0")
-	strHTTP11           = []byte("HTTP/1.1")
-	strColon            = []byte(":")
-	strColonSlashSlash  = []byte("://")
-	strColonSpace       = []byte(": ")
-	strCommaSpace       = []byte(", ")
-	strGMT              = []byte("GMT")
+	strSlash                    = []byte("/")
+	strSlashSlash               = []byte("//")
+	strSlashDotDot              = []byte("/..")
+	strSlashDotSlash            = []byte("/./")
+	strSlashDotDotSlash         = []byte("/../")
+	strBackSlashDotDot          = []byte(`\..`)
+	strBackSlashDotBackSlash    = []byte(`\.\`)
+	strSlashDotDotBackSlash     = []byte(`/..\`)
+	strBackSlashDotDotBackSlash = []byte(`\..\`)
+	strCRLF                     = []byte("\r\n")
+	strHTTP                     = []byte("http")
+	strHTTPS                    = []byte("https")
+	strHTTP10                   = []byte("HTTP/1.0")
+	strHTTP11                   = []byte("HTTP/1.1")
+	strColon                    = []byte(":")
+	strColonSlashSlash          = []byte("://")
+	strColonSpace               = []byte(": ")
+	strCommaSpace               = []byte(", ")
+	strGMT                      = []byte("GMT")
 
 	strResponseContinue = []byte("HTTP/1.1 100 Continue\r\n\r\n")
 
diff --git a/uri.go b/uri.go
index 62974df..38a431e 100644
--- a/uri.go
+++ b/uri.go
@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"path/filepath"
 	"strconv"
 	"sync"
 )
@@ -634,6 +635,60 @@ func normalizePath(dst, src []byte) []byte {
 		b = b[:nn+1]
 	}
 
+	if filepath.Separator == '\\' {
+		// remove \.\ parts
+		b = dst
+		for {
+			n := bytes.Index(b, strBackSlashDotBackSlash)
+			if n < 0 {
+				break
+			}
+			nn := n + len(strSlashDotSlash) - 1
+			copy(b[n:], b[nn:])
+			b = b[:len(b)-nn+n]
+		}
+
+		// remove /foo/..\ parts
+		for {
+			n := bytes.Index(b, strSlashDotDotBackSlash)
+			if n < 0 {
+				break
+			}
+			nn := bytes.LastIndexByte(b[:n], '/')
+			if nn < 0 {
+				nn = 0
+			}
+			n += len(strSlashDotDotBackSlash) - 1
+			copy(b[nn:], b[n:])
+			b = b[:len(b)-n+nn]
+		}
+
+		// remove /foo\..\ parts
+		for {
+			n := bytes.Index(b, strBackSlashDotDotBackSlash)
+			if n < 0 {
+				break
+			}
+			nn := bytes.LastIndexByte(b[:n], '/')
+			if nn < 0 {
+				nn = 0
+			}
+			n += len(strBackSlashDotDotBackSlash) - 1
+			copy(b[nn:], b[n:])
+			b = b[:len(b)-n+nn]
+		}
+
+		// remove trailing \foo\..
+		n := bytes.LastIndex(b, strBackSlashDotDot)
+		if n >= 0 && n+len(strSlashDotDot) == len(b) {
+			nn := bytes.LastIndexByte(b[:n], '/')
+			if nn < 0 {
+				return append(dst[:0], strSlash...)
+			}
+			b = b[:nn+1]
+		}
+	}
+
 	return b
 }
 
diff --git a/workerpool.go b/workerpool.go
index 9b1987e..f1a9a4c 100644
--- a/workerpool.go
+++ b/workerpool.go
@@ -1,6 +1,7 @@
 package fasthttp
 
 import (
+	"errors"
 	"net"
 	"runtime"
 	"strings"
@@ -226,7 +227,8 @@ func (wp *workerPool) workerFunc(ch *workerChan) {
 				strings.Contains(errStr, "reset by peer") ||
 				strings.Contains(errStr, "request headers: small read buffer") ||
 				strings.Contains(errStr, "unexpected EOF") ||
-				strings.Contains(errStr, "i/o timeout")) {
+				strings.Contains(errStr, "i/o timeout") ||
+				errors.Is(err, ErrBadTrailer)) {
 				wp.Logger.Printf("error when serving connection %q<->%q: %s", c.LocalAddr(), c.RemoteAddr(), err)
 			}
 		}


Test patch:
diff --git a/header_test.go b/header_test.go
index f041577..e313f49 100644
--- a/header_test.go
+++ b/header_test.go
@@ -1381,6 +1381,33 @@ func TestRequestContentTypeDefaultNotEmpty(t *testing.T) {
 	}
 }
 
+func TestRequestContentTypeNoDefault(t *testing.T) {
+	t.Parallel()
+
+	var h RequestHeader
+	h.SetMethod(MethodDelete)
+	h.SetNoDefaultContentType(true)
+
+	w := &bytes.Buffer{}
+	bw := bufio.NewWriter(w)
+	if err := h.Write(bw); err != nil {
+		t.Fatalf("Unexpected error: %s", err)
+	}
+	if err := bw.Flush(); err != nil {
+		t.Fatalf("Unexpected error: %s", err)
+	}
+
+	var h1 RequestHeader
+	br := bufio.NewReader(w)
+	if err := h1.Read(br); err != nil {
+		t.Fatalf("Unexpected error: %s", err)
+	}
+
+	if string(h1.contentType) != "" {
+		t.Fatalf("unexpected Content-Type %q. Expecting %q", h1.contentType, "")
+	}
+}
+
 func TestResponseDateNoDefaultNotEmpty(t *testing.T) {
 	t.Parallel()
 
diff --git a/http_test.go b/http_test.go
index 426b70a..12ca1f1 100644
--- a/http_test.go
+++ b/http_test.go
@@ -32,6 +32,9 @@ func TestInvalidTrailers(t *testing.T) {
 	if err := (&Response{}).Read(bufio.NewReader(bytes.NewReader([]byte{0x54, 0x52, 0x61, 0x49, 0x4c, 0x65, 0x52, 0x3a, 0x2c, 0x0a, 0x0a}))); !strings.Contains(err.Error(), "cannot find whitespace in the first line of response") {
 		t.Fatal(err)
 	}
+	if err := (&Request{}).Read(bufio.NewReader(bytes.NewReader([]byte{0xff, 0x20, 0x0a, 0x54, 0x52, 0x61, 0x49, 0x4c, 0x65, 0x52, 0x3a, 0x2c, 0x0a, 0x0a}))); !strings.Contains(err.Error(), "contain forbidden trailer") {
+		t.Fatal(err)
+	}
 
 	b, _ := base64.StdEncoding.DecodeString("tCAKIDoKCToKICAKCToKICAKCToKIAogOgoJOgogIAoJOgovIC8vOi4KOh0KVFJhSUxlUjo9HT09HQpUUmFJTGVSOicQAApUUmFJTGVSOj0gHSAKCT09HQoKOgoKCgo=")
 	if err := (&Request{}).Read(bufio.NewReader(bytes.NewReader(b))); !strings.Contains(err.Error(), "error when reading request headers: invalid header key") {