Repository: valyala/fasthttp
Pull request #787 (2020-04-21 03:20:35)

allow the expect 100 continue workflow to deny requests
Fixes https://github.com/valyala/fasthttp/issues/783

An okay solution right now which allows servers to deny reading the request body after reading the request headers via a server callback configuration.

A more long term solution later on might be to give full control to the user during their handler callback, but that would definitely require a lot more work and be more error prone for the user.

One unfortunate issue with this is, if you do any major work in processing headers before accepting or denying a request, you likely will have to re-process the headers on the other side.  It might be worth sticking a

````
func() interface[}
````
callback out of the deny which can be passed to a new handler, so people can pass state between the two.


Patch:
diff --git a/.travis.yml b/.travis.yml
index 2f5eaba..12d9230 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,10 +18,19 @@ go:
   - 1.10.x
   - 1.9.x
 
-matrix:
-  allow_failures:
-    - tip
-  fast_finish: true
+script:
+  # build test for supported platforms
+  - GOOS=linux go build
+  - GOOS=darwin go build
+  - GOOS=freebsd go build
+  - GOOS=windows go build
+  - GOARCH=386 go build
+
+  # run tests on a standard platform
+  - go test -v ./...
+
+  # run tests with the race detector as well
+  - go test -race -v ./...
 
 env:
   global:
@@ -31,21 +40,9 @@ before_install:
   - go get -t -v ./...
 
 jobs:
+  allow_failures:
+    - go: tip
   include:
-    - stage: test
-      script:
-        # build test for supported platforms
-        - GOOS=linux go build
-        - GOOS=darwin go build
-        - GOOS=freebsd go build
-        - GOOS=windows go build
-        - GOARCH=386 go build
-
-        # run tests on a standard platform
-        - go test -v ./...
-
-        # run tests with the race detector as well
-        - go test -race -v ./...
     - stage: fuzzit.dev
       os:
         - linux
diff --git a/bytesconv.go b/bytesconv.go
index a3b82e2..222435a 100644
--- a/bytesconv.go
+++ b/bytesconv.go
@@ -185,11 +185,12 @@ func parseUintBuf(b []byte) (int, int, error) {
 			}
 			return v, i, nil
 		}
+		vNew := 10*v + int(k)
 		// Test for overflow.
-		if v*10 < v {
+		if vNew < v {
 			return -1, i, errTooLongInt
 		}
-		v = 10*v + int(k)
+		v = vNew
 	}
 	return v, n, nil
 }
diff --git a/client.go b/client.go
index 7665312..4262274 100644
--- a/client.go
+++ b/client.go
@@ -1350,7 +1350,7 @@ func (c *HostClient) doNonNilReqResp(req *Request, resp *Response) (bool, error)
 		}
 	}
 
-	if customSkipBody || !req.Header.IsGet() && req.Header.IsHead() {
+	if customSkipBody || req.Header.IsHead() {
 		resp.SkipBody = true
 	}
 	if c.DisableHeaderNamesNormalizing {
@@ -1467,8 +1467,8 @@ func (c *HostClient) acquireConn(reqTimeout time.Duration) (cc *clientConn, err
 		timeoutOverridden := false
 		// reqTimeout == 0 means not set
 		if reqTimeout > 0 && reqTimeout < timeout {
-				timeout = reqTimeout
-				timeoutOverridden = true
+			timeout = reqTimeout
+			timeoutOverridden = true
 		}
 
 		// wait for a free connection
diff --git a/fasthttpproxy/http.go b/fasthttpproxy/http.go
new file mode 100644
index 0000000..62de1d0
--- /dev/null
+++ b/fasthttpproxy/http.go
@@ -0,0 +1,59 @@
+package fasthttpproxy
+
+import (
+	"bufio"
+	"encoding/base64"
+	"fmt"
+	"net"
+	"strings"
+
+	"github.com/valyala/fasthttp"
+)
+
+// FasthttpHTTPDialer returns a fasthttp.DialFunc that dials using
+// the provided HTTP proxy.
+//
+// Example usage:
+//	c := &fasthttp.Client{
+//		Dial: fasthttpproxy.FasthttpHTTPDialer("username:password@localhost:9050"),
+//	}
+func FasthttpHTTPDialer(proxy string) fasthttp.DialFunc {
+	var auth string
+	if strings.Contains(proxy, "@") {
+		split := strings.Split(proxy, "@")
+		auth = base64.StdEncoding.EncodeToString([]byte(split[0]))
+		proxy = split[1]
+	}
+
+	return func(addr string) (net.Conn, error) {
+		conn, err := fasthttp.Dial(proxy)
+		if err != nil {
+			return nil, err
+		}
+
+		req := "CONNECT " + addr + " HTTP/1.1\r\n"
+		if auth != "" {
+			req += "Proxy-Authorization: Basic " + auth + "\r\n"
+		}
+		req += "\r\n"
+
+		if _, err := conn.Write([]byte(req)); err != nil {
+			return nil, err
+		}
+
+		res := fasthttp.AcquireResponse()
+		defer fasthttp.ReleaseResponse(res)
+
+		res.SkipBody = true
+
+		if err := res.Read(bufio.NewReader(conn)); err != nil {
+			conn.Close()
+			return nil, err
+		}
+		if res.Header.StatusCode() != 200 {
+			conn.Close()
+			return nil, fmt.Errorf("could not connect to proxy")
+		}
+		return conn, nil
+	}
+}
diff --git a/go.mod b/go.mod
index 3e1c929..ba38c27 100644
--- a/go.mod
+++ b/go.mod
@@ -3,9 +3,8 @@ module github.com/valyala/fasthttp
 go 1.11
 
 require (
-	github.com/klauspost/compress v1.8.2
-	github.com/klauspost/cpuid v1.2.1 // indirect
+	github.com/klauspost/compress v1.10.4
 	github.com/valyala/bytebufferpool v1.0.0
 	github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a
-	golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297
+	golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e
 )
diff --git a/go.sum b/go.sum
index 52407f3..96298ae 100644
--- a/go.sum
+++ b/go.sum
@@ -1,13 +1,12 @@
-github.com/klauspost/compress v1.8.2 h1:Bx0qjetmNjdFXASH02NSAREKpiaDwkO1DRZ3dV2KCcs=
-github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w=
-github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/compress v1.10.4 h1:jFzIFaf586tquEB5EhzQG0HwGNSlgAJpG53G6Ss11wc=
+github.com/klauspost/compress v1.10.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM=
-golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
diff --git a/server.go b/server.go
index 29ce909..80ffd39 100644
--- a/server.go
+++ b/server.go
@@ -172,6 +172,17 @@ type Server struct {
 	// non zero RequestConfig field values will overwrite the default configs
 	HeaderReceived func(header *RequestHeader) RequestConfig
 
+	// ContinueHandler is called after receiving the Expect 100 Continue Header
+	//
+	// https://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html#sec8.2.3
+	// https://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.1.1
+	// Using ContinueHandler a server can make decisioning on whether or not
+	// to read a potentially large request body based on the headers
+	//
+	// The default is to automatically read request bodies of Expect 100 Continue requests
+	// like they are normal requests
+	ContinueHandler func(header *RequestHeader) bool
+
 	// Server name for sending in response headers.
 	//
 	// Default server name is used if left blank.
@@ -1935,7 +1946,8 @@ func (s *Server) serveConn(c net.Conn) (err error) {
 		connectionClose bool
 		isHTTP11        bool
 
-		reqReset bool
+		reqReset               bool
+		continueReadingRequest bool = true
 	)
 	for {
 		connRequestNum++
@@ -2007,6 +2019,7 @@ func (s *Server) serveConn(c net.Conn) (err error) {
 				//read body
 				err = ctx.Request.readLimitBody(br, maxRequestBodySize, s.GetOnly, !s.DisablePreParseMultipartForm)
 			}
+
 			if err == nil {
 				// If we read any bytes off the wire, we're active.
 				s.setState(c, StateActive)
@@ -2041,37 +2054,53 @@ func (s *Server) serveConn(c net.Conn) (err error) {
 		}
 
 		// 'Expect: 100-continue' request handling.
-		// See http://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html for details.
+		// See https://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html#sec8.2.3 for details.
 		if ctx.Request.MayContinue() {
-			// Send 'HTTP/1.1 100 Continue' response.
-			if bw == nil {
-				bw = acquireWriter(ctx)
-			}
-			_, err = bw.Write(strResponseContinue)
-			if err != nil {
-				break
-			}
-			err = bw.Flush()
-			if err != nil {
-				break
-			}
-			if s.ReduceMemoryUsage {
-				releaseWriter(s, bw)
-				bw = nil
-			}
 
-			// Read request body.
-			if br == nil {
-				br = acquireReader(ctx)
-			}
-			err = ctx.Request.ContinueReadBody(br, maxRequestBodySize, !s.DisablePreParseMultipartForm)
-			if (s.ReduceMemoryUsage && br.Buffered() == 0) || err != nil {
-				releaseReader(s, br)
-				br = nil
+			// Allow the ability to deny reading the incoming request body
+			if s.ContinueHandler != nil {
+				if continueReadingRequest = s.ContinueHandler(&ctx.Request.Header); !continueReadingRequest {
+					if br != nil {
+						br.Reset(ctx.c)
+					}
+
+					ctx.SetStatusCode(StatusExpectationFailed)
+				}
 			}
-			if err != nil {
-				bw = s.writeErrorResponse(bw, ctx, serverName, err)
-				break
+
+			if continueReadingRequest {
+				if bw == nil {
+					bw = acquireWriter(ctx)
+				}
+
+				// Send 'HTTP/1.1 100 Continue' response.
+				_, err = bw.Write(strResponseContinue)
+				if err != nil {
+					break
+				}
+				err = bw.Flush()
+				if err != nil {
+					break
+				}
+				if s.ReduceMemoryUsage {
+					releaseWriter(s, bw)
+					bw = nil
+				}
+
+				// Read request body.
+				if br == nil {
+					br = acquireReader(ctx)
+				}
+
+				err = ctx.Request.ContinueReadBody(br, maxRequestBodySize, !s.DisablePreParseMultipartForm)
+				if (s.ReduceMemoryUsage && br.Buffered() == 0) || err != nil {
+					releaseReader(s, br)
+					br = nil
+				}
+				if err != nil {
+					bw = s.writeErrorResponse(bw, ctx, serverName, err)
+					break
+				}
 			}
 		}
 
@@ -2084,7 +2113,11 @@ func (s *Server) serveConn(c net.Conn) (err error) {
 		ctx.connID = connID
 		ctx.connRequestNum = connRequestNum
 		ctx.time = time.Now()
-		s.Handler(ctx)
+
+		// If a client denies a request the handler should not be called
+		if continueReadingRequest {
+			s.Handler(ctx)
+		}
 
 		timeoutResponse = ctx.timeoutResponse
 		if timeoutResponse != nil {
diff --git a/uri.go b/uri.go
index c9a81d4..9d64db8 100644
--- a/uri.go
+++ b/uri.go
@@ -263,6 +263,10 @@ func (u *URI) Parse(host, uri []byte) {
 func (u *URI) parse(host, uri []byte, isTLS bool) {
 	u.Reset()
 
+	if stringContainsCTLByte(uri) {
+		return
+	}
+
 	if len(host) == 0 || bytes.Contains(uri, strColonSlashSlash) {
 		scheme, newHost, newURI := splitHostURI(host, uri)
 		u.scheme = append(u.scheme, scheme...)
@@ -581,3 +585,14 @@ func (u *URI) parseQueryArgs() {
 	u.queryArgs.ParseBytes(u.queryString)
 	u.parsedQueryArgs = true
 }
+
+// stringContainsCTLByte reports whether s contains any ASCII control character.
+func stringContainsCTLByte(s []byte) bool {
+	for i := 0; i < len(s); i++ {
+		b := s[i]
+		if b < ' ' || b == 0x7f {
+			return true
+		}
+	}
+	return false
+}


Test patch:
diff --git a/bytesconv_32_test.go b/bytesconv_32_test.go
index 405230b..f0647bd 100644
--- a/bytesconv_32_test.go
+++ b/bytesconv_32_test.go
@@ -38,10 +38,22 @@ func TestReadHexIntSuccess(t *testing.T) {
 	testReadHexIntSuccess(t, "1234ZZZ", 0x1234)
 }
 
+func TestParseUintError32(t *testing.T) {
+	t.Parallel()
+
+	// Overflow by last digit: 2 ** 32 / 2 * 10 ** n
+	testParseUintError(t, "2147483648")
+	testParseUintError(t, "21474836480")
+	testParseUintError(t, "214748364800")
+}
+
 func TestParseUintSuccess(t *testing.T) {
 	t.Parallel()
 
 	testParseUintSuccess(t, "0", 0)
 	testParseUintSuccess(t, "123", 123)
 	testParseUintSuccess(t, "123456789", 123456789)
+
+	// Max supported value: 2 ** 32 / 2 - 1
+	testParseUintSuccess(t, "2147483647", 2147483647)
 }
diff --git a/bytesconv_64_test.go b/bytesconv_64_test.go
index 1ba3237..d65f34f 100644
--- a/bytesconv_64_test.go
+++ b/bytesconv_64_test.go
@@ -39,6 +39,15 @@ func TestReadHexIntSuccess(t *testing.T) {
 	testReadHexIntSuccess(t, "7ffffffffffffff", 0x7ffffffffffffff)
 }
 
+func TestParseUintError64(t *testing.T) {
+	t.Parallel()
+
+	// Overflow by last digit: 2 ** 64 / 2 * 10 ** n
+	testParseUintError(t, "9223372036854775808")
+	testParseUintError(t, "92233720368547758080")
+	testParseUintError(t, "922337203685477580800")
+}
+
 func TestParseUintSuccess(t *testing.T) {
 	t.Parallel()
 
@@ -46,5 +55,7 @@ func TestParseUintSuccess(t *testing.T) {
 	testParseUintSuccess(t, "123", 123)
 	testParseUintSuccess(t, "1234567890", 1234567890)
 	testParseUintSuccess(t, "123456789012345678", 123456789012345678)
+
+	// Max supported value: 2 ** 64 / 2 - 1
 	testParseUintSuccess(t, "9223372036854775807", 9223372036854775807)
 }
diff --git a/client_test.go b/client_test.go
index 59bc413..3f9bc77 100644
--- a/client_test.go
+++ b/client_test.go
@@ -20,6 +20,38 @@ import (
 	"github.com/valyala/fasthttp/fasthttputil"
 )
 
+func TestClientInvalidURI(t *testing.T) {
+	t.Parallel()
+
+	ln := fasthttputil.NewInmemoryListener()
+	requests := int64(0)
+	s := &Server{
+		Handler: func(ctx *RequestCtx) {
+			atomic.AddInt64(&requests, 1)
+		},
+	}
+	go s.Serve(ln)
+	c := &Client{
+		Dial: func(addr string) (net.Conn, error) {
+			return ln.Dial()
+		},
+	}
+	req, res := AcquireRequest(), AcquireResponse()
+	defer func() {
+		ReleaseRequest(req)
+		ReleaseResponse(res)
+	}()
+	req.Header.SetMethod(MethodGet)
+	req.SetRequestURI("http://example.com\r\n\r\nGET /\r\n\r\n")
+	err := c.Do(req, res)
+	if err == nil {
+		t.Fatal("expected error (missing required Host header in request)")
+	}
+	if n := atomic.LoadInt64(&requests); n != 0 {
+		t.Fatalf("0 requests expected, got %d", n)
+	}
+}
+
 func TestClientGetWithBody(t *testing.T) {
 	t.Parallel()
 
diff --git a/server_test.go b/server_test.go
index a7ee71c..4252a6a 100644
--- a/server_test.go
+++ b/server_test.go
@@ -1654,6 +1654,96 @@ func TestServerExpect100Continue(t *testing.T) {
 	}
 }
 
+func TestServerContinueHandler(t *testing.T) {
+	t.Parallel()
+
+	acceptContentLength := 5
+	s := &Server{
+		ContinueHandler: func(headers *RequestHeader) bool {
+			if !headers.IsPost() {
+				t.Errorf("unexpected method %q. Expecting POST", headers.Method())
+			}
+
+			ct := headers.ContentType()
+			if string(ct) != "a/b" {
+				t.Errorf("unexpectected content-type: %q. Expecting %q", ct, "a/b")
+			}
+
+			// Pass on any request that isn't the accepted content length
+			return headers.contentLength == acceptContentLength
+		},
+		Handler: func(ctx *RequestCtx) {
+			if ctx.Request.Header.contentLength != acceptContentLength {
+				t.Errorf("all requests with content-length: other than %d, should be denied", acceptContentLength)
+			}
+			if !ctx.IsPost() {
+				t.Errorf("unexpected method %q. Expecting POST", ctx.Method())
+			}
+			if string(ctx.Path()) != "/foo" {
+				t.Errorf("unexpected path %q. Expecting %q", ctx.Path(), "/foo")
+			}
+			ct := ctx.Request.Header.ContentType()
+			if string(ct) != "a/b" {
+				t.Errorf("unexpectected content-type: %q. Expecting %q", ct, "a/b")
+			}
+			if string(ctx.PostBody()) != "12345" {
+				t.Errorf("unexpected body: %q. Expecting %q", ctx.PostBody(), "12345")
+			}
+			ctx.WriteString("foobar") //nolint:errcheck
+		},
+	}
+
+	sendRequest := func(rw *readWriter, expectedStatusCode int, expectedResponse string) {
+		ch := make(chan error)
+		go func() {
+			ch <- s.ServeConn(rw)
+		}()
+
+		select {
+		case err := <-ch:
+			if err != nil {
+				t.Fatalf("Unexpected error from serveConn: %s", err)
+			}
+		case <-time.After(100 * time.Millisecond):
+			t.Fatal("timeout")
+		}
+
+		br := bufio.NewReader(&rw.w)
+		verifyResponse(t, br, expectedStatusCode, string(defaultContentType), expectedResponse)
+
+		data, err := ioutil.ReadAll(br)
+		if err != nil {
+			t.Fatalf("Unexpected error when reading remaining data: %s", err)
+		}
+		if len(data) > 0 {
+			t.Fatalf("unexpected remaining data %q", data)
+		}
+	}
+
+	// The same server should not fail when handling the three different types of requests
+	// Regular requests
+	// Expect 100 continue accepted
+	// Exepect 100 continue denied
+	rw := &readWriter{}
+	for i := 0; i < 25; i++ {
+
+		// Regular requests without Expect 100 continue header
+		rw.r.Reset()
+		rw.r.WriteString("POST /foo HTTP/1.1\r\nHost: gle.com\r\nContent-Length: 5\r\nContent-Type: a/b\r\n\r\n12345")
+		sendRequest(rw, StatusOK, "foobar")
+
+		// Regular Expect 100 continue reqeuests that are accepted
+		rw.r.Reset()
+		rw.r.WriteString("POST /foo HTTP/1.1\r\nHost: gle.com\r\nExpect: 100-continue\r\nContent-Length: 5\r\nContent-Type: a/b\r\n\r\n12345")
+		sendRequest(rw, StatusOK, "foobar")
+
+		// Requests being denied
+		rw.r.Reset()
+		rw.r.WriteString("POST /foo HTTP/1.1\r\nHost: gle.com\r\nExpect: 100-continue\r\nContent-Length: 6\r\nContent-Type: a/b\r\n\r\n123456")
+		sendRequest(rw, StatusExpectationFailed, "")
+	}
+}
+
 func TestCompressHandler(t *testing.T) {
 	t.Parallel()
 
diff --git a/uri_test.go b/uri_test.go
index 9c4a67b..d41c067 100644
--- a/uri_test.go
+++ b/uri_test.go
@@ -354,6 +354,9 @@ func TestURIParse(t *testing.T) {
 
 	testURIParse(t, &u, "", "//aaa.com//absolute",
 		"http://aaa.com/absolute", "aaa.com", "/absolute", "//absolute", "", "")
+
+	testURIParse(t, &u, "", "//aaa.com\r\n\r\nGET x",
+		"http:///", "", "/", "", "", "")
 }
 
 func testURIParse(t *testing.T, u *URI, host, uri,