Repository: valyala/fasthttp
Pull request #672 (2019-10-10 19:31:41)

Allow client to disable path normalization
See the test for context - basically I have a URL like http://google.com/encoded/Y%2BY%2FY%3D/stuff, but when I use fasthttp, due to path normalization the actual request that gets made is to http://google.com/encoded/Y+Y/Y=/stuff.  On some servers (e.g. fasthttp's server), these URLs are semantically equivalent, but on some servers (e.g. those google urls) they are not. If you click through those URLs you can see the difference in the 404 page that google returns. https://stackoverflow.com/questions/1957115/is-a-slash-equivalent-to-an-encoded-slash-2f-in-the-path-portion-of-a has some more context on the server-side situation as well.

This wires https://github.com/valyala/fasthttp/pull/649 through the client layer to solve this issue.


Patch:
diff --git a/client.go b/client.go
index bff692a..4bde3f0 100644
--- a/client.go
+++ b/client.go
@@ -246,6 +246,15 @@ type Client struct {
 	//     * cONTENT-lenGTH -> Content-Length
 	DisableHeaderNamesNormalizing bool
 
+	// Path values are sent as-is without normalization
+	//
+	// Disabled path normalization may be useful for proxying incoming requests
+	// to servers that are expecting paths to be forwarded as-is.
+	//
+	// By default path values are normalized, i.e.
+	// extra slashes are removed, special characters are encoded.
+	DisablePathNormalizing bool
+
 	mLock sync.Mutex
 	m     map[string]*HostClient
 	ms    map[string]*HostClient
@@ -423,6 +432,7 @@ func (c *Client) Do(req *Request, resp *Response) error {
 			WriteTimeout:                  c.WriteTimeout,
 			MaxResponseBodySize:           c.MaxResponseBodySize,
 			DisableHeaderNamesNormalizing: c.DisableHeaderNamesNormalizing,
+			DisablePathNormalizing:        c.DisablePathNormalizing,
 		}
 		m[string(host)] = hc
 		if len(m) == 1 {
@@ -614,6 +624,15 @@ type HostClient struct {
 	//     * cONTENT-lenGTH -> Content-Length
 	DisableHeaderNamesNormalizing bool
 
+	// Path values are sent as-is without normalization
+	//
+	// Disabled path normalization may be useful for proxying incoming requests
+	// to servers that are expecting paths to be forwarded as-is.
+	//
+	// By default path values are normalized, i.e.
+	// extra slashes are removed, special characters are encoded.
+	DisablePathNormalizing bool
+
 	clientName  atomic.Value
 	lastUseTime uint32
 
@@ -1154,6 +1173,10 @@ func (c *HostClient) doNonNilReqResp(req *Request, resp *Response) (bool, error)
 	resp.Reset()
 	resp.SkipBody = customSkipBody
 
+	if c.DisablePathNormalizing {
+		req.URI().DisablePathNormalizing = true
+	}
+
 	// If we detected a redirect to another schema
 	if req.schemaUpdate {
 		c.IsTLS = bytes.Equal(req.URI().Scheme(), strHTTPS)


Test patch:
diff --git a/client_test.go b/client_test.go
index 579a123..2ffdd39 100644
--- a/client_test.go
+++ b/client_test.go
@@ -699,7 +699,7 @@ func testPipelineClientDo(t *testing.T, c *PipelineClient) {
 	ReleaseResponse(resp)
 }
 
-func TestClientDoTimeoutDisableNormalizing(t *testing.T) {
+func TestClientDoTimeoutDisableHeaderNamesNormalizing(t *testing.T) {
 	ln := fasthttputil.NewInmemoryListener()
 
 	s := &Server{
@@ -751,6 +751,57 @@ func TestClientDoTimeoutDisableNormalizing(t *testing.T) {
 	}
 }
 
+func TestClientDoTimeoutDisablePathNormalizing(t *testing.T) {
+	ln := fasthttputil.NewInmemoryListener()
+
+	s := &Server{
+		Handler: func(ctx *RequestCtx) {
+			uri := ctx.URI()
+			uri.DisablePathNormalizing = true
+			ctx.Response.Header.Set("received-uri", string(uri.FullURI()))
+		},
+	}
+
+	serverStopCh := make(chan struct{})
+	go func() {
+		if err := s.Serve(ln); err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		close(serverStopCh)
+	}()
+
+	c := &Client{
+		Dial: func(addr string) (net.Conn, error) {
+			return ln.Dial()
+		},
+		DisablePathNormalizing: true,
+	}
+
+	urlWithEncodedPath := "http://example.com/encoded/Y%2BY%2FY%3D/stuff"
+
+	var req Request
+	req.SetRequestURI(urlWithEncodedPath)
+	var resp Response
+	for i := 0; i < 5; i++ {
+		if err := c.DoTimeout(&req, &resp, time.Second); err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		hv := resp.Header.Peek("received-uri")
+		if string(hv) != urlWithEncodedPath {
+			t.Fatalf("request uri was normalized: %q. Expecting %q", hv, urlWithEncodedPath)
+		}
+	}
+
+	if err := ln.Close(); err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	select {
+	case <-serverStopCh:
+	case <-time.After(time.Second):
+		t.Fatalf("timeout")
+	}
+}
+
 func TestHostClientPendingRequests(t *testing.T) {
 	const concurrency = 10
 	doneCh := make(chan struct{})