Repository: fortio/fortio (go)

Issue:
Fortio to automatically generate UUID per request and replace it in URL's path.
We use fortio for load testing but it requires a single URL to be used for all requests. In one of the scenarios, we would like part of the URL to be a UUID, this way we can hit the API directly and avoid adding one more HTTP layer to handle the request to fortio and rewrite the URL.

Instead of URL being set in Fortio to  a proxy ("http://localhost:3000/test, for example)" and that proxy forward each request to "https://targetapp/something/{UUID}", Fortio would understand "https://targetapp/something/{UUID}" directly and automatically replace {UUID} with a UUID per request.

I searched the docs and could not find a reference to such feature. Sorry if it already exists and I could not find.

Pull request description:
Adding support for uuid in url.
Closes #425 

Patch:
diff --git a/fhttp/http_client.go b/fhttp/http_client.go
index 8f80ccf..afacd00 100644
--- a/fhttp/http_client.go
+++ b/fhttp/http_client.go
@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math/rand"
 	"net"
 	"net/http"
 	"net/http/httputil"
@@ -34,6 +35,7 @@ import (
 	"fortio.org/fortio/fnet"
 	"fortio.org/fortio/log"
 	"fortio.org/fortio/version"
+	"github.com/google/uuid"
 )
 
 // Fetcher is the Url content fetcher that the different client implements.
@@ -46,6 +48,10 @@ type Fetcher interface {
 	Close() int
 }
 
+const (
+	uuidToken = "{uuid}"
+)
+
 var (
 	// BufferSizeKb size of the buffer (max data) for optimized client in kilobytes defaults to 128k.
 	BufferSizeKb = 128
@@ -55,6 +61,8 @@ var (
 	contentLengthHeader   = []byte("\r\ncontent-length:")
 	connectionCloseHeader = []byte("\r\nconnection: close")
 	chunkedHeader         = []byte("\r\nTransfer-Encoding: chunked")
+	//nolint // G404 is expected because we are using weak random number generator due to performance.
+	rander = NewSyncReader(rand.New(rand.NewSource(time.Now().UnixNano())))
 )
 
 // NewHTTPOptions creates and initialize a HTTPOptions object.
@@ -295,10 +303,16 @@ func newHTTPRequest(o *HTTPOptions) (*http.Request, error) {
 // Client object for making repeated requests of the same URL using the same
 // http client (net/http).
 type Client struct {
-	url       string
-	req       *http.Request
-	client    *http.Client
-	transport *http.Transport
+	url                  string
+	path                 string // original path of the request's url
+	rawQuery             string // original query params
+	body                 string // original body of the request
+	req                  *http.Request
+	client               *http.Client
+	transport            *http.Transport
+	pathContainsUUID     bool // if url contains the "{uuid}" pattern (lowercase)
+	rawQueryContainsUUID bool // if any query params contains the "{uuid}" pattern (lowercase)
+	bodyContainsUUID     bool // if body contains the "{uuid}" pattern (lowercase)
 }
 
 // Close cleans up any resources used by NewStdClient.
@@ -328,6 +342,30 @@ func (c *Client) ChangeURL(urlStr string) (err error) {
 // Fetch fetches the byte and code for pre created client.
 func (c *Client) Fetch() (int, []byte, int) {
 	// req can't be null (client itself would be null in that case)
+	if c.pathContainsUUID {
+		path := c.path
+		for strings.Contains(path, uuidToken) {
+			path = strings.Replace(path, uuidToken, generateUUID(), 1)
+		}
+		c.req.URL.Path = path
+	}
+	if c.rawQueryContainsUUID {
+		rawQuery := c.rawQuery
+		for strings.Contains(rawQuery, uuidToken) {
+			rawQuery = strings.Replace(rawQuery, uuidToken, generateUUID(), 1)
+		}
+
+		c.req.URL.RawQuery = rawQuery
+	}
+	if c.bodyContainsUUID {
+		body := c.body
+		for strings.Contains(body, uuidToken) {
+			body = strings.Replace(body, uuidToken, generateUUID(), 1)
+		}
+		bodyBytes := []byte(body)
+		c.req.ContentLength = int64(len(bodyBytes))
+		c.req.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes))
+	}
 	resp, err := c.client.Do(c.req)
 	if err != nil {
 		log.Errf("Unable to send %s request for %s : %v", c.req.Method, c.url, err)
@@ -408,9 +446,16 @@ func NewStdClient(o *HTTPOptions) (*Client, error) {
 			}
 		}
 	}
+
 	client := Client{
-		url: o.URL,
-		req: req,
+		url:                  o.URL,
+		path:                 req.URL.Path,
+		pathContainsUUID:     strings.Contains(req.URL.Path, uuidToken),
+		rawQuery:             req.URL.RawQuery,
+		rawQueryContainsUUID: strings.Contains(req.URL.RawQuery, uuidToken),
+		body:                 o.PayloadString(),
+		bodyContainsUUID:     strings.Contains(o.PayloadString(), uuidToken),
+		req:                  req,
 		client: &http.Client{
 			Timeout:   o.HTTPReqTimeOut,
 			Transport: &tr,
@@ -466,6 +511,7 @@ type FastClient struct {
 	parseHeaders bool // don't bother in http/1.0
 	halfClose    bool // allow/do half close when keepAlive is false
 	reqTimeout   time.Duration
+	uuidMarkers  [][]byte
 }
 
 // Close cleans up any resources used by FastClient.
@@ -491,10 +537,26 @@ func NewFastClient(o *HTTPOptions) (Fetcher, error) {
 	if o.HTTP10 {
 		proto = "1.0"
 	}
+
+	uuidStrings := []string{}
+	urlString := o.URL
+	for strings.Contains(urlString, uuidToken) {
+		uuidString := generateUUID()
+		uuidStrings = append(uuidStrings, uuidString)
+		urlString = strings.Replace(urlString, uuidToken, uuidString, 1)
+	}
+	payload := o.PayloadString()
+	for strings.Contains(payload, uuidToken) {
+		uuidString := generateUUID()
+		uuidStrings = append(uuidStrings, uuidString)
+		payload = strings.Replace(payload, uuidToken, uuidString, 1)
+	}
+	o.Payload = []byte(payload)
+
 	// Parse the url, extract components.
-	url, err := url.Parse(o.URL)
+	url, err := url.Parse(urlString)
 	if err != nil {
-		log.Errf("Bad url '%s' : %v", o.URL, err)
+		log.Errf("Bad url '%s' : %v", urlString, err)
 		return nil, err
 	}
 	if url.Scheme != "http" {
@@ -562,6 +624,12 @@ func NewFastClient(o *HTTPOptions) (Fetcher, error) {
 		buf.Write(o.Payload)
 	}
 	bc.req = buf.Bytes()
+	bc.uuidMarkers = [][]byte{}
+	if len(uuidStrings) > 0 {
+		for _, uuidString := range uuidStrings {
+			bc.uuidMarkers = append(bc.uuidMarkers, []byte(uuidString))
+		}
+	}
 	log.Debugf("Created client:\n%+v\n%s", bc.dest, bc.req)
 	return &bc, nil
 }
@@ -610,7 +678,13 @@ func (c *FastClient) Fetch() (int, []byte, int) {
 	c.socket = nil // because of error returns and single retry
 	conErr := conn.SetReadDeadline(time.Now().Add(c.reqTimeout))
 	// Send the request:
-	n, err := conn.Write(c.req)
+	req := c.req
+	if len(c.uuidMarkers) > 0 {
+		for _, uuidMarker := range c.uuidMarkers {
+			req = bytes.Replace(req, uuidMarker, []byte(generateUUID()), 1)
+		}
+	}
+	n, err := conn.Write(req)
 	if err != nil || conErr != nil {
 		if reuse {
 			// it's ok for the (idle) socket to die once, auto reconnect:
@@ -841,3 +915,8 @@ func (c *FastClient) readResponse(conn net.Conn, reusedSocket bool) {
 		// we cleared c.socket in caller already
 	}
 }
+
+func generateUUID() string {
+	// We use math random instead of crypto random generator due to performance.
+	return uuid.Must(uuid.NewRandomFromReader(rander)).String()
+}
diff --git a/fhttp/http_utils.go b/fhttp/http_utils.go
index 3bb2c22..23e0d75 100644
--- a/fhttp/http_utils.go
+++ b/fhttp/http_utils.go
@@ -23,6 +23,7 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 	"unicode/utf8"
 
@@ -456,3 +457,23 @@ func AddHTTPS(url string) string {
 func generateBase64UserCredentials(userCredentials string) string {
 	return "Basic " + base64.StdEncoding.EncodeToString([]byte(userCredentials))
 }
+
+// SyncReader is a thread-safe wrapper for a reader.
+type SyncReader struct {
+	lk     sync.Mutex
+	reader io.Reader
+}
+
+// NewSyncReader returns a new thread-safe reader.
+func NewSyncReader(reader io.Reader) *SyncReader {
+	return &SyncReader{
+		reader: reader,
+	}
+}
+
+func (r *SyncReader) Read(p []byte) (n int, err error) {
+	r.lk.Lock()
+	defer r.lk.Unlock()
+
+	return r.reader.Read(p)
+}
diff --git a/go.mod b/go.mod
index f34a783..4cbc355 100644
--- a/go.mod
+++ b/go.mod
@@ -7,6 +7,7 @@ require (
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/golang/protobuf v1.4.2
 	github.com/google/go-cmp v0.5.0 // indirect
+	github.com/google/uuid v1.1.2
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/stretchr/testify v1.4.0
 	golang.org/x/net v0.0.0-20200602114024-627f9648deb9
diff --git a/go.sum b/go.sum
index 068970a..f2b9920 100644
--- a/go.sum
+++ b/go.sum
@@ -32,6 +32,8 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=


Test patch:
diff --git a/fhttp/http_test.go b/fhttp/http_test.go
index 5c51097..6f1aa9d 100644
--- a/fhttp/http_test.go
+++ b/fhttp/http_test.go
@@ -16,6 +16,7 @@ package fhttp
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net"
@@ -29,10 +30,14 @@ import (
 
 	"fortio.org/fortio/fnet"
 	"fortio.org/fortio/log"
+	"github.com/google/uuid"
 )
 
+var uuids map[string]bool
+
 func init() {
 	log.SetLogLevel(log.Debug)
+	uuids = map[string]bool{}
 }
 
 func TestGetHeaders(t *testing.T) {
@@ -841,6 +846,178 @@ func TestPayloadSizeLarge(t *testing.T) {
 	}
 }
 
+func TestUUIDFastClient(t *testing.T) {
+	m, a := DynamicHTTPServer(false)
+	m.HandleFunc("/", ValidateUUIDPath)
+	url := fmt.Sprintf("http://localhost:%d/{uuid}", a.Port)
+	o := HTTPOptions{URL: url, DisableFastClient: false}
+	client, _ := NewClient(&o)
+	code, data, header := client.Fetch()
+	t.Logf("TestPayloadSize result code %d, data len %d, headerlen %d", code, len(data), header)
+	if code != 200 {
+		t.Errorf("Got %d instead of 200", code)
+	}
+}
+
+func TestUUIDPayloadFastClient(t *testing.T) {
+	m, a := DynamicHTTPServer(false)
+	m.HandleFunc("/", ValidateManyUUID)
+	url := fmt.Sprintf("http://localhost:%d/", a.Port)
+	o := HTTPOptions{
+		URL:               url,
+		DisableFastClient: false,
+		Payload:           []byte("[\"{uuid}\", \"{uuid}\"]"),
+	}
+	client, _ := NewClient(&o)
+	code, data, header := client.Fetch()
+	t.Logf("TestPayloadSize result code %d, data len %d, headerlen %d", code, len(data), header)
+	if code != 200 {
+		t.Errorf("Got %d instead of 200", code)
+	}
+}
+
+func TestQueryUUIDFastClient(t *testing.T) {
+	m, a := DynamicHTTPServer(false)
+	m.HandleFunc("/", ValidateUUIDQueryParam)
+	url := fmt.Sprintf("http://localhost:%d/?uuid={uuid}", a.Port)
+	o := HTTPOptions{URL: url, DisableFastClient: false}
+	client, _ := NewClient(&o)
+	code, data, header := client.Fetch()
+	t.Logf("TestPayloadSize result code %d, data len %d, headerlen %d", code, len(data), header)
+	if code != 200 {
+		t.Errorf("Got %d instead of 200", code)
+	}
+}
+
+func TestManyUUIDsFastClient(t *testing.T) {
+	m, a := DynamicHTTPServer(false)
+	m.HandleFunc("/", ValidateManyUUID)
+	url := fmt.Sprintf("http://localhost:%d/{uuid}/{uuid}?uuid={uuid}&uuid2={uuid}", a.Port)
+	for i := 0; i < 10; i++ {
+		o := HTTPOptions{URL: url, DisableFastClient: false}
+		client, _ := NewClient(&o)
+		for j := 0; j < 3; j++ {
+			code, data, header := client.Fetch()
+			t.Logf("TestPayloadSize result code %d, data len %d, headerlen %d", code, len(data), header)
+			if code != 200 {
+				t.Errorf("Got %d instead of 200", code)
+			}
+		}
+	}
+}
+
+func TestBadUUIDFastClient(t *testing.T) {
+	m, a := DynamicHTTPServer(false)
+	m.HandleFunc("/", ValidateUUIDPath)
+	url := fmt.Sprintf("http://localhost:%d/{not_uuid}", a.Port)
+	o := HTTPOptions{URL: url, DisableFastClient: false}
+	client, _ := NewClient(&o)
+	code, data, header := client.Fetch()
+	t.Logf("TestPayloadSize result code %d, data len %d, headerlen %d", code, len(data), header)
+	if code != 400 {
+		t.Errorf("Got %d instead of 400", code)
+	}
+}
+
+func TestBadQueryUUIDFastClient(t *testing.T) {
+	m, a := DynamicHTTPServer(false)
+	m.HandleFunc("/", ValidateUUIDQueryParam)
+	url := fmt.Sprintf("http://localhost:%d/?uuid={not_uuid}", a.Port)
+	o := HTTPOptions{URL: url, DisableFastClient: false}
+	client, _ := NewClient(&o)
+	code, data, header := client.Fetch()
+	t.Logf("TestPayloadSize result code %d, data len %d, headerlen %d", code, len(data), header)
+	if code != 400 {
+		t.Errorf("Got %d instead of 400", code)
+	}
+}
+
+func TestUUIDClient(t *testing.T) {
+	m, a := DynamicHTTPServer(false)
+	m.HandleFunc("/", ValidateUUIDPath)
+	url := fmt.Sprintf("http://localhost:%d/{uuid}", a.Port)
+	o := HTTPOptions{URL: url, DisableFastClient: true}
+	client, _ := NewClient(&o)
+	code, data, header := client.Fetch()
+	t.Logf("TestPayloadSize result code %d, data len %d, headerlen %d", code, len(data), header)
+	if code != 200 {
+		t.Errorf("Got %d instead of 200", code)
+	}
+}
+
+func TestUUIDPayloadClient(t *testing.T) {
+	m, a := DynamicHTTPServer(false)
+	m.HandleFunc("/", ValidateManyUUID)
+	url := fmt.Sprintf("http://localhost:%d/", a.Port)
+	o := HTTPOptions{
+		URL:               url,
+		DisableFastClient: true,
+		Payload:           []byte("[\"{uuid}\", \"{uuid}\"]"),
+	}
+	client, _ := NewClient(&o)
+	code, data, header := client.Fetch()
+	t.Logf("TestPayloadSize result code %d, data len %d, headerlen %d", code, len(data), header)
+	if code != 200 {
+		t.Errorf("Got %d instead of 200", code)
+	}
+}
+
+func TestQueryUUIDClient(t *testing.T) {
+	m, a := DynamicHTTPServer(false)
+	m.HandleFunc("/", ValidateUUIDQueryParam)
+	url := fmt.Sprintf("http://localhost:%d/?uuid={uuid}", a.Port)
+	o := HTTPOptions{URL: url, DisableFastClient: true}
+	client, _ := NewClient(&o)
+	code, data, header := client.Fetch()
+	t.Logf("TestPayloadSize result code %d, data len %d, headerlen %d", code, len(data), header)
+	if code != 200 {
+		t.Errorf("Got %d instead of 200", code)
+	}
+}
+
+func TestManyUUIDsClient(t *testing.T) {
+	m, a := DynamicHTTPServer(false)
+	m.HandleFunc("/", ValidateManyUUID)
+	url := fmt.Sprintf("http://localhost:%d/{uuid}/{uuid}?uuid={uuid}&uuid2={uuid}", a.Port)
+	for i := 0; i < 10; i++ {
+		o := HTTPOptions{URL: url, DisableFastClient: true}
+		client, _ := NewClient(&o)
+		for j := 0; j < 3; j++ {
+			code, data, header := client.Fetch()
+			t.Logf("TestPayloadSize result code %d, data len %d, headerlen %d", code, len(data), header)
+			if code != 200 {
+				t.Errorf("Got %d instead of 200", code)
+			}
+		}
+	}
+}
+
+func TestBadUUIDClient(t *testing.T) {
+	m, a := DynamicHTTPServer(false)
+	m.HandleFunc("/", ValidateUUIDPath)
+	url := fmt.Sprintf("http://localhost:%d/{not_uuid}", a.Port)
+	o := HTTPOptions{URL: url, DisableFastClient: true}
+	client, _ := NewClient(&o)
+	code, data, header := client.Fetch()
+	t.Logf("TestPayloadSize result code %d, data len %d, headerlen %d", code, len(data), header)
+	if code != 400 {
+		t.Errorf("Got %d instead of 400", code)
+	}
+}
+
+func TestBadQueryUUIDClient(t *testing.T) {
+	m, a := DynamicHTTPServer(false)
+	m.HandleFunc("/", ValidateUUIDQueryParam)
+	url := fmt.Sprintf("http://localhost:%d/?uuid={not_uuid}", a.Port)
+	o := HTTPOptions{URL: url, DisableFastClient: true}
+	client, _ := NewClient(&o)
+	code, data, header := client.Fetch()
+	t.Logf("TestPayloadSize result code %d, data len %d, headerlen %d", code, len(data), header)
+	if code != 400 {
+		t.Errorf("Got %d instead of 400", code)
+	}
+}
+
 func TestDebugHandlerSortedHeaders(t *testing.T) {
 	m, a := DynamicHTTPServer(false)
 	m.HandleFunc("/debug", DebugHandler)
@@ -1190,6 +1367,124 @@ func TestInsecureRequestWithResolve(t *testing.T) {
 	}
 }
 
+// ValidateUUIDPath is an http server handler validating /{uuid}.
+func ValidateUUIDPath(w http.ResponseWriter, r *http.Request) {
+	if log.LogVerbose() {
+		LogRequest(r, "ValidateUUIDPath")
+	}
+
+	uuidParam := strings.TrimPrefix(r.URL.Path, "/")
+	_, err := uuid.Parse(uuidParam)
+	if err != nil {
+		log.Errf("Error parsing uuid %v: %v", uuidParam, err)
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	w.WriteHeader(http.StatusOK)
+}
+
+// ValidateUUIDQueryparam is an http server handler validating /?uuid={uuid}.
+func ValidateUUIDQueryParam(w http.ResponseWriter, r *http.Request) {
+	if log.LogVerbose() {
+		LogRequest(r, "ValidateUUIDQueryParam")
+	}
+
+	uuidParam := r.URL.Query().Get("uuid")
+	_, err := uuid.Parse(uuidParam)
+	if err != nil {
+		log.Errf("Error parsing uuid %v: %v", uuidParam, err)
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	w.WriteHeader(http.StatusOK)
+}
+
+// ValidateManyUUID is an http server handler validating /{uuid}?uuid={uuid},
+//   including payload in JSON following the format: ["{uuid}","{uuid}"]
+func ValidateManyUUID(w http.ResponseWriter, r *http.Request) {
+	if log.LogVerbose() {
+		LogRequest(r, "ValidateManyUUID")
+	}
+
+	uuidParams := strings.Split(r.URL.Path, "/")
+	for _, uuidParam := range uuidParams {
+		if uuidParam == "" {
+			continue
+		}
+		_, err := uuid.Parse(uuidParam)
+		if err != nil {
+			log.Errf("Error parsing uuid %v: %v", uuidParam, err)
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+	}
+
+	uuidParam1 := r.URL.Query().Get("uuid")
+	if uuidParam1 != "" {
+		_, err := uuid.Parse(uuidParam1)
+		if err != nil {
+			log.Errf("Error parsing uuid %v: %v", uuidParam1, err)
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		uuidParams = append(uuidParams, uuidParam1)
+	}
+
+	uuidParam2 := r.URL.Query().Get("uuid2")
+	if uuidParam2 != "" {
+		_, err := uuid.Parse(uuidParam2)
+		if err != nil {
+			log.Errf("Error parsing uuid %v: %v", uuidParam2, err)
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		uuidParams = append(uuidParams, uuidParam2)
+	}
+
+	uuidPayloadList := []string{}
+	dec := json.NewDecoder(r.Body)
+	err := dec.Decode(&uuidPayloadList)
+	if err == nil {
+		for _, uuidPayload := range uuidPayloadList {
+			if uuidPayload == "" {
+				continue
+			}
+			_, err := uuid.Parse(uuidPayload)
+			if err != nil {
+				log.Errf("Error parsing uuid from payload %v: %v", uuidPayload, err)
+				w.WriteHeader(http.StatusBadRequest)
+				return
+			}
+
+			uuidParams = append(uuidParams, uuidPayload)
+		}
+	}
+
+	for _, uuid := range uuidParams {
+		if uuid == "" {
+			continue
+		}
+		if dedupUUID(uuid) != nil {
+			log.Errf("duplicate uuid: %v", uuid)
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+	}
+
+	w.WriteHeader(http.StatusOK)
+}
+
+func dedupUUID(uuid string) error {
+	if uuids[uuid] {
+		return fmt.Errorf("duplicate uuid: %v", uuid)
+	}
+
+	uuids[uuid] = true
+	return nil
+}
+
 // --- for bench mark/comparison
 
 func asciiFold0(str string) []byte {