Repository: fortio/fortio (go)

Issue:
Add a more direct way to switch HTTP method (POST, PUT,...)
Right now it's implicit from having a payload or `-content-type` (which isn't in the UI - and we should treat at least -H Content-Type like -content-type) and can only be a POST or GET


Pull request description:
add -X to override method. treat content-type in header as ContentType on options
- [x] New -X (same as curl's name) works in manual tests
- [x] Need a test for -X
- [x] Content-Type in headers (`-H`) should work like ContentType in http options (`-content-type`) to also change the method from GET to POST
- [x] added cli test using fortio.org/testscript
- [x] add "X" form to UI and rapi

Fixes #780 

Patch:
diff --git a/README.md b/README.md
index 7ab0fe3..fd9c579 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-<!-- 1.55.2 -->
+<!-- 1.56.0 -->
 # Fortio
 
 [![Awesome Go](https://fortio.org/mentioned-badge.svg)](https://github.com/avelino/awesome-go#networking)
@@ -60,13 +60,13 @@ You can install from source:
 The [releases](https://github.com/fortio/fortio/releases) page has binaries for many OS/architecture combinations (see assets):
 
 ```shell
-curl -L https://github.com/fortio/fortio/releases/download/v1.55.2/fortio-linux_amd64-1.55.2.tgz \
+curl -L https://github.com/fortio/fortio/releases/download/v1.56.0/fortio-linux_amd64-1.56.0.tgz \
  | sudo tar -C / -xvzpf -
 # or the debian package
-wget https://github.com/fortio/fortio/releases/download/v1.55.2/fortio_1.55.2_amd64.deb
-dpkg -i fortio_1.55.2_amd64.deb
+wget https://github.com/fortio/fortio/releases/download/v1.56.0/fortio_1.56.0_amd64.deb
+dpkg -i fortio_1.56.0_amd64.deb
 # or the rpm
-rpm -i https://github.com/fortio/fortio/releases/download/v1.55.2/fortio-1.55.2-1.x86_64.rpm
+rpm -i https://github.com/fortio/fortio/releases/download/v1.56.0/fortio-1.56.0-1.x86_64.rpm
 # and more, see assets in release page
 ```
 
@@ -76,7 +76,7 @@ On a MacOS you can also install Fortio using [Homebrew](https://brew.sh/):
 brew install fortio
 ```
 
-On Windows, download https://github.com/fortio/fortio/releases/download/v1.55.2/fortio_win_1.55.2.zip and extract `fortio.exe` to any location, then using the Windows Command Prompt:
+On Windows, download https://github.com/fortio/fortio/releases/download/v1.56.0/fortio_win_1.56.0.zip and extract `fortio.exe` to any location, then using the Windows Command Prompt:
 ```
 fortio.exe server
 ```
@@ -127,7 +127,7 @@ Full list of command line flags (`fortio help`):
 <!-- use release/updateFlags.sh to update this section -->
 <pre>
 <!-- USAGE_START -->
-Φορτίο 1.55.2 usage:
+Φορτίο 1.56.0 usage:
         fortio command [flags] target
 where command is one of: load (load testing), server (starts ui, rest api,
  http-echo, redirect, proxies, tcp-echo, udp-echo and grpc ping servers),
@@ -150,6 +150,8 @@ passed using multiple -H.
   -P value
         Tcp proxies to run, e.g -P "localport1 dest_host1:dest_port1" -P "[::1]:0
 www.google.com:443" ...
+  -X string
+        HTTP method to use instead of GET/POST depending on payload/content-type
   -a    Automatically save JSON result with filename based on labels & timestamp
   -abort-on code
         Http code that if encountered aborts the run. e.g. 503 or -1 for socket errors.
@@ -248,6 +250,9 @@ unless -a is used)
 URL and hostname
   -log-errors
         Log http non 2xx/418 error codes as they occur (default true)
+  -logger-file-line
+        Filename and line numbers emitted in JSON logs, use -logger-file-line=false to
+disable (default true)
   -logger-json
         Log in JSON format, use -logger-json=false to disable (default true)
   -logger-timestamp
diff --git a/bincommon/commonflags.go b/bincommon/commonflags.go
index a01100f..f010f2b 100644
--- a/bincommon/commonflags.go
+++ b/bincommon/commonflags.go
@@ -106,6 +106,7 @@ var (
 	// NoReResolveFlag is false if we want to resolve the DNS name for each new connection.
 	NoReResolveFlag = flag.Bool("no-reresolve", false, "Keep the initial DNS resolution and "+
 		"don't re-resolve when making new connections (because of error or reuse limit reached)")
+	MethodFlag = flag.String("X", "", "HTTP method to use instead of GET/POST depending on payload/content-type")
 )
 
 // SharedMain is the common part of main from fortio_main and fcurl.
@@ -207,7 +208,10 @@ func SharedHTTPOptions() *fhttp.HTTPOptions {
 	httpOpts.Insecure = TLSInsecure()
 	httpOpts.Resolve = *resolve
 	httpOpts.UserCredentials = *userCredentialsFlag
-	httpOpts.ContentType = *contentTypeFlag
+	if len(*contentTypeFlag) > 0 {
+		// only set content-type from flag if flag isn't empty as it can come also from -H content-type:...
+		httpOpts.ContentType = *contentTypeFlag
+	}
 	if *PayloadStreamFlag {
 		httpOpts.PayloadReader = os.Stdin
 	} else {
@@ -230,5 +234,6 @@ func SharedHTTPOptions() *fhttp.HTTPOptions {
 	httpOpts.LogErrors = *LogErrorsFlag
 	httpOpts.SequentialWarmup = *warmupFlag
 	httpOpts.NoResolveEachConn = *NoReResolveFlag
+	httpOpts.MethodOverride = *MethodFlag
 	return &httpOpts
 }
diff --git a/fcurl/fcurl.go b/fcurl/fcurl.go
index 6aeb630..e2a608a 100644
--- a/fcurl/fcurl.go
+++ b/fcurl/fcurl.go
@@ -17,16 +17,25 @@ package main
 // Do not add any external dependencies we want to keep fortio minimal.
 
 import (
+	"os"
+
 	"fortio.org/cli"
 	"fortio.org/fortio/bincommon"
+	"fortio.org/log"
 )
 
-func main() {
+func Main() int {
 	cli.ProgramName = "Φορτίο fortio-curl"
 	cli.ArgsHelp = "url"
 	cli.MinArgs = 1
 	bincommon.SharedMain()
 	cli.Main()
 	o := bincommon.SharedHTTPOptions()
+	log.Debugf("Running curl with %+v", o)
 	bincommon.FetchURL(o)
+	return 0
+}
+
+func main() {
+	os.Exit(Main())
 }
diff --git a/fhttp/http_client.go b/fhttp/http_client.go
index fc4fdfa..d289553 100644
--- a/fhttp/http_client.go
+++ b/fhttp/http_client.go
@@ -201,6 +201,7 @@ type HTTPOptions struct {
 	UserCredentials  string        // user credentials for authorization
 	ContentType      string        // indicates request body type, implies POST instead of GET
 	Payload          []byte        // body for http request, implies POST if not empty.
+	MethodOverride   string        // optional http method override. Otherwise GET or POST when a payload or ContentType is set.
 	LogErrors        bool          // whether to log non 2xx code as they occur or not
 	ID               int           `json:"-"` // thread/connect id to use for logging (thread id when used as a runner)
 	UniqueID         int64         `json:"-"` // Run identifier when used through a runner, copied from RunnerOptions.RunID
@@ -287,6 +288,9 @@ func (h *HTTPOptions) AllHeaders() http.Header {
 
 // Method returns the method of the http req.
 func (h *HTTPOptions) Method() string {
+	if len(h.MethodOverride) > 0 {
+		return h.MethodOverride
+	}
 	if len(h.Payload) > 0 || h.ContentType != "" {
 		return fnet.POST
 	}
@@ -308,18 +312,23 @@ func (h *HTTPOptions) AddAndValidateExtraHeader(hdr string) error {
 	// No TrimSpace for the value, so we can set empty "" vs just whitespace " " which
 	// will get trimmed later but treated differently: not emitted vs emitted empty for User-Agent.
 	value := s[1]
+	// 2 headers need trimmed to not have extra spaces:
+	trimmedValue := strings.TrimSpace(value)
 	switch strings.ToLower(key) {
 	case "host":
-		log.LogVf("Will be setting special Host header to %s", value)
-		h.hostOverride = strings.TrimSpace(value) // This one needs to be trimmed
+		log.LogVf("Will be setting special Host header to %s", trimmedValue)
+		h.hostOverride = trimmedValue // This one needs to be trimmed
 	case "user-agent":
 		if value == "" {
 			log.Infof("Deleting default User-Agent: header.")
 			h.extraHeaders.Del(key)
 		} else {
-			log.Infof("User-Agent being Set to %q", value)
+			log.Infof("User-Agent being set to %q", value)
 			h.extraHeaders.Set(key, value)
 		}
+	case "content-type":
+		log.LogVf("Content-Type being set to %q", trimmedValue)
+		h.ContentType = trimmedValue
 	default:
 		log.LogVf("Setting regular extra header %s: %s", key, value)
 		h.extraHeaders.Add(key, value)
@@ -362,6 +371,7 @@ func (h *HTTPOptions) ValidateAndSetConnectionReuseRange(inp string) error {
 // newHttpRequest makes a new http GET request for url with User-Agent.
 func newHTTPRequest(o *HTTPOptions) (*http.Request, error) {
 	method := o.Method()
+	log.Debugf("newHTTPRequest %s %s", method, o.URL)
 	var body io.Reader
 	if o.PayloadReader != nil {
 		body = o.PayloadReader
@@ -753,6 +763,7 @@ func (c *FastClient) Close() {
 // the beginning and then reused many times.
 func NewFastClient(o *HTTPOptions) (Fetcher, error) { //nolint:funlen
 	method := o.Method()
+	log.Debugf("NewFastClient %s %s", method, o.URL)
 	payloadLen := len(o.Payload)
 	o.Init(o.URL)
 	proto := "1.1"
diff --git a/go.mod b/go.mod
index c19b7d7..dafa3a3 100644
--- a/go.mod
+++ b/go.mod
@@ -7,7 +7,8 @@ require (
 	fortio.org/cli v1.1.0
 	fortio.org/dflag v1.5.2
 	fortio.org/log v1.5.0
-	fortio.org/scli v1.6.0
+	fortio.org/scli v1.7.0
+	fortio.org/testscript v0.3.1
 	fortio.org/version v1.0.2
 	github.com/golang/protobuf v1.5.3
 	github.com/google/uuid v1.3.0
@@ -31,6 +32,7 @@ require (
 	golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 // indirect
 	golang.org/x/sys v0.9.0 // indirect
 	golang.org/x/text v0.10.0 // indirect
+	golang.org/x/tools v0.8.0 // indirect
 	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 )
diff --git a/go.sum b/go.sum
index 7ec4fce..6290bb3 100644
--- a/go.sum
+++ b/go.sum
@@ -6,10 +6,12 @@ fortio.org/dflag v1.5.2 h1:F9XVRj4Qr2IbJP7BMj7XZc9wB0Q/RZ61Ool+4YPVad8=
 fortio.org/dflag v1.5.2/go.mod h1:ppb/A8u+KKg+qUUYZNYuvRnXuVb8IsdHb/XGzsmjkN8=
 fortio.org/log v1.5.0 h1:0f/O7QPXQoDSnRjy8t0IyxTlQOYQsDOe0EO4Wnw8yCA=
 fortio.org/log v1.5.0/go.mod h1:u/8/2lyczXq52aT5Nw6reD+3cR6m/EbS2jBiIYhgiTU=
-fortio.org/scli v1.6.0 h1:orn3xqUVLtgkD9LgYtAovVZtfzOzN0qCuItRTd5Z+d4=
-fortio.org/scli v1.6.0/go.mod h1:HV+Ucg3jV0n+i+mQt/MWlyLXINHmPwrQhney5j5oYW0=
+fortio.org/scli v1.7.0 h1:NC3z2k+2NY5zTB+XoQGX2MKJulMq61gltK8OrkR3U9U=
+fortio.org/scli v1.7.0/go.mod h1:HV+Ucg3jV0n+i+mQt/MWlyLXINHmPwrQhney5j5oYW0=
 fortio.org/sets v1.0.3 h1:HzewdGjH69YmyW06yzplL35lGr+X4OcqQt0qS6jbaO4=
 fortio.org/sets v1.0.3/go.mod h1:QZVj0r6KP/ZD9ebySW9SgxVNy/NjghUfyHW9NN+WU+4=
+fortio.org/testscript v0.3.1 h1:MmRO64AsmzaU1KlYMzAbotJIMKRGxD1XXssJnBRiMGQ=
+fortio.org/testscript v0.3.1/go.mod h1:7OJ+U4avooRNqc7p/VHKJadYgj9fA6+N0SbGU8FVWGs=
 fortio.org/version v1.0.2 h1:8NwxdX58aoeKx7T5xAPO0xlUu1Hpk42nRz5s6e6eKZ0=
 fortio.org/version v1.0.2/go.mod h1:2JQp9Ax+tm6QKiGuzR5nJY63kFeANcgrZ0osoQFDVm0=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
@@ -30,6 +32,8 @@ golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
 golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=
 golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y=
+golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A=
 google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU=
diff --git a/main.go b/main.go
index d5c5ce2..295a81d 100644
--- a/main.go
+++ b/main.go
@@ -22,3 +22,9 @@ import "fortio.org/fortio/cli"
 func main() {
 	cli.FortioMain(nil /* no hook needed */)
 }
+
+// Same as real above but for testscript/txtar tests.
+func Main() int {
+	cli.FortioMain(nil /* no hook needed */)
+	return 0
+}
diff --git a/rapi/restHandler.go b/rapi/restHandler.go
index c71bdc7..1d296d5 100644
--- a/rapi/restHandler.go
+++ b/rapi/restHandler.go
@@ -224,6 +224,7 @@ func RESTRunHandler(w http.ResponseWriter, r *http.Request) { //nolint:funlen
 	log.Infof("Starting API run %s load request from %v for %s", runner, r.RemoteAddr, url)
 	async := (FormValue(r, jd, "async") == "on")
 	payload := FormValue(r, jd, "payload")
+	methodOverride := FormValue(r, jd, "X")
 	labels := FormValue(r, jd, "labels")
 	resolution, _ := strconv.ParseFloat(FormValue(r, jd, "r"), 64)
 	percList, _ := stats.ParsePercentiles(FormValue(r, jd, "p"))
@@ -287,6 +288,7 @@ func RESTRunHandler(w http.ResponseWriter, r *http.Request) { //nolint:funlen
 	httpopts.Resolve = resolve
 	httpopts.H2 = h2
 	httpopts.LogErrors = logErrors
+	httpopts.MethodOverride = methodOverride
 	// Set the connection reuse range.
 	err = bincommon.ConnectionReuseRange.
 		WithValidator(bincommon.ConnectionReuseRangeValidator(httpopts)).
diff --git a/release/bumpRelease.sh b/release/bumpRelease.sh
index d209a1e..a20b4bb 100755
--- a/release/bumpRelease.sh
+++ b/release/bumpRelease.sh
@@ -16,8 +16,12 @@ if [ -z "$RELEASE" ]; then
     exit 1
 fi
 if [ "$CURRENT" = "$RELEASE" ]; then
-    echo "Release $RELEASE is already in $FILENAME"
-    exit 0
+    if [ "$2" = "-f" ]; then
+        echo "Forcing regen of same $CURRENT"
+    else
+        echo "Current release $CURRENT is the same as the requested, nothing to do, add -f to force"
+        exit 0
+    fi
 fi
 echo "Changing $FILENAME from $CURRENT to release $RELEASE and updating usage section"
 ./release/updateFlags.sh | sed -e "s/Φορτίο dev/Φορτίο $RELEASE/" > /tmp/fortio_flags.txt
diff --git a/ui/templates/main.html b/ui/templates/main.html
index a8daf7d..8136a1c 100644
--- a/ui/templates/main.html
+++ b/ui/templates/main.html
@@ -54,6 +54,7 @@
     No Catch-Up (qps is a ceiling): <input type="checkbox" name="nocatchup" checked /><br />
     Percentiles: <input type="text" name="p" size="20" value="50, 75, 90, 99, 99.9" /> <br />
     Histogram Resolution: <input type="text" name="r" size="8" value="0.0001" /> <br />
+    Method override (empty for GET/POST based on payload): <input type="text" name="X" size="8" value="" /> <br />
     Extra Headers:<br />
     <input type="text" name="H" size=40 value="" /> <br />
     <button type="button" onclick="addCustomHeader()">+</button>
diff --git a/ui/uihandler.go b/ui/uihandler.go
index 385d0fa..4b44b99 100644
--- a/ui/uihandler.go
+++ b/ui/uihandler.go
@@ -122,6 +122,7 @@ func Handler(w http.ResponseWriter, r *http.Request) {
 	}
 	// Those only exist/make sense on run mode but go variable declaration...
 	payload := r.FormValue("payload")
+	methodOverride := r.FormValue("X")
 	labels := r.FormValue("labels")
 	resolution, _ := strconv.ParseFloat(r.FormValue("r"), 64)
 	percList, _ := stats.ParsePercentiles(r.FormValue("p"))
@@ -197,6 +198,7 @@ func Handler(w http.ResponseWriter, r *http.Request) {
 	httpopts.Resolve = resolve
 	httpopts.H2 = h2
 	httpopts.LogErrors = logErrors
+	httpopts.MethodOverride = methodOverride
 	// Set the connection reuse range.
 	err := bincommon.ConnectionReuseRange.
 		WithValidator(bincommon.ConnectionReuseRangeValidator(httpopts)).


Test patch:
diff --git a/cli_test.go b/cli_test.go
new file mode 100644
index 0000000..9c35cf2
--- /dev/null
+++ b/cli_test.go
@@ -0,0 +1,18 @@
+package main
+
+import (
+	"os"
+	"testing"
+
+	"fortio.org/testscript"
+)
+
+func TestMain(m *testing.M) {
+	os.Exit(testscript.RunMain(m, map[string]func() int{
+		"fortio": Main,
+	}))
+}
+
+func TestDNSPing(t *testing.T) {
+	testscript.Run(t, testscript.Params{Dir: "./"})
+}
diff --git a/cli_test.txtar b/cli_test.txtar
new file mode 100644
index 0000000..1ce1554
--- /dev/null
+++ b/cli_test.txtar
@@ -0,0 +1,25 @@
+# testscript framework tests for fortio's main binary / command line
+# Eventually we can convert most of Webtest.sh to here (except parts specifically about testing the Docker image)
+
+# Basic usage test
+!fortio
+!stdout .
+stderr 'Missing command argument'
+
+# (short) version
+fortio version
+stdout '^dev$'
+!stderr .
+
+# (long) version
+fortio buildinfo
+stdout '^dev  go'
+stdout 'path	fortio.org/fortio'
+!stderr .
+
+# Content-Type override through headers
+fortio curl -H 'content-TYPE: foo/bar' -H 'xyz: bar blah' https://debug.fortio.org/test-path
+stderr 'HTTP/1.1 200 OK'
+stdout 'Xyz: bar blah'
+stdout 'Content-Type: foo/bar'
+stdout 'POST /test-path HTTP/1.1'
diff --git a/fhttp/http_test.go b/fhttp/http_test.go
index 7ed69ad..2451bff 100644
--- a/fhttp/http_test.go
+++ b/fhttp/http_test.go
@@ -863,16 +863,32 @@ func TestPayloadForClient(t *testing.T) {
 			nil,
 			"GET",
 		},
+		{
+			"",
+			[]byte{},
+			"GET",
+		},
+		{
+			"Foo", // once we set a content-type we get a POST instead of GET (alternative to -X)
+			nil,
+			"POST",
+		},
 	}
 	for _, test := range tests {
 		hOptions := HTTPOptions{}
 		hOptions.URL = "www.google.com"
-		hOptions.ContentType = test.contentType
+		err := hOptions.AddAndValidateExtraHeader("conTENT-tYPE: " + test.contentType)
+		if err != nil {
+			t.Errorf("Got error %v adding header", err)
+		}
+		if hOptions.ContentType != test.contentType {
+			t.Errorf("Got %q, expected %q as a content type from header addition", hOptions.ContentType, test.contentType)
+		}
 		hOptions.Payload = test.payload
 		client, _ := NewStdClient(&hOptions)
 		contentType := client.req.Header.Get("Content-Type")
 		if contentType != test.contentType {
-			t.Errorf("Got %s, expected %s as a content type", contentType, test.contentType)
+			t.Errorf("Got %q, expected %q as a content type", contentType, test.contentType)
 		}
 		method := client.req.Method
 		if method != test.expectedMethod {
@@ -894,6 +910,47 @@ func TestPayloadForClient(t *testing.T) {
 	}
 }
 
+func TestMethodOverride(t *testing.T) {
+	_, addr := ServeTCP("0", "/debug")
+	o := HTTPOptions{URL: fmt.Sprintf("http://localhost:%d/debug", addr.Port), MethodOverride: "FOO"}
+	for _, fastClient := range []bool{true, false} {
+		o.DisableFastClient = !fastClient
+		cli, _ := NewClient(&o)
+		ctx := context.Background()
+		code, data, _ := cli.Fetch(ctx)
+		if code != 200 {
+			t.Errorf("Non ok code %d for debug override method %q", code, o.MethodOverride)
+		}
+		expected := []byte("FOO /debug HTTP/1.1")
+		if !bytes.Contains(data, expected) {
+			t.Errorf("Method not echoed back in fastClient %v client %s (expecting %s)", fastClient, DebugSummary(data, 512), expected)
+		}
+	}
+}
+
+func TestPostFromContentType(t *testing.T) {
+	_, addr := ServeTCP("0", "/debug")
+	o := HTTPOptions{URL: fmt.Sprintf("http://localhost:%d/debug", addr.Port)}
+	o.AddAndValidateExtraHeader("content-type: foo/bar; xyz")
+	for _, fastClient := range []bool{true, false} {
+		o.DisableFastClient = !fastClient
+		cli, _ := NewClient(&o)
+		ctx := context.Background()
+		code, data, _ := cli.Fetch(ctx)
+		if code != 200 {
+			t.Errorf("Non ok code %d for debug override method %q", code, o.MethodOverride)
+		}
+		expected := []byte("POST /debug HTTP/1.1")
+		if !bytes.Contains(data, expected) {
+			t.Errorf("POST not found in fastClient %v client %s (expecting %s)", fastClient, DebugSummary(data, 512), expected)
+		}
+		expected = []byte("Content-Type: foo/bar; xyz")
+		if !bytes.Contains(data, expected) {
+			t.Errorf("Content-type not found in fastClient %v client %s (expecting %s)", fastClient, DebugSummary(data, 512), expected)
+		}
+	}
+}
+
 func TestPayloadForFastClient(t *testing.T) {
 	tests := []struct {
 		contentType     string