Repository: istio/fortio
Pull request #210 (2018-03-20 05:29:55)

Bug fix with std client init, create a smaller executable with http client/curl only: fcurl
Moved common code between the 2 (http client flags) into bincommon/

TODO: make a binary release (extend the tgz one) in addition to the
fortio.fcurl docker image

compressed image sizes:
https://hub.docker.com/r/istio/fortio.fcurl/tags/ 2mb
https://hub.docker.com/r/istio/fortio/tags/ 3mb

uncompressed:
```
REPOSITORY             TAG                 IMAGE ID            CREATED             SIZE
istio/fortio.fcurl     0.8.1-pre1          8d76b0e6bb26        41 minutes ago      6.19MB
istio/fortio.echosrv   0.8.1-pre1          9587d7af4c79        42 minutes ago      5.88MB
istio/fortio           0.8.1-pre1          9707ad2fdc2f        42 minutes ago      8.6MB
```

not tremendously smaller but maybe convenient for health checks etc/alternative to curl (3mb compressed) for scratch images or for go programs

also has less cognitive pressures on the flags:
```
$ fcurl
Φορτίο fortio-curl 0.8.1-pre usage:
	fcurl [flags] url
flags are:
  -H value
    	Additional Header(s)
  -L	Follow redirects (implies -std-client) - do not use for load test
  -compression
    	Enable http compression
  -halfclose
    	When not keepalive, whether to half close the connection (only for fast http)
  -http1.0
    	Use http1.0 (instead of http 1.1)
  -httpbufferkb int
    	Size of the buffer (max data size) for the optimized http client in kbytes (default 128)
  -httpccch
    	Check for Connection: Close Header
  -k	Do not verify certs in https connections
  -keepalive
    	Keep connection alive (only for fast http 1.1) (default true)
  -logcaller
    	Logs filename and line number of callers to log (default true)
  -loglevel value
    	loglevel, one of [Debug Verbose Info Warning Error Critical Fatal] (default Info)
  -logprefix string
    	Prefix to log lines before logged messages (default "> ")
  -quiet
    	Quiet mode: sets the loglevel to Error and reduces the output.
  -stdclient
    	Use the slower net/http standard client (works for TLS)
  -timeout duration
    	Connection and read timeout value (for http) (default 15s)
Error: need a url as parameter
```

codecov[bot]: # [Codecov](https://codecov.io/gh/istio/fortio/pull/210?src=pr&el=h1) Report
> Merging [#210](https://codecov.io/gh/istio/fortio/pull/210?src=pr&el=desc) into [master](https://codecov.io/gh/istio/fortio/commit/5a3ec24bbeece35860d5625bc5ddab5c4a0b1f11?src=pr&el=desc) will **not change** coverage.
> The diff coverage is `n/a`.

[![Impacted file tree graph](https://codecov.io/gh/istio/fortio/pull/210/graphs/tree.svg?height=150&src=pr&token=6HLno2jpvU&width=650)](https://codecov.io/gh/istio/fortio/pull/210?src=pr&el=tree)

```diff
@@          Coverage Diff           @@
##           master    #210   +/-   ##
======================================
  Coverage    90.6%   90.6%           
======================================
  Files          10      10           
  Lines        1786    1786           
======================================
  Hits         1618    1618           
  Misses        109     109           
  Partials       59      59
```



------

[Continue to review full report at Codecov](https://codecov.io/gh/istio/fortio/pull/210?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/istio/fortio/pull/210?src=pr&el=footer). Last update [5a3ec24...257a838](https://codecov.io/gh/istio/fortio/pull/210?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).

ldemailly: example:
```
$ docker run istio/fortio.fcurl:0.8.1-pre1 -L https://dl.google.com/go/go1.10.src.tar.gz | shasum -a 256
f3de49289405fda5fd1483a8fe6bd2fa5469e005fd567df64485c4fa000c7f24  -
```
matches:
<img width="977" alt="screen shot 2018-03-19 at 11 05 03 pm" src="https://user-images.githubusercontent.com/3664595/37638414-016e6c94-2bca-11e8-9cfb-dfe1780f904c.png">

ldemailly: I found the bug because
https://github.com/istio/fortio/blob/master/fhttp/http_client.go#L200
errorred out (doing fcurl -loglevel debug -L localhost:8080 )
```
15:23:18 E http_client.go:200> Unable to dump request http: nil Request.Header
```


Patch:
diff --git a/Dockerfile b/Dockerfile
index f62bbcb..649cc25 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -6,19 +6,20 @@ COPY . fortio
 RUN make -C fortio submodule
 # Putting spaces in linker replaced variables is hard but does work.
 RUN echo "$(date +'%Y-%m-%d %H:%M') $(cd fortio; git rev-parse HEAD)" > /build-info.txt
+# Sets up the static directory outside of the go source tree and
+# the default data directory to a /var/lib/... volume
+# + rest of build time/git/version magic.
 RUN echo "-s -X istio.io/fortio/ui.resourcesDir=/usr/local/lib/fortio -X main.defaultDataDir=/var/lib/istio/fortio \
   -X \"istio.io/fortio/version.buildInfo=$(cat /build-info.txt)\" \
   -X istio.io/fortio/version.tag=$(cd fortio; git describe --tags) \
   -X istio.io/fortio/version.gitstatus=$(cd fortio; git status --porcelain | wc -l)" > /link-flags.txt
-# Sets up the static directory outside of the go source tree and
-# the default data directory to a /var/lib/... volume
 RUN go version
 RUN CGO_ENABLED=0 GOOS=linux go build -a -ldflags "$(cat /link-flags.txt)" -o fortio_go1.10.bin istio.io/fortio
 RUN ./fortio_go1.10.bin version
 # Check we still build with go 1.8 (and macos does not break)
 RUN /usr/local/go/bin/go version
 RUN CGO_ENABLED=0 GOOS=darwin /usr/local/go/bin/go build -a -ldflags "$(cat /link-flags.txt)" -o fortio_go1.8.mac istio.io/fortio
-# Build with 1.8 for perf comparaison
+# Build with 1.8 for perf comparison
 #RUN CGO_ENABLED=0 GOOS=linux /usr/local/go/bin/go build -a -ldflags "$(cat /link-flags.txt)" -o fortio_go1.8.bin istio.io/fortio
 #RUN ./fortio_go1.8.bin version
 # Just check it stays compiling on Windows (would need to set the rsrcDir too)
diff --git a/Dockerfile.fcurl b/Dockerfile.fcurl
new file mode 100644
index 0000000..8accca0
--- /dev/null
+++ b/Dockerfile.fcurl
@@ -0,0 +1,15 @@
+# Build the binaries in larger image
+FROM istio/fortio.build:v6 as build
+WORKDIR /go/src/istio.io
+COPY . fortio
+# TODO: share common part with main Dockerfile (have the main Dockerfile build more than 1 image - how?)
+RUN echo "$(date +'%Y-%m-%d %H:%M') $(cd fortio; git rev-parse HEAD)" > /build-info.txt
+RUN echo "-s -X \"istio.io/fortio/version.buildInfo=$(cat /build-info.txt)\" \
+  -X istio.io/fortio/version.tag=$(cd fortio; git describe --tags) \
+  -X istio.io/fortio/version.gitstatus=$(cd fortio; git status --porcelain | wc -l)" > /link-flags.txt
+RUN CGO_ENABLED=0 GOOS=linux go build -a -ldflags "$(cat /link-flags.txt)" -o fcurl.bin istio.io/fortio/fcurl
+# Minimal image with just the binary
+FROM scratch
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/src/istio.io/fcurl.bin /usr/local/bin/fcurl
+ENTRYPOINT ["/usr/local/bin/fcurl"]
diff --git a/Makefile b/Makefile
index 5ec2b6f..432baa1 100755
--- a/Makefile
+++ b/Makefile
@@ -4,7 +4,7 @@
 # See also release/README.md
 #
 
-IMAGES=echosrv # plus the combo image / Dockerfile without ext.
+IMAGES=echosrv fcurl # plus the combo image / Dockerfile without ext.
 
 DOCKER_PREFIX := docker.io/istio/fortio
 BUILD_IMAGE_TAG := v6
@@ -110,7 +110,7 @@ all: test install lint docker-version docker-push-internal
 
 # Makefile should be edited first
 FILES_WITH_IMAGE:= .circleci/config.yml Dockerfile Dockerfile.echosrv \
-	Dockerfile.test release/Dockerfile.in
+	Dockerfile.test Dockerfile.fcurl release/Dockerfile.in
 # Ran make update-build-image BUILD_IMAGE_TAG=v1 DOCKER_PREFIX=fortio/fortio
 update-build-image:
 	$(MAKE) docker-push-internal IMAGE=.build TAG=$(BUILD_IMAGE_TAG)
diff --git a/README.md b/README.md
index 544f932..4d04378 100644
--- a/README.md
+++ b/README.md
@@ -28,7 +28,7 @@ docker run istio/fortio load http://www.google.com/ # For a test run
 Or download the binary distribution, for instance:
 
 ```shell
-curl -L https://github.com/istio/fortio/releases/download/v0.8.0/fortio-linux_x64-0.8.0.tgz \
+curl -L https://github.com/istio/fortio/releases/download/v0.8.1/fortio-linux_x64-0.8.1.tgz \
  | sudo tar -C / -xvzpf -
 ```
 
@@ -41,17 +41,17 @@ You can get a preview of the reporting/graphing UI at https://fortio.istio.io/
 Fortio can be an http or grpc load generator, gathering statistics using the `load` subcommand, or start simple http and grpc ping servers, as well as a basic web UI, result graphing and https redirector, with the `server` command or issue grpc ping messages using the `grpcping` command. It can also fetch a single URL's for debugging when using the `curl` command (or the `-curl` flag to the load command). You can run just the redirector with `redirect`. Lastly if you saved JSON results (using the web UI or directly from the command line), you can browse and graph those results using the `report` command.
 <!-- use release/updateFlags.sh to update this section -->
 ```
-Φορτίο 0.8.0 usage:
+Φορτίο 0.8.1 usage:
 	fortio command [flags] target
 where command is one of: load (load testing), server (starts grpc ping and http
 echo/ui/redirect/proxy servers), grpcping (grpc client), report (report only UI
 server), redirect (redirect only server), or curl (single URL debug).  where
-target is a url (http load tests) or host:port (grpc health test) and flags are:
+target is a url (http load tests) or host:port (grpc health test).  flags are:
   -H value
 	Additional Header(s)
   -L	Follow redirects (implies -std-client) - do not use for load test
   -P value
-	Proxies to run, e.g -P "localport1 dest_host1:dest_port1" 
+	Proxies to run, e.g -P "localport1 dest_host1:dest_port1"
 	-P "[::1]:0 www.google.com:443" ...
   -a	Automatically save JSON result with filename based on labels & timestamp
   -allow-initial-errors
@@ -97,10 +97,11 @@ target is a url (http load tests) or host:port (grpc health test) and flags are:
   -httpccch
 	Check for Connection: Close Header
   -https-insecure
-	Do not verify certs in https connections
+	Long form of the -k flag
   -json string
 	Json output to provided file or '-' for stdout (empty = no json output,
 	unless -a is used)
+  -k	Do not verify certs in https connections
   -keepalive
 	Keep connection alive (only for fast http 1.1) (default true)
   -labels string
@@ -155,12 +156,12 @@ target is a url (http load tests) or host:port (grpc health test) and flags are:
 * Start the internal servers:
 ```
 $ fortio server &
-Fortio 0.8.0 grpc 'ping' server listening on [::]:8079
-Fortio 0.8.0 https redirector server listening on [::]:8081
-Fortio 0.8.0 echo server listening on [::]:8080
+Fortio 0.8.1 grpc 'ping' server listening on [::]:8079
+Fortio 0.8.1 https redirector server listening on [::]:8081
+Fortio 0.8.1 echo server listening on [::]:8080
 UI started - visit:
 http://localhost:8080/fortio/   (or any host/ip reachable on this server)
-21:45:23 I fortio_main.go:195> All fortio 0.8.0 buildinfo go1.10 servers started!
+21:45:23 I fortio_main.go:195> All fortio 0.8.1 buildinfo go1.10 servers started!
 ```
 
 * By default, Fortio's web/echo servers listen on port 8080 on all interfaces.
@@ -170,8 +171,8 @@ $ fortio server -http-port 10.10.10.10:8088
 UI starting - visit:
 http://10.10.10.10:8088/fortio/
 Https redirector running on :8081
-Fortio 0.8.0 grpc ping server listening on port :8079
-Fortio 0.8.0 echo server listening on port 10.10.10.10:8088
+Fortio 0.8.1 grpc ping server listening on port :8079
+Fortio 0.8.1 echo server listening on port 10.10.10.10:8088
 ```
 * Simple grpc ping:
 ```
@@ -241,14 +242,14 @@ Content-Type: text/plain; charset=UTF-8
 Date: Mon, 08 Jan 2018 22:26:26 GMT
 Content-Length: 230
 
-Φορτίο version 0.8.0 echo debug server up for 39s on ldemailly-macbookpro - request from [::1]:65055
+Φορτίο version 0.8.1 echo debug server up for 39s on ldemailly-macbookpro - request from [::1]:65055
 
 GET /debug HTTP/1.1
 
 headers:
 
 Host: localhost:8080
-User-Agent: istio/fortio-0.8.0
+User-Agent: istio/fortio-0.8.1
 Foo: Bar
 
 body:
@@ -297,6 +298,9 @@ You may find fortio's [logger](log/logger.go) useful as well.
 
 You can run the histogram code standalone as a command line in [histogram/](histogram/), a basic echo http server in [echosrv/](echosrv/), or both the http echo and GRPC ping server through `fortio server`, the fortio command line interface lives in this top level directory [fortio_main.go](fortio_main.go)
 
+There is also [fcurl/](fcurl/) which is the `fortio curl` part of the code (if you need a light http client without grpc or server side).
+A matching tiny (2Mb compressed) docker image is [istio/fortio.fcurl](https://hub.docker.com/r/istio/fortio.fcurl/tags/)
+
 ## Another example output
 
 With 5k qps: (includes envoy and mixer in the calls)
diff --git a/bincommon/commonflags.go b/bincommon/commonflags.go
new file mode 100644
index 0000000..aef57fa
--- /dev/null
+++ b/bincommon/commonflags.go
@@ -0,0 +1,126 @@
+// Copyright 2018 Istio Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package bincommon is the common code and flag handling between the fortio
+// (fortio_main.go) and fcurl (fcurl.go) executables.
+package bincommon
+
+// Do not add any external dependencies we want to keep fortio minimal.
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+	"strings"
+
+	"istio.io/fortio/fhttp"
+	"istio.io/fortio/log"
+	"istio.io/fortio/version"
+)
+
+// -- Support for multiple instances of -H flag on cmd line:
+type headersFlagList struct {
+}
+
+func (f *headersFlagList) String() string {
+	return ""
+}
+func (f *headersFlagList) Set(value string) error {
+	return httpOpts.AddAndValidateExtraHeader(value)
+}
+
+// -- end of functions for -H support
+
+// FlagsUsage prints end of the usage() (flags part + error message).
+func FlagsUsage(msgs ...interface{}) {
+	// nolint: gas
+	fmt.Fprintf(os.Stderr, "flags are:\n")
+	flag.PrintDefaults()
+	fmt.Fprint(os.Stderr, msgs...) // nolint: gas
+	os.Stderr.WriteString("\n")    // nolint: gas, errcheck
+	os.Exit(1)
+}
+
+var (
+	compressionFlag = flag.Bool("compression", false, "Enable http compression")
+	keepAliveFlag   = flag.Bool("keepalive", true, "Keep connection alive (only for fast http 1.1)")
+	halfCloseFlag   = flag.Bool("halfclose", false,
+		"When not keepalive, whether to half close the connection (only for fast http)")
+	httpReqTimeoutFlag  = flag.Duration("timeout", fhttp.HTTPReqTimeOutDefaultValue, "Connection and read timeout value (for http)")
+	stdClientFlag       = flag.Bool("stdclient", false, "Use the slower net/http standard client (works for TLS)")
+	http10Flag          = flag.Bool("http1.0", false, "Use http1.0 (instead of http 1.1)")
+	httpsInsecureFlag   = flag.Bool("k", false, "Do not verify certs in https connections")
+	headersFlags        headersFlagList
+	httpOpts            fhttp.HTTPOptions
+	followRedirectsFlag = flag.Bool("L", false, "Follow redirects (implies -std-client) - do not use for load test")
+	// QuietFlag is the value of -quiet.
+	QuietFlag = flag.Bool("quiet", false, "Quiet mode: sets the loglevel to Error and reduces the output.")
+)
+
+// SharedMain is the common part of main from fortio_main and fcurl.
+func SharedMain() {
+	flag.Var(&headersFlags, "H", "Additional Header(s)")
+	flag.IntVar(&fhttp.BufferSizeKb, "httpbufferkb", fhttp.BufferSizeKb,
+		"Size of the buffer (max data size) for the optimized http client in kbytes")
+	flag.BoolVar(&fhttp.CheckConnectionClosedHeader, "httpccch", fhttp.CheckConnectionClosedHeader,
+		"Check for Connection: Close Header")
+	// Special case so `fcurl -version` and `--version` and `version` and ... work
+	if len(os.Args) >= 2 && strings.Contains(os.Args[1], "version") {
+		if len(os.Args) >= 3 && strings.Contains(os.Args[2], "s") {
+			// so `fortio version -s` is the short version; everything else is long/full
+			fmt.Println(version.Short())
+		} else {
+			fmt.Println(version.Long())
+		}
+		os.Exit(0)
+	}
+}
+
+// FetchURL is fetching url content and exiting with 1 upon error.
+// common part between fortio_main and fcurl.
+func FetchURL(o *fhttp.HTTPOptions) {
+	// keepAlive could be just false when making 1 fetch but it helps debugging
+	// the http client when making a single request if using the flags
+	client := fhttp.NewClient(o)
+	if client == nil {
+		return // error logged already
+	}
+	code, data, header := client.Fetch()
+	log.LogVf("Fetch result code %d, data len %d, headerlen %d", code, len(data), header)
+	os.Stdout.Write(data) //nolint: errcheck
+	if code != http.StatusOK {
+		log.Errf("Error status %d : %s", code, fhttp.DebugSummary(data, 512))
+		os.Exit(1)
+	}
+}
+
+// SharedHTTPOptions is the flag->httpoptions transfer code shared between
+// fortio_main and fcurl.
+func SharedHTTPOptions() *fhttp.HTTPOptions {
+	url := strings.TrimLeft(flag.Arg(0), " \t\r\n")
+	httpOpts.URL = url
+	httpOpts.HTTP10 = *http10Flag
+	httpOpts.DisableFastClient = *stdClientFlag
+	httpOpts.DisableKeepAlive = !*keepAliveFlag
+	httpOpts.AllowHalfClose = *halfCloseFlag
+	httpOpts.Compression = *compressionFlag
+	httpOpts.HTTPReqTimeOut = *httpReqTimeoutFlag
+	httpOpts.Insecure = *httpsInsecureFlag
+	if *followRedirectsFlag {
+		httpOpts.FollowRedirects = true
+		httpOpts.DisableFastClient = true
+	}
+	return &httpOpts
+}
diff --git a/fcurl/fcurl.go b/fcurl/fcurl.go
new file mode 100644
index 0000000..ea9a55b
--- /dev/null
+++ b/fcurl/fcurl.go
@@ -0,0 +1,49 @@
+// Copyright 2018 Istio Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+// Do not add any external dependencies we want to keep fortio minimal.
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"istio.io/fortio/bincommon"
+	"istio.io/fortio/log"
+	"istio.io/fortio/version"
+)
+
+// Prints usage
+func usage(msgs ...interface{}) {
+	// nolint: gas
+	fmt.Fprintf(os.Stderr, "Φορτίο fortio-curl %s usage:\n\t%s [flags] url\n",
+		version.Short(),
+		os.Args[0])
+	bincommon.FlagsUsage(msgs...)
+}
+
+func main() {
+	bincommon.SharedMain()
+	if len(os.Args) < 2 {
+		usage("Error: need a url as parameter")
+	}
+	flag.Parse()
+	if *bincommon.QuietFlag {
+		log.SetLogLevelQuiet(log.Error)
+	}
+	o := bincommon.SharedHTTPOptions()
+	bincommon.FetchURL(o)
+}
diff --git a/fhttp/http_client.go b/fhttp/http_client.go
index c39e55c..2c54cf9 100644
--- a/fhttp/http_client.go
+++ b/fhttp/http_client.go
@@ -272,7 +272,8 @@ func (c *Client) Fetch() (int, []byte, int) {
 // NewClient creates either a standard or fast client (depending on
 // the DisableFastClient flag)
 func NewClient(o *HTTPOptions) Fetcher {
-	o.URLSchemeCheck()
+	o.Init(o.URL)      // For completely new options
+	o.URLSchemeCheck() // For changes to options after init
 	if o.DisableFastClient {
 		return NewStdClient(o)
 	}
@@ -281,6 +282,7 @@ func NewClient(o *HTTPOptions) Fetcher {
 
 // NewStdClient creates a client object that wraps the net/http standard client.
 func NewStdClient(o *HTTPOptions) *Client {
+	o.Init(o.URL)
 	req := newHTTPRequest(o)
 	if req == nil {
 		return nil
@@ -382,6 +384,7 @@ func (c *FastClient) Close() int {
 // This function itself doesn't need to be super efficient as it is created at
 // the beginning and then reused many times.
 func NewFastClient(o *HTTPOptions) Fetcher {
+	o.Init(o.URL)
 	proto := "1.1"
 	if o.HTTP10 {
 		proto = "1.0"
diff --git a/fortio_main.go b/fortio_main.go
index c89f62c..3e059a5 100644
--- a/fortio_main.go
+++ b/fortio_main.go
@@ -20,12 +20,12 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
-	"net/http"
 	"os"
 	"path"
 	"runtime"
 	"strings"
 
+	"istio.io/fortio/bincommon"
 	"istio.io/fortio/fnet"
 
 	"istio.io/fortio/fgrpc"
@@ -37,18 +37,6 @@ import (
 	"istio.io/fortio/version"
 )
 
-// -- Support for multiple instances of -H flag on cmd line:
-type headersFlagList struct {
-}
-
-func (f *headersFlagList) String() string {
-	return ""
-}
-func (f *headersFlagList) Set(value string) error {
-	return httpOpts.AddAndValidateExtraHeader(value)
-}
-
-// -- end of functions for -H support
 // -- Support for multiple proxies (-P) flags on cmd line:
 type proxiesFlagList struct {
 }
@@ -66,41 +54,34 @@ func (f *proxiesFlagList) Set(value string) error {
 // Prints usage
 func usage(msgs ...interface{}) {
 	// nolint: gas
-	fmt.Fprintf(os.Stderr, "Φορτίο %s usage:\n\t%s command [flags] target\n%s\n%s\n%s\n%s\n%s\n",
+	fmt.Fprintf(os.Stderr, "Φορτίο %s usage:\n\t%s command [flags] target\n%s\n%s\n%s\n%s\n",
 		version.Short(),
 		os.Args[0],
 		"where command is one of: load (load testing), server (starts grpc ping and",
 		"http echo/ui/redirect/proxy servers), grpcping (grpc client), report (report only UI",
 		"server), redirect (redirect only server), or curl (single URL debug).",
-		"where target is a url (http load tests) or host:port (grpc health test)",
-		"and flags are:")
-	flag.PrintDefaults()
-	fmt.Fprint(os.Stderr, msgs...) // nolint: gas
-	os.Stderr.WriteString("\n")    // nolint: gas, errcheck
-	os.Exit(1)
+		"where target is a url (http load tests) or host:port (grpc health test).")
+	bincommon.FlagsUsage(msgs...)
 }
 
+// Attention: every flag that is common to http client goes to bincommon/
+// for sharing between fortio and fcurl binaries
+
 var (
 	defaults = &periodic.DefaultRunnerOptions
 	// Very small default so people just trying with random URLs don't affect the target
-	qpsFlag            = flag.Float64("qps", defaults.QPS, "Queries Per Seconds or 0 for no wait/max qps")
-	numThreadsFlag     = flag.Int("c", defaults.NumThreads, "Number of connections/goroutine/threads")
-	durationFlag       = flag.Duration("t", defaults.Duration, "How long to run the test or 0 to run until ^C")
-	percentilesFlag    = flag.String("p", "50,75,90,99,99.9", "List of pXX to calculate")
-	resolutionFlag     = flag.Float64("r", defaults.Resolution, "Resolution of the histogram lowest buckets in seconds")
-	compressionFlag    = flag.Bool("compression", false, "Enable http compression")
-	goMaxProcsFlag     = flag.Int("gomaxprocs", 0, "Setting for runtime.GOMAXPROCS, <1 doesn't change the default")
-	profileFlag        = flag.String("profile", "", "write .cpu and .mem profiles to file")
-	keepAliveFlag      = flag.Bool("keepalive", true, "Keep connection alive (only for fast http 1.1)")
-	halfCloseFlag      = flag.Bool("halfclose", false, "When not keepalive, whether to half close the connection (only for fast http)")
-	httpReqTimeoutFlag = flag.Duration("timeout", fhttp.HTTPReqTimeOutDefaultValue, "Connection and read timeout value (for http)")
-	stdClientFlag      = flag.Bool("stdclient", false, "Use the slower net/http standard client (works for TLS)")
-	http10Flag         = flag.Bool("http1.0", false, "Use http1.0 (instead of http 1.1)")
-	grpcFlag           = flag.Bool("grpc", false, "Use GRPC (health check) for load testing")
-	grpcSecureFlag     = flag.Bool("grpc-secure", false, "Use secure transport (tls) for GRPC")
-	httpsInsecureFlag  = flag.Bool("https-insecure", false, "Do not verify certs in https connections")
-	echoPortFlag       = flag.String("http-port", "8080", "http echo server port. Can be in the form of host:port, ip:port or port.")
-	grpcPortFlag       = flag.String("grpc-port", fgrpc.DefaultGRPCPort,
+	qpsFlag           = flag.Float64("qps", defaults.QPS, "Queries Per Seconds or 0 for no wait/max qps")
+	numThreadsFlag    = flag.Int("c", defaults.NumThreads, "Number of connections/goroutine/threads")
+	durationFlag      = flag.Duration("t", defaults.Duration, "How long to run the test or 0 to run until ^C")
+	percentilesFlag   = flag.String("p", "50,75,90,99,99.9", "List of pXX to calculate")
+	resolutionFlag    = flag.Float64("r", defaults.Resolution, "Resolution of the histogram lowest buckets in seconds")
+	goMaxProcsFlag    = flag.Int("gomaxprocs", 0, "Setting for runtime.GOMAXPROCS, <1 doesn't change the default")
+	profileFlag       = flag.String("profile", "", "write .cpu and .mem profiles to file")
+	grpcFlag          = flag.Bool("grpc", false, "Use GRPC (health check) for load testing")
+	grpcSecureFlag    = flag.Bool("grpc-secure", false, "Use secure transport (tls) for GRPC")
+	httpsInsecureFlag = flag.Bool("https-insecure", false, "Long form of the -k flag")
+	echoPortFlag      = flag.String("http-port", "8080", "http echo server port. Can be in the form of host:port, ip:port or port.")
+	grpcPortFlag      = flag.String("grpc-port", fgrpc.DefaultGRPCPort,
 		"grpc server port. Can be in the form of host:port, ip:port or port.")
 	echoDbgPathFlag = flag.String("echo-debug-path", "/debug",
 		"http echo server URI for debug, empty turns off that part (more secure)")
@@ -112,14 +93,11 @@ var (
 		"Additional config data/labels to add to the resulting JSON, defaults to target URL and hostname")
 	staticDirFlag = flag.String("static-dir", "", "Absolute path to the dir containing the static files dir")
 	dataDirFlag   = flag.String("data-dir", defaultDataDir, "Directory where JSON results are stored/read")
-	headersFlags  headersFlagList
-	httpOpts      fhttp.HTTPOptions
 	proxiesFlags  proxiesFlagList
 	proxies       = make([]string, 0)
 
 	defaultDataDir = "."
 
-	followRedirectsFlag    = flag.Bool("L", false, "Follow redirects (implies -std-client) - do not use for load test")
 	allowInitialErrorsFlag = flag.Bool("allow-initial-errors", false, "Allow and don't abort on initial warmup errors")
 	autoSaveFlag           = flag.Bool("a", false, "Automatically save JSON result with filename based on labels & timestamp")
 	redirectFlag           = flag.String("redirect-port", "8081", "Redirect all incoming traffic to https URL"+
@@ -127,7 +105,6 @@ var (
 	exactlyFlag = flag.Int64("n", 0,
 		"Run for exactly this number of calls instead of duration. Default (0) is to use duration (-t). "+
 			"Default is 1 when used as grpc ping count.")
-	quietFlag   = flag.Bool("quiet", false, "Quiet mode: sets the loglevel to Error and reduces the output.")
 	syncFlag    = flag.String("sync", "", "index.tsv or s3/gcs bucket xml URL to fetch at startup for server modes.")
 	baseURLFlag = flag.String("base-url", "",
 		"base URL used as prefix for data/index.tsv generation. (when empty, the url from the first request is used)")
@@ -143,22 +120,8 @@ var (
 )
 
 func main() {
-	flag.Var(&headersFlags, "H", "Additional Header(s)")
+	bincommon.SharedMain()
 	flag.Var(&proxiesFlags, "P", "Proxies to run, e.g -P \"localport1 dest_host1:dest_port1\" -P \"[::1]:0 www.google.com:443\" ...")
-	flag.IntVar(&fhttp.BufferSizeKb, "httpbufferkb", fhttp.BufferSizeKb,
-		"Size of the buffer (max data size) for the optimized http client in kbytes")
-	flag.BoolVar(&fhttp.CheckConnectionClosedHeader, "httpccch", fhttp.CheckConnectionClosedHeader,
-		"Check for Connection: Close Header")
-	// Special case so `fortio -version` and `--version` and `version` and ... work
-	if len(os.Args) >= 2 && strings.Contains(os.Args[1], "version") {
-		if len(os.Args) >= 3 && strings.Contains(os.Args[2], "s") {
-			// so `fortio version -s` is the short version; everything else is long/full
-			fmt.Println(version.Short())
-		} else {
-			fmt.Println(version.Long())
-		}
-		os.Exit(0)
-	}
 	if len(os.Args) < 2 {
 		usage("Error: need at least 1 command parameter")
 	}
@@ -166,7 +129,7 @@ func main() {
 	os.Args = append([]string{os.Args[0]}, os.Args[2:]...)
 	flag.Parse()
 	fhttp.ChangeMaxPayloadSize(*newMaxPayloadSizeKb * 1024)
-	if *quietFlag {
+	if *bincommon.QuietFlag {
 		log.SetLogLevelQuiet(log.Error)
 	}
 	percList, err := stats.ParsePercentiles(*percentilesFlag)
@@ -225,43 +188,19 @@ func main() {
 	}
 }
 
-func fetchURL(o *fhttp.HTTPOptions) {
-	// keepAlive could be just false when making 1 fetch but it helps debugging
-	// the http client when making a single request if using the flags
-	client := fhttp.NewClient(o)
-	if client == nil {
-		return // error logged already
-	}
-	code, data, header := client.Fetch()
-	log.LogVf("Fetch result code %d, data len %d, headerlen %d", code, len(data), header)
-	os.Stdout.Write(data) //nolint: errcheck
-	if code != http.StatusOK {
-		log.Errf("Error status %d : %s", code, fhttp.DebugSummary(data, 512))
-		os.Exit(1)
-	}
-}
-
 func fortioLoad(justCurl bool, percList []float64) {
 	if len(flag.Args()) != 1 {
 		usage("Error: fortio load/curl needs a url or destination")
 	}
-	url := strings.TrimLeft(flag.Arg(0), " \t\r\n")
-	httpOpts.URL = url
-	httpOpts.HTTP10 = *http10Flag
-	httpOpts.DisableFastClient = *stdClientFlag
-	httpOpts.DisableKeepAlive = !*keepAliveFlag
-	httpOpts.AllowHalfClose = *halfCloseFlag
-	httpOpts.Compression = *compressionFlag
-	httpOpts.HTTPReqTimeOut = *httpReqTimeoutFlag
-	httpOpts.Insecure = *httpsInsecureFlag
-	if *followRedirectsFlag {
-		httpOpts.FollowRedirects = true
-		httpOpts.DisableFastClient = true
+	httpOpts := bincommon.SharedHTTPOptions()
+	if *httpsInsecureFlag {
+		httpOpts.Insecure = true
 	}
 	if justCurl {
-		fetchURL(&httpOpts)
+		bincommon.FetchURL(httpOpts)
 		return
 	}
+	url := httpOpts.URL
 	prevGoMaxProcs := runtime.GOMAXPROCS(*goMaxProcsFlag)
 	out := os.Stderr
 	qps := *qpsFlag // TODO possibly use translated <=0 to "max" from results/options normalization in periodic/
@@ -317,7 +256,7 @@ func fortioLoad(justCurl bool, percList []float64) {
 		res, err = fgrpc.RunGRPCTest(&o)
 	} else {
 		o := fhttp.HTTPRunnerOptions{
-			HTTPOptions:        httpOpts,
+			HTTPOptions:        *httpOpts,
 			RunnerOptions:      ro,
 			Profiler:           *profileFlag,
 			AllowInitialErrors: *allowInitialErrorsFlag,


Test patch:
diff --git a/fhttp/http_test.go b/fhttp/http_test.go
index b0e9d29..1287a65 100644
--- a/fhttp/http_test.go
+++ b/fhttp/http_test.go
@@ -600,6 +600,9 @@ func TestDefaultPort(t *testing.T) {
 		t.Errorf("fast client isn't supposed to support https (yet), got %v", cli)
 	}
 	cli = NewClient(opts)
+	if cli == nil {
+		t.Fatalf("Couldn't get a client using NewClient on modified opts.")
+	}
 	// currently fast client fails with https:
 	code, _, _ = cli.Fetch()
 	if code != 200 {
@@ -815,6 +818,31 @@ func TestNewHTMLEscapeWriterError(t *testing.T) {
 	}
 }
 
+func TestDefaultHeadersAndOptionsInit(t *testing.T) {
+	_, addr := Serve("0", "/debug")
+	// Un initialized http options:
+	o := HTTPOptions{URL: fmt.Sprintf("http://localhost:%d/debug", addr.Port)}
+	o1 := o
+	cli1 := NewStdClient(&o1)
+	code, data, _ := cli1.Fetch()
+	if code != 200 {
+		t.Errorf("Non ok code %d for debug default fetch1", code)
+	}
+	expected := []byte("User-Agent: istio/fortio-")
+	if !bytes.Contains(data, expected) {
+		t.Errorf("Didn't find default header echoed back in std client1 %s (expecting %s)", DebugSummary(data, 512), expected)
+	}
+	o2 := o
+	cli2 := NewFastClient(&o2)
+	code, data, _ = cli2.Fetch()
+	if code != 200 {
+		t.Errorf("Non ok code %d for debug default fetch2", code)
+	}
+	if !bytes.Contains(data, expected) {
+		t.Errorf("Didn't find default header echoed back in fast client1 %s (expecting %s)", DebugSummary(data, 512), expected)
+	}
+}
+
 // --- for bench mark/comparaison
 
 func asciiFold0(str string) []byte {