Repository: vmware/govmomi
Pull request #3227 (2023-09-08 20:13:07)

Use Envoy sidecar for guest and datastore file transfer.
## Description

- vCenter has a local Envoy process exposed over a Unix socket dedicated for host connectivity.
- During requests that involve traffic to ESX, when the envoy sidecar is in use (localhost:1080) in the VC client, leverage the envoy host gateway.
- Avoid keeping the connection alive since we're only using it for one request to the host. (ie. no connection pool to the host(s) being maintained here)

## Type of change

Please mark options that are relevant:

- [ ] Bug fix (non-breaking change which fixes an issue)
- [X] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing functionality to
  not work as expected)
- [ ] This change requires a documentation update
- [ ] Build related change

## How Has This Been Tested?

- [X] With a Go based service using localhost:1080, used iptables to block outbound traffic to 443 (and thereby prevent ESX access), verified that guest file transfer, datastore file upload continued to work.

- [X] Added unit test to verify socket creation.

Note: I think it would make sense to enhance vcsim to support this host gateway socket, but from what I could tell, the existing vcsim tests for things like datastore upload only run vcsim with an ESX - I could not find anything in place to simulate both VC and ESX.


## Checklist:

- [X] My code follows the `CONTRIBUTION` [guidelines] of this project
- [X] I have commented my code, particularly in hard-to-understand areas
- [X] I have made corresponding changes to the documentation
- [X] I have added tests that prove my fix is effective or that my feature works
- [X] New and existing unit tests pass locally with my changes
- [X] Any dependent changes have been merged

[guidelines]: https://github.com/vmware/govmomi/blob/main/CONTRIBUTING.md



Patch:
diff --git a/guest/file_manager.go b/guest/file_manager.go
index 9dc0c194..1141f54c 100644
--- a/guest/file_manager.go
+++ b/guest/file_manager.go
@@ -168,6 +168,15 @@ func (m FileManager) TransferURL(ctx context.Context, u string) (*url.URL, error
 		return turl, nil // won't matter if the VM was powered off since the call to InitiateFileTransfer will fail
 	}
 
+	// VC supports the use of a Unix domain socket for guest file transfers.
+	if internal.UsingEnvoySidecar(m.c) {
+		// Rewrite the URL in the format unix://
+		// Reciever must use a custom dialer.
+		// Nil check performed above, so Host is safe to access.
+		return internal.HostGatewayTransferURL(turl, *vm.Runtime.Host), nil
+	}
+
+	// Determine host thumbprint, address etc. to be able to trust host.
 	props := []string{
 		"name",
 		"runtime.connectionState",
diff --git a/guest/toolbox/client.go b/guest/toolbox/client.go
index 9d492296..38cc5946 100644
--- a/guest/toolbox/client.go
+++ b/guest/toolbox/client.go
@@ -29,6 +29,7 @@ import (
 	"time"
 
 	"github.com/vmware/govmomi/guest"
+	"github.com/vmware/govmomi/internal"
 	"github.com/vmware/govmomi/property"
 	"github.com/vmware/govmomi/vim25"
 	"github.com/vmware/govmomi/vim25/mo"
@@ -284,6 +285,10 @@ func (c *Client) Download(ctx context.Context, src string) (io.ReadCloser, int64
 
 	p := soap.DefaultDownload
 
+	if internal.UsingEnvoySidecar(c.ProcessManager.Client()) {
+		vc = internal.ClientWithEnvoyHostGateway(vc)
+	}
+
 	f, n, err := vc.Download(ctx, u, &p)
 	if err != nil {
 		return nil, n, err
@@ -341,5 +346,9 @@ func (c *Client) Upload(ctx context.Context, src io.Reader, dst string, p soap.U
 		return err
 	}
 
+	if internal.UsingEnvoySidecar(c.ProcessManager.Client()) {
+		vc = internal.ClientWithEnvoyHostGateway(vc)
+	}
+
 	return vc.Client.Upload(ctx, src, u, &p)
 }
diff --git a/internal/helpers.go b/internal/helpers.go
index 6c2dda0d..41e533fd 100644
--- a/internal/helpers.go
+++ b/internal/helpers.go
@@ -17,15 +17,25 @@ limitations under the License.
 package internal
 
 import (
+	"context"
+	"fmt"
 	"net"
+	"net/http"
+	"net/url"
 	"os"
 	"path"
 
 	"github.com/vmware/govmomi/vim25"
 	"github.com/vmware/govmomi/vim25/mo"
+	"github.com/vmware/govmomi/vim25/soap"
 	"github.com/vmware/govmomi/vim25/types"
 )
 
+const (
+	vCenterHostGatewaySocket    = "/var/run/envoy-hgw/hgw-pipe"
+	vCenterHostGatewaySocketEnv = "VCENTER_ENVOY_HOST_GATEWAY"
+)
+
 // InventoryPath composed of entities by Name
 func InventoryPath(entities []mo.ManagedEntity) string {
 	val := "/"
@@ -78,3 +88,54 @@ func UsingEnvoySidecar(c *vim25.Client) bool {
 	}
 	return c.URL().Hostname() == envoySidecarHost && c.URL().Scheme == "http" && c.URL().Port() == envoySidecarPort
 }
+
+// ClientWithEnvoyHostGateway clones the provided soap.Client and returns a new
+// one that uses a Unix socket to leverage vCenter's local Envoy host
+// gateway.
+// This should be used to construct clients that talk to ESX.
+// This method returns a new *vim25.Client and does not modify the original input.
+// This client disables HTTP keep alives and is intended for a single round
+// trip. (eg. guest file transfer, datastore file transfer)
+func ClientWithEnvoyHostGateway(vc *vim25.Client) *vim25.Client {
+	// Override the vim client with a new one that wraps a Unix socket transport.
+	// Using HTTP here so secure means nothing.
+	sc := soap.NewClient(vc.URL(), true)
+	// Clone the underlying HTTP transport, only replacing the dialer logic.
+	transport := sc.DefaultTransport().Clone()
+	hostGatewaySocketPath := os.Getenv(vCenterHostGatewaySocketEnv)
+	if hostGatewaySocketPath == "" {
+		hostGatewaySocketPath = vCenterHostGatewaySocket
+	}
+	transport.DialContext = func(_ context.Context, _, _ string) (net.Conn, error) {
+		return net.Dial("unix", hostGatewaySocketPath)
+	}
+	// We use this client for a single request, so we don't require keepalives.
+	transport.DisableKeepAlives = true
+	sc.Client = http.Client{
+		Transport: transport,
+	}
+	newVC := &vim25.Client{
+		Client: sc,
+	}
+	return newVC
+}
+
+// HostGatewayTransferURL rewrites the provided URL to be suitable for use
+// with the Envoy host gateway on vCenter.
+// It returns a copy of the provided URL with the host, scheme rewritten as needed.
+// Receivers of such URLs must typically also use ClientWithEnvoyHostGateway to
+// use the appropriate http.Transport to be able to make use of the host
+// gateway.
+// nil input yields an uninitialized struct.
+func HostGatewayTransferURL(u *url.URL, hostMoref types.ManagedObjectReference) *url.URL {
+	if u == nil {
+		return &url.URL{}
+	}
+	// Make a copy of the provided URL.
+	turl := *u
+	turl.Host = "localhost"
+	turl.Scheme = "http"
+	oldPath := turl.Path
+	turl.Path = fmt.Sprintf("/hgw/%s%s", hostMoref.Value, oldPath)
+	return &turl
+}
diff --git a/object/datastore.go b/object/datastore.go
index e2586a17..77d51262 100644
--- a/object/datastore.go
+++ b/object/datastore.go
@@ -27,6 +27,7 @@ import (
 	"path"
 	"strings"
 
+	"github.com/vmware/govmomi/internal"
 	"github.com/vmware/govmomi/property"
 	"github.com/vmware/govmomi/session"
 	"github.com/vmware/govmomi/vim25"
@@ -229,8 +230,18 @@ func (d Datastore) ServiceTicket(ctx context.Context, path string, method string
 	delete(q, "dcPath")
 	u.RawQuery = q.Encode()
 
+	// Now that we have a host selected, take a copy of the URL.
+	transferURL := *u
+
+	if internal.UsingEnvoySidecar(d.Client()) {
+		// Rewrite the host URL to go through the Envoy sidecar on VC.
+		// Reciever must use a custom dialer.
+		u = internal.HostGatewayTransferURL(u, host.Reference())
+	}
+
 	spec := types.SessionManagerHttpServiceRequestSpec{
-		Url: u.String(),
+		// Use the original URL (without rewrites) for the session ticket.
+		Url: transferURL.String(),
 		// See SessionManagerHttpServiceRequestSpecMethod enum
 		Method: fmt.Sprintf("http%s%s", method[0:1], strings.ToLower(method[1:])),
 	}
@@ -303,7 +314,13 @@ func (d Datastore) UploadFile(ctx context.Context, file string, path string, par
 	if err != nil {
 		return err
 	}
-	return d.Client().UploadFile(ctx, file, u, p)
+	vc := d.Client()
+	if internal.UsingEnvoySidecar(vc) {
+		// Override the vim client with a new one that wraps a Unix socket transport.
+		// Using HTTP here so secure means nothing.
+		vc = internal.ClientWithEnvoyHostGateway(vc)
+	}
+	return vc.UploadFile(ctx, file, u, p)
 }
 
 // Download via soap.Download with an http service ticket
@@ -321,7 +338,13 @@ func (d Datastore) DownloadFile(ctx context.Context, path string, file string, p
 	if err != nil {
 		return err
 	}
-	return d.Client().DownloadFile(ctx, file, u, p)
+	vc := d.Client()
+	if internal.UsingEnvoySidecar(vc) {
+		// Override the vim client with a new one that wraps a Unix socket transport.
+		// Using HTTP here so secure means nothing.
+		vc = internal.ClientWithEnvoyHostGateway(vc)
+	}
+	return vc.DownloadFile(ctx, file, u, p)
 }
 
 // AttachedHosts returns hosts that have this Datastore attached, accessible and writable.


Test patch:
diff --git a/internal/helpers_test.go b/internal/helpers_test.go
index f811d1b8..8afe2cbc 100644
--- a/internal/helpers_test.go
+++ b/internal/helpers_test.go
@@ -14,9 +14,17 @@ limitations under the License.
 package internal_test
 
 import (
+	"crypto/rand"
+	"encoding/hex"
 	"fmt"
+	"io"
+	"net"
+	"net/http"
 	"net/url"
+	"os"
+	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 
@@ -24,6 +32,7 @@ import (
 	"github.com/vmware/govmomi/simulator/esx"
 	"github.com/vmware/govmomi/vim25"
 	"github.com/vmware/govmomi/vim25/soap"
+	"github.com/vmware/govmomi/vim25/types"
 )
 
 func TestHostSystemManagementIPs(t *testing.T) {
@@ -57,3 +66,86 @@ func TestUsingVCEnvoySidecar(t *testing.T) {
 		require.True(t, usingSidecar)
 	})
 }
+
+func TestClientUsingEnvoyHostGateway(t *testing.T) {
+	prefix := "hgw"
+	suffix := ".sock"
+	randBytes := make([]byte, 16)
+	_, err := rand.Read(randBytes)
+	require.NoError(t, err)
+
+	testSocketPath := filepath.Join(os.TempDir(), prefix+hex.EncodeToString(randBytes)+suffix)
+
+	l, err := net.Listen("unix", testSocketPath)
+	require.NoError(t, err)
+	handler := &testHTTPServer{
+		expectedURL: "http://localhost/foo",
+		response:    "Hello, Unix socket!",
+		t:           t,
+	}
+	server := http.Server{
+		Handler: handler,
+	}
+	go server.Serve(l)
+	defer server.Close()
+	defer l.Close()
+
+	// First make sure the test server works fine, since we're starting a goroutine.
+	unixDialer := func(proto, addr string) (conn net.Conn, err error) {
+		return net.Dial("unix", testSocketPath)
+	}
+	tr := &http.Transport{
+		Dial: unixDialer,
+	}
+	client := &http.Client{Transport: tr}
+
+	require.Eventually(t, func() bool {
+		_, err := client.Get(handler.expectedURL)
+		return err == nil
+	}, 15*time.Second, 1*time.Second, "Expected test HTTP server to be up")
+
+	envVar := "VCENTER_ENVOY_HOST_GATEWAY"
+	oldValue := os.Getenv(envVar)
+	defer os.Setenv(envVar, oldValue)
+	os.Setenv(envVar, testSocketPath)
+
+	// Build a new client using the test unix socket.
+	vc := &vim25.Client{Client: soap.NewClient(&url.URL{}, true)}
+	newClient := internal.ClientWithEnvoyHostGateway(vc)
+
+	// An HTTP request made using the new client should hit the server listening on the Unix socket.
+	resp, err := newClient.Get(handler.expectedURL)
+
+	// ...but should successfully connect to the Unix socket set up for testing.
+	require.NoError(t, err)
+	response, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+
+	require.Equal(t, response, []byte(handler.response))
+}
+
+type testHTTPServer struct {
+	expectedURL string
+	response    string
+	t           *testing.T
+}
+
+func (t *testHTTPServer) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+	require.Equal(t.t, "/foo", req.URL.Path)
+	resp.Write([]byte(t.response))
+}
+
+func TestRewriteURLForHostGateway(t *testing.T) {
+	testURL, err := url.Parse("https://foo.bar/baz?query_param=1")
+	require.NoError(t, err)
+
+	hostMoref := types.ManagedObjectReference{
+		Type:  "HostSystem",
+		Value: "host-123",
+	}
+	result := internal.HostGatewayTransferURL(testURL, hostMoref)
+	require.Equal(t, "localhost", result.Host)
+	require.Equal(t, "/hgw/host-123/baz", result.Path)
+	values := url.Values{"query_param": []string{"1"}}
+	require.Equal(t, values, result.Query())
+}