Repository: googleforgames/open-match
Pull request #696 (2019-07-26 21:50:32)

Payload logging
HTTP and gRPC server/client payload logging now will be available if debug/trace level logging is turned on. 


Patch:
diff --git a/Makefile b/Makefile
index fcebb00f..3fa0d461 100644
--- a/Makefile
+++ b/Makefile
@@ -359,6 +359,7 @@ install-large-chart: build/toolchain/bin/helm$(EXE_EXTENSION) install/helm/open-
 		--set global.telemetry.grafana.enabled=true \
 		--set global.telemetry.jaeger.enabled=true \
 		--set global.telemetry.prometheus.enabled=true \
+		--set global.logging.rpc.enabled=true \
 		--set global.telemetry.stackdriver.gcpProjectId=$(GCP_PROJECT_ID)
 
 install-chart: build/toolchain/bin/helm$(EXE_EXTENSION) install/helm/open-match/secrets/
diff --git a/install/helm/open-match/templates/global-configmap.yaml b/install/helm/open-match/templates/global-configmap.yaml
index fcab195e..cfa3f47d 100644
--- a/install/helm/open-match/templates/global-configmap.yaml
+++ b/install/helm/open-match/templates/global-configmap.yaml
@@ -25,6 +25,8 @@ metadata:
 data:
   global_config.yaml: |-
     telemetry:
+      zpages:
+        enable: "{{ .Values.global.telemetry.zpages.enabled }}"
       jaeger:
         enable: "{{ .Values.global.telemetry.jaeger.enabled }}"
         agentEndpoint: "{{ .Values.global.telemetry.jaeger.agentEndpoint }}"
diff --git a/install/helm/open-match/templates/om-configmap.yaml b/install/helm/open-match/templates/om-configmap.yaml
index 056a38c3..03a3f9b5 100644
--- a/install/helm/open-match/templates/om-configmap.yaml
+++ b/install/helm/open-match/templates/om-configmap.yaml
@@ -32,7 +32,7 @@ data:
       {{- else }}
       format: text
       {{- end }}
-      rpc: false
+      rpc: {{ .Values.global.logging.rpc.enabled }}
     # Open Match applies the exponential backoff strategy for its retryable gRPC calls.
     # The settings below are the default backoff configuration used in Open Match.
     # See https://github.com/cenkalti/backoff/blob/v3/exponential.go for detailed explanations
diff --git a/install/helm/open-match/values.yaml b/install/helm/open-match/values.yaml
index 1c90d6a5..aa9576d7 100644
--- a/install/helm/open-match/values.yaml
+++ b/install/helm/open-match/values.yaml
@@ -153,6 +153,10 @@ global:
     rootca:
       mountPath: /app/secrets/tls/rootca
   
+  logging:
+    rpc:
+      enabled: true
+
   configs:
     global-configmap:
       volumeName: global-config-volume
@@ -166,6 +170,8 @@ global:
   # requires pod-level annotation to customize its scrape path.
   # See definitions in templates/_helpers.tpl - "prometheus.annotations" section for details
   telemetry:
+    zpages:
+      enabled: true
     jaeger:
       enabled: false
       agentEndpoint: "open-match-jaeger-agent:6831"
diff --git a/internal/logging/logging.go b/internal/logging/logging.go
index cd588551..17dfd669 100644
--- a/internal/logging/logging.go
+++ b/internal/logging/logging.go
@@ -16,6 +16,8 @@
 package logging
 
 import (
+	"strings"
+
 	stackdriver "github.com/TV4/logrus-stackdriver-formatter"
 	"github.com/sirupsen/logrus"
 	"open-match.dev/open-match/internal/config"
@@ -24,35 +26,61 @@ import (
 // ConfigureLogging sets up open match logrus instance using the logging section of the matchmaker_config.json
 //  - log line format (text[default] or json)
 //  - min log level to include (debug, info [default], warn, error, fatal, panic)
-//  - include source file and line number for every event (false [default], true)
 func ConfigureLogging(cfg config.View) {
-	switch cfg.GetString("logging.format") {
+	logrus.SetFormatter(newFormatter(cfg.GetString("logging.format")))
+	level := toLevel(cfg.GetString("logging.level"))
+	logrus.SetLevel(level)
+	if isDebugLevel(level) {
+		logrus.Warn("Trace logging level configured. Not recommended for production!")
+	}
+}
+
+func newFormatter(formatter string) logrus.Formatter {
+	switch strings.ToLower(formatter) {
 	case "stackdriver":
-		logrus.SetFormatter(stackdriver.NewFormatter())
+		return stackdriver.NewFormatter()
 	case "json":
-		logrus.SetFormatter(&logrus.JSONFormatter{})
-	case "text":
-	default:
-		logrus.SetFormatter(&logrus.TextFormatter{})
+		return &logrus.JSONFormatter{}
 	}
+	return &logrus.TextFormatter{}
+}
 
-	switch cfg.GetString("logging.level") {
+func toLevel(level string) logrus.Level {
+	switch strings.ToLower(level) {
 	case "trace":
-		logrus.SetLevel(logrus.TraceLevel)
-		logrus.Warn("Trace logging level configured. Not recommended for production!")
+		return logrus.TraceLevel
 	case "debug":
-		logrus.SetLevel(logrus.DebugLevel)
-		logrus.Warn("Debug logging level configured. Not recommended for production!")
+		return logrus.DebugLevel
 	case "warn":
-		logrus.SetLevel(logrus.WarnLevel)
+		fallthrough
+	case "warning":
+		return logrus.WarnLevel
 	case "error":
-		logrus.SetLevel(logrus.ErrorLevel)
+		return logrus.ErrorLevel
 	case "fatal":
-		logrus.SetLevel(logrus.FatalLevel)
+		return logrus.FatalLevel
 	case "panic":
-		logrus.SetLevel(logrus.PanicLevel)
-	case "info":
-	default:
-		logrus.SetLevel(logrus.InfoLevel)
+		return logrus.PanicLevel
+	}
+	return logrus.InfoLevel
+}
+
+// IsDebugEnabled returns true if the logging level is debug or more granular.
+func IsDebugEnabled(cfg config.View) bool {
+	return IsDebugLevel(cfg.GetString("logging.level"))
+}
+
+// IsDebugLevel returns true if the logging level is debug or more granular.
+func IsDebugLevel(level string) bool {
+	return isDebugLevel(toLevel(level))
+}
+
+func isDebugLevel(level logrus.Level) bool {
+	switch level {
+	case logrus.TraceLevel:
+		fallthrough
+	case logrus.DebugLevel:
+		return true
 	}
+	return false
 }
diff --git a/internal/rpc/clients.go b/internal/rpc/clients.go
index dd4d543d..82eda28c 100644
--- a/internal/rpc/clients.go
+++ b/internal/rpc/clients.go
@@ -15,6 +15,7 @@
 package rpc
 
 import (
+	"context"
 	"crypto/tls"
 	"fmt"
 	"io/ioutil"
@@ -36,6 +37,7 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
 	"open-match.dev/open-match/internal/config"
+	"open-match.dev/open-match/internal/logging"
 	"open-match.dev/open-match/internal/telemetry"
 )
 
@@ -55,10 +57,11 @@ var (
 
 // ClientParams contains the connection parameters to connect to an Open Match service.
 type ClientParams struct {
-	Address            string
-	TrustedCertificate []byte
-	EnableRPCLogging   bool
-	EnableMetrics      bool
+	Address                 string
+	TrustedCertificate      []byte
+	EnableRPCLogging        bool
+	EnableRPCPayloadLogging bool
+	EnableMetrics           bool
 }
 
 func (p *ClientParams) usingTLS() bool {
@@ -68,9 +71,10 @@ func (p *ClientParams) usingTLS() bool {
 // GRPCClientFromConfig creates a gRPC client connection from a configuration.
 func GRPCClientFromConfig(cfg config.View, prefix string) (*grpc.ClientConn, error) {
 	clientParams := &ClientParams{
-		Address:          toAddress(cfg.GetString(prefix+".hostname"), cfg.GetInt(prefix+".grpcport")),
-		EnableRPCLogging: cfg.GetBool(ConfigNameEnableRPCLogging),
-		EnableMetrics:    cfg.GetBool(telemetry.ConfigNameEnableMetrics),
+		Address:                 toAddress(cfg.GetString(prefix+".hostname"), cfg.GetInt(prefix+".grpcport")),
+		EnableRPCLogging:        cfg.GetBool(ConfigNameEnableRPCLogging),
+		EnableRPCPayloadLogging: logging.IsDebugEnabled(cfg),
+		EnableMetrics:           cfg.GetBool(telemetry.ConfigNameEnableMetrics),
 	}
 
 	// If TLS support is enabled in the config, fill in the trusted certificates for decrpting server certificate.
@@ -94,7 +98,7 @@ func GRPCClientFromConfig(cfg config.View, prefix string) (*grpc.ClientConn, err
 // GRPCClientFromEndpoint creates a gRPC client connection from endpoint.
 func GRPCClientFromEndpoint(cfg config.View, address string) (*grpc.ClientConn, error) {
 	// TODO: investigate if it is possible to keep a cache of the certpool and transport credentials
-	grpcOptions := newGRPCDialOptions(cfg.GetBool(telemetry.ConfigNameEnableMetrics), cfg.GetBool(ConfigNameEnableRPCLogging))
+	grpcOptions := newGRPCDialOptions(cfg.GetBool(telemetry.ConfigNameEnableMetrics), cfg.GetBool(ConfigNameEnableRPCLogging), logging.IsDebugEnabled(cfg))
 
 	if cfg.GetString(configNameClientTrustedCertificatePath) != "" {
 		_, err := os.Stat(cfg.GetString(configNameClientTrustedCertificatePath))
@@ -126,7 +130,7 @@ func GRPCClientFromEndpoint(cfg config.View, address string) (*grpc.ClientConn,
 
 // GRPCClientFromParams creates a gRPC client connection from the parameters.
 func GRPCClientFromParams(params *ClientParams) (*grpc.ClientConn, error) {
-	grpcOptions := newGRPCDialOptions(params.EnableMetrics, params.EnableRPCLogging)
+	grpcOptions := newGRPCDialOptions(params.EnableMetrics, params.EnableRPCLogging, params.EnableRPCPayloadLogging)
 
 	if params.usingTLS() {
 		trustedCertPool, err := trustedCertificateFromFileData(params.TrustedCertificate)
@@ -145,9 +149,10 @@ func GRPCClientFromParams(params *ClientParams) (*grpc.ClientConn, error) {
 // HTTPClientFromConfig creates a HTTP client from from a configuration.
 func HTTPClientFromConfig(cfg config.View, prefix string) (*http.Client, string, error) {
 	clientParams := &ClientParams{
-		Address:          toAddress(cfg.GetString(prefix+".hostname"), cfg.GetInt(prefix+".httpport")),
-		EnableRPCLogging: cfg.GetBool(ConfigNameEnableRPCLogging),
-		EnableMetrics:    cfg.GetBool(telemetry.ConfigNameEnableMetrics),
+		Address:                 toAddress(cfg.GetString(prefix+".hostname"), cfg.GetInt(prefix+".httpport")),
+		EnableRPCLogging:        cfg.GetBool(ConfigNameEnableRPCLogging),
+		EnableRPCPayloadLogging: logging.IsDebugEnabled(cfg),
+		EnableMetrics:           cfg.GetBool(telemetry.ConfigNameEnableMetrics),
 	}
 
 	// If TLS support is enabled in the config, fill in the trusted certificates for decrpting server certificate.
@@ -189,9 +194,10 @@ func sanitizeHTTPAddress(address string, preferHTTPS bool) (string, error) {
 func HTTPClientFromEndpoint(cfg config.View, address string) (*http.Client, string, error) {
 	// TODO: investigate if it is possible to keep a cache of the certpool and transport credentials
 	params := &ClientParams{
-		Address:          address,
-		EnableRPCLogging: cfg.GetBool(ConfigNameEnableRPCLogging),
-		EnableMetrics:    cfg.GetBool(telemetry.ConfigNameEnableMetrics),
+		Address:                 address,
+		EnableRPCLogging:        cfg.GetBool(ConfigNameEnableRPCLogging),
+		EnableRPCPayloadLogging: logging.IsDebugEnabled(cfg),
+		EnableMetrics:           cfg.GetBool(telemetry.ConfigNameEnableMetrics),
 	}
 	if cfg.GetString(configNameClientTrustedCertificatePath) != "" {
 		_, err := os.Stat(cfg.GetString(configNameClientTrustedCertificatePath))
@@ -248,7 +254,8 @@ func HTTPClientFromParams(params *ClientParams) (*http.Client, string, error) {
 	if params.EnableRPCLogging {
 		attachTransport(httpClient, func(transport http.RoundTripper) http.RoundTripper {
 			return &loggingHTTPClient{
-				transport: transport,
+				transport:   transport,
+				logPayloads: params.EnableRPCPayloadLogging,
 			}
 		})
 	}
@@ -264,7 +271,7 @@ func HTTPClientFromParams(params *ClientParams) (*http.Client, string, error) {
 	return httpClient, baseURL, nil
 }
 
-func newGRPCDialOptions(enableMetrics bool, enableRPCLogging bool) []grpc.DialOption {
+func newGRPCDialOptions(enableMetrics bool, enableRPCLogging bool, enableRPCPayloadLogging bool) []grpc.DialOption {
 	si := []grpc.StreamClientInterceptor{
 		grpc_retry.StreamClientInterceptor(),
 	}
@@ -276,8 +283,17 @@ func newGRPCDialOptions(enableMetrics bool, enableRPCLogging bool) []grpc.DialOp
 			"app":       "openmatch",
 			"component": "grpc.client",
 		})
-		si = append(si, grpc_logrus.StreamClientInterceptor(grpcLogger))
-		ui = append(ui, grpc_logrus.UnaryClientInterceptor(grpcLogger))
+		grpcLogger.Level = logrus.DebugLevel
+		if enableRPCPayloadLogging {
+			logEverythingFromClient := func(_ context.Context, _ string) bool {
+				return true
+			}
+			si = append(si, grpc_logrus.PayloadStreamClientInterceptor(grpcLogger, logEverythingFromClient))
+			ui = append(ui, grpc_logrus.PayloadUnaryClientInterceptor(grpcLogger, logEverythingFromClient))
+		} else {
+			si = append(si, grpc_logrus.StreamClientInterceptor(grpcLogger))
+			ui = append(ui, grpc_logrus.UnaryClientInterceptor(grpcLogger))
+		}
 	}
 	opts := []grpc.DialOption{
 		grpc.WithStreamInterceptor(grpc_middleware.ChainStreamClient(si...)),
@@ -294,31 +310,32 @@ func toAddress(hostname string, port int) string {
 }
 
 type loggingHTTPClient struct {
-	transport http.RoundTripper
+	transport   http.RoundTripper
+	logPayloads bool
 }
 
 func (c *loggingHTTPClient) RoundTrip(req *http.Request) (*http.Response, error) {
-	dumpReqLog, dumpReqErr := httputil.DumpRequest(req, true)
+	dumpReqLog, dumpReqErr := httputil.DumpRequest(req, c.logPayloads)
 	fields := logrus.Fields{
 		"method": req.Method,
 		"url":    req.URL,
 		"proto":  req.Proto,
 	}
 	if dumpReqErr == nil {
-		clientLogger.WithFields(fields).Info(string(dumpReqLog))
+		clientLogger.WithFields(fields).Debug(string(dumpReqLog))
 	} else {
-		clientLogger.WithError(dumpReqErr).WithFields(fields).Info("cannot dump request")
+		clientLogger.WithError(dumpReqErr).WithFields(fields).Debug("cannot dump request")
 	}
 	resp, err := c.transport.RoundTrip(req)
 	if err == nil {
-		dumpRespLog, dumpRespErr := httputil.DumpResponse(resp, true)
+		dumpRespLog, dumpRespErr := httputil.DumpResponse(resp, c.logPayloads)
 		if dumpRespErr == nil {
-			clientLogger.WithFields(fields).Info(string(dumpRespLog))
+			clientLogger.WithFields(fields).Debug(string(dumpRespLog))
 		} else {
-			clientLogger.WithError(dumpRespErr).WithFields(fields).Info("request was successful but cannot dump response")
+			clientLogger.WithError(dumpRespErr).WithFields(fields).Debug("request was successful but cannot dump response")
 		}
 	} else {
-		clientLogger.WithError(err).WithFields(fields).Info("request failed")
+		clientLogger.WithError(err).WithFields(fields).Debug("request failed")
 	}
 	return resp, err
 }
diff --git a/internal/rpc/insecure.go b/internal/rpc/insecure.go
index 8381b371..cce23fcd 100644
--- a/internal/rpc/insecure.go
+++ b/internal/rpc/insecure.go
@@ -79,7 +79,7 @@ func (s *insecureServer) start(params *ServerParams) (func(), error) {
 	ctx, cancel := context.WithCancel(context.Background())
 
 	for _, handlerFunc := range params.handlersForGrpcProxy {
-		dialOpts := newGRPCDialOptions(params.enableMetrics, params.enableRPCLogging)
+		dialOpts := newGRPCDialOptions(params.enableMetrics, params.enableRPCLogging, params.enableRPCPayloadLogging)
 		dialOpts = append(dialOpts, grpc.WithInsecure())
 		if err = handlerFunc(ctx, s.proxyMux, grpcListener.Addr().String(), dialOpts); err != nil {
 			cancel()
diff --git a/internal/rpc/server.go b/internal/rpc/server.go
index 6479eabe..625cbf00 100644
--- a/internal/rpc/server.go
+++ b/internal/rpc/server.go
@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"net/http/httputil"
 
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
@@ -32,6 +33,7 @@ import (
 	"go.opencensus.io/plugin/ochttp/propagation/b3"
 	"google.golang.org/grpc"
 	"open-match.dev/open-match/internal/config"
+	"open-match.dev/open-match/internal/logging"
 	"open-match.dev/open-match/internal/signal"
 	"open-match.dev/open-match/internal/telemetry"
 )
@@ -75,9 +77,10 @@ type ServerParams struct {
 	// Private key in PEM format.
 	privateKeyFileData []byte
 
-	enableRPCLogging bool
-	enableMetrics    bool
-	closer           func()
+	enableRPCLogging        bool
+	enableRPCPayloadLogging bool
+	enableMetrics           bool
+	closer                  func()
 }
 
 // NewServerParamsFromConfig returns server Params initialized from the configuration file.
@@ -131,6 +134,7 @@ func NewServerParamsFromConfig(cfg config.View, prefix string) (*ServerParams, e
 
 	p.enableMetrics = cfg.GetBool(telemetry.ConfigNameEnableMetrics)
 	p.enableRPCLogging = cfg.GetBool(ConfigNameEnableRPCLogging)
+	p.enableRPCPayloadLogging = logging.IsDebugEnabled(cfg)
 	// TODO: This isn't ideal since telemetry requires config for it to be initialized.
 	// This forces us to initialize readiness probes earlier than necessary.
 	p.closer = telemetry.Setup(p.ServeMux, cfg)
@@ -259,13 +263,39 @@ func MustServeForever(params *ServerParams) {
 	serveUntilKilledFunc()
 }
 
+type loggingHTTPHandler struct {
+	handler     http.Handler
+	logPayloads bool
+}
+
+func (l *loggingHTTPHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	dumpReqLog, dumpReqErr := httputil.DumpRequest(req, l.logPayloads)
+	fields := logrus.Fields{
+		"method": req.Method,
+		"url":    req.URL,
+		"proto":  req.Proto,
+	}
+	if dumpReqErr == nil {
+		serverLogger.WithFields(fields).Debug(string(dumpReqLog))
+	} else {
+		serverLogger.WithError(dumpReqErr).WithFields(fields).Debug("cannot dump request")
+	}
+	l.handler.ServeHTTP(w, req)
+}
+
 func instrumentHTTPHandler(handler http.Handler, params *ServerParams) http.Handler {
 	if params.enableMetrics {
-		return &ochttp.Handler{
+		handler = &ochttp.Handler{
 			Handler:     handler,
 			Propagation: &b3.HTTPFormat{},
 		}
 	}
+	if params.enableRPCLogging {
+		handler = &loggingHTTPHandler{
+			handler:     handler,
+			logPayloads: params.enableRPCPayloadLogging,
+		}
+	}
 	return handler
 }
 
@@ -284,8 +314,17 @@ func newGRPCServerOptions(params *ServerParams) []grpc.ServerOption {
 			"app":       "openmatch",
 			"component": "grpc.server",
 		})
-		si = append(si, grpc_logrus.StreamServerInterceptor(grpcLogger))
-		ui = append(ui, grpc_logrus.UnaryServerInterceptor(grpcLogger))
+		grpcLogger.Level = logrus.DebugLevel
+		if params.enableRPCPayloadLogging {
+			logEverythingFromServer := func(_ context.Context, _ string, _ interface{}) bool {
+				return true
+			}
+			si = append(si, grpc_logrus.PayloadStreamServerInterceptor(grpcLogger, logEverythingFromServer))
+			ui = append(ui, grpc_logrus.PayloadUnaryServerInterceptor(grpcLogger, logEverythingFromServer))
+		} else {
+			si = append(si, grpc_logrus.StreamServerInterceptor(grpcLogger))
+			ui = append(ui, grpc_logrus.UnaryServerInterceptor(grpcLogger))
+		}
 	}
 
 	if params.enableMetrics {
diff --git a/internal/rpc/tls_server.go b/internal/rpc/tls_server.go
index 0e9eda6f..da0bd630 100644
--- a/internal/rpc/tls_server.go
+++ b/internal/rpc/tls_server.go
@@ -103,7 +103,7 @@ func (s *tlsServer) start(params *ServerParams) (func(), error) {
 	// Bind gRPC handlers
 	ctx, cancel := context.WithCancel(context.Background())
 
-	httpsToGrpcProxyOptions := newGRPCDialOptions(params.enableMetrics, params.enableRPCLogging)
+	httpsToGrpcProxyOptions := newGRPCDialOptions(params.enableMetrics, params.enableRPCLogging, params.enableRPCPayloadLogging)
 	httpsToGrpcProxyOptions = append(httpsToGrpcProxyOptions, grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(certPoolForGrpcEndpoint, "")))
 
 	for _, handlerFunc := range params.handlersForGrpcProxy {


Test patch:
diff --git a/internal/logging/logging_test.go b/internal/logging/logging_test.go
new file mode 100644
index 00000000..89a0494f
--- /dev/null
+++ b/internal/logging/logging_test.go
@@ -0,0 +1,95 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package logging
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+
+	stackdriver "github.com/TV4/logrus-stackdriver-formatter"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewFormatter(t *testing.T) {
+	testCases := []struct {
+		in       string
+		expected interface{}
+	}{
+		{"", &logrus.TextFormatter{}},
+		{"json", &logrus.JSONFormatter{}},
+		{"stackdriver", stackdriver.NewFormatter()},
+		{"text", &logrus.TextFormatter{}},
+	}
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(fmt.Sprintf("newFormatter(%s) => %s", tc.in, tc.expected), func(t *testing.T) {
+			assert := assert.New(t)
+			actual := newFormatter(tc.in)
+			assert.Equal(reflect.TypeOf(tc.expected), reflect.TypeOf(actual))
+		})
+	}
+}
+
+func TestIsDebugLevel(t *testing.T) {
+	testCases := []struct {
+		in       logrus.Level
+		expected bool
+	}{
+		{logrus.TraceLevel, true},
+		{logrus.DebugLevel, true},
+		{logrus.InfoLevel, false},
+		{logrus.WarnLevel, false},
+		{logrus.ErrorLevel, false},
+		{logrus.FatalLevel, false},
+		{logrus.PanicLevel, false},
+	}
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(fmt.Sprintf("isDebugLevel(%s) => %t", tc.in, tc.expected), func(t *testing.T) {
+			assert := assert.New(t)
+			actual := isDebugLevel(tc.in)
+			assert.Equal(tc.expected, actual)
+		})
+	}
+}
+
+func TestToLevel(t *testing.T) {
+	testCases := []struct {
+		in       string
+		expected logrus.Level
+	}{
+		{"trace", logrus.TraceLevel},
+		{"debug", logrus.DebugLevel},
+		{"info", logrus.InfoLevel},
+		{"warn", logrus.WarnLevel},
+		{"warning", logrus.WarnLevel},
+		{"error", logrus.ErrorLevel},
+		{"fatal", logrus.FatalLevel},
+		{"panic", logrus.PanicLevel},
+		{"info", logrus.InfoLevel},
+		{"", logrus.InfoLevel},
+		{"nothing", logrus.InfoLevel},
+	}
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(fmt.Sprintf("toLevel(%s) => %s", tc.in, tc.expected), func(t *testing.T) {
+			assert := assert.New(t)
+			actual := toLevel(tc.in)
+			assert.Equal(tc.expected, actual)
+		})
+	}
+}
diff --git a/internal/rpc/testing/helpers.go b/internal/rpc/testing/helpers.go
index 6d883c16..5da2cb62 100644
--- a/internal/rpc/testing/helpers.go
+++ b/internal/rpc/testing/helpers.go
@@ -143,10 +143,11 @@ func (tc *TestContext) MustHTTP() (*http.Client, string) {
 
 func (tc *TestContext) newClientParams(address string) *rpc.ClientParams {
 	return &rpc.ClientParams{
-		Address:            address,
-		TrustedCertificate: tc.trustedCertificate,
-		EnableRPCLogging:   true,
-		EnableMetrics:      false,
+		Address:                 address,
+		TrustedCertificate:      tc.trustedCertificate,
+		EnableRPCLogging:        true,
+		EnableRPCPayloadLogging: true,
+		EnableMetrics:           false,
 	}
 }
 
diff --git a/internal/testing/e2e/cluster.go b/internal/testing/e2e/cluster.go
index 652e7bdc..55946595 100644
--- a/internal/testing/e2e/cluster.go
+++ b/internal/testing/e2e/cluster.go
@@ -33,6 +33,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	"k8s.io/client-go/tools/clientcmd"
+	"open-match.dev/open-match/internal/logging"
 	"open-match.dev/open-match/internal/rpc"
 	"open-match.dev/open-match/internal/util"
 	pb "open-match.dev/open-match/pkg/pb"
@@ -128,9 +129,10 @@ func (com *clusterOM) getHTTPAddressFromServiceName(serviceName string) (string,
 func (com *clusterOM) getGRPCClientFromServiceName(serviceName string) (*grpc.ClientConn, error) {
 	ipAddress, port := com.getGRPCAddressFromServiceName(serviceName)
 	conn, err := rpc.GRPCClientFromParams(&rpc.ClientParams{
-		Address:          fmt.Sprintf("%s:%d", ipAddress, int(port)),
-		EnableRPCLogging: *testOnlyEnableRPCLoggingFlag,
-		EnableMetrics:    *testOnlyEnableMetrics,
+		Address:                 fmt.Sprintf("%s:%d", ipAddress, int(port)),
+		EnableRPCLogging:        *testOnlyEnableRPCLoggingFlag,
+		EnableRPCPayloadLogging: logging.IsDebugLevel(*testOnlyLoggingLevel),
+		EnableMetrics:           *testOnlyEnableMetrics,
 	})
 	if err != nil {
 		return nil, errors.Wrapf(err, "cannot connect to gRPC %s:%d", ipAddress, port)
diff --git a/internal/testing/e2e/common.go b/internal/testing/e2e/common.go
index 47353bf3..8dfb3e05 100644
--- a/internal/testing/e2e/common.go
+++ b/internal/testing/e2e/common.go
@@ -27,6 +27,7 @@ import (
 var (
 	testOnlyEnableMetrics        = flag.Bool("test_only_metrics", true, "Enables metrics exporting for tests.")
 	testOnlyEnableRPCLoggingFlag = flag.Bool("test_only_rpc_logging", false, "Enables RPC Logging for tests. This output is very verbose.")
+	testOnlyLoggingLevel         = flag.String("test_only_log_level", "info", "Sets the log level for tests.")
 )
 
 // OM is the interface for communicating with Open Match.
diff --git a/internal/testing/e2e/in_memory.go b/internal/testing/e2e/in_memory.go
index ce3881e1..af2966da 100644
--- a/internal/testing/e2e/in_memory.go
+++ b/internal/testing/e2e/in_memory.go
@@ -144,6 +144,7 @@ func createMinimatchForTest(t *testing.T, evalTc *rpcTesting.TestContext) *rpcTe
 	cfg.Set("api.evaluator.httpport", evalTc.GetHTTPPort())
 	cfg.Set("synchronizer.enabled", true)
 	cfg.Set(rpc.ConfigNameEnableRPCLogging, *testOnlyEnableRPCLoggingFlag)
+	cfg.Set("logging.level", *testOnlyLoggingLevel)
 	cfg.Set(telemetry.ConfigNameEnableMetrics, *testOnlyEnableMetrics)
 
 	// TODO: This is very ugly. Need a better story around closing resources.