Repository: envoyproxy/gateway

Issue:
OpenTelemetry access log sink failure with "otel.Text is nil" error (EG v1.3.1)
*Description*:
After upgrading our development clusters to v1.3.1 of Envoy Gateway, we noticed that none of our HTTPRoutes in the cluster were not registering to the proxy listeners, making all services unavailable. This was validated with `egctl config envoy-proxy listener` and `... routes`, showing that none of the configured routes were registered.

```
❯ curl https://<url> -vvv
* Host <url>:443 was resolved.
* IPv6: (none)
* IPv4: ...
*   Trying ...
* Connected to ... port 443
* ALPN: curl offers h2,http/1.1
* (304) (OUT), TLS handshake, Client hello (1):
*  CAfile: /etc/ssl/cert.pem
*  CApath: none
* LibreSSL SSL_connect: SSL_ERROR_SYSCALL in connection to ...:443
* Closing connection
curl: (35) LibreSSL SSL_connect: SSL_ERROR_SYSCALL in connection to ...:443 

❯ openssl s_client -connect ...:443
Connecting to ...
CONNECTED(00000005)
4088A9F501000000:error:0A000126:SSL routines::unexpected eof while reading:ssl/record/rec_layer_s3.c:693:
---
no peer certificate available
---
No client certificate CA names sent
---
SSL handshake has read 0 bytes and written 343 bytes
Verification: OK
---
New, (NONE), Cipher is (NONE)
Protocol: TLSv1.3
This TLS version forbids renegotiation.
Compression: NONE
Expansion: NONE
No ALPN negotiated
Early data was not sent
Verify return code: 0 (ok)
---
```

In the envoy logs, these log lines were visible:

```
2025-03-12T11:13:00.577Z	ERROR	xds-translator	runner/runner.go:85	failed to translate xds ir	{"runner": "xds-translator", "error": "otel.Text is nil\notel.Text is nil"}
2025-03-12T11:13:00.577Z	INFO	provider.demo-keyauth.demo-keyauth	kubernetes/status_updater.go:109	status unchanged, bypassing update	{"runner": "provider"}
2025-03-12T11:13:00.577Z	INFO	provider	kubernetes/status_updater.go:145	received a status update	{"runner": "provider", "namespace": "<namespace>", "name": "<name>"}
2025-03-12T11:13:00.577Z	ERROR	watchable	message/watchutil.go:80	observed an error	{"runner": "xds-translator", "error": "otel.Text is nil\notel.Text is nil"}
```

After disabling the `openTelemetry` sink from the proxy access log config, the routes were configured and services became accessible.

The lines generating the logs seems to originate here: https://github.com/envoyproxy/gateway/blob/df2bb96c32604554f50ec08702cbea834ce968f5/internal/xds/translator/accesslog.go#L339-L341

When downgrading back to v1.3.0, the existing telemetry sink configuration works again.

*Repro steps*:

Envoy proxy configmap includes following telemetry config:

```
...
  telemetry:
    accessLog:
      settings:
      - format:
          json:
            ...
          type: JSON
        sinks:
        - openTelemetry:
            host: telemetry-opentelemetry-collector.telemetry.svc.cluster.local
            port: 4317
            resources:
              k8s.cluster.name: <cluster-name>
          type: OpenTelemetry
        - file:
            path: /dev/stdout
          type: File
```

*Environment*:

`envoy-gateway v1.3.1`, templated and installed via latest Helm Chart.

*Logs*:

Included in description.

Hints from issue comments:
@zirain: cc @guydc 

Solution patch:
diff --git a/internal/xds/translator/accesslog.go b/internal/xds/translator/accesslog.go
index 991c8e342..304f22da4 100644
--- a/internal/xds/translator/accesslog.go
+++ b/internal/xds/translator/accesslog.go
@@ -336,10 +336,10 @@ func buildXdsAccessLog(al *ir.AccessLog, accessLogType ir.ProxyAccessLogType) ([
 			ResourceAttributes: convertToKeyValueList(otel.Resources, false),
 		}
 
-		if otel.Text == nil {
-			return nil, errors.New("otel.Text is nil")
+		var format string
+		if otel.Text != nil && *otel.Text != "" {
+			format = *otel.Text
 		}
-		format := *otel.Text
 
 		if format != "" {
 			al.Body = &otlpcommonv1.AnyValue{
@@ -349,9 +349,17 @@ func buildXdsAccessLog(al *ir.AccessLog, accessLogType ir.ProxyAccessLogType) ([
 			}
 		}
 
-		al.Attributes = convertToKeyValueList(otel.Attributes, true)
+		var attrs map[string]string
+		if len(otel.Attributes) != 0 {
+			attrs = otel.Attributes
+		} else if len(otel.Attributes) == 0 && format == "" {
+			// if there are no attributes and text format is unset, use the default EnvoyJSONLogFields
+			attrs = EnvoyJSONLogFields
+		}
+
+		al.Attributes = convertToKeyValueList(attrs, true)
+		formatters := accessLogOpenTelemetryFormatters(format, attrs)
 
-		formatters := accessLogOpenTelemetryFormatters(format, otel.Attributes)
 		if len(formatters) != 0 {
 			al.Formatters = formatters
 		}
diff --git a/release-notes/current.yaml b/release-notes/current.yaml
index b332fc830..3b02988dc 100644
--- a/release-notes/current.yaml
+++ b/release-notes/current.yaml
@@ -36,6 +36,7 @@ bug fixes: |
   Added support for BackendTLSPolicy and EnvoyExtensionPolicy parsing in Standalone mode.
   Retrigger reconciliation when backendRef of type ServiceImport is updated or when EndpointSlice(s) for a ServiceImport are updated.
   Fix not logging an error and returning it in the K8s Reconcile method when a GatewayClass is not accepted.
+  Fix allowing empty text field for opentelemetry sink when using JSON format.
 
 # Enhancements that improve performance.
 performance improvements: |



Test patch:
diff --git a/test/e2e/testdata/accesslog-otel-default.yaml b/test/e2e/testdata/accesslog-otel-default.yaml
new file mode 100644
index 000000000..764e8e02e
--- /dev/null
+++ b/test/e2e/testdata/accesslog-otel-default.yaml
@@ -0,0 +1,61 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: accesslog-gtw
+  namespace: gateway-conformance-infra
+spec:
+  gatewayClassName: {GATEWAY_CLASS_NAME}
+  listeners:
+    - name: http
+      port: 80
+      protocol: HTTP
+      allowedRoutes:
+        namespaces:
+          from: Same
+  infrastructure:
+    parametersRef:
+      group: gateway.envoyproxy.io
+      kind: EnvoyProxy
+      name: accesslog-otel
+---
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: EnvoyProxy
+metadata:
+  name: accesslog-otel
+  namespace: gateway-conformance-infra
+spec:
+  ipFamily: IPv4
+  telemetry:
+    accessLog:
+      settings:
+        - matches:
+            - "'x-envoy-logged' in request.headers"
+          sinks:
+            - type: OpenTelemetry
+              openTelemetry:
+                backendRefs:
+                  - name: otel-collector
+                    namespace: monitoring
+                    port: 4317
+                resources:
+                  k8s.cluster.name: "envoy-gateway"
+  shutdown:
+    drainTimeout: 5s
+    minDrainDuration: 1s
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: accesslog-otel
+  namespace: gateway-conformance-infra
+spec:
+  parentRefs:
+    - name: accesslog-gtw
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /otel
+      backendRefs:
+        - name: infra-backend-v2
+          port: 8080
diff --git a/test/e2e/testdata/accesslog-otel-json.yaml b/test/e2e/testdata/accesslog-otel-json.yaml
new file mode 100644
index 000000000..219315112
--- /dev/null
+++ b/test/e2e/testdata/accesslog-otel-json.yaml
@@ -0,0 +1,79 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: accesslog-gtw
+  namespace: gateway-conformance-infra
+spec:
+  gatewayClassName: {GATEWAY_CLASS_NAME}
+  listeners:
+    - name: http
+      port: 80
+      protocol: HTTP
+      allowedRoutes:
+        namespaces:
+          from: Same
+  infrastructure:
+    parametersRef:
+      group: gateway.envoyproxy.io
+      kind: EnvoyProxy
+      name: accesslog-otel
+---
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: EnvoyProxy
+metadata:
+  name: accesslog-otel
+  namespace: gateway-conformance-infra
+spec:
+  ipFamily: IPv4
+  telemetry:
+    accessLog:
+      settings:
+        - format:
+            type: JSON
+            json:
+              time: "%START_TIME%"
+              method: "%REQ(:METHOD)%"
+              metadata: "%METADATA(ROUTE:envoy-gateway:resources)%"
+              x-envoy-original-path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%"
+              protocol: "%PROTOCOL%"
+              responseCode: "%RESPONSE_CODE%"
+              responseFlags: "%RESPONSE_FLAGS%"
+              bytesReceived: "%BYTES_RECEIVED%"
+              bytesSent: "%BYTES_SENT%"
+              duration: "%DURATION%"
+              x-forwarded-for: "%REQ(X-FORWARDED-FOR)%"
+              user-agent: "%REQ(USER-AGENT)%"
+              x-request-id: "%REQ(X-REQUEST-ID)%"
+              authority: "%REQ(:AUTHORITY)%"
+              upstreamHost: "%UPSTREAM_HOST%"
+          matches:
+            - "'x-envoy-logged' in request.headers"
+          sinks:
+            - type: OpenTelemetry
+              openTelemetry:
+                backendRefs:
+                  - name: otel-collector
+                    namespace: monitoring
+                    port: 4317
+                resources:
+                  k8s.cluster.name: "envoy-gateway"
+  shutdown:
+    drainTimeout: 5s
+    minDrainDuration: 1s
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: accesslog-otel
+  namespace: gateway-conformance-infra
+spec:
+  parentRefs:
+    - name: accesslog-gtw
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /otel
+      backendRefs:
+        - name: infra-backend-v2
+          port: 8080
diff --git a/test/e2e/tests/accesslog.go b/test/e2e/tests/accesslog.go
index ea5773495..5df762868 100644
--- a/test/e2e/tests/accesslog.go
+++ b/test/e2e/tests/accesslog.go
@@ -21,7 +21,7 @@ import (
 )
 
 func init() {
-	ConformanceTests = append(ConformanceTests, FileAccessLogTest, OpenTelemetryTest, ALSTest)
+	ConformanceTests = append(ConformanceTests, FileAccessLogTest, OpenTelemetryTestText, OpenTelemetryTestJSON, ALSTest, OpenTelemetryTestJSONAsDefault)
 }
 
 var FileAccessLogTest = suite.ConformanceTest{
@@ -116,9 +116,9 @@ var FileAccessLogTest = suite.ConformanceTest{
 	},
 }
 
-var OpenTelemetryTest = suite.ConformanceTest{
-	ShortName:   "OpenTelemetryAccessLog",
-	Description: "Make sure OpenTelemetry access log is working",
+var OpenTelemetryTestText = suite.ConformanceTest{
+	ShortName:   "OpenTelemetryTextAccessLog",
+	Description: "Make sure OpenTelemetry text access log is working",
 	Manifests:   []string{"testdata/accesslog-otel.yaml"},
 	Test: func(t *testing.T, suite *suite.ConformanceTestSuite) {
 		labels := map[string]string{
@@ -169,6 +169,112 @@ var OpenTelemetryTest = suite.ConformanceTest{
 	},
 }
 
+var OpenTelemetryTestJSONAsDefault = suite.ConformanceTest{
+	ShortName:   "OpenTelemetryAccessLogJSONAsDefault",
+	Description: "Make sure OpenTelemetry JSON access log is working as default when no format or type is specified",
+	Manifests:   []string{"testdata/accesslog-otel-default.yaml"},
+	Test: func(t *testing.T, suite *suite.ConformanceTestSuite) {
+		labels := map[string]string{
+			"k8s_namespace_name": "envoy-gateway-system",
+			"exporter":           "OTLP",
+		}
+
+		ns := "gateway-conformance-infra"
+		routeNN := types.NamespacedName{Name: "accesslog-otel", Namespace: ns}
+		gwNN := types.NamespacedName{Name: "accesslog-gtw", Namespace: ns}
+		gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN)
+
+		t.Run("Positive", func(t *testing.T) {
+			expectedResponse := httputils.ExpectedResponse{
+				Request: httputils.Request{
+					Path: "/otel",
+					Headers: map[string]string{
+						"x-envoy-logged": "1",
+					},
+				},
+				Response: httputils.Response{
+					StatusCode: 200,
+				},
+				Namespace: ns,
+			}
+			// make sure listener is ready
+			httputils.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expectedResponse)
+
+			runLogTest(t, suite, gwAddr, expectedResponse, labels, "", 1)
+		})
+
+		t.Run("Negative", func(t *testing.T) {
+			expectedResponse := httputils.ExpectedResponse{
+				Request: httputils.Request{
+					Path: "/otel",
+					// envoy will not log this request without the header x-envoy-logged
+				},
+				Response: httputils.Response{
+					StatusCode: 200,
+				},
+				Namespace: ns,
+			}
+			// make sure listener is ready
+			httputils.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expectedResponse)
+
+			runLogTest(t, suite, gwAddr, expectedResponse, labels, "", 0)
+		})
+	},
+}
+
+var OpenTelemetryTestJSON = suite.ConformanceTest{
+	ShortName:   "OpenTelemetryAccessLogJSON",
+	Description: "Make sure OpenTelemetry JSON access log is working with custom JSON attributes",
+	Manifests:   []string{"testdata/accesslog-otel-json.yaml"},
+	Test: func(t *testing.T, suite *suite.ConformanceTestSuite) {
+		labels := map[string]string{
+			"k8s_namespace_name": "envoy-gateway-system",
+			"exporter":           "OTLP",
+		}
+
+		ns := "gateway-conformance-infra"
+		routeNN := types.NamespacedName{Name: "accesslog-otel", Namespace: ns}
+		gwNN := types.NamespacedName{Name: "accesslog-gtw", Namespace: ns}
+		gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN)
+
+		t.Run("Positive", func(t *testing.T) {
+			expectedResponse := httputils.ExpectedResponse{
+				Request: httputils.Request{
+					Path: "/otel",
+					Headers: map[string]string{
+						"x-envoy-logged": "1",
+					},
+				},
+				Response: httputils.Response{
+					StatusCode: 200,
+				},
+				Namespace: ns,
+			}
+			// make sure listener is ready
+			httputils.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expectedResponse)
+
+			runLogTest(t, suite, gwAddr, expectedResponse, labels, "", 1)
+		})
+
+		t.Run("Negative", func(t *testing.T) {
+			expectedResponse := httputils.ExpectedResponse{
+				Request: httputils.Request{
+					Path: "/otel",
+					// envoy will not log this request without the header x-envoy-logged
+				},
+				Response: httputils.Response{
+					StatusCode: 200,
+				},
+				Namespace: ns,
+			}
+			// make sure listener is ready
+			httputils.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expectedResponse)
+
+			runLogTest(t, suite, gwAddr, expectedResponse, labels, "", 0)
+		})
+	},
+}
+
 var ALSTest = suite.ConformanceTest{
 	ShortName:   "ALS",
 	Description: "Make sure ALS access log is working",