Repository: viamrobotics/rdk
Pull request #4939: APP-8080: Fixing RDK crash when processing logs with certain data types
Author: login: jmatth
Labels: name: safe to test
description: This pull request is marked safe to test from a trusted zone
Created: 2025-04-25T19:03:07Z | Merged: 2025-04-28T18:36:13Z

The server was crashing when modules sent logs containing certain types. This seems to be because zap enforces some invariants on `zap.Field` instances that we break when serializing them to protobuf. This PR:

1) Adds code to sanitize incoming log fields so they can be printed.
2) Adds a recover just above where we convert those fields into json to be printed so if we break zap again in the future we at least won't crash the entire server.


Diff:
diff --git a/logging/appender.go b/logging/appender.go
index 317524b4eab..78e0307845d 100644
--- a/logging/appender.go
+++ b/logging/appender.go
@@ -1,6 +1,7 @@
 package logging
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
@@ -61,10 +62,23 @@ func NewFileAppender(filename string) (Appender, io.Closer) {
 
 // ZapcoreFieldsToJSON will serialize the Field objects into a JSON map of key/value pairs. It's
 // unclear what circumstances will result in an error being returned.
-func ZapcoreFieldsToJSON(fields []zapcore.Field) (string, error) {
+func ZapcoreFieldsToJSON(fields []zapcore.Field) (result string, err error) {
 	// Use zap's json encoder which will encode our slice of fields in-order. As opposed to the
 	// random iteration order of a map. Call it with an empty Entry object such that only the fields
 	// become "map-ified".
+	// The json encoder can panic if there is a mismatch between the value in zapcore.Field.Type and
+	// the data in the other fields, which happens in several cases as a result of proto serialization.
+	// We attempt to sanitize incoming data in FieldFromProto, but recover here in case something slips
+	// through to avoid crashing the entire goroutine.
+	defer func() {
+		if r := recover(); r != nil {
+			if perr, ok := r.(error); ok {
+				err = fmt.Errorf("panic serializing log fields: %w", perr)
+				return
+			}
+			err = fmt.Errorf("panic serializing log fields: %v", r)
+		}
+	}()
 	jsonEncoder := zapcore.NewJSONEncoder(zapcore.EncoderConfig{SkipLineEnding: true})
 	buf, err := jsonEncoder.EncodeEntry(zapcore.Entry{}, fields)
 	if err != nil {
@@ -94,9 +108,16 @@ func (appender ConsoleAppender) Write(entry zapcore.Entry, fields []zapcore.Fiel
 
 	fieldsJSON, err := ZapcoreFieldsToJSON(fields)
 	if err != nil {
-		fmt.Fprintln(appender.Writer, strings.Join(toPrint, "\t")) //nolint:errcheck
+		errJSON, err := json.Marshal(map[string]string{"logging_err": err.Error()})
+		if err != nil {
+			// This should never happen but append the raw sting as a last resort just in case.
+			toPrint = append(toPrint, err.Error())
+		} else {
+			toPrint = append(toPrint, string(errJSON))
+		}
+	} else {
+		toPrint = append(toPrint, fieldsJSON)
 	}
-	toPrint = append(toPrint, fieldsJSON)
 
 	fmt.Fprintln(appender.Writer, strings.Join(toPrint, "\t")) //nolint:errcheck
 	return nil
diff --git a/logging/proto_conversions.go b/logging/proto_conversions.go
index 569e846764e..6f1c6192e42 100644
--- a/logging/proto_conversions.go
+++ b/logging/proto_conversions.go
@@ -52,10 +52,41 @@ func FieldFromProto(field *structpb.Struct) (zap.Field, error) {
 		return zap.Field{}, err
 	}
 
-	// Handle poorly serialized error fields (force them into a string type with
-	// an empty value). Newer Golang modules should serialize correctly (turn
-	// errors into strings client-side) per RSDK-9097.
-	if zf.Type == zapcore.ErrorType {
+	//nolint:exhaustive
+	switch zf.Type {
+	case zapcore.TimeType:
+		if _, ok := zf.Interface.(*time.Location); !ok {
+			// Proto serialization put garbage in the field zap assumes is a timezone,
+			// nil it out to stop zap from panicking
+			zf.Interface = nil
+		}
+	case zapcore.ArrayMarshalerType:
+		if _, ok := zf.Interface.(zapcore.ArrayMarshaler); !ok {
+			// Proto serialization erased the type information, fall back on reflection
+			zf.Type = zapcore.ReflectType
+		}
+	case zapcore.BinaryType:
+		if _, ok := zf.Interface.([]byte); !ok {
+			zf.Type = zapcore.ReflectType
+		}
+	case zapcore.Float64Type:
+		// See FieldToProto above: we encode float64s as strings to avoid loss in
+		// proto conversion. Old logs from the app DB may still have float64s in
+		// the Integer field: do nothing in such cases.
+		if zf.String == "" {
+			break
+		}
+
+		parsedFloat, err := strconv.ParseFloat(zf.String, 64)
+		if err != nil {
+			return zf, err
+		}
+		zf.Integer = int64(math.Float64bits(parsedFloat))
+		zf.String = ""
+	case zapcore.ErrorType:
+		// Handle poorly serialized error fields (force them into a string type with
+		// an empty value). Newer Golang modules should serialize correctly (turn
+		// errors into strings client-side) per RSDK-9097.
 		if _, ok := zf.Interface.(error); !ok {
 			zf.Type = zapcore.StringType
 			zf.String = ""
diff --git a/logging/proto_conversions_test.go b/logging/proto_conversions_test.go
index a3ade1248e5..ed46db9f716 100644
--- a/logging/proto_conversions_test.go
+++ b/logging/proto_conversions_test.go
@@ -24,8 +24,9 @@ func TestFieldConversion(t *testing.T) {
 	testStruct := &s{"foo", true}
 
 	testCases := []struct {
-		field       zap.Field
-		expectedVal any
+		field         zap.Field
+		expectedVal   any
+		expectedField zap.Field
 	}{
 		{
 			field: zap.Field{
@@ -107,6 +108,12 @@ func TestFieldConversion(t *testing.T) {
 			},
 			// Error types retain only their message. Stacks are contained in log.Stack.
 			expectedVal: "error message",
+			expectedField: zap.Field{
+				Key:       "error",
+				Type:      zapcore.StringType,
+				String:    "error message",
+				Interface: map[string]any{},
+			},
 		},
 		{
 			field: zap.Field{
@@ -117,6 +124,11 @@ func TestFieldConversion(t *testing.T) {
 			},
 			// Ensure that UTC is used instead of the Local location from original time.
 			expectedVal: testTime.In(time.UTC),
+			expectedField: zap.Field{
+				Key:     "time",
+				Type:    zapcore.TimeType,
+				Integer: testTime.UnixNano(),
+			},
 		},
 		{
 			field: zap.Field{
@@ -127,6 +139,32 @@ func TestFieldConversion(t *testing.T) {
 			// Types of structs cannot actually be preserved; we convert to
 			// map[string]interface{}.
 			expectedVal: map[string]interface{}{"Field1": "foo", "Field2": true},
+			expectedField: zap.Field{
+				Key:  "struct",
+				Type: zapcore.ReflectType,
+				Interface: map[string]any{
+					"Field1": "foo",
+					"Field2": true,
+				},
+			},
+		},
+		{
+			field:       zap.Strings("array", []string{"a", "b", "c"}),
+			expectedVal: []any{"a", "b", "c"},
+			expectedField: zap.Field{
+				Key:       "array",
+				Type:      zapcore.ReflectType,
+				Interface: []any{"a", "b", "c"},
+			},
+		},
+		{
+			field:       zap.Binary("binary", []byte{244, 244, 244}),
+			expectedVal: []any{float64(244), float64(244), float64(244)},
+			expectedField: zap.Field{
+				Key:       "binary",
+				Type:      zapcore.ReflectType,
+				Interface: []any{float64(244), float64(244), float64(244)},
+			},
 		},
 	}
 	for _, tc := range testCases {
@@ -141,6 +179,18 @@ func TestFieldConversion(t *testing.T) {
 			test.That(t, err, test.ShouldBeNil)
 			test.That(t, key, test.ShouldEqual, tc.field.Key)
 			test.That(t, val, test.ShouldResemble, tc.expectedVal)
+
+			zapField, err := FieldFromProto(field)
+			test.That(t, err, test.ShouldBeNil)
+			if tc.expectedField.Type != zapcore.UnknownType {
+				test.That(t, zapField, test.ShouldResemble, tc.expectedField)
+			} else {
+				test.That(t, zapField, test.ShouldResemble, tc.field)
+			}
+			// ZapcoreFieldsToJSON will fail if we did not properly sanitize a field
+			jsonField, err := ZapcoreFieldsToJSON([]zapcore.Field{zapField})
+			test.That(t, err, test.ShouldBeNil)
+			test.That(t, jsonField, test.ShouldNotBeEmpty)
 		})
 	}
 }


Reviews:
user: login: dgottlieb
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-28T16:52:24Z
user: login: jmatth
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-28T17:06:28Z

Review comments:
diff_hunk: @@ -141,6 +179,18 @@ func TestFieldConversion(t *testing.T) {
 			test.That(t, err, test.ShouldBeNil)
 			test.That(t, key, test.ShouldEqual, tc.field.Key)
 			test.That(t, val, test.ShouldResemble, tc.expectedVal)
+
+			zapField, err := FieldFromProto(field)
+			test.That(t, err, test.ShouldBeNil)
+			if tc.expectedField.Type != zapcore.UnknownType {
path: logging/proto_conversions_test.go
user: login: dgottlieb
body: I'm guessing `UnknownType` is the default value for a default `expectedField` (i.e: most test cases)?
created_at: 2025-04-28T16:51:29Z
author_association: MEMBER
diff_hunk: @@ -141,6 +179,18 @@ func TestFieldConversion(t *testing.T) {
 			test.That(t, err, test.ShouldBeNil)
 			test.That(t, key, test.ShouldEqual, tc.field.Key)
 			test.That(t, val, test.ShouldResemble, tc.expectedVal)
+
+			zapField, err := FieldFromProto(field)
+			test.That(t, err, test.ShouldBeNil)
+			if tc.expectedField.Type != zapcore.UnknownType {
path: logging/proto_conversions_test.go
user: login: jmatth
body: Correct. It's the default value and should never actually be set at runtime because zap will panic in the json serializer (and possibly other places) if it encounters a field with `UnknownType`
created_at: 2025-04-28T17:06:28Z
author_association: MEMBER

PR comments:
user: login: CLAassistant
created_at: 2025-04-25T19:03:14Z
body: [![CLA assistant check](https://cla-assistant.io/pull/badge/signed)](https://cla-assistant.io/viamrobotics/rdk?pullRequest=4939) <br/>All committers have signed the CLA.
author_association: NONE
user: login: CLAassistant
created_at: 2025-04-25T19:03:14Z
body: [![CLA assistant check](https://cla-assistant.io/pull/badge/not_signed)](https://cla-assistant.io/viamrobotics/rdk?pullRequest=4939) <br/>Thank you for your submission! We really appreciate it. Like many open source projects, we ask that you sign our [Contributor License Agreement](https://cla-assistant.io/viamrobotics/rdk?pullRequest=4939) before we can accept your contribution.<br/><sub>You have signed the CLA already but the status is still pending? Let us [recheck](https://cla-assistant.io/check/viamrobotics/rdk?pullRequest=4939) it.</sub>
author_association: NONE