Repository: kmesh-net/kmesh

Issue:
Metric `sent_bytes` `received_bytes` not consistent with accesslog
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!
-->

**What happened**:

I have been testing the long connection metrics feature. And deployed a websocket client  and a server.

The server sent a message to the client every 1 second.

After a long time, i can see the access log 

```
accesslog: 2025-05-08 07:41:31.436041742 +0000 UTC src.addr=10.244.0.14:49056, src.workload=ws-client, src.namespace=default, dst.addr=10.244.0.11:8080, dst.service=ws-server-service.default.svc.cluster.local, dst.workload=ws-server, dst.namespace=default, start_time=2025-05-08 03:38:34.201259611 +0000 UTC, direction=INBOUND, state=BPF_TCP_ESTABLISHED, sent_bytes=14566, received_bytes=280, packet_loss=0, retransmissions=0, srtt=457us, min_rtt=38us, duration=1.4577234782131e+07ms
accesslog: 2025-05-08 07:41:31.436079898 +0000 UTC src.addr=10.244.0.14:49056, src.workload=ws-client, src.namespace=default, dst.addr=10.244.0.11:8080, dst.service=ws-server-service.default.svc.cluster.local, dst.workload=ws-server, dst.namespace=default, start_time=2025-05-08 03:38:34.201135327 +0000 UTC, direction=OUTBOUND, state=BPF_TCP_ESTABLISHED, sent_bytes=3, received_bytes=29257, packet_loss=0, retransmissions=0, srtt=41600us, min_rtt=40us, duration=1.4577234944571e+07ms
```

And also queried the metrics, they are quite different.

```
kmesh_tcp_connection_received_bytes_total{connection_security_policy="mutual_tls",destination_address="10.244.0.11:8080",destination_app="ws-server",destination_canonical_revision="latest",destination_canonical_service="ws-server",destination_cluster="Kubernetes",destination_pod_address="10.244.0.11",destination_pod_name="ws-server",destination_pod_namespace="default",destination_principal="-",destination_service="ws-server-service.default.svc.cluster.local",destination_service_name="ws-server-service",destination_service_namespace="default",destination_version="latest",destination_workload="ws-server",destination_workload_namespace="default",reporter="destination",request_protocol="tcp",response_flags="-",source_address="10.244.0.14:49056",source_app="ws-client",source_canonical_revision="latest",source_canonical_service="ws-client",source_cluster="Kubernetes",source_principal="-",source_version="latest",source_workload="ws-client",source_workload_namespace="default",start_time="2025-05-08 03:38:34.201259611 +0000 UTC"} 801360
kmesh_tcp_connection_received_bytes_total{connection_security_policy="mutual_tls",destination_address="10.244.0.11:8080",destination_app="ws-server",destination_canonical_revision="latest",destination_canonical_service="ws-server",destination_cluster="Kubernetes",destination_pod_address="10.244.0.11",destination_pod_name="ws-server",destination_pod_namespace="default",destination_principal="-",destination_service="ws-server-service.default.svc.cluster.local",destination_service_name="ws-server-service",destination_service_namespace="default",destination_version="latest",destination_workload="ws-server",destination_workload_namespace="default",reporter="source",request_protocol="tcp",response_flags="-",source_address="10.244.0.14:49056",source_app="ws-client",source_canonical_revision="latest",source_canonical_service="ws-client",source_cluster="Kubernetes",source_principal="-",source_version="latest",source_workload="ws-client",source_workload_namespace="default",start_time="2025-05-08 03:38:34.201135327 +0000 UTC"} 4.3193304e+07
kmesh_tcp_connection_sent_bytes_total{connection_security_policy="mutual_tls",destination_address="10.244.0.11:8080",destination_app="ws-server",destination_canonical_revision="latest",destination_canonical_service="ws-server",destination_cluster="Kubernetes",destination_pod_address="10.244.0.11",destination_pod_name="ws-server",destination_pod_namespace="default",destination_principal="-",destination_service="ws-server-service.default.svc.cluster.local",destination_service_name="ws-server-service",destination_service_namespace="default",destination_version="latest",destination_workload="ws-server",destination_workload_namespace="default",reporter="destination",request_protocol="tcp",response_flags="-",source_address="10.244.0.14:49056",source_app="ws-client",source_canonical_revision="latest",source_canonical_service="ws-client",source_cluster="Kubernetes",source_principal="-",source_version="latest",source_workload="ws-client",source_workload_namespace="default",start_time="2025-05-08 03:38:34.201259611 +0000 UTC"} 2.1417777e+07
kmesh_tcp_connection_sent_bytes_total{connection_security_policy="mutual_tls",destination_address="10.244.0.11:8080",destination_app="ws-server",destination_canonical_revision="latest",destination_canonical_service="ws-server",destination_cluster="Kubernetes",destination_pod_address="10.244.0.11",destination_pod_name="ws-server",destination_pod_namespace="default",destination_principal="-",destination_service="ws-server-service.default.svc.cluster.local",destination_service_name="ws-server-service",destination_service_namespace="default",destination_version="latest",destination_workload="ws-server",destination_workload_namespace="default",reporter="source",request_protocol="tcp",response_flags="-",source_address="10.244.0.14:49056",source_app="ws-client",source_canonical_revision="latest",source_canonical_service="ws-client",source_cluster="Kubernetes",source_principal="-",source_version="latest",source_workload="ws-client",source_workload_namespace="default",start_time="2025-05-08 03:38:34.201135327 +0000 UTC"} 8586
```

And the workload sent bytes is also not consistent

```
kmesh_tcp_workload_sent_bytes_total{connection_security_policy="mutual_tls",destination_app="ws-server",destination_canonical_revision="latest",destination_canonical_service="ws-server",destination_cluster="Kubernetes",destination_pod_address="10.244.0.11",destination_pod_name="ws-server",destination_pod_namespace="default",destination_principal="-",destination_version="latest",destination_workload="ws-server",destination_workload_namespace="default",reporter="destination",request_protocol="tcp",response_flags="-",source_app="ws-client",source_canonical_revision="latest",source_canonical_service="ws-client",source_cluster="Kubernetes",source_principal="-",source_version="latest",source_workload="ws-client",source_workload_namespace="default"} 2.2251539e+07
kmesh_tcp_workload_sent_bytes_total{connection_security_policy="mutual_tls",destination_app="ws-server",destination_canonical_revision="latest",destination_canonical_service="ws-server",destination_cluster="Kubernetes",destination_pod_address="10.244.0.11",destination_pod_name="ws-server",destination_pod_namespace="default",destination_principal="-",destination_version="latest",destination_workload="ws-server",destination_workload_namespace="default",reporter="source",request_protocol="tcp",response_flags="-",source_app="ws-client",source_canonical_revision="latest",source_canonical_service="ws-client",source_cluster="Kubernetes",source_principal="-",source_version="latest",source_workload="ws-client",source_workload_namespace="default"} 1.2536987285488e+13
```

**What you expected to happen**:

**How to reproduce it (as minimally and precisely as possible)**:

**Anything else we need to know?**:

**Environment**:

- Kmesh version:
- Kmesh mode(kmesh has `Kernel-Native Mode` and `Duel-Engine Mode`):
- Istio version:
- Kernel version:
- Others:


Hints from issue comments:
@hzxuzhonghu: @yp969803 @yp969803: Let me check it out@yp969803: ```
accesslog: 2025-05-06 07:01:05.344948704 +0000 UTC src.addr=10.244.1.6:42008, src.workload=busybox-telnet, src.namespace=default, dst.addr=10.244.1.7:23, dst.service=telnet-service.default.svc.cluster.local, dst.workload=busybox-telnet-server, dst.namespace=default, start_time=2025-05-06 06:59:12.043322592 +0000 UTC, direction=INBOUND, state=BPF_TCP_ESTABLISHED, sent_bytes=31, received_bytes=53, packet_loss=0, retransmissions=0, srtt=1154us, min_rtt=11us, duration=113301.626112ms

```

```
kmesh_tcp_sent_bytes_total{connection_security_policy="mutual_tls",destination_app="busybox-telnet",destination_canonical_revision="latest",destination_canonical_service="busybox-telnet",destination_cluster="Kubernetes",destination_principal="-",destination_service="telnet-service.default.svc.cluster.local",destination_service_name="telnet-service",destination_service_namespace="default",destination_version="latest",destination_workload="busybox-telnet-server",destination_workload_namespace="default",reporter="source",request_protocol="tcp",response_flags="-",source_app="busybox",source_canonical_revision="latest",source_canonical_service="busybox",source_cluster="Kubernetes",source_principal="-",source_version="latest",source_workload="busybox-telnet",source_workload_namespace="default"} 31

```

tcp_sent_bytes is working fine for serviceMetrics@yp969803: ```
kmesh_tcp_received_bytes_total{connection_security_policy="mutual_tls",destination_app="busybox-telnet",destination_canonical_revision="latest",destination_canonical_service="busybox-telnet",destination_cluster="Kubernetes",destination_principal="-",destination_service="telnet-service.default.svc.cluster.local",destination_service_name="telnet-service",destination_service_namespace="default",destination_version="latest",destination_workload="busybox-telnet-server",destination_workload_namespace="default",reporter="source",request_protocol="tcp",response_flags="-",source_app="busybox",source_canonical_revision="latest",source_canonical_service="busybox",source_cluster="Kubernetes",source_principal="-",source_version="latest",source_workload="busybox-telnet",source_workload_namespace="default"} 661

```

```
accesslog: 2025-05-06 07:01:05.344918035 +0000 UTC src.addr=10.244.1.6:42008, src.workload=busybox-telnet, src.namespace=default, dst.addr=10.244.1.7:23, dst.service=telnet-service.default.svc.cluster.local, dst.workload=busybox-telnet-server, dst.namespace=default, start_time=2025-05-06 06:59:12.043248215 +0000 UTC, direction=OUTBOUND, state=BPF_TCP_ESTABLISHED, sent_bytes=28, received_bytes=633, packet_loss=0, retransmissions=0, srtt=3444us, min_rtt=14us, duration=113301.66982ms
```

service_recieve_bytes are also fine@hzxuzhonghu: It only happens when both client and server deployed in same node

Solution patch:
diff --git a/pkg/controller/telemetry/metric.go b/pkg/controller/telemetry/metric.go
index 8585c397..12168dae 100644
--- a/pkg/controller/telemetry/metric.go
+++ b/pkg/controller/telemetry/metric.go
@@ -521,7 +521,7 @@ func (m *MetricController) Run(ctx context.Context, mapOfTcpInfo *ebpf.Map) {
 			}
 			m.updateServiceMetricCache(reqMetric, serviceLabels, tcpConns[reqMetric.conSrcDstInfo])
 			if m.EnableConnectionMetric.Load() && reqMetric.duration > LONG_CONN_METRIC_THRESHOLD {
-				m.updateConnectionMetricCache(reqMetric, tcpConns[reqMetric.conSrcDstInfo], connectionLabels)
+				m.updateConnectionMetricCache(reqMetric, connectionLabels)
 			}
 			m.mutex.Unlock()
 
@@ -907,7 +907,7 @@ func (m *MetricController) updateServiceMetricCache(reqMetric requestMetric, lab
 	}
 }
 
-func (m *MetricController) updateConnectionMetricCache(reqMetric requestMetric, connMetric connMetric, labels connectionMetricLabels) {
+func (m *MetricController) updateConnectionMetricCache(reqMetric requestMetric, labels connectionMetricLabels) {
 	v, ok := m.connectionMetricCache[labels]
 	if ok {
 		v.ConnSentBytes = v.ConnSentBytes + float64(reqMetric.sentBytes)
@@ -916,10 +916,10 @@ func (m *MetricController) updateConnectionMetricCache(reqMetric requestMetric,
 		v.ConnTotalRetrans = v.ConnTotalRetrans + float64(reqMetric.totalRetrans)
 	} else {
 		newConnectionMetricInfo := connectionMetricInfo{}
-		newConnectionMetricInfo.ConnSentBytes = float64(connMetric.sentBytes)
-		newConnectionMetricInfo.ConnReceivedBytes = float64(connMetric.receivedBytes)
-		newConnectionMetricInfo.ConnPacketLost = float64(connMetric.packetLost)
-		newConnectionMetricInfo.ConnTotalRetrans = float64(connMetric.totalRetrans)
+		newConnectionMetricInfo.ConnSentBytes = float64(reqMetric.sentBytes)
+		newConnectionMetricInfo.ConnReceivedBytes = float64(reqMetric.receivedBytes)
+		newConnectionMetricInfo.ConnPacketLost = float64(reqMetric.packetLost)
+		newConnectionMetricInfo.ConnTotalRetrans = float64(reqMetric.totalRetrans)
 		m.connectionMetricCache[labels] = &newConnectionMetricInfo
 	}
 	if reqMetric.state == TCP_CLOSED {
diff --git a/samples/addons/grafana.yaml b/samples/addons/grafana.yaml
index 9b10c4fd..0c036977 100644
--- a/samples/addons/grafana.yaml
+++ b/samples/addons/grafana.yaml
@@ -707,7 +707,7 @@ data:
                 "uid": "PBFA97CFB590B2093"
               },
               "editorMode": "code",
-              "expr": "sum(kmesh_operation_duration_seconds_sum{app=\"kmesh\", namespace=\"kmesh-system\", operation_type=\"SOCK_TRAFFIC_CONTROL\"}) by (node_name) \r\n/\r\nsum(kmesh_operation_duration_seconds_count{app=\"kmesh\", namespace=\"kmesh-system\", operation_type=\"SOCK_TRAFFIC_CONTROL\"}) by (node_name)\r\n",
+              "expr": "sum(kmesh_bpf_prog_operation_duration_sum{app=\"kmesh\", namespace=\"kmesh-system\", operation_type=\"SOCK_TRAFFIC_CONTROL\"}) by (node_name) \r\n/\r\nsum(kmesh_bpf_prog_operation_duration_count{app=\"kmesh\", namespace=\"kmesh-system\", operation_type=\"SOCK_TRAFFIC_CONTROL\"}) by (node_name)\r\n",
               "instant": false,
               "legendFormat": "__auto",
               "range": true,
diff --git a/samples/addons/kmesh_performance-monitoring.json b/samples/addons/kmesh_performance-monitoring.json
index 0471c35b..b3f47633 100644
--- a/samples/addons/kmesh_performance-monitoring.json
+++ b/samples/addons/kmesh_performance-monitoring.json
@@ -486,7 +486,7 @@
             "uid": "PBFA97CFB590B2093"
           },
           "editorMode": "code",
-          "expr": "sum(kmesh_operation_duration_seconds_sum{app=\"kmesh\", namespace=\"kmesh-system\", operation_type=\"SOCK_TRAFFIC_CONTROL\"}) by (node_name) \r\n/\r\nsum(kmesh_operation_duration_seconds_count{app=\"kmesh\", namespace=\"kmesh-system\", operation_type=\"SOCK_TRAFFIC_CONTROL\"}) by (node_name)\r\n",
+          "expr": "sum(kmesh_bpf_prog_operation_duration_sum{app=\"kmesh\", namespace=\"kmesh-system\", operation_type=\"SOCK_TRAFFIC_CONTROL\"}) by (node_name) \r\n/\r\nsum(kmesh_bpf_prog_operation_duration_count{app=\"kmesh\", namespace=\"kmesh-system\", operation_type=\"SOCK_TRAFFIC_CONTROL\"}) by (node_name)\r\n",
           "instant": false,
           "legendFormat": "__auto",
           "range": true,



Test patch:
diff --git a/pkg/controller/telemetry/metric_test.go b/pkg/controller/telemetry/metric_test.go
index e141f459..ab54e1a6 100644
--- a/pkg/controller/telemetry/metric_test.go
+++ b/pkg/controller/telemetry/metric_test.go
@@ -491,9 +491,8 @@ func TestBuildServiceMetricsToPrometheus(t *testing.T) {
 
 func TestBuildConnectionMetricsToPrometheus(t *testing.T) {
 	type args struct {
-		data     requestMetric
-		labels   connectionMetricLabels
-		tcpConns map[connectionSrcDst]connMetric
+		data   requestMetric
+		labels connectionMetricLabels
 	}
 	tests := []struct {
 		id   int32
@@ -542,26 +541,6 @@ func TestBuildConnectionMetricsToPrometheus(t *testing.T) {
 					responseFlags:                "-",
 					connectionSecurityPolicy:     "mutual_tls",
 				},
-				tcpConns: map[connectionSrcDst]connMetric{
-					{
-						src:       [4]uint32{183763210, 0, 0, 0},
-						dst:       [4]uint32{183762951, 0, 0, 0},
-						direction: constants.INBOUND,
-					}: {
-						sentBytes:     0x0000003,
-						receivedBytes: 0x0000004,
-						packetLost:    0x0000001,
-						totalRetrans:  0x0000002,
-						totalReports:  1,
-					},
-					{
-						src:       [4]uint32{183763210, 0, 0, 0},
-						dst:       [4]uint32{183762951, 0, 0, 0},
-						direction: constants.OUTBOUND,
-					}: {
-						totalReports: 3,
-					},
-				},
 			},
 			want: []float64{
 				3,
@@ -611,26 +590,6 @@ func TestBuildConnectionMetricsToPrometheus(t *testing.T) {
 					responseFlags:                "-",
 					connectionSecurityPolicy:     "mutual_tls",
 				},
-				tcpConns: map[connectionSrcDst]connMetric{
-					{
-						src:       [4]uint32{183763210, 0, 0, 0},
-						dst:       [4]uint32{183762951, 0, 0, 0},
-						direction: constants.INBOUND,
-					}: {
-						sentBytes:     0x0000003,
-						receivedBytes: 0x0000004,
-						packetLost:    0x0000001,
-						totalRetrans:  0x0000002,
-						totalReports:  1,
-					},
-					{
-						src:       [4]uint32{183763210, 0, 0, 0},
-						dst:       [4]uint32{183762951, 0, 0, 0},
-						direction: constants.OUTBOUND,
-					}: {
-						totalReports: 3,
-					},
-				},
 			},
 			want: []float64{
 				4,
@@ -681,7 +640,7 @@ func TestBuildConnectionMetricsToPrometheus(t *testing.T) {
 			}
 
 			deleteConnection = []*connectionMetricLabels{}
-			m.updateConnectionMetricCache(tt.args.data, tt.args.tcpConns[tt.args.data.conSrcDstInfo], tt.args.labels)
+			m.updateConnectionMetricCache(tt.args.data, tt.args.labels)
 			assert.Equal(t, m.connectionMetricCache[tt.args.labels].ConnSentBytes, tt.want[0])
 			assert.Equal(t, m.connectionMetricCache[tt.args.labels].ConnReceivedBytes, tt.want[1])
 			assert.Equal(t, m.connectionMetricCache[tt.args.labels].ConnPacketLost, tt.want[2])