Repository: kmesh-net/kmesh

Issue:
Delete pod with long running connection met error
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!
-->

**What happened**:

```
time="2025-05-09T01:32:09Z" level=info msg="default/ws-server: enable Kmesh manage" subsys=manage_controller
time="2025-05-09T01:32:09Z" level=info msg="add annotation for pod default/ws-server" subsys=manage_controller
time="2025-05-09T01:32:09Z" level=info msg="default/ws-client: enable Kmesh manage" subsys=manage_controller
time="2025-05-09T01:32:09Z" level=info msg="add annotation for pod default/ws-client" subsys=manage_controller
time="2025-05-09T01:32:10Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:10Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:11Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:11Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:12Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:12Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:13Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:13Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:14Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:14Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:15Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:15Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:16Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:16Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:17Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:17Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:18Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:18Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:19Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:19Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:20Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:20Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:21Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:21Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:22Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:22Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:23Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:23Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:24Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:24Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:25Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:25Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:26Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:26Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:27Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:27Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:28Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:28Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:29Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:29Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:30Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:30Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:31Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:31Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:32Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:32Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:33Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:33Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:34Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:34Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:35Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:35Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:36Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:36Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:37Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:37Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:38Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:38Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:39Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:39Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:39Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:39Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:39Z" level=info msg="[PROBE] ERR: on close: bpf_sk_storage_get failed\n" subsys=ebpf
time="2025-05-09T01:32:39Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
time="2025-05-09T01:32:39Z" level=info msg="[PROBE] ERR: on close: bpf_sk_storage_get failed\n" subsys=ebpf
time="2025-05-09T01:32:40Z" level=info msg="default/ws-client: Pod managed by Kmesh is deleted" subsys=manage_controller
time="2025-05-09T01:32:40Z" level=info msg="default/ws-server: Pod managed by Kmesh is deleted" subsys=manage_controller
```

**What you expected to happen**:

Deploy app 

```
apiVersion: v1
kind: Pod
metadata:
  name: ws-server
  labels:
    app: ws-server
spec:
  containers:
  - name: ws-server
    image: node:18
    command: ["/bin/sh", "-c"]
    args:
      - |
        mkdir server 
        cd server 
        npm init -y
        npm install ws
        cat << EOF > server.js
        const WebSocket = require('ws');
        const server = new WebSocket.Server({ port: 8080 });
        server.on('listening', () => {
          console.log('🚀 WebSocket server started on ws://localhost:8080');
        });

        server.on('connection', socket => {
          console.log('Client connected');
          
          // Send data every 2 seconds
          setInterval(() => {
            socket.send('Hello world');
            console.log('Sent message to client');
          }, 1000);

          socket.on('message', message => {
            console.log("Received %s", message) ;
          });

          socket.on('close', () => {
            console.log('Client disconnected');
          });
        });
        EOF
        node server.js

    ports:
    - containerPort: 8080

---
apiVersion: v1
kind: Service
metadata:
  name: ws-server-service
  labels:
    app: ws-server
spec:
  selector:
    app: ws-server
  ports:
  - protocol: TCP
    port: 8080        # Port exposed by the service
    targetPort: 8080  # Port the server container listens on

--- 

apiVersion: v1
kind: Pod
metadata:
  name: ws-client
  labels:
    app: ws-client
spec:
  containers:
  - name: ws-client
    image: node:18
    command: ["/bin/sh", "-c"]
    args:
      - |
        mkdir client
        cd client
        npm init -y
        npm install ws 
        cat <<EOF > client.js
        const WebSocket = require('ws');
        const socket = new WebSocket('ws://ws-server-service.default.svc.cluster.local:8080');

        socket.on('open', () => {
          console.log('Connected to server');
          socket.send('Hello from client');
          console.log('Sent message to server');
        });

        socket.on('message', data => {
          console.log("Received %s",data);
        });

        socket.on('close', () => {
          console.log('Connection closed');
        });
        EOF
        node client.js

```

2. enable kmesh monitoring
3. Then delete the app

BTW, i remember i have ever made kmesh daemon restart. So i feel `[PROBE] ERR: on close: bpf_sk_storage_get failed` is because the connection  is not managed by the new kmesh daemon.  

**How to reproduce it (as minimally and precisely as possible)**:

**Anything else we need to know?**:

**Environment**:

- Kmesh version:
- Kmesh mode(kmesh has `Kernel-Native Mode` and `Duel-Engine Mode`):
- Istio version:
- Kernel version:
- Others:


Hints from issue comments:
@hzxuzhonghu: `failed to get frontend in xdp` does mean that the pod has been removed from the frontendmap. We should think about support graceful shutdown@hzxuzhonghu: /assign@hzxuzhonghu: ```
time="2025-05-09T01:32:27Z" level=info msg="[XDP] INFO: failed to get frontend in xdp" subsys=ebpf
```

This error log is hard to fix, it is happening because workload is deleted when the pod is being shutdown.

But authorization rely on searching frontend map to get the auth policy. If this is not found, then during pod shutdown, it will miss authz completely

```
static inline wl_policies_v *get_workload_policies(struct xdp_info *info, struct bpf_sock_tuple *tuple_info)
{
    frontend_key frontend_k = {};
    frontend_value *frontend_v = NULL;
    __u32 workload_uid = 0;

    if (info->iph->version == 4) {
        frontend_k.addr.ip4 = tuple_info->ipv4.daddr;
    } else if (is_ipv4_mapped_addr(tuple_info->ipv6.daddr)) {
        frontend_k.addr.ip4 = tuple_info->ipv6.daddr[3];
    } else {
        bpf_memcpy(frontend_k.addr.ip6, tuple_info->ipv6.daddr, IPV6_ADDR_LEN);
    }
    frontend_v = kmesh_map_lookup_elem(&map_of_frontend, &frontend_k);
    if (!frontend_v) {
        BPF_LOG(INFO, XDP, "failed to get frontend in xdp");
        return AUTH_ALLOW;
    }
```

@hzxuzhonghu: One possible solution is recording workload health status in the back end map. During pod deletion, its status will be marked unhealthy.

So during cgroup/connect dnat, we should check the workload status and only dnat to healthy one.

And so in this case, xdp prog could find the corresponding auth policies.


@hzxuzhonghu: cc @nlgwcy WDYT https://github.com/kmesh-net/kmesh/issues/1381#issuecomment-2874908239

Solution patch:
diff --git a/bpf/kmesh/workload/include/frontend.h b/bpf/kmesh/workload/include/frontend.h
index 8bb9a21f..a1649298 100644
--- a/bpf/kmesh/workload/include/frontend.h
+++ b/bpf/kmesh/workload/include/frontend.h
@@ -34,7 +34,7 @@ static inline int frontend_manager(struct kmesh_context *kmesh_ctx, frontend_val
         direct_backend = true;
     }
 
-    if (direct_backend) {
+    if (direct_backend) { // in this case, we donot check the healthy status of the backend, just let it go
         // For pod direct access, if a pod has waypoint captured, we will redirect to waypoint, otherwise we do nothing.
         if (backend_v->waypoint_port != 0) {
             BPF_LOG(
diff --git a/bpf/kmesh/workload/xdp.c b/bpf/kmesh/workload/xdp.c
index ad6d61fe..c8e9857d 100644
--- a/bpf/kmesh/workload/xdp.c
+++ b/bpf/kmesh/workload/xdp.c
@@ -70,7 +70,7 @@ static inline wl_policies_v *get_workload_policies(struct xdp_info *info, struct
     }
     frontend_v = kmesh_map_lookup_elem(&map_of_frontend, &frontend_k);
     if (!frontend_v) {
-        BPF_LOG(INFO, XDP, "failed to get frontend in xdp");
+        BPF_LOG(DEBUG, XDP, "failed to get frontend in xdp");
         return AUTH_ALLOW;
     }
     workload_uid = frontend_v->upstream_id;
diff --git a/go.mod b/go.mod
index 4a652f97..6998f5ac 100644
--- a/go.mod
+++ b/go.mod
@@ -16,7 +16,7 @@ require (
 	github.com/envoyproxy/go-control-plane v0.13.2-0.20241125134052-fc612d4a3afa
 	github.com/fsnotify/fsnotify v1.8.0
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/miekg/dns v1.1.62
+	github.com/miekg/dns v1.1.66
 	github.com/prometheus/client_golang v1.21.0
 	github.com/prometheus/common v0.62.0
 	github.com/safchain/ethtool v0.5.10
@@ -25,7 +25,7 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.10.0
 	github.com/vishvananda/netlink v1.3.0
-	golang.org/x/sys v0.30.0
+	golang.org/x/sys v0.32.0
 	google.golang.org/grpc v1.70.0
 	google.golang.org/protobuf v1.36.3
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
@@ -202,16 +202,16 @@ require (
 	go.uber.org/mock v0.5.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
-	golang.org/x/crypto v0.32.0 // indirect
+	golang.org/x/crypto v0.37.0 // indirect
 	golang.org/x/exp v0.0.0-20241215155358-4a5509556b9e // indirect
-	golang.org/x/mod v0.22.0 // indirect
-	golang.org/x/net v0.34.0 // indirect
+	golang.org/x/mod v0.24.0 // indirect
+	golang.org/x/net v0.39.0 // indirect
 	golang.org/x/oauth2 v0.24.0 // indirect
-	golang.org/x/sync v0.10.0 // indirect
-	golang.org/x/term v0.28.0 // indirect
-	golang.org/x/text v0.21.0 // indirect
+	golang.org/x/sync v0.13.0 // indirect
+	golang.org/x/term v0.31.0 // indirect
+	golang.org/x/text v0.24.0 // indirect
 	golang.org/x/time v0.8.0 // indirect
-	golang.org/x/tools v0.28.0 // indirect
+	golang.org/x/tools v0.32.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/api v0.199.0 // indirect
 	google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect
diff --git a/go.sum b/go.sum
index e6348e47..1e71e17b 100644
--- a/go.sum
+++ b/go.sum
@@ -300,8 +300,8 @@ github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU
 github.com/mdlayher/socket v0.1.1/go.mod h1:mYV5YIZAfHh4dzDVzI8x8tWLWCliuX8Mon5Awbj+qDs=
 github.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos=
 github.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTanQE37IQ=
-github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ=
-github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ=
+github.com/miekg/dns v1.1.66 h1:FeZXOS3VCVsKnEAd+wBkjMC3D2K+ww66Cq3VnCINuJE=
+github.com/miekg/dns v1.1.66/go.mod h1:jGFzBsSNbJw6z1HYut1RKBKHA9PBdxeHrZG8J+gC2WE=
 github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
@@ -495,8 +495,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
-golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
+golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
+golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20241215155358-4a5509556b9e h1:4qufH0hlUYs6AO6XmZC3GqfDPGSXHVXUFR6OND+iJX4=
 golang.org/x/exp v0.0.0-20241215155358-4a5509556b9e/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c=
@@ -508,8 +508,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4=
-golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
+golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
+golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -525,8 +525,8 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
-golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
+golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
+golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=
 golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
@@ -538,8 +538,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
-golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
+golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -564,15 +564,15 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
-golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
+golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
-golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg=
-golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
+golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o=
+golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -580,8 +580,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
-golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
+golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
 golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg=
 golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -596,8 +596,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8=
-golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw=
+golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU=
+golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/pkg/controller/workload/workload_processor.go b/pkg/controller/workload/workload_processor.go
index 7fdb587e..c87bece9 100644
--- a/pkg/controller/workload/workload_processor.go
+++ b/pkg/controller/workload/workload_processor.go
@@ -201,6 +201,24 @@ func (p *Processor) removeWorkload(uid string) error {
 	return p.removeWorkloadFromBpfMap(wl)
 }
 
+// handleUnhealthyWorkload is used to handle unhealthy workload, we only leave it in the frontend and backend map.
+// Because we rely on the backend map to be aware of direct access in `frontend_manager`,
+// and xdp rely on frontend map to get the authz policy and do authz.
+func (p *Processor) handleUnhealthyWorkload(workload *workloadapi.Workload) error {
+	backendUid := p.hashName.Hash(workload.Uid)
+
+	// 1. find all endpoint keys related to this workload and delete them
+	if eks := p.bpf.GetEndpointKeys(backendUid); len(eks) > 0 {
+		if err := p.deleteEndpointRecords(eks.UnsortedList()); err != nil {
+			return fmt.Errorf("handleUnhealthyWorkload: deleteEndpointRecords for %s failed: %v", workload.Uid, err)
+		}
+	}
+
+	// TODO(hzxuzhonghu), maybe need to update backend map to update the workload status
+
+	return nil
+}
+
 func (p *Processor) removeWorkloadFromBpfMap(workload *workloadapi.Workload) error {
 	var (
 		err      error
@@ -486,11 +504,11 @@ func (p *Processor) handleWorkload(workload *workloadapi.Workload) error {
 		p.locality.SetLocality(p.nodeName, workload.GetClusterId(), workload.GetNetwork(), workload.GetLocality())
 	}
 
-	// Exclude unhealthy workload, which is not ready to serve traffic
+	// Exclude unhealthy workload, which is not ready to serve traffic, but keep it in the frontend
+	// backend map for authz
 	if workload.Status == workloadapi.WorkloadStatus_UNHEALTHY {
 		log.Debugf("workload %s is unhealthy", workload.ResourceName())
-		// If the workload is updated to unhealthy, we should remove it from the bpf map
-		return p.removeWorkloadFromBpfMap(workload)
+		return p.handleUnhealthyWorkload(workload)
 	}
 
 	// 1. update workload in backend map



Test patch:
diff --git a/pkg/controller/workload/workload_processor_test.go b/pkg/controller/workload/workload_processor_test.go
index cebcff46..b55026f1 100644
--- a/pkg/controller/workload/workload_processor_test.go
+++ b/pkg/controller/workload/workload_processor_test.go
@@ -160,10 +160,12 @@ func Test_handleWorkload(t *testing.T) {
 	assert.Equal(t, got.Status, workloadapi.WorkloadStatus_UNHEALTHY)
 	checkNotExistInFrontEndMap(t, workload3.Addresses[0], p)
 
-	// 8. update workload from healthy to unhealthy, should remove it from bpf map
+	// 8. update workload from healthy to unhealthy, should remove it from bpf endpoint map
 	workload2.Status = workloadapi.WorkloadStatus_UNHEALTHY
 	_ = p.handleWorkload(workload2)
-	checkNotExistInFrontEndMap(t, workload2.Addresses[0], p)
+	checkNotExistInEndpointMap(t, p, fakeSvc, []uint32{workload2ID})
+	checkFrontEndMap(t, workload2.Addresses[0], p)
+	checkBackendMap(t, p, workload2ID, workload2)
 
 	// 9. delete service
 	p.handleRemovedAddresses([]string{fakeSvc.ResourceName()})
@@ -276,6 +278,18 @@ func checkServiceMap(t *testing.T, p *Processor, svcId uint32, fakeSvc *workload
 	assert.Equal(t, sv.WaypointPort, nets.ConvertPortToBigEndian(fakeSvc.Waypoint.GetHboneMtlsPort()))
 }
 
+func checkNotExistInEndpointMap(t *testing.T, p *Processor, fakeSvc *workloadapi.Service, backendUid []uint32) {
+	endpoints := p.bpf.GetAllEndpointsForService(p.hashName.Hash(fakeSvc.ResourceName()))
+	t.Logf("endpoints number: %v", len(endpoints))
+
+	all := sets.New[uint32](backendUid...)
+	for _, endpoint := range endpoints {
+		if all.Contains(endpoint.BackendUid) {
+			t.Fatalf("endpoint %v still exists", endpoint.BackendUid)
+		}
+	}
+}
+
 func checkEndpointMap(t *testing.T, p *Processor, fakeSvc *workloadapi.Service, backendUid []uint32) {
 	endpoints := p.bpf.GetAllEndpointsForService(p.hashName.Hash(fakeSvc.ResourceName()))
 	assert.Equal(t, len(endpoints), len(backendUid))