Repository: neuvector/neuvector
Pull request #1937: NV9905: k8s workload scan
Author: login: jayhuang-suse
Created: 2025-04-28T23:57:09Z | Merged: 2025-04-30T17:26:56Z

Recreated a PR to reduce the confusions.

The original PR is https://github.com/neuvector/neuvector/pull/1933


Diff:
diff --git a/agent/grpc.go b/agent/grpc.go
index be5cf219e..8ad32e4e9 100644
--- a/agent/grpc.go
+++ b/agent/grpc.go
@@ -63,7 +63,7 @@ func (ss *ScanService) ScanGetFiles(ctx context.Context, req *share.ScanRunningR
 	var pid int
 	var data share.ScanData
 	var pidHost bool
-	var k8sImageRepo string
+	var k8sAppString string
 
 	gInfoRLock()
 	if req.Type == share.ScanObjectType_HOST {
@@ -82,9 +82,15 @@ func (ss *ScanService) ScanGetFiles(ctx context.Context, req *share.ScanRunningR
 		}
 
 		if Host.Platform == share.PlatformKubernetes {
-			// include imageRepo
-			k8sImageRepo = c.info.Image
-			log.WithFields(log.Fields{"k8sImageRepo": k8sImageRepo}).Debug()
+			if c.info.Labels != nil {
+				if name, ok := c.info.Labels["app.kubernetes.io/name"]; ok {
+					k8sAppString = name
+					if version, ok := c.info.Labels["app.kubernetes.io/version"]; ok {
+						k8sAppString += ":" + version
+					}
+				}
+			}
+			log.WithFields(log.Fields{"k8sAppString": k8sAppString}).Debug()
 		}
 	}
 	gInfoRUnlock()
@@ -103,12 +109,12 @@ func (ss *ScanService) ScanGetFiles(ctx context.Context, req *share.ScanRunningR
 	global.SYS.ReCalculateMemoryMetrics(memStatsEnforcerResetMark)
 
 	taskReq := workerlet.WalkGetPackageRequest{
-		Pid:       pid,
-		Id:        req.ID,
-		Kernel:    Host.Kernel,
-		ObjType:   req.Type,
-		PidHost:   pidHost,
-		ImageRepo: k8sImageRepo,
+		Pid:          pid,
+		Id:           req.ID,
+		Kernel:       Host.Kernel,
+		ObjType:      req.Type,
+		PidHost:      pidHost,
+		K8sAppString: k8sAppString,
 	}
 
 	bytesValue, _, err := walkerTask.Run(taskReq, req.ID)
diff --git a/agent/workerlet/pathWalker/pathWalker.go b/agent/workerlet/pathWalker/pathWalker.go
index 0773e5424..042b128c0 100644
--- a/agent/workerlet/pathWalker/pathWalker.go
+++ b/agent/workerlet/pathWalker/pathWalker.go
@@ -300,7 +300,8 @@ func (tm *taskMain) WalkPathTask(req workerlet.WalkPathRequest) {
 func (tm *taskMain) WalkPackageTask(req workerlet.WalkGetPackageRequest) {
 	var data share.ScanData
 	scanUtil := scan.NewScanUtil(tm.sys)
-	data.Buffer, data.Error = scanUtil.GetRunningPackages(req.Id, req.ObjType, req.Pid, req.Kernel, req.ImageRepo, req.PidHost)
+	data.Buffer, data.Error = scanUtil.GetRunningPackages(req.Id, req.ObjType, req.Pid,
+		req.Kernel, req.K8sAppString, req.PidHost)
 
 	// outputs:
 	output, err := json.Marshal(data)
diff --git a/agent/workerlet/types.go b/agent/workerlet/types.go
index bdde2db3f..64dcb58cf 100644
--- a/agent/workerlet/types.go
+++ b/agent/workerlet/types.go
@@ -22,12 +22,12 @@ type WalkPathRequest struct {
 }
 
 type WalkGetPackageRequest struct {
-	Pid       int                  `json:"pid"`
-	Id        string               `json:"id"`
-	Kernel    string               `json:"kernel"`
-	ObjType   share.ScanObjectType `json:"objType"`
-	PidHost   bool                 `json:"pidHost"`
-	ImageRepo string               `json:"k8sImage"`
+	Pid          int                  `json:"pid"`
+	Id           string               `json:"id"`
+	Kernel       string               `json:"kernel"`
+	ObjType      share.ScanObjectType `json:"objType"`
+	PidHost      bool                 `json:"pidHost"`
+	K8sAppString string               `json:"K8sAppString"`
 }
 
 type WalkSecretRequest struct {
diff --git a/share/scan/scan_utils.go b/share/scan/scan_utils.go
index 66341798a..447ad5e40 100644
--- a/share/scan/scan_utils.go
+++ b/share/scan/scan_utils.go
@@ -93,7 +93,7 @@ type CacherData struct {
 }
 
 const (
-	TypeK8sRepo = "k8sRepo"
+	TypeK8sApp = "k8sApp"
 )
 
 type KubernetesResource struct {
@@ -186,7 +186,7 @@ func (s *ScanUtil) readRunningPackages(pid int, prefix, kernel string, pidHost b
 }
 
 func (s *ScanUtil) GetRunningPackages(id string, objType share.ScanObjectType, pid int,
-	kernel, imageRepo string, pidHost bool) ([]byte, share.ScanErrorCode) {
+	kernel, k8sAppResourceStr string, pidHost bool) ([]byte, share.ScanErrorCode) {
 	files, hasPkgMgr := s.readRunningPackages(pid, "/", kernel, pidHost)
 	if len(files) == 0 && !hasPkgMgr && objType == share.ScanObjectType_HOST {
 		// In RancherOS, host os-release file is at /host/proc/1/root/usr/etc/os-release
@@ -204,11 +204,11 @@ func (s *ScanUtil) GetRunningPackages(id string, objType share.ScanObjectType, p
 			files[AppFileName] = data
 		}
 
-		if imageRepo != "" {
+		if k8sAppResourceStr != "" {
 			// include k8s image repo meta data
 			k8sResource := KubernetesResource{
-				ResourceType: TypeK8sRepo,
-				Name:         imageRepo,
+				ResourceType: TypeK8sApp,
+				Name:         k8sAppResourceStr,
 			}
 
 			if data, err := json.Marshal(&k8sResource); err == nil {


Reviews:
user: login: holyspectral
body: Thanks for your clarification.  It makes much more sense now.  Some thoughts:

- Would it be beneficial if we include the whole `c.info.Labels` instead of returning `k8sResourceString`? This way we will have more materials to detect next time. For example, labels like `"org.opencontainers.image.version": "24.04"` would be very useful in your scenario.

```
# ctr -n k8s.io containers info 89d3b87c322c6504a13649116cca960c14cebd0fdf82ccdf939c26b1f24db327
...
    "Labels": {
        "io.cri-containerd.kind": "container",
        "io.kubernetes.container.name": "ubuntu",
        "io.kubernetes.pod.name": "ubuntu-deployment-5fccb98689-4kmmw",
        "io.kubernetes.pod.namespace": "default",
        "io.kubernetes.pod.uid": "b11ab672-25f7-45fb-8b4a-b2e29cc60f25",
        "org.opencontainers.image.ref.name": "ubuntu",
        "org.opencontainers.image.version": "24.04"
    },
```

state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-29T01:42:33Z
user: login: jayhuang-suse
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-29T16:14:12Z
user: login: holyspectral
body: Generally looks good to me except a small issue.  
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-29T17:35:26Z
user: login: holyspectral
body: LGTM.  Please squash changes into one commit before you merge. 
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-30T17:25:25Z

Review comments:
diff_hunk: @@ -300,7 +300,8 @@ func (tm *taskMain) WalkPathTask(req workerlet.WalkPathRequest) {
 func (tm *taskMain) WalkPackageTask(req workerlet.WalkGetPackageRequest) {
 	var data share.ScanData
 	scanUtil := scan.NewScanUtil(tm.sys)
-	data.Buffer, data.Error = scanUtil.GetRunningPackages(req.Id, req.ObjType, req.Pid, req.Kernel, req.ImageRepo, req.PidHost)
+	data.Buffer, data.Error = scanUtil.GetRunningPackages(req.Id, req.ObjType, req.Pid,
path: agent/workerlet/pathWalker/pathWalker.go
user: login: holyspectral
body: Could you rename the `imageRepo` in `scanUtil.GetRunningPackages()`'s parameters as well?
created_at: 2025-04-29T01:26:12Z
author_association: COLLABORATOR
diff_hunk: @@ -300,7 +300,8 @@ func (tm *taskMain) WalkPathTask(req workerlet.WalkPathRequest) {
 func (tm *taskMain) WalkPackageTask(req workerlet.WalkGetPackageRequest) {
 	var data share.ScanData
 	scanUtil := scan.NewScanUtil(tm.sys)
-	data.Buffer, data.Error = scanUtil.GetRunningPackages(req.Id, req.ObjType, req.Pid, req.Kernel, req.ImageRepo, req.PidHost)
+	data.Buffer, data.Error = scanUtil.GetRunningPackages(req.Id, req.ObjType, req.Pid,
path: agent/workerlet/pathWalker/pathWalker.go
user: login: jayhuang-suse
body: will do
created_at: 2025-04-29T16:14:12Z
author_association: CONTRIBUTOR
diff_hunk: @@ -22,12 +22,12 @@ type WalkPathRequest struct {
 }
 
 type WalkGetPackageRequest struct {
-	Pid       int                  `json:"pid"`
-	Id        string               `json:"id"`
-	Kernel    string               `json:"kernel"`
-	ObjType   share.ScanObjectType `json:"objType"`
-	PidHost   bool                 `json:"pidHost"`
-	ImageRepo string               `json:"k8sImage"`
+	Pid          int                  `json:"pid"`
+	Id           string               `json:"id"`
+	Kernel       string               `json:"kernel"`
+	ObjType      share.ScanObjectType `json:"objType"`
+	PidHost      bool                 `json:"pidHost"`
+	K8sAppString string               `json:"k8sAppRscString"`
path: agent/workerlet/types.go
user: login: holyspectral
body: ```suggestion
	K8sAppString string               `json:"k8sAppString"`
```
created_at: 2025-04-29T17:33:25Z
author_association: COLLABORATOR

PR comments:
user: login: pohanhuangtw
created_at: 2025-04-29T03:15:31Z
body: Hi @jayhuang-suse,

Thank you for recreating the PR and providing important context about this change.

However, based on the previous [PR discussion](https://github.com/neuvector/neuvector/pull/1933#discussion_r2064322152), would it be possible to first try fetching the resource name and version using the native API, similar to the [GetResource](https://github.com/neuvector/neuvector/blob/c459e5c7e4e84330166d037e8af91ae9d3bf51d0/controller/resource/kubernetes_resource.go#L1650) function in kubernetes_resource.go?


### examples

1. [neuvector\controller\controller.go](https://github.com/neuvector/neuvector/blob/c459e5c7e4e84330166d037e8af91ae9d3bf51d0/controller/controller.go#L1138):

  - global.ORCH.GetResource(resource.RscTypePod, domain, podname)
2. [controller\resource\kubernetes_resource.go](https://github.com/neuvector/neuvector/blob/c459e5c7e4e84330166d037e8af91ae9d3bf51d0/controller/resource/kubernetes_resource.go#L2233):
  - pods, err := global.ORCH.ListResource(RscTypePod, NvAdmSvcNamespace)

If that attempt fails, we could then fall back to the new function you created.

Thanks again for your efforts on this!
author_association: CONTRIBUTOR
user: login: jayhuang-suse
created_at: 2025-04-29T16:12:10Z
body: > Thanks for your clarification. It makes much more sense now. Some thoughts:
> 
> * Would it be beneficial if we include the whole `c.info.Labels` instead of returning `k8sResourceString`? This way we will have more materials to detect next time. For example, labels like `"org.opencontainers.image.version": "24.04"` would be very useful in your scenario.
> 
> ```
> # ctr -n k8s.io containers info 89d3b87c322c6504a13649116cca960c14cebd0fdf82ccdf939c26b1f24db327
> ...
>     "Labels": {
>         "io.cri-containerd.kind": "container",
>         "io.kubernetes.container.name": "ubuntu",
>         "io.kubernetes.pod.name": "ubuntu-deployment-5fccb98689-4kmmw",
>         "io.kubernetes.pod.namespace": "default",
>         "io.kubernetes.pod.uid": "b11ab672-25f7-45fb-8b4a-b2e29cc60f25",
>         "org.opencontainers.image.ref.name": "ubuntu",
>         "org.opencontainers.image.version": "24.04"
>     },
> ```

After researching the existing k8s CVE, they are not merely from the labels (not all labels are useful). If we need to add more cases from the k8s resources, it needs more study, This PR is partial and a beginning solution. Not suggest to add more information now.   

By the way, those image resources, "org.opencontainers.image.version", are not parts of the k8s resource criteria.
author_association: CONTRIBUTOR
user: login: jayhuang-suse
created_at: 2025-04-29T16:13:40Z
body: > Hi @jayhuang-suse,
> 
> Thank you for recreating the PR and providing important context about this change.
> 
> However, based on the previous [PR discussion](https://github.com/neuvector/neuvector/pull/1933#discussion_r2064322152), would it be possible to first try fetching the resource name and version using the native API, similar to the [GetResource](https://github.com/neuvector/neuvector/blob/c459e5c7e4e84330166d037e8af91ae9d3bf51d0/controller/resource/kubernetes_resource.go#L1650) function in kubernetes_resource.go?
> 
> ### examples
> 1. [neuvector\controller\controller.go](https://github.com/neuvector/neuvector/blob/c459e5c7e4e84330166d037e8af91ae9d3bf51d0/controller/controller.go#L1138):
> 
> * global.ORCH.GetResource(resource.RscTypePod, domain, podname)
> 
> 2. [controller\resource\kubernetes_resource.go](https://github.com/neuvector/neuvector/blob/c459e5c7e4e84330166d037e8af91ae9d3bf51d0/controller/resource/kubernetes_resource.go#L2233):
> 
> * pods, err := global.ORCH.ListResource(RscTypePod, NvAdmSvcNamespace)
> 
> If that attempt fails, we could then fall back to the new function you created.
> 
> Thanks again for your efforts on this!

Unfortunately, at enforcer (data plane), it is not allowed to make k8s queries to minimize the data usages,  
author_association: CONTRIBUTOR
user: login: jayhuang-suse
created_at: 2025-04-29T16:28:14Z
body: Updated and the the scanner's PR needs to sync with the new shared neuvector packages. 
author_association: CONTRIBUTOR
user: login: holyspectral
created_at: 2025-05-05T14:04:19Z
body: The commits were not squashed before this PR is merged.  Is my comment not clear enough or?  

This is not how we are supposed to do with rebase and merge and not what we agreed on with git approach.  Unfortunately I will not give my approval until the changes are arranged into reasonable commits next time.  
author_association: COLLABORATOR
user: login: jayhuang-suse
created_at: 2025-05-05T16:04:36Z
body: I am confused, Do we have a button of the "Squash and Merge"? 
author_association: CONTRIBUTOR
user: login: holyspectral
created_at: 2025-05-05T16:20:51Z
body: You can use `git rebase -i` and merge commits together using `squash`.  This [article](https://medium.com/@slamflipstrom/a-beginners-guide-to-squashing-commits-with-git-rebase-8185cf6e62ec) provides a good example regarding how to do it.  

If you use GUI, there would be slight difference, but the concept is the same.  
author_association: COLLABORATOR