Repository: google/turbinia

Issue:
Filter Container/Docker enumeration output
### What would this feature improve or what problem would it solve?

Having fewer tasks makes it easier to see the results.  This would also give more flexibility in processing things selectively.

### What is the feature you are proposing?

Filter the Container/Docker Enumeration output to exclude snaphots by default and add a `TASK_CONFIG` variable to be able to control what get filtered out.  Make sure that the *Enumeration report output makes it clear what is being filtered out.  Also set a limit to the number of containers enumerated by default (and have corresponding `TASK_CONFIG` var).

### What alternatives have you considered?

_No response_


Discussion:
It'd be nice if it somehow can tag or filter k8s system level pods as I think that is one of the main causes of so many timelines/tasks and can pick out "user deployed ones"
As per side conversation:
- filter out default system ones
- have an option to include *all* containers if user wants that
- have an option where the user can specifically provide the container names they want to have analysed (and ignore all others)
I'm consolidating the other filtering of system containers into this issue as well.

Some other filtering ideas from an old thread:
- Filter last N started containers
- Show only containers with an exposed port

Resolving patch:
diff --git a/k8s/tools/get-system-pods-list.py b/k8s/tools/get-system-pods-list.py
new file mode 100644
--- /dev/null
+++ b/k8s/tools/get-system-pods-list.py
@@ -0,0 +1,70 @@
+# Copyright 2023 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Script to enumerate system pods.
+
+This is to generate a list of containers that we can filter out by default in
+the containr/docker enumeration tasks.
+"""
+
+import json
+import subprocess
+import sys
+
+if len(sys.argv) < 3:
+  print(
+      f'usage: {sys.argv[0]} <project> <cluster to list> [<zone if other than us-central-f>]'
+  )
+  sys.exit(1)
+
+project = sys.argv[1]
+cluster = sys.argv[2]
+if len(sys.argv) >= 4:
+  zone = sys.argv[3]
+else:
+  zone = 'us-central1-f'
+namespaces = {}
+
+# Authenticate to cluster
+auth_cmd = f'gcloud container clusters get-credentials {cluster} --zone {zone} --project {project}'
+print(f'Authenticating to project {project} cluster {cluster} zone {zone}')
+subprocess.check_call(auth_cmd.split(' '))
+
+# Get pods data
+cmd = f"kubectl get pods -o json -A"
+pods_data = subprocess.check_output(cmd.split(' '))
+pods_data = json.loads(pods_data)
+
+filtered_data = []
+
+for item in pods_data['items']:
+  if not item.get('metadata'):
+    continue
+
+  name = item.get('metadata').get('name')
+  namespace = item.get('metadata').get('namespace')
+
+  if namespace not in namespaces:
+    namespaces[namespace] = []
+
+  for container in item['spec']['containers']:
+    image = container["image"].split('@')[0]
+    image = image.split(':')[0]
+    namespaces[namespace].append(
+        f'Pod Name: {name}, Container Name: {container["name"]} Image: {image}')
+
+print()
+for namespace, containers in namespaces.items():
+  print(f'Namespace: {namespace}')
+  for container_info in containers:
+    print(f'\t{container_info}')
\ No newline at end of file
diff --git a/turbinia/evidence.py b/turbinia/evidence.py
--- a/turbinia/evidence.py
+++ b/turbinia/evidence.py
@@ -1266,11 +1266,15 @@ class ContainerdContainer(Evidence):
   REQUIRED_ATTRIBUTES = ['namespace', 'container_id']
   POSSIBLE_STATES = [EvidenceState.CONTAINER_MOUNTED]
 
-  def __init__(self, namespace=None, container_id=None, *args, **kwargs):
+  def __init__(
+      self, namespace=None, container_id=None, image_name=None, pod_name=None,
+      *args, **kwargs):
     """Initialization of containerd container."""
     super(ContainerdContainer, self).__init__(*args, **kwargs)
     self.namespace = namespace
     self.container_id = container_id
+    self.image_name = image_name if image_name else 'UnknownImageName'
+    self.pod_name = pod_name if pod_name else 'UnknownPodName'
     self._image_path = None
     self._container_fs_path = None
 
@@ -1282,9 +1286,12 @@ def name(self):
       return self._name
 
     if self.parent_evidence:
-      return ':'.join((self.parent_evidence.name, self.container_id))
+      return ':'.join((
+          self.parent_evidence.name, self.image_name, self.pod_name,
+          self.container_id))
     else:
-      return ':'.join((self.type, self.container_id))
+      return ':'.join(
+          (self.type, self.image_name, self.pod_name, self.container_id))
 
   def _preprocess(self, _, required_states):
     if EvidenceState.CONTAINER_MOUNTED in required_states:
diff --git a/turbinia/workers/containerd.py b/turbinia/workers/containerd.py
--- a/turbinia/workers/containerd.py
+++ b/turbinia/workers/containerd.py
@@ -32,6 +32,7 @@
 
 CE_BINARY = '/opt/container-explorer/bin/ce'
 CE_SUPPORT_FILE = '/opt/container-explorer/etc/supportcontainer.yaml'
+POD_NAME_LABEL = 'io.kubernetes.pod.name'
 
 
 class ContainerdEnumerationTask(TurbiniaTask):
@@ -39,6 +40,46 @@ class ContainerdEnumerationTask(TurbiniaTask):
 
   REQUIRED_STATES = [state.ATTACHED, state.MOUNTED]
 
+  TASK_CONFIG = {
+      # These filters will all match on partial matches, e.g. an image filter of
+      # ['gke.gcr.io/'] will filter out image `gke.gcr.io/event-exporter`.
+      #
+      # Which k8 namespaces to filter out by default
+      'filter_namespaces': ['kube-system'],
+      'filter_pod_names': ['sidecar', 'k8s-sidecar', 'konnectivity-agent'],
+      # Taken from
+      # https://github.com/google/container-explorer/blob/main/supportcontainer.yaml
+      'filter_images': [
+          'gcr.io/gke-release-staging/cluster-proportional-autoscaler-amd64',
+          'gcr.io/k8s-ingress-image-push/ingress-gce-404-server-with-metrics',
+          'gke.gcr.io/ingress-gce-404-server-with-metrics',
+          'gke.gcr.io/cluster-proportional-autoscaler',
+          'gke.gcr.io/csi-node-driver-registrar',
+          'gke.gcr.io/event-exporter',
+          'gke.gcr.io/fluent-bit',
+          'gke.gcr.io/fluent-bit-gke-exporter',
+          'gke.gcr.io/gcp-compute-persistent-disk-csi-driver',
+          'gke.gcr.io/gke-metrics-agent',
+          'gke.gcr.io/k8s-dns-dnsmasq-nanny',
+          'gke.gcr.io/k8s-dns-kube-dns',
+          'gke.gcr.io/k8s-dns-sidecar',
+          'gke.gcr.io/kube-proxy-amd64',
+          'gke.gcr.io/prometheus-to-sd',
+          'gke.gcr.io/proxy-agent',
+          'k8s.gcr.io/metrics-server/metrics-server',
+          'gke.gcr.io/metrics-server',
+          'k8s.gcr.io/pause',
+          'gke.gcr.io/pause',
+          'gcr.io/gke-release-staging/addon-resizer',
+          'gcr.io/gke-release-staging/cpvpa-amd64',
+          'gcr.io/google-containers/pause-amd64',
+          'gke.gcr.io/addon-resizer',
+          'gke.gcr.io/cpvpa-amd64',
+          'k8s.gcr.io/kube-proxy-amd64',
+          'k8s.gcr.io/prometheus-to-sd',
+      ],
+  }
+
   def list_containers(self, evidence, _, detailed_output=False):
     """List containerd containers in the evidence.
 
@@ -95,8 +136,8 @@ def _list_containers_result(self, containers, detailed_output):
       return containers
 
     basic_fields = [
-        'Namespace', 'Image', 'ContainerType', 'ID', 'Hostname', 'CreatedAt'
-        'Labels'
+        'Name', 'Namespace', 'Image', 'ContainerType', 'ID', 'Hostname',
+        'CreatedAt', 'Labels'
     ]
     basic_containers = []
 
@@ -123,10 +164,16 @@ def run(self, evidence, result):
     summary = ''
     success = False
     report_data = []
+    filter_namespaces = self.task_config.get('filter_namespaces')
+    filter_pod_names = self.task_config.get('filter_pod_names')
+    filter_images = self.task_config.get('filter_images')
+    filtered_container_list = []
 
     image_path = evidence.mount_path
     if not image_path:
-      summary = f'Evidence {evidence.name}:{evidence.source_path} is not mounted'
+      summary = (
+          f'Evidence {evidence.name}:{evidence.source_path} is not '
+          'mounted')
       result.close(self, success=False, status=summary)
       return result
 
@@ -142,20 +189,62 @@ def run(self, evidence, result):
           f'Found {len(container_ids)} containers: {", ".join(container_ids)}')
 
       # 2. Add containers as evidences
+      new_evidence = []
       for container in containers:
-        namespace = container.get('Namespace')
-        container_id = container.get('ID')
+        namespace = container.get('Namespace', 'UnknownNamespace')
+        container_id = container.get('ID', 'UnknownContainerID')
+        if container.get('Labels'):
+          pod_name = container.get('Labels').get(
+              POD_NAME_LABEL, 'UnknownPodName')
+        else:
+          pod_name = 'UnknownPodName'
         container_type = container.get('ContainerType') or None
+        image = container.get('Image')
+        if image:
+          image_short = image.split('@')[0]
+          image_short = image_short.split(':')[0]
+        else:
+          image_short = 'UnknownImageName'
 
         if not namespace or not container_id:
-          result.log(
-              f'Value is empty. namespace={namespace}, container_id={container_id}'
-          )
-          report_data.append(
+          message = (
               f'Skipping container with empty value namespace ({namespace})'
               f' or container_id ({container_id})')
+          result.log(message)
+          report_data.append(message)
           continue
 
+        # Filter out configured namespaces/containers/images.  Even though we
+        # could let container explorer filter these before we get them we want
+        # to do it here so that we can report on what was available and filtered
+        # out to give the analyst the option to reprocess these containers.
+        if filter_namespaces:
+          if namespace in filter_namespaces:
+            message = (
+                f'Filtering out container {container_id} because namespace '
+                f'matches filter.')
+            result.log(message)
+            report_data.append(message)
+            filtered_container_list.append(container_id)
+            continue
+        if filter_images:
+          if image_short in filter_images:
+            message = (
+                f'Filtering out image {image} because image matches filter')
+            result.log(message)
+            report_data.append(message)
+            filtered_container_list.append(container_id)
+            continue
+        if filter_pod_names:
+          if pod_name in filter_pod_names:
+            message = (
+                f'Filtering out container {container_id} because container '
+                f'name matches filter')
+            result.log(message)
+            report_data.append(message)
+            filtered_container_list.append(container_id)
+            continue
+
         # We want to process docker managed container using Docker-Explorer
         if container_type and container_type.lower() == 'docker':
           result.log(
@@ -165,12 +254,29 @@ def run(self, evidence, result):
           continue
 
         container_evidence = ContainerdContainer(
-            namespace=namespace, container_id=container_id)
+            namespace=namespace, container_id=container_id,
+            image_name=image_short, pod_name=pod_name)
+        new_evidence.append(container_evidence.name)
 
         result.add_evidence(container_evidence, evidence.config)
+        result.log(
+            f'Adding container evidence {container_evidence.name} '
+            f'type {container_type}')
+
       summary = (
-          f'Found {len(container_ids)} containers: {", ".join(container_ids)}')
+          f'Found {len(container_ids)} containers, added {len(new_evidence)} '
+          f'(filtered out {len(filtered_container_list)})')
       success = True
+      if filtered_container_list:
+        report_data.append(
+            f'Filtered out {len(filtered_container_list)} containers: '
+            f'{", ".join(filtered_container_list)}')
+        report_data.append(
+            f'Container filter lists: Namespaces: {filter_namespaces}, Images: {filter_images}, '
+            f'Pod Names: {filter_pod_names}')
+        report_data.append(
+            'To process filtered containers, adjust the ContainerEnumeration '
+            'Task config filter* parameters with a recipe')
     except TurbiniaException as e:
       summary = f'Error enumerating containerd containers: {e}'
       report_data.append(summary)


Test patch:
diff --git a/turbinia/workers/containerd_test.py b/turbinia/workers/containerd_test.py
--- a/turbinia/workers/containerd_test.py
+++ b/turbinia/workers/containerd_test.py
@@ -47,10 +47,12 @@ def testContainerdEnumerationTaskRun(self, list_containers_mock, _):
         {
             'Namespace': 'default',
             'ID': 'nginx01',
+            'Image': 'nginx01-image',
         },
         {
             'Namespace': 'default',
             'ID': 'apache01',
+            'Image': 'apache01-image',
         },
     ]
     result = self.task.run(self.evidence, self.result)