Repository: google/turbinia
Pull request #1224 (2023-01-11 01:01:21)

Add node autoscaling to pubsub script
GKE e2e tests failing due to not having same logic to enable node autoscaling/variables changed


Patch:
diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile
index b6ed944..f40538c 100644
--- a/docker/worker/Dockerfile
+++ b/docker/worker/Dockerfile
@@ -87,8 +87,8 @@ COPY turbinia/config/rules/*.yar /opt/signature-base/yara/
 
 RUN mkdir -p /opt/fraken/yara && chown -R turbinia:turbinia /opt/fraken
 COPY --chown=turbinia:turbinia tools/fraken/* /opt/fraken/
-RUN cd /opt/fraken/yara && curl -s https://api.github.com/repos/VirusTotal/Yara/releases/latest | sed -n 's/.*"tarball_url": "\(.*\)",.*/\1/p' | xargs -n1 wget -O - -q | tar -xz --strip-components=1 \
-    && ./bootstrap.sh \
+RUN cd /opt/fraken/yara && curl -s -L --retry 5 "$(curl -s -L --retry 5 https://api.github.com/repos/VirusTotal/Yara/releases/latest | sed -n 's/.*"tarball_url": "\(.*\)",.*/\1/p')"  | tar -xz --strip-components=1
+RUN cd /opt/fraken/yara && ./bootstrap.sh \
     && ./configure \
     && make && sudo make install && sudo ldconfig
 RUN cd /opt/fraken && go build -ldflags="-linkmode=external -extldflags=-ljemalloc" -o fraken
diff --git a/k8s/tools/deploy-pubsub-gke.sh b/k8s/tools/deploy-pubsub-gke.sh
index 1af42a1..5f85448 100755
--- a/k8s/tools/deploy-pubsub-gke.sh
+++ b/k8s/tools/deploy-pubsub-gke.sh
@@ -26,6 +26,7 @@ if [[ "$*" == *--help ]] ; then
   echo "--no-appengine                 Do not enable App Engine"
   echo "--no-datastore                 Do not configure Turbinia Datastore"
   echo "--no-filestore                 Do not deploy Turbinia Filestore"
+  echo "--no-node-autoscale            Do not enable Node autoscaling"
   echo "--no-gcs                       Do not create a GCS bucket"
   echo "--no-pubsub                    Do not create the PubSub and PSQ topic/subscription"
   echo "--no-cluster                   Do not create the cluster"
@@ -170,8 +171,14 @@ fi
 if [[ "$*" != *--no-cluster* ]] ; then
   echo "Enabling Container API"
   gcloud -q --project $DEVSHELL_PROJECT_ID services enable container.googleapis.com
-  echo "Creating cluser $CLUSTER_NAME with $CLUSTER_NODE_SIZE node(s) configured with machine type $CLUSTER_MACHINE_TYPE and disk size $CLUSTER_MACHINE_SIZE"
-  gcloud -q --project $DEVSHELL_PROJECT_ID container clusters create $CLUSTER_NAME --machine-type $CLUSTER_MACHINE_TYPE --disk-size $CLUSTER_MACHINE_SIZE --num-nodes $CLUSTER_NODE_SIZE --master-ipv4-cidr $VPC_CONTROL_PANE --network $VPC_NETWORK --zone $ZONE --shielded-secure-boot --shielded-integrity-monitoring --no-enable-master-authorized-networks --enable-private-nodes --enable-ip-alias --scopes "https://www.googleapis.com/auth/cloud-platform" --labels "turbinia-infra=true" --workload-pool=$DEVSHELL_PROJECT_ID.svc.id.goog
+  if [[ "$*" != *--no-node-autoscale* ]] ; then
+    echo "Creating cluster $CLUSTER_NAME with a minimum node size of $CLUSTER_MIN_NODE_SIZE to scale up to a maximum node size of $CLUSTER_MAX_NODE_SIZE. Each node will be configured with a machine type $CLUSTER_MACHINE_TYPE and disk size of $CLUSTER_MACHINE_SIZE"
+    gcloud -q --project $DEVSHELL_PROJECT_ID container clusters create $CLUSTER_NAME --machine-type $CLUSTER_MACHINE_TYPE --disk-size $CLUSTER_MACHINE_SIZE --num-nodes $CLUSTER_MIN_NODE_SIZE --master-ipv4-cidr $VPC_CONTROL_PANE --network $VPC_NETWORK --zone $ZONE --shielded-secure-boot --shielded-integrity-monitoring --no-enable-master-authorized-networks --enable-private-nodes --enable-ip-alias --scopes "https://www.googleapis.com/auth/cloud-platform" --labels "turbinia-infra=true" --workload-pool=$DEVSHELL_PROJECT_ID.svc.id.goog --default-max-pods-per-node=50 --enable-autoscaling --min-nodes=$CLUSTER_MIN_NODE_SIZE --max-nodes=$CLUSTER_MAX_NODE_SIZE
+  else
+    echo "--no-node-autoscale specified. Node size will remain constant at $CLUSTER_MIN_NODE_SIZE node(s)"
+    echo "Creating cluster $CLUSTER_NAME with a node size of $CLUSTER_MIN_NODE_SIZE. Each node will be configured with a machine type $CLUSTER_MACHINE_TYPE and disk size of $CLUSTER_MACHINE_SIZE"
+    gcloud -q --project $DEVSHELL_PROJECT_ID container clusters create $CLUSTER_NAME --machine-type $CLUSTER_MACHINE_TYPE --disk-size $CLUSTER_MACHINE_SIZE --num-nodes $CLUSTER_MIN_NODE_SIZE --master-ipv4-cidr $VPC_CONTROL_PANE --network $VPC_NETWORK --zone $ZONE --shielded-secure-boot --shielded-integrity-monitoring --no-enable-master-authorized-networks --enable-private-nodes --enable-ip-alias --scopes "https://www.googleapis.com/auth/cloud-platform" --labels "turbinia-infra=true" --workload-pool=$DEVSHELL_PROJECT_ID.svc.id.goog --default-max-pods-per-node=50
+  fi
 else
   echo "--no-cluster specified. Authenticating to pre-existing cluster $CLUSTER_NAME"
 fi
diff --git a/turbinia/api/cli/requirements.txt b/turbinia/api/cli/requirements.txt
index b03cb4c..dca9a90 100644
--- a/turbinia/api/cli/requirements.txt
+++ b/turbinia/api/cli/requirements.txt
@@ -1,2 +1,1 @@
-turbinia-api-client
 click
diff --git a/turbinia/processors/mount_local.py b/turbinia/processors/mount_local.py
index 7b77937..9b4b7d3 100644
--- a/turbinia/processors/mount_local.py
+++ b/turbinia/processors/mount_local.py
@@ -495,7 +495,7 @@ def PreprocessMountPartition(partition_path, filesystem_type):
     # everything read-only.
     mount_cmd.extend(['-o', 'noload'])
   elif filesystem_type == 'XFS':
-    mount_cmd.extend(['-o', 'norecovery'])
+    mount_cmd.extend(['-o', 'norecovery', '-o', 'nouuid'])
   mount_cmd.extend([partition_path, mount_path])
 
   log.info('Running: {0:s}'.format(' '.join(mount_cmd)))


Test patch:
diff --git a/turbinia/processors/mount_local_test.py b/turbinia/processors/mount_local_test.py
index 6790e19..6f2e9b4 100644
--- a/turbinia/processors/mount_local_test.py
+++ b/turbinia/processors/mount_local_test.py
@@ -371,8 +371,8 @@ class MountLocalProcessorTest(unittest.TestCase):
     mock_subprocess.reset_mock()
     mount_path = mount_local.PreprocessMountPartition('/dev/loop0', 'XFS')
     expected_args = [
-        'sudo', 'mount', '-o', 'ro', '-o', 'norecovery', '/dev/loop0',
-        '/mnt/turbinia/turbinia0ckdntz0'
+        'sudo', 'mount', '-o', 'ro', '-o', 'norecovery', '-o', 'nouuid',
+        '/dev/loop0', '/mnt/turbinia/turbinia0ckdntz0'
     ]
     mock_subprocess.assert_called_once_with(expected_args)
     self.assertEqual(mount_path, '/mnt/turbinia/turbinia0ckdntz0')