Repository: google/turbinia (language: python)

Issue:
Scale GKE nodes in addition to pods
Currently we scale on pods, but it would be nice to scale on nodes as well.


Discussion / hints:
FYI @wajihyassine 
Briefely tested this while I was building out the Celery/Redis k8s scripts and it seemed fine ,just need to run a few tests to confirm but changes should hopefully be minor. Will assign to add this in sometime when the Celery/Redis k8s deployment is merged.
GKE autoscaling docs: https://cloud.google.com/kubernetes-engine/docs/concepts/cluster-autoscaler

Would just need to add in `--enable-autoscaling --min-nodes=1 --max-nodes=MAX` then adjust `.clusterconfig` accepting a minimum node and maximum node assuming no other problems


Fix patch:
diff --git a/k8s/common/turbinia-autoscale-cpu.yaml b/k8s/common/turbinia-autoscale-cpu.yaml
index 51d6685aa..ff8e99691 100644
--- a/k8s/common/turbinia-autoscale-cpu.yaml
+++ b/k8s/common/turbinia-autoscale-cpu.yaml
@@ -8,5 +8,5 @@ spec:
     kind: Deployment
     name: turbinia-worker
   minReplicas: 3
-  maxReplicas: 50
+  maxReplicas: 500
   targetCPUUtilizationPercentage: 90
diff --git a/k8s/common/turbinia-worker.yaml b/k8s/common/turbinia-worker.yaml
index 4a6676837..793b1987e 100644
--- a/k8s/common/turbinia-worker.yaml
+++ b/k8s/common/turbinia-worker.yaml
@@ -65,8 +65,8 @@ spec:
             - containerPort: 9200
           resources: 
             requests:
-              memory: "256Mi"
-              cpu: "500m"
+              memory: "2048Mi"
+              cpu: "1500m"
             limits:
               memory: "8192Mi"
               cpu: "32000m"
diff --git a/k8s/tools/.clusterconfig b/k8s/tools/.clusterconfig
index c0fbdb4a1..3a5cdf9e6 100644
--- a/k8s/tools/.clusterconfig
+++ b/k8s/tools/.clusterconfig
@@ -28,17 +28,18 @@ VPC_NETWORK='default'
 # to communicate privately.
 VPC_CONTROL_PANE='172.16.0.0/28' # Set to default
 
-# The cluster name, number of nodes, machine type and disk size of the
-# deployed cluster and nodes within it.
+# The cluster name, number of minimum and maximum nodes, machine type and disk 
+# size of the deployed cluster and nodes within it.
 CLUSTER_NAME=$INSTANCE_ID
-CLUSTER_NODE_SIZE='1'
+CLUSTER_MIN_NODE_SIZE='1'
+CLUSTER_MAX_NODE_SIZE='20'
 CLUSTER_MACHINE_TYPE='e2-standard-32'
-CLUSTER_MACHINE_SIZE='1000'
+CLUSTER_MACHINE_SIZE='200'
 
 # The Filestore share name, and size. Filestore will be used to retain shared output
 # from Turbinia, such as logs. Please specify size in terabytes(TB).
 FILESTORE_NAME='turbiniavolume'
-FILESTORE_CAPACITY='1T'
+FILESTORE_CAPACITY='10T'
 
 # Filestore share names, and sizes for dfDewey datastores (if deployed).
 FILESTORE_DFDEWEY_NAME='dfdeweyvolume'
diff --git a/k8s/tools/deploy-celery-gke.sh b/k8s/tools/deploy-celery-gke.sh
index 503a08e7e..b3a41e3d8 100755
--- a/k8s/tools/deploy-celery-gke.sh
+++ b/k8s/tools/deploy-celery-gke.sh
@@ -25,6 +25,7 @@ if [[ "$*" == *--help ||  "$*" == *-h ]] ; then
   echo "--build-experimental           Deploy Turbinia experimental docker image"
   echo "--no-cluster                   Do not create the cluster"
   echo "--no-filestore                 Do not deploy Turbinia Filestore"
+  echo "--no-node-autoscale            Do not enable Node autoscaling"
   echo "--deploy-controller            Deploy Turbinia controller for load testing and troubleshooting" 
   echo "--deploy-dfdewey               Deploy dfDewey datastores"
   exit 1
@@ -126,8 +127,14 @@ cp ../turbinia/config/turbinia_config_tmpl.py $DEPLOYMENT_FOLDER/$TURBINIA_CONFI
 if [[ "$*" != *--no-cluster* ]] ; then
   echo "Enabling Container API"
   gcloud -q --project $DEVSHELL_PROJECT_ID services enable container.googleapis.com
-  echo "Creating cluser $CLUSTER_NAME with $CLUSTER_NODE_SIZE node(s) configured with machine type $CLUSTER_MACHINE_TYPE and disk size $CLUSTER_MACHINE_SIZE"
-  gcloud -q --project $DEVSHELL_PROJECT_ID container clusters create $CLUSTER_NAME --machine-type $CLUSTER_MACHINE_TYPE --disk-size $CLUSTER_MACHINE_SIZE --num-nodes $CLUSTER_NODE_SIZE --master-ipv4-cidr $VPC_CONTROL_PANE --network $VPC_NETWORK --zone $ZONE --shielded-secure-boot --shielded-integrity-monitoring --no-enable-master-authorized-networks --enable-private-nodes --enable-ip-alias --scopes "https://www.googleapis.com/auth/cloud-platform" --labels "turbinia-infra=true" --workload-pool=$DEVSHELL_PROJECT_ID.svc.id.goog
+  if [[ "$*" != *--no-node-autoscale* ]] ; then
+    echo "Creating cluster $CLUSTER_NAME with a minimum node size of $CLUSTER_MIN_NODE_SIZE to scale up to a maximum node size of $CLUSTER_MAX_NODE_SIZE. Each node will be configured with a machine type $CLUSTER_MACHINE_TYPE and disk size of $CLUSTER_MACHINE_SIZE"
+    gcloud -q --project $DEVSHELL_PROJECT_ID container clusters create $CLUSTER_NAME --machine-type $CLUSTER_MACHINE_TYPE --disk-size $CLUSTER_MACHINE_SIZE --num-nodes $CLUSTER_MIN_NODE_SIZE --master-ipv4-cidr $VPC_CONTROL_PANE --network $VPC_NETWORK --zone $ZONE --shielded-secure-boot --shielded-integrity-monitoring --no-enable-master-authorized-networks --enable-private-nodes --enable-ip-alias --scopes "https://www.googleapis.com/auth/cloud-platform" --labels "turbinia-infra=true" --workload-pool=$DEVSHELL_PROJECT_ID.svc.id.goog --default-max-pods-per-node=50 --enable-autoscaling --min-nodes=$CLUSTER_MIN_NODE_SIZE --max-nodes=$CLUSTER_MAX_NODE_SIZE
+  else
+    echo "--no-node-autoscale specified. Node size will remain constant at $CLUSTER_MIN_NODE_SIZE node(s)"
+    echo "Creating cluster $CLUSTER_NAME with a node size of $CLUSTER_MIN_NODE_SIZE. Each node will be configured with a machine type $CLUSTER_MACHINE_TYPE and disk size of $CLUSTER_MACHINE_SIZE"
+    gcloud -q --project $DEVSHELL_PROJECT_ID container clusters create $CLUSTER_NAME --machine-type $CLUSTER_MACHINE_TYPE --disk-size $CLUSTER_MACHINE_SIZE --num-nodes $CLUSTER_MIN_NODE_SIZE --master-ipv4-cidr $VPC_CONTROL_PANE --network $VPC_NETWORK --zone $ZONE --shielded-secure-boot --shielded-integrity-monitoring --no-enable-master-authorized-networks --enable-private-nodes --enable-ip-alias --scopes "https://www.googleapis.com/auth/cloud-platform" --labels "turbinia-infra=true" --workload-pool=$DEVSHELL_PROJECT_ID.svc.id.goog --default-max-pods-per-node=50
+  fi
 else
   echo "--no-cluster specified. Authenticating to pre-existing cluster $CLUSTER_NAME"
 fi


Test patch:
diff --git a/k8s/tools/load-test.sh b/k8s/tools/load-test.sh
index c32ae4089..00c09ec9c 100644
--- a/k8s/tools/load-test.sh
+++ b/k8s/tools/load-test.sh
@@ -16,11 +16,11 @@ fi
 # Please ensure the disks you are testing have been created prior to running
 # this script as it does not handle test disk creation. Once that is done, update
 # the values below with the correct test disks.
-MAX_DISKS=5
-DISK_NAME='test-disk-25gb'
+MAX_DISKS=100
+DISK_NAME='test-disk-20gb'
 DISK_ZONE='us-central1-f'
 
-for i in $(seq 1 $MAX_DISKS};
+for i in $(seq 1 $MAX_DISKS);
 do
     echo -n "Start" > $OUTPUT_LOG/$DISK_NAME-$i.log
     date >> $OUTPUT_LOG/$DISK_NAME-$i.log