Repository: openshift/sandboxed-containers-operator
Pull request #584: podvm: Check for pre-existing image before image creation
Author: login: bpradipt
Labels: name: lgtm
description: Indicates that a PR is ready to be merged.
Created: 2025-03-19T10:54:32Z | Merged: 2025-03-25T12:14:51Z

For OSC, we expect only one image to be created by the operator. The IMAGE_VERSION (for Azure) or AMI_VERSION (for AWS) uses clusterID as the unique id.  This helps to check if the image already exists before recreating the image again.

<!--
If this is a bug fix, make sure your description includes "Fixes: #xxxx", or
"Closes: #xxxx"

Please provide the following information:
-->
**- Description of the problem which is fixed/What is the use case**

**- What I did**

**- How to verify it**

**- Description for the changelog**
<!--
Write a short (one line) summary that describes the changes in this
pull request for inclusion in the changelog:
-->



Diff:
diff --git a/config/peerpods/podvm/aws-podvm-image-cm.yaml b/config/peerpods/podvm/aws-podvm-image-cm.yaml
index dce44d67c..a3903cd3c 100644
--- a/config/peerpods/podvm/aws-podvm-image-cm.yaml
+++ b/config/peerpods/podvm/aws-podvm-image-cm.yaml
@@ -9,7 +9,11 @@ data:
 
   # Image
   AMI_BASE_NAME: "podvm-image"
-  AMI_VERSION_MAJ_MIN: "0.0"
+  # Must be in Major(int).Minor(int).Patch(int) format. Not mandatory for AWS, but
+  # for consistency sake with Azure
+  # When deployed from operator, update the AMI_VERSION to something like
+  # "$clusterid.0.1"
+  AMI_VERSION: "0.0.1"
   AMI_VOLUME_SIZE: "30"
 
   # Packer Instance type
diff --git a/config/peerpods/podvm/aws-podvm-image-handler.sh b/config/peerpods/podvm/aws-podvm-image-handler.sh
index c492d10aa..2caa7b2b0 100755
--- a/config/peerpods/podvm/aws-podvm-image-handler.sh
+++ b/config/peerpods/podvm/aws-podvm-image-handler.sh
@@ -63,7 +63,7 @@ function verify_vars() {
     [[ ! "${AWS_SUBNET_ID}" ]] && error_exit "AWS_SUBNET_ID is not set"
 
     [[ -z "${AMI_BASE_NAME}" ]] && error_exit "AMI_BASE_NAME is not set"
-    [[ -z "${AMI_VERSION_MAJ_MIN}" ]] && error_exit "AMI_VERSION_MAJ_MIN is not set"
+    [[ -z "${AMI_VERSION}" ]] && error_exit "AMI_VERSION is not set"
     [[ -z "${AMI_VOLUME_SIZE}" ]] && error_exit "AMI_VOLUME_SIZE is not set"
 
     [[ -z "${PODVM_DISTRO}" ]] && error_exit "PODVM_DISTRO is not set"
@@ -116,7 +116,7 @@ function create_ami_using_packer() {
     # The variables are set before calling the function
 
     # If PODVM_DISTRO is not set to rhel then exit
-    [[ "${PODVM_DISTRO}" != "rhel" ]] && error_exit "unsupport distro"
+    [[ "${PODVM_DISTRO}" != "rhel" ]] && error_exit "unsupported distro"
 
     # Set the packer variables
 
@@ -142,11 +142,6 @@ function create_ami_using_packer() {
 }
 
 function set_ami_name() {
-    # Set the AMI version
-    # It should follow the Major(int).Minor(int).Patch(int)
-    AMI_VERSION="${AMI_VERSION_MAJ_MIN}.$(date +'%Y%m%d%S')"
-    export AMI_VERSION
-
     # Set the image name
     AMI_NAME="${AMI_BASE_NAME}-${AMI_VERSION}"
     export AMI_NAME
@@ -379,6 +374,22 @@ function create_ami_from_scratch() {
 function create_ami() {
     echo "Creating AWS AMI"
 
+    # generate & set the ami name
+    set_ami_name
+
+    ami_exists
+    ami_status=$?
+
+    if [[ "${ami_status}" -eq 0 ]]; then
+        echo "AMI exists. Skipping creation"
+        return
+    elif [[ "${ami_status}" -eq 2 ]]; then
+        echo "De-registering AMI (${AMI_NAME}) and deleting associated snapshot, before recreating"
+        delete_ami_using_name
+    fi
+
+    echo "AMI does not exist. Proceeding to create the AMI"
+
     # Create the AWS image
     # If any error occurs, exit the script with an error message
 
@@ -391,16 +402,13 @@ function create_ami() {
         install_binary_packages
     fi
 
-    # generate & set the ami name
-    set_ami_name
-
     if [[ "${IMAGE_TYPE}" == "operator-built" ]]; then
         create_ami_from_scratch
     elif [[ "${IMAGE_TYPE}" == "pre-built" ]]; then
         create_ami_from_prebuilt_artifact
     fi
 
-    # Get the ami id of the newly created image
+    # Get the ami id
     # This will set the AMI_ID environment variable
     get_ami_id
 
@@ -476,6 +484,80 @@ function delete_ami_using_id() {
 
 }
 
+# Function to delete AMI and associated snapshots by name
+# AMI_NAME must be set as an environment variable
+
+function delete_ami_using_name() {
+    echo "Deleting AWS AMI by name"
+
+    # Delete the AMI by name
+    # If any error occurs, exit the script with an error message
+
+    # AMI_NAME shouldn't be empty
+    [[ -z "${AMI_NAME}" ]] && error_exit "AMI_NAME is empty"
+
+    # Retrieve the AMI ID before deleting the ami
+    AMI_ID=$(
+        aws ec2 describe-images \
+            --region "${AWS_REGION}" \
+            --filters "Name=name,Values=${AMI_NAME}" \
+            --query 'Images[*].ImageId' \
+            --output text
+    ) || error_exit "Failed to get the ami id"
+
+    # If AMI_ID is empty, then nothing to do. Return
+    [[ -z "${AMI_ID}" ]] && return
+
+    delete_ami_using_id
+
+}
+
+# Check if AMI name exists
+# This checks if AMI already exist in Aws
+# and the LATEST_AMI_ID annotation is set in the peer-pods-cm configmap
+# 0: AMI exists and matches
+# 1: AMI does not exist
+# 2: AMI mismatch (caller should delete AMI & snapshot)
+# The required variables are assumed to be set
+function ami_exists() {
+    echo "Checking if AMI exists"
+
+    local ami_id latest_ami_id ami_return_code
+
+    # Retrieve the AMI ID from AWS
+    ami_id=$(aws ec2 describe-images --region "${AWS_REGION}" --filters "Name=name,Values=${AMI_NAME}" --query 'Images[*].ImageId' --output text)
+    ami_return_code=$?
+
+    # Retrieve the latest AMI ID from the configmap
+    latest_ami_id=$(kubectl get configmap peer-pods-cm -n openshift-sandboxed-containers-operator -o jsonpath='{.metadata.annotations.LATEST_AMI_ID}')
+
+    # Handle AWS command failure
+    if [[ "${ami_return_code}" -ne 0 ]]; then
+        error_exit "Error querying AWS for AMI."
+    fi
+
+    # Case 1: AMI exists and matches the configmap.
+    # In AWS ami_id will be empty if AMI doesn't exist. Unlike Azure which returns an error
+    # So we need to additionally check that ami_id is not empty
+    if [[ -n "${ami_id}" && -n "${latest_ami_id}" && "${ami_id}" == "${latest_ami_id}" ]]; then
+        echo "AMI (${ami_id}) is up-to-date in AWS and configmap."
+        return 0
+    # Case 2: No AMI in AWS and no record in the configmap
+    elif [[ -z "${ami_id}" && -z "${latest_ami_id}" ]]; then
+        echo "No AMI found in AWS, and no record in configmap."
+        return 1
+    # Case 3: AMI missing in AWS but present in configmap
+    elif [[ -z "${ami_id}" && -n "${latest_ami_id}" ]]; then
+        echo "No AMI found in AWS, but configmap has record (${latest_ami_id}). AMI might have been deleted."
+        return 1
+    # Case 4: AMI exists in AWS but does not match configmap.
+    else
+        echo "AMI ID mismatch: AWS AMI (${ami_id}) differs from ConfigMap AMI (${latest_ami_id})."
+        return 2 # Caller should delete AMI and snapshot and recreate AMI
+    fi
+
+}
+
 # display help message
 
 function display_help() {
diff --git a/config/peerpods/podvm/azure-podvm-image-cm.yaml b/config/peerpods/podvm/azure-podvm-image-cm.yaml
index d326378b7..28609f274 100644
--- a/config/peerpods/podvm/azure-podvm-image-cm.yaml
+++ b/config/peerpods/podvm/azure-podvm-image-cm.yaml
@@ -33,7 +33,10 @@ data:
 
   # Image
   IMAGE_BASE_NAME: "podvm-image"
-  IMAGE_VERSION_MAJ_MIN: "0.0"
+  # Must be in Major(int).Minor(int).Patch(int) format
+  # When deployed from operator, update the IMAGE_VERSION to something like
+  # "$clusterid.0.1"
+  IMAGE_VERSION: "0.0.1"
 
   # Packer VM size
   VM_SIZE: "Standard_D2as_v5"
diff --git a/config/peerpods/podvm/azure-podvm-image-handler.sh b/config/peerpods/podvm/azure-podvm-image-handler.sh
index 7855a40d4..d4b99893f 100755
--- a/config/peerpods/podvm/azure-podvm-image-handler.sh
+++ b/config/peerpods/podvm/azure-podvm-image-handler.sh
@@ -63,7 +63,7 @@ function verify_vars() {
 
     # Ensure that the image variables are set
     [[ -z "${IMAGE_BASE_NAME}" ]] && error_exit "IMAGE_BASE_NAME is empty"
-    [[ -z "${IMAGE_VERSION_MAJ_MIN}" ]] && error_exit "IMAGE_VERSION_MAJ_MIN is empty"
+    [[ -z "${IMAGE_VERSION}" ]] && error_exit "IMAGE_VERSION is empty"
 
     [[ -z "${CAA_SRC}" ]] && error_exit "CAA_SRC is empty"
     [[ -z "${CAA_REF}" ]] && error_exit "CAA_REF is empty"
@@ -98,14 +98,11 @@ function add_azure_repositories() {
     echo "Azure yum repositories added successfully"
 }
 
-function set_image_version_and_name() {
-    # Set the image version
-    # It should follow the Major(int).Minor(int).Patch(int)
-    IMAGE_VERSION="${IMAGE_VERSION_MAJ_MIN}.$(date +'%Y%m%d%S')"
-    export IMAGE_VERSION
+function set_image_name() {
 
     # Set the image name
     IMAGE_NAME="${IMAGE_BASE_NAME}-${IMAGE_VERSION}"
+    echo "Image name: ${IMAGE_NAME}"
     export IMAGE_NAME
 }
 
@@ -242,11 +239,16 @@ function create_image_definition() {
 function create_image_using_packer() {
     echo "Creating Azure image using packer"
 
+    echo "Deleting any leftover managed image (${IMAGE_NAME}) due to abrupt exit of packer build"
+    # This will be of the form
+    # /subscriptions/<subscription-id>/resourceGroups/<resource-group>/providers/Microsoft.Compute/images/<image-name>
+    # Note that this is not tied to gallery
+    # We ignore any errors here.
+    az image delete --name "${IMAGE_NAME}" --resource-group "${AZURE_RESOURCE_GROUP}" || true
+
     # If any error occurs, exit the script with an error message
     # The variables are set before calling the function
 
-    # Set the image version and name
-    set_image_version_and_name
     # Set the base image details
 
     if [[ "${PODVM_DISTRO}" != "rhel" ]]; then
@@ -274,6 +276,10 @@ function create_image_using_packer() {
     packer init "${PODVM_DISTRO}"/
     make BINARIES= PAUSE_BUNDLE= image
 
+    # Check if make resulted in error
+    return_code=$?
+    [[ "$return_code" -ne 0 ]] && error_exit "Failed to create Azure image using packer"
+
     # Wait for the image to be created
 
     echo "Azure image created successfully"
@@ -368,6 +374,22 @@ function create_image() {
     echo "Creating Azure image"
     # If any error occurs, exit the script with an error message
 
+    # Set the image version and name
+    set_image_name
+
+    image_exists
+    image_status=$?
+
+    if [[ "${image_status}" -eq 0 ]]; then
+        echo "Image exists. Skipping creation"
+        return
+    elif [[ "${image_status}" -eq 2 ]]; then
+        echo "Deleting image version (${IMAGE_VERSION}), before recreating"
+        delete_image_version
+    fi
+
+    echo "Image does not exist. Proceeding to create the image"
+
     # Install packages if INSTALL_PACKAGES is set to yes
     if [[ "${INSTALL_PACKAGES}" == "yes" ]]; then
         # Add Azure yum repositories
@@ -387,7 +409,7 @@ function create_image() {
         create_azure_image_from_prebuilt_artifact
     fi
 
-    # Get the image id of the newly created image.
+    # Get the image id
     # This will set the IMAGE_ID variable
     get_image_id
 
@@ -425,9 +447,6 @@ function create_azure_image_from_scratch() {
 function create_azure_image_from_prebuilt_artifact() {
     echo "Creating Azure image from prebuilt artifact"
 
-    # Set the IMAGE_VERSION and IMAGE_NAME
-    set_image_version_and_name
-
     echo "Pulling the podvm image from the provided path"
     image_src="/tmp/image"
     extraction_destination_path="/image"
@@ -556,7 +575,7 @@ function upload_vhd_image() {
 # Function to delete a specific image version from Azure
 
 function delete_image_version() {
-    echo "Deleting Azure image version"
+    echo "Deleting Azure image version (${IMAGE_VERSION})"
     # If any error occurs, exit the script with an error message
 
     # Delete the image version
@@ -566,7 +585,7 @@ function delete_image_version() {
         --gallery-image-version "${IMAGE_VERSION}" ||
         error_exit "Failed to delete the image version"
 
-    echo "Azure image version deleted successfully"
+    echo "Azure image version (${IMAGE_VERSION}) deleted successfully"
 }
 
 # Function delete all image versions from Azure image-definition
@@ -672,9 +691,17 @@ function delete_image_gallery() {
     delete_image_definition
 
     # Delete the image gallery
-    az sig delete --resource-group "${AZURE_RESOURCE_GROUP}" \
-        --gallery-name "${IMAGE_GALLERY_NAME}" ||
-        error_exit "Failed to delete the image gallery"
+    #az sig delete --resource-group "${AZURE_RESOURCE_GROUP}" \
+    #    --gallery-name "${IMAGE_GALLERY_NAME}" ||
+    #    error_exit "Failed to delete the image gallery"
+
+    # Sometimes the delete fails with teh following command:
+    # "ERROR: (CannotDeleteResource) Cannot delete resource while nested resources exist"
+    # This could be temporary due to image definition deletion not being immediate
+    # Hence add retry logic to delete the image gallery
+
+    retry_command az sig delete --resource-group "${AZURE_RESOURCE_GROUP}" --gallery-name "${IMAGE_GALLERY_NAME}" ||
+        echo "Failed to delete the image gallery."
 
     # Remove the image gallery annotation from peer-pods-cm configmap
     delete_image_gallery_annotation_from_peer_pods_cm
@@ -718,6 +745,52 @@ function delete_image_using_id() {
     echo "Azure image deleted successfully"
 }
 
+# Function to check if image already exists
+# This checks if IMAGE_VERSION exist in Azure
+# and the LATEST_IMAGE_ID annotation is set in the peer-pods-cm configmap
+# 0: Image exists and matches
+# 1: Image does not exist
+# 2: Image mismatch (caller should delete Image)
+# The required variables are assumed to be set
+function image_exists() {
+
+    local image_id
+    # Get the image id
+    image_id=$(az sig image-version show --resource-group "${AZURE_RESOURCE_GROUP}" \
+        --gallery-name "${IMAGE_GALLERY_NAME}" \
+        --gallery-image-definition "${IMAGE_DEFINITION_NAME}" \
+        --gallery-image-version "${IMAGE_VERSION}" \
+        --query "id" --output tsv)
+
+    img_return_code=$?
+
+    # Get the latest image id from the peer-pods-cm configmap
+    local latest_image_id
+    latest_image_id=$(kubectl get configmap peer-pods-cm -n openshift-sandboxed-containers-operator -o jsonpath='{.metadata.annotations.LATEST_IMAGE_ID}')
+
+    # Handle Azure command failure
+    if [[ "${img_return_code}" -ne 0 ]]; then
+        # Treating any error as non-existent image as unlike AWS, Azure does not return empty string for non-existent image
+        echo "Image does not exist in Azure."
+        return 1
+    fi
+
+    # Case 1: Image exists and matches the configmap
+    if [[ "${image_id}" == "${latest_image_id}" ]]; then
+        echo "Image (${image_id}) is up-to-date in configmap."
+        return 0
+    # Case 2: Image missing in Azure but present in configmap
+    elif [[ -z "${image_id}" && -n "${latest_image_id}" ]]; then
+        echo "No Image found in Azure, but configmap has record (${latest_image_id}). Image might have been deleted."
+        return 1
+    # Case 3: Image exists in Azure but does not match configmap (it may be empty or have any other image listed).
+    else
+        echo "Image mismatch: Azure Image (${image_id}) differs from ConfigMap Image (${latest_image_id})."
+        return 2 # Caller should delete Azure image version and recreate
+    fi
+
+}
+
 # display help message
 
 function display_help() {
diff --git a/config/peerpods/podvm/gcp-podvm-image-cm.yaml b/config/peerpods/podvm/gcp-podvm-image-cm.yaml
index 8425f13dc..f3d168326 100644
--- a/config/peerpods/podvm/gcp-podvm-image-cm.yaml
+++ b/config/peerpods/podvm/gcp-podvm-image-cm.yaml
@@ -11,6 +11,10 @@ data:
   CAA_REF: "main"
 
   IMAGE_BASE_NAME: "podvm-image"
+  # Must be in Major(int).Minor(int).Patch(int) format
+  # When deployed from operator, update the IMAGE_VERSION to something like
+  # "$clusterid-0-1"
+  IMAGE_VERSION: "0-0-1"
 
   # Booleans
   INSTALL_PACKAGES: "no"
diff --git a/config/peerpods/podvm/gcp-podvm-image-handler.sh b/config/peerpods/podvm/gcp-podvm-image-handler.sh
index faef1c005..da8eed50d 100755
--- a/config/peerpods/podvm/gcp-podvm-image-handler.sh
+++ b/config/peerpods/podvm/gcp-podvm-image-handler.sh
@@ -28,6 +28,7 @@ function verify_vars() {
 
     # From gcp-podvm-image-cm:
     "IMAGE_BASE_NAME"
+    "IMAGE_VERSION"
     "INSTALL_PACKAGES"
     "DISABLE_CLOUD_CONFIG"
 
@@ -101,10 +102,7 @@ function create_image() {
   echo "GCP image created successfully"
 }
 
-function set_image_version_and_name() {
-  IMAGE_VERSION="$(date +'%Y%m%d%S')"
-  export IMAGE_VERSION
-
+function set_image_name() {
   # Set the image name
   IMAGE_NAME="${IMAGE_BASE_NAME}-${IMAGE_VERSION}"
   export IMAGE_NAME
@@ -113,8 +111,8 @@ function set_image_version_and_name() {
 function create_image_from_prebuilt_artifact() {
   echo "Creating GCP image from prebuilt artifact"
 
-  # Set the IMAGE_VERSION and IMAGE_NAME
-  set_image_version_and_name
+  # Set the IMAGE_NAME
+  set_image_name
 
   echo "Pulling the podvm image from the provided path"
   image_src="/tmp/image"
diff --git a/config/peerpods/podvm/lib.sh b/config/peerpods/podvm/lib.sh
index 5995db154..d8022cb13 100644
--- a/config/peerpods/podvm/lib.sh
+++ b/config/peerpods/podvm/lib.sh
@@ -34,7 +34,7 @@ function install_rpm_packages() {
         "skopeo"
         "jq"
         "qemu-img" # for handling pre-built images. Note that this rpm requires subscription
-        "podman" # TODO: can we drop podman/skopeo?
+        "podman"   # TODO: can we drop podman/skopeo?
     )
 
     # Create a new array to store rpm packages that are not installed
@@ -221,14 +221,14 @@ to edit policy.conf"
         else
             error_exit "Invalid AGENT_POLICY value set, expected base64 encoded valid agent policy, got: \"${AGENT_POLICY}\""
         fi
-	echo "~~~ Fallback default policy is set to: ~~~" && cat "${podvm_dir}"/files/etc/kata-opa/custom.rego
+        echo "~~~ Fallback default policy is set to: ~~~" && cat "${podvm_dir}"/files/etc/kata-opa/custom.rego
     elif [[ "$CONFIDENTIAL_COMPUTE_ENABLED" == "yes" ]]; then
         echo "Setting custom agent policy to CoCo's recommended policy"
         sed 's/default ReadStreamRequest := true/default ReadStreamRequest := false/;
             s/default ExecProcessRequest := true/default ExecProcessRequest := false/' \
             "${podvm_dir}"/files/etc/kata-opa/allow-all.rego >"${podvm_dir}"/files/etc/kata-opa/coco-default-policy.rego
         sed -i 's/allow-all.rego/coco-default-policy.rego/g' "${podvm_dir}"/files/etc/tmpfiles.d/policy.conf || error_exit "Failed to edit policy.conf"
-	echo "~~~ Fallback default policy is set to: ~~~" && cat "${podvm_dir}"/files/etc/kata-opa/coco-default-policy.rego
+        echo "~~~ Fallback default policy is set to: ~~~" && cat "${podvm_dir}"/files/etc/kata-opa/coco-default-policy.rego
     fi
 
     # Fix disk mounts for CoCo
@@ -254,7 +254,7 @@ to edit policy.conf"
     if [[ "$CONFIDENTIAL_COMPUTE_ENABLED" == "yes" ]]; then
         local agent_config_file="/etc/agent-config.toml"
 
-        cat<<EOF>"${podvm_dir}/files${agent_config_file}"
+        cat <<EOF >"${podvm_dir}/files${agent_config_file}"
 server_addr = "unix:///run/kata-containers/agent.sock"
 guest_components_procs = "none"
 image_registry_auth = "file:///run/peerpod/auth.json"
@@ -482,7 +482,7 @@ function bootc_image_builder_conversion() {
     # some VM customizations
     if [[ -n "${BOOTC_BUILD_CONFIG}" ]]; then
         echo "Using Custom Bootc Build Configuration"
-        echo "${BOOTC_BUILD_CONFIG}" > ./custom-config.toml
+        echo "${BOOTC_BUILD_CONFIG}" >./custom-config.toml
         BOOTC_BUILD_CONFIG_PATH=$(pwd)/custom-config.toml
     else
         echo "Using Default Bootc Build Configuration"
@@ -494,8 +494,8 @@ function bootc_image_builder_conversion() {
     # login for local registry pulling # TODO: can we use token instead?
     if [[ "${container_image_repo_url}" == *"image-registry.openshift-image-registry.svc"* ]]; then
         echo "login to local registry"
-        mkdir  /etc/containers/certs.d/image-registry.openshift-image-registry.svc:5000
-        ln -s /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt  /etc/containers/certs.d/image-registry.openshift-image-registry.svc:5000/service-ca.crt
+        mkdir /etc/containers/certs.d/image-registry.openshift-image-registry.svc:5000
+        ln -s /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt /etc/containers/certs.d/image-registry.openshift-image-registry.svc:5000/service-ca.crt
         podman login -u kubeadmin -p $(cat /var/run/secrets/kubernetes.io/serviceaccount/token) image-registry.openshift-image-registry.svc:5000
         podman pull "${container_image_repo_url}:${image_tag}" || error_exit "Failed to pull local bootc image"
     else
@@ -503,7 +503,6 @@ function bootc_image_builder_conversion() {
         REGISTRY_AUTH_FILE=${auth_json_file} podman pull "${container_image_repo_url}:${image_tag}" || error_exit "Failed to pull bootc image"
     fi
 
-
     # execute bootc-image-builder
     # TODO: check if we can avoid this to drop the /store volumeMount
     # REGISTRY_AUTH_FILE is needed to access bib image
@@ -681,6 +680,31 @@ delete_cm_annotation() {
     echo "Annotation ${key} deleted successfully from ${cm}"
 }
 
+# Function to retry a command with exponential backoff
+retry_command() {
+    local max_retries=5
+    local delay=10
+    local attempt=1
+
+    while [ $attempt -le $max_retries ]; do
+        echo "Attempt $attempt: Executing command..."
+        "$@" # Execute the passed command
+
+        if [ $? -eq 0 ]; then
+            echo "Command succeeded."
+            return 0
+        fi
+
+        echo "Command failed. Retrying in $delay seconds..."
+        sleep $delay
+        delay=$((delay * 2)) # Exponential backoff
+        ((attempt++))
+    done
+
+    echo "Command failed after $max_retries attempts."
+    return 1
+}
+
 # Global variables
 
 # Set global variable for the source code directory
diff --git a/config/peerpods/podvm/osc-podvm-create-job.yaml b/config/peerpods/podvm/osc-podvm-create-job.yaml
index 9ab1951cf..84d8ac5a8 100644
--- a/config/peerpods/podvm/osc-podvm-create-job.yaml
+++ b/config/peerpods/podvm/osc-podvm-create-job.yaml
@@ -35,6 +35,10 @@ spec:
           # aws-podvm-image-handler.sh script under /scripts/aws-podvm-image-handler.sh
           # gcp-podvm-image-handler.sh script under /scripts/gcp-podvm-image-handler.sh
           # sources for cloud-api-adaptor under /src/cloud-api-adaptor
+          lifecycle: 
+            preStop: 
+              exec: 
+                command: ["/scripts/packer-resource-cleanup.sh"]
           securityContext: # TODO: evaluate if these are actually needed
             privileged: true
             allowPrivilegeEscalation: true
@@ -77,7 +81,8 @@ spec:
               readOnly: true
             - name: store-volume
               mountPath: /store
-
+      # Default is 30s which is too less for the preStop hook to fully execute        
+      terminationGracePeriodSeconds: 180
       volumes:
         - name: payload
           emptyDir: {}
diff --git a/config/peerpods/podvm/packer-resource-cleanup.sh b/config/peerpods/podvm/packer-resource-cleanup.sh
new file mode 100755
index 000000000..729be0cba
--- /dev/null
+++ b/config/peerpods/podvm/packer-resource-cleanup.sh
@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+set -e
+
+CLOUD_PROVIDER=${CLOUD_PROVIDER:-"azure"}
+
+# Define log file path
+# This needs to be the PID 1 stdout, which is what is read by `kubectl logs`
+# This log will be available in the pod logs
+LOG_FILE=${LOG_FILE:-"/proc/1/fd/1"}
+
+# Function to log messages with timestamp
+function log() {
+    echo "$(date '+%Y-%m-%d %H:%M:%S') $1" >>"$LOG_FILE"
+}
+
+
+function azure_cleanup() {
+    # Capture Packer created VM names in an array
+    log "Deleting Packer VMs"
+    mapfile -t vm_names < <(az vm list --resource-group "$AZURE_RESOURCE_GROUP" --query "[?osProfile.adminUsername=='packer' && starts_with(osProfile.computerName, 'pkrvm')].name" --output tsv)
+
+    # Log the captured VM names
+    for vm in "${vm_names[@]}"; do
+        log "Captured VM: $vm"
+    done
+
+    # Check if any VMs were found
+    if [ ${#vm_names[@]} -eq 0 ]; then
+        log "No VMs found matching the criteria."
+    else
+        # Loop through the array and delete each VM
+        for vm in "${vm_names[@]}"; do
+            log "Deleting VM: $vm"
+            az vm delete --name "$vm" --resource-group "$AZURE_RESOURCE_GROUP" --yes >>"$LOG_FILE" 2>&1
+        done
+    fi
+
+    log "Disassociating and Deleting Packer Public IPs..."
+    mapfile -t public_ip_names < <(az network public-ip list --resource-group "$AZURE_RESOURCE_GROUP" --query "[?starts_with(name, 'pkrip')].name" --output tsv)
+
+    # Log the captured Public IP names
+    for public_ip_name in "${public_ip_names[@]}"; do
+        log "Captured Public IP: $public_ip_name"
+    done
+
+    for public_ip_name in "${public_ip_names[@]}"; do
+        # Get the public IP ID dynamically
+        public_ip_id=$(az network public-ip show --name "$public_ip_name" --resource-group "$AZURE_RESOURCE_GROUP" --query "id" --output tsv)
+
+        if [[ -z "$public_ip_id" ]]; then
+            log "Skipping $public_ip_name as it has no associated ID."
+            continue
+        fi
+
+        # Find the NIC associated with this Public IP
+        nic_name=$(az network nic list --resource-group "$AZURE_RESOURCE_GROUP" --query "[?ipConfigurations[?publicIPAddress.id=='$public_ip_id']].name" --output tsv)
+
+        if [[ -n "$nic_name" ]]; then
+            log "Disassociating Public IP $public_ip_name from NIC $nic_name..."
+            az network nic ip-config update --name ipconfig --nic-name "$nic_name" --resource-group "$AZURE_RESOURCE_GROUP" --remove publicIPAddress >>"$LOG_FILE" 2>&1
+        fi
+
+        log "Deleting Public IP: $public_ip_name"
+        az network public-ip delete --name "$public_ip_name" --resource-group "$AZURE_RESOURCE_GROUP" >>"$LOG_FILE" 2>&1
+    done
+
+    log "Deleting Packer Network Interfaces..."
+    mapfile -t nics < <(az network nic list --resource-group "$AZURE_RESOURCE_GROUP" --query "[?starts_with(name, 'pkrni')].name" --output tsv)
+
+    # Log the captured NIC names
+    for name in "${nics[@]}"; do
+        log "Captured NIC: $name"
+    done
+
+    for name in "${nics[@]}"; do
+        log "Deleting NIC: $name"
+        az network nic delete --name "$name" --resource-group "$AZURE_RESOURCE_GROUP" >>"$LOG_FILE" 2>&1
+    done
+
+    log "Deleting Packer disks..."
+    mapfile -t disks < <(az disk list --resource-group "$AZURE_RESOURCE_GROUP" --query "[?starts_with(name, 'pkrdisk')].name" --output tsv)
+
+    # Log the captured Disk names
+    for name in "${disks[@]}"; do
+        log "Captured Disk: $name"
+    done
+
+    for name in "${disks[@]}"; do
+        log "Deleting Disk: $name"
+        az disk delete --name "$name" --resource-group "$AZURE_RESOURCE_GROUP" --yes >>"$LOG_FILE" 2>&1
+    done
+
+    log "Deleting Packer Virtual Networks..."
+    mapfile -t vnets < <(az network vnet list --resource-group "$AZURE_RESOURCE_GROUP" --query "[?starts_with(name, 'pkrvn')].name" --output tsv)
+    for vnet in "${vnets[@]}"; do
+        log "Deleting VNet: $vnet (this will also remove subnets)"
+        az network vnet delete --name "$vnet" --resource-group "$AZURE_RESOURCE_GROUP" >>"$LOG_FILE" 2>&1
+    done
+}
+
+# Log the start of the script
+log "Starting Packer resource cleanup script."
+
+# Execute cleanup if CLOUD_PROVIDER==azure
+if [ "$CLOUD_PROVIDER" == "azure" ]; then
+    azure_cleanup
+else
+    log "Cleanup not supported for CLOUD_PROVIDER: $CLOUD_PROVIDER"
+fi
+
+log "cleanup hook completed."
diff --git a/controllers/image_generator.go b/controllers/image_generator.go
index abcd2345f..f7e5cd352 100644
--- a/controllers/image_generator.go
+++ b/controllers/image_generator.go
@@ -745,8 +745,25 @@ func (r *ImageGenerator) createImageConfigMapFromFile() error {
 		return err
 	}
 
+	// This is to ensure that the image version (for azure) or ami version (for aws)
+	// is unique for each cluster
+	// image version expects the format <major>.<minor>.<patch> - all digits
+	// clusterID is a hex string, so we need to convert it to an integer
+	// This is to ensure that the image version is unique for each cluster
+	imageVersion := "0.0.1"
+	clusterIdInt, err := strconv.ParseUint(r.clusterId, 16, 32)
+	if err == nil {
+		imageVersion = fmt.Sprintf("%d.0.1", clusterIdInt)
+	} else {
+		igLogger.Info("clusterId is not a valid hex number, using default version")
+	}
+
+	igLogger.Info("using image version", "image_version", imageVersion)
+
 	// For Azure provider set the IMAGE_GALLERY_NAME if its empty
 	// The IMAGE_GALLERY_NAME is set to azureImageGalleryPrefix + "_" + clusterID
+	// Also set the IMAGE_VERSION or AMI_VERSION
+
 	if r.provider == AzureProvider {
 		if cm.Data["IMAGE_GALLERY_NAME"] == "" {
 			image_gallery_name := azureImageGalleryPrefix + "_" + r.clusterId
@@ -754,6 +771,21 @@ func (r *ImageGenerator) createImageConfigMapFromFile() error {
 			igLogger.Info("Setting IMAGE_GALLERY_NAME", "image_gallery_name", image_gallery_name)
 		}
 
+		cm.Data["IMAGE_VERSION"] = imageVersion
+		igLogger.Info("Setting IMAGE_VERSION", "image_version", imageVersion)
+
+	}
+
+	if r.provider == AWSProvider {
+		cm.Data["AMI_VERSION"] = imageVersion
+		igLogger.Info("Setting AMI_VERSION", "ami_version", imageVersion)
+	}
+
+	if r.provider == GCPProvider {
+		// GCP doesn't allow dot. It needs to use dash
+		imageVersion = strings.ReplaceAll(imageVersion, ".", "-")
+		cm.Data["IMAGE_VERSION"] = imageVersion
+		igLogger.Info("Setting IMAGE_VERSION", "image_version", imageVersion)
 	}
 
 	if err := r.client.Create(context.TODO(), cm); err != nil {


Reviews:
user: login: beraldoleal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-19T16:42:25Z
user: login: beraldoleal
body: LGTM, thanks.
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-20T12:35:17Z
user: login: beraldoleal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-21T12:53:26Z
user: login: bpradipt
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-21T13:02:30Z
user: login: beraldoleal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-21T13:05:31Z
user: login: littlejawa
body: /lgtm

state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-25T10:30:01Z

Review comments:
diff_hunk: @@ -745,15 +745,40 @@ func (r *ImageGenerator) createImageConfigMapFromFile() error {
 		return err
 	}
 
+	// This is to ensure that the image version (for azure) or ami version (for aws)
+	// is unique for each cluster
+	// image version expects the format <major>.<minor>.<patch> - all digits
+	// clusterID is a hex string, so we need to convert it to an integer
+	// This is to ensure that the image version is unique for each cluster
+	imageVersion := "0.0.1"
+	clusterIdInt, err := strconv.ParseUint(r.clusterId, 16, 32)
+	if err == nil {
+		imageVersion = fmt.Sprintf("%d.0.1", clusterIdInt)
+	} else {
+		igLogger.Info("clusterId is not a valid hex number, using default version")
+	}
+
+	igLogger.Info("using image version", "image_version", imageVersion)
+
 	// For Azure provider set the IMAGE_GALLERY_NAME if its empty
 	// The IMAGE_GALLERY_NAME is set to azureImageGalleryPrefix + "_" + clusterID
+	// Also set the IMAGE_VERSION or AMI_VERSION
+
 	if r.provider == AzureProvider {
 		if cm.Data["IMAGE_GALLERY_NAME"] == "" {
 			image_gallery_name := azureImageGalleryPrefix + "_" + r.clusterId
 			cm.Data["IMAGE_GALLERY_NAME"] = image_gallery_name
 			igLogger.Info("Setting IMAGE_GALLERY_NAME", "image_gallery_name", image_gallery_name)
 		}
 
+		cm.Data["IMAGE_VERSION"] = imageVersion
path: controllers/image_generator.go
user: login: beraldoleal
body: Since GCP the image version will be part of the name... and the name can't have DOTs. IIUC, we should add another if here to check if the provider == GCP and set  `cm.Data["IMAGE_VERSION"] = imageVersionWithoutdots` ?
created_at: 2025-03-19T16:41:46Z
author_association: MEMBER
diff_hunk: @@ -745,15 +745,40 @@ func (r *ImageGenerator) createImageConfigMapFromFile() error {
 		return err
 	}
 
+	// This is to ensure that the image version (for azure) or ami version (for aws)
+	// is unique for each cluster
+	// image version expects the format <major>.<minor>.<patch> - all digits
+	// clusterID is a hex string, so we need to convert it to an integer
+	// This is to ensure that the image version is unique for each cluster
+	imageVersion := "0.0.1"
+	clusterIdInt, err := strconv.ParseUint(r.clusterId, 16, 32)
+	if err == nil {
+		imageVersion = fmt.Sprintf("%d.0.1", clusterIdInt)
+	} else {
+		igLogger.Info("clusterId is not a valid hex number, using default version")
+	}
+
+	igLogger.Info("using image version", "image_version", imageVersion)
+
 	// For Azure provider set the IMAGE_GALLERY_NAME if its empty
 	// The IMAGE_GALLERY_NAME is set to azureImageGalleryPrefix + "_" + clusterID
+	// Also set the IMAGE_VERSION or AMI_VERSION
+
 	if r.provider == AzureProvider {
 		if cm.Data["IMAGE_GALLERY_NAME"] == "" {
 			image_gallery_name := azureImageGalleryPrefix + "_" + r.clusterId
 			cm.Data["IMAGE_GALLERY_NAME"] = image_gallery_name
 			igLogger.Info("Setting IMAGE_GALLERY_NAME", "image_gallery_name", image_gallery_name)
 		}
 
+		cm.Data["IMAGE_VERSION"] = imageVersion
path: controllers/image_generator.go
user: login: beraldoleal
body: Since GCP the image version will be part of the name... and the name can't have DOTs. IIUC, we should add another if here to check if the provider == GCP and set  `cm.Data["IMAGE_VERSION"] = imageVersionWithoutdots` ?
created_at: 2025-03-19T16:41:54Z
author_association: MEMBER
diff_hunk: @@ -718,6 +737,52 @@ function delete_image_using_id() {
     echo "Azure image deleted successfully"
 }
 
+# Function to check if image already exists
+# This checks if IMAGE_VERSION exist in Azure
+# and the LATEST_IMAGE_ID annotation is set in the peer-pods-cm configmap
+# 0: Image exists and matches
+# 1: Image does not exist
+# 2: Image mismatch (caller should delete Image)
+# The required variables are assumed to be set
+function image_exists() {
path: config/peerpods/podvm/azure-podvm-image-handler.sh
user: login: beraldoleal
body: I missed this method under gcp. IIUC, most of it could be generic to all providers?
created_at: 2025-03-21T12:53:23Z
author_association: MEMBER
diff_hunk: @@ -718,6 +737,52 @@ function delete_image_using_id() {
     echo "Azure image deleted successfully"
 }
 
+# Function to check if image already exists
+# This checks if IMAGE_VERSION exist in Azure
+# and the LATEST_IMAGE_ID annotation is set in the peer-pods-cm configmap
+# 0: Image exists and matches
+# 1: Image does not exist
+# 2: Image mismatch (caller should delete Image)
+# The required variables are assumed to be set
+function image_exists() {
path: config/peerpods/podvm/azure-podvm-image-handler.sh
user: login: bpradipt
body: A portion of it yes, but querying is specific to individual provider. So I kept it for each provider..
created_at: 2025-03-21T13:02:30Z
author_association: CONTRIBUTOR
diff_hunk: @@ -718,6 +737,52 @@ function delete_image_using_id() {
     echo "Azure image deleted successfully"
 }
 
+# Function to check if image already exists
+# This checks if IMAGE_VERSION exist in Azure
+# and the LATEST_IMAGE_ID annotation is set in the peer-pods-cm configmap
+# 0: Image exists and matches
+# 1: Image does not exist
+# 2: Image mismatch (caller should delete Image)
+# The required variables are assumed to be set
+function image_exists() {
path: config/peerpods/podvm/azure-podvm-image-handler.sh
user: login: beraldoleal
body: And where is the gcp implementation? I saw you changed the gcp-podvm-image-handler.sh but I could not find any image_exists method.
created_at: 2025-03-21T13:05:31Z
author_association: MEMBER

PR comments:
user: login: openshift-ci[bot]
created_at: 2025-03-19T10:54:36Z
body: Skipping CI for Draft Pull Request.
If you want CI signal for your change, please convert it to an actual PR.
You can still manually trigger a test run with `/test all`
author_association: NONE
user: login: beraldoleal
created_at: 2025-03-21T12:43:20Z
body: @bpradipt thanks for handling the gcp part, saved me some time. ;)
author_association: MEMBER
user: login: openshift-ci[bot]
created_at: 2025-03-25T08:57:42Z
body: @bpradipt: all tests **passed!**


[Full PR test history](https://prow.ci.openshift.org/pr-history?org=openshift&repo=sandboxed-containers-operator&pr=584). [Your PR dashboard](https://prow.ci.openshift.org/pr?query=is:pr+state:open+author:bpradipt).

<details>

Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository. I understand the commands that are listed [here](https://go.k8s.io/bot-commands).
</details>
<!-- test report -->
author_association: NONE