Repository: AlexsLemonade/refinebio

Issue:
Clean up AWS Batch job definition list
### Problem or idea

The Batch job definition section contains 100+ pages of jobs (~2500 items). They haven't been cleaned up properly during deploy process for a while.


### Solution or next step

Clean up stale items, make sure job deregistering script takes care of old job definitions in a right way.



Discussion:

Resolving patch:
diff --git a/infrastructure/delete_batch_job_queue.py b/infrastructure/delete_batch_job_queue.py
--- a/infrastructure/delete_batch_job_queue.py
+++ b/infrastructure/delete_batch_job_queue.py
@@ -2,19 +2,22 @@
 from time import sleep
 
 import boto3
+from botocore.exceptions import ClientError
 
-AWS_REGION = os.environ["AWS_REGION"]
 AWS_BATCH_QUEUE_ALL_NAMES = os.environ["REFINEBIO_JOB_QUEUE_ALL_NAMES"].split(",")
 
-batch = boto3.client("batch", region_name=AWS_REGION)
+batch = boto3.client("batch", region_name=os.environ["AWS_REGION"])
 
 # First disable each job queue.
 for batch_queue_name in AWS_BATCH_QUEUE_ALL_NAMES:
     try:
         batch.update_job_queue(jobQueue=batch_queue_name, state="DISABLED")
-    except Exception as e:
+    except ClientError as e:
         # If the job queue doesn't exist, that's cool, we were trying to delete it anyway.
-        pass
+        if str(e).endswith(" does not exist."):
+            pass
+        else:
+            raise e
 
 # Then wait for each one to be disabled so it can be deleted.
 for batch_queue_name in AWS_BATCH_QUEUE_ALL_NAMES:
diff --git a/infrastructure/deregister_batch_job_definitions.py b/infrastructure/deregister_batch_job_definitions.py
--- a/infrastructure/deregister_batch_job_definitions.py
+++ b/infrastructure/deregister_batch_job_definitions.py
@@ -2,25 +2,36 @@
 
 import boto3
 
-AWS_REGION = os.environ["AWS_REGION"]
-
-batch = boto3.client("batch", region_name=AWS_REGION)
+batch = boto3.client("batch", region_name=os.environ["AWS_REGION"])
 
 # TODO: stop repeating this construction everywhere. Just set it once somewhere.
 JOB_DEFINITION_PREFIX = os.environ["USER"] + "_" + os.environ["STAGE"] + "_"
 
-job_definition_files = os.listdir("batch-job-templates")
-
-job_definition_list = list(
-    {JOB_DEFINITION_PREFIX + job_def.upper().split(".")[0] for job_def in job_definition_files}
+job_names = (
+    JOB_DEFINITION_PREFIX + batch_job_template.upper().split(".")[0]
+    for batch_job_template in os.listdir("batch-job-templates")
 )
+nextToken = ""
 
 # Have to go one by one because providing a list of job names doesn't work:
 # https://github.com/boto/boto3/issues/2908
-for job_definition in job_definition_list:
-    job_definitions = batch.describe_job_definitions(
-        jobDefinitionName=job_definition, status="ACTIVE"
-    )
-    # There can be multiple revisions per job definition. We want them all gone.
-    for job_definition_revision in job_definitions["jobDefinitions"]:
-        batch.deregister_job_definition(jobDefinition=job_definition_revision["jobDefinitionArn"])
+for job_name in sorted(job_names):
+    while True:
+        data = {
+            "jobDefinitionName": job_name,
+            "maxResults": 100,
+            "status": "ACTIVE",
+        }
+        if nextToken:
+            data["nextToken"] = nextToken
+
+        response = batch.describe_job_definitions(**data)
+        nextToken = response.get("nextToken", "")
+
+        job_definitions = response.get("jobDefinitions")
+        if not job_definitions:
+            break
+
+        # There can be multiple revisions per job definition. We want them all gone.
+        for job_definition in job_definitions:
+            batch.deregister_job_definition(jobDefinition=job_definition["jobDefinitionArn"])


Test patch: