Repository: ansible/awx
File: awx/main/tasks.py (Python)

Code before:


@task(queue=get_local_queuename)
def purge_old_stdout_files():
    nowtime = time.time()
    for f in os.listdir(settings.JOBOUTPUT_ROOT):
        if os.path.getctime(os.path.join(settings.JOBOUTPUT_ROOT, f)) < nowtime - settings.LOCAL_STDOUT_EXPIRE_TIME:
            os.unlink(os.path.join(settings.JOBOUTPUT_ROOT, f))
            logger.debug("Removing {}".format(os.path.join(settings.JOBOUTPUT_ROOT, f)))


@task(queue=get_local_queuename)
def cleanup_execution_environment_images():
    if settings.IS_K8S:
        return
    images_in_use = [ee.image for ee in ExecutionEnvironment.objects.all()]
    images_system = subprocess.run("podman images -a --format json".split(" "), capture_output=True)
    if len(images_system.stdout) > 0:
        images_system = json.loads(images_system.stdout)
        for e in images_system:
            if 'Names' in e:
                image_name = e['Names'][0]
            else:
                image_name = e["Id"]
            image_size = e['Size'] / 1e6
            if image_name not in images_in_use:
                logger.debug(f"Cleanup execution environment images: deleting {image_name}, {image_size:.0f} MB")
                process = subprocess.run(['podman', 'rmi', image_name, '-f'], stdout=subprocess.DEVNULL)
                if process.returncode != 0:
                    logger.debug(f"Unsuccessfully deleted image {image_name}")


@task(queue=get_local_queuename)
def cluster_node_heartbeat():
    logger.debug("Cluster node heartbeat task.")
    nowtime = now()
    instance_list = list(Instance.objects.all_non_isolated())
    this_inst = None
    lost_instances = []

    (changed, instance) = Instance.objects.get_or_register()
    if changed:
        logger.info("Registered tower node '{}'".format(instance.hostname))

    for inst in list(instance_list):
        if inst.hostname == settings.CLUSTER_HOST_ID:
            this_inst = inst
            instance_list.remove(inst)
        elif inst.is_lost(ref_time=nowtime):
            lost_instances.append(inst)
            instance_list.remove(inst)

Review comment (bug):
What if my EE has set `quay.io/project-receptor/receptor` but the reported output from `podman images -a --format json` gives:

```json
    {
        "Id": "bf4de35112248c7c5b1db46c43f87346f31e2717850831433af1fee6dfd0c2b4",
        "ParentId": "",
        "Size": 387129820,
        "Labels": {
            "license": "ASL2",
            "name": "receptor",
            "vendor": "project-receptor",
            "version": "0.9.1"
        },
        "Names": [
            "quay.io/project-receptor/receptor:latest"
        ],
        "Digest": "sha256:11d9a25f07ea27146d94772eb3c6f2faf5d5c75d9c3313fcad7390c7e417c32a",
        "Digests": [
            "sha256:11d9a25f07ea27146d94772eb3c6f2faf5d5c75d9c3313fcad7390c7e417c32a"
        ],
        "Created": 1602187764,
        "CreatedAt": "2020-10-08T20:09:24Z"
    }
```

I think it's valid to not include `:latest` so I'm wondering if it might confuse this code.

It's also going to be tricky to validate this. If it incorrectly deletes images and layers that it shouldn't, it will be quite difficult for QE to track down what was _successfully_ deleted, but shouldn't have been (triggering an expensive download on the next run).

Code after:

@task(queue=get_local_queuename)
def purge_old_stdout_files():
    nowtime = time.time()
    for f in os.listdir(settings.JOBOUTPUT_ROOT):
        if os.path.getctime(os.path.join(settings.JOBOUTPUT_ROOT, f)) < nowtime - settings.LOCAL_STDOUT_EXPIRE_TIME:
            os.unlink(os.path.join(settings.JOBOUTPUT_ROOT, f))
            logger.debug("Removing {}".format(os.path.join(settings.JOBOUTPUT_ROOT, f)))


@task(queue=get_local_queuename)
def cleanup_execution_environment_images():
    if settings.IS_K8S:
        return
    process = subprocess.run('podman images --filter="dangling=true" --format json'.split(" "), capture_output=True)
    if process.returncode != 0:
        logger.debug("Cleanup execution environment images: could not get list of images")
        return
    if len(process.stdout) > 0:
        images_system = json.loads(process.stdout)
        for e in images_system:
            image_name = e["Id"]
            logger.debug(f"Cleanup execution environment images: deleting {image_name}")
            process = subprocess.run(['podman', 'rmi', image_name, '-f'], stdout=subprocess.DEVNULL)
            if process.returncode != 0:
                logger.debug(f"Failed to delete image {image_name}")


@task(queue=get_local_queuename)
def cluster_node_heartbeat():
    logger.debug("Cluster node heartbeat task.")
    nowtime = now()
    instance_list = list(Instance.objects.all_non_isolated())
    this_inst = None
    lost_instances = []

    (changed, instance) = Instance.objects.get_or_register()
    if changed:
        logger.info("Registered tower node '{}'".format(instance.hostname))

    for inst in list(instance_list):
        if inst.hostname == settings.CLUSTER_HOST_ID:
            this_inst = inst
            instance_list.remove(inst)
        elif inst.is_lost(ref_time=nowtime):
            lost_instances.append(inst)
            instance_list.remove(inst)
    if this_inst:
        startup_event = this_inst.is_lost(ref_time=nowtime)
        this_inst.refresh_capacity()
        if startup_event: