Repository: ansible/awx
File: awx/main/tasks/facts.py (Python)

Code before:
    msg='Inventory {inventory_id} host facts: updated {updated_ct}, cleared {cleared_ct}, unchanged {unmodified_ct}, took {delta:.3f} s',
    add_log_data=True,
)
def finish_fact_cache(destination, log_data, job_id=None, inventory_id=None):
    log_data['inventory_id'] = inventory_id
    log_data['updated_ct'] = 0
    log_data['unmodified_ct'] = 0
    log_data['cleared_ct'] = 0

    summary_path = os.path.join(destination, 'host_cache_summary.json')
    if not os.path.exists(summary_path):
        system_tracking_logger.error(f'Missing summary file at {summary_path}')
        return

    try:
        with open(summary_path, 'r', encoding='utf-8') as f:
            summary = json.load(f)
    except Exception:
        system_tracking_logger.error(f'Unable to read summary file at {summary_path}')
        return

    host_names = summary.get('hosts_cached', [])
    facts_write_time = os.path.getmtime(summary_path)

    # Lookup Host objects
    hosts_cached = list(Host.objects.filter(name__in=host_names, inventory_id=inventory_id).order_by('id'))

    hosts_to_update = []
    host_facts_dir = os.path.join(settings.ANSIBLE_FACT_CACHE, str(inventory_id))  # or however the path is built

    for host in hosts_cached:
        filepath = os.path.join(host_facts_dir, host.name)
        if not os.path.realpath(filepath).startswith(host_facts_dir):
            system_tracking_logger.error(f'Invalid path for facts file: {filepath}')
            continue

        if os.path.exists(filepath):
            modified = os.path.getmtime(filepath)
            if not facts_write_time or modified > facts_write_time:
                try:
                    with codecs.open(filepath, 'r', encoding='utf-8') as f:
                        ansible_facts = json.load(f)
                except ValueError:
                    continue

                host.ansible_facts = ansible_facts
                host.ansible_facts_modified = now()
                hosts_to_update.append(host)

                system_tracking_logger.info(
                    f'New fact for inventory {smart_str(host.inventory.name)} host {smart_str(host.name)}',

Review comment (suggestion):
This materializes all the ansible facts for all the hosts in the inventory into memory.

With the current structure, I believe you can avoid that by simply removing the `list(` and adding an `.iterator()` to the queryset. It won't load the queryset into memory right away. And the iterator allows garbage collecting hosts that have gone out of scope in the loop.

Code after:
    logger,
    debug_cutoff=0.01,
    msg='Inventory {inventory_id} host facts: updated {updated_ct}, cleared {cleared_ct}, unchanged {unmodified_ct}, took {delta:.3f} s',
    add_log_data=True,
)
def finish_fact_cache(artifacts_dir, job_id=None, inventory_id=None, log_data=None):
    log_data = log_data or {}
    log_data['inventory_id'] = inventory_id
    log_data['updated_ct'] = 0
    log_data['unmodified_ct'] = 0
    log_data['cleared_ct'] = 0
    # The summary file is directly inside the artifacts dir
    summary_path = os.path.join(artifacts_dir, 'host_cache_summary.json')
    if not os.path.exists(summary_path):
        logger.error(f'Missing summary file at {summary_path}')
        return

    try:
        with open(summary_path, 'r', encoding='utf-8') as f:
            summary = json.load(f)
        facts_write_time = os.path.getmtime(summary_path)  # After successful read
    except (json.JSONDecodeError, OSError) as e:
        logger.error(f'Error reading summary file at {summary_path}: {e}')
        return

    host_names = summary.get('hosts_cached', [])
    hosts_cached = Host.objects.filter(name__in=host_names).order_by('id').iterator()
    # Path where individual fact files were written
    fact_cache_dir = os.path.join(artifacts_dir, 'fact_cache')
    hosts_to_update = []

    for host in hosts_cached:
        filepath = os.path.join(fact_cache_dir, host.name)
        if not os.path.realpath(filepath).startswith(fact_cache_dir):
            logger.error(f'Invalid path for facts file: {filepath}')
            continue

        if os.path.exists(filepath):
            # If the file changed since we wrote the last facts file, pre-playbook run...
            modified = os.path.getmtime(filepath)
            if not facts_write_time or modified >= facts_write_time:
                try:
                    with codecs.open(filepath, 'r', encoding='utf-8') as f:
                        ansible_facts = json.load(f)
                except ValueError:
                    continue

                if ansible_facts != host.ansible_facts:
                    host.ansible_facts = ansible_facts
                    host.ansible_facts_modified = now()
                    hosts_to_update.append(host)