Repository: ansible/awx
File: awx/main/tasks/facts.py (Python)

Code before:

    # Write summary file to artifacts dir
    if inventory_id is not None:
        artifact_dir = os.path.join('artifacts', f'inventory_{inventory_id}')
        os.makedirs(artifact_dir, exist_ok=True)
        summary_file = os.path.join(artifact_dir, 'host_cache_summary.json')
        summary_data = {'last_write_time': last_write_time, 'hosts_cached': hosts_cached, 'written_ct': log_data['written_ct']}
        with open(summary_file, 'w', encoding='utf-8') as f:
            json.dump(summary_data, f, indent=2)


@log_excess_runtime(
    logger,
    debug_cutoff=0.01,
    msg='Inventory {inventory_id} host facts: updated {updated_ct}, cleared {cleared_ct}, unchanged {unmodified_ct}, took {delta:.3f} s',
    add_log_data=True,
)
def finish_fact_cache(destination, log_data, job_id=None, inventory_id=None):
    log_data['inventory_id'] = inventory_id
    log_data['updated_ct'] = 0
    log_data['unmodified_ct'] = 0
    log_data['cleared_ct'] = 0

    summary_path = os.path.join(destination, 'host_cache_summary.json')
    if not os.path.exists(summary_path):
        system_tracking_logger.error(f'Missing summary file at {summary_path}')
        return

    try:
        with open(summary_path, 'r', encoding='utf-8') as f:
            summary = json.load(f)
        facts_write_time = os.path.getmtime(summary_path)  # Get mtime *after* successful read
    except (json.JSONDecodeError, OSError) as e:
        system_tracking_logger.error(f'Error reading summary file at {summary_path}: {e}')
        return

    host_names = summary.get('hosts_cached', [])

    # Lookup Host objects.  Use iterator() for large queries.  Order by 'id'.
    hosts_cached = Host.objects.filter(name__in=host_names, inventory_id=inventory_id).order_by('id').iterator()

    hosts_to_update = []
    host_facts_dir = os.path.join(settings.ANSIBLE_FACT_CACHE, str(inventory_id))

    for host in hosts_cached:
        filepath = os.path.join(host_facts_dir, host.name)
        if not os.path.realpath(filepath).startswith(host_facts_dir):
            system_tracking_logger.error(f'Invalid path for facts file: {filepath}')
            continue

        if os.path.exists(filepath):

Review comment (bug):
I looked at your test failures, and was awfully perplexed. It seems that what is making this very difficult to debut is the use of `system_tracking_logger` as opposed to `logger`. The former will present the logs in normal logs. I would avoid the former except for the very narrow cases where it is already being used. Because of this, we're not getting legitimate output from tests, which should show this log is being emitted, which is likely the current problem you're on.

Code after:

    # Write summary file directly to the artifacts_dir
    if inventory_id is not None:
        summary_file = os.path.join(artifacts_dir, 'host_cache_summary.json')
        summary_data = {
            'last_write_time': last_write_time,
            'hosts_cached': hosts_cached,
            'written_ct': log_data['written_ct'],
        }
        with open(summary_file, 'w', encoding='utf-8') as f:
            json.dump(summary_data, f, indent=2)


@log_excess_runtime(
    logger,
    debug_cutoff=0.01,
    msg='Inventory {inventory_id} host facts: updated {updated_ct}, cleared {cleared_ct}, unchanged {unmodified_ct}, took {delta:.3f} s',
    add_log_data=True,
)
def finish_fact_cache(artifacts_dir, job_id=None, inventory_id=None, log_data=None):
    log_data = log_data or {}
    log_data['inventory_id'] = inventory_id
    log_data['updated_ct'] = 0
    log_data['unmodified_ct'] = 0
    log_data['cleared_ct'] = 0
    # The summary file is directly inside the artifacts dir
    summary_path = os.path.join(artifacts_dir, 'host_cache_summary.json')
    if not os.path.exists(summary_path):
        logger.error(f'Missing summary file at {summary_path}')
        return

    try:
        with open(summary_path, 'r', encoding='utf-8') as f:
            summary = json.load(f)
        facts_write_time = os.path.getmtime(summary_path)  # After successful read
    except (json.JSONDecodeError, OSError) as e:
        logger.error(f'Error reading summary file at {summary_path}: {e}')
        return

    host_names = summary.get('hosts_cached', [])
    hosts_cached = Host.objects.filter(name__in=host_names).order_by('id').iterator()
    # Path where individual fact files were written
    fact_cache_dir = os.path.join(artifacts_dir, 'fact_cache')
    hosts_to_update = []

    for host in hosts_cached:
        filepath = os.path.join(fact_cache_dir, host.name)
        if not os.path.realpath(filepath).startswith(fact_cache_dir):
            logger.error(f'Invalid path for facts file: {filepath}')
            continue