Repository: ansible/awx
File: awx/main/tasks/facts.py (Python)

Code before:
    try:
        os.makedirs(destination, mode=0o700)
    except FileExistsError:
        pass

    if timeout is None:
        timeout = settings.ANSIBLE_FACT_CACHE_TIMEOUT

    last_filepath_written = None

    for host in hosts:
        if not host.ansible_facts_modified or (timeout and host.ansible_facts_modified < now() - datetime.timedelta(seconds=timeout)):
            continue  # facts are expired - do not write them

        filepath = os.sep.join(map(str, [destination, host.name]))
        if not os.path.realpath(filepath).startswith(destination):
            system_tracking_logger.error('facts for host {} could not be cached'.format(smart_str(host.name)))
            continue

        try:
            with codecs.open(filepath, 'w', encoding='utf-8') as f:
                os.chmod(f.name, 0o600)
                json.dump(host.ansible_facts, f)
                log_data['written_ct'] += 1
                last_filepath_written = filepath
            hosts_cached.append(CachedHost(name=host.name, inventory=host.inventory, ansible_facts=host.ansible_facts))
        except IOError:
            system_tracking_logger.error('facts for host {} could not be cached'.format(smart_str(host.name)))
            continue

    if last_filepath_written:
        return os.path.getmtime(last_filepath_written), hosts_cached

    return None, hosts_cached


def raw_update_hosts(host_list):
    Host.objects.bulk_update(host_list, ['ansible_facts', 'ansible_facts_modified'])


def update_hosts(host_list, max_tries=5):
    if not host_list:
        return
    for i in range(max_tries):
        try:
            raw_update_hosts(host_list)
        except OperationalError as exc:
            # Deadlocks can happen if this runs at the same time as another large query
            # inventory updates and updating last_job_host_summary are candidates for conflict
            # but these would resolve easily on a retry
            if i + 1 < max_tries:

Review comment (performance):
I still expect that this won't work. Again, when going through the `for host in hosts:` loop, many hosts will hit the `continue` statement before here, meaning they won't get into the list. Really, if you run this for the first time, **no** hosts will be in the list, and I legitimately think it will be `[]`. This is not what you want. For this to work as the variable of iteration in `finish_fact_cache`, it needs to basically contain all hosts in the inventory (of the section of inventory for sliced jobs).

And this is a very creative approach to create a new class, but keeping all ansible_facts in memory is too much. You also shouldn't need them, they shouldn't be used in `finish_fact_cache`.

Code after:
    last_filepath_written = None
    for host in hosts:
        hosts_cached.append(host)
        if not host.ansible_facts_modified or (timeout and host.ansible_facts_modified < now() - datetime.timedelta(seconds=timeout)):
            continue  # facts are expired - do not write them

        filepath = os.sep.join(map(str, [destination, host.name]))
        if not os.path.realpath(filepath).startswith(destination):
            system_tracking_logger.error('facts for host {} could not be cached'.format(smart_str(host.name)))
            continue

        try:
            with codecs.open(filepath, 'w', encoding='utf-8') as f:
                os.chmod(f.name, 0o600)
                json.dump(host.ansible_facts, f)
                log_data['written_ct'] += 1
                last_filepath_written = filepath
        except IOError:
            system_tracking_logger.error('facts for host {} could not be cached'.format(smart_str(host.name)))
            continue

    if last_filepath_written:
        return os.path.getmtime(last_filepath_written), hosts_cached

    return None, hosts_cached


def raw_update_hosts(host_list):
    Host.objects.bulk_update(host_list, ['ansible_facts', 'ansible_facts_modified'])


def update_hosts(host_list, max_tries=5):
    if not host_list:
        return
    for i in range(max_tries):
        try:
            raw_update_hosts(host_list)
        except OperationalError as exc:
            # Deadlocks can happen if this runs at the same time as another large query
            # inventory updates and updating last_job_host_summary are candidates for conflict
            # but these would resolve easily on a retry
            if i + 1 < max_tries:
                logger.info(f'OperationalError (suspected deadlock) saving host facts retry {i}, message: {exc}')
                continue
            else:
                raise
        break


@log_excess_runtime(
    logger,