def to_dict(multi_dict):
    """Converts a multi dict into a dict where only list entries are not flat"""
    return {
        k: v if len(v) > 1 else v[0]
        for (k, v) in multi_dict.to_dict(flat=False).items()
    }

def to_dict ( multi_dict ) : return { k : v if len ( v ) > 1 else v [ 0 ] for ( k , v ) in multi_dict . to_dict ( flat = False ) . items ( ) }


Gets current_user_id from args featuring a user_id key

def get_current_user_id(args):
    """Gets current_user_id from args featuring a "user_id" key"""
    if args.get("user_id"):
        return decode_string_id(args["user_id"])
    return None

def get_current_user_id ( args ) : if args . get ( "user_id" ) : return decode_string_id ( args [ "user_id" ] ) return None


Resolves an Audius URL into the cannonical API route . Accepts fully formed urls as well as just url paths .

def resolve_url(session, url):
    """
    Resolves an Audius URL into the cannonical API route.
    Accepts fully formed urls as well as just url paths.
    """
    parsed = urlparse(url)
    # Will strip out any preceding protocol & domain (e.g. https://audius.co)
    path = parsed.path

    match = track_url_regex.match(path)
    if match:
        slug = match.group("slug")
        handle = match.group("handle")
        return ns_url_for(tracks_ns, "track_by_route", slug=slug, handle=handle)

    match = playlist_url_regex.match(path)
    if match:
        playlist_id = match.group("id")
        hashed_id = encode_int_id(int(playlist_id))
        return ns_url_for(playlists_ns, "playlist", playlist_id=hashed_id)

    match = user_url_regex.match(path)
    if match:
        handle = match.group("handle")
        user = (
            session.query(User)
            .filter(User.handle_lc == handle.lower(), User.is_current == True)
            .one()
        )
        hashed_id = encode_int_id(user.user_id)
        return ns_url_for(users_ns, "user", user_id=hashed_id)

    return None


Gets the size of the entire filesystem ( bytes )

def get_filesystem_size(**kwargs):
    """
    Gets the size of the entire filesystem (bytes)

    Kwargs:
        db: global database instance
        redis: global redis instance
    """
    disk = None
    try:
        disk = psutil.disk_usage("/var/k8s")
    except:
        disk = psutil.disk_usage("/")

    return disk.total

def get_filesystem_size ( * * kwargs ) : disk = None try : disk = psutil . disk_usage ( "/var/k8s" ) except : disk = psutil . disk_usage ( "/" ) return disk . total


Gets the used portion of the filesystem ( bytes )

def get_filesystem_used(**kwargs):
    """
    Gets the used portion of the filesystem (bytes)

    Kwargs:
        db: global database instance
        redis: global redis instance
    """
    disk = None
    try:
        disk = psutil.disk_usage("/var/k8s")
    except:
        disk = psutil.disk_usage("/")

    return disk.used

def get_filesystem_used ( * * kwargs ) : disk = None try : disk = psutil . disk_usage ( "/var/k8s" ) except : disk = psutil . disk_usage ( "/" ) return disk . used


def get_received_bytes_per_sec(**kwargs):
    """
    Gets the rate of received bytes per sec over the network

    Kwargs:
        db: global database instance
        redis: global redis instance
    """
    # pylint: disable=W0603
    global bytes_recv, bytes_recv_time

    net_io = psutil.net_io_counters()
    recv = net_io.bytes_recv
    t = time.time()

    rx_sec = 0
    if bytes_recv and bytes_recv_time:
        rx_sec = (recv - bytes_recv) / float(t - bytes_recv_time)

    bytes_recv = recv
    bytes_recv_time = t

    return rx_sec

def
get_received_bytes_per_sec
(
*
*
kwargs
)
:
# pylint: disable=W0603
global
bytes_recv
,
bytes_recv_time
net_io
=
psutil
.
net_io_counters
(
)
recv
=
net_io
.
bytes_recv
t
=
time
.
time
(
)
rx_sec
=
0
if
bytes_recv
and
bytes_recv_time
:
rx_sec
=
(
recv
bytes_recv
)
/
float
(
t
bytes_recv_time
)
bytes_recv
=
recv
bytes_recv_time
=
t
return
rx_sec


def get_transferred_bytes_per_sec(**kwargs):
    """
    Gets the rate of transferred bytes per sec over the network

    Kwargs:
        db: global database instance
        redis: global redis instance
    """
    # pylint: disable=W0603
    global bytes_sent, bytes_sent_time

    net_io = psutil.net_io_counters()
    sent = net_io.bytes_sent
    t = time.time()

    tx_sec = 0
    if bytes_sent and bytes_sent_time:
        tx_sec = (sent - bytes_sent) / float(t - bytes_sent_time)

    bytes_sent = sent
    bytes_sent_time = t
    return tx_sec

def
get_transferred_bytes_per_sec
(
*
*
kwargs
)
:
# pylint: disable=W0603
global
bytes_sent
,
bytes_sent_time
net_io
=
psutil
.
net_io_counters
(
)
sent
=
net_io
.
bytes_sent
t
=
time
.
time
(
)
tx_sec
=
0
if
bytes_sent
and
bytes_sent_time
:
tx_sec
=
(
sent
bytes_sent
)
/
float
(
t
bytes_sent_time
)
bytes_sent
=
sent
bytes_sent_time
=
t
return
tx_sec


def parse_value(monitor, value):
    """
    Parses a string value into the corresponding type

    Args:
        monitor: dict The monitor dictionary qwith name, func, ttl, and type
        value: string The value to parse
    """
    try:
        if str(value) == "None":
            return None
        # pylint: disable=R1705
        if monitor[monitor_names.type] == "bool":
            return value == "True"
        elif monitor[monitor_names.type] == "int":
            return int(value)
        elif monitor[monitor_names.type] == "float":
            return float(value)
        elif monitor[monitor_names.type] == "json":
            return json.loads(value)
        else:  # string
            return str(value)
    except Exception:
        return str(value)

def
parse_value
(
monitor
,
value
)
:
try
:
if
str
(
value
)
==
"None"
:
return
# pylint: disable=R1705
if
monitor
[
monitor_names
.
type
]
==
"bool"
:
return
value
==
"True"
elif
monitor
[
monitor_names
.
type
]
==
"int"
:
return
int
(
value
)
elif
monitor
[
monitor_names
.
type
]
==
"float"
:
return
float
(
value
)
elif
monitor
[
monitor_names
.
type
]
==
"json"
:
return
json
.
loads
(
value
)
else
:
# string
return
str
(
value
)
except
Exception
:
return
str
(
value
)


def get_monitors(monitors):
    """
    Gets monitor values

    Args:
        monitors: list(dict) The list of monitors to get values of
    """
    pipe = redis.pipeline()
    for monitor in monitors:
        key = get_monitor_redis_key(monitor)
        pipe.get(key)
    ret = {}
    results = pipe.execute()
    for i, result in enumerate(results):
        ret[monitors[i][monitor_names.name]] = parse_value(monitors[i], result)
    return ret

def get_monitors ( monitors ) : pipe = redis . pipeline ( ) for monitor in monitors : key = get_monitor_redis_key ( monitor ) pipe . get ( key ) ret = { } results = pipe . execute ( ) for i , result in enumerate ( results ) : ret [ monitors [ i ] [ monitor_names . name ] ] = parse_value ( monitors [ i ] , result ) return ret


def get_total_memory(**kwargs):
    """
    Gets the total vitual memory of the system

    Kwargs:
        db: global database instance
        redis: global redis instance
    """
    mem = psutil.virtual_memory()
    return mem.total

def get_total_memory ( * * kwargs ) : mem = psutil . virtual_memory ( ) return mem . total


def get_used_memory(**kwargs):
    """
    Gets the used virtual memory of the system

    Kwargs:
        db: global database instance
        redis: global redis instance
    """
    mem = psutil.virtual_memory()
    return mem.used

def get_used_memory ( * * kwargs ) : mem = psutil . virtual_memory ( ) return mem . used


def refresh(redis, db, monitor):
    """
    Refreshes the cached value for a monitor

    Args:
        redis: Singleton redis instance
        db: Singleton database instance
        monitor: dict The monitor dictionary qwith name, func, ttl, and type
    """
    key = get_monitor_redis_key(monitor)
    ttl_key = f"{key}:ttl"

    is_fresh = redis.get(ttl_key)
    if is_fresh:
        return

    # Invoke the monitor function with kwargs for db and redis.
    # This allows any monitor to access the db and/or redis connection.
    value = monitor[monitor_names.func](db=db, redis=redis)
    logger.info(
        f"monitoring_queue.py | Computed value for {monitor[monitor_names.name]}"
    )

    redis.set(key, value)

    if "ttl" in monitor:
        # Set a TTL (in seconds) key to track when this value needs refreshing.
        # We store a separate TTL key rather than expiring the value itself
        # so that in the case of an error, the current value can still be read
        redis.set(ttl_key, 1, monitor["ttl"])


A persistent cron - style queue that periodically monitors various health metrics and caches values in redis .

def monitoring_queue_task(self):
    """
    A persistent cron-style queue that periodically monitors various
    health metrics and caches values in redis.

    The queue runs every minute on cron, but individual monitors establish
    their own freshness/refresh rate to operate on.
        1. The queue spins up and for each monitor checks to see if it needs a refresh
        2. Refreshes the value and stores the update in redis
    """
    db = monitoring_queue_task.db
    redis = monitoring_queue_task.redis

    have_lock = False
    update_lock = redis.lock("monitoring_queue_lock", timeout=2000)

    try:
        have_lock = update_lock.acquire(blocking=False)

        if have_lock:
            start_time = time.time()

            for monitor in MONITORS.values():
                try:
                    refresh(redis, db, monitor)
                except Exception as e:
                    logger.warning(
                        f"monitoring_queue.py | Error computing {monitor['name']} {e}"
                    )

            end_time = time.time()
            logger.info(
                f"monitoring_queue.py | Finished monitoring_queue in {end_time - start_time} seconds"
            )
        else:
            logger.info("monitoring_queue.py | Failed to acquire lock")
    except Exception as e:
        logger.error("monitoring_queue.py | Fatal error in main loop", exc_info=True)
        raise e
    finally:
        if have_lock:
            update_lock.release()


Gets database liveness with a select 1 query

def get_database_liveness(**kwargs):
    """
    Gets database liveness with a `select 1` query

    Kwargs:
        db: global database instance
        redis: global redis instance
    """
    db = kwargs["db"]
    try:
        with db.scoped_session() as session:
            q = sqlalchemy.text("SELECT 1")
            session.execute(q).fetchone()
            return str(True)
    except Exception:
        return str(False)

def
get_database_liveness
(
*
*
kwargs
)
:
db
=
kwargs
[
"db"
]
try
:
with
db
.
scoped_session
(
)
as
session
:
q
=
sqlalchemy
.
text
(
"SELECT 1"
)
session
.
execute
(
q
)
.
fetchone
(
)
return
str
(
True
)
except
Exception
:
return
str
(
False
)


def get_database_size(**kwargs):
    """
    Gets the size of the database in bytes

    Kwargs:
        db: global database instance
        redis: global redis instance
    """
    db = kwargs["db"]
    with db.scoped_session() as session:
        q = sqlalchemy.text("SELECT pg_database_size(current_database())")
        res = session.execute(q).fetchone()[0]
        return res


def get_database_connections(**kwargs):
    """
    Gets the number of active database connections

    Kwargs:
        db: global database instance
        redis: global redis instance
    """
    db = kwargs["db"]
    with db.scoped_session() as session:
        q = sqlalchemy.text(
            "SELECT numbackends from pg_stat_database where datname = current_database()"
        )
        res = session.execute(q).fetchone()[0]
        return res


Gets full database query connection information ( waits state query string )

def get_database_connection_info(**kwargs):
    """
    Gets full database query connection information (waits, state, query string)

    Kwargs:
        db: global database instance
        redis: global redis instance
    """
    db = kwargs["db"]
    with db.scoped_session() as session:
        q = sqlalchemy.text(
            "select wait_event_type, wait_event, state, query, to_char(query_start, 'DD Mon YYYY HH:MI:SSPM')"
            + 'as "query_start" from pg_stat_activity where datname = current_database()'
        )

        result = session.execute(q).fetchall()
        connection_info = [dict(row) for row in result]
        return json.dumps(connection_info)


def get_database_index_count(**kwargs):
    """
    Gets number of indexes in the database

    Kwargs:
        db: global database instance
        redis: global redis instance
    """
    db = kwargs["db"]
    with db.scoped_session() as session:
        q = sqlalchemy.text(
            "SELECT COUNT(*) FROM pg_indexes WHERE schemaname = 'public'"
        )
        res = session.execute(q).fetchone()[0]
        return res


Gets full database index information ( tablename indexname indexdef )

def get_database_index_info(**kwargs):
    """
    Gets full database index information (tablename, indexname, indexdef)

    Kwargs:
        db: global database instance
        redis: global redis instance
    """
    db = kwargs["db"]
    with db.scoped_session() as session:
        q = sqlalchemy.text(
            "SELECT tablename, indexname, indexdef FROM pg_indexes WHERE schemaname = 'public'"
            + "ORDER BY tablename, indexname"
        )
        result = session.execute(q).fetchall()
        connection_info = [dict(row) for row in result]
        return json.dumps(connection_info)


Gets table information ( number of rows data size ) .

def get_table_size_info(**kwargs):
    """
    Gets table information (number of rows, data size).

    Kwargs:
        db: global database instance
        redis: global redis instance
    """
    db = kwargs["db"]
    with db.scoped_session() as session:
        sql_statement = """SELECT c.relname AS table_name,
                c.reltuples::text AS rows,
                pg_size_pretty(pg_relation_size(s.relid)) AS data_size
            FROM pg_class c
            JOIN pg_catalog.pg_statio_user_tables s ON s.relname = c.relname
            WHERE c.relkind = 'r'
            ORDER BY c.reltuples DESC;"""

        q = sqlalchemy.text(sql_statement)
        result = session.execute(q).fetchall()
        table_size_info = [dict(row) for row in result]
        return json.dumps(table_size_info)


def get_frequent_queries(**kwargs):
    """
    Gets the most frequent queries

    Kwargs:
        db: global database instance
        redis: global redis instance
    """
    db = kwargs["db"]
    with db.scoped_session() as session:
        q = sqlalchemy.text(
            "SELECT query, calls FROM pg_stat_statements ORDER BY calls DESC LIMIT 100"
        )
        result = session.execute(q).fetchall()
        frequent_queries = [dict(row) for row in result]
        return json.dumps(frequent_queries)


def get_slow_queries(**kwargs):
    """
    Gets the queries with the highest average latency

    Kwargs:
        db: global database instance
        redis: global redis instance
    """
    db = kwargs["db"]
    with db.scoped_session() as session:
        q = sqlalchemy.text(
            "SELECT query, mean_time FROM pg_stat_statements ORDER BY mean_time DESC LIMIT 100"
        )
        result = session.execute(q).fetchall()
        slow_queries = [dict(row) for row in result]

        return json.dumps(slow_queries)


def get_redis_num_keys(**kwargs):
    """
    Gets the total number of keys

    Kwargs:
        db: global database instance
        redis: global redis instance
    """
    redis = kwargs["redis"]
    num_keys = redis.dbsize()
    return num_keys

def get_redis_num_keys ( * * kwargs ) : redis = kwargs [ "redis" ] num_keys = redis . dbsize ( ) return num_keys


def get_redis_used_memory(**kwargs):
    """
    Gets the total memory used by redis

    Kwargs:
        db: global database instance
        redis: global redis instance
    """
    redis = kwargs["redis"]
    info = redis.info()
    return info["used_memory"]

def get_redis_used_memory ( * * kwargs ) : redis = kwargs [ "redis" ] info = redis . info ( ) return info [ "used_memory" ]


def get_redis_total_memory(**kwargs):
    """
    Gets the total memory available to redis

    Kwargs:
        db: global database instance
        redis: global redis instance
    """
    # The current version of redis used does not
    # support `total_system_memory`.
    #
    # Future versions of redis (> 3.2) add support
    # TODO: change this code to the following after upgrading.
    # redis = kwargs['redis']
    # info = redis.info()
    # return info['total_system_memory']
    #
    mem = psutil.virtual_memory()
    return mem.total


Return Tuple containing int representing number of User Library model state changes found in transaction and empty Set ( to align with fn signature of other _state_update functions .

def user_library_state_update(
    self,
    update_task: DatabaseTask,
    session: Session,
    user_library_factory_txs,
    block_number,
    block_timestamp,
    block_hash,
    _ipfs_metadata,  # prefix unused args with underscore to prevent pylint
    _blacklisted_cids,
) -> Tuple[int, Set]:
    """Return Tuple containing int representing number of User Library model state changes found in transaction and empty Set (to align with fn signature of other _state_update functions."""
    empty_set: Set[int] = set()
    num_total_changes = 0
    if not user_library_factory_txs:
        return num_total_changes, empty_set

    user_library_abi = update_task.abi_values["UserLibraryFactory"]["abi"]
    user_library_contract = update_task.web3.eth.contract(
        address=get_contract_addresses()["user_library_factory"], abi=user_library_abi
    )
    challenge_bus = update_task.challenge_event_bus
    block_datetime = datetime.utcfromtimestamp(block_timestamp)

    track_save_state_changes: Dict[int, Dict[int, Save]] = {}
    playlist_save_state_changes: Dict[int, Dict[int, Save]] = {}

    for tx_receipt in user_library_factory_txs:
        try:
            add_track_save(
                self,
                user_library_contract,
                update_task,
                session,
                tx_receipt,
                block_number,
                block_datetime,
                track_save_state_changes,
            )

            add_playlist_save(
                self,
                user_library_contract,
                update_task,
                session,
                tx_receipt,
                block_number,
                block_datetime,
                playlist_save_state_changes,
            )

            delete_track_save(
                self,
                user_library_contract,
                update_task,
                session,
                tx_receipt,
                block_number,
                block_datetime,
                track_save_state_changes,
            )

            delete_playlist_save(
                self,
                user_library_contract,
                update_task,
                session,
                tx_receipt,
                block_number,
                block_datetime,
                playlist_save_state_changes,
            )
        except Exception as e:
            logger.info("Error in user library transaction")
            txhash = update_task.web3.toHex(tx_receipt.transactionHash)
            blockhash = update_task.web3.toHex(block_hash)
            raise IndexingError(
                "user_library", block_number, blockhash, txhash, str(e)
            ) from e

    for user_id, track_ids in track_save_state_changes.items():
        for track_id in track_ids:
            invalidate_old_save(session, user_id, track_id, SaveType.track)
            save = track_ids[track_id]
            session.add(save)
            dispatch_favorite(challenge_bus, save, block_number)
        num_total_changes += len(track_ids)

    for user_id, playlist_ids in playlist_save_state_changes.items():
        for playlist_id in playlist_ids:
            invalidate_old_save(
                session,
                user_id,
                playlist_id,
                playlist_ids[playlist_id].save_type,
            )
            save = playlist_ids[playlist_id]
            session.add(save)
            dispatch_favorite(challenge_bus, save, block_number)
        num_total_changes += len(playlist_ids)

    return num_total_changes, empty_set


Return Tuple containing int representing number of Playlist model state changes found in transaction and set of processed playlist IDs .

def playlist_state_update(
    self,
    update_task: DatabaseTask,
    session: Session,
    playlist_factory_txs,
    block_number,
    block_timestamp,
    block_hash,
    _ipfs_metadata,  # prefix unused args with underscore to prevent pylint
    _blacklisted_cids,
) -> Tuple[int, Set]:
    """Return Tuple containing int representing number of Playlist model state changes found in transaction and set of processed playlist IDs."""
    blockhash = update_task.web3.toHex(block_hash)
    num_total_changes = 0
    skipped_tx_count = 0
    # This stores the playlist_ids created or updated in the set of transactions
    playlist_ids: Set[int] = set()

    if not playlist_factory_txs:
        return num_total_changes, playlist_ids

    playlist_events_lookup: Dict[int, Dict[str, Any]] = {}
    for tx_receipt in playlist_factory_txs:
        txhash = update_task.web3.toHex(tx_receipt.transactionHash)
        for event_type in playlist_event_types_arr:
            playlist_events_tx = get_playlist_events_tx(
                update_task, event_type, tx_receipt
            )
            processedEntries = 0  # if record does not get added, do not count towards num_total_changes
            for entry in playlist_events_tx:
                existing_playlist_record = None
                playlist_id = helpers.get_tx_arg(entry, "_playlistId")
                try:
                    # look up or populate existing record
                    if playlist_id in playlist_events_lookup:
                        existing_playlist_record = playlist_events_lookup[playlist_id][
                            "playlist"
                        ]
                    else:
                        existing_playlist_record = lookup_playlist_record(
                            update_task, session, entry, block_number, txhash
                        )

                    # parse playlist event to add metadata to record
                    playlist_record = parse_playlist_event(
                        self,
                        update_task,
                        entry,
                        event_type,
                        existing_playlist_record,
                        block_timestamp,
                        session,
                    )

                    # process playlist record
                    if playlist_record is not None:
                        if playlist_id not in playlist_events_lookup:
                            playlist_events_lookup[playlist_id] = {
                                "playlist": playlist_record,
                                "events": [],
                            }
                        else:
                            playlist_events_lookup[playlist_id][
                                "playlist"
                            ] = playlist_record
                        playlist_events_lookup[playlist_id]["events"].append(event_type)
                        playlist_ids.add(playlist_id)
                        processedEntries += 1
                except EntityMissingRequiredFieldError as e:
                    logger.warning(f"Skipping tx {txhash} with error {e}")
                    skipped_tx_count += 1
                    add_node_level_skipped_transaction(
                        session, block_number, blockhash, txhash
                    )
                    pass
                except Exception as e:
                    logger.info("Error in parse playlist transaction")
                    raise IndexingError(
                        "playlist", block_number, blockhash, txhash, str(e)
                    ) from e
            num_total_changes += processedEntries

    logger.info(
        f"index.py | playlists.py | There are {num_total_changes} events processed and {skipped_tx_count} skipped transactions."
    )

    for playlist_id, value_obj in playlist_events_lookup.items():
        logger.info(f"index.py | playlists.py | Adding {value_obj['playlist']})")
        if value_obj["events"]:
            invalidate_old_playlist(session, playlist_id)
            session.add(value_obj["playlist"])

    return num_total_changes, playlist_ids


def update_user_balances_task(self):
    """Caches user Audio balances, in wei."""

    db = update_user_balances_task.db
    redis = update_user_balances_task.redis
    eth_web3 = update_user_balances_task.eth_web3
    solana_client_manager = update_user_balances_task.solana_client_manager

    have_lock = False
    update_lock = redis.lock("update_user_balances_lock", timeout=7200)

    try:
        have_lock = update_lock.acquire(blocking=False)

        if have_lock:
            start_time = time.time()

            delegate_manager_inst = get_delegate_manager_contract(eth_web3)
            staking_inst = get_staking_contract(eth_web3)
            token_inst = get_token_contract(
                eth_web3, update_user_balances_task.shared_config
            )
            waudio_token = get_audio_token(solana_client_manager.get_client())
            refresh_user_ids(
                redis,
                db,
                token_inst,
                delegate_manager_inst,
                staking_inst,
                eth_web3,
                waudio_token,
            )

            end_time = time.time()
            redis.set(user_balances_refresh_last_completion_redis_key, int(end_time))
            logger.info(
                f"cache_user_balance.py | Finished cache_user_balance in {end_time - start_time} seconds"
            )
        else:
            logger.info("cache_user_balance.py | Failed to acquire lock")
    except Exception as e:
        logger.error("cache_user_balance.py | Fatal error in main loop", exc_info=True)
        raise e
    finally:
        if have_lock:
            update_lock.release()


Return Tuple containing int representing number of social feature related state changes in this transaction and empty Set ( to align with other _state_update function signatures )

def social_feature_state_update(
    self,
    update_task: DatabaseTask,
    session: Session,
    social_feature_factory_txs,
    block_number,
    block_timestamp,
    block_hash,
    _ipfs_metadata,  # prefix unused args with underscore to prevent pylint
    _blacklisted_cids,
) -> Tuple[int, Set]:
    """Return Tuple containing int representing number of social feature related state changes in this transaction and empty Set (to align with other _state_update function signatures)"""
    empty_set: Set[int] = set()
    num_total_changes = 0
    if not social_feature_factory_txs:
        return num_total_changes, empty_set

    social_feature_factory_abi = update_task.abi_values["SocialFeatureFactory"]["abi"]
    social_feature_factory_contract = update_task.web3.eth.contract(
        address=get_contract_addresses()["social_feature_factory"],
        abi=social_feature_factory_abi,
    )
    challenge_bus = update_task.challenge_event_bus
    block_datetime = datetime.utcfromtimestamp(block_timestamp)

    # stores net state changes of all reposts and follows and corresponding events in current block
    #   track_repost_state_changes = { "user_id": { "track_id": {__Repost__} } }
    #   playlist_repost_state_changes = { "user_id": { "playlist_id": {__Repost__} } }
    #   follow_state_changes = { "follower_user_id": { "followee_user_id": {__Follow__} } }
    track_repost_state_changes: Dict[int, Dict[int, Repost]] = {}
    playlist_repost_state_changes: Dict[int, Dict[int, Repost]] = {}
    follow_state_changes: Dict[int, Dict[int, Follow]] = {}

    for tx_receipt in social_feature_factory_txs:
        try:
            add_track_repost(
                self,
                social_feature_factory_contract,
                update_task,
                session,
                tx_receipt,
                block_number,
                block_datetime,
                track_repost_state_changes,
            )
            delete_track_repost(
                self,
                social_feature_factory_contract,
                update_task,
                session,
                tx_receipt,
                block_number,
                block_datetime,
                track_repost_state_changes,
            )
            add_playlist_repost(
                self,
                social_feature_factory_contract,
                update_task,
                session,
                tx_receipt,
                block_number,
                block_datetime,
                playlist_repost_state_changes,
            )
            delete_playlist_repost(
                self,
                social_feature_factory_contract,
                update_task,
                session,
                tx_receipt,
                block_number,
                block_datetime,
                playlist_repost_state_changes,
            )
            add_follow(
                self,
                social_feature_factory_contract,
                update_task,
                session,
                tx_receipt,
                block_number,
                block_datetime,
                follow_state_changes,
            )
            delete_follow(
                self,
                social_feature_factory_contract,
                update_task,
                session,
                tx_receipt,
                block_number,
                block_datetime,
                follow_state_changes,
            )
        except Exception as e:
            logger.info("Error in parse track transaction")
            txhash = update_task.web3.toHex(tx_receipt.transactionHash)
            blockhash = update_task.web3.toHex(block_hash)
            raise IndexingError(
                "social_feature", block_number, blockhash, txhash, str(e)
            ) from e

    # bulk process all repost and follow changes

    for repost_user_id, repost_track_ids in track_repost_state_changes.items():
        for repost_track_id in repost_track_ids:
            invalidate_old_repost(
                session, repost_user_id, repost_track_id, RepostType.track
            )
            repost = repost_track_ids[repost_track_id]
            session.add(repost)
            dispatch_challenge_repost(challenge_bus, repost, block_number)
        num_total_changes += len(repost_track_ids)

    for repost_user_id, repost_playlist_ids in playlist_repost_state_changes.items():
        for repost_playlist_id in repost_playlist_ids:
            invalidate_old_repost(
                session,
                repost_user_id,
                repost_playlist_id,
                repost_playlist_ids[repost_playlist_id].repost_type,
            )
            repost = repost_playlist_ids[repost_playlist_id]
            session.add(repost)
            dispatch_challenge_repost(challenge_bus, repost, block_number)
        num_total_changes += len(repost_playlist_ids)

    for follower_user_id, followee_user_ids in follow_state_changes.items():
        for followee_user_id in followee_user_ids:
            invalidate_old_follow(session, follower_user_id, followee_user_id)
            follow = followee_user_ids[followee_user_id]
            session.add(follow)
            dispatch_challenge_follow(challenge_bus, follow, block_number)
            queue_related_artist_calculation(update_task.redis, followee_user_id)
        num_total_changes += len(followee_user_ids)
    return num_total_changes, empty_set


def parse_transfer_instruction_data(data: str) -> RewardsManagerTransfer:
    """Parse Transfer instruction data submitted to Audius Rewards Manager program

    Instruction struct:
    pub struct TransferArgs {
        /// Amount to transfer
        pub amount: u64,
        /// ID generated on backend
        pub id: String,
        /// Recipient's Eth address
        pub eth_recipient: EthereumAddress,
    }

    Decodes the data and parses each param into the correct type
    """

    return parse_instruction_data(data, rewards_manager_transfer_instr)

def parse_transfer_instruction_data ( data : str ) -> RewardsManagerTransfer : return parse_instruction_data ( data , rewards_manager_transfer_instr )


Parses the transfer instruction id into [ challenge_id specifier ] The id in the transfer instruction is formatted as <CHALLENGE_ID > : <SPECIFIER >

def parse_transfer_instruction_id(transfer_id: str) -> Optional[List[str]]:
    """Parses the transfer instruction id into [challenge_id, specifier]
    The id in the transfer instruction is formatted as "<CHALLENGE_ID>:<SPECIFIER>"
    """
    id_parts = transfer_id.split(":", 1)
    if len(id_parts) != 2:
        logger.error(
            "index_rewards_manager.py | Unable to parse transfer instruction id"
            f"into challenge_id and specifier {transfer_id}"
        )
        return None
    return id_parts


def get_valid_instruction(
    tx_message: TransactionMessage, meta: ResultMeta
) -> Optional[TransactionMessageInstruction]:
    """Checks that the tx is valid
    checks for the transaction message for correct instruction log
    checks accounts keys for rewards manager account
    checks for rewards manager program in instruction
    """
    try:
        account_keys = tx_message["accountKeys"]
        has_transfer_instruction = any(
            log == "Program log: Instruction: Transfer" for log in meta["logMessages"]
        )

        if not has_transfer_instruction:
            return None

        if not any(REWARDS_MANAGER_ACCOUNT == key for key in account_keys):
            logger.error(
                "index_rewards_manager.py | Rewards manager account missing from account keys"
            )
            return None

        instructions = tx_message["instructions"]
        rewards_manager_program_index = account_keys.index(REWARDS_MANAGER_PROGRAM)
        for instruction in instructions:
            if instruction["programIdIndex"] == rewards_manager_program_index:
                return instruction

        return None
    except Exception as e:
        logger.error(
            f"index_rewards_manager.py | Error processing instruction valid, {e}",
            exc_info=True,
        )
        return None


def fetch_and_parse_sol_rewards_transfer_instruction(
    solana_client_manager: SolanaClientManager, tx_sig: str
) -> RewardManagerTransactionInfo:
    """Fetches metadata for rewards transfer transactions and parses data

    Fetches the transaction metadata from solana using the tx signature
    Checks the metadata for a transfer instruction
    Decodes and parses the transfer instruction metadata
    Validates the metadata fields
    """
    try:
        tx_info = solana_client_manager.get_sol_tx_info(tx_sig)
        result: TransactionInfoResult = tx_info["result"]
        # Create transaction metadata
        tx_metadata: RewardManagerTransactionInfo = {
            "tx_sig": tx_sig,
            "slot": result["slot"],
            "timestamp": result["blockTime"],
            "transfer_instruction": None,
        }
        meta = result["meta"]
        if meta["err"]:
            logger.info(
                f"index_rewards_manager.py | Skipping error transaction from chain {tx_info}"
            )
            return tx_metadata
        tx_message = result["transaction"]["message"]
        instruction = get_valid_instruction(tx_message, meta)
        if instruction is None:
            return tx_metadata
        transfer_instruction_data = parse_transfer_instruction_data(instruction["data"])
        amount = transfer_instruction_data["amount"]
        eth_recipient = transfer_instruction_data["eth_recipient"]
        id = transfer_instruction_data["id"]
        transfer_instruction = parse_transfer_instruction_id(id)
        if transfer_instruction is None:
            return tx_metadata

        challenge_id, specifier = transfer_instruction
        tx_metadata["transfer_instruction"] = {
            "amount": amount,
            "eth_recipient": eth_recipient,
            "challenge_id": challenge_id,
            "specifier": specifier,
        }
        return tx_metadata
    except Exception as e:
        logger.error(
            f"index_rewards_manager.py | Error processing {tx_sig}, {e}", exc_info=True
        )
        raise e


def process_batch_sol_reward_manager_txs(
    session: Session,
    reward_manager_txs: List[RewardManagerTransactionInfo],
    redis: Redis,
):
    """Validates that the transfer instruction is consistent with DB and inserts ChallengeDisbursement DB entries"""
    try:
        logger.error(f"index_reward_manager | {reward_manager_txs}")
        eth_recipients = [
            tx["transfer_instruction"]["eth_recipient"]
            for tx in reward_manager_txs
            if tx["transfer_instruction"] is not None
        ]
        users = (
            session.query(User.wallet, User.user_id)
            .filter(User.wallet.in_(eth_recipients), User.is_current == True)
            .all()
        )
        users_map = {user[0]: user[1] for user in users}

        specifiers = [
            tx["transfer_instruction"]["specifier"]
            for tx in reward_manager_txs
            if tx["transfer_instruction"] is not None
        ]

        user_challenges = (
            session.query(UserChallenge.specifier)
            .filter(
                UserChallenge.specifier.in_(specifiers),
            )
            .all()
        )
        user_challenge_specifiers = {challenge[0] for challenge in user_challenges}

        challenge_disbursements = []
        for tx in reward_manager_txs:
            # Add transaction
            session.add(
                RewardManagerTransaction(
                    signature=tx["tx_sig"],
                    slot=tx["slot"],
                    created_at=datetime.datetime.utcfromtimestamp(tx["timestamp"]),
                )
            )
            # No instruction found
            if tx["transfer_instruction"] is None:
                logger.warning(
                    f"index_rewards_manager.py | No transfer instruction found in {tx}"
                )
                continue
            transfer_instr: RewardTransferInstruction = tx["transfer_instruction"]
            specifier = transfer_instr["specifier"]
            eth_recipient = transfer_instr["eth_recipient"]
            if specifier not in user_challenge_specifiers:
                logger.error(
                    f"index_rewards_manager.py | Challenge specifier {specifier} not found"
                    "while processing disbursement"
                )
            if eth_recipient not in users_map:
                logger.error(
                    f"index_rewards_manager.py | eth_recipient {eth_recipient} not found while processing disbursement"
                )
                tx_signature = tx["tx_sig"]
                raise MissingEthRecipientError(
                    eth_recipient,
                    transfer_instr["challenge_id"],
                    specifier,
                    tx["tx_sig"],
                    tx["slot"],
                    f"Error: eth_recipient {eth_recipient} not found while indexing rewards manager for tx signature {tx_signature}",
                )

            user_id = users_map[eth_recipient]
            logger.info(
                f"index_rewards_manager.py | found successful disbursement for user_id: [{user_id}]"
            )

            challenge_disbursements.append(
                ChallengeDisbursement(
                    challenge_id=transfer_instr["challenge_id"],
                    user_id=user_id,
                    specifier=specifier,
                    amount=str(transfer_instr["amount"]),
                    slot=tx["slot"],
                    signature=tx["tx_sig"],
                )
            )

        if challenge_disbursements:
            # Save out the disbursements
            session.bulk_save_objects(challenge_disbursements)
            # Enqueue balance refreshes for the users
            user_ids = [c.user_id for c in challenge_disbursements]
            enqueue_immediate_balance_refresh(redis, user_ids)

    except Exception as e:
        logger.error(f"index_rewards_manager.py | Error processing {e}", exc_info=True)
        raise e


def get_latest_reward_disbursment_slot(session: Session):
    """Fetches the most recent slot for Challenge Disburements"""
    latest_slot = None
    highest_slot_query = (
        session.query(RewardManagerTransaction.slot).order_by(
            desc(RewardManagerTransaction.slot)
        )
    ).first()
    # Can be None prior to first write operations
    if highest_slot_query is not None:
        latest_slot = highest_slot_query[0]

    # If no slots have yet been recorded, assume all are valid
    if latest_slot is None:
        latest_slot = 0

    return latest_slot


def get_tx_in_db(session: Session, tx_sig: str) -> bool:
    """Checks if the transaction signature already exists for Challenge Disburements"""
    tx_sig_db_count = (
        session.query(RewardManagerTransaction).filter(
            RewardManagerTransaction.signature == tx_sig
        )
    ).count()
    exists = tx_sig_db_count > 0
    return exists

def get_tx_in_db ( session : Session , tx_sig : str ) -> bool : tx_sig_db_count = ( session . query ( RewardManagerTransaction ) . filter ( RewardManagerTransaction . signature == tx_sig ) ) . count ( ) exists = tx_sig_db_count > 0 return exists


def get_transaction_signatures(
    solana_client_manager: SolanaClientManager,
    db: SessionManager,
    program: str,
    get_latest_slot: Callable[[Session], int],
    check_tx_exists: Callable[[Session, str], bool],
    min_slot=None,
) -> List[List[str]]:
    """Fetches next batch of transaction signature offset from the previous latest processed slot

    Fetches the latest processed slot for the rewards manager program
    Iterates backwards from the current tx until an intersection is found with the latest processed slot
    Returns the next set of transaction signature from the current offset slot to process
    """
    # List of signatures that will be populated as we traverse recent operations
    transaction_signatures = []

    last_tx_signature = None

    # Loop exit condition
    intersection_found = False

    # Query for solana transactions until an intersection is found
    with db.scoped_session() as session:
        latest_processed_slot = get_latest_slot(session)
        while not intersection_found:
            transactions_history = solana_client_manager.get_signatures_for_address(
                program, before=last_tx_signature, limit=TX_SIGNATURES_BATCH_SIZE
            )

            transactions_array = transactions_history["result"]
            if not transactions_array:
                intersection_found = True
                logger.info(
                    f"index_rewards_manager.py | No transactions found before {last_tx_signature}"
                )
            else:
                # Current batch of transactions
                transaction_signature_batch = []
                for tx_info in transactions_array:
                    tx_sig = tx_info["signature"]
                    tx_slot = tx_info["slot"]
                    logger.info(
                        f"index_rewards_manager.py | Processing tx={tx_sig} | slot={tx_slot}"
                    )
                    if tx_info["slot"] > latest_processed_slot:
                        transaction_signature_batch.append(tx_sig)
                    elif tx_info["slot"] <= latest_processed_slot and (
                        min_slot is None or tx_info["slot"] > min_slot
                    ):
                        # Check the tx signature for any txs in the latest batch,
                        # and if not present in DB, add to processing
                        logger.info(
                            f"index_rewards_manager.py | Latest slot re-traversal\
                            slot={tx_slot}, sig={tx_sig},\
                            latest_processed_slot(db)={latest_processed_slot}"
                        )
                        exists = check_tx_exists(session, tx_sig)
                        if exists:
                            intersection_found = True
                            break
                        # Ensure this transaction is still processed
                        transaction_signature_batch.append(tx_sig)

                # Restart processing at the end of this transaction signature batch
                last_tx = transactions_array[-1]
                last_tx_signature = last_tx["signature"]

                # Append batch of processed signatures
                if transaction_signature_batch:
                    transaction_signatures.append(transaction_signature_batch)

                # Ensure processing does not grow unbounded
                if len(transaction_signatures) > TX_SIGNATURES_MAX_BATCHES:
                    # Only take the oldest transaction from the transaction_signatures array
                    # transaction_signatures is sorted from newest to oldest
                    transaction_signatures = transaction_signatures[
                        -TX_SIGNATURES_RESIZE_LENGTH:
                    ]

    # Reverse batches aggregated so oldest transactions are processed first
    transaction_signatures.reverse()
    return transaction_signatures


def process_transaction_signatures(
    solana_client_manager: SolanaClientManager,
    db: SessionManager,
    redis: Redis,
    transaction_signatures: List[List[str]],
):
    """Concurrently processes the transactions to update the DB state for reward transfer instructions"""
    last_tx_sig: Optional[str] = None
    last_tx: Optional[RewardManagerTransactionInfo] = None
    if transaction_signatures and transaction_signatures[-1]:
        last_tx_sig = transaction_signatures[-1][0]

    for tx_sig_batch in transaction_signatures:
        logger.info(f"index_rewards_manager.py | processing {tx_sig_batch}")
        batch_start_time = time.time()

        transfer_instructions: List[RewardManagerTransactionInfo] = []
        # Process each batch in parallel
        with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
            parse_sol_tx_futures = {
                executor.submit(
                    fetch_and_parse_sol_rewards_transfer_instruction,
                    solana_client_manager,
                    tx_sig,
                ): tx_sig
                for tx_sig in tx_sig_batch
            }
            for future in concurrent.futures.as_completed(parse_sol_tx_futures):
                try:
                    # No return value expected here so we just ensure all futures are resolved
                    parsed_solana_transfer_instruction = future.result()
                    if parsed_solana_transfer_instruction is not None:
                        transfer_instructions.append(parsed_solana_transfer_instruction)
                        if (
                            last_tx_sig
                            and last_tx_sig
                            == parsed_solana_transfer_instruction["tx_sig"]
                        ):
                            last_tx = parsed_solana_transfer_instruction
                except Exception as exc:
                    logger.error(f"index_rewards_manager.py | {exc}")
                    raise exc
        with db.scoped_session() as session:
            process_batch_sol_reward_manager_txs(session, transfer_instructions, redis)
        batch_end_time = time.time()
        batch_duration = batch_end_time - batch_start_time
        logger.info(
            f"index_rewards_manager.py | processed batch {len(tx_sig_batch)} txs in {batch_duration}s"
        )

    if last_tx:
        cache_latest_sol_rewards_manager_db_tx(
            redis,
            {
                "signature": last_tx["tx_sig"],
                "slot": last_tx["slot"],
                "timestamp": last_tx["timestamp"],
            },
        )


def process_solana_rewards_manager(
    solana_client_manager: SolanaClientManager, db: SessionManager, redis: Redis
):
    """Fetches the next set of reward manager transactions and updates the DB with Challenge Disbursements"""
    if not is_valid_rewards_manager_program:
        logger.error(
            "index_rewards_manager.py | no valid reward manager program passed"
        )
        return
    if not REWARDS_MANAGER_ACCOUNT:
        logger.error("index_rewards_manager.py | reward manager account missing")
        return
    # List of signatures that will be populated as we traverse recent operations
    transaction_signatures = get_transaction_signatures(
        solana_client_manager,
        db,
        REWARDS_MANAGER_PROGRAM,
        get_latest_reward_disbursment_slot,
        get_tx_in_db,
        MIN_SLOT,
    )
    logger.info(f"index_rewards_manager.py | {transaction_signatures}")

    process_transaction_signatures(
        solana_client_manager, db, redis, transaction_signatures
    )


def parse_sol_tx_batch(
    db, solana_client_manager, redis, tx_sig_batch_records, retries=10
):
    """
    Parse a batch of solana transactions in parallel by calling parse_sol_play_transaction
    with a ThreaPoolExecutor

    This function also has a recursive retry upto a certain limit in case a future doesn't complete
    within the alloted time. It clears the futures thread queue and the batch is retried
    """
    batch_start_time = time.time()
    challenge_bus_events = []
    plays = []

    # Last record in this batch to be cached
    # Important to note that the batch records are in time DESC order
    last_tx_in_batch = tx_sig_batch_records[0]
    challenge_bus = index_solana_plays.challenge_event_bus

    # Process each batch in parallel
    with concurrent.futures.ThreadPoolExecutor() as executor:
        parse_sol_tx_futures = {
            executor.submit(
                parse_sol_play_transaction,
                solana_client_manager,
                tx_sig,
            ): tx_sig
            for tx_sig in tx_sig_batch_records
        }
        try:
            for future in concurrent.futures.as_completed(
                parse_sol_tx_futures, timeout=45
            ):
                # Returns the properties for a Play object to be created in the db
                # can be None so check the value exists
                result = future.result()
                if result:
                    user_id, track_id, created_at, source, slot, tx_sig = result

                    # Append plays to a list that will be written if all plays are successfully retrieved
                    # from the rpc pool
                    play: PlayInfo = {
                        "user_id": user_id,
                        "play_item_id": track_id,
                        "created_at": created_at,
                        "updated_at": datetime.now(),
                        "source": source,
                        "slot": slot,
                        "signature": tx_sig,
                    }
                    plays.append(play)
                    # Only enqueue a challenge event if it's *not*
                    # an anonymous listen
                    if user_id is not None:
                        challenge_bus_events.append(
                            {
                                "slot": slot,
                                "user_id": user_id,
                                "created_at": created_at.timestamp(),
                            }
                        )

        except Exception as exc:
            logger.error(
                f"index_solana_plays.py | Error parsing sol play transaction: {exc}"
            )
            # timeout in a ThreadPoolExecutor doesn't actually stop execution of the underlying thread
            # in order to do that we need to actually clear the queue which we do here to force this
            # task to stop execution
            executor._threads.clear()
            concurrent.futures.thread._threads_queues.clear()

            # if we have retries left, recursively call this function again
            if retries > 0:
                return parse_sol_tx_batch(
                    db, solana_client_manager, redis, tx_sig_batch_records, retries - 1
                )

            # if no more retries, raise
            raise exc

        # if the thread pool executor completes successfully without raising an exception
        # the data is successfully fetched so we can add it to the db session and dispatch
        # events to challenge bus

    # Cache the latest play from this batch
    # This reflects the ordering from chain
    for play in plays:
        if play.get("signature") == last_tx_in_batch:
            most_recent_db_play = {
                "signature": play.get("signature"),
                "slot": play.get("slot"),
                "timestamp": int(play.get("created_at").timestamp()),
            }
            cache_latest_sol_play_db_tx(redis, most_recent_db_play)
            break

    db_save_start = time.time()
    logger.info(
        f"index_solana_plays.py | DB | Saving test to DB, fetched batch tx details in {db_save_start - batch_start_time}"
    )

    with db.scoped_session() as session:
        logger.info(
            f"index_solana_plays.py | DB | Acquired session in {time.time() - db_save_start}"
        )
        session_execute_start = time.time()
        # Save in bulk
        session.execute(Play.__table__.insert().values(plays))
        logger.info(
            f"index_solana_plays.py | DB | Session execute completed in {time.time() - session_execute_start}"
        )

    logger.info(
        f"index_solana_plays.py | DB | Saved to DB in {time.time() - db_save_start}"
    )

    track_play_ids = [play["play_item_id"] for play in plays]
    if track_play_ids:
        redis.sadd(TRACK_LISTEN_IDS, *track_play_ids)

    logger.info("index_solana_plays.py | Dispatching listen events")
    listen_dispatch_start = time.time()
    for event in challenge_bus_events:
        challenge_bus.dispatch(
            ChallengeEvent.track_listen,
            event.get("slot"),
            event.get("user_id"),
            {"created_at": event.get("created_at")},
        )
    listen_dispatch_end = time.time()
    listen_dispatch_diff = listen_dispatch_end - listen_dispatch_start
    logger.info(
        f"index_solana_plays.py | Dispatched listen events in {listen_dispatch_diff}"
    )

    batch_end_time = time.time()
    batch_duration = batch_end_time - batch_start_time
    logger.info(
        f"index_solana_plays.py | processed batch {len(tx_sig_batch_records)} txs in {batch_duration}s"
    )
    return None


def get_genres(session: Session) -> List[str]:
    """Returns all genres"""
    genres: List[Tuple[str]] = (session.query(Track.genre).distinct(Track.genre)).all()
    genres = filter(  # type: ignore
        lambda x: x[0] is not None and x[0] != "" and x[0] in genre_allowlist, genres
    )
    return list(map(lambda x: x[0], genres))

def
get_genres
(
session
:
Session
)
->
List
[
str
]
:
genres
:
List
[
Tuple
[
str
]
]
=
(
session
.
query
(
Track
.
genre
)
.
distinct
(
Track
.
genre
)
)
.
all
(
)
genres
=
filter
(
# type: ignore
lambda
x
:
x
[
0
]
is
not
and
x
[
0
]
!=
""
and
x
[
0
]
in
genre_allowlist
,
genres
)
return
list
(
map
(
lambda
x
:
x
[
0
]
,
genres
)
)


Floor a datetime object to a time - span in seconds interval_seconds : Closest number of seconds to floor to

def floor_time(dt: datetime, interval_seconds: int):
    """
    Floor a datetime object to a time-span in seconds
    interval_seconds: Closest number of seconds to floor to

    For example, if floor_time is invoked with `interval_seconds` of 15,
    the provided datetime is rounded down to the nearest 15 minute interval.
    E.g. 10:48 rounds to 10:45, 11:02 rounds to 11:00, etc.
    """
    seconds = (dt.replace(tzinfo=None) - dt.min).seconds
    rounding = seconds // interval_seconds * interval_seconds
    return dt + timedelta(0, rounding - seconds, -dt.microsecond)

def floor_time ( dt : datetime , interval_seconds : int ) : seconds = ( dt . replace ( tzinfo = None ) - dt . min ) . seconds rounding = seconds // interval_seconds * interval_seconds return dt + timedelta ( 0 , rounding - seconds , - dt . microsecond )


Checks if the trending job should re - run based off the last trending run s timestamp and the most recently indexed block s timestamp . If the most recently indexed block ( rounded down to the nearest interval ) is interval_seconds ahead of the last trending job run then the job should re - run . The function returns the an int representing the timestamp if the jobs should re - run else None

def get_should_update_trending(
    db: SessionManager, web3: Web3, redis: Redis, interval_seconds: int
) -> Optional[int]:
    """
    Checks if the trending job should re-run based off the last trending run's timestamp and
    the most recently indexed block's timestamp.
    If the most recently indexed block (rounded down to the nearest interval) is `interval_seconds`
    ahead of the last trending job run, then the job should re-run.
    The function returns the an int, representing the timestamp, if the jobs should re-run, else None
    """
    with db.scoped_session() as session:
        current_db_block = (
            session.query(Block.blockhash).filter(Block.is_current == True).first()
        )
        current_block = web3.eth.getBlock(current_db_block[0], True)
        current_timestamp = current_block["timestamp"]
        block_datetime = floor_time(
            datetime.fromtimestamp(current_timestamp), interval_seconds
        )

        last_trending_datetime = get_last_trending_datetime(redis)
        if not last_trending_datetime:
            return int(block_datetime.timestamp())

        duration_since_last_index = block_datetime - last_trending_datetime
        if duration_since_last_index.total_seconds() >= interval_seconds:
            return int(block_datetime.timestamp())

    return None

def get_should_update_trending ( db : SessionManager , web3 : Web3 , redis : Redis , interval_seconds : int ) -> Optional [ int ] : with db . scoped_session ( ) as session : current_db_block = ( session . query ( Block . blockhash ) . filter ( Block . is_current == True ) . first ( ) ) current_block = web3 . eth . getBlock ( current_db_block [ 0 ] , True ) current_timestamp = current_block [ "timestamp" ] block_datetime = floor_time ( datetime . fromtimestamp ( current_timestamp ) , interval_seconds ) last_trending_datetime = get_last_trending_datetime ( redis ) if not last_trending_datetime : return int ( block_datetime . timestamp ( ) ) duration_since_last_index = block_datetime - last_trending_datetime if duration_since_last_index . total_seconds ( ) >= interval_seconds : return int ( block_datetime . timestamp ( ) ) return None


Caches all trending combination of time - range and genre ( including no genre ) .

def index_trending_task(self):
    """Caches all trending combination of time-range and genre (including no genre)."""
    db = index_trending_task.db
    redis = index_trending_task.redis
    web3 = index_trending_task.web3
    have_lock = False
    update_lock = redis.lock("index_trending_lock", timeout=7200)
    try:
        should_update_timestamp = get_should_update_trending(
            db, web3, redis, UPDATE_TRENDING_DURATION_DIFF_SEC
        )
        have_lock = update_lock.acquire(blocking=False)
        if should_update_timestamp and have_lock:
            index_trending(self, db, redis, should_update_timestamp)
        else:
            logger.info(
                f"index_trending.py | \
                skip indexing: have lock {have_lock}, \
                shoud update {should_update_timestamp}"
            )
    except Exception as e:
        logger.error("index_trending.py | Fatal error in main loop", exc_info=True)
        raise e
    finally:
        if have_lock:
            update_lock.release()


def calculate_trending_challenges_task(self, date=None):
    """Caches all trending combination of time-range and genre (including no genre)."""
    if date is None:
        logger.error("calculate_trending_challenges.py | Must be called with a date")
        return
    # Celery gives this to us formatted as '2022-01-01T00:00:00', need to parse into datetime
    date = datetime.fromisoformat(date)
    db = calculate_trending_challenges_task.db
    redis = calculate_trending_challenges_task.redis
    challenge_bus = calculate_trending_challenges_task.challenge_event_bus
    have_lock = False
    update_lock = redis.lock("calculate_trending_challenges_lock", timeout=7200)
    try:
        have_lock = update_lock.acquire(blocking=False)
        if have_lock:
            enqueue_trending_challenges(db, redis, challenge_bus, date)
        else:
            logger.info(
                "calculate_trending_challenges.py | Failed to acquire index trending lock"
            )
    except Exception as e:
        logger.error(
            "calculate_trending_challenges.py | Fatal error in main loop", exc_info=True
        )
        raise e
    finally:
        if have_lock:
            update_lock.release()


def parse_create_token_data(data: str) -> CreateTokenAccount:
    """Parse Transfer instruction data submitted to Audius Claimable Token program

    Instruction struct:
    pub struct TransferArgs {
        pub eth_address: EthereumAddress,
    }

    Decodes the data and parses each param into the correct type
    """

    return parse_instruction_data(data, create_token_account_instr)

def parse_create_token_data ( data : str ) -> CreateTokenAccount : return parse_instruction_data ( data , create_token_account_instr )


def get_valid_instruction(
    tx_message: TransactionMessage, meta: ResultMeta
) -> Optional[TransactionMessageInstruction]:
    """Checks that the tx is valid
    checks for the transaction message for correct instruction log
    checks accounts keys for claimable token program
    """
    try:
        account_keys = tx_message["accountKeys"]
        instructions = tx_message["instructions"]
        user_bank_program_index = account_keys.index(USER_BANK_ADDRESS)
        for instruction in instructions:
            if instruction["programIdIndex"] == user_bank_program_index:
                return instruction

        return None
    except Exception as e:
        logger.error(
            f"index_user_bank.py | Error processing instruction valid, {e}",
            exc_info=True,
        )
        return None


Return tuple containing int representing number of User model state changes found in transaction and set of processed user IDs .

def user_state_update(
    self,
    update_task: DatabaseTask,
    session: Session,
    user_factory_txs,
    block_number,
    block_timestamp,
    block_hash,
    ipfs_metadata,
    blacklisted_cids,
) -> Tuple[int, Set]:
    """Return tuple containing int representing number of User model state changes found in transaction and set of processed user IDs."""

    blockhash = update_task.web3.toHex(block_hash)
    num_total_changes = 0
    skipped_tx_count = 0
    user_ids: Set[int] = set()
    if not user_factory_txs:
        return num_total_changes, user_ids

    challenge_bus = update_task.challenge_event_bus

    # This stores the state of the user object along with all the events applied to it
    # before it gets committed to the db
    # Data format is {"user_id": {"user", "events": []}}
    # NOTE - events are stored only for debugging purposes and not used or persisted anywhere
    user_events_lookup: Dict[int, Dict[str, Any]] = {}

    # for each user factory transaction, loop through every tx
    # loop through all audius event types within that tx and get all event logs
    # for each event, apply changes to the user in user_events_lookup
    for tx_receipt in user_factory_txs:
        txhash = update_task.web3.toHex(tx_receipt.transactionHash)
        for event_type in user_event_types_arr:
            user_events_tx = get_user_events_tx(update_task, event_type, tx_receipt)
            # if record does not get added, do not count towards num_total_changes
            processedEntries = 0
            for entry in user_events_tx:
                existing_user_record = None
                user_id = helpers.get_tx_arg(entry, "_userId")
                try:
                    # look up or populate existing record
                    if user_id in user_events_lookup:
                        existing_user_record = user_events_lookup[user_id]["user"]
                    else:
                        existing_user_record = lookup_user_record(
                            update_task,
                            session,
                            entry,
                            block_number,
                            block_timestamp,
                            txhash,
                        )

                    # parse user event to add metadata to record
                    if event_type == user_event_types_lookup["update_multihash"]:
                        metadata_multihash = helpers.multihash_digest_to_cid(
                            helpers.get_tx_arg(entry, "_multihashDigest")
                        )
                        user_record = (
                            parse_user_event(
                                self,
                                update_task,
                                session,
                                tx_receipt,
                                block_number,
                                entry,
                                event_type,
                                existing_user_record,
                                ipfs_metadata[metadata_multihash],
                                block_timestamp,
                            )
                            if metadata_multihash not in blacklisted_cids
                            else None
                        )
                    else:
                        user_record = parse_user_event(
                            self,
                            update_task,
                            session,
                            tx_receipt,
                            block_number,
                            entry,
                            event_type,
                            existing_user_record,
                            None,
                            block_timestamp,
                        )

                    # process user record
                    if user_record is not None:
                        if user_id not in user_events_lookup:
                            user_events_lookup[user_id] = {
                                "user": user_record,
                                "events": [],
                            }
                        else:
                            user_events_lookup[user_id]["user"] = user_record
                        user_events_lookup[user_id]["events"].append(event_type)
                        user_ids.add(user_id)
                        processedEntries += 1
                except EntityMissingRequiredFieldError as e:
                    logger.warning(f"Skipping tx {txhash} with error {e}")
                    skipped_tx_count += 1
                    add_node_level_skipped_transaction(
                        session, block_number, blockhash, txhash
                    )
                    pass
                except Exception as e:
                    logger.error("Error in parse user transaction")
                    raise IndexingError(
                        "user", block_number, blockhash, txhash, str(e)
                    ) from e

            num_total_changes += processedEntries

    logger.info(
        f"index.py | users.py | There are {num_total_changes} events processed and {skipped_tx_count} skipped transactions."
    )

    # for each record in user_events_lookup, invalidate the old record and add the new record
    # we do this after all processing has completed so the user record is atomic by block, not tx
    for user_id, value_obj in user_events_lookup.items():
        logger.info(f"index.py | users.py | Adding {value_obj['user']}")
        if value_obj["events"]:
            invalidate_old_user(session, user_id)
            challenge_bus.dispatch(ChallengeEvent.profile_update, block_number, user_id)
            session.add(value_obj["user"])

    return num_total_changes, user_ids


def update_user_associated_wallets(
    session, update_task, user_record, associated_wallets, chain
):
    """Updates the user associated wallets table"""
    try:
        if not isinstance(associated_wallets, dict):
            # With malformed associated wallets, we update the associated wallets
            # to be an empty dict. This has the effect of generating new rows for the
            # already associated wallets and marking them as deleted.
            associated_wallets = {}

        prev_user_associated_wallets_response = (
            session.query(AssociatedWallet.wallet)
            .filter_by(
                user_id=user_record.user_id,
                is_current=True,
                is_delete=False,
                chain=chain,
            )
            .all()
        )

        previous_wallets = [
            wallet for [wallet] in prev_user_associated_wallets_response
        ]
        added_associated_wallets = set()

        session.query(AssociatedWallet).filter_by(
            user_id=user_record.user_id, chain=chain
        ).update({"is_current": False})

        # Verify the wallet signatures and create the user id to wallet associations
        for associated_wallet, wallet_metadata in associated_wallets.items():
            if "signature" not in wallet_metadata or not isinstance(
                wallet_metadata["signature"], str
            ):
                continue
            is_valid_signature = validate_signature(
                chain,
                update_task.web3,
                user_record.user_id,
                associated_wallet,
                wallet_metadata["signature"],
            )

            if is_valid_signature:
                # Check that the wallet doesn't already exist
                wallet_exists = (
                    session.query(AssociatedWallet)
                    .filter_by(
                        wallet=associated_wallet,
                        is_current=True,
                        is_delete=False,
                        chain=chain,
                    )
                    .count()
                    > 0
                )
                if not wallet_exists:
                    added_associated_wallets.add(associated_wallet)
                    associated_wallet_entry = AssociatedWallet(
                        user_id=user_record.user_id,
                        wallet=associated_wallet,
                        chain=chain,
                        is_current=True,
                        is_delete=False,
                        blocknumber=user_record.blocknumber,
                        blockhash=user_record.blockhash,
                    )
                    session.add(associated_wallet_entry)

        # Mark the previously associated wallets as deleted
        for previously_associated_wallet in previous_wallets:
            if previously_associated_wallet not in added_associated_wallets:
                associated_wallet_entry = AssociatedWallet(
                    user_id=user_record.user_id,
                    wallet=previously_associated_wallet,
                    chain=chain,
                    is_current=True,
                    is_delete=True,
                    blocknumber=user_record.blocknumber,
                    blockhash=user_record.blockhash,
                )
                session.add(associated_wallet_entry)

        is_updated_wallets = set(previous_wallets) != added_associated_wallets
        if is_updated_wallets:
            enqueue_immediate_balance_refresh(update_task.redis, [user_record.user_id])
    except Exception as e:
        logger.error(
            f"index.py | users.py | Fatal updating user associated wallets while indexing {e}",
            exc_info=True,
        )


def update_user_events(
    session: Session,
    user_record: User,
    events: UserEventsMetadata,
    bus: ChallengeEventBus,
) -> None:
    """Updates the user events table"""
    try:
        if not isinstance(events, dict):
            # There is something wrong with events, don't process it
            return

        # Get existing UserEvents entry
        existing_user_events = (
            session.query(UserEvents)
            .filter_by(user_id=user_record.user_id, is_current=True)
            .one_or_none()
        )
        existing_referrer = (
            existing_user_events.referrer if existing_user_events else None
        )
        existing_mobile_user = (
            existing_user_events.is_mobile_user if existing_user_events else False
        )
        user_events = UserEvents(
            user_id=user_record.user_id,
            is_current=True,
            blocknumber=user_record.blocknumber,
            blockhash=user_record.blockhash,
            referrer=existing_referrer,
            is_mobile_user=existing_mobile_user,
        )
        for event, value in events.items():
            if (
                event == "referrer"
                and isinstance(value, int)
                and user_events.referrer is None
            ):
                user_events.referrer = value
                bus.dispatch(
                    ChallengeEvent.referral_signup,
                    user_record.blocknumber,
                    value,
                    {"referred_user_id": user_record.user_id},
                )
                bus.dispatch(
                    ChallengeEvent.referred_signup,
                    user_record.blocknumber,
                    user_record.user_id,
                )
            elif (
                event == "is_mobile_user"
                and isinstance(value, bool)
                and not user_events.is_mobile_user
            ):
                user_events.is_mobile_user = value
                if value:
                    bus.dispatch(
                        ChallengeEvent.mobile_install,
                        user_record.blocknumber,
                        user_record.user_id,
                    )
        # Only add a row if there's an update
        if (
            existing_user_events is None
            or user_events.is_mobile_user != existing_mobile_user
            or user_events.referrer != existing_referrer
        ):
            # Mark existing UserEvents entries as not current
            session.query(UserEvents).filter_by(
                user_id=user_record.user_id, is_current=True
            ).update({"is_current": False})
            session.add(user_events)

    except Exception as e:
        logger.error(
            f"index.py | users.py | Fatal updating user events while indexing {e}",
            exc_info=True,
        )


Gets the next hightest milstone threshold avaiable given the play count if past the last threshold or given an invalid previous milestone will return None

def get_next_track_milestone(play_count: int, prev_milestone: Optional[int] = None):
    """
    Gets the next hightest milstone threshold avaiable given the play count,
    if past the last threshold or given an invalid previous milestone, will return None
    """
    next_milestone = milestone_threshold[0]
    if prev_milestone:
        if prev_milestone in next_threshold:
            next_milestone = next_threshold[prev_milestone]
        else:
            # track is past the last milestone, so return none and stop
            return None

    # If play counts have not passed the next threshold, return None
    if play_count < next_milestone:
        return None

    # If play counts have pasted the next milestone threshold, continue to compare against higher thresholds
    next_next_milestone = (
        next_threshold[next_milestone] if next_milestone in next_threshold else None
    )
    while next_next_milestone and play_count >= next_next_milestone:
        next_milestone = next_next_milestone
        next_next_milestone = (
            next_threshold[next_milestone] if next_milestone in next_threshold else None
        )

    return next_milestone


def index_listen_count_milestones_task(self):
    """Caches all trending combination of time-range and genre (including no genre)."""
    db = index_listen_count_milestones_task.db
    redis = index_listen_count_milestones_task.redis
    have_lock = False
    # Max timeout is 60 sec * 10 min
    update_lock = redis.lock("index_listen_count_milestones_lock", timeout=600)
    try:
        have_lock = update_lock.acquire(blocking=False)
        if have_lock:
            index_listen_count_milestones(db, redis)
        else:
            logger.info(
                "index_listen_count_milestones.py | Failed to acquire index trending lock"
            )
    except Exception as e:
        logger.error(
            "index_listen_count_milestones.py | Fatal error in main loop", exc_info=True
        )
        raise e
    finally:
        if have_lock:
            update_lock.release()


def user_replica_set_state_update(
    self,
    update_task: DatabaseTask,
    session: Session,
    user_replica_set_mgr_txs,
    block_number,
    block_timestamp,
    block_hash,
    _ipfs_metadata,  # prefix unused args with underscore to prevent pylint
    _blacklisted_cids,
) -> Tuple[int, Set]:
    """Return Tuple containing int representing number of User model state changes found in transaction and set of user_id values"""

    event_blockhash = update_task.web3.toHex(block_hash)
    num_user_replica_set_changes = 0
    skipped_tx_count = 0

    user_ids: Set[int] = set()
    if not user_replica_set_mgr_txs:
        return num_user_replica_set_changes, user_ids

    # This stores the state of the user object along with all the events applied to it
    # before it gets committed to the db
    # Data format is {"user_id": {"user", "events": []}}
    # NOTE - events are stored only for debugging purposes and not used or persisted anywhere
    user_replica_set_events_lookup = {}

    # This stores the state of the cnode object along with all events applied
    # Data format is {"cnode_sp_id": {"cnode_record", "events":[]}}
    cnode_events_lookup = {}

    # pylint: disable=too-many-nested-blocks
    for tx_receipt in user_replica_set_mgr_txs:
        txhash = update_task.web3.toHex(tx_receipt.transactionHash)
        for event_type in user_replica_set_manager_event_types_arr:
            user_events_tx = get_user_replica_set_mgr_tx(
                update_task, event_type, tx_receipt
            )
            processedEntries = 0  # if record does not get added, do not count towards num_total_changes
            for entry in user_events_tx:
                args = entry["args"]
                existing_user_record = None
                existing_cnode_record = None
                user_id = (
                    helpers.get_tx_arg(entry, "_userId") if "_userId" in args else None
                )
                cnode_sp_id = (
                    helpers.get_tx_arg(entry, "_cnodeSpId")
                    if "_cnodeSpId" in args
                    else None
                )
                try:
                    # if the user id is not in the lookup object, it hasn't been initialized yet
                    # first, get the user object from the db(if exists or create a new one)
                    # then set the lookup object for user_id with the appropriate props
                    if user_id and (user_id not in user_replica_set_events_lookup):
                        existing_user_record = lookup_user_record(
                            update_task,
                            session,
                            entry,
                            block_number,
                            block_timestamp,
                            txhash,
                        )

                    if cnode_sp_id and (cnode_sp_id not in cnode_events_lookup):
                        existing_cnode_record = lookup_ursm_cnode(
                            update_task,
                            session,
                            entry,
                            block_number,
                            block_timestamp,
                            txhash,
                        )

                    # Add or update the value of the user record for this block in user_replica_set_events_lookup,
                    # ensuring that multiple events for a single user result in only 1 row insert operation
                    # (even if multiple operations are present)
                    if (
                        event_type
                        == user_replica_set_manager_event_types_lookup[
                            "update_replica_set"
                        ]
                    ):
                        parsed_user_record = parse_user_record(
                            update_task,
                            entry,
                            existing_user_record,
                            block_timestamp,
                        )
                        if user_id not in user_replica_set_events_lookup:
                            user_replica_set_events_lookup[user_id] = {
                                "user": parsed_user_record,
                                "events": [],
                            }
                        else:
                            user_replica_set_events_lookup[user_id][
                                "user"
                            ] = parsed_user_record
                        user_replica_set_events_lookup[user_id]["events"].append(
                            event_type
                        )
                        user_ids.add(user_id)
                        processedEntries += 1
                    # Process L2 Content Node operations
                    elif (
                        event_type
                        == user_replica_set_manager_event_types_lookup[
                            "add_or_update_content_node"
                        ]
                    ):
                        parsed_cnode_record = parse_ursm_cnode_record(
                            update_task,
                            entry,
                            existing_cnode_record,
                        )
                        if cnode_sp_id not in cnode_events_lookup:
                            cnode_events_lookup[cnode_sp_id] = {
                                "content_node": parsed_cnode_record,
                                "events": [],
                            }
                        else:
                            cnode_events_lookup[cnode_sp_id][
                                "content_node"
                            ] = parsed_cnode_record
                        cnode_events_lookup[cnode_sp_id]["events"].append(event_type)
                        processedEntries += 1
                except EntityMissingRequiredFieldError as e:
                    logger.warning(f"Skipping tx {txhash} with error {e}")
                    skipped_tx_count += 1
                    add_node_level_skipped_transaction(
                        session, block_number, event_blockhash, txhash
                    )
                    pass
                except Exception as e:
                    logger.info("Error in parse user replica set transaction")
                    raise IndexingError(
                        "user_replica_set",
                        block_number,
                        event_blockhash,
                        txhash,
                        str(e),
                    ) from e
            num_user_replica_set_changes += processedEntries

    logger.info(
        f"index.py | user_replica_set.py | [URSM indexing] There are {num_user_replica_set_changes} events processed and {skipped_tx_count} skipped transactions."
    )

    # for each record in user_replica_set_events_lookup, invalidate the old record and add the new record
    # we do this after all processing has completed so the user record is atomic by block, not tx
    for user_id, value_obj in user_replica_set_events_lookup.items():
        logger.info(
            f"index.py | user_replica_set.py | Replica Set Processing Adding {value_obj['user']}"
        )
        invalidate_old_user(session, user_id)
        session.add(value_obj["user"])

    for content_node_id, value_obj in cnode_events_lookup.items():
        logger.info(
            f"index.py | user_replica_set.py | Content Node Processing Adding {value_obj['content_node']}"
        )
        invalidate_old_cnode_record(session, content_node_id)
        session.add(value_obj["content_node"])

    return num_user_replica_set_changes, user_ids


def get_tx_hash_to_skip(session, redis):
    """Fetch if there is a tx_hash to be skipped because of continuous errors"""
    indexing_error = get_indexing_error(redis)
    if (
        isinstance(indexing_error, dict)
        and "has_consensus" in indexing_error
        and indexing_error["has_consensus"]
    ):
        return indexing_error["txhash"]
    else:
        return None

def get_tx_hash_to_skip ( session , redis ) : indexing_error = get_indexing_error ( redis ) if ( isinstance ( indexing_error , dict ) and "has_consensus" in indexing_error and indexing_error [ "has_consensus" ] ) : return indexing_error [ "txhash" ] else : return None


For a redis hset storing a mapping of routes to the number of times they are hit parse each key out into the version path and query string . Create a new entry in the DB for the each route .

def process_route_keys(session, redis, key, ip, date):
    """
    For a redis hset storing a mapping of routes to the number of times they are hit,
    parse each key out into the version, path, and query string.
    Create a new entry in the DB for the each route.
    """
    try:
        route_metrics = []
        routes = redis.hgetall(key)
        for key_bstr in routes:
            route = key_bstr.decode("utf-8").strip("/")
            val = int(routes[key_bstr].decode("utf-8"))

            version = "0"  # default value if version is not present
            path = route
            query_string = None

            route_subpaths = route.split("/")

            # Extract the version out of the path
            if route_subpaths[0].startswith("v") and len(route_subpaths[0]) > 1:
                version = route_subpaths[0][1:]
                path = "/".join(route_subpaths[1:])

            # Extract the query string out of the path
            route_query = path.split("?")
            if len(route_query) > 1:
                path = route_query[0]
                query_string = route_query[1]
            route_metrics.append(
                RouteMetrics(
                    version=version,
                    route_path=path,
                    query_string=query_string,
                    count=val,
                    ip=ip,
                    timestamp=date,
                )
            )

        if route_metrics:
            session.bulk_save_objects(route_metrics)
        redis.delete(key)
    except Exception as e:
        raise Exception(f"Error processing route key {key} with error {e}") from e


For a redis hset storing a mapping of app_name usage in request parameters to count Create a new entry in the DB for each app_name .

def process_app_name_keys(session, redis, key, ip, date):
    """
    For a redis hset storing a mapping of app_name usage in request parameters to count,
    Create a new entry in the DB for each app_name.
    """
    try:
        app_name_metrics = []
        app_names = redis.hgetall(key)
        for key_bstr in app_names:
            app_name = key_bstr.decode("utf-8")
            val = int(app_names[key_bstr].decode("utf-8"))

            app_name_metrics.append(
                AppNameMetrics(
                    application_name=app_name, count=val, ip=ip, timestamp=date
                )
            )
        if app_name_metrics:
            session.bulk_save_objects(app_name_metrics)
        redis.delete(key)

    except Exception as e:
        raise Exception(f"Error processing app name key {key} with error {e}") from e


Move the metrics values from redis to the DB .

def sweep_metrics(db, redis):
    """
    Move the metrics values from redis to the DB.

    Get all the redis keys with the metrics prefix,
    parse the key to get the timestamp in the key.
    If it is before the current time, then process the redis hset.
    """
    with db.scoped_session() as session:
        for key_byte in redis.scan_iter(f"{metrics_prefix}:*"):
            key = key_byte.decode("utf-8")
            try:
                parsed_key = parse_metrics_key(key)

                if parsed_key is None:
                    raise KeyError(
                        f"index_metrics.py | Unable to parse key {key} | Skipping process key"
                    )
                source, ip, key_date = parsed_key

                current_date_time = get_rounded_date_time()

                if key_date < current_date_time:
                    if source == metrics_routes:
                        process_route_keys(session, redis, key, ip, key_date)
                    elif source == metrics_applications:
                        process_app_name_keys(session, redis, key, ip, key_date)
            except KeyError as e:
                logger.warning(e)
                redis.delete(key)
            except Exception as e:
                logger.error(e)
                redis.delete(key)


Get recent route and app metrics from all other discovery nodes and merge with this node s metrics so that this node will be aware of all the metrics across users hitting different providers

def consolidate_metrics_from_other_nodes(self, db, redis):
    """
    Get recent route and app metrics from all other discovery nodes
    and merge with this node's metrics so that this node will be aware
    of all the metrics across users hitting different providers
    """
    all_other_nodes = get_all_other_nodes()[0]

    visited_node_timestamps_str = redis_get_or_restore(redis, metrics_visited_nodes)
    visited_node_timestamps = (
        json.loads(visited_node_timestamps_str) if visited_node_timestamps_str else {}
    )

    now = datetime.utcnow()
    one_iteration_ago = now - timedelta(minutes=METRICS_INTERVAL)
    one_iteration_ago_str = one_iteration_ago.strftime(datetime_format_secondary)
    end_time = now.strftime(datetime_format_secondary)

    # personal unique metrics for the day and the month
    summed_unique_metrics = get_summed_unique_metrics(now)
    summed_unique_daily_count = summed_unique_metrics["daily"]
    summed_unique_monthly_count = summed_unique_metrics["monthly"]

    # Merge & persist metrics for our personal node
    personal_route_metrics_str = redis_get_or_restore(redis, personal_route_metrics)
    personal_route_metrics_dict = (
        json.loads(personal_route_metrics_str) if personal_route_metrics_str else {}
    )
    new_personal_route_metrics = {}
    for timestamp, metrics in personal_route_metrics_dict.items():
        if timestamp > one_iteration_ago_str:
            for ip, count in metrics.items():
                if ip in new_personal_route_metrics:
                    new_personal_route_metrics[ip] += count
                else:
                    new_personal_route_metrics[ip] = count

    personal_app_metrics_str = redis_get_or_restore(redis, personal_app_metrics)
    personal_app_metrics_dict = (
        json.loads(personal_app_metrics_str) if personal_app_metrics_str else {}
    )
    new_personal_app_metrics = {}
    for timestamp, metrics in personal_app_metrics_dict.items():
        if timestamp > one_iteration_ago_str:
            for app_name, count in metrics.items():
                if app_name in new_personal_app_metrics:
                    new_personal_app_metrics[app_name] += count
                else:
                    new_personal_app_metrics[app_name] = count

    merge_route_metrics(new_personal_route_metrics, end_time, db)
    merge_app_metrics(new_personal_app_metrics, end_time, db)

    # Merge & persist metrics for other nodes
    for node in all_other_nodes:
        start_time_str = (
            visited_node_timestamps[node]
            if node in visited_node_timestamps
            else one_iteration_ago_str
        )
        start_time_obj = datetime.strptime(start_time_str, datetime_format_secondary)
        start_time = int(start_time_obj.timestamp())
        new_route_metrics, new_app_metrics = get_metrics(node, start_time)

        logger.info(
            f"did attempt to receive route and app metrics from {node} at {start_time_obj} ({start_time})"
        )

        # add other nodes' summed unique daily and monthly counts to this node's
        if new_route_metrics:
            logger.info(
                f"summed unique metrics from {node}: {new_route_metrics['summed']}"
            )
            summed_unique_daily_count += new_route_metrics["summed"]["daily"]
            summed_unique_monthly_count += new_route_metrics["summed"]["monthly"]
            new_route_metrics = new_route_metrics["deduped"]

        merge_route_metrics(new_route_metrics or {}, end_time, db)
        merge_app_metrics(new_app_metrics or {}, end_time, db)

        if new_route_metrics is not None and new_app_metrics is not None:
            visited_node_timestamps[node] = end_time
            redis_set_and_dump(
                redis, metrics_visited_nodes, json.dumps(visited_node_timestamps)
            )

    # persist updated summed unique counts
    persist_summed_unique_counts(
        db, end_time, summed_unique_daily_count, summed_unique_monthly_count
    )

    logger.info(f"visited node timestamps: {visited_node_timestamps}")


def vacuum_db(self):
    """Vacuum the db"""

    db = vacuum_db.db
    redis = vacuum_db.redis

    have_lock = False
    update_lock = redis.lock("vacuum_db", timeout=3600)

    try:
        have_lock = update_lock.acquire(blocking=False)

        if have_lock:
            engine = db._engine
            with engine.connect().execution_options(
                isolation_level="AUTOCOMMIT"
            ) as connection:
                connection.execute("VACUUM ANALYZE")
        else:
            logger.info("vacuum_db.py | Failed to acquire lock")
    except Exception as e:
        logger.error("vacuum_db.py | Fatal error in main loop", exc_info=True)
        raise e
    finally:
        if have_lock:
            update_lock.release()