Gets a global view into the most saved of type amongst followees . Requires an account . This endpoint is useful in generating views like : - Most favorited

def get_top_followee_saves_route(type):
    """
    Gets a global view into the most saved of `type` amongst followees. Requires an account.
    This endpoint is useful in generating views like:
        - Most favorited

    Args:
        type: (string) The `type` (same as repost/save type) to query from. Currently only
            track is supported.
        limit?: (number) default=25, max=100
    """
    args = to_dict(request.args)
    if "limit" in request.args:
        args["limit"] = min(request.args.get("limit", type=int), 100)
    else:
        args["limit"] = 25
    if "with_users" in request.args:
        args["with_users"] = parse_bool_param(request.args.get("with_users"))

    try:
        tracks = get_top_followee_saves(type, args)
        return api_helpers.success_response(tracks)
    except exceptions.ArgumentError as e:
        return api_helpers.error_response(str(e), 400)

def get_top_followee_saves_route ( type ) : args = to_dict ( request . args ) if "limit" in request . args : args [ "limit" ] = min ( request . args . get ( "limit" , type = int ) , 100 ) else : args [ "limit" ] = 25 if "with_users" in request . args : args [ "with_users" ] = parse_bool_param ( request . args . get ( "with_users" ) ) try : tracks = get_top_followee_saves ( type , args ) return api_helpers . success_response ( tracks ) except exceptions . ArgumentError as e : return api_helpers . error_response ( str ( e ) , 400 )


Executes a function with retries across the iterable . If all executions fail raise an exception .

def _try_all(iterable, func, message, randomize=False):
    """Executes a function with retries across the iterable.
    If all executions fail, raise an exception."""
    items = list(enumerate(iterable))
    items = items if not randomize else random.sample(items, k=len(items))
    for index, value in items:
        try:
            return func(value, index)
        except Exception:
            logger.error(
                f"solana_client_manager.py | _try_all | Failed attempt at index {index} for function {func}"
            )
            if index < len(items) - 1:
                logger.info("solana_client_manager.py | _try_all | Retrying")
            continue
    raise Exception(message)


Do not use this function with ThreadPoolExecutor doesn t play well with futures

def _try_all_with_timeout(iterable, func, message, randomize=False):
    """Do not use this function with ThreadPoolExecutor,
    doesn't play well with futures

    Executes a function with retries across the iterable.
    If all executions fail, raise an exception."""
    items = list(enumerate(iterable))
    items = items if not randomize else random.sample(items, k=len(items))
    for index, value in items:
        try:
            with timeout(30):
                return func(value, index)
        except Exception:
            logger.error(
                f"solana_client_manager.py | _try_all | Failed attempt at index {index} for function {func}"
            )
            if index < len(items) - 1:
                logger.info("solana_client_manager.py | _try_all | Retrying")
            continue
    raise Exception(message)


Fetches a solana transaction by signature with retries and a delay .

def get_sol_tx_info(self, tx_sig: str, retries=DEFAULT_MAX_RETRIES):
        """Fetches a solana transaction by signature with retries and a delay."""

        def handle_get_sol_tx_info(client, index):
            endpoint = self.endpoints[index]
            num_retries = retries
            while num_retries > 0:
                try:
                    logger.info(
                        f"solana_client_manager.py | get_sol_tx_info | Fetching tx {tx_sig} {endpoint}"
                    )
                    tx_info = client.get_confirmed_transaction(tx_sig)
                    logger.info(
                        f"solana_client_manager.py | get_sol_tx_info | Finished fetching tx {tx_sig} {endpoint}"
                    )
                    if tx_info["result"] is not None:
                        return tx_info
                except Exception as e:
                    logger.error(
                        f"solana_client_manager.py | get_sol_tx_info | \
                            Error fetching tx {tx_sig} from endpoint {endpoint}, {e}",
                        exc_info=True,
                    )
                num_retries -= 1
                time.sleep(DELAY_SECONDS)
                logger.error(
                    f"solana_client_manager.py | get_sol_tx_info | Retrying tx fetch: {tx_sig} with endpoint {endpoint}"
                )
            raise Exception(
                f"solana_client_manager.py | get_sol_tx_info | Failed to fetch {tx_sig} with endpoint {endpoint}"
            )

        return _try_all(
            self.clients,
            handle_get_sol_tx_info,
            f"solana_client_manager.py | get_sol_tx_info | All requests failed to fetch {tx_sig}",
        )


Fetches confirmed signatures for transactions given an address .

def get_signatures_for_address(
        self,
        account: Union[str, Account, PublicKey],
        before: Optional[str] = None,
        limit: Optional[int] = None,
        retries: Optional[int] = DEFAULT_MAX_RETRIES,
    ):
        """Fetches confirmed signatures for transactions given an address."""

        def handle_get_signatures_for_address(client, index):
            endpoint = self.endpoints[index]
            num_retries = retries
            while num_retries > 0:
                try:
                    logger.info(
                        f"solana_client_manager.py | handle_get_signatures_for_address | Fetching {before} {endpoint}"
                    )
                    transactions: ConfirmedSignatureForAddressResponse = (
                        client.get_signatures_for_address(account, before, limit)
                    )
                    logger.info(
                        f"solana_client_manager.py | handle_get_signatures_for_address | Finished fetching {before} {endpoint}"
                    )
                    return transactions
                except Exception as e:
                    logger.error(
                        f"solana_client_manager.py | handle_get_signatures_for_address | \
                            Error fetching account {account} from endpoint {endpoint}, {e}",
                        exc_info=True,
                    )
                num_retries -= 1
                time.sleep(DELAY_SECONDS)
                logger.error(
                    f"solana_client_manager.py | handle_get_signatures_for_address | Retrying account fetch: {account} with endpoint {endpoint}"
                )
            raise Exception(
                f"solana_client_manager.py | handle_get_signatures_for_address | Failed to fetch account {account} with endpoint {endpoint}"
            )

        return _try_all_with_timeout(
            self.clients,
            handle_get_signatures_for_address,
            "solana_client_manager.py | get_signatures_for_address | All requests failed",
        )


def parse_instruction_data(data: str, instructionFormat: List[InstructionFormat]):
    """Parses encoded instruction data into a dictionary based on instruction format"""
    decoded = base58.b58decode(data)[1:]
    last_end = 0
    decoded_params: Dict = {}
    for intr in instructionFormat:
        name = intr["name"]
        type = intr["type"]

        if type == SolanaInstructionType.u64:
            type_len = solanaInstructionSpace[type]
            decoded_params[name] = int.from_bytes(
                decoded[last_end : last_end + type_len], "little"
            )
            last_end = last_end + type_len
        elif type == SolanaInstructionType.string:
            type_len = solanaInstructionSpace[type]
            instr_len = int.from_bytes(
                decoded[last_end : last_end + type_len], "little"
            )
            start, end = last_end + type_len, last_end + type_len + instr_len
            decoded_value: bytes = decoded[start:end]
            decoded_params[name] = str(decoded_value, "utf-8")
            last_end = end
        elif type == SolanaInstructionType.EthereumAddress:
            type_len = solanaInstructionSpace[type]
            decoded_int = int.from_bytes(decoded[last_end : last_end + type_len], "big")
            # Ensure stored address is of length 40 characters
            # Pads zeros if present at start of string
            # https://stackoverflow.com/a/12638477
            decoded_params[name] = f"0x{decoded_int:040x}"
            last_end = last_end + type_len
        elif type == SolanaInstructionType.UnixTimestamp:
            type_len = solanaInstructionSpace[type]
            decoded_params[name] = int.from_bytes(
                decoded[last_end : last_end + type_len], "little"
            )
            last_end = last_end + type_len

    return decoded_params


Construct the gateway url for an image directory .

def construct_image_dir_gateway_url(address, CID):
    """Construct the gateway url for an image directory.

    Args:
        args.address - base url of gateway
        args.CID - CID of the image directory
    """
    if not address:
        return None

    return urljoin(address, f"/ipfs/{CID}/original.jpg")

def construct_image_dir_gateway_url ( address , CID ) : if not address : return None return urljoin ( address , f"/ipfs/{CID}/original.jpg" )


Retrieve file from IPFS or gateway validating metadata requirements prior to returning an object with no missing entries

def get_metadata(self, multihash, default_metadata_fields, user_replica_set=None):
        """Retrieve file from IPFS or gateway, validating metadata requirements prior to
        returning an object with no missing entries
        """
        logger.warning(f"IPFSCLIENT | get_metadata - {multihash}")
        api_metadata = default_metadata_fields
        retrieved_from_gateway = False
        retrieved_from_ipfs_node = False
        start_time = time.time()

        with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
            metadata_futures = {}
            metadata_futures[
                executor.submit(
                    self.get_metadata_from_ipfs_node, multihash, default_metadata_fields
                )
            ] = "metadata_from_ipfs_node"
            metadata_futures[
                executor.submit(
                    self.get_metadata_from_gateway,
                    multihash,
                    default_metadata_fields,
                    user_replica_set,
                )
            ] = "metadata_from_gateway"
            for get_metadata_future in concurrent.futures.as_completed(
                metadata_futures, timeout=NEW_BLOCK_TIMEOUT_SECONDS
            ):
                metadata_fetch_source = metadata_futures[get_metadata_future]
                try:
                    api_metadata = get_metadata_future.result()
                    retrieved = api_metadata != default_metadata_fields
                    if retrieved:
                        logger.info(
                            f"IPFSCLIENT | retrieved metadata successfully, \
                            {api_metadata}, \
                            source: {metadata_fetch_source}"
                        )
                        if metadata_fetch_source == "metadata_from_gateway":
                            retrieved_from_gateway = True
                        else:
                            retrieved_from_ipfs_node = True
                        self.force_clear_queue_and_stop_task_execution(executor)
                        break  # use first returned result
                except Exception as e:
                    logger.error(
                        f"IPFSCLIENT | ipfs_lib.py | \
                        ERROR in metadata_futures parallel processing \
                        generated {e}, multihash: {multihash}, source: {metadata_fetch_source}",
                        exc_info=True,
                    )

        retrieved_metadata = retrieved_from_gateway or retrieved_from_ipfs_node
        # Raise error if metadata is not retrieved.
        # Ensure default values are not written into database.
        if not retrieved_metadata:
            logger.error(
                f"IPFSCLIENT | Retrieved metadata: {retrieved_metadata}. "
                f"retrieved from gateway : {retrieved_from_gateway}, "
                f"retrieved from local node : {retrieved_from_ipfs_node}"
            )
            logger.error(api_metadata)
            logger.error(default_metadata_fields)
            raise Exception(
                f"IPFSCLIENT | Failed to retrieve metadata. Using default values for {multihash}"
            )

        duration = time.time() - start_time
        logger.info(
            f"IPFSCLIENT | get_metadata ${multihash} {duration} seconds \
                | from ipfs:{retrieved_from_ipfs_node} |from gateway:{retrieved_from_gateway}"
        )

        return api_metadata


Args : args . user_replica_set - comma - separated string of user s replica urls

def get_metadata_from_gateway(
        self, multihash, default_metadata_fields, user_replica_set: str = None
    ):
        """Args:
        args.user_replica_set - comma-separated string of user's replica urls
        """

        # Default return initial metadata format
        gateway_metadata_json = default_metadata_fields
        logger.warning(
            f"IPFSCLIENT | get_metadata_from_gateway, {multihash} replica set: {user_replica_set}"
        )
        gateway_endpoints = self._cnode_endpoints

        # first attempt to first fetch metadata from user replica set, if provided & non-empty
        if user_replica_set and isinstance(user_replica_set, str):
            user_replicas = user_replica_set.split(",")
            try:
                query_urls = [f"{addr}/ipfs/{multihash}" for addr in user_replicas]
                data = self.query_ipfs_metadata_json(
                    query_urls, default_metadata_fields
                )
                if data is None:
                    raise Exception()
                return data
            except Exception:
                logger.error(
                    "IPFSCLIENT | get_metadata_from_gateway \
                        \nfailed to fetch metadata from user replica gateways"
                )
                # Remove replica set from gateway endpoints before querying
                gateway_endpoints = list(
                    filter(
                        lambda endpoint: endpoint not in user_replicas,
                        gateway_endpoints,
                    )
                )

        logger.warning(
            f"IPFSCLIENT | get_metadata_from_gateway, \
                \ncombined addresses: {gateway_endpoints}, \
                \ncnode_endpoints: {self._cnode_endpoints}"
        )

        query_urls = [f"{addr}/ipfs/{multihash}" for addr in gateway_endpoints]
        data = self.query_ipfs_metadata_json(query_urls, default_metadata_fields)
        if data is None:
            raise Exception(
                f"IPFSCLIENT | Failed to retrieve CID {multihash} from gateway"
            )
        gateway_metadata_json = data
        return gateway_metadata_json


Checks whether a multihash code is valid . : param code : Code to check . : return : Boolean indicating whether code is a valid multihash code .

def is_valid_code(code: int) -> bool:
    """
    Checks whether a multihash code is valid.
    :param code: Code to check.
    :return: Boolean indicating whether `code` is a valid multihash code.
    """

    if is_app_code(code):
        return True

    if code in constants.codes:
        return True

    return False

def is_valid_code ( code : int ) -> bool : if is_app_code ( code ) : return True if code in constants . codes : return True return False


Checks whether a code is part of the app range . : param code : Code to check . : return : Boolean indicating whether code is within the app range .

def is_app_code(code: int) -> bool:
    """
    Checks whether a code is part of the app range.
    :param code: Code to check.
    :return: Boolean indicating whether `code` is within the app range.
    """
    return 0 < code < 0x10

def is_app_code ( code : int ) -> bool : return 0 < code < 0x10


Converts a hash function name into the matching code . If passed a number it will return the number if it s a valid code . : param name : The input hash function code . : return : Output hash function number / code .

def coerce_code(name: Union[str, int]) -> int:
    """
    Converts a hash function name into the matching code.
    If passed a number it will return the number if it's a valid code.
    :param name: The input hash function code.
    :return: Output hash function number/code.
    """
    code = name
    if isinstance(name, str):
        if name not in constants.names:
            raise ValueError(f"Unrecognized hash function name: {name}")
        code = constants.names[name]

    if not isinstance(code, int):
        raise TypeError(f"Hash function code should be a number. Got: {code}")

    if code not in constants.codes and not is_app_code(code):
        raise ValueError(f"Unrecognized function code: {code}")

    return code


Encode a hash digest along with the specified function code . : param digest : Input hash digest as an array of bytes . : param code : The hash function code as an int or string . : param length : The digest length . Defaults to None in which case length is derived from the digest itself . : return : Output digest bytes array .

def encode(digest: bytes, code: Union[str, int], length: Optional[int] = None) -> bytes:
    """
    Encode a hash digest along with the specified function code.
    :param digest: Input hash digest as an array of bytes.
    :param code: The hash function code as an int or string.
    :param length: The digest length. Defaults to None, in which case length is derived from the digest itself.
    :return: Output digest bytes array.
    """
    # Ensure it's a hash function code.
    hash_function = coerce_code(code)

    if not isinstance(digest, bytes):
        raise TypeError("digest should be `bytes` object")

    length = len(digest) if length is None else length

    if length != len(digest):
        raise ValueError("digest length should be equal to specified length")

    return varint.to_varint(hash_function) + varint.to_varint(length) + digest


Decode a hash from the given multihash . : param buf : Input multihash as an array of bytes . : return : Dictionary of type { code : int name : str length : int digest : bytes }

def decode(buf: bytes) -> Dict[str, Any]:
    """
    Decode a hash from the given multihash.
    :param buf: Input multihash as an array of bytes.
    :return: Dictionary of type {code: int, name: str, length: int, digest: bytes}
    """
    if not isinstance(buf, bytes):
        raise TypeError("multihash should be `bytes` object")

    if len(buf) < 3:
        raise ValueError("multihash too short. must be > 3 bytes.")

    code, n = varint.from_varint(buf)
    if not is_valid_code(code):
        raise ValueError(f"multihash unknown function code: {code}")

    length, n = varint.from_varint(buf, n)
    if length < 1:
        raise ValueError(f"multihash invalid length: {length}")

    buf = buf[n:]

    if len(buf) != length:
        raise ValueError(f"multihash length inconsistent: {len(buf)} != {length}")

    return dict(code=code, name=constants.codes[code], length=length, digest=buf)


Convert the given multihash to a base58 encoded string . : param multihash : The input multihash as an array of bytes . : return : Output Base58 encoded string representation of input hash .

def to_b58_string(multihash: bytes) -> str:
    """
    Convert the given multihash to a base58 encoded string.
    :param multihash: The input multihash as an array of bytes.
    :return: Output Base58 encoded string representation of input hash.
    """
    if not isinstance(multihash, bytes):
        raise TypeError("input must be bytes array")
    return base58.b58encode(multihash).decode()

def
to_b58_string
(
multihash
:
bytes
)
->
str
:
if
not
isinstance
(
multihash
,
bytes
)
:
raise
TypeError
(
"input must be bytes array"
)
return
base58
.
b58encode
(
multihash
)
.
decode
(
)


Convert the given base58 encoded string to a multi - hash . : param multihash : The input multihash as a base58 encoded string . : return : Output byte array representation of input hash .

def from_b58_string(multihash: str) -> bytes:
    """
    Convert the given base58 encoded string to a multi-hash.
    :param multihash: The input multihash as a base58 encoded string.
    :return: Output byte array representation of input hash.
    """
    if not isinstance(multihash, str):
        raise TypeError("input must be string")
    return base58.b58decode(multihash)

def
from_b58_string
(
multihash
:
str
)
->
bytes
:
if
not
isinstance
(
multihash
,
str
)
:
raise
TypeError
(
"input must be string"
)
return
base58
.
b58decode
(
multihash
)


Get endpoint for this discovery node At each node get the service info which includes the endpoint return node endpoint of node with matching delegate_owner_wallet

def get_node_endpoint() -> Optional[str]:
    """
    Get endpoint for this discovery node
    At each node, get the service info which includes the endpoint
    return node endpoint of node with matching delegate_owner_wallet
    """
    eth_web3 = web3_provider.get_eth_web3()

    eth_registry_address = eth_web3.toChecksumAddress(
        shared_config["eth_contracts"]["registry"]
    )
    eth_registry_instance = eth_web3.eth.contract(
        address=eth_registry_address, abi=eth_abi_values["Registry"]["abi"]
    )
    sp_factory_address = eth_registry_instance.functions.getContract(
        SP_FACTORY_REGISTRY_KEY
    ).call()
    sp_factory_inst = eth_web3.eth.contract(
        address=sp_factory_address, abi=eth_abi_values["ServiceProviderFactory"]["abi"]
    )
    num_discovery_nodes = sp_factory_inst.functions.getTotalServiceTypeProviders(
        DISCOVERY_NODE_SERVICE_TYPE
    ).call()
    logger.info(f"number of discovery nodes: {num_discovery_nodes}")

    ids_list = list(range(1, num_discovery_nodes + 1))

    endpoint: Optional[str] = None

    # fetch all discovery nodes info in parallel
    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
        discovery_node_futures = {
            executor.submit(fetch_discovery_node_info, i, sp_factory_inst): i
            for i in ids_list
        }
        for future in concurrent.futures.as_completed(discovery_node_futures):
            node_op = discovery_node_futures[future]
            try:
                node_info = future.result()
                wallet = node_info[3]
                if wallet == shared_config["delegate"]["owner_wallet"]:
                    endpoint = node_info[1]
                    break
            except Exception as e:
                logger.error(
                    f"get_all_other_nodes.py | ERROR in discovery_node_futures {node_op} generated {e}"
                )

    logger.info(f"this node's endpoint: {endpoint}")

    return endpoint


Get number of discovery nodes At each node get the service info which includes the endpoint Return all a tuple of node endpoints except that of this node and all wallets ( endpoints wallets )

def get_all_other_nodes() -> Tuple[List[str], List[str]]:
    """
    Get number of discovery nodes
    At each node, get the service info which includes the endpoint
    Return all a tuple of node endpoints except that of this node and all wallets
    (endpoints, wallets)
    """
    eth_web3 = web3_provider.get_eth_web3()

    eth_registry_address = eth_web3.toChecksumAddress(
        shared_config["eth_contracts"]["registry"]
    )
    eth_registry_instance = eth_web3.eth.contract(
        address=eth_registry_address, abi=eth_abi_values["Registry"]["abi"]
    )
    sp_factory_address = eth_registry_instance.functions.getContract(
        SP_FACTORY_REGISTRY_KEY
    ).call()
    sp_factory_inst = eth_web3.eth.contract(
        address=sp_factory_address, abi=eth_abi_values["ServiceProviderFactory"]["abi"]
    )
    num_discovery_nodes = sp_factory_inst.functions.getTotalServiceTypeProviders(
        DISCOVERY_NODE_SERVICE_TYPE
    ).call()
    logger.info(f"number of discovery nodes: {num_discovery_nodes}")

    ids_list = list(range(1, num_discovery_nodes + 1))
    all_other_nodes = []
    all_other_wallets = []

    # fetch all discovery nodes info in parallel
    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
        discovery_node_futures = {
            executor.submit(fetch_discovery_node_info, i, sp_factory_inst): i
            for i in ids_list
        }
        for future in concurrent.futures.as_completed(discovery_node_futures):
            node_op = discovery_node_futures[future]
            try:
                node_info = future.result()
                wallet = node_info[3]
                if wallet != shared_config["delegate"]["owner_wallet"]:
                    endpoint = node_info[1]
                    all_other_wallets.append(wallet)
                    if is_fqdn(endpoint):
                        all_other_nodes.append(endpoint)
            except Exception as e:
                logger.error(
                    f"index_metrics.py | ERROR in discovery_node_futures {node_op} generated {e}"
                )

    logger.info(
        f"this node's delegate owner wallet: {shared_config['delegate']['owner_wallet']}"
    )
    logger.info(f"all the other nodes: {all_other_nodes}")
    return all_other_nodes, all_other_wallets


Before the engine tries to execute a statement try to comment the caller s function name .

def comment_sql_calls(
        self, conn, cursor, statement, parameters, context, executemany
    ):
        """
        Before the engine tries to execute a statement,
        try to comment the caller's function name.
        """
        if "src" in conn.info:
            statement = f"-- {conn.info.pop('src')} \n{statement}"

        return statement, parameters


After a transaction has begun try to add the caller s function name to the connection .

def session_on_after_begin(self, session, transaction, connection):
        """
        After a transaction has begun, try to add the caller's function
        name to the connection.

        This serves as a bridge between the ORM session object and the connection
        which will be used for statements.
        """
        if "src" in session.info:
            connection.info["src"] = session.info["src"]

def session_on_after_begin ( self , session , transaction , connection ) : if "src" in session . info : connection . info [ "src" ] = session . info [ "src" ]


Callback invoked with a raw DBAPI connection every time the engine assigns a new connection to the session manager .

def on_connect(self, dbapi_conn, connection_record):
        """
        Callback invoked with a raw DBAPI connection every time the engine assigns a new
        connection to the session manager.

        Actions that should be fired on new connection should be performed here.
        For example, pg_trgm.similarity_threshold needs to be set once for each connection,
        but not if that connection is recycled and used in another session.
        """
        logger.debug("Using new DBAPI connection")
        cursor = dbapi_conn.cursor()
        set_search_similarity(cursor)
        cursor.close()

def
on_connect
(
self
,
dbapi_conn
,
connection_record
)
:
logger
.
debug
(
"Using new DBAPI connection"
)
cursor
=
dbapi_conn
.
cursor
(
)
set_search_similarity
(
cursor
)
cursor
.
close
(
)


Get a session for direct management / use . Use not recommended unless absolutely necessary .

def session(self):
        """
        Get a session for direct management/use. Use not recommended unless absolutely
        necessary.
        """
        return self._session_factory()

def session ( self ) : return self . _session_factory ( )


Usage : with scoped_session () as session : use the session ...

def scoped_session(self, expire_on_commit=True):
        """
        Usage:
            with scoped_session() as session:
                use the session ...

        Session commits when leaving the block normally, or rolls back if an exception
        is thrown.

        Taken from: http://docs.sqlalchemy.org/en/latest/orm/session_basics.html
        """
        session = self._session_factory()
        session.expire_on_commit = expire_on_commit

        try:
            session.info["src"] = inspect.stack()[2][3]  # get caller's function name
        except Exception:
            pass

        try:
            yield session
            session.commit()
        except:
            session.rollback()
            raise
        finally:
            session.close()

def
scoped_session
(
self
,
expire_on_commit
=
True
)
:
session
=
self
.
_session_factory
(
)
session
.
expire_on_commit
=
expire_on_commit
try
:
session
.
info
[
"src"
]
=
inspect
.
stack
(
)
[
2
]
[
3
]
# get caller's function name
except
Exception
:
pass
try
:
yield
session
session
.
commit
(
)
except
:
session
.
rollback
(
)
raise
finally
:
session
.
close
(
)


Gets the IP address from a request using the X - Forwarded - For header if present

def get_ip(request_obj):
    """Gets the IP address from a request using the X-Forwarded-For header if present"""
    ip = request_obj.headers.get("X-Forwarded-For", request_obj.remote_addr)
    if not ip:
        return ""
    return ip.split(",")[0].strip()

def get_ip ( request_obj ) : ip = request_obj . headers . get ( "X-Forwarded-For" , request_obj . remote_addr ) if not ip : return "" return ip . split ( "," ) [ 0 ] . strip ( )


def get_openresty_public_key():
    """Get public key for openresty if it is running"""
    try:
        resp = requests.get("http://localhost:5000/openresty_pubkey", timeout=1)
        resp.raise_for_status()
        return resp.text
    except requests.exceptions.RequestException:
        return None


Context manager that changes to directory path and return to CWD when exited .

def cd(path):
    """Context manager that changes to directory `path` and return to CWD
    when exited.
    """
    old_path = os.getcwd()
    os.chdir(path)
    try:
        yield
    finally:
        os.chdir(old_path)

def cd ( path ) : old_path = os . getcwd ( ) os . chdir ( path ) try : yield finally : os . chdir ( old_path )


Converts the given SQLAlchemy model into a dictionary primarily used for serialization to JSON .

def model_to_dictionary(model, exclude_keys=None):
    """Converts the given SQLAlchemy model into a dictionary, primarily used
    for serialization to JSON.

    - Includes columns, relationships, and properties decorated with
    `@property` (useful for calculated values from relationships).
    - Excludes the keys in `exclude_keys` and the keys in the given model's
    `exclude_keys` property or attribute.
    - Excludes any property or attribute with a leading underscore.
    """
    model_dict = {}

    columns = model.__table__.columns.keys()
    relationships = model.__mapper__.relationships.keys()
    properties = []
    for key in list(set(dir(model)) - set(columns) - set(relationships)):
        if hasattr(type(model), key):
            attr = getattr(type(model), key)
            if not callable(attr) and isinstance(attr, property):
                properties.append(key)

    if exclude_keys is None:
        exclude_keys = []
    if hasattr(model, "exclude_keys"):
        exclude_keys.extend(model.exclude_keys)

    assert set(exclude_keys).issubset(set(properties).union(columns))

    for key in columns:
        if key not in exclude_keys and not key.startswith("_"):
            model_dict[key] = getattr(model, key)

    for key in properties:
        if key not in exclude_keys and not key.startswith("_"):
            model_dict[key] = getattr(model, key)

    for key in relationships:
        if key not in exclude_keys and not key.startswith("_"):
            attr = getattr(model, key)
            if isinstance(attr, list):
                model_dict[key] = query_result_to_list(attr)
            else:
                model_dict[key] = model_to_dictionary(attr)

    return model_dict

def model_to_dictionary ( model , exclude_keys = None ) : model_dict = { } columns = model . __table__ . columns . keys ( ) relationships = model . __mapper__ . relationships . keys ( ) properties = [ ] for key in list ( set ( dir ( model ) ) - set ( columns ) - set ( relationships ) ) : if hasattr ( type ( model ) , key ) : attr = getattr ( type ( model ) , key ) if not callable ( attr ) and isinstance ( attr , property ) : properties . append ( key ) if exclude_keys is None : exclude_keys = [ ] if hasattr ( model , "exclude_keys" ) : exclude_keys . extend ( model . exclude_keys ) assert set ( exclude_keys ) . issubset ( set ( properties ) . union ( columns ) ) for key in columns : if key not in exclude_keys and not key . startswith ( "_" ) : model_dict [ key ] = getattr ( model , key ) for key in properties : if key not in exclude_keys and not key . startswith ( "_" ) : model_dict [ key ] = getattr ( model , key ) for key in relationships : if key not in exclude_keys and not key . startswith ( "_" ) : attr = getattr ( model , key ) if isinstance ( attr , list ) : model_dict [ key ] = query_result_to_list ( attr ) else : model_dict [ key ] = model_to_dictionary ( attr ) return model_dict


Converts the given tuple into the proper SQLAlchemy model object in dictionary form .

def tuple_to_model_dictionary(t, model):
    """Converts the given tuple into the proper SQLAlchemy model object in dictionary form."""
    keys = model.__table__.columns.keys()
    assert len(t) == len(keys)

    return dict(zip(keys, t))

def tuple_to_model_dictionary ( t , model ) : keys = model . __table__ . columns . keys ( ) assert len ( t ) == len ( keys ) return dict ( zip ( keys , t ) )


Try and remove a file no - op if not present

def remove_test_file(filepath):
    """Try and remove a file, no-op if not present"""
    try:
        os.remove(filepath)
    except OSError:
        pass

def remove_test_file ( filepath ) : try : os . remove ( filepath ) except OSError : pass


Constructs a track s route_id from an unsanitized title and handle . Resulting route_ids are of the shape <handle > / <sanitized_title > .

def create_track_route_id(title, handle):
    """
    Constructs a track's route_id from an unsanitized title and handle.
    Resulting route_ids are of the shape `<handle>/<sanitized_title>`.
    """
    sanitized_title = title.encode("utf-8", "ignore").decode("utf-8", "ignore")
    # Strip out invalid character
    sanitized_title = re.sub(
        r"!|%|#|\$|&|\'|\(|\)|&|\*|\+|,|\/|:|;|=|\?|@|\[|\]|\x00", "", sanitized_title
    )

    # Convert whitespaces to dashes
    sanitized_title = re.sub(r"\s+", "-", sanitized_title)

    # Convert multiple dashes to single dashes
    sanitized_title = re.sub(r"-+", "-", sanitized_title)

    # Lowercase it
    sanitized_title = sanitized_title.lower()

    # Lowercase the handle
    sanitized_handle = handle.lower()

    return f"{sanitized_handle}/{sanitized_title}"


Converts the title of a track into a URL - friendly slug

def create_track_slug(title, track_id, collision_id=0):
    """Converts the title of a track into a URL-friendly 'slug'

    Strips special characters, replaces spaces with dashes, converts to
    lowercase, and appends a collision_id if non-zero.

    If the sanitized title is entirely escaped (empty string), use the
    hashed track_id.

    Example:
    (Title="My Awesome Track!", collision_id=2) => "my-awesome-track-2"
    """
    sanitized_title = title.encode("utf-8", "ignore").decode("utf-8", "ignore")
    # Strip out invalid character
    sanitized_title = re.sub(
        r"!|%|#|\$|&|\'|\(|\)|&|\*|\+|,|\/|:|;|=|\?|@|\[|\]|\x00|\^|\.|\{|\}|\"",
        "",
        sanitized_title,
    )

    # Convert whitespaces to dashes
    sanitized_title = re.sub(r"\s+", "-", sanitized_title.strip())
    sanitized_title = re.sub(r"-+", "-", sanitized_title)

    sanitized_title = sanitized_title.lower()
    # This means that the entire title was sanitized away, use the id
    # for the slug.
    if not sanitized_title:
        sanitized_title = encode_int_id(track_id)

    if collision_id > 0:
        sanitized_title = f"{sanitized_title}-{collision_id}"

    return sanitized_title


Validates that a key is correctly formatted and returns the source : ( routes|applications ) ip address and date of key

def parse_metrics_key(key):
    """
    Validates that a key is correctly formatted and returns
    the source: (routes|applications), ip address, and date of key
    """
    if not key.startswith(metrics_prefix):
        logger.warning(f"Bad redis key inserted w/out metrics prefix {key}")
        return None

    fragments = key.split(":")
    if len(fragments) != 5:
        logger.warning(f"Bad redis key inserted: must have 5 parts {key}")
        return None

    _, source, ip, date, time = fragments
    # Replace the ipv6 _ delimiter back to :
    ip = ip.replace("_", ":")
    if source not in (metrics_routes, metrics_applications):
        logger.warning(f"Bad redis key inserted: must be routes or application {key}")
        return None
    date_time = datetime.strptime(f"{date}:{time}", datetime_format)

    return source, ip, date_time


Merge this node s metrics to those received from other discovery nodes : Update unique and total daily and monthly metrics for routes and apps

def merge_metrics(metrics, end_time, metric_type, db):
    """
    Merge this node's metrics to those received from other discovery nodes:
        Update unique and total, daily and monthly metrics for routes and apps

        Dump the cached metrics so that if this node temporarily goes down,
        we can recover the IPs and app names to perform the calculation and deduplication
        when the node comes back up

        Clean up old metrics from cache

        Persist metrics in the database
    """
    logger.info(f"about to merge {metric_type} metrics: {len(metrics)} new entries")
    day = end_time.split(":")[0]
    month = f"{day[:7]}/01"

    daily_key = daily_route_metrics if metric_type == "route" else daily_app_metrics
    daily_metrics_str = redis_get_or_restore(REDIS, daily_key)
    daily_metrics = json.loads(daily_metrics_str) if daily_metrics_str else {}

    monthly_key = (
        monthly_route_metrics if metric_type == "route" else monthly_app_metrics
    )
    monthly_metrics_str = redis_get_or_restore(REDIS, monthly_key)
    monthly_metrics = json.loads(monthly_metrics_str) if monthly_metrics_str else {}

    if day not in daily_metrics:
        daily_metrics[day] = {}
    if month not in monthly_metrics:
        monthly_metrics[month] = {}

    # only relevant for unique users metrics
    unique_daily_count = 0
    unique_monthly_count = 0

    # only relevant for app metrics
    app_count = {}

    # update daily and monthly metrics, which could be route metrics or app metrics
    # if route metrics, new_value and new_count would be an IP and the number of requests from it
    # otherwise, new_value and new_count would be an app and the number of requests from it
    for new_value, new_count in metrics.items():
        if metric_type == "route" and new_value not in daily_metrics[day]:
            unique_daily_count += 1
        if metric_type == "route" and new_value not in monthly_metrics[month]:
            unique_monthly_count += 1
        if metric_type == "app":
            app_count[new_value] = new_count
        daily_metrics[day][new_value] = (
            daily_metrics[day][new_value] + new_count
            if new_value in daily_metrics[day]
            else new_count
        )
        monthly_metrics[month][new_value] = (
            monthly_metrics[month][new_value] + new_count
            if new_value in monthly_metrics[month]
            else new_count
        )

    # clean up metrics METRICS_INTERVAL after the end of the day from daily_metrics
    yesterday_str = (datetime.utcnow() - timedelta(days=1)).strftime(
        datetime_format_secondary
    )
    daily_metrics = {
        timestamp: metrics
        for timestamp, metrics in daily_metrics.items()
        if timestamp > yesterday_str
    }
    if daily_metrics:
        redis_set_and_dump(REDIS, daily_key, json.dumps(daily_metrics))
    logger.info(f"updated cached daily {metric_type} metrics")

    # clean up metrics METRICS_INTERVAL after the end of the month from monthly_metrics
    thirty_one_days_ago = (datetime.utcnow() - timedelta(days=31)).strftime(
        datetime_format_secondary
    )
    monthly_metrics = {
        timestamp: metrics
        for timestamp, metrics in monthly_metrics.items()
        if timestamp > thirty_one_days_ago
    }
    if monthly_metrics:
        redis_set_and_dump(REDIS, monthly_key, json.dumps(monthly_metrics))
    logger.info(f"updated cached monthly {metric_type} metrics")

    # persist aggregated metrics from other nodes
    day_obj = datetime.strptime(day, day_format).date()
    month_obj = datetime.strptime(month, day_format).date()
    if metric_type == "route":
        persist_route_metrics(
            db,
            day_obj,
            month_obj,
            sum(metrics.values()),
            unique_daily_count,
            unique_monthly_count,
        )
    else:
        persist_app_metrics(db, day_obj, month_obj, app_count)


Extracts the application name redis key and hash from the request The key should be of format : <metrics_prefix > : <metrics_application > : <ip > : <rounded_date_time_format > ie : API_METRICS : applications : 192 . 168 . 0 . 1 : 2020 / 08 / 04 : 14 The hash should be of format : <app_name > ie : audius_dapp

def extract_app_name_key():
    """
    Extracts the application name redis key and hash from the request
    The key should be of format:
        <metrics_prefix>:<metrics_application>:<ip>:<rounded_date_time_format>
        ie: "API_METRICS:applications:192.168.0.1:2020/08/04:14"
    The hash should be of format:
        <app_name>
        ie: "audius_dapp"
    """
    application_name = request.args.get(app_name_param, type=str, default=None)
    ip = get_request_ip(request)
    date_time = get_rounded_date_time().strftime(datetime_format)

    application_key = f"{metrics_prefix}:{metrics_applications}:{ip}:{date_time}"
    return (application_key, application_name)


Extracts the route redis key and hash from the request The key should be of format : <metrics_prefix > : <metrics_routes > : <ip > : <rounded_date_time_format > ie : API_METRICS : routes : 192 . 168 . 0 . 1 : 2020 / 08 / 04 : 14 The hash should be of format : <path > <sorted_query_params > ie : / v1 / tracks / search?genre = rap&query = best

def extract_route_key():
    """
    Extracts the route redis key and hash from the request
    The key should be of format:
        <metrics_prefix>:<metrics_routes>:<ip>:<rounded_date_time_format>
        ie: "API_METRICS:routes:192.168.0.1:2020/08/04:14"
    The hash should be of format:
        <path><sorted_query_params>
        ie: "/v1/tracks/search?genre=rap&query=best"
    """
    path = request.path
    req_args = request.args.items()
    req_args = stringify_query_params(req_args)
    route = f"{path}?{req_args}" if req_args else path
    ip = get_request_ip(request)
    date_time = get_rounded_date_time().strftime(datetime_format)

    route_key = f"{metrics_prefix}:{metrics_routes}:{ip}:{date_time}"
    return (route_key, route)


The metrics decorator records each time a route is hit in redis The number of times a route is hit and an app_name query param are used are recorded . A redis a redis hash map is used to store each of these values .

def record_metrics(func):
    """
    The metrics decorator records each time a route is hit in redis
    The number of times a route is hit and an app_name query param are used are recorded.
    A redis a redis hash map is used to store each of these values.

    NOTE: This must be placed before the cache decorator in order for the redis incr to occur
    """

    @functools.wraps(func)
    def wrap(*args, **kwargs):
        try:
            application_key, application_name = extract_app_name_key()
            route_key, route = extract_route_key()
            REDIS.hincrby(route_key, route, 1)
            if application_name:
                REDIS.hincrby(application_key, application_name, 1)

            record_aggregate_metrics()
        except Exception as e:
            logger.error("Error while recording metrics: %s", e.message)

        return func(*args, **kwargs)

    return wrap

def
record_metrics
(
func
)
:
@
functools
.
wraps
(
func
)
def
wrap
(
*
args
,
*
*
kwargs
)
:
try
:
application_key
,
application_name
=
extract_app_name_key
(
)
route_key
,
route
=
extract_route_key
(
)
REDIS
.
hincrby
(
route_key
,
route
,
1
)
if
application_name
:
REDIS
.
hincrby
(
application_key
,
application_name
,
1
)
record_aggregate_metrics
(
)
except
Exception
as
e
:
logger
.
error
(
"Error while recording metrics: %s"
,
e
.
message
)
return
func
(
*
args
,
*
*
kwargs
)
return
wrap


def to_varint(number: int) -> bytes:
    """
    Pack `number` into varint bytes.

    :param number: Integer value to pack into varint bytes.
    :return: Varint bytes array.
    """
    buf = b""
    while True:
        towrite = number & 0x7F
        number >>= 7
        if number:
            buf += _byte(towrite | 0x80)
        else:
            buf += _byte(towrite)
            break
    return buf

def to_varint ( number : int ) -> bytes : buf = b"" while True : towrite = number & 0x7F number >>= 7 if number : buf += _byte ( towrite | 0x80 ) else : buf += _byte ( towrite ) break return buf


def from_stream(stream: BytesIO, offset: int = 0) -> Tuple[int, int]:
    """
    Read a varint from `stream`.

    :param stream: Input bytes stream.
    :param offset: Offset in bytes from which to begin decoding.
    :return: Tuple containing decoded integer and offset of the first byte after encoded integer in source stream.
    """
    shift = 0
    result = 0
    stream.seek(offset)
    while True:
        i = _read_one(stream)
        result |= (i & 0x7F) << shift
        shift += 7
        if not i & 0x80:
            return result, offset + shift // 7

def from_stream ( stream : BytesIO , offset : int = 0 ) -> Tuple [ int , int ] : shift = 0 result = 0 stream . seek ( offset ) while True : i = _read_one ( stream ) result |= ( i & 0x7F ) << shift shift += 7 if not i & 0x80 : return result , offset + shift // 7


Return an integer value obtained by decoding varint data .

def from_varint(data: bytes, offset: int = 0) -> Tuple[int, int]:
    """
    Return an integer value obtained by decoding varint data.

    :param data: Input varint bytes array.
    :param offset: Offset in bytes from which to begin decoding.
    :return: Tuple containing decoded integer and offset of the first byte after encoded integer in source bytes array.
    """
    return from_stream(BytesIO(data), offset)

def from_varint ( data : bytes , offset : int = 0 ) -> Tuple [ int , int ] : return from_stream ( BytesIO ( data ) , offset )


Read a byte from a file - like object ( as an integer )

def _read_one(stream: BytesIO) -> int:
    """
    Read a byte from a file-like object (as an integer)

    :param stream: Input bytes stream.
    :return: Decoded integer value
    :raises: EOFError if the stream ends while reading bytes.
    """
    c = stream.read(1)
    if c == b"":
        raise EOFError("Unexpected EOF while reading bytes")
    return ord(c)

def
_read_one
(
stream
:
BytesIO
)
->
int
:
c
=
stream
.
read
(
1
)
if
c
==
b""
:
raise
EOFError
(
"Unexpected EOF while reading bytes"
)
return
ord
(
c
)


def auth_middleware(**kwargs):
    """
    Auth middleware decorator.

    Should decorate a route and be used to supply an authed user to
    the query behind a route.

    Example:

    @auth_middleware
    def get(self):
        args = track_slug_parser.parse_args()
        slug, handle = (args.get("slug"), args.get("handle"))
        routes = args.get("route")

    @functools.wraps simply ensures that if Python introspects `inner_wrap`, it refers to
    `func` rather than `inner_wrap`.
    """

    def outer_wrap(func):
        @functools.wraps(func)
        def inner_wrap(*args, **kwargs):
            message = request.headers.get(MESSAGE_HEADER)
            signature = request.headers.get(SIGNATURE_HEADER)

            authed_user_id = None
            if message and signature:
                web3 = web3_provider.get_web3()
                encoded_to_recover = encode_defunct(text=message)
                wallet = web3.eth.account.recover_message(
                    encoded_to_recover, signature=signature
                )
                db = db_session.get_db_read_replica()
                with db.scoped_session() as session:
                    user = (
                        session.query(User.user_id)
                        .filter(
                            # Convert checksum wallet to lowercase
                            User.wallet == wallet.lower(),
                            User.is_current == True,
                        )
                        .first()
                    )
                    if user:
                        authed_user_id = user.user_id
            return func(*args, **kwargs, authed_user_id=authed_user_id)

        return inner_wrap

    return outer_wrap

def
auth_middleware
(
*
*
kwargs
)
:
def
outer_wrap
(
func
)
:
@
functools
.
wraps
(
func
)
def
inner_wrap
(
*
args
,
*
*
kwargs
)
:
message
=
request
.
headers
.
get
(
MESSAGE_HEADER
)
signature
=
request
.
headers
.
get
(
SIGNATURE_HEADER
)
authed_user_id
=
if
message
and
signature
:
web3
=
web3_provider
.
get_web3
(
)
encoded_to_recover
=
encode_defunct
(
text
=
message
)
wallet
=
web3
.
eth
.
account
.
recover_message
(
encoded_to_recover
,
signature
=
signature
)
db
=
db_session
.
get_db_read_replica
(
)
with
db
.
scoped_session
(
)
as
session
:
user
=
(
session
.
query
(
User
.
user_id
)
.
filter
(
# Convert checksum wallet to lowercase
User
.
wallet
==
wallet
.
lower
(
)
,
User
.
is_current
==
True
,
)
.
first
(
)
)
if
user
:
authed_user_id
=
user
.
user_id
return
func
(
*
args
,
*
*
kwargs
,
authed_user_id
=
authed_user_id
)
return
inner_wrap
return
outer_wrap


Connect to the configured database . The connection is unique for each request and will be reused if this is called again .

def get_db():
    """Connect to the configured database. The connection
    is unique for each request and will be reused if this is called
    again.
    """
    return current_app.db_session_manager

def get_db ( ) : return current_app . db_session_manager


def get_db_read_replica():
    """Connect to the configured database. The connection
    is unique for each request and will be reused if this is called
    again.
    """
    return current_app.db_read_replica_session_manager

def get_db_read_replica ( ) : return current_app . db_read_replica_session_manager


Attempts to return value by key otherwise caches and returns work_func

def use_redis_cache(key, ttl_sec, work_func):
    """Attempts to return value by key, otherwise caches and returns `work_func`"""
    redis = redis_connection.get_redis()
    cached_value = get_pickled_key(redis, key)
    if cached_value:
        return cached_value
    to_cache = work_func()
    pickle_and_set(redis, key, to_cache, ttl_sec)
    return to_cache

def use_redis_cache ( key , ttl_sec , work_func ) : redis = redis_connection . get_redis ( ) cached_value = get_pickled_key ( redis , key ) if cached_value : return cached_value to_cache = work_func ( ) pickle_and_set ( redis , key , to_cache , ttl_sec ) return to_cache


Cache decorator . Should be called with @cache ( ttl_sec = 123 transform = transform_response cache_prefix_override = some - prefix )

def cache(**kwargs):
    """
    Cache decorator.
    Should be called with `@cache(ttl_sec=123, transform=transform_response, cache_prefix_override='some-prefix')`

    Arguments:
        ttl_sec: optional,number The time in seconds to cache the response if
            status code < 400
        transform: optional,func The transform function of the wrapped function
            to convert the function response to request response
        cache_prefix_override: optional,the prefix for the cache key to use
            currently the cache decorator function has a default prefix for public API routes
            this param allows us to override the prefix for the internal API routes and avoid confusion

    Usage Notes:
        If the wrapped function returns a tuple, the transform function will not
        be run on the response. The first item of the tuple must be serializable.

        If the wrapped function returns a single response, the transform function
        must be passed to the decorator. The wrapper function response must be
        serializable.

    Decorators in Python are just higher-order-functions that accept a function
    as a single parameter, and return a function that wraps the input function.

    In this case, because we need to pass kwargs into our decorator function,
    we need an additional layer of wrapping; the outermost function accepts the kwargs,
    and when called, returns the decorating function `outer_wrap`, which in turn returns
    the wrapped input function, `inner_wrap`.

    @functools.wraps simply ensures that if Python introspects `inner_wrap`, it refers to
    `func` rather than `inner_wrap`.
    """
    ttl_sec = kwargs["ttl_sec"] if "ttl_sec" in kwargs else default_ttl_sec
    transform = kwargs["transform"] if "transform" in kwargs else None
    cache_prefix_override = (
        kwargs["cache_prefix_override"] if "cache_prefix_override" in kwargs else None
    )
    redis = redis_connection.get_redis()

    def outer_wrap(func):
        @functools.wraps(func)
        def inner_wrap(*args, **kwargs):
            has_user_id = (
                "user_id" in request.args and request.args["user_id"] is not None
            )
            key = extract_key(request.path, request.args.items(), cache_prefix_override)
            if not has_user_id:
                cached_resp = redis.get(key)

                if cached_resp:
                    logger.debug(f"Redis Cache - hit {key}")
                    try:
                        deserialized = pickle.loads(cached_resp)
                        if transform is not None:
                            return transform(deserialized)  # pylint: disable=E1102
                        return deserialized, 200
                    except Exception as e:
                        logger.warning(f"Unable to deserialize cached response: {e}")

                logger.debug(f"Redis Cache - miss {key}")
            response = func(*args, **kwargs)

            if len(response) == 2:
                resp, status_code = response
                if status_code < 400:
                    serialized = pickle.dumps(resp)
                    redis.set(key, serialized, ttl_sec)
                return resp, status_code
            serialized = pickle.dumps(response)
            redis.set(key, serialized, ttl_sec)
            return transform(response)  # pylint: disable=E1102

        return inner_wrap

    return outer_wrap


Overridden read () method to call parse_flask_section () at the end

def read(self, filenames, encoding=None):
        """Overridden read() method to call parse_flask_section() at the end"""
        ret = configparser.ConfigParser.read(self, filenames, encoding)
        self.parse_flask_section()
        return ret

def read ( self , filenames , encoding = None ) : ret = configparser . ConfigParser . read ( self , filenames , encoding ) self . parse_flask_section ( ) return ret


Parse the [ flask ] section of your config and hand off the config to the app in context .

def parse_flask_section(self):
        """Parse the [flask] section of your config and hand off the config
        to the app in context.

        Config vars should have the same name as their flask equivalent except
        in all lower-case."""
        for section_name in self.sections():
            current_app.config[section_name] = {}

        for section_name in self.sections():
            for item in self.items(section_name):
                self._load_item(section_name, item[0])

        # Set db_read_replica url to same as db url if none provided
        if ("url_read_replica" not in current_app.config["db"]) or (
            not current_app.config["db"]["url_read_replica"]
        ):
            current_app.config["db"]["url_read_replica"] = current_app.config["db"][
                "url"
            ]

        # Always disable (not included in app.default_config)
        # See https://flask-restx.readthedocs.io/en/latest/mask.html#usage
        current_app.config["RESTX_MASK_SWAGGER"] = False


Load the specified item from the [ flask ] section . Type is determined by the type of the equivalent value in app . default_config or string if unknown .

def _load_item(self, section_name, key):
        """Load the specified item from the [flask] section. Type is
        determined by the type of the equivalent value in app.default_config
        or string if unknown."""
        default = current_app.default_config.get(key)
        if isinstance(default, datetime.timedelta):
            # One of the default config vars is a timedelta - interpret it
            # as an int and construct using it
            current_app.config[section_name][key] = datetime.timedelta(
                self.getint(section_name, key)
            )
        elif isinstance(default, bool):
            current_app.config[section_name][key] = self.getboolean(section_name, key)
        elif isinstance(default, float):
            current_app.config[section_name][key] = self.getfloat(section_name, key)
        elif isinstance(default, int):
            current_app.config[section_name][key] = self.getint(section_name, key)
        else:
            # All the string keys need to be coerced into str()
            # because Flask expects some of them not to be unicode
            current_app.config[section_name][key] = str(self.get(section_name, key))
        env_config_update(current_app.config, section_name, key)


A custom retry loop to throttle down block range .

def _retry_web3_call(  # type: ignore
    func,
    start_block,
    end_block,
    retries=MAX_REQUEST_RETRIES,
    delay=REQUEST_RETRY_SECONDS,
) -> Tuple[int, list]:  # type: ignore
    """A custom retry loop to throttle down block range.

    If our JSON-RPC server cannot serve all incoming `eth_get_logs` in a single request,
    we retry and throttle down block range for every retry.

    For example, Go Ethereum does not indicate what is an acceptable response size.
    It just fails on the server-side with a "context was cancelled" warning.

    :param func: A callable that triggers Ethereum JSON-RPC, as func(start_block, end_block)
    :param start_block: The initial start block of the block range
    :param end_block: The initial start block of the block range
    :param retries: How many times we retry
    :param delay: Time to sleep between retries
    """
    for i in range(retries):
        try:
            return end_block, func(start_block, end_block)
        except Exception as e:
            # Assume this is HTTPConnectionPool(host='localhost', port=8545): Read timed out. (read timeout=10)
            # from Go Ethereum. This translates to the error "context was cancelled" on the server side:
            # https://github.com/ethereum/go-ethereum/issues/20426
            if i < retries - 1:
                # Give some more verbose info than the default middleware
                logger.warning(
                    "event_scanner.py | Retrying events for block range %d - %d (%d) failed with %s, retrying in %s seconds",
                    start_block,
                    end_block,
                    end_block - start_block,
                    e,
                    delay,
                )
                # Decrease the `eth_get_blocks` range
                end_block = start_block + ((end_block - start_block) // 2)
                # Let the JSON-RPC to recover e.g. from restart
                time.sleep(delay)
                continue
            logger.warning("event_scanner.py | Out of retries")
            raise


def _fetch_events_for_all_contracts(
    web3,
    event_type,
    argument_filters: dict,
    from_block: BlockIdentifier,
    to_block: BlockIdentifier,
) -> Iterable:
    """Get events using eth_get_logs API.

    This method is detached from any contract instance.

    This is a stateless method, as opposed to createFilter.
    It can be safely called against nodes which do not provide `eth_newFilter` API, like Infura.
    """

    if from_block is None:
        raise TypeError("Missing mandatory keyword argument to get_logs: fromBlock")

    # Currently no way to poke this using a public Web3.py API.
    # This will return raw underlying ABI JSON object for the event
    abi = event_type._get_event_abi()

    # Depending on the Solidity version used to compile
    # the contract that uses the ABI,
    # it might have Solidity ABI encoding v1 or v2.
    # We just assume the default that you set on Web3 object here.
    # More information here https://eth-abi.readthedocs.io/en/latest/index.html
    codec: ABICodec = web3.codec

    # Here we need to poke a bit into Web3 internals, as this
    # functionality is not exposed by default.
    # Construct JSON-RPC raw filter presentation based on human readable Python descriptions
    # Namely, convert event names to their keccak signatures
    # More information here:
    # https://github.com/ethereum/web3.py/blob/e176ce0793dafdd0573acc8d4b76425b6eb604ca/web3/_utils/filters.py#L71
    _, event_filter_params = construct_event_filter_params(
        abi,
        codec,
        address=argument_filters.get("address"),
        argument_filters=argument_filters,
        fromBlock=from_block,
        toBlock=to_block,
    )

    logger.debug(
        "event_scanner.py | Querying eth_get_logs with the following parameters: %s",
        event_filter_params,
    )

    # Call JSON-RPC API on your Ethereum node.
    # get_logs() returns raw AttributedDict entries
    logs = web3.eth.getLogs(event_filter_params)

    # Convert raw binary data to Python proxy objects as described by ABI
    all_events = []
    for log in logs:
        # Convert raw JSON-RPC log result to human readable event by using ABI data
        # More information how processLog works here
        # https://github.com/ethereum/web3.py/blob/fbaf1ad11b0c7fac09ba34baff2c256cffe0a148/web3/_utils/events.py#L200
        event = get_event_data(codec, abi, log)
        all_events.append(event)
    return all_events


: param db : database handle : param redis : redis handle : param web3 : Web3 instantiated with provider url : param contract : Contract : param state : state manager to keep tracks of last scanned block and persisting events to db : param event_type : web3 Event we scan : param filters : Filters passed to get_logs e . g . { address : <token - address > }

def __init__(
        self,
        db,
        redis,
        web3: Web3,
        contract: Type[Contract],
        event_type: Type[ContractEvent],
        filters: dict,
    ):
        """
        :param db: database handle
        :param redis: redis handle
        :param web3: Web3 instantiated with provider url
        :param contract: Contract
        :param state: state manager to keep tracks of last scanned block and persisting events to db
        :param event_type: web3 Event we scan
        :param filters: Filters passed to get_logs e.g. { "address": <token-address> }
        """

        self.logger = logger
        self.db = db
        self.redis = redis
        self.contract = contract
        self.web3 = web3
        self.event_type = event_type
        self.filters = filters
        self.last_scanned_block = MIN_SCAN_START_BLOCK
        self.latest_chain_block = self.web3.eth.blockNumber

def __init__ ( self , db , redis , web3 : Web3 , contract : Type [ Contract ] , event_type : Type [ ContractEvent ] , filters : dict , ) : self . logger = logger self . db = db self . redis = redis self . contract = contract self . web3 = web3 self . event_type = event_type self . filters = filters self . last_scanned_block = MIN_SCAN_START_BLOCK self . latest_chain_block = self . web3 . eth . blockNumber


Restore the last scan state from redis . If value not found in redis restore from database .

def restore(self):
        """Restore the last scan state from redis.
        If value not found in redis, restore from database."""
        restored = self.redis.get(eth_indexing_last_scanned_block_key)
        if not restored:
            with self.db.scoped_session() as session:
                result = session.query(EthBlock.last_scanned_block).first()
                restored = result[0] if result else restored
        self.last_scanned_block = int(restored) if restored else MIN_SCAN_START_BLOCK
        logger.info(
            f"event_scanner.py | Restored last scanned block ({self.last_scanned_block})"
        )


Save at the end of each chunk of blocks so we can resume in the case of a crash or CTRL + C Next time the scanner is started we will resume from this block

def save(self, block_number: int):
        """Save at the end of each chunk of blocks, so we can resume in the case of a crash or CTRL+C
        Next time the scanner is started we will resume from this block
        """
        self.last_scanned_block = block_number
        logger.info(
            f"event_scanner.py | Saving last scanned block ({self.last_scanned_block}) to redis"
        )
        self.redis.set(
            eth_indexing_last_scanned_block_key,
            str(self.last_scanned_block),
        )
        with self.db.scoped_session() as session:
            record = session.query(EthBlock).first()
            if record:
                record.last_scanned_block = self.last_scanned_block
            else:
                record = EthBlock(last_scanned_block=self.last_scanned_block)
            session.add(record)


def get_block_timestamp(self, block_num) -> Union[datetime.datetime, None]:
        """Get Ethereum block timestamp"""
        try:
            block_info = self.web3.eth.getBlock(block_num)
        except BlockNotFound:
            # Block was not mined yet,
            # minor chain reorganisation?
            return None
        last_time = block_info["timestamp"]
        return datetime.datetime.utcfromtimestamp(last_time)

def
get_block_timestamp
(
self
,
block_num
)
->
Union
[
datetime
.
datetime
,
]
:
try
:
block_info
=
self
.
web3
.
eth
.
getBlock
(
block_num
)
except
BlockNotFound
:
# Block was not mined yet,
# minor chain reorganisation?
return
last_time
=
block_info
[
"timestamp"
]
return
datetime
.
datetime
.
utcfromtimestamp
(
last_time
)


Get the last mined block on Ethereum chain we are following .

def get_suggested_scan_end_block(self):
        """Get the last mined block on Ethereum chain we are following."""

        # Do not scan all the way to the final block, as this
        # block might not be mined yet
        return self.latest_chain_block - ETH_BLOCK_TAIL_OFFSET


The number of the last block we have stored .

def get_last_scanned_block(self) -> int:
        """The number of the last block we have stored."""
        return self.last_scanned_block

def get_last_scanned_block ( self ) -> int : return self . last_scanned_block


Record a ERC - 20 transfer in our database .

def process_event(
        self, block_timestamp: datetime.datetime, event: TransferEvent
    ) -> str:
        """Record a ERC-20 transfer in our database."""
        # Events are keyed by their transaction hash and log index
        # One transaction may contain multiple events
        # and each one of those gets their own log index

        log_index = event["logIndex"]  # Log index within the block
        # transaction_index = event.transactionIndex  # Transaction index within the block
        txhash = event["transactionHash"].hex()  # Transaction hash
        block_number = event["blockNumber"]

        # Convert ERC-20 Transfer event to our internal format
        args = event["args"]
        transfer = {
            "from": args["from"],
            "to": args["to"],
            "value": args["value"],
            "timestamp": block_timestamp,
        }

        # Add user ids from the transfer event into the balance refresh queue.
        # Depending on the wallet connection, we may have the address stored as
        # lower cased, so to be safe, we refresh check-summed and lower-cased adddresses.
        transfer_event_wallets = [
            transfer["from"],
            transfer["to"],
            transfer["from"].lower(),
            transfer["to"].lower(),
        ]
        with self.db.scoped_session() as session:
            user_result = (
                session.query(User.user_id)
                .filter(User.is_current == True)
                .filter(User.wallet.in_(transfer_event_wallets))
            ).all()
            user_set = {user_id for [user_id] in user_result}

            associated_wallet_result = (
                session.query(AssociatedWallet.user_id)
                .filter(AssociatedWallet.is_current == True)
                .filter(AssociatedWallet.is_delete == False)
                .filter(AssociatedWallet.wallet.in_(transfer_event_wallets))
            ).all()
            associated_wallet_set = {user_id for [user_id] in associated_wallet_result}

            user_ids = list(user_set.union(associated_wallet_set))
            if user_ids:
                logger.info(
                    f"event_scanner.py | Enqueueing user ids {user_ids} to immediate balance refresh queue"
                )
                enqueue_immediate_balance_refresh(self.redis, user_ids)

        # Return a pointer that allows us to look up this event later if needed
        return f"{block_number}-{txhash}-{log_index}"


Read and process events between to block numbers .

def scan_chunk(self, start_block, end_block) -> Tuple[int, list]:
        """Read and process events between to block numbers.

        Dynamically decrease the size of the chunk in case the JSON-RPC server pukes out.

        :return: tuple(actual end block number, when this block was mined, processed events)
        """

        block_timestamps = {}
        get_block_timestamp = self.get_block_timestamp

        # Cache block timestamps to reduce some RPC overhead
        # Real solution might include smarter models around block
        def get_block_mined_timestamp(block_num):
            if block_num not in block_timestamps:
                block_timestamps[block_num] = get_block_timestamp(block_num)
            return block_timestamps[block_num]

        all_processed = []

        # Callable that takes care of the underlying web3 call
        def _fetch_events(from_block, to_block):
            return _fetch_events_for_all_contracts(
                self.web3,
                self.event_type,
                self.filters,
                from_block=from_block,
                to_block=to_block,
            )

        # Do `n` retries on `eth_get_logs`,
        # throttle down block range if needed
        end_block, events = _retry_web3_call(
            _fetch_events, start_block=start_block, end_block=end_block
        )

        for evt in events:
            idx = evt[
                "logIndex"
            ]  # Integer of the log index position in the block, null when its pending

            # We cannot avoid minor chain reorganisations, but
            # at least we must avoid blocks that are not mined yet
            assert idx is not None, "Somehow tried to scan a pending block"

            block_number = evt["blockNumber"]

            # Get UTC time when this event happened (block mined timestamp)
            # from our in-memory cache
            block_timestamp = get_block_mined_timestamp(block_number)

            logger.debug(
                f'event_scanner.py | Processing event {evt["event"]}, block:{evt["blockNumber"]}'
            )
            processed = self.process_event(block_timestamp, evt)
            all_processed.append(processed)

        return end_block, all_processed


def estimate_next_chunk_size(self, current_chuck_size: int, event_found_count: int):
        """Try to figure out optimal chunk size

        Our scanner might need to scan the whole blockchain for all events

        * We want to minimize API calls over empty blocks
        * We want to make sure that one scan chunk does not try to process too many entries once, as we try to control commit buffer size and potentially asynchronous busy loop
        * Do not overload node serving JSON-RPC API by asking data for too many events at a time

        Currently Ethereum JSON-API does not have an API to tell when a first event occured in a blockchain
        and our heuristics try to accelerate block fetching (chunk size) until we see the first event.

        These heurestics exponentially increase the scan chunk size depending on if we are seeing events or not.
        When any transfers are encountered, we are back to scanning only a few blocks at a time.
        It does not make sense to do a full chain scan starting from block 1, doing one JSON-RPC call per 20 blocks.
        """

        if event_found_count > 0:
            # When we encounter first events, reset the chunk size window
            current_chuck_size = MIN_SCAN_CHUNK_SIZE
        else:
            current_chuck_size *= CHUNK_SIZE_INCREASE

        current_chuck_size = max(MIN_SCAN_CHUNK_SIZE, current_chuck_size)
        current_chuck_size = min(MAX_CHUNK_SCAN_SIZE, current_chuck_size)
        return int(current_chuck_size)


def scan(
        self,
        start_block,
        end_block,
        start_chunk_size=START_CHUNK_SIZE,
    ) -> Tuple[list, int]:
        """Perform a token events scan.

        :param start_block: The first block included in the scan
        :param end_block: The last block included in the scan
        :param start_chunk_size: How many blocks we try to fetch over JSON-RPC on the first attempt

        :return: [All processed events, number of chunks used]
        """

        current_block = start_block

        # Scan in chunks, commit between
        chunk_size = start_chunk_size
        last_scan_duration = last_logs_found = 0
        total_chunks_scanned = 0

        # All processed entries we got on this scan cycle
        all_processed = []

        while current_block <= end_block:

            # Print some diagnostics to logs to try to fiddle with real world JSON-RPC API performance
            estimated_end_block = min(
                current_block + chunk_size, self.get_suggested_scan_end_block()
            )
            logger.debug(
                "event_scanner.py | Scanning token transfers for blocks: %d - %d, chunk size %d, last chunk scan took %f, last logs found %d",
                current_block,
                estimated_end_block,
                chunk_size,
                last_scan_duration,
                last_logs_found,
            )

            start = time.time()
            actual_end_block, new_entries = self.scan_chunk(
                current_block, estimated_end_block
            )

            # Where does our current chunk scan ends - are we out of chain yet?
            current_end = actual_end_block

            last_scan_duration = int(time.time() - start)
            all_processed += new_entries

            # Try to guess how many blocks to fetch over `eth_get_logs` API next time
            chunk_size = self.estimate_next_chunk_size(chunk_size, len(new_entries))

            # Set where the next chunk starts
            current_block = current_end + 1
            total_chunks_scanned += 1
            self.save(min(current_end, self.get_suggested_scan_end_block()))

        return all_processed, total_chunks_scanned


Registers a listener ( ChallengeManager ) to listen for a particular event type .

def register_listener(self, event: ChallengeEvent, listener: ChallengeManager):
        """Registers a listener (`ChallengeManager`) to listen for a particular event type."""
        self._listeners[event].append(listener)
        if listener.challenge_id not in self._managers:
            self._managers[listener.challenge_id] = listener

def register_listener ( self , event : ChallengeEvent , listener : ChallengeManager ) : self . _listeners [ event ] . append ( listener ) if listener . challenge_id not in self . _managers : self . _managers [ listener . challenge_id ] = listener


def get_manager(self, challenge_id: str) -> ChallengeManager:
        """Gets a manager for a given challenge_id"""
        return self._managers[challenge_id]

def get_manager ( self , challenge_id : str ) -> ChallengeManager : return self . _managers [ challenge_id ]


def does_manager_exist(self, challenge_id: str) -> bool:
        """Returns whether or not a manager exists for a given challenge_id"""
        return challenge_id in self._managers

def does_manager_exist ( self , challenge_id : str ) -> bool : return challenge_id in self . _managers


Makes the bus only dispatch the events once out of the new scope created with with

def use_scoped_dispatch_queue(self):
        """Makes the bus only dispatch the events once out of the new scope created with 'with'"""
        if len(self._in_memory_queue) > 0:
            logger.warning("ChallengeEventBus: Already using in-memory queue")
        try:
            yield self._in_memory_queue
        finally:
            self.flush()


Dispatches an event + block_number + user_id to an in memory queue .

def dispatch(
        self,
        event: ChallengeEvent,
        block_number: int,
        user_id: int,
        extra: Dict = None,
    ):
        """Dispatches an event + block_number + user_id to an in memory queue.

        Does not dispatch to Redis until flush is called or a scoped dispatch queue goes out of scope
        """
        if extra is None:
            extra = {}
        # Sanitize input, drop the event if it's malformed
        valid_event = event is not None and isinstance(event, str)
        valid_block = block_number is not None and isinstance(block_number, int)
        valid_user = user_id is not None and isinstance(user_id, int)
        valid_extra = extra is not None and isinstance(extra, dict)
        if not (valid_event and valid_block and valid_user and valid_extra):
            logger.warning(
                f"ChallengeEventBus: ignoring invalid event: {(event, block_number, user_id, extra)}"
            )
            return

        self._in_memory_queue.append(
            {
                "event": event,
                "block_number": block_number,
                "user_id": user_id,
                "extra": extra,
            }
        )


Flushes the in - memory queue of events and enqueues them to Redis

def flush(self):
        """Flushes the in-memory queue of events and enqueues them to Redis"""
        logger.info(
            f"ChallengeEventBus: Flushing {len(self._in_memory_queue)} events from in-memory queue"
        )
        for event in self._in_memory_queue:
            try:
                event_json = self._event_to_json(
                    event["event"],
                    event["block_number"],
                    event["user_id"],
                    event.get("extra", {}),
                )
                logger.info(f"ChallengeEventBus: dispatch {event_json}")
                self._redis.rpush(REDIS_QUEUE_PREFIX, event_json)
            except Exception as e:
                logger.warning(f"ChallengeEventBus: error enqueuing to Redis: {e}")
        self._in_memory_queue.clear()


Dequeues max_events from Redis queue and processes them forwarding to listening ChallengeManagers . Returns ( num_processed_events did_error ) . Will return - 1 as num_processed_events if an error prevented any events from being processed ( i . e . some error deserializing from Redis )

def process_events(self, session: Session, max_events=1000) -> Tuple[int, bool]:
        """Dequeues `max_events` from Redis queue and processes them, forwarding to listening ChallengeManagers.
        Returns (num_processed_events, did_error).
        Will return -1 as num_processed_events if an error prevented any events from
        being processed (i.e. some error deserializing from Redis)
        """
        try:
            # get the first max_events elements.
            events_json = self._redis.lrange(REDIS_QUEUE_PREFIX, 0, max_events)
            logger.info(f"ChallengeEventBus: dequeued {len(events_json)} events")
            # trim the first from the front of the list
            self._redis.ltrim(REDIS_QUEUE_PREFIX, len(events_json), -1)
            events_dicts = list(map(self._json_to_event, events_json))

            # Consolidate event types for processing
            # map of {"event_type": [{ user_id: number, block_number: number, extra: {} }]}}
            event_user_dict: DefaultDict[
                ChallengeEvent, List[EventMetadata]
            ] = defaultdict(lambda: [])
            for event_dict in events_dicts:
                event_type = event_dict["event"]
                event_user_dict[event_type].append(
                    {
                        "user_id": event_dict["user_id"],
                        "block_number": event_dict["block_number"],
                        "extra": event_dict.get(  # use .get to be safe since prior versions didn't have `extra`
                            "extra", {}
                        ),
                    }
                )
        except Exception as e:
            logger.warning(f"ChallengeEventBus: error processing from Redis: {e}")
            return (-1, True)

        did_error = False
        for (event_type, event_dicts) in event_user_dict.items():
            listeners = self._listeners[event_type]
            for listener in listeners:
                try:
                    listener.process(session, event_type, event_dicts)
                except Exception as e:
                    # We really shouldn't see errors from a ChallengeManager (they should handle on their own),
                    # but in case we do, swallow it and continue on
                    logger.warning(
                        f"ChallengeEventBus: manager [{listener.challenge_id} unexpectedly propogated error: [{e}]"
                    )
                    did_error = True

        return (len(events_json), did_error)


Checks if is daylight savings time During daylight savings the clock moves forward one hr

def is_dst(zonename, dt):
    """Checks if is daylight savings time
    During daylight savings, the clock moves forward one hr
    """
    tz = pytz.timezone(zonename)
    localized = pytz.utc.localize(dt)
    return localized.astimezone(tz).dst() != timedelta(0)

def is_dst ( zonename , dt ) : tz = pytz . timezone ( zonename ) localized = pytz . utc . localize ( dt ) return localized . astimezone ( tz ) . dst ( ) != timedelta ( 0 )


(
    session: Session, timestamp: int
)

Checks if the timestamp is after a week and there is no pending trending update Returns a tuple of boolean if the challenge should be updated and if it s set to true the date

def should_trending_challenge_update(
    session: Session, timestamp: int
) -> Tuple[bool, Optional[date]]:
    """Checks if the timestamp is after a week and there is no pending trending update
    Returns a tuple of boolean if the challenge should be updated, and if it's set to true, the date
    """

    dt = datetime.fromtimestamp(timestamp)
    is_valid_timestamp = get_is_valid_timestamp(dt)
    if not is_valid_timestamp:
        return (False, None)

    # DB query for most recent db row of trending's date
    # using that, figure out new date threshold -> next friday at noon
    most_recent_user_challenge = (
        session.query(TrendingResult.week).order_by(desc(TrendingResult.week)).first()
    )

    if most_recent_user_challenge is None:
        # do somthing
        return (True, dt.date())
    week = most_recent_user_challenge[0]

    if week == dt.date():
        return (False, None)

    return (True, dt.date())


This is usually the main required method to fill out when implementing a new challenge . Given an event type a list of existing user challenges and the base challenge type update the given user_challenges .

def update_user_challenges(
        self,
        session: Session,
        event: str,
        user_challenges: List[UserChallenge],
        step_count: Optional[int],
        event_metadatas: List[FullEventMetadata],
        starting_block: Optional[int],
    ):
        """This is usually the main required method to fill out when implementing a new challenge.
        Given an event type, a list of existing user challenges, and the base challenge type,
        update the given user_challenges.

        In the case of aggregate challenges, where UserChallenges are created in an
        already completed state, this method can be left as is.
        """

def update_user_challenges ( self , session : Session , event : str , user_challenges : List [ UserChallenge ] , step_count : Optional [ int ] , event_metadatas : List [ FullEventMetadata ] , starting_block : Optional [ int ] , ) :


Optional method to do some work after the ChallengeManager creates new challenges . If a challenge is backed by it s own table for instance create those rows here .

def on_after_challenge_creation(
        self, session: Session, metadatas: List[FullEventMetadata]
    ):
        """Optional method to do some work after the `ChallengeManager` creates new challenges.
        If a challenge is backed by it's own table, for instance, create those rows here.
        """

def on_after_challenge_creation ( self , session : Session , metadatas : List [ FullEventMetadata ] ) :


Optional method to provide a custom specifier for a challenge given a user_id

def generate_specifier(self, user_id: int, extra: Dict) -> str:
        """Optional method to provide a custom specifier for a challenge, given a user_id"""
        return str(user_id)

def generate_specifier ( self , user_id : int , extra : Dict ) -> str : return str ( user_id )


Optional method called for aggregate challenges to allow for overriding default behavior of creating a new UserChallenge whenever 1 ) we see a relevant event and 2 ) the parent challenge is not yet complete .

def should_create_new_challenge(
        self, session: Session, event: str, user_id: int, extra: Dict
    ) -> bool:
        """Optional method called for aggregate challenges to allow for overriding default
        behavior of creating a new UserChallenge whenever 1) we see a relevant event and
        2) the parent challenge is not yet complete.
        """
        return True

def should_create_new_challenge ( self , session : Session , event : str , user_id : int , extra : Dict ) -> bool : return True


Optional method to show / hide a challenge for a particular user .

def should_show_challenge_for_user(self, session: Session, user_id: int) -> bool:
        """Optional method to show/hide a challenge for a particular user."""
        return True

def should_show_challenge_for_user ( self , session : Session , user_id : int ) -> bool : return True


Optional method to provide any extra metadata required for client to properly display a challenge .

def get_metadata(self, session: Session, specifiers: List[str]) -> List[Dict]:
        """Optional method to provide any extra metadata required for client to properly display a challenge."""
        return [{} for s in specifiers]

def get_metadata ( self , session : Session , specifiers : List [ str ] ) -> List [ Dict ] : return [ { } for s in specifiers ]


Optional method to provide default metadata for an challenge with no progress .

def get_default_metadata(self) -> Dict:
        """Optional method to provide default metadata for an challenge with no progress."""
        return {}

def get_default_metadata ( self ) -> Dict : return { }


def get_override_challenge_step_count(
        self, session: Session, user_id: int
    ) -> Optional[int]:
        """Optional method to override the step count value"""
        return None

def get_override_challenge_step_count ( self , session : Session , user_id : int ) -> Optional [ int ] : return None


Processes a number of events for a particular event type updating UserChallengeEvents as needed .

def process(self, session, event_type: str, event_metadatas: List[EventMetadata]):
        """Processes a number of events for a particular event type, updating
        UserChallengeEvents as needed.
        """
        logger.info(
            f"ChallengeManager: processing event type [{event_type}] for challenge [{self.challenge_id}]"
        )
        if not self._did_init:  # lazy init
            self._init_challenge(session)

        # If inactive, do nothing
        if not self._is_active:
            return

        # filter out events that took place before the starting block, returning
        # early if need be
        if self._starting_block is not None:
            event_metadatas = list(
                filter(
                    lambda x: x["block_number"] >= cast(int, self._starting_block),
                    event_metadatas,
                )
            )
        if not event_metadatas:
            return

        # Add specifiers
        events_with_specifiers: List[FullEventMetadata] = [
            {
                "user_id": event["user_id"],
                "block_number": event["block_number"],
                "extra": event["extra"],
                "specifier": self._updater.generate_specifier(
                    event["user_id"], event["extra"]
                ),
            }
            for event in event_metadatas
        ]

        # Drop any duplicate specifiers
        events_with_specifiers_map = {
            event["specifier"]: event for event in events_with_specifiers
        }
        events_with_specifiers = list(events_with_specifiers_map.values())

        specifiers: List[str] = [e["specifier"] for e in events_with_specifiers]

        # Because we reuse a single session between multiple
        # challenge managers, we have to be extra careful in the case
        # that we run into a Postgres level error, to rollback
        # the session so it remains usable - hence, all the sensitive
        # code belongs in a `try` block here.
        try:
            # Gets all user challenges,
            existing_user_challenges = fetch_user_challenges(
                session, self.challenge_id, specifiers
            )

            # Create users that need challenges still
            existing_specifiers = {
                challenge.specifier for challenge in existing_user_challenges
            }

            # Create new challenges

            new_challenge_metadata = [
                metadata
                for metadata in events_with_specifiers
                if metadata["specifier"] not in existing_specifiers
            ]
            to_create_metadata: List[FullEventMetadata] = []
            if self._challenge_type == ChallengeType.aggregate:
                # For aggregate challenges, only create them
                # if we haven't maxed out completion yet, and
                # we haven't overriden this via should_create_new_challenge

                # Get *all* UserChallenges per user
                user_ids = list({e["user_id"] for e in event_metadatas})
                all_user_challenges: List[Tuple[int, int]] = (
                    session.query(
                        UserChallenge.user_id, func.count(UserChallenge.specifier)
                    )
                    .filter(
                        UserChallenge.challenge_id == self.challenge_id,
                        UserChallenge.user_id.in_(user_ids),
                    )
                    .group_by(UserChallenge.user_id)
                ).all()
                challenges_per_user = dict(all_user_challenges)
                new_user_challenges_specifiers: Dict[int, Set[str]] = defaultdict(set)
                for new_metadata in new_challenge_metadata:
                    user_id = new_metadata["user_id"]
                    completion_count = challenges_per_user.get(user_id, 0) + len(
                        new_user_challenges_specifiers[user_id]
                    )
                    if self._step_count and completion_count >= self._step_count:
                        continue
                    if not self._updater.should_create_new_challenge(
                        session,
                        event_type,
                        new_metadata["user_id"],
                        new_metadata["extra"],
                    ):
                        continue
                    new_user_challenges_specifiers[user_id].add(
                        new_metadata["specifier"]
                    )
                    to_create_metadata.append(new_metadata)
            else:
                to_create_metadata = new_challenge_metadata

            new_user_challenges = [
                self._create_new_user_challenge(
                    metadata["user_id"], metadata["specifier"]
                )
                for metadata in to_create_metadata
            ]
            logger.warning(f"new challenges ${new_user_challenges}")

            # Get the other challenges to update (the ones in progress)
            in_progress_challenges = [
                challenge
                for challenge in existing_user_challenges
                if not challenge.is_complete
            ]
            to_update = in_progress_challenges + new_user_challenges

            # Filter out challenges for deactivated users
            to_update_user_ids = list({c.user_id for c in to_update})
            deactivated_user_ids = (
                session.query(User.user_id)
                .filter(
                    User.user_id.in_(to_update_user_ids),
                    User.is_deactivated == True,
                )
                .all()
            )
            to_create_metadata = list(
                filter(
                    lambda c: c["user_id"] not in deactivated_user_ids,
                    to_create_metadata,
                )
            )
            to_update = list(
                filter(lambda c: c.user_id not in deactivated_user_ids, to_update)
            )

            # Do any other custom work needed after creating a challenge event
            self._updater.on_after_challenge_creation(session, to_create_metadata)

            # Update all the challenges
            self._updater.update_user_challenges(
                session,
                event_type,
                to_update,
                self._step_count,
                events_with_specifiers,
                self._starting_block,
            )

            # Add block # to newly completed challenges
            for challenge in to_update:
                if challenge.is_complete:
                    block_number = events_with_specifiers_map[challenge.specifier][
                        "block_number"
                    ]
                    challenge.completed_blocknumber = block_number

            logger.debug(
                f"ChallengeManager: Updated challenges from event [{event_type}]: [{to_update}]"
            )
            # Only add the new ones
            session.add_all(new_user_challenges)

            # Commit, so if there are DB errors
            # we encounter now and can roll back
            # to keep the session valid
            # for the next manager
            session.commit()
        except Exception as e:
            logger.warning(
                f"ChallengeManager: caught error in manager [{self.challenge_id}]: [{e}]. Rolling back"
            )
            session.rollback()


Gets additional metadata to render the challenge if needed .

def get_metadata(self, session: Session, specifiers: List[str]) -> List[Dict]:
        """Gets additional metadata to render the challenge if needed."""
        return self._updater.get_metadata(session, specifiers)

def get_metadata ( self , session : Session , specifiers : List [ str ] ) -> List [ Dict ] : return self . _updater . get_metadata ( session , specifiers )


Gets default metadata for an challenge with no progress .

def get_default_metadata(self):
        """Gets default metadata for an challenge with no progress."""
        return self._updater.get_default_metadata()

def get_default_metadata ( self ) : return self . _updater . get_default_metadata ( )


def should_show_challenge_for_user(self, session: Session, user_id: int) -> bool:
        """Optional method to show/hide a challenge for a particular user."""
        return self._updater.should_show_challenge_for_user(session, user_id)

def should_show_challenge_for_user ( self , session : Session , user_id : int ) -> bool : return self . _updater . should_show_challenge_for_user ( session , user_id )


Optional method to override a challenge step count for a particular user .

def get_override_challenge_step_count(
        self, session: Session, user_id: int
    ) -> Optional[int]:
        """Optional method to override a challenge step count for a particular user."""
        return self._updater.get_override_challenge_step_count(session, user_id)

def get_override_challenge_step_count ( self , session : Session , user_id : int ) -> Optional [ int ] : return self . _updater . get_override_challenge_step_count ( session , user_id )


def get_playlist(playlist_id, current_user_id):
    """Returns a single playlist, or None"""
    args = {
        "playlist_id": [playlist_id],
        "with_users": True,
        "current_user_id": current_user_id,
    }
    playlists = get_playlists(args)
    if playlists:
        return extend_playlist(playlists[0])
    return None

def get_playlist ( playlist_id , current_user_id ) : args = { "playlist_id" : [ playlist_id ] , "with_users" : True , "current_user_id" : current_user_id , } playlists = get_playlists ( args ) if playlists : return extend_playlist ( playlists [ 0 ] ) return None


Monkey patch for HTTPS or else swagger docs do not serve over HTTPS https : // stackoverflow . com / questions / 47508257 / serving - flask - restplus - on - https - server

def specs_url(self):
        """
        Monkey patch for HTTPS or else swagger docs do not serve over HTTPS
        https://stackoverflow.com/questions/47508257/serving-flask-restplus-on-https-server
        """
        scheme = "https" if "https" in self.base_url else "http"
        return url_for(self.endpoint("specs"), _external=True, _scheme=scheme)

def specs_url ( self ) : scheme = "https" if "https" in self . base_url else "http" return url_for ( self . endpoint ( "specs" ) , _external = True , _scheme = scheme )