def get_word_set_total_score(board, word_set, num_move_locations):
    total_score = 0
    word_score = 0

    for word_location_set in word_set:
        word_score = 0
        word_multiplier = 1

        for location in word_location_set:
            square = board.board_square_dict[location]
            word_multiplier *= square.word_multiplier
            word_score += square.tile.point_value * square.letter_multiplier

        word_score *= word_multiplier
        total_score += word_score

    if num_move_locations == config.PLAYER_RACK_SIZE:
        total_score += config.BINGO_SCORE

    return total_score

get_word_set_total_score(board=  abcdefghijklmno1 _______________2 _______________3 _______________4 _______________5 _______________6 _______________7 _______________8 _______BAKER___9 _______________10_______________11_______________12_______________13_______________14_______________15_______________, word_set={frozenset(), frozenset({('h', 8), ('j', 8), ('l', 8), ('i', 8), ('k', 8)})}, num_move_locations=5)


def rolling_median_by_h(x, h, w, name):
    """Compute a rolling median of x, after first aggregating by h.

    Right-aligned. Computes a single median for each unique value of h. Each
    median is over at least w samples.

    For each h where there are fewer than w samples, we take samples from the previous h,
    moving backwards. (In other words, we ~ assume that the x's are shuffled within each h.)

    Parameters
    ----------
    x: Array.
    h: Array of horizon for each value in x.
    w: Integer window size (number of elements).
    name: Name for metric in result dataframe

    Returns
    -------
    Dataframe with columns horizon and name, the rolling median of x.
    """
    # Aggregate over h
    df = pd.DataFrame({'x': x, 'h': h})
    grouped = df.groupby('h')
    df2 = grouped.size().reset_index().sort_values('h')
    hs = df2['h']

    res_h = []
    res_x = []
    # Start from the right and work backwards
    i = len(hs) - 1
    while i >= 0:
        h_i = hs[i]
        xs = grouped.get_group(h_i).x.tolist()

        # wrap in array so this works if h is pandas Series with custom index or numpy array
        next_idx_to_add = np.array(h == h_i).argmax() - 1
        while (len(xs) < w) and (next_idx_to_add >= 0):
            # Include points from the previous horizon. All of them if still
            # less than w, otherwise just enough to get to w.
            xs.append(x[next_idx_to_add])
            next_idx_to_add -= 1
        if len(xs) < w:
            # Ran out of points before getting enough.
            break
        res_h.append(hs[i])
        res_x.append(np.median(xs))
        i -= 1
    res_h.reverse()
    res_x.reverse()
    return pd.DataFrame({'horizon': res_h, name: res_x})

rolling_median_by_h(x=array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), h=array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), w=1, name='x')

<pandas.core.groupby.generic.DataFrameGroupBy object at 0x7ff75ca12880>


def create_request_parameters(parent, request_model, params=None, index=None):
    """
    Handle request parameters that can be filled in from identifiers,
    resource data members or constants.

    By passing ``params``, you can invoke this method multiple times and
    build up a parameter dict over time, which is particularly useful
    for reverse JMESPath expressions that append to lists.

    :type parent: ServiceResource
    :param parent: The resource instance to which this action is attached.
    :type request_model: :py:class:`~boto3.resources.model.Request`
    :param request_model: The action request model.
    :type params: dict
    :param params: If set, then add to this existing dict. It is both
                   edited in-place and returned.
    :type index: int
    :param index: The position of an item within a list
    :rtype: dict
    :return: Pre-filled parameters to be sent to the request operation.
    """
    if params is None:
        params = {}

    for param in request_model.params:
        source = param.source
        target = param.target

        if source == 'identifier':
            # Resource identifier, e.g. queue.url
            value = getattr(parent, xform_name(param.name))
        elif source == 'data':
            # If this is a data member then it may incur a load
            # action before returning the value.
            value = get_data_member(parent, param.path)
        elif source in ['string', 'integer', 'boolean']:
            # These are hard-coded values in the definition
            value = param.value
        elif source == 'input':
            # This is provided by the user, so ignore it here
            continue
        else:
            raise NotImplementedError(f'Unsupported source type: {source}')

        build_param_structure(params, target, value, index)

    return params

create_request_parameters(parent=dynamodb.Table(name='MyTable'), request_model={_definition=OrderedDict([('operation', 'Scan'), ('params', [OrderedDict([('target', 'TableName'), ('source', 'identifier'), ('name', 'Name')])])]), operation='Scan'}, params=None, index=None)

{target='TableName', source='identifier', name='Name', path=None, value=None}


def get_mylogin_cnf_path():
    """Return the path to the login path file or None if it doesn't exist."""
    mylogin_cnf_path = os.getenv('MYSQL_TEST_LOGIN_FILE')

    if mylogin_cnf_path is None:
        app_data = os.getenv('APPDATA')
        default_dir = os.path.join(app_data, 'MySQL') if app_data else '~'
        mylogin_cnf_path = os.path.join(default_dir, '.mylogin.cnf')

    mylogin_cnf_path = os.path.expanduser(mylogin_cnf_path)

    if exists(mylogin_cnf_path):
        logger.debug("Found login path file at '{0}'".format(mylogin_cnf_path))
        return mylogin_cnf_path
    return None

get_mylogin_cnf_path()


def _check(self) -> None:
        """Check the syntax of the expression.

        :raises ValueError:
            If the expression is incomplete.  This error is aimed at debugging
            the expression so it is very verbose.
        """
        stack_size = 0
        for i, token_ in enumerate(self._tokens):
            stack_size -= token_.pops
            if stack_size < 0:
                raise ValueError(
                    self._format_syntax_error(
                        f"'{token_}' takes {token_.pops} argument(s) but the stack"
                        f" will only have {stack_size + token_.pops} element(s)",
                        i,
                    )
                )
            stack_size += token_.puts
        if stack_size == 0:
            raise ValueError(
                self._format_syntax_error("expression does not produce a result")
            )
        if stack_size > 1:
            raise ValueError(
                self._format_syntax_error(
                    f"expression produces too many results ({stack_size}), "
                    "expected 1"
                )
            )

_check(self=CompleteExpression([Literal(1)]), self[0]=Literal(1))


def contains_sublist(list_: List[Any], sublist: List[Any]) -> bool:
    """Determine if a `list` contains a `sublist`.

    :param list_:
        list to search for the `sublist` in.
    :param sublist:
        Sub list to search for.

    :return:
        True if `list` contains `sublist`.

    """
    # Adapted from: https://stackoverflow.com/a/12576755
    if not sublist:
        return False
    for i in range(len(list_)):
        if list_[i] == sublist[0] and list_[i : i + len(sublist)] == sublist:
            return True
    return False

contains_sublist(list_=[1, 2, 3, 4], sublist=[1, 2])


def delete_sublist(list_: List[Any], sublist: List[Any]) -> List[Any]:
    """Remove a `sublist` from the given `list_`.

    :param list_:
        List to remove the `sublist` from.
    :param sublist:
        Sublist to remove from `list_`.

    :return:
        A copy of `list_` with the `sublist` removed.
    """
    if not sublist:
        return list_[:]
    for i in range(len(list_)):
        if list_[i] == sublist[0] and list_[i : i + len(sublist)] == sublist:
            return list_[:i] + list_[i + len(sublist) :]
    return list_[:]

delete_sublist(list_=[1, 2, 3, 4], sublist=[1, 2])


def bounded_product(sequence, n=None, seed=None):
    """
    Returns a shuffled, bounded cartesian product of the input sequence.
    Designed to cover as wide a range of permutations as possible with a limited number of iterations.
    Will manifest the whole list in memory, so not suitable for super large sequences.

    :param sequence: iterable
    :param n: length of returned list
    :param seed: random seed for reproducibility
    :return: list
    """
    p = list(itertools.product(*sequence))
    if seed is not None:
        random.seed(seed)
    random.shuffle(p)
    return p if n is None else p[:n]

bounded_product(sequence=([[0, 1, 1], [1, 2, 2], [0, 2, 2]], [True, False], [[[['global'], 'all']], [[['local'], 'all']], [[['sparse_variable'], 'all']], [[['sparse_fixed'], 'all']]], [[True, False], [False, True]], [[{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]]), n=50, seed=None)

[([0, 1, 1], True, [[['global'], 'all']], [True, False], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 1, 1], True, [[['global'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 1, 1], True, [[['global'], 'all']], [False, True], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 1, 1], True, [[['global'], 'all']], [False, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 1, 1], True, [[['local'], 'all']], [True, False], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 1, 1], True, [[['local'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 1, 1], True, [[['local'], 'all']], [False, True], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 1, 1], True, [[['local'], 'all']], [False, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 1, 1], True, [[['sparse_variable'], 'all']], [True, False], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 1, 1], True, [[['sparse_variable'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 1, 1], True, [[['sparse_variable'], 'all']], [False, True], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 1, 1], True, [[['sparse_variable'], 'all']], [False, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 1, 1], True, [[['sparse_fixed'], 'all']], [True, False], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 1, 1], True, [[['sparse_fixed'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 1, 1], True, [[['sparse_fixed'], 'all']], [False, True], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 1, 1], True, [[['sparse_fixed'], 'all']], [False, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 1, 1], False, [[['global'], 'all']], [True, False], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 1, 1], False, [[['global'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 1, 1], False, [[['global'], 'all']], [False, True], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 1, 1], False, [[['global'], 'all']], [False, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 1, 1], False, [[['local'], 'all']], [True, False], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 1, 1], False, [[['local'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 1, 1], False, [[['local'], 'all']], [False, True], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis... True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 2, 2], True, [[['sparse_variable'], 'all']], [False, True], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 2, 2], True, [[['sparse_variable'], 'all']], [False, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 2, 2], True, [[['sparse_fixed'], 'all']], [True, False], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 2, 2], True, [[['sparse_fixed'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 2, 2], True, [[['sparse_fixed'], 'all']], [False, True], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 2, 2], True, [[['sparse_fixed'], 'all']], [False, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 2, 2], False, [[['global'], 'all']], [True, False], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 2, 2], False, [[['global'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 2, 2], False, [[['global'], 'all']], [False, True], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 2, 2], False, [[['global'], 'all']], [False, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 2, 2], False, [[['local'], 'all']], [True, False], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 2, 2], False, [[['local'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 2, 2], False, [[['local'], 'all']], [False, True], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 2, 2], False, [[['local'], 'all']], [False, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 2, 2], False, [[['sparse_variable'], 'all']], [True, False], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 2, 2], False, [[['sparse_variable'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 2, 2], False, [[['sparse_variable'], 'all']], [False, True], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 2, 2], False, [[['sparse_variable'], 'all']], [False, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 2, 2], False, [[['sparse_fixed'], 'all']], [True, False], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 2, 2], False, [[['sparse_fixed'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 2, 2], False, [[['sparse_fixed'], 'all']], [False, True], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 2, 2], False, [[['sparse_fixed'], 'all']], [False, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False])]

[([0, 1, 1], False, [[['sparse_variable'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 1, 1], False, [[['local'], 'all']], [False, True], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([1, 2, 2], True, [[['sparse_fixed'], 'all']], [True, False], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([1, 2, 2], True, [[['sparse_variable'], 'all']], [False, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([1, 2, 2], True, [[['local'], 'all']], [True, False], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 2, 2], True, [[['sparse_fixed'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 2, 2], True, [[['sparse_variable'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 2, 2], True, [[['local'], 'all']], [False, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([1, 2, 2], True, [[['global'], 'all']], [True, False], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([1, 2, 2], False, [[['global'], 'all']], [True, False], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 1, 1], True, [[['global'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 2, 2], True, [[['global'], 'all']], [True, False], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 1, 1], True, [[['global'], 'all']], [False, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 1, 1], False, [[['sparse_fixed'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 1, 1], True, [[['sparse_fixed'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 1, 1], False, [[['sparse_variable'], 'all']], [False, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([1, 2, 2], False, [[['sparse_fixed'], 'all']], [False, True], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 2, 2], False, [[['sparse_fixed'], 'all']], [False, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([1, 2, 2], True, [[['local'], 'all']], [False, True], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 1, 1], True, [[['local'], 'all']], [True, False], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 1, 1], True, [[['sparse_variable'], 'all']], [True, False], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 1, 1], True, [[['sparse_variable'], 'all']], [False, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([1, 2, 2], False, [[['local'], 'all']], [False, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, '...abled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([1, 2, 2], False, [[['local'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 2, 2], False, [[['local'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([1, 2, 2], True, [[['sparse_variable'], 'all']], [False, True], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([1, 2, 2], True, [[['sparse_fixed'], 'all']], [False, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 1, 1], True, [[['local'], 'all']], [False, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([1, 2, 2], False, [[['sparse_fixed'], 'all']], [True, False], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([1, 2, 2], True, [[['local'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([1, 2, 2], True, [[['sparse_fixed'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 2, 2], False, [[['global'], 'all']], [False, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 2, 2], False, [[['sparse_fixed'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 2, 2], True, [[['sparse_fixed'], 'all']], [False, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 2, 2], False, [[['sparse_variable'], 'all']], [True, False], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 2, 2], True, [[['sparse_variable'], 'all']], [False, True], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 2, 2], False, [[['global'], 'all']], [False, True], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 1, 1], False, [[['global'], 'all']], [True, False], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([1, 2, 2], False, [[['sparse_fixed'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 2, 2], False, [[['sparse_variable'], 'all']], [False, True], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 2, 2], False, [[['sparse_fixed'], 'all']], [True, False], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 2, 2], False, [[['sparse_fixed'], 'all']], [False, True], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True]), ([0, 1, 1], True, [[['sparse_variable'], 'all']], [True, False], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 1, 1], True, [[['sparse_fixed'], 'all']], [False, True], [{'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, False]), ([0, 2, 2], False, [[['local'], 'all']], [True, False], [{'enabled': True, 'type': 'bfloat16', 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, True])]


def set_up_autotuning(encoded_config, overwrite_values):
        config = json.loads(base64.urlsafe_b64decode(encoded_config).decode("utf-8"))
        overwrite_values = overwrite_values if overwrite_values else {}
        for tuning_param in AUTOTUNING_ARGS:
            # TODO: This is for autotuning specifically, may cause surprises for someone with a weird setup
            if tuning_param in config:
                overwrite_values[tuning_param] = config[tuning_param]
        return overwrite_values

set_up_autotuning(encoded_config='eyJ0cmFpbl9iYXRjaF9zaXplIjogNCwgInRyYWluX21pY3JvX2JhdGNoX3NpemVfcGVyX2dwdSI6IDQsICJvcHRpbWl6ZXIiOiB7InR5cGUiOiAic20zIiwgInBhcmFtcyI6IHt9fSwgImZwMTYiOiB7InR5cGUiOiAiZnAxNiIsICJlbmFibGVkIjogdHJ1ZX0sICJ6ZXJvX29wdGltaXphdGlvbiI6IHsic3RhZ2UiOiAwLCAiYWxsZ2F0aGVyX3BhcnRpdGlvbnMiOiB0cnVlLCAicmVkdWNlX3NjYXR0ZXIiOiB0cnVlLCAiYWxsZ2F0aGVyX2J1Y2tldF9zaXplIjogNTAwMDAwMDAwLCAib3ZlcmxhcF9jb21tIjogZmFsc2UsICJyZWR1Y2VfYnVja2V0X3NpemUiOiA1MDAwMDAwMDAsICJjb250aWd1b3VzX2dyYWRpZW50cyI6IGZhbHNlfSwgIndhbGxfY2xvY2tfYnJlYWtkb3duIjogdHJ1ZSwgImNvbW1zX2xvZ2dlciI6IHsiZW5hYmxlZCI6IHRydWUsICJ2ZXJib3NlIjogdHJ1ZSwgInByb2ZfYWxsIjogdHJ1ZSwgImRlYnVnIjogZmFsc2V9fQ==', overwrite_values={'train_iters': 32})

{'train_batch_size': 4, 'train_micro_batch_size_per_gpu': 4, 'optimizer': {'type': 'sm3', 'params': {}}, 'fp16': {'type': 'fp16', 'enabled': True}, 'zero_optimization': {'stage': 0, 'allgather_partitions': True, 'reduce_scatter': True, 'allgather_bucket_size': 500000000, 'overlap_comm': False, 'reduce_bucket_size': 500000000, 'contiguous_gradients': False}, 'wall_clock_breakdown': True, 'comms_logger': {'enabled': True, 'verbose': True, 'prof_all': True, 'debug': False}}

{'train_iters': 32, 'train_batch_size': 4}

{'train_iters': 32, 'train_batch_size': 4, 'train_micro_batch_size_per_gpu': 4}

{'train_iters': 32, 'train_batch_size': 4, 'train_micro_batch_size_per_gpu': 4, 'zero_optimization': {'stage': 0, 'allgather_partitions': True, 'reduce_scatter': True, 'allgather_bucket_size': 500000000, 'overlap_comm': False, 'reduce_bucket_size': 500000000, 'contiguous_gradients': False}}


def test_model_training_options(monkeypatch, key, value):
    # TODO: Possibly add testing over world_size=2 back in
    neox_args = NeoXArgs.from_dict(BASE_CONFIG)
    if getattr(neox_args, key) == value:
        pytest.skip("Skipping to avoid redundancy as no change in base config")
    if key == "precision" and value == "bfloat16":
        pytest.xfail(
            reason="Assumes that ZeRO optimization stage has been set in the YAML"
        )
    param_dict = {key: value}
    run_train_test(monkeypatch, overwrite_values=param_dict)

test_model_training_options(monkeypatch={_setattr=[], _setitem=[], _cwd=None, _savesyspath=None}, key='gpt_j_residual', value=False)

NeoXArgs(distributed_backend='nccl', local_rank=None, rank=None, lazy_mpu_init=False, short_seq_prob=0.1, eod_mask_loss=False, adlr_autoresume=False, adlr_autoresume_interval=1000, seed=1234, onnx_safe=False, deepscale=False, deepscale_config=None, deepspeed_mpi=False, deepspeed_slurm=False, user_script=None, iteration=None, do_train=None, do_valid=None, do_test=None, save_iters=[], global_num_gpus=1, text_gen_type='unconditional', temperature=0.0, top_p=0.0, top_k=0, return_logits=False, maximum_tokens=64, prompt_end='\n', sample_input_file=None, sample_output_file='samples.txt', num_samples=1, recompute=False, eval_results_prefix='', eval_tasks=None, use_wandb=False, wandb_group=None, wandb_team=None, wandb_project='neox', wandb_host='https://api.wandb.ai', wandb_init_all_ranks=False, git_hash='7a8fa2f0', log_dir=None, tensorboard_dir=None, log_interval=10, log_grad_pct_zeros=False, log_param_norm=False, log_grad_norm=False, log_optimizer_states=False, log_gradient_noise_scale=False, gradient_noise_scale_n_batches=5, gradient_noise_scale_cpu_offload=False, pipe_parallel_size=1, model_parallel_size=1, pipe_partition_method='type:transformer|mlp', world_size=None, is_pipe_parallel=True, data_path='data/enwik8/enwik8_text_document', use_shared_fs=True, train_data_paths=None, label_data_paths=None, test_data_paths=None, valid_data_paths=None, train_data_weights=None, valid_data_weights=None, test_data_weights=None, weight_by_num_documents=False, weighted_sampler_alpha=1.0, data_impl='mmap', mmap_warmup=False, save=None, s3_path=None, s3_chunk_size=104857600, config_files=None, load=None, checkpoint_validation_with_forward_pass=False, checkpoint_scale='linear', checkpoint_factor=1000, extra_save_iters=None, no_save_optim=False, no_save_rng=False, no_load_optim=False, no_load_rng=False, finetune=False, batch_size=4, train_iters=20, eval_iters=10, keep_last_n_checkpoints=None, eval_interval=100000, split='969, 30, 1', vocab_file='data/gpt2-vocab.json', merge_file='data/gpt2-merges.txt', num_workers=1, exit_interval=None, attention_dropout=0, hidden_dropout=0, weight_decay=0.1, checkpoint_activations=True, checkpoint_num_layers=1, deepspeed_activation_checkpointing=True, contiguous_checkpointing=False, checkpoint_in_cpu=False, synchronize_each_layer=True, profile_backward=False, partition_activations=True, gas=1, clip_grad=1.0, hysteresis=2, dynamic_loss_scale=True, loss_scale=None, loss_scale_window=1000.0, min_scale=1.0, char_level_ppl=False, use_mup=False, coord_check=False, save_base_shapes=False, base_shapes_file=None, mup_init_scale=1.0, mup_attn_temp=1.0, mup_output_temp=1.0, mup_embedding_mult=1.0, mup_rp_embedding_mult=1.0, mup_width_scale=2, tokenizer_type='GPT2BPETokenizer', padded_vocab_size=None, optimizer_type='sm3', use_bnb_optimizer=False, zero_stage=0, zero_reduce_scatter=True, zero_contiguous_gradients=False, zero_reduce_bucket_size=500000000, zero_allgather_bucket_size=500000000, lr=0.001, lr_decay_style='cosine', lr_decay_iters=20, min_lr=0.0, warmup=0.01, override_lr_scheduler=False, use_checkpoint_lr_scheduler=False, precision='fp16', num_layers=2, hidden_size=8, num_attention_heads=4, seq_length=1024, max_position_embeddings=1024, norm='layernorm', use_qk_layernorm=False, layernorm_epsilon=1e-05, rms_norm_epsilon=1e-08, scalenorm_epsilon=1e-08, pos_emb='rotary', rpe_num_buckets=32, rpe_max_distance=128, opt_pos_emb_offset=0, no_weight_tying=True, attention_config=['global', 'global'], sparsity_config={}, num_unique_layers=None, param_sharing_style='grouped', make_vocab_size_divisible_by=128, activation='gelu', scaled_upper_triang_masked_softmax_fusion=False, scaled_masked_softmax_fusion=False, bias_gelu_fusion=False, bias_dropout_fusion=False, rope_fusion=False, fp16_lm_cross_entropy=False, init_method_std=0.02, apply_query_key_layer_scaling=False, use_cpu_initialization=False, attention_softmax_in_fp32=False, rotary_pct=1.0, rotary_emb_base=10000, init_method='small_init', output_layer_init_method='wang_init', gmlp_attn_dim=64, gpt_j_residual=False, gpt_j_tied=False, use_bias_in_norms=True, use_bias_in_attn_linear=True, mlp_type='regular', soft_prompt_tuning=None, output_layer_parallelism='column', deepspeed=True, train_batch_size=4, train_micro_batch_size_per_gpu=4, gradient_accumulation_steps=1, optimizer={'type': 'sm3', 'params': {}}, scheduler=None, fp32_allreduce=False, prescale_gradients=False, gradient_predivide_factor=1.0, sparse_gradients=False, fp16={'type': 'fp16', 'enabled': True}, bf16=None, amp=None, gradient_clipping=1.0, zero_optimization={'stage': 0, 'allgather_partitions': True, 'reduce_scatter': True, 'allgather_bucket_size': 500000000, 'overlap_comm': False, 'reduce_bucket_size': 500000000, 'contiguous_gradients': False}, curriculum_learning=None, curriculum_seqlen=0, steps_per_print=10, wall_clock_breakdown=True, dump_state=False, flops_profiler=None, communication_data_type=None, autotuning=None, activation_checkpointing=None, sparse_attention=None, data_efficiency=None, tensorboard=None, wandb=None, csv_monitor=None, elasticity=None, comms_logger=None, compression_training=None, checkpoint=None, data_types=None, deepspeed_extra_args={'comms_logger': {'enabled': True, 'verbose': True, 'prof_all': True, 'debug': False}}, hostfile='None', include='localhost:1', exclude=None, num_nodes=-1, num_gpus=None, master_port=29500, master_addr=None, launcher='pdsh', force_multi=False, detect_nvlink_pairs=False, autotuning_run=None, no_ssh_check=False, comment=None, account=None)


def _get_nested(data, key):
    """
    Return value for a hierrachical key (like a.b.c).
    Return None if nothing found.
    If there is a key with . in the name, and a subdictionary,
    the former is preferred:

    >>> print(_get_nested({'a.b': 10, 'a':{'b': 20}}, 'a.b'))
    10
    >>> print(_get_nested({'a': {'b': 20}}, 'a.b'))
    20
    >>> print(_get_nested({'a': {'b': {'c': 30}}}, 'a.b.c'))
    30
    """

    if not data or not isinstance(data, dict):
        return None
    if '.' not in key:
        return data.get(key)
    if key in data:
        return data[key]

    parts = key.split('.')
    for i in range(len(parts))[::-1]:
        prefix = ".".join(parts[:i])
        if prefix in data:
            return _get_nested(data[prefix], ".".join(parts[i:]))

    return None

_get_nested(data={'server': {'address': '0.0.0.0'}, 'cache': {'type': 'redis'}}, key='adapters.active')


def sample(logits: torch.Tensor, temperature: float = 1.0, top_k: Optional[int] = None) -> torch.Tensor:
    logits = logits[0, -1]
    # optionally crop the logits to only the top k options
    if top_k is not None:
        v, i = torch.topk(logits, min(top_k, logits.size(-1)))
        # do not use `torch.where` as in nanogpt because it will repeat top-k collisions
        logits = torch.full_like(logits, float("-inf")).scatter_(-1, i, v)
    # optionally scale the logits and sample from a probability distribution
    if temperature > 0.0:
        probs = torch.nn.functional.softmax(logits / temperature, dim=-1)
        return multinomial_num_samples_1(probs)
    return torch.argmax(logits, dim=-1, keepdim=True)

sample(logits=tensor([[[ 0.2595,  0.0058,  0.6498, -0.5155,  0.0366, -0.2944,  0.0555,          -0.4013],         [ 0.5342, -0.9790, -0.0861,  0.2530,  0.1378, -0.4913,  0.1915,          -0.3248],         [ 0.5289, -0.3149,  0.4052,  0.2107, -0.8788, -0.9871,  0.6539,          -0.2213]]], device='cuda:0'), temperature=0.8, top_k=None)


def test_generate(monkeypatch, generated, stop_tokens, expected):
    import chat.base as chat
    import generate.base as generate

    input_idx = torch.tensor([5, 3])
    max_returned_tokens = len(input_idx) + 8
    model = MagicMock()
    model.config.block_size = 100
    model.max_seq_length = 100
    it = iter(generated)

    def multinomial(*_, **__):
        out = next(it)
        return torch.tensor([out])

    monkeypatch.setattr(generate, "multinomial_num_samples_1", multinomial)
    actual = chat.generate(model, input_idx, max_returned_tokens, stop_tokens=stop_tokens)
    actual = list(actual)

    assert len(actual) == len(expected)
    if not actual:
        assert actual == expected
    else:
        for t in actual:
            assert t.dtype == torch.long
        assert torch.cat(actual).tolist() == expected

test_generate(monkeypatch={_setattr=[], _setitem=[], _cwd=None, _savesyspath=None}, generated=repeat(1), stop_tokens=(), expected=[1, 1, 1, 1, 1, 1, 1, 1])

{_setattr=[(<module 'generate.base' from '/local/rcs/XXX/code/pytrace-collector/logs/self_collected/tried/Lightning-AI+lit-gpt/Lightning-AI+lit-gpt/generate/base.py'>, 'multinomial_num_samples_1', <function multinomial_num_samples_1 at 0x7f805238a160>)], _setitem=[], _cwd=None, _savesyspath=None}

<function test_generate.<locals>.multinomial at 0x7f7fd0f7e430>

<generator object generate at 0x7f803c55ff90>


def render_pep440_feature(pieces):
    """Build up version string, used within "feature" branch of repository.

    Our goal: MERGE-POINT.post.devN+gHEX.BRANCH-NAME.M[.dirty]
        +) MERGE-POINT = Most recent common ancestor for `develop` and `master`
          *) Does not yet handle branch from `release-*`
        +) N = DISTANCE from the MERGE-POINT of `develop` and `master`
        +) M = DISTANCE from the MERGE-POINT of "feature" and `develop`

    Exceptions:
    1: no tags. 0.post.devDISTANCE+gHEX[.dirty]
    """
    if pieces["closest-tag"] and pieces["develop"]:
        rendered = pieces["closest-tag"]
        distance_to_develop = pieces["distance-to-develop"]
        distance_to_merge = pieces["distance-to-master"]
        distance_merge_to_tag = (pieces["distance"] - distance_to_merge)
        distance_dev_to_merge = (distance_to_merge - distance_to_develop)
        if (distance_merge_to_tag > 0):
            rendered += ".%d" % distance_merge_to_tag
        rendered += ".post.dev%d" % distance_dev_to_merge
        rendered += plus_or_dot(pieces)
        rendered += "g%s" % pieces["short"]
        rendered += ".%s" % pieces["branch"]
        rendered += ".%d" % distance_to_develop
    else:
        # exception #1
        rendered = "0.post.dev%d" % (pieces["distance"] - 1)
        rendered += plus_or_dot(pieces)
        rendered += "g%s" % pieces["short"]
    if pieces["dirty"]:
        rendered += ".dirty"
    return rendered

render_pep440_feature(pieces={'long': 'e60d005d389cb31b6e99f937e35adbe3fccb7aaf', 'short': 'e60d005', 'error': None, 'dirty': False, 'closest-tag': '0.8', 'distance': 3, 'branch': 'HEAD', 'distance-to-master': 0, 'develop': None, 'distance-to-develop': None, 'date': '2018-05-05T22:17:58-0700', 'authors': ['Padraic Shafer']})


def encode_number(n):
    b128_digits = []
    while n:
        b128_digits.insert(0, (n & 0x7f) | 0x80)
        n = n >> 7
    if not b128_digits:
        b128_digits.append(0)
    b128_digits[-1] &= 0x7f
    return b''.join([int.to_bytes(d, 1, 'big') for d in b128_digits])

encode_number(n=840)


def _import_plugins(plugin_names):
    plugins = []
    for name, path in plugin_names.items():
        log.debug("Importing plugin %s: %s", name, path)
        if '.' not in path:
            plugins.append(__import__(path))
        else:
            package, module = path.rsplit('.', 1)
            module = __import__(path, fromlist=[module])
            plugins.append(module)
    return plugins

_import_plugins(plugin_names={'__builtin__': 'awscli.handlers'})


def _make_stage(self, planes: int, num_blocks: int, num_se_blocks: int) -> nn.Sequential:
        """Build a stage of MobileOne model.

        :param planes: Number of output channels.
        :param num_blocks: Number of blocks in this stage.
        :param num_se_blocks: Number of SE blocks in this stage.
        :return: A stage of MobileOne model.
        """
        # Get strides for all layers
        strides = [2] + [1] * (num_blocks - 1)
        blocks = []
        for ix, stride in enumerate(strides):
            use_se = False
            if num_se_blocks > num_blocks:
                raise ValueError("Number of SE blocks cannot " "exceed number of layers.")
            if ix >= (num_blocks - num_se_blocks):
                use_se = True

            # Depthwise conv
            blocks.append(
                MobileOneBlock(
                    in_channels=self.in_planes,
                    out_channels=self.in_planes,
                    kernel_size=3,
                    stride=stride,
                    padding=1,
                    groups=self.in_planes,
                    inference_mode=self.inference_mode,
                    use_se=use_se,
                    num_conv_branches=self.num_conv_branches,
                )
            )
            # Pointwise conv
            blocks.append(
                MobileOneBlock(
                    in_channels=self.in_planes,
                    out_channels=planes,
                    kernel_size=1,
                    stride=1,
                    padding=0,
                    groups=1,
                    inference_mode=self.inference_mode,
                    use_se=use_se,
                    num_conv_branches=self.num_conv_branches,
                )
            )
            self.in_planes = planes
            self.cur_layer_idx += 1
        return nn.Sequential(*blocks)

_make_stage(self=MobileOne(  (stage0): MobileOneBlock(    (se): Identity()    (activation): ReLU()    (rbr_conv): ModuleList(      (0): Sequential(        (conv): Conv2d(3, 48, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)        (bn): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)      )    )    (rbr_scale): Sequential(      (conv): Conv2d(3, 48, kernel_size=(1, 1), stride=(2, 2), bias=False)      (bn): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)    )  )), planes=48, num_blocks=2, num_se_blocks=0, self._backward_hooks=OrderedDict(), self._backward_pre_hooks=OrderedDict(), self._buffers=OrderedDict(), self._depth=3, self._forward_hooks=OrderedDict(), self._forward_hooks_always_called=OrderedDict(), self._forward_hooks_with_kwargs=OrderedDict(), self._forward_pre_hooks=OrderedDict(), self._forward_pre_hooks_with_kwargs=OrderedDict(), self._in_channels=3, self._is_full_backward_hook=None, self._load_state_dict_post_hooks=OrderedDict(), self._load_state_dict_pre_hooks=OrderedDict(), self._modules=OrderedDict([('stage0', MobileOneBlock(  (se): Identity()  (activation): ReLU()  (rbr_conv): ModuleList(    (0): Sequential(      (conv): Conv2d(3, 48, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)      (bn): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)    )  )  (rbr_scale): Sequential(    (conv): Conv2d(3, 48, kernel_size=(1, 1), stride=(2, 2), bias=False)    (bn): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)  )))]), self._non_persistent_buffers_set=set(), self._out_channels=(3, 48, 48, 128, 256, 1024), self._parameters=OrderedDict(), self._state_dict_hooks=OrderedDict(), self._state_dict_pre_hooks=OrderedDict(), self.cur_layer_idx=1, self.in_planes=48, self.inference_mode=False, self.num_conv_branches=4, self.training=True, self.use_se=False)

[MobileOneBlock(  (se): Identity()  (activation): ReLU()  (rbr_conv): ModuleList(    (0-3): 4 x Sequential(      (conv): Conv2d(48, 48, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=48, bias=False)      (bn): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)    )  )  (rbr_scale): Sequential(    (conv): Conv2d(48, 48, kernel_size=(1, 1), stride=(2, 2), groups=48, bias=False)    (bn): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)  ))]

[MobileOneBlock(  (se): Identity()  (activation): ReLU()  (rbr_conv): ModuleList(    (0-3): 4 x Sequential(      (conv): Conv2d(48, 48, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=48, bias=False)      (bn): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)    )  )  (rbr_scale): Sequential(    (conv): Conv2d(48, 48, kernel_size=(1, 1), stride=(2, 2), groups=48, bias=False)    (bn): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)  )), MobileOneBlock(  (se): Identity()  (activation): ReLU()  (rbr_skip): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)  (rbr_conv): ModuleList(    (0-3): 4 x Sequential(      (conv): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1), bias=False)      (bn): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)    )  ))]

[MobileOneBlock(  (se): Identity()  (activation): ReLU()  (rbr_conv): ModuleList(    (0-3): 4 x Sequential(      (conv): Conv2d(48, 48, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=48, bias=False)      (bn): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)    )  )  (rbr_scale): Sequential(    (conv): Conv2d(48, 48, kernel_size=(1, 1), stride=(2, 2), groups=48, bias=False)    (bn): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)  )), MobileOneBlock(  (se): Identity()  (activation): ReLU()  (rbr_skip): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)  (rbr_conv): ModuleList(    (0-3): 4 x Sequential(      (conv): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1), bias=False)      (bn): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)    )  )), MobileOneBlock(  (se): Identity()  (activation): ReLU()  (rbr_skip): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)  (rbr_conv): ModuleList(    (0-3): 4 x Sequential(      (conv): Conv2d(48, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=48, bias=False)      (bn): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)    )  )  (rbr_scale): Sequential(    (conv): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1), groups=48, bias=False)    (bn): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)  ))]

[MobileOneBlock(  (se): Identity()  (activation): ReLU()  (rbr_conv): ModuleList(    (0-3): 4 x Sequential(      (conv): Conv2d(48, 48, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=48, bias=False)      (bn): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)    )  )  (rbr_scale): Sequential(    (conv): Conv2d(48, 48, kernel_size=(1, 1), stride=(2, 2), groups=48, bias=False)    (bn): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)  )), MobileOneBlock(  (se): Identity()  (activation): ReLU()  (rbr_skip): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)  (rbr_conv): ModuleList(    (0-3): 4 x Sequential(      (conv): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1), bias=False)      (bn): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)    )  )), MobileOneBlock(  (se): Identity()  (activation): ReLU()  (rbr_skip): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)  (rbr_conv): ModuleList(    (0-3): 4 x Sequential(      (conv): Conv2d(48, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=48, bias=False)      (bn): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)    )  )  (rbr_scale): Sequential(    (conv): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1), groups=48, bias=False)    (bn): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)  )), MobileOneBlock(  (se): Identity()  (activation): ReLU()  (rbr_skip): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)  (rbr_conv): ModuleList(    (0-3): 4 x Sequential(      (conv): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1), bias=False)      (bn): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)    )  ))]


def __setitem__(self, key, value):
        # type: (_K, _V) -> None
        """Store a new views, potentially discarding an old value.
        """
        if key not in self:
            if len(self) >= self.cache_size:
                self.popitem(last=False)
        OrderedDict.__setitem__(self, key, value)

__setitem__(self=LRUCache(), key='foo', value=1)


def config(_config=None, **kwargs):
    """
    A decorator for setting the default kwargs of `BaseHandler.crawl`.
    Any self.crawl with this callback will use this config.
    """
    if _config is None:
        _config = {}
    _config.update(kwargs)

    def wrapper(func):
        func._config = _config
        return func
    return wrapper

config(_config=None, kwargs={'age': 864000})

<function config.<locals>.wrapper at 0x7f6a10388ee0>


def base64url_decode(input: Union[bytes, str]) -> bytes:
    input_bytes = force_bytes(input)

    rem = len(input_bytes) % 4

    if rem > 0:
        input_bytes += b"=" * (4 - rem)

    return base64.urlsafe_b64decode(input_bytes)

base64url_decode(input='hJtXIZ2uSN5kbQfbtTNWbpdmhkV8FJG-Onbc6mxCcYg')


def to_dict(self, default=None):
        """
        Converts sequence of (Key, Value) pairs to a dictionary.

        >>> type(seq([('a', 1)]).to_dict())
        dict

        >>> seq([('a', 1), ('b', 2)]).to_dict()
        {'a': 1, 'b': 2}

        :param default: Can be a callable zero argument function. When not None, the returned
            dictionary is a collections.defaultdict with default as value for missing keys. If the
            value is not callable, then a zero argument lambda function is created returning the
            value and used for collections.defaultdict
        :return: dictionary from sequence of (Key, Value) elements
        """
        dictionary = {}
        for e in self.sequence:
            dictionary[e[0]] = e[1]
        if default is None:
            return dictionary
        else:
            if hasattr(default, '__call__'):
                return collections.defaultdict(default, dictionary)
            else:
                return collections.defaultdict(lambda: default, dictionary)

to_dict(self=[(1, 2), (2, 10), (7, 2)], default=None, self._base_sequence=[(1, 2), (2, 10), (7, 2)], self._lineage=Lineage: sequence)


def sqlite3(conn, sql, parameters=None, *args, **kwargs):
    """
    Additional entry point to Sequence which query data from sqlite db file.

    >>> seq.sqlite3('examples/users.db', 'select id, name from users where id = 1;').first()
    [(1, "Tom")]

    :param conn: path or sqlite connection, cursor
    :param sql: SQL query string
    :return: Sequence wrapping SQL cursor
    """

    if parameters is None:
        parameters = ()

    if isinstance(conn, (sqlite3api.Connection, sqlite3api.Cursor)):
        return seq(conn.execute(sql, parameters))
    elif isinstance(conn, str):
        with sqlite3api.connect(conn, *args, **kwargs) as input_conn:
            return seq(input_conn.execute(sql, parameters))
    else:
        raise ValueError('conn must be a must be a file path or sqlite3 Connection/Cursor')

sqlite3(conn=1, sql='SELECT * from user', parameters=None, args=(), kwargs={})


def secrets_dir(env=os.getenv('D2_ENVIRONMENT', None),
                basedir=os.getenv('D2_SECRETS_BASEDIR', None)):
    if env is not None:
        env_str = str(env)
    else:
        cwd = os.getcwd()
        default_file = os.path.join(cwd, '.python_secrets_environment')
        if os.path.exists(default_file):
            with open(default_file, 'r') as f:
                env_str = f.read().strip()
        else:
            env_str = os.path.basename(cwd)
    if basedir is None:
        basedir = os.path.join(
                HOME,
                'secrets' if sys.platform.startswith('win') else '.secrets')
    return os.path.join(basedir, env_str)

secrets_dir(env=None, basedir=None)


def _read2DArray(fp, data_type: str = "i", storage_type: str = None, file_dims: tuple = None):
        if storage_type == 'SPSE':
            raise TypeError('Sparse storage not allowed for 2D data form.')

        if data_type in ["i", "I"]:
            data_type = np.int32
            data_type_str = "i"
        elif data_type in ["f", "F"]:
            data_type = np.float32
            data_type_str = "f"
        else:
            raise ValueError(
                "Provided argument 'data_type' must be 'i' (integer) or 'f' (float). '%s' was provided." % data_type)

        # Note that 'order' refers to Fortran vs C (i.e. has nothing to do with floats or ints)
        array = np.ndarray(shape=file_dims[0:2], dtype=data_type, order='F')

        arraySize = array.size
        nread = 0
        while nread != arraySize:
            nbyte = HarFileIO._getEntrySize(fp)
            dataForm = "=4siiiiiii"
            V = HarFileIO._unpack_data(fp, dataForm)

            if fb(V[0]) != '    ':
                raise RuntimeError("Encountered characters at read2D loop")
            if V[2] != array.shape[0]:
                raise ValueError("Mismatching row sizes on header")
            if V[3] != array.shape[1]:
                raise ValueError("Mismatching col sizes on header")

            xsize = V[5] - V[4] + 1
            ysize = V[7] - V[6] + 1
            ndata = xsize * ysize
            nread += ndata
            dataForm = "=" + str(ndata) + data_type_str
            dat = HarFileIO._unpack_data(fp, dataForm)
            array[V[4] - 1:V[5], V[6] - 1:V[7]] = np.array(dat).reshape(xsize, ysize, order='F')

            if nbyte != HarFileIO._getEntrySize(fp):
                raise RuntimeError('Header corrupted.')

        # array = HAR_IO._read2DObj(fp, data_type, file_dims)
        array = np.ascontiguousarray(array)
        return array

_read2DArray(fp=<_io.BufferedReader name='/local/rcs/XXX/code/pytrace-collector/logs/pypibugs/tried/GEMPACKsoftware+HARPY/GEMPACKsoftware+HARPY/test_remove_header_array.har'>, data_type='i', storage_type='FULL', file_dims=(4, 4))


<StringPool #strings=85, #styles=0, UTF8=False>

<ARSCHeader idx='0x00000008' type='1' header_size='28' size='4252'>


def _write2DArray(fp, array, dtype):
        dataForm = "=i4siiiiiii"
        maxData = 7991
        nrec = (array.size - 1) // maxData + 1
        ndata = 0
        indexTuple=(None,)
        nbyte=0
        for st, end in HarFileIO._slice_inds(array, maxData):
            if array.ndim == 1:
                indexTuple = (slice(st[0], end[0]))
                ndata = (end[0] - st[0])
                nbyte = ndata * 4 + 32
                fp.write(struct.pack(dataForm, nbyte, tb('    '), nrec, array.size, 1, st[0] + 1, end[0], 1, 1))
            elif array.ndim == 2:
                indexTuple = (slice(st[0], end[0]), slice(st[1], end[1]))
                ndata = (end[0] - st[0]) * (end[1] - st[1])
                nbyte = ndata * 4 + 32
                fp.write(struct.pack(dataForm, nbyte, tb('    '), nrec, array.shape[0],
                                         array.shape[1], st[0] + 1, end[0], st[1] + 1, end[1]))

            dataForm1 = '=' + str(ndata) + dtype
            fp.write(struct.pack(dataForm1, *array[indexTuple].flatten('F')))
            fp.write(struct.pack('=i', nbyte))
            nrec = nrec - 1

_write2DArray(fp=<_io.BufferedWriter name='test_overwrite_header.har'>, array=array([[ 0,  1,  2,  3],       [ 4,  5,  6,  7],       [ 8,  9, 10, 11],       [12, 13, 14, 15]], dtype=int32), dtype='i')


def _readREFullObj(fp, array, dtype):

        nbyte = HarFileIO._getEntrySize(fp)
        dataForm = '=4sii'

        V = HarFileIO._unpack_data(fp, dataForm)

        if fb(V[0]) != '    ':
            raise Exception("Encountered characters at read7D[1]")

        nrec = V[1]
        NDim = V[2]
        dataForm = "=" + ('i' * NDim)

        V = HarFileIO._unpack_data(fp, dataForm)

        if nbyte != HarFileIO._getEntrySize(fp):
            raise RuntimeError("Header corrupted read7D[0] @ %d" % fp.tell())

        oldshape = array.shape
        array = array.flatten('F')
        idata = 0
        while nrec > 1:
            nbyte = HarFileIO._getEntrySize(fp)
            dataForm = '4s15i'
            V = HarFileIO._unpack_data(fp, dataForm)

            if fb(V[0]) != '    ':
                raise RuntimeError("Encountered characters at first four positions at)SetEl")

            if nbyte != HarFileIO._getEntrySize(fp):
                raise RuntimeError('read7D data[2] corrupted')

            nbyte = HarFileIO._getEntrySize(fp)
            ndata = (nbyte - 8) // struct.calcsize(dtype)
            dataForm = '4si' + str(ndata) + dtype
            V = HarFileIO._unpack_data(fp, dataForm)

            if nbyte != HarFileIO._getEntrySize(fp):
                raise RuntimeError('read7D data[2])corrupted')

            if fb(V[0]) != '    ':
                raise RuntimeError("Encountered characters at read7D[2]")

            nrec = V[1]
            array[idata:idata + ndata] = V[2:]
            idata += ndata
        array = array.reshape(oldshape, order='F')
        return array

_readREFullObj(fp=<_io.BufferedReader name='/local/rcs/XXX/code/pytrace-collector/logs/pypibugs/tried/GEMPACKsoftware+HARPY/GEMPACKsoftware+HARPY/test_remove_header_array.har'>, array=array([[0., 0.],       [0., 0.]], dtype=float32), dtype='f')


array[V[4] - 1:V[5], V[6] - 1:V[7]] = np.array(dat).reshape(xsize, ysize, order='F')


def __new__(cls, latitude=None, longitude=None, altitude=None):
        """
        :param float latitude: Latitude of point.
        :param float longitude: Longitude of point.
        :param float altitude: Altitude of point.
        """
        single_arg = latitude is not None and longitude is None and altitude is None
        if single_arg and not isinstance(latitude, util.NUMBER_TYPES):
            arg = latitude
            if isinstance(arg, Point):
                return cls.from_point(arg)
            elif isinstance(arg, str):
                return cls.from_string(arg)
            else:
                try:
                    seq = iter(arg)
                except TypeError:
                    raise TypeError(
                        "Failed to create Point instance from %r." % (arg,)
                    )
                else:
                    return cls.from_sequence(seq)

        if single_arg:
            raise ValueError(
                'A single number has been passed to the Point '
                'constructor. This is probably a mistake, because '
                'constructing a Point with just a latitude '
                'seems senseless. If this is exactly what was '
                'meant, then pass the zero longitude explicitly '
                'to get rid of this error.'
            )

        latitude, longitude, altitude = \
            _normalize_coordinates(latitude, longitude, altitude)

        self = super().__new__(cls)
        self.latitude = latitude
        self.longitude = longitude
        self.altitude = altitude
        return self

__new__(cls=<class 'geopy.point.Point'>, latitude=40.74113, longitude=-73.989656, altitude=None, __class__=<class 'geopy.point.Point'>)


def format_decimal(self, altitude=None):
        """
        Format decimal degrees with altitude::

            >>> p = Point(41.5, -81.0, 12.3)
            >>> p.format_decimal()
            '41.5, -81.0, 12.3km'
            >>> p = Point(41.5, 0, 0)
            >>> p.format_decimal()
            '41.5, 0.0'

        :param bool altitude: Whether to include ``altitude`` value.
            By default it is automatically included if it is non-zero.
        """
        coordinates = [str(self.latitude), str(self.longitude)]

        if altitude is None:
            altitude = bool(self.altitude)
        if altitude:
            if not isinstance(altitude, str):
                altitude = 'km'
            coordinates.append(self.format_altitude(altitude))

        return ", ".join(coordinates)

format_decimal(self=Point(41.5, 81.0, 2.5), altitude=None, self.altitude=2.5, self.latitude=41.5, self.longitude=81.0)


def _write7DFullArray(fp, array, dtype):
        StEndList = []
        for i, j in HarFileIO._slice_inds(array, 7996):
            StEndList.append([i[:], j[:]])
        nrec = len(StEndList) * 2 + 1
        dataForm = '=i4sii7ii'
        nbyte = struct.calcsize(dataForm) - 8
        writeList = [nbyte, tb('    '), nrec, 7]
        writeList.extend([array.shape[i] if i < array.ndim else 1 for i in range(0, 7)])
        writeList.append(nbyte)

        fp.write(struct.pack(dataForm, *writeList))

        array1=array.flatten('F')
        nWritten=0
        for StEnd in StEndList:
            nrec = nrec - 1
            st = StEnd[0]
            end = StEnd[1]

            PosList = [[st[i] + 1, end[i]][ind] if i < array.ndim else [1, 1][ind] for i in range(0, 7) for ind in
                       range(0, 2)]
            dataForm = '=i4s16i'
            nbyte = struct.calcsize(dataForm) - 8
            writeList = [nbyte, tb('    '), nrec]
            writeList.extend(PosList)
            writeList.append(nbyte)
            fp.write(struct.pack(dataForm, *writeList))

            nrec = nrec - 1
            ndata = 1
            for i, j in zip(st, end): ndata *= (j - i)
            if dtype == 'f' or dtype == 'i': nbyte = ndata * 4 + 8

            dataForm = '=i4si'
            fp.write(struct.pack(dataForm, nbyte, tb('    '), nrec))
            dataForm = '=' + str(ndata) + dtype
            fp.write(struct.pack(dataForm, *array1[nWritten:nWritten+ndata].flatten('F')))
            nWritten+=ndata
            dataForm = '=i'
            fp.write(struct.pack(dataForm, nbyte))

_write7DFullArray(fp=<_io.BufferedWriter name='test_overwrite_header.har'>, array=array([[1.  , 3.14],       [3.14, 5.  ]], dtype=float32), dtype='f')


def rowcol_to_a1(row, col):
    """Translates a row and column cell address to A1 notation.

    :param row: The row of the cell to be converted.
        Rows start at index 1.
    :type row: int, str

    :param col: The column of the cell to be converted.
        Columns start at index 1.
    :type row: int, str

    :returns: a string containing the cell's coordinates in A1 notation.

    Example:

    >>> rowcol_to_a1(1, 1)
    A1

    """
    row = int(row)
    col = int(col)

    if row < 1 or col < 1:
        raise IncorrectCellLabel("({}, {})".format(row, col))

    div = col
    column_label = ""

    while div:
        (div, mod) = divmod(div, 26)
        if mod == 0:
            mod = 26
            div -= 1
        column_label = chr(mod + MAGIC_NUMBER) + column_label

    label = "{}{}".format(column_label, row)

    return label

rowcol_to_a1(row=4, col=4)


def import_ext(name, optional=False):
  components = name.split(".")
  if (len(components) > 1):
    __import__(".".join(components[:-1]))
  previous_dlopenflags = None
  if (sys.platform.startswith("linux")) :
    previous_dlopenflags = sys.getdlopenflags()
    sys.setdlopenflags(0x100|0x2)
  try: mod = __import__(name)
  except ImportError as e:
    if (optional): return None
    error_msg = str(e)
    m = symbol_not_found_pat.search(error_msg)
    if m:
      error_msg = (  error_msg[:m.start(1)]
                   + cpp_function_name.demangle(m.group(1))
                   + error_msg[m.end(1):])
    raise ImportError(
      "\n  ".join(['__import__("%s"): %s' % (name, error_msg), "sys.path:"]
      + ["  "+p for p in sys.path]))
  for comp in components[1:]:
    mod = getattr(mod, comp)
  if (previous_dlopenflags is not None):
    sys.setdlopenflags(previous_dlopenflags)
  if (python_libstdcxx_so is not None):
    mod_file = getattr(mod, "__file__", None)
    if (mod_file is not None):
      for line in easy_run.fully_buffered(
                    command='/usr/bin/ldd "%s"' % mod_file).stdout_lines:
        if (line.strip().startswith("libstdc++.so")):
          mod_libstdcxx_so = line.split()[0]
          if (mod_libstdcxx_so != python_libstdcxx_so):
            raise SystemError("""\
FATAL: libstdc++.so mismatch:
  %s: %s
  %s: %s""" % (sys.executable, python_libstdcxx_so,
               mod_file, mod_libstdcxx_so))
          break
  return mod

import_ext(name='boost_python_meta_ext', optional=False)

ModuleNotFoundError("No module named 'boost_python_meta_ext'")


def client_factory(client_name, **kwargs):
    """Return a client for an external data set"""
    # set up
    dir_name = os.path.dirname(os.path.abspath(__file__))
    error_msg = 'No client found for name %s' % client_name
    client_key = client_name.upper()

    # find client
    try:
        client_vals = BALANCING_AUTHORITIES[client_key]
        module_name = client_vals['module']

        class_name = client_vals['class']
    except KeyError:
        raise ValueError(error_msg)

    # find module
    try:
        fp, pathname, description = imp.find_module(module_name, [dir_name])
    except ImportError:
        raise ValueError(error_msg)

    # load
    try:
        mod = imp.load_module(module_name, fp, pathname, description)
    finally:
        # Since we may exit via an exception, close fp explicitly.
        if fp:
            fp.close()

    # instantiate class
    try:
        client_inst = getattr(mod, class_name)(**kwargs)
    except AttributeError:
        raise ValueError(error_msg)

    # set name
    client_inst.NAME = client_name

    return client_inst

client_factory(client_name='MISO', kwargs={})

{'module': 'miso', 'class': 'MISOClient'}

('.py', 'r', 1)


def prepare_docstring_help(N):
    """Replace docstrings to include the parameters (schema)"""
    # at this point, the params have not yet been populated

    args = []
    if hasattr(N, '__annotations__'):
        for attr_name, cls in N.__annotations__.items():

            filtered = filter_params(N)
            parsed = parse_source_for_params(filtered)
            attr = attr_map(parsed).get(attr_name)
            if attr is None:
                continue

            args.append(argument_help(attr_name, attr))

    return '\n'.join(args)

prepare_docstring_help(N={})

{'type': 'int', 'default': '0', 'description': ''}


def parse_csv(csv):
    lines = csv.strip().split('\n')
    lines = [line for line in lines if line.strip()]
    csv = '\n'.join(lines)
    if autoeq_pattern.match(csv):  # Matches AutoEq produced CSV files
        columns = lines[0].split(',')
        return {column: [float(line.split(',')[i]) for line in lines[1:]] for i, column in enumerate(columns)}

    if rew_pattern.match(csv) or crinacle_pattern.match(csv):
        # These two have all sort of junk in them but the first column is frequency and the second SPL, so all good
        csv = '\n'.join([re.sub(r'(?:, ?| |\t)', '\t', line) for line in lines if numeric_start.match(line) and '?' not in line])
        lines = csv.split('\n')

    column_separator, decimal_separator = find_csv_separators(csv)
    columns = find_csv_columns(csv, column_separator)

    # Find indexes of frequency and raw columns
    if columns is None:
        # No header, assume first column is frequency and the second is SPL
        ixs = {'frequency': 0, 'raw': 1}
    else:
        ixs = {'frequency': None, 'raw': None}
        for i, column in enumerate(columns):
            if re.match(r'^freq', column, flags=re.IGNORECASE):
                ixs['frequency'] = i
            if re.match(r'^(?:spl|gain|ampl|raw)', column, flags=re.IGNORECASE):
                ixs['raw'] = i
        if ixs['frequency'] is None:
            if len(columns) == 2:  # Can't find proper columns but there's only two, assuming freq + raw
                ixs = {'frequency': 0, 'raw': 1}
            else:
                raise CsvParseError('Failed to find frequency column')
        if ixs['raw'] is None:
            raise CsvParseError('Failed to find SPL column')

    # Read and parse data lines
    data_line_pattern = re.compile(rf'^-?\d+(?:{column_separator}\d+)?')
    data = {'frequency': [], 'raw': []}
    for line in lines:
        if not data_line_pattern.match(line):
            continue
        cells = line.split(column_separator)
        if decimal_separator == ',':
            cells = [float(cell.replace(',', '.')) for cell in cells]
        else:
            cells = [float(cell) for cell in cells]
        for column, ix in ixs.items():
            data[column].append(cells[ix])
    return data

parse_csv(csv='frequency,raw\n20,0\n1000,3\n20000,0\n')


def abs_path(path):
    path = os.path.expanduser(path)
    if not os.path.isabs(path):
        cwd = os.environ.get('PWD') or os.getcwd()
        path = os.path.join(cwd, path)
    return path

abs_path(path='~/file.txt')


def load_permissions(apilevel, permtype='permissions'):
    """
    Load the Permissions for the given apilevel.

    The permissions lists are generated using this tool: https://github.com/U039b/aosp_permissions_extraction

    Has a fallback to select the maximum or minimal available API level.
    For example, if 28 is requested but only 26 is available, 26 is returned.
    If 5 is requested but 16 is available, 16 is returned.

    If an API level is requested which is in between of two API levels we got,
    the lower level is returned. For example, if 5,6,7,10 is available and 8 is
    requested, 7 is returned instead.

    :param apilevel:  integer value of the API level
    :param permtype: either load permissions (:code:`'permissions'`) or
    permission groups (:code:`'groups'`)
    :return: a dictionary of {Permission Name: {Permission info}
    """

    if permtype not in ['permissions', 'groups']:
        raise ValueError("The type of permission list is not known.")

    # Usually apilevel is supplied as string...
    apilevel = int(apilevel)

    root = os.path.dirname(os.path.realpath(__file__))
    permissions_file = os.path.join(root, "aosp_permissions", "permissions_{}.json".format(apilevel))

    levels = filter(lambda x: re.match(r'^permissions_\d+\.json$', x), os.listdir(os.path.join(root, "aosp_permissions")))
    levels = list(map(lambda x: int(x[:-5].split('_')[1]), levels))

    if not levels:
        logger.error("No Permissions available, can not load!")
        return {}

    logger.debug("Available API levels: {}".format(", ".join(map(str, sorted(levels)))))

    if not os.path.isfile(permissions_file):
        if apilevel > max(levels):
            logger.warning("Requested API level {} is larger than maximum we have, returning API level {} instead.".format(apilevel, max(levels)))
            return load_permissions(max(levels), permtype)
        if apilevel < min(levels):
            logger.warning("Requested API level {} is smaller than minimal we have, returning API level {} instead.".format(apilevel, max(levels)))
            return load_permissions(min(levels), permtype)

        # Missing level between existing ones, return the lower level
        lower_level = max(filter(lambda x: x < apilevel, levels))
        logger.warning("Requested API Level could not be found, using {} instead".format(lower_level))
        return load_permissions(lower_level, permtype)

    with open(permissions_file, "r") as fp:
        return json.load(fp)[permtype]

load_permissions(apilevel='25', permtype='permissions')


def _writeSecondRecord(fp: BinaryIO, inList):
        nint = len(inList) - 4

        if len(inList[3]) != 70:
            raise ValueError("'long_name' must be precisely 70 characters long. 'long_name' is: %s (%d characters long)." % (inList[3], len(inList[3])))

        inList = [tb(x) if isinstance(x, str) else x for x in inList]
        dataForm = '=i4s2s4s70s' + 'i' * nint + 'i' # For reading it is "=4s2s4s70si"
        byteLen = struct.calcsize(dataForm) - 8
        inList.append(byteLen)
        inList.insert(0, byteLen)
        packed = struct.pack(dataForm, *inList)
        fp.write(packed)

_writeSecondRecord(fp=<_io.BufferedWriter name='test_overwrite_header.har'>, inList=['    ', '1C', 'FULL', 'Creation Date and Time                                                ', 2, 1, 70])

[b'    ', b'1C', b'FULL', b'Creation Date and Time                                                ', 2, 1, 70, 92]

[b'    ', b'1C', b'FULL', b'Creation Date and Time                                                ', 2, 1, 70]

[92, b'    ', b'1C', b'FULL', b'Creation Date and Time                                                ', 2, 1, 70, 92]


idx_pad1_l = _get_idx(ndim, dim, slice(-pw[1]-2, -pw[1]-1, None))


def strip_quoted_wrapping(res, fname=None, fence=DEFAULT_FENCE):
    """
    Given an input string which may have extra "wrapping" around it, remove the wrapping.
    For example:

    filename.ext
    ```
    We just want this content
    Not the filename and triple quotes
    ```
    """
    if not res:
        return res

    res = res.splitlines()

    if fname and res[0].strip().endswith(Path(fname).name):
        res = res[1:]

    if res[0].startswith(fence[0]) and res[-1].startswith(fence[1]):
        res = res[1:-1]

    res = "\n".join(res)
    if res and res[-1] != "\n":
        res += "\n"

    return res

strip_quoted_wrapping(res='two\n', fname='/tmp/tmp7g7a2csg/file.txt', fence=('```', '```'))


def load_json_dict(filename, *args):
    """Checks if file exists. Returns {} if something fails."""
    data = {}
    if os.path.exists(filename):
        lock.acquire()
        with open(filename, "r") as f:
            try:
                data = _json.load(f)
                if not isinstance(data, dict):
                    data = {}
            except:
                data = {}  # TODO: issue a warning and bubble it up
        lock.release()
        if args:
            return {key: data[key] for key in args if key in data}
    return data

load_json_dict(filename='/home/XXX/.plotly/.config', args=())

<_io.TextIOWrapper name='/home/XXX/.plotly/.config' mode='r' encoding='UTF-8'>

{'plotly_domain': 'https://plot.ly', 'plotly_streaming_domain': 'stream.plot.ly', 'plotly_api_domain': 'https://api.plot.ly', 'plotly_ssl_verification': True, 'plotly_proxy_authorization': False, 'world_readable': True, 'sharing': 'public', 'auto_open': True}


def _ParseFn(args):
    """Parses the list of `args` into (varargs, kwargs), remaining_args."""
    kwargs, remaining_kwargs, remaining_args = _ParseKeywordArgs(
        args, all_args, fn_spec.varkw)

    # Note: _ParseArgs modifies kwargs.
    parsed_args, kwargs, remaining_args, capacity = _ParseArgs(
        fn_spec.args, fn_spec.defaults, num_required_args, kwargs,
        remaining_args, metadata)

    if fn_spec.varargs or fn_spec.varkw:
      # If we're allowed *varargs or **kwargs, there's always capacity.
      capacity = True

    extra_kw = set(kwargs) - set(fn_spec.kwonlyargs)
    if fn_spec.varkw is None and extra_kw:
      raise FireError('Unexpected kwargs present:', extra_kw)

    missing_kwonly = set(required_kwonly) - set(kwargs)
    if missing_kwonly:
      raise FireError('Missing required flags:', missing_kwonly)

    # If we accept *varargs, then use all remaining arguments for *varargs.
    if fn_spec.varargs is not None:
      varargs, remaining_args = remaining_args, []
    else:
      varargs = []

    for index, value in enumerate(varargs):
      varargs[index] = _ParseValue(value, None, None, metadata)

    varargs = parsed_args + varargs
    remaining_args += remaining_kwargs

    consumed_args = args[:len(args) - len(remaining_args)]
    return (varargs, kwargs), consumed_args, remaining_args, capacity

_ParseFn(args=['x'], all_args=[], fn_spec={args=[], varargs=None, varkw='cli_args', defaults=(), kwonlyargs=[], kwonlydefaults={}, annotations={}}, metadata={'ACCEPTS_POSITIONAL_ARGS': False}, num_required_args=0, required_kwonly=set())


def _insert(self, key, value):
        flat_key = self._serialize_key(key)
        i = self._index.get(flat_key, -1)
        if i >= 0:
            self._items[i] = (key, value)
        else:
            self._items.append((key, value))
            self._index[flat_key] = len(self._items) - 1

_insert(self=OrderedMapSerializedKey([]), key='みbob', value=199)


def get_mylogin_cnf_path():
    """Return the path to the login path file or None if it doesn't exist."""
    mylogin_cnf_path = os.getenv('MYSQL_TEST_LOGIN_FILE')

    if mylogin_cnf_path is None:
        app_data = os.getenv('APPDATA')
        default_dir = os.path.join(app_data, 'MySQL') if app_data else '~'
        mylogin_cnf_path = os.path.join(default_dir, '.mylogin.cnf')

    mylogin_cnf_path = os.path.expanduser(mylogin_cnf_path)

    if exists(mylogin_cnf_path):
        logger.debug("Found login path file at '{0}'".format(mylogin_cnf_path))
        return mylogin_cnf_path
    return None

get_mylogin_cnf_path()


def extend(*args):
        args = list(args)
        dest = args.pop(0)
        for source in args:
            if source:
                dest.update(source)
        return dest

extend(args=({}, {}))


def _readREFullObj(fp, array, dtype):

        nbyte = HarFileIO._getEntrySize(fp)
        dataForm = '=4sii'

        V = HarFileIO._unpack_data(fp, dataForm)

        if fb(V[0]) != '    ':
            raise Exception("Encountered characters at read7D[1]")

        nrec = V[1]
        NDim = V[2]
        dataForm = "=" + ('i' * NDim)

        V = HarFileIO._unpack_data(fp, dataForm)

        if nbyte != HarFileIO._getEntrySize(fp):
            raise RuntimeError("Header corrupted read7D[0] @ %d" % fp.tell())

        oldshape = array.shape
        array = array.flatten('F')
        idata = 0
        while nrec > 1:
            nbyte = HarFileIO._getEntrySize(fp)
            dataForm = '4s15i'
            V = HarFileIO._unpack_data(fp, dataForm)

            if fb(V[0]) != '    ':
                raise RuntimeError("Encountered characters at first four positions at)SetEl")

            if nbyte != HarFileIO._getEntrySize(fp):
                raise RuntimeError('read7D data[2] corrupted')

            nbyte = HarFileIO._getEntrySize(fp)
            ndata = (nbyte - 8) // struct.calcsize(dtype)
            dataForm = '4si' + str(ndata) + dtype
            V = HarFileIO._unpack_data(fp, dataForm)

            if nbyte != HarFileIO._getEntrySize(fp):
                raise RuntimeError('read7D data[2])corrupted')

            if fb(V[0]) != '    ':
                raise RuntimeError("Encountered characters at read7D[2]")

            nrec = V[1]
            array[idata:idata + ndata] = V[2:]
            idata += ndata
        array = array.reshape(oldshape, order='F')
        return array

_readREFullObj(fp=<_io.BufferedReader name='/local/rcs/XXX/code/pytrace-collector/logs/pypibugs/tried/GEMPACKsoftware+HARPY/GEMPACKsoftware+HARPY/test_remove_header_array.har'>, array=array([[0., 0.],       [0., 0.]], dtype=float32), dtype='f')


def task_subclasses(cls) -> Set[Type]:
    # Get subclasses of task, which not abstract

    subclasses = set()
    for sc in cls.__subclasses__():
        # Skip if last MRO base is ABC
        if abc.ABC != sc.__mro__[1]:
            subclasses.add(sc)

        subclasses.update(task_subclasses(sc))

    return subclasses

task_subclasses(cls=<class 'carnival.task.SimpleTask'>)


def get_air_quality(city):
    """
    通过城市名获取空气质量
    官网：http://aqicn.org/here/
    token 申请地址：http://aqicn.org/data-platform/token/#/
    :param city: 城市
    :return:
    """

    if not city or not city.strip():
        return
    print('获取 {} 的空气质量...'.format(city))
    try:

        url = 'http://api.waqi.info/feed/{city}/?token={token}'.format(city=city, token=AQICN_TOKEN)
        resp = requests.get(url)
        if resp.status_code == 200:
            # print(resp.text)
            content_dict = resp.json()
            if content_dict.get('status') == 'ok':
                data_dict = content_dict['data']
                aqi = data_dict['aqi']
                air_status = '严重污染'
                for key in sorted(AIR_STATUS_DICT):
                    if key >= aqi:
                        air_status = AIR_STATUS_DICT[key]
                        break
                aqi_info = '{city} PM2.5：{aqi} {air_status}'.format(city=city, aqi=aqi, air_status=air_status)
                # print(aqi_info)
                return aqi_info
            else:
                print('获取空气质量失败:{}'.format(content_dict['data']))
                return None
        print('获取空气质量失败。')
    except Exception as exception:
        print(str(exception))
    return None

get_air_quality(city='桂林')


def _compare_parts(self, a, b, decimal):
        if decimal:
            if a == "": a = "0"
            if b == "": b = "0"
            return int(a) - int(b)
        else:
            i = 0
            while i < (min(len(a), len(b)) + 1):
                res = self._order(self._get_empty_str_on_index_error(a, i)) \
                        - self._order(self._get_empty_str_on_index_error(b, i))
                if res != 0:
                    return res
                i += 1
            else:
                return 0

_compare_parts(self=2.2.0~rc5, a='', b='', decimal=False, self.epoch=0, self.revision='0', self.revision_allowed_chars=('.', '+', '~'), self.upstream_version='2.2.0~rc5', self.upstream_version_allowed_chars=('.', '+', '~', '-', ':'), self.version='2.2.0~rc5')


def _yvalue(b, a, c):
    """
    Get the two possible y-values for the upper point of a triangle.

    where c is the length of the down side starting in the origin and
    lying on the x-axes, a is the distance of the unknown point to the origen
    and b is the distance of the unknown point to the righter given point
    """
    # ckeck flatness to eliminate numerical errors when the triangle is flat
    if a + b <= c or a + c <= b or b + c <= a:
        return 0.0, -0.0

    res = 2 * ((a * b) ** 2 + (a * c) ** 2 + (b * c) ** 2)
    res -= a ** 4 + b ** 4 + c ** 4
    # in case of numerical errors set res to 0 (hope you check validty before)
    res = max(res, 0.0)
    res = np.sqrt(res)
    res /= 2 * c
    return res, -res

_yvalue(b=4.0, a=2.0, c=3.0)


def find_diffs(content):
    # We can always fence with triple-quotes, because all the udiff content
    # is prefixed with +/-/space.

    if not content.endswith("\n"):
        content = content + "\n"

    lines = content.splitlines(keepends=True)
    line_num = 0
    edits = []
    while line_num < len(lines):
        while line_num < len(lines):
            line = lines[line_num]
            if line.startswith("```diff"):
                line_num, these_edits = process_fenced_block(lines, line_num + 1)
                edits += these_edits
                break
            line_num += 1

    # For now, just take 1!
    # edits = edits[:1]

    return edits

find_diffs(content='\nSome text...\n\n```diff\n--- /dev/null\n+++ file.txt\n@@ ... @@\n-Original\n+Modified\n```\n')


def _global_import(name):
    p = __import__(name, globals(), locals(), level=1)
    lst = p.__all__ if '__all__' in dir(p) else dir(p)
    if lst:
        globals().pop(name, None)
        for k in lst:
            if not k.startswith('__'):
                globals()[k] = p.__dict__[k]
                __all__.append(k)

_global_import(name='base')

['DataFlow', 'ProxyDataFlow', 'RNGDataFlow', 'DataFlowTerminated']


def lineAfterContext(line, prefix):
    if line.startswith(prefix):
        line = line[len(prefix):]

    toks = line.split(' in ', 1)
    if len(toks) == 2:
        rest = toks[1].split(' ')
        line = ' '.join(rest[1:])

    return line

lineAfterContext(line='ic| eins: zwei', prefix='ic| ')


def show_ios_class_methods(args: list) -> None:
    """
        Displays the methods available in a class.

        :param args:
        :return:
    """

    if len(clean_argument_flags(args)) <= 0:
        click.secho('Usage: ios hooking list class_methods <class name> (--include-parents)', bold=True)
        return

    classname = args[0]

    api = state_connection.get_api()
    methods = api.ios_hooking_get_class_methods(classname, _should_include_parent_methods(args))

    if len(methods) > 0:

        # dump the methods to screen
        for method in methods:
            click.secho(method)

        click.secho('\nFound {0} methods'.format(len(methods)), bold=True)

    else:
        click.secho('No class / methods found')

show_ios_class_methods(args=['TEKeychainManager'])


def perfect_replace(whole_lines, part_lines, replace_lines):
    part_tup = tuple(part_lines)
    part_len = len(part_lines)

    for i in range(len(whole_lines) - part_len + 1):
        whole_tup = tuple(whole_lines[i : i + part_len])
        if part_tup == whole_tup:
            res = whole_lines[:i] + replace_lines + whole_lines[i + part_len :]
            return "".join(res)

perfect_replace(whole_lines=['two\n'], part_lines=['two\n'], replace_lines=['three\n'])


def generate_k(order, secexp, hash_func, data, retry_gen=0, extra_entropy=b''):
    '''
        order - order of the DSA generator used in the signature
        secexp - secure exponent (private key) in numeric form
        hash_func - reference to the same hash function used for generating hash
        data - hash in binary form of the signing data
        retry_gen - int - how many good 'k' values to skip before returning
        extra_entropy - extra added data in binary form as per section-3.6 of
            rfc6979
    '''

    qlen = bit_length(order)
    holen = hash_func().digest_size
    rolen = (qlen + 7) / 8
    bx = number_to_string(secexp, order) + bits2octets(data, order) + \
        extra_entropy

    # Step B
    v = b'\x01' * holen

    # Step C
    k = b'\x00' * holen

    # Step D

    k = hmac.new(k, v + b'\x00' + bx, hash_func).digest()

    # Step E
    v = hmac.new(k, v, hash_func).digest()

    # Step F
    k = hmac.new(k, v + b'\x01' + bx, hash_func).digest()

    # Step G
    v = hmac.new(k, v, hash_func).digest()

    # Step H
    while True:
        # Step H1
        t = b''

        # Step H2
        while len(t) < rolen:
            v = hmac.new(k, v, hash_func).digest()
            t += v

        # Step H3
        secret = bits2int(t, qlen)

        if secret >= 1 and secret < order:
            if retry_gen <= 0:
                return secret
            else:
                retry_gen -= 1

        k = hmac.new(k, v + b'\x00', hash_func).digest()
        v = hmac.new(k, v, hash_func).digest()

generate_k(order=115792089237316195423570985008687907852837564279074904382605163141518161494337, secexp=11806252235961651298089590628336806290921645495320214372650577192963691649562, hash_func=<built-in function openssl_sha256>, data=b'\xa7?\xcf3\x96@\x92\x92\x07(\x1f\xb8\xe08\x88H\x06\xe2\xeb\x08@\xf2$V\x94\xdb\xba\x1d\\\xc8\x9ee', retry_gen=0, extra_entropy=b'')


def _readSetElementInfoRecord(fp):

        SetNames = []
        ElementList = []
        SetStatus = []

        nbyte = HarFileIO._getEntrySize(fp)

        # read the data, has to be in chunks as it is dependent on interanl size specifications
        dataForm = '=' + '4siii12si'

        V = HarFileIO._unpack_data(fp, dataForm)

        if fb(V[0]) != '    ':
            raise RuntimeError("Encountered characters at first four positions at SetEl")

        NSets = V[3]
        Coefficient = fb(V[4])
        setKnown=V[5]!=0
        if not setKnown:
            dataForm = '=i'
        else:
            dataForm = "=" + str(NSets * 12) + 's' + str(NSets) + 's' + str(NSets) + 'i' + 'i'

        V = HarFileIO._unpack_data(fp, dataForm)
        if setKnown:
            SetNames = [fb(V[0][i:i + 12]) for i in range(0, NSets * 12, 12)]
            SetStatus = [fb(V[1][i:i + 1]) for i in range(0, NSets)]

        Nexplicit = V[-1]
        dataForm = '=' + str(Nexplicit * 12) + 's'
        V = HarFileIO._unpack_data(fp, dataForm)

        if Nexplicit > 0:
            dataForm = '=' + str(Nexplicit * 12) + 's'
            V = HarFileIO._unpack_data(fp, dataForm)

            ElementList = [fb(V[-1][i:i + 12]) for i in range(0, NSets * 12, 12)]

        HarFileIO._checkRead(fp, nbyte)

        return Coefficient, SetNames, SetStatus, ElementList

_readSetElementInfoRecord(fp=<_io.BufferedReader name='/local/rcs/XXX/code/pytrace-collector/logs/pypibugs/tried/GEMPACKsoftware+HARPY/GEMPACKsoftware+HARPY/test_remove_header_array.har'>)

(b'SimpleSet   SimpleSet2  ', b'kk', 0, 0, 0)


def guessed_wrong_repo(io, git_root, fnames, git_dname):
    """After we parse the args, we can determine the real repo. Did we guess wrong?"""

    try:
        check_repo = Path(GitRepo(io, fnames, git_dname).root).resolve()
    except FileNotFoundError:
        return

    # we had no guess, rely on the "true" repo result
    if not git_root:
        return str(check_repo)

    git_root = Path(git_root).resolve()
    if check_repo == git_root:
        return

    return str(check_repo)

guessed_wrong_repo(io=<MagicMock id='140339689280704'>, git_root='/tmp/tmp65cfsod1', fnames=['/tmp/tmp65cfsod1/foo.py'], git_dname=None)