jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def _combinations(
        n_features, min_degree, max_degree, interaction_only, include_bias
    ):
        comb = combinations if interaction_only else combinations_w_r
        start = max(1, min_degree)
        iter = chain.from_iterable(
            comb(range(n_features), i) for i in range(start, max_degree + 1)
        )
        if include_bias:
            iter = chain(comb(range(n_features), 0), iter)
        return iter
//  [python]
def compute_contrastive_loss(self, pos_h, neg_h):
        binomial_coefficient, loss = s.comb(len(pos_h), 2), torch.tensor(0.0).type_as(pos_h[0])
        
        for i in range(len(pos_h)):
            for j in range(len(pos_h)):
                if i == j: continue
                numerator, denominator = torch.exp(self.cos(pos_h[i], pos_h[j]) / self.hparams.tau).type_as(pos_h[0]), torch.tensor(0.0).type_as(pos_h[0])

                for k in range(len(neg_h)):
                    denominator += torch.exp(self.cos(pos_h[i], neg_h[k]) / self.hparams.tau)

                for k in range(len(pos_h)):
                    if i == k: continue
                    denominator += torch.exp(self.cos(pos_h[i], pos_h[k]) / self.hparams.tau)

                loss += torch.log(numerator / denominator)
        
        return -1 / s.comb(len(pos_h), 2) * loss
//  [python]
def compute_contrastive_loss(self, pos_h, neg_h):
        binomial_coefficient, loss = scipy.special.comb(len(pos_h), 2), torch.tensor(0.0).type_as(pos_h[0])
        
        for i in range(len(pos_h)):
            for j in range(len(pos_h)):
                if i == j: continue
                numerator, denominator = torch.exp(self.cos(pos_h[i], pos_h[j]) / self.hparams.tau).type_as(pos_h[0]), torch.tensor(0.0).type_as(pos_h[0])

                for k in range(len(neg_h)):
                    denominator += torch.exp(self.cos(pos_h[i], neg_h[k]) / self.hparams.tau)

                for k in range(len(pos_h)):
                    if i == k: continue
                    denominator += torch.exp(self.cos(pos_h[i], pos_h[k]) / self.hparams.tau)

                loss += torch.log(numerator / denominator)
        
        return -1 / scipy.special.comb(len(pos_h), 2) * loss
//  [python]
def _num_combinations(
        n_features, min_degree, max_degree, interaction_only, include_bias
    ):
        """Calculate number of terms in polynomial expansion

        This should be equivalent to counting the number of terms returned by
        _combinations(...) but much faster.
        """

        if interaction_only:
            combinations = sum(
                [
                    s.comb(n_features, i, exact=True)
                    for i in range(max(1, min_degree), min(max_degree, n_features) + 1)
                ]
            )
        else:
            combinations = s.comb(n_features + max_degree, max_degree, exact=True) - 1
            if min_degree > 0:
                d = min_degree - 1
                combinations -= s.comb(n_features + d, d, exact=True) - 1

        if include_bias:
            combinations += 1

        return combinations
//  [python]
def _num_combinations(
        n_features, min_degree, max_degree, interaction_only, include_bias
    ):
        """Calculate number of terms in polynomial expansion

        This should be equivalent to counting the number of terms returned by
        _combinations(...) but much faster.
        """

        if interaction_only:
            combinations = sum(
                [
                    comb(n_features, i, exact=True)
                    for i in range(max(1, min_degree), min(max_degree, n_features) + 1)
                ]
            )
        else:
            combinations = comb(n_features + max_degree, max_degree, exact=True) - 1
            if min_degree > 0:
                d = min_degree - 1
                combinations -= comb(n_features + d, d, exact=True) - 1

        if include_bias:
            combinations += 1

        return combinations
//  [python]
def m(epsilon, delta, r):
    """
    The r'th uncentered moment. Given by Jones et al.
    """
    frac1 = 1 / np.power(2, r)
    acc = 0
    for i in range(r + 1):
        combs = s.comb(r, i)
        flip = np.power(-1, i)
        ex = np.exp((r - 2 * i) * epsilon / delta)
        p = P((r - 2 * i) / delta)
        acc += combs * flip * ex * p
    return frac1 * acc
//  [python]
def m(epsilon, delta, r):
    """
    The r'th uncentered moment. Given by Jones et al.
    """
    frac1 = 1 / np.power(2, r)
    acc = 0
    for i in range(r + 1):
        combs = spp.comb(r, i)
        flip = np.power(-1, i)
        ex = np.exp((r - 2 * i) * epsilon / delta)
        p = P((r - 2 * i) / delta)
        acc += combs * flip * ex * p
    return frac1 * acc
//  [python]
def generate_hypothesis(self, uv_img, pts):

        # Determining the number of pts present
        num_of_pts = pts.shape[0]

        # Selecting random pairs of pts (0-n) [n = number of pts]
        # By using torch.multinomial: https://pytorch.org/docs/stable/generated/torch.multinomial.html?highlight=multinomial
        
        # limit N based on the maximum number of possible combinations given the 
        # number of points
        max_num_of_pairs = int(scipy.misc.comb(num_of_pts, 2))
        N = min(self.HPARAM.HV_NUM_OF_HYPOTHESES, max_num_of_pairs)

        # first creating uniform probability distribution
        weights = torch.ones(num_of_pts, device=uv_img.device).expand(N, -1)

        # Obtaining the point pairs via sequence numbers: (N, 2)
        point_pair_idx = torch.multinomial(weights, num_samples=2, replacement=False)

        # Indexing the pts locations among the image (random_pt_pairs: 2xNx2)
        # first index = (pair division), second index = (pt index), third index = (pt's x and y)
        pt_pairs = torch.stack([pts[point_pair_idx[:,0]], pts[point_pair_idx[:,1]]])

        # Indexing the pts unit vector values
        uv_img = uv_img.permute(1, 2, 0)
        uv_pt_pairs = uv_img[pt_pairs[:,:,0], pt_pairs[:,:,1]]

        # Remove pairs if one of the pts has nan in it.
        is_nan = torch.sum(torch.isnan(uv_pt_pairs), dim=(0,2)) != 0
        
        # If all are nan, then return nan
        if is_nan.all():
            return torch.tensor([float('nan'), float('nan')], device=uv_img.device)
        
        # Elif any nan are present, keep only non_nan values
        elif is_nan.any():
            is_not_nan = ~is_nan
            uv_pt_pairs = uv_pt_pairs[:,is_not_nan,:]
            pt_pairs = pt_pairs[:,is_not_nan,:]

        # Construct the system of equations
        A = torch.stack((uv_pt_pairs[0], -uv_pt_pairs[1]), dim=-1)
        B = torch.stack((-pt_pairs[0], pt_pairs[1]), dim=-1)
        B = (B[:,:,0] + B[:,:,1]).reshape((pt_pairs.shape[1],-1,1))

        # Solving the linear system of equations
        #Y = lstsq_solver(A, B, pt_pairs, uv_pt_pairs)
        Y = self.batched_pinverse_solver(A, B, pt_pairs, uv_pt_pairs)

        # Prun any outliers using std trimming for Y
        Y = self.std_trimming_mean(Y)

        return Y
//  [python]
def generate_hypothesis(self, uv_img, pts):

        # Determining the number of pts present
        num_of_pts = pts.shape[0]

        # Selecting random pairs of pts (0-n) [n = number of pts]
        # By using torch.multinomial: https://pytorch.org/docs/stable/generated/torch.multinomial.html?highlight=multinomial
        
        # limit N based on the maximum number of possible combinations given the 
        # number of points
        max_num_of_pairs = int(scipy.special.comb(num_of_pts, 2))
        N = min(self.HPARAM.HV_NUM_OF_HYPOTHESES, max_num_of_pairs)

        # first creating uniform probability distribution
        weights = torch.ones(num_of_pts, device=uv_img.device).expand(N, -1)

        # Obtaining the point pairs via sequence numbers: (N, 2)
        point_pair_idx = torch.multinomial(weights, num_samples=2, replacement=False)

        # Indexing the pts locations among the image (random_pt_pairs: 2xNx2)
        # first index = (pair division), second index = (pt index), third index = (pt's x and y)
        pt_pairs = torch.stack([pts[point_pair_idx[:,0]], pts[point_pair_idx[:,1]]])

        # Indexing the pts unit vector values
        uv_img = uv_img.permute(1, 2, 0)
        uv_pt_pairs = uv_img[pt_pairs[:,:,0], pt_pairs[:,:,1]]

        # Remove pairs if one of the pts has nan in it.
        is_nan = torch.sum(torch.isnan(uv_pt_pairs), dim=(0,2)) != 0
        
        # If all are nan, then return nan
        if is_nan.all():
            return torch.tensor([float('nan'), float('nan')], device=uv_img.device)
        
        # Elif any nan are present, keep only non_nan values
        elif is_nan.any():
            is_not_nan = ~is_nan
            uv_pt_pairs = uv_pt_pairs[:,is_not_nan,:]
            pt_pairs = pt_pairs[:,is_not_nan,:]

        # Construct the system of equations
        A = torch.stack((uv_pt_pairs[0], -uv_pt_pairs[1]), dim=-1)
        B = torch.stack((-pt_pairs[0], pt_pairs[1]), dim=-1)
        B = (B[:,:,0] + B[:,:,1]).reshape((pt_pairs.shape[1],-1,1))

        # Solving the linear system of equations
        #Y = lstsq_solver(A, B, pt_pairs, uv_pt_pairs)
        Y = self.batched_pinverse_solver(A, B, pt_pairs, uv_pt_pairs)

        # Prun any outliers using std trimming for Y
        Y = self.std_trimming_mean(Y)

        return Y
//  [python]
def Sxyz(a, b, aa, bb, Ra, Rb, R):
    """
    Compute overlap integral between two unnormalized Cartesian gaussian functions along one direction.

    INPUT:
        A: Angular momentum along the chosen direction for Gaussian 1
        B: Angular momentum along the chosen direction for Gaussian 2
        AA: Exponential coefficient for Gaussian 1
        BB: Exponential coefficient for Gaussian 2
        RA: Coordinate (along chosen direction) of the center of Gaussian 1
        RB: Coordinate (along chosen direction) of the center of Gaussian 2
        R: Coordinate (along chosen direction) of the center of the product of the two gaussians
    OUTPUT:
        S: Overlap of the two gaussians along the chosen direction

    Source:
        The Mathematica Journal
        Evaluation of Gaussian Molecular Integrals
        I. Overlap Integrals
        Minhhuy Hô and Julio Manuel Hernández-Pérez
    """

    S = 0

    for i in range(a + 1):
        for j in range(b + 1):
            if (i + j) % 2 == 0:
                tmp = s.comb(a, i, exact=True)
                tmp *= s.comb(b, j, exact=True)
                tmp *= s.factorial2(i + j - 1, exact=True)
                tmp /= (2.0 * (aa + bb)) ** ((i + j) / 2.0)
                tmp *= (R - Ra) ** (a - i)
                tmp *= (R - Rb) ** (b - j)

                S += tmp

    return S
//  [python]
def Sxyz(a, b, aa, bb, Ra, Rb, R):
    """
    Compute overlap integral between two unnormalized Cartesian gaussian functions along one direction.

    INPUT:
        A: Angular momentum along the chosen direction for Gaussian 1
        B: Angular momentum along the chosen direction for Gaussian 2
        AA: Exponential coefficient for Gaussian 1
        BB: Exponential coefficient for Gaussian 2
        RA: Coordinate (along chosen direction) of the center of Gaussian 1
        RB: Coordinate (along chosen direction) of the center of Gaussian 2
        R: Coordinate (along chosen direction) of the center of the product of the two gaussians
    OUTPUT:
        S: Overlap of the two gaussians along the chosen direction

    Source:
        The Mathematica Journal
        Evaluation of Gaussian Molecular Integrals
        I. Overlap Integrals
        Minhhuy Hô and Julio Manuel Hernández-Pérez
    """

    S = 0

    for i in range(a + 1):
        for j in range(b + 1):
            if (i + j) % 2 == 0:
                tmp = special.comb(a, i, exact=True)
                tmp *= special.comb(b, j, exact=True)
                tmp *= special.factorial2(i + j - 1, exact=True)
                tmp /= (2.0 * (aa + bb)) ** ((i + j) / 2.0)
                tmp *= (R - Ra) ** (a - i)
                tmp *= (R - Rb) ** (b - j)

                S += tmp

    return S
//  [python]
def get_global_trigger_rate(single_rate, n_channels, number_coincidences, coinc_window):
    '''calculates the global trigger rate with a given trigger rate from one channel including
     number of channels with trigger (n_channels), number_coincidences and the time coinc_window
     of the coincidence (coinc_window)

     Parameters
    ----------
    single_rate: float or array
        trigger rate of a single antenna
    n_channels: int
        number of channels with trigger
    number_coincidences: int
        number of coincidences with antennas
    coinc_window: float
        time window of coincidence in ns


    Returns
    -------
    global trigger rate of all trigger channels combined
    '''

    return number_coincidences * s.comb(n_channels, number_coincidences) \
           * single_rate ** number_coincidences * coinc_window ** (number_coincidences - 1)
//  [python]
def get_global_trigger_rate(single_rate, n_channels, number_coincidences, coinc_window):
    '''calculates the global trigger rate with a given trigger rate from one channel including
     number of channels with trigger (n_channels), number_coincidences and the time coinc_window
     of the coincidence (coinc_window)

     Parameters
    ----------
    single_rate: float or array
        trigger rate of a single antenna
    n_channels: int
        number of channels with trigger
    number_coincidences: int
        number of coincidences with antennas
    coinc_window: float
        time window of coincidence in ns


    Returns
    -------
    global trigger rate of all trigger channels combined
    '''

    return number_coincidences * scipy.special.comb(n_channels, number_coincidences) \
           * single_rate ** number_coincidences * coinc_window ** (number_coincidences - 1)
//  [python]
def _data_partitions(data, permutations, size_a, axis=-1, random_state=None):
    """All partitions of data into sets of given lengths, ignoring order"""

    random_state = check_random_state(random_state)
    if axis < 0:  # we'll be adding a new dimension at the end
        axis = data.ndim + axis

    # prepare permutation indices
    size = data.shape[axis]
    # number of distinct combinations
    n_max = s.comb(size, size_a)

    if permutations < n_max:
        indices = np.array([random_state.permutation(size)
                            for i in range(permutations)]).T
    else:
        permutations = n_max
        indices = np.array([np.concatenate(z)
                            for z in _all_partitions(size_a, size-size_a)]).T

    data = data.swapaxes(axis, -1)   # so we can index along a new dimension
    data = data[..., indices]        # generate permutations
    data = data.swapaxes(-2, axis)   # restore original axis order
    data = np.moveaxis(data, -1, 0)  # permutations indexed along axis 0
    return data, permutations
//  [python]
def _data_partitions(data, permutations, size_a, axis=-1, random_state=None):
    """All partitions of data into sets of given lengths, ignoring order"""

    random_state = check_random_state(random_state)
    if axis < 0:  # we'll be adding a new dimension at the end
        axis = data.ndim + axis

    # prepare permutation indices
    size = data.shape[axis]
    # number of distinct combinations
    n_max = special.comb(size, size_a)

    if permutations < n_max:
        indices = np.array([random_state.permutation(size)
                            for i in range(permutations)]).T
    else:
        permutations = n_max
        indices = np.array([np.concatenate(z)
                            for z in _all_partitions(size_a, size-size_a)]).T

    data = data.swapaxes(axis, -1)   # so we can index along a new dimension
    data = data[..., indices]        # generate permutations
    data = data.swapaxes(-2, axis)   # restore original axis order
    data = np.moveaxis(data, -1, 0)  # permutations indexed along axis 0
    return data, permutations
//  [python]
def test_canoncial_purification(conserve_ancilla, L=6, charge_sector=0, eps=1.e-14):
    site = spin_half
    psi = purification_mps.PurificationMPS.from_infiniteT_canonical(
        [site] * L, [charge_sector], conserve_ancilla_charge=conserve_ancilla)
    psi.test_sanity()
    Szs = psi.expectation_value('Sz')
    assert abs(sum(Szs) - charge_sector) < 1.e-13
    total_psi = psi.get_theta(0, L).take_slice(0, 'vL').take_slice(0, 'vR')
    total_psi.itranspose(['p' + str(i) for i in range(L)] + ['q' + str(i) for i in range(L)])
    # note: don't `combine_legs`: it will permute the p legs differently than q due to charges
    total_psi_dense = total_psi.to_ndarray().reshape(2**L, 2**L)
    # now it should be diagonal
    diag = np.diag(total_psi_dense)
    assert np.all(np.abs(total_psi_dense - np.diag(diag) < eps))  # is it diagonal?
    # and the diagonal should be sqrt(L choose L//2) for states with fitting numbers
    pref = 1. / scipy.misc.comb(L, L // 2 + charge_sector)**0.5
    Q_p = site.leg.to_qflat()[:, 0]
    for i, entry in enumerate(diag):
        Q_i = sum([Q_p[int(b)] for b in format(i, 'b').zfill(L)])
        if Q_i == charge_sector:
            assert abs(entry - pref) < eps
        else:
            assert abs(entry) < eps

    # and one quick test of TEBD
    xxz_pars = dict(L=L, Jxx=1., Jz=3., hz=0., bc_MPS='finite', sort_charge=False)
    # sort_charge should be same as for global `spin_half`.
    M = XXZChain(xxz_pars)
    TEBD_params = {
        'trunc_params': {
            'chi_max': 16,
            'svd_min': 1.e-8
        },
        'disentangle': None,  # 'renyi' should work as well, 'backwards' not.
        'dt': 0.1,
        'N_steps': 2
    }
    if conserve_ancilla:
        M = purification_mps.convert_model_purification_canonical_conserve_ancilla_charge(M)
    eng = PurificationTEBD(psi, M, TEBD_params)
    eng.run_imaginary(0.2)
    eng.run()
    N = psi.expectation_value('Id')  # check normalization : <1> =?= 1
    npt.assert_array_almost_equal_nulp(N, np.ones([L]), 100)
//  [python]
def test_canoncial_purification(conserve_ancilla, L=6, charge_sector=0, eps=1.e-14):
    site = spin_half
    psi = purification_mps.PurificationMPS.from_infiniteT_canonical(
        [site] * L, [charge_sector], conserve_ancilla_charge=conserve_ancilla)
    psi.test_sanity()
    Szs = psi.expectation_value('Sz')
    assert abs(sum(Szs) - charge_sector) < 1.e-13
    total_psi = psi.get_theta(0, L).take_slice(0, 'vL').take_slice(0, 'vR')
    total_psi.itranspose(['p' + str(i) for i in range(L)] + ['q' + str(i) for i in range(L)])
    # note: don't `combine_legs`: it will permute the p legs differently than q due to charges
    total_psi_dense = total_psi.to_ndarray().reshape(2**L, 2**L)
    # now it should be diagonal
    diag = np.diag(total_psi_dense)
    assert np.all(np.abs(total_psi_dense - np.diag(diag) < eps))  # is it diagonal?
    # and the diagonal should be sqrt(L choose L//2) for states with fitting numbers
    pref = 1. / scipy.special.comb(L, L // 2 + charge_sector)**0.5
    Q_p = site.leg.to_qflat()[:, 0]
    for i, entry in enumerate(diag):
        Q_i = sum([Q_p[int(b)] for b in format(i, 'b').zfill(L)])
        if Q_i == charge_sector:
            assert abs(entry - pref) < eps
        else:
            assert abs(entry) < eps

    # and one quick test of TEBD
    xxz_pars = dict(L=L, Jxx=1., Jz=3., hz=0., bc_MPS='finite', sort_charge=False)
    # sort_charge should be same as for global `spin_half`.
    M = XXZChain(xxz_pars)
    TEBD_params = {
        'trunc_params': {
            'chi_max': 16,
            'svd_min': 1.e-8
        },
        'disentangle': None,  # 'renyi' should work as well, 'backwards' not.
        'dt': 0.1,
        'N_steps': 2
    }
    if conserve_ancilla:
        M = purification_mps.convert_model_purification_canonical_conserve_ancilla_charge(M)
    eng = PurificationTEBD(psi, M, TEBD_params)
    eng.run_imaginary(0.2)
    eng.run()
    N = psi.expectation_value('Id')  # check normalization : <1> =?= 1
    npt.assert_array_almost_equal_nulp(N, np.ones([L]), 100)
//  [python]
def get_single_channel_trigger_rate(global_rate, n_channels, number_coincidences, coinc_window):
    '''calculates the trigger rate of a singel channel for a global trigger rate including
     number of channels with trigger (n_channels), number_coincidences and the time coinc_window
     of the coincidence (coinc_window)

     Parameters
        ----------
    global_rate: float or array
        trigger rate of a all antenna together
    n_channels: int
        number of channels with trigger
    number_coincidences: int
        number of coincidences with antennas
    coinc_window: float
        time window of coincidence in ns

    Returns
    -------
    single trigger rate of one channel
    '''
    r_single_pow_n_coincidences = global_rate / (
            number_coincidences * s.comb(n_channels, number_coincidences) *
            coinc_window ** (number_coincidences - 1))
    r_single = r_single_pow_n_coincidences ** (1 / float(number_coincidences))
    return r_single
//  [python]
def get_single_channel_trigger_rate(global_rate, n_channels, number_coincidences, coinc_window):
    '''calculates the trigger rate of a singel channel for a global trigger rate including
     number of channels with trigger (n_channels), number_coincidences and the time coinc_window
     of the coincidence (coinc_window)

     Parameters
        ----------
    global_rate: float or array
        trigger rate of a all antenna together
    n_channels: int
        number of channels with trigger
    number_coincidences: int
        number of coincidences with antennas
    coinc_window: float
        time window of coincidence in ns

    Returns
    -------
    single trigger rate of one channel
    '''
    r_single_pow_n_coincidences = global_rate / (
            number_coincidences * scipy.special.comb(n_channels, number_coincidences) *
            coinc_window ** (number_coincidences - 1))
    r_single = r_single_pow_n_coincidences ** (1 / float(number_coincidences))
    return r_single
//  [python]
def bernstein_poly(self, n, i, t):
        return s.comb(n, i) * t ** i * (1 - t) ** (n - i)
//  [python]
def bernstein_poly(self, n, i, t):
        return scipy.special.comb(n, i) * t ** i * (1 - t) ** (n - i)
//  [python]
def __init__(self, n: int, *, k: int = 1, retraction: str = "qr"):
        self._n = n
        self._k = k

        if k == 1:
            name = f"Special orthogonal group SO({n})"
        elif k > 1:
            name = f"Sphecial orthogonal group SO({n})^{k}"
        else:
            raise ValueError("k must be an integer no less than 1.")
        dimension = int(k * s.comb(n, 2))
        super().__init__(name, dimension, retraction)
//  [python]
def __init__(self, n: int, *, k: int = 1, retraction: str = "qr"):
        self._n = n
        self._k = k

        if k == 1:
            name = f"Special orthogonal group SO({n})"
        elif k > 1:
            name = f"Sphecial orthogonal group SO({n})^{k}"
        else:
            raise ValueError("k must be an integer no less than 1.")
        dimension = int(k * scipy.special.comb(n, 2))
        super().__init__(name, dimension, retraction)
//  [python]
def StabilityUsage(k_min, k_max, step, n_rep, dists, n_cells, filepath_usage, filepath_consensus=None, 
                   n_cells_max=20000, cluster=True):
    
    '''
    Borrowed from Brunet et al. 2004: Metagenes and molecular pattern discovery using
    matrix factorization
    '''
    np.random.seed(520)
    
    K = np.arange(k_min, k_max+1, step)
    results = []
    
    if n_cells > n_cells_max:
        cells_choice = np.random.choice(n_cells, n_cells_max, replace=False)
        n_cells = n_cells_max
    else:
        cells_choice = np.arange(n_cells)
    
    for k in K:
        for dist in dists:
            d = np.zeros(int(s.comb(n_cells, 2)))
            # Read all cell usage results
            for rs in np.arange(1, n_rep+1):
                usage = load_df_from_npz(filepath_usage%(dist, rs, k))
                usage = usage.to_numpy()
                usage = usage[cells_choice,]               
                if cluster:
                    assign = usage.argmax(1)
                    usage = np.zeros_like(usage)
                    usage[np.arange(len(usage)), assign] = 1
                    d += scipy.spatial.distance.pdist(usage, 'braycurtis')
                else:
                    d += scipy.spatial.distance.pdist(usage)
            
            d = d/n_rep
            
            if filepath_consensus:
                if cluster:
                    # Save consensus clustering matrix
                    np.save(filepath_consensus%(dist, k), 1-d)
                else:
                    # Save distance matrix
                    np.save(filepath_consensus%(dist, k), d)
            
            # Hierarchical clustering using distance d
            HC = fc.linkage(d, method='average')
            cophen_d = cophenet(HC)
            
            # Compute Cophenetic correlation coefficient
            cophen_corr = np.corrcoef(d, cophen_d)[0,1]
            results.append([k, dist, cophen_corr])
    
    results = pd.DataFrame(results, columns=["K", "Method", "Stability"])
    
    return results
//  [python]
def StabilityUsage(k_min, k_max, step, n_rep, dists, n_cells, filepath_usage, filepath_consensus=None, 
                   n_cells_max=20000, cluster=True):
    
    '''
    Borrowed from Brunet et al. 2004: Metagenes and molecular pattern discovery using
    matrix factorization
    '''
    np.random.seed(520)
    
    K = np.arange(k_min, k_max+1, step)
    results = []
    
    if n_cells > n_cells_max:
        cells_choice = np.random.choice(n_cells, n_cells_max, replace=False)
        n_cells = n_cells_max
    else:
        cells_choice = np.arange(n_cells)
    
    for k in K:
        for dist in dists:
            d = np.zeros(int(scipy.special.comb(n_cells, 2)))
            # Read all cell usage results
            for rs in np.arange(1, n_rep+1):
                usage = load_df_from_npz(filepath_usage%(dist, rs, k))
                usage = usage.to_numpy()
                usage = usage[cells_choice,]               
                if cluster:
                    assign = usage.argmax(1)
                    usage = np.zeros_like(usage)
                    usage[np.arange(len(usage)), assign] = 1
                    d += scipy.spatial.distance.pdist(usage, 'braycurtis')
                else:
                    d += scipy.spatial.distance.pdist(usage)
            
            d = d/n_rep
            
            if filepath_consensus:
                if cluster:
                    # Save consensus clustering matrix
                    np.save(filepath_consensus%(dist, k), 1-d)
                else:
                    # Save distance matrix
                    np.save(filepath_consensus%(dist, k), d)
            
            # Hierarchical clustering using distance d
            HC = fc.linkage(d, method='average')
            cophen_d = cophenet(HC)
            
            # Compute Cophenetic correlation coefficient
            cophen_corr = np.corrcoef(d, cophen_d)[0,1]
            results.append([k, dist, cophen_corr])
    
    results = pd.DataFrame(results, columns=["K", "Method", "Stability"])
    
    return results
//  [python]
def makeHapProbs(inbam,inbai,inbed,outprob, outdiff_id):
    """takes bam, index, bed files to do random sampling and create pseudohaps"""
    diff_id=[]
    probsA=[]
    chrm=[]
    pos=[]

    bed = pybedtools.BedTool(inbed)
    with pysam.AlignmentFile(inbam) as bam:
        for i, snp in enumerate(bed):

            freq = pileup2alleles(bam, snp)
            refA=freq[0]
            altA=freq[1]
            nsites=refA+altA
            if nsites==0:
                probsA.append(-9)
                diff_id.append(-9)

            elif nsites>0 and nsites<2:

                p=altA/nsites
                probsA.append(p)

                diff_id.append(-9)


            elif nsites>=2:

                p=altA/nsites
                probsA.append(p)

                d0=s.comb(altA,1) * s.comb(refA,1)/s.comb(nsites,2)
                diff_id.append(d0)

            chrm.append(snp.chrom)
            pos.append(snp.start)

    df=pd.DataFrame({
    "chrm":chrm,
    "pos":pos,
    "alt_prob":probsA
    })

    diffs=pd.DataFrame({
    "chrm":chrm,
    "pos":pos,
    "diff":diff_id
    })

    with pd.option_context('display.max_rows', len(df.index), 'display.max_columns', len(df.columns)):
            df.to_csv(outprob, sep=',')

    with pd.option_context('display.max_rows', len(diffs.index), 'display.max_columns', len(diffs.columns)):
            diffs.to_csv(outdiff_id, sep=',')
//  [python]
def makeHapProbs(inbam,inbai,inbed,outprob, outdiff_id):
    """takes bam, index, bed files to do random sampling and create pseudohaps"""
    diff_id=[]
    probsA=[]
    chrm=[]
    pos=[]

    bed = pybedtools.BedTool(inbed)
    with pysam.AlignmentFile(inbam) as bam:
        for i, snp in enumerate(bed):

            freq = pileup2alleles(bam, snp)
            refA=freq[0]
            altA=freq[1]
            nsites=refA+altA
            if nsites==0:
                probsA.append(-9)
                diff_id.append(-9)

            elif nsites>0 and nsites<2:

                p=altA/nsites
                probsA.append(p)

                diff_id.append(-9)


            elif nsites>=2:

                p=altA/nsites
                probsA.append(p)

                d0=scp.comb(altA,1) * scp.comb(refA,1)/scp.comb(nsites,2)
                diff_id.append(d0)

            chrm.append(snp.chrom)
            pos.append(snp.start)

    df=pd.DataFrame({
    "chrm":chrm,
    "pos":pos,
    "alt_prob":probsA
    })

    diffs=pd.DataFrame({
    "chrm":chrm,
    "pos":pos,
    "diff":diff_id
    })

    with pd.option_context('display.max_rows', len(df.index), 'display.max_columns', len(df.columns)):
            df.to_csv(outprob, sep=',')

    with pd.option_context('display.max_rows', len(diffs.index), 'display.max_columns', len(diffs.columns)):
            diffs.to_csv(outdiff_id, sep=',')
//  [python]
def test_k_array_rank_arbitrary_precision():
    n, k = 100, 50
    a = np.arange(n-k, n)
    assert_array_equal(k_array_rank(a),
                       scipy.misc.comb(n, k, exact=True)-1)
//  [python]
def test_k_array_rank_arbitrary_precision():
    n, k = 100, 50
    a = np.arange(n-k, n)
    assert_array_equal(k_array_rank(a),
                       scipy.special.comb(n, k, exact=True)-1)
//  [python]
def test_comb_legacy(self, N, k, legacy, repetition):
        # test is only relevant for exact=True
        if legacy and (N != int(N) or k != int(k)):
            with pytest.warns(
                DeprecationWarning,
                match=r"Non-integer arguments are currently being cast to",
            ):
                result = s.comb(N, k, exact=True, legacy=legacy,
                                      repetition=repetition)
        else:
            result = s.comb(N, k, exact=True, legacy=legacy,
                                  repetition=repetition)
        if legacy:
            # for exact=True and legacy=True, cast input arguments, else don't
            if repetition:
                # the casting in legacy mode happens AFTER transforming N & k,
                # so rounding can change (e.g. both floats, but sum to int);
                # hence we need to emulate the repetition-transformation here
                N, k = int(N + k - 1), int(k)
                repetition = False
            else:
                N, k = int(N), int(k)
        # expected result is the same as with exact=False
        expected = s.comb(N, k, legacy=legacy, repetition=repetition)
        assert_equal(result, expected)
//  [python]
def test_comb_legacy(self, N, k, legacy, repetition):
        # test is only relevant for exact=True
        if legacy and (N != int(N) or k != int(k)):
            with pytest.warns(
                DeprecationWarning,
                match=r"Non-integer arguments are currently being cast to",
            ):
                result = special.comb(N, k, exact=True, legacy=legacy,
                                      repetition=repetition)
        else:
            result = special.comb(N, k, exact=True, legacy=legacy,
                                  repetition=repetition)
        if legacy:
            # for exact=True and legacy=True, cast input arguments, else don't
            if repetition:
                # the casting in legacy mode happens AFTER transforming N & k,
                # so rounding can change (e.g. both floats, but sum to int);
                # hence we need to emulate the repetition-transformation here
                N, k = int(N + k - 1), int(k)
                repetition = False
            else:
                N, k = int(N), int(k)
        # expected result is the same as with exact=False
        expected = special.comb(N, k, legacy=legacy, repetition=repetition)
        assert_equal(result, expected)
//  [python]
def smoothstep(x, NN=1., xmin=0., xmax=1.):
    N = math.ceil(NN)
    x = np.clip((x - xmin) / (xmax - xmin), 0, 1)
    result = 0
    for n in range(0, N+1):
         result += s.comb(N+n, n) * s.comb(2*N+1, N-n) * (-x)**n
    result *= x**(N+1)
    if NN != N: result = (x + result) / 2
    return result
//  [python]
def smoothstep(x, NN=1., xmin=0., xmax=1.):
    N = math.ceil(NN)
    x = np.clip((x - xmin) / (xmax - xmin), 0, 1)
    result = 0
    for n in range(0, N+1):
         result += scipy.special.comb(N+n, n) * scipy.special.comb(2*N+1, N-n) * (-x)**n
    result *= x**(N+1)
    if NN != N: result = (x + result) / 2
    return result
//  [python]
def test_comb_with_np_int64(self):
        n = 70
        k = 30
        np_n = np.int64(n)
        np_k = np.int64(k)
        res_np = s.comb(np_n, np_k, exact=True)
        res_py = s.comb(n, k, exact=True)
        assert res_np == res_py
//  [python]
def test_comb_with_np_int64(self):
        n = 70
        k = 30
        np_n = np.int64(n)
        np_k = np.int64(k)
        res_np = special.comb(np_n, np_k, exact=True)
        res_py = special.comb(n, k, exact=True)
        assert res_np == res_py
//  [python]
def bernstein_poly(self, n, i, t):
        """
        Bernstein polynom.
        :param n: (int) polynom degree
        :param i: (int)
        :param t: (float)
        :return: (float)
        """
        return s.comb(n, i) * t**i * (1 - t) ** (n - i)
//  [python]
def bernstein_poly(self, n, i, t):
        """
        Bernstein polynom.
        :param n: (int) polynom degree
        :param i: (int)
        :param t: (float)
        :return: (float)
        """
        return scipy.special.comb(n, i) * t**i * (1 - t) ** (n - i)
//  [python]
def __init__(self, input_gen, labels, eval_func=exact_match_instance_evaluation, token_func=None, annotators=None,
                 documents=None):
        if not (annotators and documents):
            annotators, documents = _collect_annotators_and_documents(input_gen)
            annotators.sort()
            documents.sort()
        assert len(annotators) > 1, 'At least two annotators are necessary to compute agreement!'
        num_pairs = s.comb(len(annotators), 2, exact=True)
        # (p, d, c, l) where p := annotator pairs, d := documents, c := counts (tp, total = 2*tp+fp+fn), l := labels
        self._pdcl = np.zeros((num_pairs, len(documents), 2, len(labels)))
        self._documents = list(documents)
        self._doc2idx = {d: i for i, d in enumerate(documents)}
        self._labels = list(labels)
        self._label2idx = {l: i for i, l in enumerate(labels)}
        self._annotators = list(annotators)
        self._pairs = [pair for pair in combinations(annotators, 2)]
        self._pair2idx = {p: i for i, p in enumerate(self._pairs)}
        # add pairs in reverse order (same index)
        for (a1, a2), value in self._pair2idx.copy().items():
            self._pair2idx[(a2, a1)] = value
        self._eval_func = eval_func  # function used to extract true positives, false positives and false negatives
        self._token_func = token_func  # function used for tokenization
        self._compute_tp_total(input_gen)
//  [python]
def __init__(self, input_gen, labels, eval_func=exact_match_instance_evaluation, token_func=None, annotators=None,
                 documents=None):
        if not (annotators and documents):
            annotators, documents = _collect_annotators_and_documents(input_gen)
            annotators.sort()
            documents.sort()
        assert len(annotators) > 1, 'At least two annotators are necessary to compute agreement!'
        num_pairs = comb(len(annotators), 2, exact=True)
        # (p, d, c, l) where p := annotator pairs, d := documents, c := counts (tp, total = 2*tp+fp+fn), l := labels
        self._pdcl = np.zeros((num_pairs, len(documents), 2, len(labels)))
        self._documents = list(documents)
        self._doc2idx = {d: i for i, d in enumerate(documents)}
        self._labels = list(labels)
        self._label2idx = {l: i for i, l in enumerate(labels)}
        self._annotators = list(annotators)
        self._pairs = [pair for pair in combinations(annotators, 2)]
        self._pair2idx = {p: i for i, p in enumerate(self._pairs)}
        # add pairs in reverse order (same index)
        for (a1, a2), value in self._pair2idx.copy().items():
            self._pair2idx[(a2, a1)] = value
        self._eval_func = eval_func  # function used to extract true positives, false positives and false negatives
        self._token_func = token_func  # function used for tokenization
        self._compute_tp_total(input_gen)
//  [python]
def test_comb_legacy(self, N, k, legacy, repetition):
        # test is only relevant for exact=True
        if legacy is not None:
            with pytest.warns(
                DeprecationWarning,
                match=r"Using 'legacy' keyword is deprecated"
            ):
                result = s.comb(N, k, exact=True, legacy=legacy,
                                      repetition=repetition)
        else:
            result = s.comb(N, k, exact=True, legacy=legacy,
                                  repetition=repetition)
        if legacy:
            # for exact=True and legacy=True, cast input arguments, else don't
            if repetition:
                # the casting in legacy mode happens AFTER transforming N & k,
                # so rounding can change (e.g. both floats, but sum to int);
                # hence we need to emulate the repetition-transformation here
                N, k = int(N + k - 1), int(k)
                repetition = False
            else:
                N, k = int(N), int(k)
        # expected result is the same as with exact=False
        with suppress_warnings() as sup:
            if legacy is not None:
                sup.filter(DeprecationWarning)
            expected = s.comb(N, k, legacy=legacy, repetition=repetition)
        assert_equal(result, expected)
//  [python]
def test_comb_legacy(self, N, k, legacy, repetition):
        # test is only relevant for exact=True
        if legacy is not None:
            with pytest.warns(
                DeprecationWarning,
                match=r"Using 'legacy' keyword is deprecated"
            ):
                result = special.comb(N, k, exact=True, legacy=legacy,
                                      repetition=repetition)
        else:
            result = special.comb(N, k, exact=True, legacy=legacy,
                                  repetition=repetition)
        if legacy:
            # for exact=True and legacy=True, cast input arguments, else don't
            if repetition:
                # the casting in legacy mode happens AFTER transforming N & k,
                # so rounding can change (e.g. both floats, but sum to int);
                # hence we need to emulate the repetition-transformation here
                N, k = int(N + k - 1), int(k)
                repetition = False
            else:
                N, k = int(N), int(k)
        # expected result is the same as with exact=False
        with suppress_warnings() as sup:
            if legacy is not None:
                sup.filter(DeprecationWarning)
            expected = special.comb(N, k, legacy=legacy, repetition=repetition)
        assert_equal(result, expected)
//  [python]
def memory_expensive_random_choose(l, k, n=5000):
    l = list(l)
    n = min(int(s.comb(len(l), k, repetition=False)), n)
    cvs = set()
    while len(cvs) < n:
        elm = numpy.random.choice(l,k,replace=False)
        elm = tuple(elm)
        elm = frozenset(elm)
        if elm in cvs:
            continue
        cvs.add(elm)
        yield elm
//  [python]
def memory_expensive_random_choose(l, k, n=5000):
    l = list(l)
    n = min(int(scipy.special.comb(len(l), k, repetition=False)), n)
    cvs = set()
    while len(cvs) < n:
        elm = numpy.random.choice(l,k,replace=False)
        elm = tuple(elm)
        elm = frozenset(elm)
        if elm in cvs:
            continue
        cvs.add(elm)
        yield elm
//  [python]
def get_accuracy(series_groundtruth, series_parsedlog, debug=False):
    """ Compute accuracy metrics between log parsing results and ground truth
    
    Arguments
    ---------
        series_groundtruth : pandas.Series
            A sequence of groundtruth event Ids
        series_parsedlog : pandas.Series
            A sequence of parsed event Ids
        debug : bool, default False
            print error log messages when set to True

    Returns
    -------
        precision : float
        recall : float
        f_measure : float
        accuracy : float
    """
    series_groundtruth_valuecounts = series_groundtruth.value_counts()
    real_pairs = 0
    for count in series_groundtruth_valuecounts:
        if count > 1:
            real_pairs += s.comb(count, 2)

    print('real_pairs: ', real_pairs)

    series_parsedlog_valuecounts = series_parsedlog.value_counts()
    parsed_pairs = 0
    for count in series_parsedlog_valuecounts:
        if count > 1:
            parsed_pairs += s.comb(count, 2)
    print('parsed_pairs: ', parsed_pairs)


    accurate_pairs = 0
    accurate_events = 0 # determine how many lines are correctly parsed
    for parsed_eventId in series_parsedlog_valuecounts.index:
        logIds = series_parsedlog[series_parsedlog == parsed_eventId].index
        series_groundtruth_logId_valuecounts = series_groundtruth[logIds].value_counts()
        error_eventIds = (parsed_eventId, series_groundtruth_logId_valuecounts.index.tolist())
        error = True
        if series_groundtruth_logId_valuecounts.size == 1:
            groundtruth_eventId = series_groundtruth_logId_valuecounts.index[0]
            if logIds.size == series_groundtruth[series_groundtruth == groundtruth_eventId].size:
                accurate_events += logIds.size
                error = False
        if error and debug:
            print('(parsed_eventId, groundtruth_eventId) =', error_eventIds, 'failed', logIds.size, 'messages')
        for count in series_groundtruth_logId_valuecounts:
            if count > 1:
                accurate_pairs += s.comb(count, 2)

    precision = float(accurate_pairs) / parsed_pairs
    recall = float(accurate_pairs) / real_pairs
    f_measure = 2 * precision * recall / (precision + recall)
    accuracy = float(accurate_events) / series_groundtruth.size
    return precision, recall, f_measure, accuracy
//  [python]
def get_accuracy(series_groundtruth, series_parsedlog, debug=False):
    """ Compute accuracy metrics between log parsing results and ground truth
    
    Arguments
    ---------
        series_groundtruth : pandas.Series
            A sequence of groundtruth event Ids
        series_parsedlog : pandas.Series
            A sequence of parsed event Ids
        debug : bool, default False
            print error log messages when set to True

    Returns
    -------
        precision : float
        recall : float
        f_measure : float
        accuracy : float
    """
    series_groundtruth_valuecounts = series_groundtruth.value_counts()
    real_pairs = 0
    for count in series_groundtruth_valuecounts:
        if count > 1:
            real_pairs += scipy.special.comb(count, 2)

    print('real_pairs: ', real_pairs)

    series_parsedlog_valuecounts = series_parsedlog.value_counts()
    parsed_pairs = 0
    for count in series_parsedlog_valuecounts:
        if count > 1:
            parsed_pairs += scipy.special.comb(count, 2)
    print('parsed_pairs: ', parsed_pairs)


    accurate_pairs = 0
    accurate_events = 0 # determine how many lines are correctly parsed
    for parsed_eventId in series_parsedlog_valuecounts.index:
        logIds = series_parsedlog[series_parsedlog == parsed_eventId].index
        series_groundtruth_logId_valuecounts = series_groundtruth[logIds].value_counts()
        error_eventIds = (parsed_eventId, series_groundtruth_logId_valuecounts.index.tolist())
        error = True
        if series_groundtruth_logId_valuecounts.size == 1:
            groundtruth_eventId = series_groundtruth_logId_valuecounts.index[0]
            if logIds.size == series_groundtruth[series_groundtruth == groundtruth_eventId].size:
                accurate_events += logIds.size
                error = False
        if error and debug:
            print('(parsed_eventId, groundtruth_eventId) =', error_eventIds, 'failed', logIds.size, 'messages')
        for count in series_groundtruth_logId_valuecounts:
            if count > 1:
                accurate_pairs += scipy.special.comb(count, 2)

    precision = float(accurate_pairs) / parsed_pairs
    recall = float(accurate_pairs) / real_pairs
    f_measure = 2 * precision * recall / (precision + recall)
    accuracy = float(accurate_events) / series_groundtruth.size
    return precision, recall, f_measure, accuracy
//  [python]
def _MatrixSufficientStatistics(
    x: ArrayLike,
    y: ArrayLike,
    epsilon: float,
    preproc: Optional[KendallPreproc],
    preproc_x: Optional[KendallPreproc],
    ) -> Tuple[int, int, int, int, int]:
  """Calculates tau sufficient statistics using matrices in NumPy.
  
  An absolute difference less than `epsilon` in x pairs is considered to be
  a tie.

  Args:
    x: Vector of numeric values.
    y: Vector of numeric values.
    epsilon: The threshold for which an absolute difference in x scores should
      be considered a tie.
    preproc: A `KendallPreproc` object that has been called on a vector of
      y values to be compared to the currrent x. If this is non-None, the y
      parameter is ignored.
    preproc_x: A `PairwiseDiffs` object that has been called on a vector of
      x values to be compared to the currrent y. If this is non-None, the x
      parameter is ignored.

  Returns:
    The number of concordant pairs, discordant pairs, pairs tied only in x,
    paired tied only in y, and pairs tied in both x and y.
  """
  if preproc_x is None:
    preproc_x = PairwiseDiffs(x, epsilon)
  x_diffs = preproc_x.x_diffs
  x_is_tie = preproc_x.x_is_tie

  if preproc is None:
    y1, y2 = np.meshgrid(y, y.T)
    y_diffs = y1 - y2
    y_is_tie = y_diffs == 0.0
  else:
    y_diffs = preproc.y_diffs
    y_is_tie = preproc.y_is_tie

  n = len(y) if preproc is None else len(preproc.y)
  num_pairs = int(s.comb(n, 2))
  # All of the counts are divided by 2 because each pair is double counted. The
  # double counted data will always be an even number, so dividing by 2 will
  # be an integer.
  con = int(
      ((x_diffs > 0) & (y_diffs > 0) | (x_diffs < 0) & (y_diffs < 0)).sum() / 2
  )
  t_x = int((x_is_tie & ~y_is_tie).sum() / 2)
  t_y = int((~x_is_tie & y_is_tie).sum() / 2)
  t_xy = int(((x_is_tie & y_is_tie).sum() - n) / 2)  # -n removes diagonal
  dis = num_pairs - (con + t_x + t_y + t_xy)
  return con, dis, t_x, t_y, t_xy