jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def _estimate_shapley_values_exact(
    set_func: Callable[[np.ndarray], Union[float, np.ndarray]], num_players: int, n_jobs: int
) -> np.ndarray:
    """Following Eq. (2) in
    Janzing, D., Minorics, L., & Bloebaum, P. (2020).
    Feature relevance quantification in explainable AI: A causal problem.
    In International Conference on Artificial Intelligence and Statistics (pp. 2907-2916). PMLR."""
    all_subsets = [tuple(subset) for subset in itertools.product([0, 1], repeat=num_players)]

    with Parallel(n_jobs=n_jobs) as parallel:
        subset_to_result_map = _evaluate_set_function(set_func, all_subsets, parallel)

    def compute_subset_weight(length: int) -> float:
        return 1 / (num_players * comb(num_players - 1, length))

    subset_weight_cache = {}

    shapley_values = [None] * num_players
    subsets_missing_one_player = np.array(list(itertools.product([0, 1], repeat=num_players - 1)))
    for player_index in range(num_players):
        subsets_with_player = [
            tuple(subset) for subset in np.insert(subsets_missing_one_player, player_index, 1, axis=1)
        ]
        subsets_without_player = [
            tuple(subset) for subset in np.insert(subsets_missing_one_player, player_index, 0, axis=1)
        ]

        for i in range(len(subsets_with_player)):
            subset_length = int(np.sum(subsets_without_player[i]))
            if subset_length not in subset_weight_cache:
                subset_weight_cache[subset_length] = compute_subset_weight(subset_length)

            weighted_diff = subset_weight_cache[subset_length] * (
                subset_to_result_map[subsets_with_player[i]] - subset_to_result_map[subsets_without_player[i]]
            )
            # For estimating Shapley values for multiple samples (e.g. in feature relevance) and the number of samples
            # is unknown beforehand.
            if shapley_values[player_index] is None:
                shapley_values[player_index] = weighted_diff
            else:
                shapley_values[player_index] += weighted_diff

    return np.array(shapley_values).T
//  [python]
def print_num_possible_permuted_combinations(df_clade_size, sample_original_foreground):
    import scipy
    num_possible_permutation_combination = 1
    is_fg_stem = df_clade_size.loc[:, 'is_fg_stem'].values
    for bin_no in df_clade_size.loc[:, 'bin'].unique():
        is_bin = (df_clade_size.loc[:, 'bin'] == bin_no)
        num_bin_fg = (is_bin & is_fg_stem).sum()
        if num_bin_fg == 0:
            continue
        if sample_original_foreground:
            num_bin_choice = is_bin.sum()
        else:
            num_bin_choice = (is_bin & ~is_fg_stem).sum()
        num_possible_permutation_combination_bin = scipy.misc.comb(N=num_bin_choice, k=num_bin_fg, exact=True)
        num_possible_permutation_combination *= num_possible_permutation_combination_bin
    txt = 'Number of possible clade-permuted combinations without considering branch dependency = {:,}'
    print(txt.format(int(num_possible_permutation_combination)))
//  [python]
def print_num_possible_permuted_combinations(df_clade_size, sample_original_foreground):
    import scipy
    num_possible_permutation_combination = 1
    is_fg_stem = df_clade_size.loc[:, 'is_fg_stem'].values
    for bin_no in df_clade_size.loc[:, 'bin'].unique():
        is_bin = (df_clade_size.loc[:, 'bin'] == bin_no)
        num_bin_fg = (is_bin & is_fg_stem).sum()
        if num_bin_fg == 0:
            continue
        if sample_original_foreground:
            num_bin_choice = is_bin.sum()
        else:
            num_bin_choice = (is_bin & ~is_fg_stem).sum()
        num_possible_permutation_combination_bin = scipy.special.comb(N=num_bin_choice, k=num_bin_fg, repetition=False)
        num_possible_permutation_combination *= num_possible_permutation_combination_bin
    txt = 'Number of possible clade-permuted combinations without considering branch dependency = {:,}'
    print(txt.format(int(num_possible_permutation_combination)))
//  [python]
def multivariate_rho(u: pd.DataFrame) -> float:
    """
    Helper function for extended approach to partner selection. Calculates 3 proposed estimators for
    high dimensional generalization for Spearman's rho. These implementations are present in
    Schmid, F., Schmidt, R., 2007. Multivariate extensions of Spearman’s rho and related statis-tics.

    :param u: (pd.DataFrame) : ranked returns of quadruple
    :return: (float) : mean of the three estimators of multivariate rho
    """

    n, d = u.shape  # n : Number of samples, d : Number of stocks
    h_d = (d + 1) / ((2 ** d) - d - 1)

    # # Calculating the first estimator of multivariate rho
    sum_1 = np.prod(1 - u, axis=1).sum()
    rho_1 = h_d * (-1 + (((2 ** d) / n) * sum_1))

    # # Calculating the second estimator of multivariate rho
    sum_2 = np.prod(u, axis=1).sum()
    rho_2 = h_d * (-1 + (((2 ** d) / n) * sum_2))

    # Calculating the third estimator of multivariate rho
    pairs = [x for x in itertools.combinations(range(u.shape[-1]), 2)]
    sum_3 = np.sum([(1 - u.iloc[:, k]) * (1 - u.iloc[:, l]) for (k, l) in pairs])
    dc2 = s.comb(d, 2, exact=True)
    rho_3 = -3 + (12 / (n * dc2)) * sum_3

    return (rho_1 + rho_2 + rho_3) / 3
//  [python]
def multivariate_rho(u: pd.DataFrame) -> float:
    """
    Helper function for extended approach to partner selection. Calculates 3 proposed estimators for
    high dimensional generalization for Spearman's rho. These implementations are present in
    Schmid, F., Schmidt, R., 2007. Multivariate extensions of Spearman’s rho and related statis-tics.

    :param u: (pd.DataFrame) : ranked returns of quadruple
    :return: (float) : mean of the three estimators of multivariate rho
    """

    n, d = u.shape  # n : Number of samples, d : Number of stocks
    h_d = (d + 1) / ((2 ** d) - d - 1)

    # # Calculating the first estimator of multivariate rho
    sum_1 = np.prod(1 - u, axis=1).sum()
    rho_1 = h_d * (-1 + (((2 ** d) / n) * sum_1))

    # # Calculating the second estimator of multivariate rho
    sum_2 = np.prod(u, axis=1).sum()
    rho_2 = h_d * (-1 + (((2 ** d) / n) * sum_2))

    # Calculating the third estimator of multivariate rho
    pairs = [x for x in itertools.combinations(range(u.shape[-1]), 2)]
    sum_3 = np.sum([(1 - u.iloc[:, k]) * (1 - u.iloc[:, l]) for (k, l) in pairs])
    dc2 = scipy.special.comb(d, 2, exact=True)
    rho_3 = -3 + (12 / (n * dc2)) * sum_3

    return (rho_1 + rho_2 + rho_3) / 3
//  [python]
def search_space_size(self):
        # do not consider p
        # consider the choices as a set not list, this might not be the case
        # for some search space construction
        return scipy.misc.comb(
            self.num_choices, self.size, exact=False, repetition=self.replace
        )
//  [python]
def search_space_size(self):
        # do not consider p
        # consider the choices as a set not list, this might not be the case
        # for some search space construction
        return scipy.special.comb(
            self.num_choices, self.size, exact=False, repetition=self.replace
        )
//  [python]
def num_perfect_balanced_sequences():
    """
    compute S(N), the number of sequences of N values to be inserted that lead to a
    perfect complete binary tree. Now generalize to find a recursive formula
    for c(k) to predict value of S(N) when N = 2^(k-1).
    """
    from algs.modeling import numpy_error
    print(7, compute_perfect_tree(7))
    print(15, compute_perfect_tree(15))
    print(31, compute_perfect_tree(31))

    if numpy_error:
        print('cannot compute scipy.special.comb()')
        return

    import scipy

    # https://oeis.org/A056972
    def counting(k):
        if k == 1:
            return 1

        binomial = scipy.special.comb(2 ** k - 2, 2 ** (k-1) - 1, exact=True)
        return binomial * (counting(k-1) ** 2)

    for n in range(1,8):
        print(n, counting(n))
//  [python]
def num_perfect_balanced_sequences():
    """
    compute S(N), the number of sequences of N values to be inserted that lead to a
    perfect complete binary tree. Now generalize to find a recursive formula
    for c(k) to predict value of S(N) when N = 2^(k-1).
    """
    from algs.modeling import numpy_error
    print(7, compute_perfect_tree(7))
    print(15, compute_perfect_tree(15))
    print(31, compute_perfect_tree(31))

    if numpy_error:
        print('cannot compute scipy.special.comb()')
        return

    import scipy

    # https://oeis.org/A056972
    def counting(k):
        if k == 1:
            return 1

        binomial = scipy.special.comb(2 ** k - 2, 2 ** (k-1) - 1)
        return binomial * (counting(k-1) ** 2)

    for n in range(1,8):
        print(n, counting(n))
//  [python]
def macula_matrix(d: int, k: int, n: int):
  """Produces d-separable design matrix."""
  # https://core.ac.uk/download/pdf/82758506.pdf
  n_groups = int(s.comb(n, d))
  n_cols = int(s.comb(n, k))
  new_groups = np.zeros((n_groups, n_cols), dtype=bool)
  comb_groups = itertools.combinations(range(n), d)
  comb_cols = itertools.combinations(range(n), k)
  d_vec = np.zeros((n_groups, n), dtype=bool)
  k_vec = np.zeros((n_cols, n), dtype=bool)
  for i, comb_g in enumerate(comb_groups):
    d_vec[i, comb_g] = True
  for j, comb_c in enumerate(comb_cols):
    k_vec[j, comb_c] = True

  for i in range(n_groups):
    for j in range(n_cols):
      new_groups[i, j] = np.all(
          np.logical_or(np.logical_not(d_vec[i, :]), k_vec[j, :]))
  return new_groups
//  [python]
def macula_matrix(d: int, k: int, n: int):
  """Produces d-separable design matrix."""
  # https://core.ac.uk/download/pdf/82758506.pdf
  n_groups = int(scipy.special.comb(n, d))
  n_cols = int(scipy.special.comb(n, k))
  new_groups = np.zeros((n_groups, n_cols), dtype=bool)
  comb_groups = itertools.combinations(range(n), d)
  comb_cols = itertools.combinations(range(n), k)
  d_vec = np.zeros((n_groups, n), dtype=bool)
  k_vec = np.zeros((n_cols, n), dtype=bool)
  for i, comb_g in enumerate(comb_groups):
    d_vec[i, comb_g] = True
  for j, comb_c in enumerate(comb_cols):
    k_vec[j, comb_c] = True

  for i in range(n_groups):
    for j in range(n_cols):
      new_groups[i, j] = np.all(
          np.logical_or(np.logical_not(d_vec[i, :]), k_vec[j, :]))
  return new_groups
//  [python]
def _samlmularge(x, nmom=5):
    try:
        x = np.asarray(x, dtype=np.float64)
        n = len(x)
        x.sort()
    except ValueError:
        raise ValueError("Input data to estimate L-moments must be numeric.")

    if nmom <= 0:
        raise ValueError("Invalid number of sample L-moments")

    if n < nmom:
        raise ValueError("Insufficient length of data for specified nmoments")

    ##Calculate first order
    l = [np.sum(x) / s.comb(n, 1, exact=True)]

    if nmom == 1:
        return l[0]

    #Setup comb table, where comb[i][x] refers to comb(x,i)
    comb = []
    for i in range(1, nmom):
        comb.append([])
        for j in range(n):
            comb[-1].append(s.comb(j, i, exact=True))

    for mom in range(2, nmom + 1):
        coefl = 1.0 / mom * 1.0 / s.comb(n, mom, exact=True)
        xtrans = []
        for i in range(0, n):
            coeftemp = []
            for j in range(0, mom):
                coeftemp.append(1)

            for j in range(0, mom - 1):
                coeftemp[j] = coeftemp[j] * comb[mom - j - 2][i]

            for j in range(1, mom):
                coeftemp[j] = coeftemp[j] * comb[j - 1][n - i - 1]

            for j in range(0, mom):
                coeftemp[j] = coeftemp[j] * s.comb(mom - 1, j, exact=True)

            for j in range(0, int(0.5 * mom)):
                coeftemp[j * 2 + 1] = -coeftemp[j * 2 + 1]
            coeftemp = sum(coeftemp)
            xtrans.append(x[i] * coeftemp)

        if mom > 2:
            l.append(coefl * sum(xtrans) / l[1])
        else:
            l.append(coefl * sum(xtrans))
    return l
//  [python]
def _samlmularge(x, nmom=5):
    try:
        x = np.asarray(x, dtype=np.float64)
        n = len(x)
        x.sort()
    except ValueError:
        raise ValueError("Input data to estimate L-moments must be numeric.")

    if nmom <= 0:
        raise ValueError("Invalid number of sample L-moments")

    if n < nmom:
        raise ValueError("Insufficient length of data for specified nmoments")

    ##Calculate first order
    l = [np.sum(x) / scipy.special.comb(n, 1, exact=True)]

    if nmom == 1:
        return l[0]

    #Setup comb table, where comb[i][x] refers to comb(x,i)
    comb = []
    for i in range(1, nmom):
        comb.append([])
        for j in range(n):
            comb[-1].append(scipy.special.comb(j, i, exact=True))

    for mom in range(2, nmom + 1):
        coefl = 1.0 / mom * 1.0 / scipy.special.comb(n, mom, exact=True)
        xtrans = []
        for i in range(0, n):
            coeftemp = []
            for j in range(0, mom):
                coeftemp.append(1)

            for j in range(0, mom - 1):
                coeftemp[j] = coeftemp[j] * comb[mom - j - 2][i]

            for j in range(1, mom):
                coeftemp[j] = coeftemp[j] * comb[j - 1][n - i - 1]

            for j in range(0, mom):
                coeftemp[j] = coeftemp[j] * scipy.special.comb(mom - 1, j, exact=True)

            for j in range(0, int(0.5 * mom)):
                coeftemp[j * 2 + 1] = -coeftemp[j * 2 + 1]
            coeftemp = sum(coeftemp)
            xtrans.append(x[i] * coeftemp)

        if mom > 2:
            l.append(coefl * sum(xtrans) / l[1])
        else:
            l.append(coefl * sum(xtrans))
    return l
//  [python]
def sample_maxeval_disjoint_matrix(num_cols: int,
                                   num_rows: int,
                                   n_max_test: int,
                                   d: int,
                                   max_tries=100,
                                   count_fn=count_mean):
  """Samples matrices, returns the most count-disjoint matrix."""
  random_groups = sample_groups_of_size((num_rows, num_cols), n_max_test)
  count = eval_disjoint(random_groups, d, count_fn=count_fn)
  max_count = int(s.comb(num_cols, d))
  attempt = 0
  while attempt < max_tries and count < max_count:
    attempt += 1
    random_groups_iter = sample_groups_of_size((num_rows, num_cols), n_max_test)
    count_iter = eval_disjoint(random_groups, d, count_fn=count_fn)
    if count_iter > count:
      count = count_iter
      random_groups = random_groups_iter
      if count == max_count:
        return random_groups, count
  return random_groups, count
//  [python]
def sample_maxeval_disjoint_matrix(num_cols: int,
                                   num_rows: int,
                                   n_max_test: int,
                                   d: int,
                                   max_tries=100,
                                   count_fn=count_mean):
  """Samples matrices, returns the most count-disjoint matrix."""
  random_groups = sample_groups_of_size((num_rows, num_cols), n_max_test)
  count = eval_disjoint(random_groups, d, count_fn=count_fn)
  max_count = int(scipy.special.comb(num_cols, d))
  attempt = 0
  while attempt < max_tries and count < max_count:
    attempt += 1
    random_groups_iter = sample_groups_of_size((num_rows, num_cols), n_max_test)
    count_iter = eval_disjoint(random_groups, d, count_fn=count_fn)
    if count_iter > count:
      count = count_iter
      random_groups = random_groups_iter
      if count == max_count:
        return random_groups, count
  return random_groups, count
//  [python]
def the_transform(x,Nt = 500,Np = 500):

    N  = x.shape[0] - 1
    thetas = np.linspace(1e-10,np.pi,Nt)
    phis   = np.linspace(0,2*np.pi,Np)
    
    # duplicate the signal
    X = np.kron(np.conj(x),np.ones((Np,Nt,1))).T
    
    # maps of angles 
    Thetas, Phis = np.meshgrid(thetas,phis,indexing='ij')
    
    # map of phase space variable
    z = np.cos(Thetas/2)/np.sin(Thetas/2)*np.exp(1j*Phis)
    Z = np.kron(z,np.ones((N+1,1,1)))
    
    # indices
    ell = np.arange(N+1)
    L = np.kron(ell,np.ones((Np,Nt,1))).T
    
    # compute the transform using the generative function trick
    normZ  = np.sqrt(2*(1+np.abs(Z)**2))
    sumand = X*np.sqrt(s.comb(N,L))*((1-Z)/(normZ))**L*((1+Z)/(normZ))**(N-L)
    Kz = np.sum(sumand,0)
    
    return Kz
//  [python]
def the_transform(x,Nt = 500,Np = 500):

    N  = x.shape[0] - 1
    thetas = np.linspace(1e-10,np.pi,Nt)
    phis   = np.linspace(0,2*np.pi,Np)
    
    # duplicate the signal
    X = np.kron(np.conj(x),np.ones((Np,Nt,1))).T
    
    # maps of angles 
    Thetas, Phis = np.meshgrid(thetas,phis,indexing='ij')
    
    # map of phase space variable
    z = np.cos(Thetas/2)/np.sin(Thetas/2)*np.exp(1j*Phis)
    Z = np.kron(z,np.ones((N+1,1,1)))
    
    # indices
    ell = np.arange(N+1)
    L = np.kron(ell,np.ones((Np,Nt,1))).T
    
    # compute the transform using the generative function trick
    normZ  = np.sqrt(2*(1+np.abs(Z)**2))
    sumand = X*np.sqrt(sps.comb(N,L))*((1-Z)/(normZ))**L*((1+Z)/(normZ))**(N-L)
    Kz = np.sum(sumand,0)
    
    return Kz
//  [python]
def fit(self, X, fX, U0 = None):
		r""" Given samples, fit the polynomial ridge approximation.

		Parameters
		----------
		X : array-like (M, m)
			Input coordinates
		fX : array-like (M,)
			Evaluations of the function at the samples

		"""
		kwargs = self.kwargs

		X = np.array(X)
		fX = np.array(fX).flatten()

		assert X.shape[0] == fX.shape[0], "Dimensions of input do not match"

		# Check if we have enough data to make problem overdetermined
		m = X.shape[1]
		n = self.subspace_dimension
		d = self.degree
		n_param = s.comb(n+d, d)	# Polynomial contribution
		n_param += m*n - (n*(n+1))//2			# Number of parameters in Grassmann manifold
		if len(fX) < n_param:
			mess = "A polynomial ridge approximation of degree %d and subspace dimension %d of a %d-dimensional function " % (d, n, m)
			mess += "requires at least %d samples to not be underdetermined" % (n_param, )
			raise UnderdeterminedException(mess)

		# Special case where solution is convex and no iteration is required
		if self.subspace_dimension == 1 and self.degree == 1:
			self._U = self._fit_affine(X, fX)
			self.coef = self._fit_coef(X, fX, self._U)
			return


		if U0 is not None:
			# Check that U0 has the right shape
			U0 = np.array(U0)
			assert U0.shape[0] == X.shape[1], "U0 has %d rows, expected %d based on X" % (U0.shape[0], X.shape[1])
			assert U0.shape[1] == self.subspace_dimension, "U0 has %d columns; expected %d" % (U0.shape[1], self.subspace_dimension)
		else:
			U0 = initialize_subspace(X = X, fX = fX)[:,:self.subspace_dimension]

		# Orthogonalize just to make sure the starting value satisfies constraints
		U0 = orth(U0)

		# TODO Implement multiple initializations
		if self.norm == 2 and self.bound == None:
			return self._fit_varpro(X, fX, U0, **kwargs)
		else:
			return self._fit_alternating(X, fX, U0, **kwargs)
//  [python]
def fit(self, X, fX, U0 = None):
		r""" Given samples, fit the polynomial ridge approximation.

		Parameters
		----------
		X : array-like (M, m)
			Input coordinates
		fX : array-like (M,)
			Evaluations of the function at the samples

		"""
		kwargs = self.kwargs

		X = np.array(X)
		fX = np.array(fX).flatten()

		assert X.shape[0] == fX.shape[0], "Dimensions of input do not match"

		# Check if we have enough data to make problem overdetermined
		m = X.shape[1]
		n = self.subspace_dimension
		d = self.degree
		n_param = scipy.special.comb(n+d, d)	# Polynomial contribution
		n_param += m*n - (n*(n+1))//2			# Number of parameters in Grassmann manifold
		if len(fX) < n_param:
			mess = "A polynomial ridge approximation of degree %d and subspace dimension %d of a %d-dimensional function " % (d, n, m)
			mess += "requires at least %d samples to not be underdetermined" % (n_param, )
			raise UnderdeterminedException(mess)

		# Special case where solution is convex and no iteration is required
		if self.subspace_dimension == 1 and self.degree == 1:
			self._U = self._fit_affine(X, fX)
			self.coef = self._fit_coef(X, fX, self._U)
			return


		if U0 is not None:
			# Check that U0 has the right shape
			U0 = np.array(U0)
			assert U0.shape[0] == X.shape[1], "U0 has %d rows, expected %d based on X" % (U0.shape[0], X.shape[1])
			assert U0.shape[1] == self.subspace_dimension, "U0 has %d columns; expected %d" % (U0.shape[1], self.subspace_dimension)
		else:
			U0 = initialize_subspace(X = X, fX = fX)[:,:self.subspace_dimension]

		# Orthogonalize just to make sure the starting value satisfies constraints
		U0 = orth(U0)

		# TODO Implement multiple initializations
		if self.norm == 2 and self.bound == None:
			return self._fit_varpro(X, fX, U0, **kwargs)
		else:
			return self._fit_alternating(X, fX, U0, **kwargs)
//  [python]
def hypergeometric(population, population_successes, sample, sample_successes):
    return (s.comb(population_successes, sample_successes) *\
            s.comb(population - population_successes, sample - sample_successes)) /\
            s.comb(population, sample)
//  [python]
def hypergeometric(population, population_successes, sample, sample_successes):
    return (scipy.special.comb(population_successes, sample_successes) *\
            scipy.special.comb(population - population_successes, sample - sample_successes)) /\
            scipy.special.comb(population, sample)
//  [python]
def test_cnr():
    n = np.random.randint(0, 50)
    r = np.random.randint(0, n + 1)
    assert round(NumCpp.cnr(n, r)) == round(s.comb(n, r))
//  [python]
def test_cnr():
    n = np.random.randint(0, 50)
    r = np.random.randint(0, n + 1)
    assert round(NumCpp.cnr(n, r)) == round(sp.comb(n, r))
//  [python]
def _cdf_without_replacement(i,n,N):
    return s.comb(i,n) / s.comb(N,n)
//  [python]
def _cdf_without_replacement(i,n,N):
    return scipy.special.comb(i,n) / scipy.special.comb(N,n)
//  [python]
def num_compositions(m, n):
    """
    The total number of m-part compositions of n, which is equal to
    (n+m-1) choose (m-1).

    Parameters
    ----------
    m : scalar(int)
        Number of parts of composition.

    n : scalar(int)
        Integer to decompose.

    Returns
    -------
    scalar(int)
        Total number of m-part compositions of n.

    """
    # docs.scipy.org/doc/scipy/reference/generated/scipy.special.comb.html
    return s.comb(n+m-1, m-1, exact=True)
//  [python]
def num_compositions(m, n):
    """
    The total number of m-part compositions of n, which is equal to
    (n+m-1) choose (m-1).

    Parameters
    ----------
    m : scalar(int)
        Number of parts of composition.

    n : scalar(int)
        Integer to decompose.

    Returns
    -------
    scalar(int)
        Total number of m-part compositions of n.

    """
    # docs.scipy.org/doc/scipy/reference/generated/scipy.special.comb.html
    return scipy.special.comb(n+m-1, m-1, exact=True)
//  [python]
def calcHistArray(self, pc, norm, indNeigh):
        """override this function with custom Histogram"""
        print("\tCalculating histograms naive method \n")

        N = len(pc)
        histArray = np.zeros((N, self._div**3))
        for i in range(N):
            u = np.asarray(norm[i].T).squeeze()
            k = self._nneighbors
            n = k + 1
            N_features = s.comb(n, 2)
            features = []
            p_list = [i] + indNeigh[i]
            p_list_copy = [i] + indNeigh[i]
            for z in p_list:
                p_list_copy.pop(0)
                for p in p_list_copy:
                    pi = pc[p]
                    pj = pc[z]
                    if np.arccos(np.dot(norm[p], pj - pi)) <= np.arccos(np.dot(norm[z], pi - pj)):
                        ps = pi
                        pt = pj
                        ns = np.asarray(norm[p]).squeeze()
                        nt = np.asarray(norm[z]).squeeze()
                    else:
                        ps = pj
                        pt = pi
                        ns = np.asarray(norm[z]).squeeze()
                        nt = np.asarray(norm[p]).squeeze()

                    u = ns
                    difV = pt - ps
                    dist = np.linalg.norm(difV)
                    difV = difV/dist
                    difV = np.asarray(difV).squeeze()
                    v = np.cross(difV, u)
                    w = np.cross(u, v)

                    alpha = np.dot(v, nt)
                    phi = np.dot(u, difV)
                    theta = np.arctan(np.dot(w, nt) / np.dot(u, nt))

                    features.append(np.array([alpha, phi, theta]))
                
            features = np.asarray(features)
            pfh_hist, bin_edges = self.calc_pfh_hist(features)
            histArray[i, :] = pfh_hist / (N_features)
        return histArray
//  [python]
def calcHistArray(self, pc, norm, indNeigh):
        """override this function with custom Histogram"""
        print("\tCalculating histograms naive method \n")

        N = len(pc)
        histArray = np.zeros((N, self._div**3))
        for i in range(N):
            u = np.asarray(norm[i].T).squeeze()
            k = self._nneighbors
            n = k + 1
            N_features = sp.comb(n, 2)
            features = []
            p_list = [i] + indNeigh[i]
            p_list_copy = [i] + indNeigh[i]
            for z in p_list:
                p_list_copy.pop(0)
                for p in p_list_copy:
                    pi = pc[p]
                    pj = pc[z]
                    if np.arccos(np.dot(norm[p], pj - pi)) <= np.arccos(np.dot(norm[z], pi - pj)):
                        ps = pi
                        pt = pj
                        ns = np.asarray(norm[p]).squeeze()
                        nt = np.asarray(norm[z]).squeeze()
                    else:
                        ps = pj
                        pt = pi
                        ns = np.asarray(norm[z]).squeeze()
                        nt = np.asarray(norm[p]).squeeze()

                    u = ns
                    difV = pt - ps
                    dist = np.linalg.norm(difV)
                    difV = difV/dist
                    difV = np.asarray(difV).squeeze()
                    v = np.cross(difV, u)
                    w = np.cross(u, v)

                    alpha = np.dot(v, nt)
                    phi = np.dot(u, difV)
                    theta = np.arctan(np.dot(w, nt) / np.dot(u, nt))

                    features.append(np.array([alpha, phi, theta]))
                
            features = np.asarray(features)
            pfh_hist, bin_edges = self.calc_pfh_hist(features)
            histArray[i, :] = pfh_hist / (N_features)
        return histArray
//  [python]
def untangle(graph, k_hop, with_data: bool = True, with_call: bool = True, with_name: bool = True):
    seeds, list_of_graphs = deltaPDG_to_list_of_Graphs(graph, khop_k=k_hop)
    wl_subtree = GraphKernel(kernel=[{"name": "weisfeiler_lehman", "n_iter": 10}, {"name": "subtree_wl"}],
                             normalize=True)
    if len(list_of_graphs) > 0:
        similarities = defaultdict(lambda: (0, 0.0))
        for g1, g2 in itertools.combinations(list_of_graphs, 2):
            # The graph has to be converted to {Graph, Node_Labels, Edge_Labels}
            wl_subtree.fit([graph_to_grakel(g1, with_data, with_call, with_name)])
            similarity = wl_subtree.transform([graph_to_grakel(g2, with_data, with_call, with_name)])[0][0]
            similarities[(list_of_graphs.index(g1), list_of_graphs.index(g2))] = similarity

        n = len(list_of_graphs)
        affinity = np.zeros(shape=(scipy.misc.comb(n, 2, exact=True),))
        args = list(enumerate(itertools.combinations(range(n), 2)))
        with ThreadPool(processes=min(os.cpu_count() - 1, 1)) as wp:
            for k, value in wp.imap_unordered(lambda i: (i[0], similarities[(i[-1][0], i[-1][1])]),
                                              args):
                affinity[k] += (1 - value)  # affinity is distance! so (1 - sim)

        cluster = AgglomerativeClustering(n_clusters=None, distance_threshold=0.5,
                                          affinity='precomputed',
                                          linkage='complete')
        if len(affinity) < 2:
            if len(affinity) == 1:
                labels = np.asarray([0, 0]) if affinity[0] <= 0.5 else np.asarray([0, 1])
            else:
                labels = np.asarray([0])
        else:
            labels = cluster.fit_predict(scipy.spatial.distance.squareform(affinity))
    else:
        labels = None

    label = list()
    for node, data in graph.nodes(data=True):
        if 'color' in data.keys():
            i = seeds.index(node) if node in seeds else -1

            if labels is not None and i != -1:
                data['label'] = '%d: ' % labels[i] + data['label']
                label.append(labels[i])
                graph.add_node(node, **data)
            else:
                data['label'] = '-1: ' + data['label']
                label.append(-1)
                graph.add_node(node, **data)

    return graph
//  [python]
def untangle(graph, k_hop, with_data: bool = True, with_call: bool = True, with_name: bool = True):
    seeds, list_of_graphs = deltaPDG_to_list_of_Graphs(graph, khop_k=k_hop)
    wl_subtree = GraphKernel(kernel=[{"name": "weisfeiler_lehman", "n_iter": 10}, {"name": "subtree_wl"}],
                             normalize=True)
    if len(list_of_graphs) > 0:
        similarities = defaultdict(lambda: (0, 0.0))
        for g1, g2 in itertools.combinations(list_of_graphs, 2):
            # The graph has to be converted to {Graph, Node_Labels, Edge_Labels}
            wl_subtree.fit([graph_to_grakel(g1, with_data, with_call, with_name)])
            similarity = wl_subtree.transform([graph_to_grakel(g2, with_data, with_call, with_name)])[0][0]
            similarities[(list_of_graphs.index(g1), list_of_graphs.index(g2))] = similarity

        n = len(list_of_graphs)
        affinity = np.zeros(shape=(scipy.special.comb(n, 2, exact=True),))
        args = list(enumerate(itertools.combinations(range(n), 2)))
        with ThreadPool(processes=min(os.cpu_count() - 1, 1)) as wp:
            for k, value in wp.imap_unordered(lambda i: (i[0], similarities[(i[-1][0], i[-1][1])]),
                                              args):
                affinity[k] += (1 - value)  # affinity is distance! so (1 - sim)

        cluster = AgglomerativeClustering(n_clusters=None, distance_threshold=0.5,
                                          affinity='precomputed',
                                          linkage='complete')
        if len(affinity) < 2:
            if len(affinity) == 1:
                labels = np.asarray([0, 0]) if affinity[0] <= 0.5 else np.asarray([0, 1])
            else:
                labels = np.asarray([0])
        else:
            labels = cluster.fit_predict(scipy.spatial.distance.squareform(affinity))
    else:
        labels = None

    label = list()
    for node, data in graph.nodes(data=True):
        if 'color' in data.keys():
            i = seeds.index(node) if node in seeds else -1

            if labels is not None and i != -1:
                data['label'] = '%d: ' % labels[i] + data['label']
                label.append(labels[i])
                graph.add_node(node, **data)
            else:
                data['label'] = '-1: ' + data['label']
                label.append(-1)
                graph.add_node(node, **data)

    return graph
//  [python]
def fit(self, X, fX, U0 = None):
		r""" Given samples, fit the polynomial ridge approximation.

		Parameters
		----------
		X : array-like (M, m)
			Input coordinates
		fX : array-like (M,)
			Evaluations of the function at the samples
		
		"""
		kwargs = self.kwargs

		X = np.array(X)
		fX = np.array(fX).flatten()	

		assert X.shape[0] == fX.shape[0], "Dimensions of input do not match"

		# Check if we have enough data to make problem overdetermined
		m = X.shape[1]
		n = self.subspace_dimension
		d = self.degree
		n_param = s.comb(n+d, d)	# Polynomial contribution
		n_param += m*n - (n*(n+1))//2			# Number of parameters in Grassmann manifold
		if len(fX) < n_param:
			mess = "A polynomial ridge approximation of degree %d and subspace dimension %d of a %d-dimensional function " % (d, n, m)
			mess += "requires at least %d samples to not be underdetermined" % (n_param, )
			raise UnderdeterminedException(mess) 	

		# Special case where solution is convex and no iteration is required
		if self.subspace_dimension == 1 and self.degree == 1:
			self._U = self._fit_affine(X, fX)	
			self.coef = self._fit_coef(X, fX, self._U)	
			return 


		if U0 is not None:
			# Check that U0 has the right shape
			U0 = np.array(U0)
			assert U0.shape[0] == X.shape[1], "U0 has %d rows, expected %d based on X" % (U0.shape[0], X.shape[1])
			assert U0.shape[1] == self.subspace_dimension, "U0 has %d columns; expected %d" % (U0.shape[1], self.subspace_dimension)
		else:
			U0 = initialize_subspace(X = X, fX = fX)[:,:self.subspace_dimension]
			
		# Orthogonalize just to make sure the starting value satisfies constraints	
		U0 = orth(U0)
			
		# TODO Implement multiple initializations
		if self.norm == 2 and self.bound == None:
			return self._fit_varpro(X, fX, U0, **kwargs)
		else:	
			return self._fit_alternating(X, fX, U0, **kwargs)
//  [python]
def fit(self, X, fX, U0 = None):
		r""" Given samples, fit the polynomial ridge approximation.

		Parameters
		----------
		X : array-like (M, m)
			Input coordinates
		fX : array-like (M,)
			Evaluations of the function at the samples
		
		"""
		kwargs = self.kwargs

		X = np.array(X)
		fX = np.array(fX).flatten()	

		assert X.shape[0] == fX.shape[0], "Dimensions of input do not match"

		# Check if we have enough data to make problem overdetermined
		m = X.shape[1]
		n = self.subspace_dimension
		d = self.degree
		n_param = scipy.special.comb(n+d, d)	# Polynomial contribution
		n_param += m*n - (n*(n+1))//2			# Number of parameters in Grassmann manifold
		if len(fX) < n_param:
			mess = "A polynomial ridge approximation of degree %d and subspace dimension %d of a %d-dimensional function " % (d, n, m)
			mess += "requires at least %d samples to not be underdetermined" % (n_param, )
			raise UnderdeterminedException(mess) 	

		# Special case where solution is convex and no iteration is required
		if self.subspace_dimension == 1 and self.degree == 1:
			self._U = self._fit_affine(X, fX)	
			self.coef = self._fit_coef(X, fX, self._U)	
			return 


		if U0 is not None:
			# Check that U0 has the right shape
			U0 = np.array(U0)
			assert U0.shape[0] == X.shape[1], "U0 has %d rows, expected %d based on X" % (U0.shape[0], X.shape[1])
			assert U0.shape[1] == self.subspace_dimension, "U0 has %d columns; expected %d" % (U0.shape[1], self.subspace_dimension)
		else:
			U0 = initialize_subspace(X = X, fX = fX)[:,:self.subspace_dimension]
			
		# Orthogonalize just to make sure the starting value satisfies constraints	
		U0 = orth(U0)
			
		# TODO Implement multiple initializations
		if self.norm == 2 and self.bound == None:
			return self._fit_varpro(X, fX, U0, **kwargs)
		else:	
			return self._fit_alternating(X, fX, U0, **kwargs)
//  [python]
def _solve_node(self, curr_node, k, dual_, upper_bound, tighten_bound_via_ADMM=True):
        """Solve the perspective relaxation of the node and update the dual bound

        Args:
            curr_node (CustomClass): current node to be solved
            k (int): cardinality constraint
            dual_ (list): list of minimum dual values at each depth of the tree
            upper_bound (float): current loss of the best solution found for this node
            tighten_bound_via_ADMM (bool, optional): whether to refine the dual bound computed by the Fast Solve method. Defaults to True.

        Returns:
            float: dual value of the perspective relaxation of the current node
        """
        self.number_of_nodes += 1
        total_num_bruteForce = s.comb(
            len(curr_node.unfixed_support_on_allowed_support),
            k - len(curr_node.fixed_support_on_allowed_support),
        )
        if self.useBruteForce and (total_num_bruteForce < self.bruteForceThreshold):
            curr_dual = curr_node.lower_solve_brute_force(
                k, self.upper_solver_with_cache
            )
            # print("lower bound by brute force method is", curr_dual)
        else:
            curr_dual = curr_node.lower_solve_fast(k, upper_bound)
            # print("lower bound by fast method is", curr_dual)

            if tighten_bound_via_ADMM and (curr_dual < upper_bound):
                # curr_dual = curr_node.lower_solve_improve(k, upper_bound)
                curr_dual = curr_node.lower_solve_admm(k, upper_bound)
                # print("lower bound by admm method is", curr_dual)

        dual_[curr_node.level] = min(curr_dual, dual_.get(curr_node.level, sys.maxsize))
        self.levels[curr_node.level] -= 1
        return curr_dual
//  [python]
def _solve_node(self, curr_node, k, dual_, upper_bound, tighten_bound_via_ADMM=True):
        """Solve the perspective relaxation of the node and update the dual bound

        Args:
            curr_node (CustomClass): current node to be solved
            k (int): cardinality constraint
            dual_ (list): list of minimum dual values at each depth of the tree
            upper_bound (float): current loss of the best solution found for this node
            tighten_bound_via_ADMM (bool, optional): whether to refine the dual bound computed by the Fast Solve method. Defaults to True.

        Returns:
            float: dual value of the perspective relaxation of the current node
        """
        self.number_of_nodes += 1
        total_num_bruteForce = scipy.special.comb(
            len(curr_node.unfixed_support_on_allowed_support),
            k - len(curr_node.fixed_support_on_allowed_support),
        )
        if self.useBruteForce and (total_num_bruteForce < self.bruteForceThreshold):
            curr_dual = curr_node.lower_solve_brute_force(
                k, self.upper_solver_with_cache
            )
            # print("lower bound by brute force method is", curr_dual)
        else:
            curr_dual = curr_node.lower_solve_fast(k, upper_bound)
            # print("lower bound by fast method is", curr_dual)

            if tighten_bound_via_ADMM and (curr_dual < upper_bound):
                # curr_dual = curr_node.lower_solve_improve(k, upper_bound)
                curr_dual = curr_node.lower_solve_admm(k, upper_bound)
                # print("lower bound by admm method is", curr_dual)

        dual_[curr_node.level] = min(curr_dual, dual_.get(curr_node.level, sys.maxsize))
        self.levels[curr_node.level] -= 1
        return curr_dual
//  [python]
comb = s.comb(64, 32)

N = 37*37
x_values = np.arange(N)
x_values = x_values[2:]
y_values = [s.comb(x, 2) for x in x_values]
//  [python]
comb = scipy.special.comb(64, 32)

N = 37*37
x_values = np.arange(N)
x_values = x_values[2:]
y_values = [scipy.special.comb(x, 2) for x in x_values]
//  [python]
def binomial_expansion(l, x, max_lim=None):

    if max_lim is None:
        max_lim = np.max(l)+1

    vector_x = np.zeros((max_lim, max_lim, max_lim))
    vector_y = np.zeros((max_lim, max_lim, max_lim))
    vector_z = np.zeros((max_lim, max_lim, max_lim))

    for k in range(l[0]+1):
        vector_x[k, 0, 0] += s.comb(l[0], k) * x[0]**(l[0]-k)
    for k in range(l[1] + 1):
        vector_y[0, k, 0] += s.comb(l[1], k) * x[1]**(l[1]-k)
    for k in range(l[2] + 1):
        vector_z[0, 0, k] += s.comb(l[2], k) * x[2]**(l[2]-k)

    vector = product_poly_coeff(product_poly_coeff(vector_x, vector_y), vector_z, max_lim)
    return vector
//  [python]
def binomial_expansion(l, x, max_lim=None):

    if max_lim is None:
        max_lim = np.max(l)+1

    vector_x = np.zeros((max_lim, max_lim, max_lim))
    vector_y = np.zeros((max_lim, max_lim, max_lim))
    vector_z = np.zeros((max_lim, max_lim, max_lim))

    for k in range(l[0]+1):
        vector_x[k, 0, 0] += comb(l[0], k) * x[0]**(l[0]-k)
    for k in range(l[1] + 1):
        vector_y[0, k, 0] += comb(l[1], k) * x[1]**(l[1]-k)
    for k in range(l[2] + 1):
        vector_z[0, 0, k] += comb(l[2], k) * x[2]**(l[2]-k)

    vector = product_poly_coeff(product_poly_coeff(vector_x, vector_y), vector_z, max_lim)
    return vector
//  [python]
def integrate_exponential_py(n, a, b):
    """
    integrals type  x^n exp(-(ax^2+bx))

    """
    if n == 0:
        return np.sqrt(np.pi/a)*np.exp(b**2/(4*a))
    elif n == 1:
        return np.sqrt(np.pi)/(2*a**(3/2))*b*np.exp(b**2/(4*a))
    else:
        factor = np.sum([s(n, 2*k)*(b/(2*a))**(n-2*k)*math.factorial(2*k)/(2**(2*k)*math.factorial(k)*a**k)
                          for k in range(n//2+1)])
        return factor * np.sqrt(np.pi/a)*np.exp(b**2/(4*a))
//  [python]
def integrate_exponential_py(n, a, b):
    """
    integrals type  x^n exp(-(ax^2+bx))

    """
    if n == 0:
        return np.sqrt(np.pi/a)*np.exp(b**2/(4*a))
    elif n == 1:
        return np.sqrt(np.pi)/(2*a**(3/2))*b*np.exp(b**2/(4*a))
    else:
        factor = np.sum([comb(n, 2*k)*(b/(2*a))**(n-2*k)*math.factorial(2*k)/(2**(2*k)*math.factorial(k)*a**k)
                          for k in range(n//2+1)])
        return factor * np.sqrt(np.pi/a)*np.exp(b**2/(4*a))
//  [python]
def integrate_exponential_sep_py(n, a, b):
    """
    norm for integrals type  x^n exp(-(ax^2+bx))
    separated in prefactor and exponential

    """
    if n == 0:
        return np.sqrt(np.pi/a), b**2/(4*a)
    elif n == 1:
        return np.sqrt(np.pi)/(2*a**(3/2))*b, b**2/(4*a)
    else:
        factor = np.sum([s(n, 2*k)*(b/(2*a))**(n-2*k)*math.factorial(2*k)/(2**(2*k)*math.factorial(k)*a**k)
                          for k in range(n//2+1)])
        return factor * np.sqrt(np.pi/a), b**2/(4*a)
//  [python]
def integrate_exponential_sep_py(n, a, b):
    """
    norm for integrals type  x^n exp(-(ax^2+bx))
    separated in prefactor and exponential

    """
    if n == 0:
        return np.sqrt(np.pi/a), b**2/(4*a)
    elif n == 1:
        return np.sqrt(np.pi)/(2*a**(3/2))*b, b**2/(4*a)
    else:
        factor = np.sum([comb(n, 2*k)*(b/(2*a))**(n-2*k)*math.factorial(2*k)/(2**(2*k)*math.factorial(k)*a**k)
                          for k in range(n//2+1)])
        return factor * np.sqrt(np.pi/a), b**2/(4*a)
//  [python]
def hypergeometric_distribution(plotname):
    fig_height, fig_size, fig_width = get_large_figsize()

    pdf_pages = PdfPages('./plots_output/%s' % plotname)
    params, fig_size = get_plt_params()

    plt.rcParams.update(params)
    matplotlib.rc('font', **{'size': 14, 'family': 'serif', 'serif': ['Computer Modern']})
    matplotlib.rc('text', usetex=True)

    plt.axes([0.12, 0.32, 0.85, 0.63], frameon=True)
    plt.rc('pdf', fonttype=42)  # IMPORTANT to get rid of Type 3

    colors, linestyles = get_colorful_styles()
    f, ax1 = plt.subplots()

    def hypergeom_calc(x, frac, total):
        top = s.comb(total - int(total * frac), x)
        bottom = s.comb(total, x)
        return 1.0 - (top / bottom)

    total_number_of_weights = 20000
    fractions = [0.1, 0.25, 0.5, 0.75]
    x_values = list(range(1, 101))
    perc = '%'

    for i, f in enumerate(fractions):
        y_values = [hypergeom_calc(x, f, total_number_of_weights) for x in x_values]
        x_values_perc = [float(x) / float(total_number_of_weights) for x in x_values]
        label = f"{(f * 100.0):.0f}\\%"
        plt.plot(x_values_perc, y_values, label=label, color=colors[i], linewidth=2)
    # plt.boxplot(benign)
    # plt.plot(benign_avg, label="Benign (avg)", color=colors[0], linestyle=linestyles[1], linewidth=2)
    # plt.plot(mal, label="Malicious", color=colors[1], linestyle=linestyles[1], linewidth=2)

    plt.xlabel(f'Parameters outside range (total weights = {total_number_of_weights})')
    ax1.xaxis.set_major_formatter(ticker.PercentFormatter())
    # ax1.yaxis.set_major_formatter(ticker.FuncFormatter(lambda y, pos: '%.0fk' % (y * 1e-3)))

    plt.ylabel("Detection probability")
    # plt.yscale("log")
    leg = plt.legend(bbox_to_anchor=(-0.016, 1.00, 1., .102), loc=3, ncol=4, columnspacing=0.75,
                     title="Percentage of parameters verified")
    leg._legend_box.align = "left"

    plt.grid(True, linestyle=':', color='0.8', zorder=0)
    F = plt.gcf()
    F.set_size_inches(fig_size)
    pdf_pages.savefig(F, bbox_inches='tight')
    plt.clf()
    pdf_pages.close()
//  [python]
def hypergeometric_distribution(plotname):
    fig_height, fig_size, fig_width = get_large_figsize()

    pdf_pages = PdfPages('./plots_output/%s' % plotname)
    params, fig_size = get_plt_params()

    plt.rcParams.update(params)
    matplotlib.rc('font', **{'size': 14, 'family': 'serif', 'serif': ['Computer Modern']})
    matplotlib.rc('text', usetex=True)

    plt.axes([0.12, 0.32, 0.85, 0.63], frameon=True)
    plt.rc('pdf', fonttype=42)  # IMPORTANT to get rid of Type 3

    colors, linestyles = get_colorful_styles()
    f, ax1 = plt.subplots()

    def hypergeom_calc(x, frac, total):
        top = scipy.special.comb(total - int(total * frac), x)
        bottom = scipy.special.comb(total, x)
        return 1.0 - (top / bottom)

    total_number_of_weights = 20000
    fractions = [0.1, 0.25, 0.5, 0.75]
    x_values = list(range(1, 101))
    perc = '%'

    for i, f in enumerate(fractions):
        y_values = [hypergeom_calc(x, f, total_number_of_weights) for x in x_values]
        x_values_perc = [float(x) / float(total_number_of_weights) for x in x_values]
        label = f"{(f * 100.0):.0f}\\%"
        plt.plot(x_values_perc, y_values, label=label, color=colors[i], linewidth=2)
    # plt.boxplot(benign)
    # plt.plot(benign_avg, label="Benign (avg)", color=colors[0], linestyle=linestyles[1], linewidth=2)
    # plt.plot(mal, label="Malicious", color=colors[1], linestyle=linestyles[1], linewidth=2)

    plt.xlabel(f'Parameters outside range (total weights = {total_number_of_weights})')
    ax1.xaxis.set_major_formatter(ticker.PercentFormatter())
    # ax1.yaxis.set_major_formatter(ticker.FuncFormatter(lambda y, pos: '%.0fk' % (y * 1e-3)))

    plt.ylabel("Detection probability")
    # plt.yscale("log")
    leg = plt.legend(bbox_to_anchor=(-0.016, 1.00, 1., .102), loc=3, ncol=4, columnspacing=0.75,
                     title="Percentage of parameters verified")
    leg._legend_box.align = "left"

    plt.grid(True, linestyle=':', color='0.8', zorder=0)
    F = plt.gcf()
    F.set_size_inches(fig_size)
    pdf_pages.savefig(F, bbox_inches='tight')
    plt.clf()
    pdf_pages.close()
//  [python]
def __init__(self, annotators, documents, labels, eval_func=exact_match_instance_evaluation, token_func=None):
        assert len(annotators) > 1, 'At least two annotators are necessary to compute agreement!'
        num_pairs = s.comb(len(annotators), 2, exact=True)
        # (p, d, c, l) where p := annotator pairs, d := documents, c := counts (tp, fp, fn), l := labels
        self._pdcl = np.zeros((num_pairs, len(documents), 3, len(labels)))
        self._documents = list(documents)
        self._doc2idx = {d: i for i, d in enumerate(documents)}
        self._labels = list(labels)
        self._label2idx = {l: i for i, l in enumerate(labels)}
        self._annotators = list(annotators)
        self._pairs = [pair for pair in combinations(annotators, 2)]
        self._pair2idx = {p: i for i, p in enumerate(self._pairs)}
        # add pairs in reverse order (same index)
        for (a1, a2), value in self._pair2idx.copy().items():
            self._pair2idx[(a2, a1)] = value
        self._eval_func = eval_func  # function used to extract true positives, false positives and false negatives
        self._token_func = token_func  # function used for tokenization
        self._compute_tp_fp_fn(documents)