jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def normalize_latent(self, log_pygx_unnorm):
        """Normalize the latent variable distribution

        For each sample in the training set, we estimate a probability distribution
        over y_j, each hidden factor. Here we normalize it. (Eq. 7 in paper.)
        This normalization factor is used for estimating TC.

        Parameters
        ----------
        Unnormalized distribution of hidden factors for each training sample.

        Returns
        -------
        p_y_given_x : 3D array, shape (n_hidden, n_samples)
            p(y_j|x^l), the probability distribution over all hidden factors,
            for data samples l = 1...n_samples
        log_z : 2D array, shape (n_hidden, n_samples)
            Point-wise estimate of total correlation explained by each Y_j for each sample,
            used to estimate overall total correlation.

        """
        with np.errstate(under='ignore'):
            log_z = scipy.special.logsumexp(log_pygx_unnorm, axis=0)  # Essential to maintain precision.
            log_pygx = log_pygx_unnorm[1] - log_z
            p_norm = np.exp(log_pygx)
        return p_norm.clip(1e-6, 1 - 1e-6), log_pygx, log_z
//  [python]
def log_likelihood(self, x, mu):
        """ Calculate negative binomial mixture read count log likelihood.
        
        Args:
            x (numpy.array): observed read counts
            mu (numpy.array): expected read counts
        
        Returns:
            float: log likelihood per segment
            
        The pmf of the negative binomial mixture is:

            (1 - z) * NBin(x, mu) + z * NBin(x, mu)

        """
        
        ll = np.array([
            np.log(1. - self.z) + self.negbin.log_likelihood(x, mu),
            np.log(self.z) + self.negbin_noise.log_likelihood(x, mu),
        ])

        ll = scipy.misc.logsumexp(ll, axis=0)

        return ll
//  [python]
def log_likelihood(self, x, mu):
        """ Calculate negative binomial mixture read count log likelihood.
        
        Args:
            x (numpy.array): observed read counts
            mu (numpy.array): expected read counts
        
        Returns:
            float: log likelihood per segment
            
        The pmf of the negative binomial mixture is:

            (1 - z) * NBin(x, mu) + z * NBin(x, mu)

        """
        
        ll = np.array([
            np.log(1. - self.z) + self.negbin.log_likelihood(x, mu),
            np.log(self.z) + self.negbin_noise.log_likelihood(x, mu),
        ])

        ll = scipy.special.logsumexp(ll, axis=0)

        return ll
//  [python]
def _logcdf(self, k, M, n, N):
        res = []
        for quant, tot, good, draw in zip(k, M, n, N):
            if (quant + 0.5) * (tot + 0.5) > (good - 0.5) * (draw - 0.5):
                # Less terms to sum if we calculate log(1-sf)
                res.append(log1p(-exp(self.logsf(quant, tot, good, draw))))
            else:
                # Integration over probability mass function using logsumexp
                k2 = np.arange(0, quant + 1)
                res.append(logsumexp(self._logpmf(k2, tot, good, draw)))
        return np.asarray(res)
//  [python]
def _logsf(self, k, M, n, N):
        res = []
        for quant, tot, good, draw in zip(k, M, n, N):
            if (quant + 0.5) * (tot + 0.5) < (good - 0.5) * (draw - 0.5):
                # Less terms to sum if we calculate log(1-cdf)
                res.append(log1p(-exp(self.logcdf(quant, tot, good, draw))))
            else:
                # Integration over probability mass function using logsumexp
                k2 = np.arange(quant + 1, draw + 1)
                res.append(logsumexp(self._logpmf(k2, tot, good, draw)))
        return np.asarray(res)
//  [python]
def rescale_DPD_run_length_distribution(self, log_scale_new,
                                        rescaler_for_old_obs, t):
        """Rescales joint log probs"""
        if rescaler_for_old_obs is None:
            self.joint_log_probabilities = (self.joint_log_probabilities
                                            - log_scale_new)
        else:
            self.joint_log_probabilities[0] = (self.joint_log_probabilities[0]
                    - log_scale_new)
            self.joint_log_probabilities[1:] = (
                    self.joint_log_probabilities[1:] + rescaler_for_old_obs)
        self.model_log_evidence = scipy.misc.logsumexp(
                self.joint_log_probabilities)
//  [python]
def rescale_DPD_run_length_distribution(self, log_scale_new,
                                        rescaler_for_old_obs, t):
        """Rescales joint log probs"""
        if rescaler_for_old_obs is None:
            self.joint_log_probabilities = (self.joint_log_probabilities
                                            - log_scale_new)
        else:
            self.joint_log_probabilities[0] = (self.joint_log_probabilities[0]
                    - log_scale_new)
            self.joint_log_probabilities[1:] = (
                    self.joint_log_probabilities[1:] + rescaler_for_old_obs)
        self.model_log_evidence = scipy.special.logsumexp(
                self.joint_log_probabilities)
//  [python]
def log_likelihood(self, k, n, p):
        """ Calculate beta binomial / uniform allele count log likelihood.
        
        Args:
            k (numpy.array): observed minor allelic read counts
            n (numpy.array): observed total allelic read counts
            p (numpy.array): expected minor allele fraction
        
        Returns:
            float: log likelihood per segment
            
        The pmf of the beta binomial / uniform mixture is:

            (1 - z) * BB(k, n, p) + z * (1 / (n + 1))

        """

        ll = np.array([
            np.log(1. - self.z) + self.betabin.log_likelihood(k, n, p),
            np.log(self.z) - np.log(n + 1.)
        ])

        ll = scipy.misc.logsumexp(ll, axis=0)

        return ll
//  [python]
def log_likelihood(self, k, n, p):
        """ Calculate beta binomial / uniform allele count log likelihood.
        
        Args:
            k (numpy.array): observed minor allelic read counts
            n (numpy.array): observed total allelic read counts
            p (numpy.array): expected minor allele fraction
        
        Returns:
            float: log likelihood per segment
            
        The pmf of the beta binomial / uniform mixture is:

            (1 - z) * BB(k, n, p) + z * (1 / (n + 1))

        """

        ll = np.array([
            np.log(1. - self.z) + self.betabin.log_likelihood(k, n, p),
            np.log(self.z) - np.log(n + 1.)
        ])

        ll = scipy.special.logsumexp(ll, axis=0)

        return ll
//  [python]
def logsumexp(self, a, axis=None):
        return s.logsumexp(a, axis=axis)
//  [python]
def logsumexp(self, a, axis=None):
        return special.logsumexp(a, axis=axis)
//  [python]
def _log_sum(log_p, log_q):
    return s.logsumexp([log_p, log_q], axis=0)
//  [python]
def _log_sum(log_p, log_q):
    return sc.logsumexp([log_p, log_q], axis=0)
//  [python]
def _log_diff(log_p, log_q):
    return s.logsumexp([log_p, log_q+np.pi*1j], axis=0)
//  [python]
def _log_diff(log_p, log_q):
    return sc.logsumexp([log_p, log_q+np.pi*1j], axis=0)
//  [python]
def marginal_log_densities(jointProb):
    return utils.mrow(scipy.special.logsumexp(jointProb, axis=0))
//  [python]
def amplify_rdps(unamplified_rdps, N, K, m, orders, steps):
  # m: batch_size, N: total number of samples, K: neighborhood size
  rv = hypergeom(N, K, m)
  log_densities = [rv.logpmf(i) for i in np.arange(K + 1)]

  amplified_rdps = []
  for order, unamplified_rdp in zip(orders, unamplified_rdps):
    beta = unamplified_rdp * (order - 1)
    log_fs = beta * (np.square(np.arange(K + 1) / K))
    amplified_rdp = np.logaddexp.reduce(log_densities + log_fs) / (order - 1)
    amplified_rdps.append(amplified_rdp)
  amplified_rdps = np.asarray(amplified_rdps)
  amplified_rdps_total = amplified_rdps * steps
  return amplified_rdps_total
//  [python]
def amplify_rdps(unamplified_rdps, N, K, m, orders, steps):
  # m: batch_size, N: total number of samples, K: neighborhood size
  rv = hypergeom(N, K, m)
  log_densities = [rv.logpmf(i) for i in np.arange(K + 1)]

  amplified_rdps = []
  for order, unamplified_rdp in zip(orders, unamplified_rdps):
    beta = unamplified_rdp * (order - 1)
    log_fs = beta * (np.square(np.arange(K + 1) / K))
    amplified_rdp = special.logsumexp(log_densities + log_fs) / (order - 1)
    amplified_rdps.append(amplified_rdp)
  amplified_rdps = np.asarray(amplified_rdps)
  amplified_rdps_total = amplified_rdps * steps
  return amplified_rdps_total
//  [python]
def PACTranDirichlet(prob_np_all, label_np_all, alpha):
    """Compute the PACTran-Dirichlet estimator."""
    label_np_all = one_hot(label_np_all)  # [n, v]
    soft_targets_sum = np.sum(label_np_all, axis=0)  # [v]
    soft_targets_sum = np.expand_dims(soft_targets_sum, axis=1)  # [v, 1]
    a0 = alpha * soft_targets_sum / np.sum(soft_targets_sum) + 1e-10

    # initialize
    qz = prob_np_all  # [n, d]
    log_s = np.log(prob_np_all + 1e-10)  # [n, d]

    for _ in range(10):
        aw = a0 + np.sum(np.einsum("BY,BZ->BYZ", label_np_all, qz), axis=0)
        logits_qz = (log_s +
                    np.matmul(label_np_all, scipy.special.digamma(aw)) -
                    np.reshape(scipy.special.digamma(np.sum(aw, axis=0)), [1, -1]))
        log_qz = logits_qz - np.max(logits_qz, axis=-1, keepdims=True)
        log_qz = log_qz - np.log(np.sum(np.exp(log_qz), axis=-1, keepdims=True))
        qz = np.exp(log_qz)

    log_c0 = scipy.special.loggamma(np.sum(a0)) - np.sum(
        scipy.special.loggamma(a0))
    log_c = scipy.special.loggamma(np.sum(aw, axis=0)) - np.sum(
        scipy.special.loggamma(aw), axis=0)

    pac_dir = np.sum(
        log_c0 - log_c - np.sum(qz * (log_qz - log_s), axis=0))
    pac_dir = -pac_dir / label_np_all.size
    return pac_dir
//  [python]
def PACTranDirichlet(prob_np_all, label_np_all, alpha):
    """Compute the PACTran-Dirichlet estimator."""
    label_np_all = one_hot(label_np_all)  # [n, v]
    soft_targets_sum = np.sum(label_np_all, axis=0)  # [v]
    soft_targets_sum = np.expand_dims(soft_targets_sum, axis=1)  # [v, 1]
    a0 = alpha * soft_targets_sum / np.sum(soft_targets_sum) + 1e-10

    # initialize
    qz = prob_np_all  # [n, d]
    log_s = np.log(prob_np_all + 1e-10)  # [n, d]

    for _ in range(10):
        aw = a0 + np.sum(np.einsum("BY,BZ->BYZ", label_np_all, qz), axis=0)
        logits_qz = (log_s +
                    np.matmul(label_np_all, scipy.special.digamma(aw)) -
                    np.reshape(scipy.special.digamma(np.sum(aw, axis=0)), [1, -1]))
        log_qz = logits_qz - scipy.special.logsumexp(
            logits_qz, axis=-1, keepdims=True)
        qz = np.exp(log_qz)

    log_c0 = scipy.special.loggamma(np.sum(a0)) - np.sum(
        scipy.special.loggamma(a0))
    log_c = scipy.special.loggamma(np.sum(aw, axis=0)) - np.sum(
        scipy.special.loggamma(aw), axis=0)

    pac_dir = np.sum(
        log_c0 - log_c - np.sum(qz * (log_qz - log_s), axis=0))
    pac_dir = -pac_dir / label_np_all.size
    return pac_dir
//  [python]
def _logsf(self, k, M, n, N):
        res = []
        for quant, tot, good, draw in zip(*np.broadcast_arrays(k, M, n, N)):
            if (quant + 0.5) * (tot + 0.5) < (good - 0.5) * (draw - 0.5):
                # Less terms to sum if we calculate log(1-cdf)
                res.append(log1p(-exp(self.logcdf(quant, tot, good, draw))))
            else:
                # Integration over probability mass function using logsumexp
                k2 = np.arange(quant + 1, draw + 1)
                res.append(s.logsumexp(self._logpmf(k2, tot, good, draw)))
        return np.asarray(res)
//  [python]
def _logsf(self, k, M, n, N):
        res = []
        for quant, tot, good, draw in zip(*np.broadcast_arrays(k, M, n, N)):
            if (quant + 0.5) * (tot + 0.5) < (good - 0.5) * (draw - 0.5):
                # Less terms to sum if we calculate log(1-cdf)
                res.append(log1p(-exp(self.logcdf(quant, tot, good, draw))))
            else:
                # Integration over probability mass function using logsumexp
                k2 = np.arange(quant + 1, draw + 1)
                res.append(logsumexp(self._logpmf(k2, tot, good, draw)))
        return np.asarray(res)
//  [python]
def _logcdf(self, k, M, n, N):
        res = []
        for quant, tot, good, draw in zip(*np.broadcast_arrays(k, M, n, N)):
            if (quant + 0.5) * (tot + 0.5) > (good - 0.5) * (draw - 0.5):
                # Less terms to sum if we calculate log(1-sf)
                res.append(log1p(-exp(self.logsf(quant, tot, good, draw))))
            else:
                # Integration over probability mass function using logsumexp
                k2 = np.arange(0, quant + 1)
                res.append(s(self._logpmf(k2, tot, good, draw)))
        return np.asarray(res)
//  [python]
def _logcdf(self, k, M, n, N):
        res = []
        for quant, tot, good, draw in zip(*np.broadcast_arrays(k, M, n, N)):
            if (quant + 0.5) * (tot + 0.5) > (good - 0.5) * (draw - 0.5):
                # Less terms to sum if we calculate log(1-sf)
                res.append(log1p(-exp(self.logsf(quant, tot, good, draw))))
            else:
                # Integration over probability mass function using logsumexp
                k2 = np.arange(0, quant + 1)
                res.append(logsumexp(self._logpmf(k2, tot, good, draw)))
        return np.asarray(res)
//  [python]
def merge_logprobs_topk_mean(
    private_next_logprobs: list[dict[int, float]],
    public_next_logprobs: dict[int, float],
    n_vocab: int,
    no_public_token: bool,
    normalize_max: bool,
) -> np.ndarray:
    # Compute merged distribution
    # logsumexp - np.log(...): compute mean probability of distribution
    if normalize_max:
        normalize_func = (
            log_max_normalize  # normalize max probability to 1, Exponential mechanism
        )
    else:
        normalize_func = (
            log_normalize  # normalize sum probability to 1, Gaussian mechanism
        )
    if no_public_token:
        merged_next_logprobs = np.sum(
            np.stack(
                [
                    # Turn into a 1D tensor of size n_vocab
                    densify(
                        n_vocab,
                        # Normalize distribution
                        normalize_func(
                            # Filter to the top 100 most likely next tokens according to the public prompt
                            {k: v for k, v in lps.items()}
                        ),
                    )
                    for lps in private_next_logprobs
                ]
            ),
            axis=0,
        ) - np.log(len(private_next_logprobs))

    else:
        merged_next_logprobs = np.sum(
            np.stack(
                [
                    # Turn into a 1D tensor of size n_vocab
                    densify(
                        n_vocab,
                        # Normalize distribution
                        normalize_func(
                            # Filter to the top 100 most likely next tokens according to the public prompt
                            {k: v for k, v in lps.items() if k in public_next_logprobs}
                        ),
                    )
                    for lps in private_next_logprobs
                ]
            ),
            axis=0,
        ) - np.log(len(private_next_logprobs))
    merged_next_probs = np.exp(merged_next_logprobs)
    return merged_next_probs
//  [python]
def merge_logprobs_topk_mean(
    private_next_logprobs: list[dict[int, float]],
    public_next_logprobs: dict[int, float],
    n_vocab: int,
    no_public_token: bool,
    normalize_max: bool,
) -> np.ndarray:
    # Compute merged distribution
    # logsumexp - np.log(...): compute mean probability of distribution
    if normalize_max:
        normalize_func = (
            log_max_normalize  # normalize max probability to 1, Exponential mechanism
        )
    else:
        normalize_func = (
            log_normalize  # normalize sum probability to 1, Gaussian mechanism
        )
    if no_public_token:
        merged_next_logprobs = scipy.special.logsumexp(
            np.stack(
                [
                    # Turn into a 1D tensor of size n_vocab
                    densify(
                        n_vocab,
                        # Normalize distribution
                        normalize_func(
                            # Filter to the top 100 most likely next tokens according to the public prompt
                            {k: v for k, v in lps.items()}
                        ),
                    )
                    for lps in private_next_logprobs
                ]
            ),
            axis=0,
        ) - np.log(len(private_next_logprobs))

    else:
        merged_next_logprobs = scipy.special.logsumexp(
            np.stack(
                [
                    # Turn into a 1D tensor of size n_vocab
                    densify(
                        n_vocab,
                        # Normalize distribution
                        normalize_func(
                            # Filter to the top 100 most likely next tokens according to the public prompt
                            {k: v for k, v in lps.items() if k in public_next_logprobs}
                        ),
                    )
                    for lps in private_next_logprobs
                ]
            ),
            axis=0,
        ) - np.log(len(private_next_logprobs))
    merged_next_probs = np.exp(merged_next_logprobs)
    return merged_next_probs
//  [python]
def PACTranGamma(prob_np_all, label_np_all, alpha):
    """Compute the PAC-Gamma estimator."""
    label_np_all = one_hot(label_np_all)  # [n, v]
    soft_targets_sum = np.sum(label_np_all, axis=0)  # [v]
    soft_targets_sum = np.expand_dims(soft_targets_sum, axis=1)  # [v, 1]

    a0 = alpha * soft_targets_sum / np.sum(soft_targets_sum) + 1e-10
    beta = 1.

    # initialize
    qz = prob_np_all  # [n, d]
    s = prob_np_all  # [n, d]
    log_s = np.log(prob_np_all + 1e-10)  # [n, d]
    aw = a0
    bw = beta
    lw = np.sum(s, axis=-1, keepdims=True) * np.sum(aw / bw)  # [n, 1]

    for _ in range(10):
        aw = a0 + np.sum(np.einsum("BY,BZ->BYZ", label_np_all, qz),
                        axis=0)  # [v, d]
        lw = np.matmul(
            s, np.expand_dims(np.sum(aw / bw, axis=0), axis=1))  # [n, 1]
        logits_qz = (
            log_s + np.matmul(label_np_all, scipy.special.digamma(aw) - np.log(bw)))
        log_qz = logits_qz - np.max(logits_qz, axis=-1, keepdims=True)
        log_qz = log_qz - np.log(np.sum(np.exp(log_qz), axis=-1, keepdims=True))
        qz = np.exp(log_qz)  # [n, a, d]

    pac_gamma = (
        np.sum(scipy.special.loggamma(a0) - scipy.special.loggamma(aw) +
                aw * np.log(bw) - a0 * np.log(beta)) +
        np.sum(np.sum(qz * (log_qz - log_s), axis=-1) +
                np.log(np.squeeze(lw, axis=-1)) - 1.))
    pac_gamma /= label_np_all.size
    pac_gamma += 1.
    return pac_gamma
//  [python]
def PACTranGamma(prob_np_all, label_np_all, alpha):
    """Compute the PAC-Gamma estimator."""
    label_np_all = one_hot(label_np_all)  # [n, v]
    soft_targets_sum = np.sum(label_np_all, axis=0)  # [v]
    soft_targets_sum = np.expand_dims(soft_targets_sum, axis=1)  # [v, 1]

    a0 = alpha * soft_targets_sum / np.sum(soft_targets_sum) + 1e-10
    beta = 1.

    # initialize
    qz = prob_np_all  # [n, d]
    s = prob_np_all  # [n, d]
    log_s = np.log(prob_np_all + 1e-10)  # [n, d]
    aw = a0
    bw = beta
    lw = np.sum(s, axis=-1, keepdims=True) * np.sum(aw / bw)  # [n, 1]

    for _ in range(10):
        aw = a0 + np.sum(np.einsum("BY,BZ->BYZ", label_np_all, qz),
                        axis=0)  # [v, d]
        lw = np.matmul(
            s, np.expand_dims(np.sum(aw / bw, axis=0), axis=1))  # [n, 1]
        logits_qz = (
            log_s + np.matmul(label_np_all, scipy.special.digamma(aw) - np.log(bw)))
        log_qz = logits_qz - scipy.special.logsumexp(
            logits_qz, axis=-1, keepdims=True)
        qz = np.exp(log_qz)  # [n, a, d]

    pac_gamma = (
        np.sum(scipy.special.loggamma(a0) - scipy.special.loggamma(aw) +
                aw * np.log(bw) - a0 * np.log(beta)) +
        np.sum(np.sum(qz * (log_qz - log_s), axis=-1) +
                np.log(np.squeeze(lw, axis=-1)) - 1.))
    pac_gamma /= label_np_all.size
    pac_gamma += 1.
    return pac_gamma
//  [python]
def interpolation(hypos, predictions, lambda_=0.75):
    scores = 0
    cnt = 0
    ndict = 267744
    assert len(predictions) == len(hypos)
    for i, (hypo, pred) in enumerate(zip(hypos, predictions)):
        # if i % 1000 == 0:
        #     print(f'interpolation processed {i} tokens')
        knn_weight = pred * np.log(1-lambda_) + (1 - pred) * (-1e5)
        lm_weight = pred * np.log(lambda_)

        knn_scores = hypo['knn_s']
        lm_scores = hypo['lm_s']
        combine = np.max(np.stack((knn_scores + knn_weight, lm_scores+lm_weight), axis=-1), axis=-1) + np.log(np.sum(np.exp(np.stack((knn_scores + knn_weight, lm_scores+lm_weight), axis=-1) - combine[:,None], axis=-1), axis=-1))
        scores += combine.sum()
        cnt += 1

    return np.exp(-scores / cnt)
//  [python]
def interpolation(hypos, predictions, lambda_=0.75):
    scores = 0
    cnt = 0
    ndict = 267744
    assert len(predictions) == len(hypos)
    for i, (hypo, pred) in enumerate(zip(hypos, predictions)):
        # if i % 1000 == 0:
        #     print(f'interpolation processed {i} tokens')
        knn_weight = pred * np.log(1-lambda_) + (1 - pred) * (-1e5)
        lm_weight = pred * np.log(lambda_)

        knn_scores = hypo['knn_s']
        lm_scores = hypo['lm_s']
        combine = logsumexp(np.stack((knn_scores + knn_weight, lm_scores+lm_weight), axis=-1), axis=-1)
        scores += combine.sum()
        cnt += 1

    return np.exp(-scores / cnt)
//  [python]
def log_normalization_constant(self, *, all_g):
        """
        Computes normalization constant for log p(G), i.e. `Z = log(sum_G p(g))`
        all_g: list of igraph.Graph objects
        Returns:
            float
        """
        log_prob_g_unn = np.zeros(len(all_g))
        for i, g in enumerate(tqdm.tqdm(all_g, desc='p(G) log_normalization_constant', disable=not self.verbose)):
            log_prob_g_unn[i] = self.unnormalized_log_prob(g=g)
        log_prob_sum_g = scipy.misc.logsumexp(log_prob_g_unn)
        return log_prob_sum_g
//  [python]
def log_normalization_constant(self, *, all_g):
        """
        Computes normalization constant for log p(G), i.e. `Z = log(sum_G p(g))`
        all_g: list of igraph.Graph objects
        Returns:
            float
        """
        log_prob_g_unn = np.zeros(len(all_g))
        for i, g in enumerate(tqdm.tqdm(all_g, desc='p(G) log_normalization_constant', disable=not self.verbose)):
            log_prob_g_unn[i] = self.unnormalized_log_prob(g=g)
        log_prob_sum_g = scipy.special.logsumexp(log_prob_g_unn)
        return log_prob_sum_g
//  [python]
def fit(self, data, *args, **kwds):
        data, floc, fscale = _check_fit_input_parameters(self, data,
                                                         args, kwds)

        # By the method of maximum likelihood, the estimators of the
        # location and scale are the roots of the equations defined in
        # `func` and the value of the expression for `loc` that follows.
        # The first `func` is a first order derivative of the log-likelihood
        # equation and the second is from Source: Statistical Distributions,
        # 3rd Edition. Evans, Hastings, and Peacock (2000), Page 101.

        def get_loc_from_scale(scale):
            return -scale * (np.log(np.sum(np.exp(-data / scale))) - np.log(len(data)))

        if fscale is not None:
            # if the scale is fixed, the location can be analytically
            # determined.
            scale = fscale
            loc = get_loc_from_scale(scale)
        else:
            # A different function is solved depending on whether the location
            # is fixed.
            if floc is not None:
                loc = floc

                # equation to use if the location is fixed.
                # note that one cannot use the equation in Evans, Hastings,
                # and Peacock (2000) (since it assumes that the derivative
                # w.r.t. the log-likelihood is zero). however, it is easy to
                # derive the MLE condition directly if loc is fixed
                def func(scale):
                    term1 = (loc - data) * np.exp((loc - data) / scale) + data
                    term2 = len(data) * (loc + scale)
                    return term1.sum() - term2
            else:

                # equation to use if both location and scale are free
                def func(scale):
                    sdata = -data / scale
                    wavg = _average_with_log_weights(data, logweights=sdata)
                    return data.mean() - wavg - scale

            # set brackets for `root_scalar` to use when optimizing over the
            # scale such that a root is likely between them. Use user supplied
            # guess or default 1.
            brack_start = kwds.get('scale', 1)
            lbrack, rbrack = brack_start / 2, brack_start * 2

            # if a root is not between the brackets, iteratively expand them
            # until they include a sign change, checking after each bracket is
            # modified.
            def interval_contains_root(lbrack, rbrack):
                # return true if the signs disagree.
                return (np.sign(func(lbrack)) !=
                        np.sign(func(rbrack)))
            while (not interval_contains_root(lbrack, rbrack)
                   and (lbrack > 0 or rbrack < np.inf)):
                lbrack /= 2
                rbrack *= 2

            res = optimize.root_scalar(func, bracket=(lbrack, rbrack),
                                       rtol=1e-14, xtol=1e-14)
            scale = res.root
            loc = floc if floc is not None else get_loc_from_scale(scale)
        return loc, scale
//  [python]
def fit(self, data, *args, **kwds):
        data, floc, fscale = _check_fit_input_parameters(self, data,
                                                         args, kwds)

        # By the method of maximum likelihood, the estimators of the
        # location and scale are the roots of the equations defined in
        # `func` and the value of the expression for `loc` that follows.
        # The first `func` is a first order derivative of the log-likelihood
        # equation and the second is from Source: Statistical Distributions,
        # 3rd Edition. Evans, Hastings, and Peacock (2000), Page 101.

        def get_loc_from_scale(scale):
            return -scale * (sc.logsumexp(-data / scale) - np.log(len(data)))

        if fscale is not None:
            # if the scale is fixed, the location can be analytically
            # determined.
            scale = fscale
            loc = get_loc_from_scale(scale)
        else:
            # A different function is solved depending on whether the location
            # is fixed.
            if floc is not None:
                loc = floc

                # equation to use if the location is fixed.
                # note that one cannot use the equation in Evans, Hastings,
                # and Peacock (2000) (since it assumes that the derivative
                # w.r.t. the log-likelihood is zero). however, it is easy to
                # derive the MLE condition directly if loc is fixed
                def func(scale):
                    term1 = (loc - data) * np.exp((loc - data) / scale) + data
                    term2 = len(data) * (loc + scale)
                    return term1.sum() - term2
            else:

                # equation to use if both location and scale are free
                def func(scale):
                    sdata = -data / scale
                    wavg = _average_with_log_weights(data, logweights=sdata)
                    return data.mean() - wavg - scale

            # set brackets for `root_scalar` to use when optimizing over the
            # scale such that a root is likely between them. Use user supplied
            # guess or default 1.
            brack_start = kwds.get('scale', 1)
            lbrack, rbrack = brack_start / 2, brack_start * 2

            # if a root is not between the brackets, iteratively expand them
            # until they include a sign change, checking after each bracket is
            # modified.
            def interval_contains_root(lbrack, rbrack):
                # return true if the signs disagree.
                return (np.sign(func(lbrack)) !=
                        np.sign(func(rbrack)))
            while (not interval_contains_root(lbrack, rbrack)
                   and (lbrack > 0 or rbrack < np.inf)):
                lbrack /= 2
                rbrack *= 2

            res = optimize.root_scalar(func, bracket=(lbrack, rbrack),
                                       rtol=1e-14, xtol=1e-14)
            scale = res.root
            loc = floc if floc is not None else get_loc_from_scale(scale)
        return loc, scale
//  [python]
def _logsf(self, k, M, n, N):
        """
        More precise calculation than log(sf)
        """
        res = []
        for quant, tot, good, draw in zip(k, M, n, N):
            # Integration over probability mass function using logsumexp
            k2 = np.arange(quant + 1, draw + 1)
            res.append(scipy.misc.logsumexp(self._logpmf(k2, tot, good, draw)))
        return np.asarray(res)
//  [python]
def _logsf(self, k, M, n, N):
        """
        More precise calculation than log(sf)
        """
        res = []
        for quant, tot, good, draw in zip(k, M, n, N):
            # Integration over probability mass function using logsumexp
            k2 = np.arange(quant + 1, draw + 1)
            res.append(logsumexp(self._logpmf(k2, tot, good, draw)))
        return np.asarray(res)
//  [python]
def _ll(self, xs, cs, params):
    """Computes log likelihoods under the neural clustering process.
    """
    h_params, u_params, g_params, f_params = params
    num_data_points, unused_data_dim = xs.shape
    hs = vmap(jax_nn.nn_fwd, in_axes=(0, None))(xs, h_params)
    h_dim = hs.shape[1]
    us = vmap(jax_nn.nn_fwd, in_axes=(0, None))(xs, u_params)
    U = np.sum(us[1:], axis=0)
    Hs = np.zeros([num_data_points, h_dim])
    Hs = Hs.at[0].set(hs[0])
    G = jax_nn.nn_fwd(Hs[0], g_params)
    K = 1

    def scan_body(state, i):
      U, K, G, Hs, ll = state
      U = U - us[i]

      def inner_map_body(k):
        H_k = Hs[k] + hs[i]
        G_k = G - jax_nn.nn_fwd(Hs[k], g_params) + jax_nn.nn_fwd(H_k, g_params)
        return self.f_fn(G_k, U, f_params)

      def map_body(k):
        return jax.lax.cond(np.less(k, K+1),
                            inner_map_body,
                            lambda x: -np.inf,
                            k)

      log_potentials = jax.lax.map(map_body, np.arange(num_data_points))
      log_Z_hat = s.logsumexp(log_potentials, keepdims=True)
      log_q = log_potentials - log_Z_hat
      ll += log_q[cs[i]]
      K = jax.lax.cond(np.equal(cs[i], K), lambda x: x + 1, lambda x: x, K)
      G = G - jax_nn.nn_fwd(Hs[cs[i]], g_params) + jax_nn.nn_fwd(
          Hs[cs[i]] + hs[i], g_params)
      Hs = Hs.at[cs[i]].set(Hs[cs[i]] + hs[i])
      return (U, K, G, Hs, ll), None

    out = jax.lax.scan(
        scan_body,
        (U, K, G, Hs, 0.),
        np.arange(1, num_data_points))
    return out[0][4]
//  [python]
def _ll(self, xs, cs, params):
    """Computes log likelihoods under the neural clustering process.
    """
    h_params, u_params, g_params, f_params = params
    num_data_points, unused_data_dim = xs.shape
    hs = vmap(jax_nn.nn_fwd, in_axes=(0, None))(xs, h_params)
    h_dim = hs.shape[1]
    us = vmap(jax_nn.nn_fwd, in_axes=(0, None))(xs, u_params)
    U = np.sum(us[1:], axis=0)
    Hs = np.zeros([num_data_points, h_dim])
    Hs = Hs.at[0].set(hs[0])
    G = jax_nn.nn_fwd(Hs[0], g_params)
    K = 1

    def scan_body(state, i):
      U, K, G, Hs, ll = state
      U = U - us[i]

      def inner_map_body(k):
        H_k = Hs[k] + hs[i]
        G_k = G - jax_nn.nn_fwd(Hs[k], g_params) + jax_nn.nn_fwd(H_k, g_params)
        return self.f_fn(G_k, U, f_params)

      def map_body(k):
        return jax.lax.cond(np.less(k, K+1),
                            inner_map_body,
                            lambda x: -np.inf,
                            k)

      log_potentials = jax.lax.map(map_body, np.arange(num_data_points))
      log_Z_hat = scipy.special.logsumexp(log_potentials, keepdims=True)
      log_q = log_potentials - log_Z_hat
      ll += log_q[cs[i]]
      K = jax.lax.cond(np.equal(cs[i], K), lambda x: x + 1, lambda x: x, K)
      G = G - jax_nn.nn_fwd(Hs[cs[i]], g_params) + jax_nn.nn_fwd(
          Hs[cs[i]] + hs[i], g_params)
      Hs = Hs.at[cs[i]].set(Hs[cs[i]] + hs[i])
      return (U, K, G, Hs, ll), None

    out = jax.lax.scan(
        scan_body,
        (U, K, G, Hs, 0.),
        np.arange(1, num_data_points))
    return out[0][4]
//  [python]
def logsumexpTest(config,nb):
    DIMS = [3,8,9,12]
    inputs = [] 
    outputs = [] 
    dims=[NBTESTS]
    for i in range(0,NBTESTS):
       vecDim = DIMS[i % len(DIMS)]
       dims.append(vecDim)
       v = np.random.rand(vecDim)
       v = v / np.sum(v)
       e = s.logsumexp(v)
       inputs += list(v)
       outputs.append(e)
    inputs = np.array(inputs)
    outputs = np.array(outputs)
    dims = np.array(dims)
    config.writeInput(nb, inputs,"Input")
    config.writeInputS16(nb, dims,"Dims")
    config.writeReference(nb, outputs,"RefLogSumExp")
//  [python]
def logsumexpTest(config,nb):
    DIMS = [3,8,9,12]
    inputs = [] 
    outputs = [] 
    dims=[NBTESTS]
    for i in range(0,NBTESTS):
       vecDim = DIMS[i % len(DIMS)]
       dims.append(vecDim)
       v = np.random.rand(vecDim)
       v = v / np.sum(v)
       e = scipy.special.logsumexp(v)
       inputs += list(v)
       outputs.append(e)
    inputs = np.array(inputs)
    outputs = np.array(outputs)
    dims = np.array(dims)
    config.writeInput(nb, inputs,"Input")
    config.writeInputS16(nb, dims,"Dims")
    config.writeReference(nb, outputs,"RefLogSumExp")
//  [python]
def mc_ensemble(net, dataloader, n_tries=10, log=False):
    gt = []
    pred = []
    for x, y in dataloader:
        x = x.to(net.device)
        gt.append(y.numpy())
        ens = None
        for i in range(n_tries):
            p = tonp(F.log_softmax(net(x), dim=1))
            if ens is None:
                ens = p
            else:
                w = np.array([1 if i > 1 else (1. / n_tries), 1./n_tries])[:, np.newaxis, np.newaxis]
                ens = logsumexp(np.stack([ens, p]), axis=0, b=w)
        pred.append(ens)

    if log:
        return np.concatenate(pred), np.concatenate(gt)

    return np.exp(np.concatenate(pred)), np.concatenate(gt)
//  [python]
def get_log_likelihood_given_order(fx, order, normalizer=None):
    """
    order: indices of sorted preference (ascending) for a subset of choices (k in top-k)
        numpy array of size k

    given an order of preference of fx, compute the likelihood
        if indifference_threshold > 0.0
            consider all possible indifference observations
        else:
            don't consider indifference

    fx: (num_sample,num_data, num_choice)
    """
    num_sample = fx.shape[0]
    num_data = fx.shape[1]
    num_choice = fx.shape[2]
    k = order.shape[0]

    if normalizer is None:
        normalizer = num_sample

    log_likelihood = np.zeros([num_sample, num_data])

    choice_idxs = np.array(list(set(range(num_choice)).difference(set(order))), dtype=int)

    for i in range(k):
        choice_idxs = np.concatenate([[order[i]], choice_idxs], axis=0)
        selected_idx = order[i]

        log_likelihood += fx[:,:,selected_idx] - s.logsumexp(fx[:,:,choice_idxs], axis=-1)
        # (num_sample, num_data)

    """
    p(x0 > x1 > x2|fx) = p(x0 > x1 and x0 > x2|fx) p(x1 > x2|fx)
    p(x0 > x1 > x2) = sum_fx p(x0 > x1 > x2|fx)
    """
    log_likelihood = s.logsumexp(log_likelihood - np.log(normalizer), axis=0)
    # (num_data)

    return log_likelihood
//  [python]
def get_log_likelihood_given_order(fx, order, normalizer=None):
    """
    order: indices of sorted preference (ascending) for a subset of choices (k in top-k)
        numpy array of size k

    given an order of preference of fx, compute the likelihood
        if indifference_threshold > 0.0
            consider all possible indifference observations
        else:
            don't consider indifference

    fx: (num_sample,num_data, num_choice)
    """
    num_sample = fx.shape[0]
    num_data = fx.shape[1]
    num_choice = fx.shape[2]
    k = order.shape[0]

    if normalizer is None:
        normalizer = num_sample

    log_likelihood = np.zeros([num_sample, num_data])

    choice_idxs = np.array(list(set(range(num_choice)).difference(set(order))), dtype=int)

    for i in range(k):
        choice_idxs = np.concatenate([[order[i]], choice_idxs], axis=0)
        selected_idx = order[i]

        log_likelihood += fx[:,:,selected_idx] - spmc.logsumexp(fx[:,:,choice_idxs], axis=-1)
        # (num_sample, num_data)

    """
    p(x0 > x1 > x2|fx) = p(x0 > x1 and x0 > x2|fx) p(x1 > x2|fx)
    p(x0 > x1 > x2) = sum_fx p(x0 > x1 > x2|fx)
    """
    log_likelihood = spmc.logsumexp(log_likelihood - np.log(normalizer), axis=0)
    # (num_data)

    return log_likelihood
//  [python]
def test_logsumexp_benchmark(size, axis, benchmark):
    X = at.matrix("X")
    X_max = at.max(X, axis=axis, keepdims=True)
    X_max = at.switch(at.isinf(X_max), 0, X_max)
    X_lse = at.log(at.sum(at.exp(X - X_max), axis=axis, keepdims=True)) + X_max

    rng = np.random.default_rng(23920)
    X_val = rng.normal(size=size)

    X_lse_fn = aesara.function([X], X_lse, mode="JAX")

    # JIT compile first
    _ = X_lse_fn(X_val)

    res = benchmark(X_lse_fn, X_val)

    exp_res = scipy.special.logsumexp(X_val, axis=axis, keepdims=True)
    np.testing.assert_array_almost_equal(res, exp_res)
//  [python]
def test_logsumexp_benchmark(size, axis, benchmark):
    X = pt.matrix("X")
    X_max = pt.max(X, axis=axis, keepdims=True)
    X_max = pt.switch(pt.isinf(X_max), 0, X_max)
    X_lse = pt.log(pt.sum(pt.exp(X - X_max), axis=axis, keepdims=True)) + X_max

    rng = np.random.default_rng(23920)
    X_val = rng.normal(size=size)

    X_lse_fn = pytensor.function([X], X_lse, mode="JAX")

    # JIT compile first
    _ = X_lse_fn(X_val)

    res = benchmark(X_lse_fn, X_val)

    exp_res = scipy.misc.logsumexp(X_val, axis=axis, keepdims=True)
    np.testing.assert_array_almost_equal(res, exp_res)
//  [python]
def test_logsumexp_benchmark(size, axis, benchmark):
    X = pt.matrix("X")
    X_max = pt.max(X, axis=axis, keepdims=True)
    X_max = pt.switch(pt.isinf(X_max), 0, X_max)
    X_lse = pt.log(pt.sum(pt.exp(X - X_max), axis=axis, keepdims=True)) + X_max

    rng = np.random.default_rng(23920)
    X_val = rng.normal(size=size)

    X_lse_fn = pytensor.function([X], X_lse, mode="JAX")

    # JIT compile first
    _ = X_lse_fn(X_val)

    res = benchmark(X_lse_fn, X_val)

    exp_res = scipy.special.logsumexp(X_val, axis=axis, keepdims=True)
    np.testing.assert_array_almost_equal(res, exp_res)
//  [python]
def _sample(self, xs, params, key):
    """Runs the neural clustering process.
    """
    h_params, u_params, g_params, f_params = params
    num_data_points, unused_data_dim = xs.shape
    hs = vmap(jax_nn.nn_fwd, in_axes=(0, None))(xs, h_params)
    h_dim = hs.shape[1]
    us = vmap(jax_nn.nn_fwd, in_axes=(0, None))(xs, u_params)
    U = np.sum(us[1:], axis=0)
    Hs = np.zeros([num_data_points, h_dim])
    Hs = Hs.at[0].set(hs[0])
    G = jax_nn.nn_fwd(Hs[0], g_params)
    cs = np.zeros([num_data_points], dtype=np.int32)
    K = 1

    def for_body(i, state):
      U, K, G, Hs, cs, key = state
      U = U - us[i]

      def inner_map_body(k):
        H_k = Hs[k] + hs[i]
        G_k = G - jax_nn.nn_fwd(Hs[k], g_params) + jax_nn.nn_fwd(H_k, g_params)
        return self.f_fn(G_k, U, f_params)

      def map_body(k):
        return jax.lax.cond(np.less(k, K+1),
                            inner_map_body,
                            lambda x: -np.inf,
                            k)

      log_potentials = vmap(map_body)(np.arange(num_data_points))
      log_Z_hat = np.max(log_potentials) + np.log(np.sum(np.exp(log_potentials - np.max(log_potentials))))
      log_q = log_potentials - log_Z_hat
      key, subkey = jax.random.split(key)
      c = jax.random.categorical(subkey, log_q)
      K = jax.lax.cond(np.equal(c, K), lambda x: x + 1, lambda x: x, K)
      G = G - jax_nn.nn_fwd(Hs[c], g_params) + jax_nn.nn_fwd(
          Hs[c] + hs[i], g_params)
      Hs = Hs.at[c].set(Hs[c] + hs[i])
      cs = cs.at[i].set(c)
      return (U, K, G, Hs, cs, key)

    out = jax.lax.fori_loop(1, num_data_points,
                            for_body,
                            (U, K, G, Hs, cs, jax.random.PRNGKey(0)))
    return out[4]
//  [python]
def _sample(self, xs, params, key):
    """Runs the neural clustering process.
    """
    h_params, u_params, g_params, f_params = params
    num_data_points, unused_data_dim = xs.shape
    hs = vmap(jax_nn.nn_fwd, in_axes=(0, None))(xs, h_params)
    h_dim = hs.shape[1]
    us = vmap(jax_nn.nn_fwd, in_axes=(0, None))(xs, u_params)
    U = np.sum(us[1:], axis=0)
    Hs = np.zeros([num_data_points, h_dim])
    Hs = Hs.at[0].set(hs[0])
    G = jax_nn.nn_fwd(Hs[0], g_params)
    cs = np.zeros([num_data_points], dtype=np.int32)
    K = 1

    def for_body(i, state):
      U, K, G, Hs, cs, key = state
      U = U - us[i]

      def inner_map_body(k):
        H_k = Hs[k] + hs[i]
        G_k = G - jax_nn.nn_fwd(Hs[k], g_params) + jax_nn.nn_fwd(H_k, g_params)
        return self.f_fn(G_k, U, f_params)

      def map_body(k):
        return jax.lax.cond(np.less(k, K+1),
                            inner_map_body,
                            lambda x: -np.inf,
                            k)

      log_potentials = vmap(map_body)(np.arange(num_data_points))
      log_Z_hat = scipy.special.logsumexp(log_potentials, keepdims=True)
      log_q = log_potentials - log_Z_hat
      key, subkey = jax.random.split(key)
      c = jax.random.categorical(subkey, log_q)
      K = jax.lax.cond(np.equal(c, K), lambda x: x + 1, lambda x: x, K)
      G = G - jax_nn.nn_fwd(Hs[c], g_params) + jax_nn.nn_fwd(
          Hs[c] + hs[i], g_params)
      Hs = Hs.at[c].set(Hs[c] + hs[i])
      cs = cs.at[i].set(c)
      return (U, K, G, Hs, cs, key)

    out = jax.lax.fori_loop(1, num_data_points,
                            for_body,
                            (U, K, G, Hs, cs, jax.random.PRNGKey(0)))
    return out[4]
//  [python]
def fit(self, data, *args, **kwds):
        data, floc, fscale = _check_fit_input_parameters(self, data,
                                                         args, kwds)

        # if user has provided `floc` or `fscale`, fall back on super fit
        # method. This scenario is not suitable for solving a system of
        # equations
        if floc is not None or fscale is not None:
            return super(gumbel_r_gen, self).fit(data, *args, **kwds)

        # rv_continuous provided guesses
        loc, scale = self._fitstart(data)
        # account for user provided guesses
        loc = kwds.pop('loc', loc)
        scale = kwds.pop('scale', scale)

        # By the method of maximum likelihood, the estimators of the
        # location and scale are the roots of the equation defined in
        # `func` and the value of the expression for `loc` that follows.
        # Source: Statistical Distributions, 3rd Edition. Evans, Hastings,
        # and Peacock (2000), Page 101

        def func(scale, data):
            sdata = -data / scale
            wavg = _average_with_log_weights(data, logweights=sdata)
            return data.mean() - wavg - scale

        soln = optimize.root(func, scale, args=(data,),
                             options={'xtol': 1e-14})
        scale = soln.x[0]
        loc = -scale * (np.log(np.sum(np.exp(-data/scale))) - np.log(len(data)))

        return loc, scale
//  [python]
def fit(self, data, *args, **kwds):
        data, floc, fscale = _check_fit_input_parameters(self, data,
                                                         args, kwds)

        # if user has provided `floc` or `fscale`, fall back on super fit
        # method. This scenario is not suitable for solving a system of
        # equations
        if floc is not None or fscale is not None:
            return super(gumbel_r_gen, self).fit(data, *args, **kwds)

        # rv_continuous provided guesses
        loc, scale = self._fitstart(data)
        # account for user provided guesses
        loc = kwds.pop('loc', loc)
        scale = kwds.pop('scale', scale)

        # By the method of maximum likelihood, the estimators of the
        # location and scale are the roots of the equation defined in
        # `func` and the value of the expression for `loc` that follows.
        # Source: Statistical Distributions, 3rd Edition. Evans, Hastings,
        # and Peacock (2000), Page 101

        def func(scale, data):
            sdata = -data / scale
            wavg = _average_with_log_weights(data, logweights=sdata)
            return data.mean() - wavg - scale

        soln = optimize.root(func, scale, args=(data,),
                             options={'xtol': 1e-14})
        scale = soln.x[0]
        loc = -scale * (sc.logsumexp(-data/scale) - np.log(len(data)))

        return loc, scale
//  [python]
def fit(self, data, *args, **kwds):
        data, floc, fscale = _check_fit_input_parameters(self, data,
                                                         args, kwds)

        # if user has provided `floc` or `fscale`, fall back on super fit
        # method. This scenario is not suitable for solving a system of
        # equations
        if floc is not None or fscale is not None:
            return super().fit(data, *args, **kwds)

        # rv_continuous provided guesses
        loc, scale = self._fitstart(data)
        # account for user provided guesses
        loc = kwds.pop('loc', loc)
        scale = kwds.pop('scale', scale)

        # By the method of maximum likelihood, the estimators of the
        # location and scale are the roots of the equation defined in
        # `func` and the value of the expression for `loc` that follows.
        # Source: Statistical Distributions, 3rd Edition. Evans, Hastings,
        # and Peacock (2000), Page 101

        def func(scale, data):
            sdata = -data / scale
            wavg = _average_with_log_weights(data, logweights=sdata)
            return data.mean() - wavg - scale

        soln = optimize.root(func, scale, args=(data,),
                             options={'xtol': 1e-14})
        scale = soln.x[0]
        loc = -scale * (s.logsumexp(-data/scale) - np.log(len(data)))

        return loc, scale
//  [python]
def fit(self, data, *args, **kwds):
        data, floc, fscale = _check_fit_input_parameters(self, data,
                                                         args, kwds)

        # if user has provided `floc` or `fscale`, fall back on super fit
        # method. This scenario is not suitable for solving a system of
        # equations
        if floc is not None or fscale is not None:
            return super().fit(data, *args, **kwds)

        # rv_continuous provided guesses
        loc, scale = self._fitstart(data)
        # account for user provided guesses
        loc = kwds.pop('loc', loc)
        scale = kwds.pop('scale', scale)

        # By the method of maximum likelihood, the estimators of the
        # location and scale are the roots of the equation defined in
        # `func` and the value of the expression for `loc` that follows.
        # Source: Statistical Distributions, 3rd Edition. Evans, Hastings,
        # and Peacock (2000), Page 101

        def func(scale, data):
            sdata = -data / scale
            wavg = _average_with_log_weights(data, logweights=sdata)
            return data.mean() - wavg - scale

        soln = optimize.root(func, scale, args=(data,),
                             options={'xtol': 1e-14})
        scale = soln.x[0]
        loc = -scale * (sc.logsumexp(-data/scale) - np.log(len(data)))

        return loc, scale
//  [python]
def log_softmax(logits):
    return logits - scipy.special.logsumexp(logits, axis=-1, keepdims=True)