jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def save_negative_log_likelihood_fixed_pars(self, t):
        """For comparison with GP, fix sigma^2 at the MAP when doing the 
        one-step-ahead prediction."""
        all_one_step_ahead_lklhoods_weighted = [
            (self.model_universe[m].one_step_ahead_predictive_log_probs_fixed_pars + 
             self.model_and_run_length_log_distr[m,
                        self.model_and_run_length_log_distr[m,:]> -np.inf])
            for m in range(0, self.Q)]
             #self.model_universe[m].retained_run_lengths])
             #np.where(self.model_and_run_length_log_distr[m,:]> -np.inf)])            
             
        summed_up = -special.logsumexp([item for entry in 
            all_one_step_ahead_lklhoods_weighted for item in entry])
        self.negative_log_likelihood_fixed_pars.append(summed_up)
//  [python]
def update_priors(self, t):
        """update the priors for each model in the model universe, provided 
        that the boolean auto_prior_update is true"""
        for (m, model) in zip(range(0,self.Q), self.model_universe):
            if model.auto_prior_update == True:
                """We need to weigh quantities acc. to rld"""
                model_specific_rld = (self.model_and_run_length_log_distr[m,:] -
                    s.logsumexp(self.model_and_run_length_log_distr[m,:]))
                model.prior_update(t, model_specific_rld)
//  [python]
def update_priors(self, t):
        """update the priors for each model in the model universe, provided 
        that the boolean auto_prior_update is true"""
        for (m, model) in zip(range(0,self.Q), self.model_universe):
            if model.auto_prior_update == True:
                """We need to weigh quantities acc. to rld"""
                model_specific_rld = (self.model_and_run_length_log_distr[m,:] -
                    special.logsumexp(self.model_and_run_length_log_distr[m,:]))
                model.prior_update(t, model_specific_rld)
//  [python]
def log_transition(self, x, u):
        if isinstance(x, np.ndarray) and isinstance(u, np.ndarray):
            nb_steps = np.maximum(len(x) - 1, 1)
            logtrans = np.broadcast_to(self.logmat, (nb_steps,) + self.logmat.shape)
            return logtrans - logsumexp(logtrans, axis=-1, keepdims=True)
        else:
            def inner(x, u):
                return self.log_transition.__wrapped__(self, x, u)
            return list(map(inner, x, u))
//  [python]
def log_transition(self, x, u):
        if isinstance(x, np.ndarray) and isinstance(u, np.ndarray):
            nb_steps = np.maximum(len(x) - 1, 1)
            input = np.hstack((x[:nb_steps], u[:nb_steps]))
            output = np_float(self.regressor.predict(input))
            return output - logsumexp(output, axis=-1, keepdims=True)
        else:
            def inner(x, u):
                return self.log_transition.__wrapped__(self, x, u)
            return list(map(inner, x, u))
//  [python]
def _fep_helper(target_potentials, reference_potentials, metad_rbias, kT,
                n_bootstrap_cycles=0, subsample_size=0, fixed_bootstrap_indices=None):
    """Same as fep() but it works with unitless arguments (and unitless return values)."""
    n_samples = len(target_potentials)

    # Compute the reduced work and weights.
    metad_rbias = metad_rbias / kT
    minus_work = (- target_potentials + reference_potentials) / kT

    # Compute and return the free energy.
    if 0 < subsample_size < n_samples:
        indices = np.linspace(0, n_samples-1, num=subsample_size, dtype=np.int)
        rbias = metad_rbias[indices]
        log_weights = rbias - scipy.misc.logsumexp(rbias)
        delta_f = - scipy.misc.logsumexp(minus_work[indices] + log_weights)
    else:
        log_weights = metad_rbias - scipy.misc.logsumexp(metad_rbias)
        delta_f = - scipy.misc.logsumexp(minus_work + log_weights)

    if n_bootstrap_cycles > 0:
        # Initialize bootstrap distribution of Df.
        boot_delta_f = np.empty(n_bootstrap_cycles)

        for boot_cycle_idx, bootstrap_indices in enumerate(_iterate_bootstrap_indices(
            n_bootstrap_cycles, n_samples, subsample_size, fixed_bootstrap_indices
        )):
            rbias = metad_rbias[bootstrap_indices]
            log_weights = rbias - scipy.misc.logsumexp(rbias)
            boot_delta_f[boot_cycle_idx] = - scipy.misc.logsumexp(minus_work[bootstrap_indices] + log_weights)

        # Compute the 95% bootstrap CI.
        bootstrap_ci = np.nanpercentile(boot_delta_f, [2.5, 97.5], axis=0)

        # Compute the mean and median bootstrap statistic.
        bootstrap_mean = np.mean(boot_delta_f, axis=0)
        bootstrap_median = np.median(boot_delta_f, axis=0)

        return delta_f, bootstrap_ci, bootstrap_mean, bootstrap_median
    return delta_f
//  [python]
def _fep_helper(target_potentials, reference_potentials, metad_rbias, kT,
                n_bootstrap_cycles=0, subsample_size=0, fixed_bootstrap_indices=None):
    """Same as fep() but it works with unitless arguments (and unitless return values)."""
    n_samples = len(target_potentials)

    # Compute the reduced work and weights.
    metad_rbias = metad_rbias / kT
    minus_work = (- target_potentials + reference_potentials) / kT

    # Compute and return the free energy.
    if 0 < subsample_size < n_samples:
        indices = np.linspace(0, n_samples-1, num=subsample_size, dtype=np.int)
        rbias = metad_rbias[indices]
        log_weights = rbias - scipy.special.logsumexp(rbias)
        delta_f = - scipy.special.logsumexp(minus_work[indices] + log_weights)
    else:
        log_weights = metad_rbias - scipy.special.logsumexp(metad_rbias)
        delta_f = - scipy.special.logsumexp(minus_work + log_weights)

    if n_bootstrap_cycles > 0:
        # Initialize bootstrap distribution of Df.
        boot_delta_f = np.empty(n_bootstrap_cycles)

        for boot_cycle_idx, bootstrap_indices in enumerate(_iterate_bootstrap_indices(
            n_bootstrap_cycles, n_samples, subsample_size, fixed_bootstrap_indices
        )):
            rbias = metad_rbias[bootstrap_indices]
            log_weights = rbias - scipy.special.logsumexp(rbias)
            boot_delta_f[boot_cycle_idx] = - scipy.special.logsumexp(minus_work[bootstrap_indices] + log_weights)

        # Compute the 95% bootstrap CI.
        bootstrap_ci = np.nanpercentile(boot_delta_f, [2.5, 97.5], axis=0)

        # Compute the mean and median bootstrap statistic.
        bootstrap_mean = np.mean(boot_delta_f, axis=0)
        bootstrap_median = np.median(boot_delta_f, axis=0)

        return delta_f, bootstrap_ci, bootstrap_mean, bootstrap_median
    return delta_f
//  [python]
def value(self, theta_flat, lmbda, idx):
        theta = theta_flat.reshape(self.n_features, self.n_classes)
        x = self.X[idx]
        y = self.y[idx]
        n_samples, _ = x.shape
        lbda = lmbda[idx]
        prod = x @ theta
        weights = sc.expit(lbda)
        individual_losses = -prod[y == 1] + np.log(np.sum(np.exp(prod), axis=1))
        regul = self.reg * np.dot(theta_flat, theta_flat)
        return (individual_losses * weights).sum() / n_samples + regul
//  [python]
def value(self, theta_flat, lmbda, idx):
        theta = theta_flat.reshape(self.n_features, self.n_classes)
        x = self.X[idx]
        y = self.y[idx]
        n_samples, _ = x.shape
        lbda = lmbda[idx]
        prod = x @ theta
        weights = sc.expit(lbda)
        individual_losses = -prod[y == 1] + sc.logsumexp(prod, axis=1)
        regul = self.reg * np.dot(theta_flat, theta_flat)
        return (individual_losses * weights).sum() / n_samples + regul
//  [python]
def rescale_DPD_run_length_distribution(self, log_scale_new, 
                                        rescaler_for_old_obs, t):
        """Rescales joint log probs"""
        if rescaler_for_old_obs is None:
            self.joint_log_probabilities = (self.joint_log_probabilities 
                                            - log_scale_new)
        else:
            self.joint_log_probabilities[0] = (self.joint_log_probabilities[0] 
                    - log_scale_new)
            self.joint_log_probabilities[1:] = (
                    self.joint_log_probabilities[1:] + rescaler_for_old_obs)
        self.model_log_evidence = s.logsumexp(
                self.joint_log_probabilities)
//  [python]
def solve_schroedingers_equation(self, hw, psigrid, vgrid, return_eigsys=False):
        """
        Given the frequency of the HO, the wavefunctions and
        the gridded potential, solves the schrodinger's
        equation.
        """

        # The number of basis set elements.
        nbasis = len(psigrid)

        # Constructs the Hamiltonian matrix.
        h = np.zeros((nbasis, nbasis))
        for i in range(nbasis):
            for j in range(i, nbasis, 1):
                h[i][j] = np.dot(psigrid[i] * psigrid[j], vgrid)
            h[i][i] *= 0.5
            h[i][i] += 0.5 * (i + 0.5) * hw
        h += h.T

        # Diagonalise Hamiltonian matrix and evaluate anharmonic free energy and vibrational freq
        evals, evecs = np.linalg.eigh(h)

        # Calculates the free and internal energy
        A = -np.max(evals) - self.imm.temp * np.log(np.sum(np.exp(-1.0 * (evals - np.max(evals)) / self.imm.temp))) * self.imm.temp
        E = np.sum(evals * np.exp(-1.0 * evals / self.imm.temp)) / np.sum(
            np.exp(-1.0 * evals / self.imm.temp)
        )

        if return_eigsys:
            return A, E, evals, evecs.T
        else:
            return A, E
//  [python]
def solve_schroedingers_equation(self, hw, psigrid, vgrid, return_eigsys=False):
        """
        Given the frequency of the HO, the wavefunctions and
        the gridded potential, solves the schrodinger's
        equation.
        """

        # The number of basis set elements.
        nbasis = len(psigrid)

        # Constructs the Hamiltonian matrix.
        h = np.zeros((nbasis, nbasis))
        for i in range(nbasis):
            for j in range(i, nbasis, 1):
                h[i][j] = np.dot(psigrid[i] * psigrid[j], vgrid)
            h[i][i] *= 0.5
            h[i][i] += 0.5 * (i + 0.5) * hw
        h += h.T

        # Diagonalise Hamiltonian matrix and evaluate anharmonic free energy and vibrational freq
        evals, evecs = np.linalg.eigh(h)

        # Calculates the free and internal energy
        A = -logsumexp(-1.0 * evals / self.imm.temp) * self.imm.temp
        E = np.sum(evals * np.exp(-1.0 * evals / self.imm.temp)) / np.sum(
            np.exp(-1.0 * evals / self.imm.temp)
        )

        if return_eigsys:
            return A, E, evals, evecs.T
        else:
            return A, E
//  [python]
def _logp(va, logits, act):
    logits = np.where(va, logits, -1e20)
    logp = logits - np.max(logits, axis=0, keepdims=True) - np.log(np.sum(np.exp(logits - np.max(logits, axis=0, keepdims=True)), axis=0, keepdims=True))

    logp = logp[
        act, \
        np.arange(act.shape[0]).reshape(-1, 1), \
        np.arange(act.shape[1]).reshape(1, -1), \
    ]

    is_valid = ~np.isinf(logp) & ~np.isnan(logp)
    invalid_pos = np.where(~is_valid)
    assert is_valid.all(), \
        f"invalid action at {invalid_pos}, with \n"\
        f"logits={logits[(slice(None),)+ invalid_pos]}\n"\
        f"act={act[invalid_pos]}\n"\
        f"logp={logp[invalid_pos]}\n"

    return logp
//  [python]
def _logp(va, logits, act):
    logits = np.where(va, logits, -1e20)
    logp = logits - scipy.special.logsumexp(logits, axis=0, keepdims=True)

    logp = logp[
        act, \
        np.arange(act.shape[0]).reshape(-1, 1), \
        np.arange(act.shape[1]).reshape(1, -1), \
    ]

    is_valid = ~np.isinf(logp) & ~np.isnan(logp)
    invalid_pos = np.where(~is_valid)
    assert is_valid.all(), \
        f"invalid action at {invalid_pos}, with \n"\
        f"logits={logits[(slice(None),)+ invalid_pos]}\n"\
        f"act={act[invalid_pos]}\n"\
        f"logp={logp[invalid_pos]}\n"

    return logp
//  [python]
def grad_outer_var(self, theta_flat, lmbda, idx):
        theta = theta_flat.reshape(self.n_features, self.n_classes)
        x = self.X[idx]
        y = self.y[idx]
        lbda = lmbda[idx]
        grad_lbda = np.zeros_like(lmbda)
        n_samples, n_features = x.shape
        prod = x @ theta
        weights = sc.expit(lbda)
        individual_losses = -prod[y == 1] + np.log(np.sum(np.exp(prod), axis=1))
        d_weights = weights - weights ** 2
        grad_lbda[idx] = d_weights * individual_losses / n_samples
        return grad_lbda
//  [python]
def grad_outer_var(self, theta_flat, lmbda, idx):
        theta = theta_flat.reshape(self.n_features, self.n_classes)
        x = self.X[idx]
        y = self.y[idx]
        lbda = lmbda[idx]
        grad_lbda = np.zeros_like(lmbda)
        n_samples, n_features = x.shape
        prod = x @ theta
        weights = sc.expit(lbda)
        individual_losses = -prod[y == 1] + sc.logsumexp(prod, axis=1)
        d_weights = weights - weights ** 2
        grad_lbda[idx] = d_weights * individual_losses / n_samples
        return grad_lbda
//  [python]
def grad(self, theta_flat, lmbda, idx):
        theta = theta_flat.reshape(self.n_features, self.n_classes)
        x = self.X[idx]
        y = self.y[idx]
        lbda = lmbda[idx]
        grad_lbda = np.zeros_like(lmbda)
        n_samples, n_features = x.shape
        prod = x @ theta
        Y_proba = sc.softmax(prod, axis=1)
        weights = sc.expit(lbda)
        individual_losses = -prod[y == 1] + np.max(prod, axis=1) + np.log(np.sum(np.exp(prod - np.max(prod, axis=1, keepdims=True)), axis=1))
        grad_theta = x.T @ ((Y_proba - y) * weights[:, None]) / n_samples
        d_weights = weights - weights ** 2
        grad_lbda[idx] = d_weights * individual_losses / n_samples
        return grad_theta.ravel() + 2 * self.reg * theta_flat, grad_lbda
//  [python]
def grad(self, theta_flat, lmbda, idx):
        theta = theta_flat.reshape(self.n_features, self.n_classes)
        x = self.X[idx]
        y = self.y[idx]
        lbda = lmbda[idx]
        grad_lbda = np.zeros_like(lmbda)
        n_samples, n_features = x.shape
        prod = x @ theta
        Y_proba = sc.softmax(prod, axis=1)
        weights = sc.expit(lbda)
        individual_losses = -prod[y == 1] + sc.logsumexp(prod, axis=1)
        grad_theta = x.T @ ((Y_proba - y) * weights[:, None]) / n_samples
        d_weights = weights - weights ** 2
        grad_lbda[idx] = d_weights * individual_losses / n_samples
        return grad_theta.ravel() + 2 * self.reg * theta_flat, grad_lbda
//  [python]
def MVG(DTE, DTR, LTR):
    h = {}

    for i in range(2):
        mu, C = ML_GAU(DTR[:, LTR == i])
        h[i] = (mu, C)

    SJoint = numpy.zeros((2, DTE.shape[1]))
    logSJoint = numpy.zeros((2, DTE.shape[1]))
    dens = numpy.zeros((2, DTE.shape[1]))
    classPriors = [0.5, 0.5]

    for label in range(2):
        mu, C = h[label]
        dens[label, :] = numpy.exp(logpdf_GAU_ND(DTE, mu, C).ravel())
        SJoint[label, :] = dens[label, :] * classPriors[label]
        logSJoint[label, :] = logpdf_GAU_ND(DTE, mu, C).ravel() + numpy.log(classPriors[label])

    SMarginal = SJoint.sum(0)
    logSMarginal = numpy.logaddexp.reduce(logSJoint, axis=0)

    Post1 = SJoint / mrow(SMarginal)
    logPost = logSJoint - mrow(logSMarginal)
    Post2 = numpy.exp(logPost)
    LPred1 = Post1.argmax(0)
    LPred2 = Post2.argmax(0)
    return LPred1, LPred2, numpy.log(dens[1] / dens[0])
//  [python]
def MVG(DTE, DTR, LTR):
    h = {}

    for i in range(2):
        mu, C = ML_GAU(DTR[:, LTR == i])
        h[i] = (mu, C)

    SJoint = numpy.zeros((2, DTE.shape[1]))
    logSJoint = numpy.zeros((2, DTE.shape[1]))
    dens = numpy.zeros((2, DTE.shape[1]))
    classPriors = [0.5, 0.5]

    for label in range(2):
        mu, C = h[label]
        dens[label, :] = numpy.exp(logpdf_GAU_ND(DTE, mu, C).ravel())
        SJoint[label, :] = dens[label, :] * classPriors[label]
        logSJoint[label, :] = logpdf_GAU_ND(DTE, mu, C).ravel() + numpy.log(classPriors[label])

    SMarginal = SJoint.sum(0)
    logSMarginal = scipy.special.logsumexp(logSJoint, axis=0)

    Post1 = SJoint / mrow(SMarginal)
    logPost = logSJoint - mrow(logSMarginal)
    Post2 = numpy.exp(logPost)
    LPred1 = Post1.argmax(0)
    LPred2 = Post2.argmax(0)
    return LPred1, LPred2, numpy.log(dens[1] / dens[0])
//  [python]
def marginal_density_GMM (S):
    return np.log(np.sum(np.exp(S), axis = 0))
//  [python]
def marginal_density_GMM (S):
    return scipy.special.logsumexp(S, axis = 0)
//  [python]
def log_softmax(x, axis):
  return x - scipy.special.logsumexp(x, axis=axis, keepdims=True)
//  [python]
def naive_MVG(DTE, DTR, LTR):
    h = {}

    for i in range(2):
        mu, C = ML_GAU(DTR[:, LTR == i])
        C = C * numpy.identity(C.shape[0])
        h[i] = (mu, C)

    SJoint = numpy.zeros((2, DTE.shape[1]))
    logSJoint = numpy.zeros((2, DTE.shape[1]))
    dens = numpy.zeros((2, DTE.shape[1]))
    classPriors = [0.5, 0.5]

    for label in range(2):
        mu, C = h[label]
        dens[label, :] = numpy.exp(logpdf_GAU_ND(DTE, mu, C).ravel())
        SJoint[label, :] = dens[label, :] * classPriors[label]
        logSJoint[label, :] = logpdf_GAU_ND(DTE, mu, C).ravel() + numpy.log(classPriors[label])

    SMarginal = SJoint.sum(0)
    logSMarginal = numpy.logaddexp.reduce(logSJoint, axis=0)

    Post1 = SJoint / mrow(SMarginal)
    logPost = logSJoint - mrow(logSMarginal)
    Post2 = numpy.exp(logPost)

    LPred1 = Post1.argmax(0)
    LPred2 = Post2.argmax(0)
    return LPred1, LPred2, numpy.log(dens[1] / dens[0])
//  [python]
def naive_MVG(DTE, DTR, LTR):
    h = {}

    for i in range(2):
        mu, C = ML_GAU(DTR[:, LTR == i])
        C = C * numpy.identity(C.shape[0])
        h[i] = (mu, C)

    SJoint = numpy.zeros((2, DTE.shape[1]))
    logSJoint = numpy.zeros((2, DTE.shape[1]))
    dens = numpy.zeros((2, DTE.shape[1]))
    classPriors = [0.5, 0.5]

    for label in range(2):
        mu, C = h[label]
        dens[label, :] = numpy.exp(logpdf_GAU_ND(DTE, mu, C).ravel())
        SJoint[label, :] = dens[label, :] * classPriors[label]
        logSJoint[label, :] = logpdf_GAU_ND(DTE, mu, C).ravel() + numpy.log(classPriors[label])

    SMarginal = SJoint.sum(0)
    logSMarginal = scipy.special.logsumexp(logSJoint, axis=0)

    Post1 = SJoint / mrow(SMarginal)
    logPost = logSJoint - mrow(logSMarginal)
    Post2 = numpy.exp(logPost)

    LPred1 = Post1.argmax(0)
    LPred2 = Post2.argmax(0)
    return LPred1, LPred2, numpy.log(dens[1] / dens[0])
//  [python]
def computeLogLikelihood(X, gmm):
    tempSum=np.zeros((len(gmm), X.shape[1]))
    for i in range(len(gmm)):
        tempSum[i,:]=np.log(gmm[i][0])+multivariateGaussian.logpdf_GAU_ND(X, gmm[i][1], gmm[i][2])
    return scipy.misc.logsumexp(tempSum, axis=0)
//  [python]
def computeLogLikelihood(X, gmm):
    # SHOULD BE FIXED
    tempSum=np.zeros((len(gmm), X.shape[1]))
    for i in range(len(gmm)):
        tempSum[i,:]=np.log(gmm[i][0])+multivariateGaussian.logpdf_GAU_ND(X, gmm[i][1], gmm[i][2])
    return scipy.special.logsumexp(tempSum, axis=0)
//  [python]
def test_autoregressive_have_normalize_distribution(machine_input, machine_class, machine_args):
    with graph.as_default():
        machine = machine_class(machine_input, **machine_args)
        model = Model(inputs=machine_input, outputs=machine.predictions)
        log_wave_function = to_log_wave_function_vector(model)
        log_distribution = numpy.real(2 * log_wave_function)
        log_distribution_sum = numpy.sum(numpy.exp(log_distribution))
        distribution_sum = numpy.exp(numpy.log(log_distribution_sum))
        assert distribution_sum == pytest.approx(1.0, 0.00001)
//  [python]
def test_autoregressive_have_normalize_distribution(machine_input, machine_class, machine_args):
    with graph.as_default():
        machine = machine_class(machine_input, **machine_args)
        model = Model(inputs=machine_input, outputs=machine.predictions)
        log_wave_function = to_log_wave_function_vector(model)
        log_distribution = numpy.real(2 * log_wave_function)
        log_distribution_sum = scipy.special.logsumexp(log_distribution)
        distribution_sum = numpy.exp(log_distribution_sum)
        assert distribution_sum == pytest.approx(1.0, 0.00001)
//  [python]
def tied_cov_GC(DTE, DTR, LTR):
    h = {}
    Ctot = 0
    for i in range(2):
        mu, C = ML_GAU(DTR[:, LTR == i])
        Ctot += DTR[:, LTR == i].shape[1] * C
        h[i] = (mu)

    Ctot = Ctot / DTR.shape[1]

    SJoint = numpy.zeros((2, DTE.shape[1]))
    logSJoint = numpy.zeros((2, DTE.shape[1]))
    dens = numpy.zeros((2, DTE.shape[1]))
    classPriors = [0.5, 0.5]

    for label in range(2):
        mu = h[label]
        dens[label, :] = numpy.exp(logpdf_GAU_ND(DTE, mu, Ctot).ravel())
        SJoint[label, :] = dens[label, :] * classPriors[label]
        logSJoint[label, :] = logpdf_GAU_ND(DTE, mu, Ctot).ravel() + numpy.log(classPriors[label])

    SMarginal = SJoint.sum(0)
    logSMarginal = s.logsumexp(logSJoint, axis=0)

    Post1 = SJoint / mrow(SMarginal)
    logPost = logSJoint - mrow(logSMarginal)
    Post2 = numpy.exp(logPost)

    LPred1 = Post1.argmax(0)
    LPred2 = Post2.argmax(0)
    return LPred1, LPred2, numpy.log(dens[1] / dens[0])
//  [python]
def tied_cov_GC(DTE, DTR, LTR):
    h = {}
    Ctot = 0
    for i in range(2):
        mu, C = ML_GAU(DTR[:, LTR == i])
        Ctot += DTR[:, LTR == i].shape[1] * C
        h[i] = (mu)

    Ctot = Ctot / DTR.shape[1]

    SJoint = numpy.zeros((2, DTE.shape[1]))
    logSJoint = numpy.zeros((2, DTE.shape[1]))
    dens = numpy.zeros((2, DTE.shape[1]))
    classPriors = [0.5, 0.5]

    for label in range(2):
        mu = h[label]
        dens[label, :] = numpy.exp(logpdf_GAU_ND(DTE, mu, Ctot).ravel())
        SJoint[label, :] = dens[label, :] * classPriors[label]
        logSJoint[label, :] = logpdf_GAU_ND(DTE, mu, Ctot).ravel() + numpy.log(classPriors[label])

    SMarginal = SJoint.sum(0)
    logSMarginal = scipy.special.logsumexp(logSJoint, axis=0)

    Post1 = SJoint / mrow(SMarginal)
    logPost = logSJoint - mrow(logSMarginal)
    Post2 = numpy.exp(logPost)

    LPred1 = Post1.argmax(0)
    LPred2 = Post2.argmax(0)
    return LPred1, LPred2, numpy.log(dens[1] / dens[0])
//  [python]
def tied_cov_naive_GC(DTE, DTR, LTR):
    h = {}
    Ctot = 0
    for i in range(2):
        mu, C = ML_GAU(DTR[:, LTR == i])
        Ctot += DTR[:, LTR == i].shape[1] * C
        h[i] = (mu)

    Ctot = Ctot / DTR.shape[1]
    Ctot = Ctot * numpy.identity(Ctot.shape[0])

    SJoint = numpy.zeros((2, DTE.shape[1]))
    logSJoint = numpy.zeros((2, DTE.shape[1]))
    dens = numpy.zeros((2, DTE.shape[1]))
    classPriors = [0.5, 0.5]

    for label in range(2):
        mu = h[label]
        dens[label, :] = numpy.exp(logpdf_GAU_ND(DTE, mu, Ctot).ravel())
        SJoint[label, :] = dens[label, :] * classPriors[label]
        logSJoint[label, :] = logpdf_GAU_ND(DTE, mu, Ctot).ravel() + numpy.log(classPriors[label])

    SMarginal = SJoint.sum(0)
    logSMarginal = s.logsumexp(logSJoint, axis=0)

    Post1 = SJoint / mrow(SMarginal)
    logPost = logSJoint - mrow(logSMarginal)
    Post2 = numpy.exp(logPost)

    LPred1 = Post1.argmax(0)
    LPred2 = Post2.argmax(0)
    return LPred1, LPred2, numpy.log(dens[1] / dens[0])
//  [python]
def tied_cov_naive_GC(DTE, DTR, LTR):
    h = {}
    Ctot = 0
    for i in range(2):
        mu, C = ML_GAU(DTR[:, LTR == i])
        Ctot += DTR[:, LTR == i].shape[1] * C
        h[i] = (mu)

    Ctot = Ctot / DTR.shape[1]
    Ctot = Ctot * numpy.identity(Ctot.shape[0])

    SJoint = numpy.zeros((2, DTE.shape[1]))
    logSJoint = numpy.zeros((2, DTE.shape[1]))
    dens = numpy.zeros((2, DTE.shape[1]))
    classPriors = [0.5, 0.5]

    for label in range(2):
        mu = h[label]
        dens[label, :] = numpy.exp(logpdf_GAU_ND(DTE, mu, Ctot).ravel())
        SJoint[label, :] = dens[label, :] * classPriors[label]
        logSJoint[label, :] = logpdf_GAU_ND(DTE, mu, Ctot).ravel() + numpy.log(classPriors[label])

    SMarginal = SJoint.sum(0)
    logSMarginal = scipy.special.logsumexp(logSJoint, axis=0)

    Post1 = SJoint / mrow(SMarginal)
    logPost = logSJoint - mrow(logSMarginal)
    Post2 = numpy.exp(logPost)

    LPred1 = Post1.argmax(0)
    LPred2 = Post2.argmax(0)
    return LPred1, LPred2, numpy.log(dens[1] / dens[0])
//  [python]
def association_probabilities(
        self,
        # measurements of shape=(M, m)=(#measurements, dim)
        Z: np.ndarray,
        filter_state: ET,
        *,
        sensor_state: Optional[Dict[str, Any]] = None,
    ) -> np.ndarray:  # beta, shape=(M + 1,): the association probabilities (normalized likelihood ratios)
        """calculate the poseterior event/association probabilities."""

        # log likelihoods
        lls = self.loglikelihood_ratios(Z, filter_state, sensor_state=sensor_state)

        # lls_normalized[ak] = log(alpha) + lls[ak]
        # log(alpha) = -logsumexp(ll[:])
        log_a = -np.logaddexp.reduce(lls)
        lls_normalized = log_a + lls

        # probabilities
        beta = np.exp(lls_normalized)
        return beta
//  [python]
def association_probabilities(
        self,
        # measurements of shape=(M, m)=(#measurements, dim)
        Z: np.ndarray,
        filter_state: ET,
        *,
        sensor_state: Optional[Dict[str, Any]] = None,
    ) -> np.ndarray:  # beta, shape=(M + 1,): the association probabilities (normalized likelihood ratios)
        """calculate the poseterior event/association probabilities."""

        # log likelihoods
        lls = self.loglikelihood_ratios(Z, filter_state, sensor_state=sensor_state)

        # lls_normalized[ak] = log(alpha) + lls[ak]
        # log(alpha) = -logsumexp(ll[:])
        log_a = -scipy.special.logsumexp(lls)
        lls_normalized = log_a + lls

        # probabilities
        beta = np.exp(lls_normalized)
        return beta
//  [python]
def _GMM_ll_per_sample(self, X, gmm):
        G = len(gmm)
        N = X.shape[1]
        S = numpy.zeros((G, N))
        
        for g in range(G):
            S[g, :] = self._logpdf_GAU_ND_Opt(X, gmm[g][1], gmm[g][2]) + numpy.log(gmm[g][0])
        return scipy.misc.logsumexp(S, axis=0)
//  [python]
def _GMM_ll_per_sample(self, X, gmm):
        G = len(gmm)
        N = X.shape[1]
        S = numpy.zeros((G, N))
        
        for g in range(G):
            S[g, :] = self._logpdf_GAU_ND_Opt(X, gmm[g][1], gmm[g][2]) + numpy.log(gmm[g][0])
        return scipy.special.logsumexp(S, axis=0)
//  [python]
def _value(self, x):
        Ax = self.mat_vec_product(x)
        regularization = 0
        if self.l2 != 0:
            regularization = self.l2/2 * self.norm(x)**2
        if self.least_squares_term:
            regularization += 1/2 * np.linalg.norm(Ax)**2
        return self.max_smoothing*scipy.misc.logsumexp(((Ax-self.b)/self.max_smoothing)) + regularization
//  [python]
def _value(self, x):
        Ax = self.mat_vec_product(x)
        regularization = 0
        if self.l2 != 0:
            regularization = self.l2/2 * self.norm(x)**2
        if self.least_squares_term:
            regularization += 1/2 * np.linalg.norm(Ax)**2
        return self.max_smoothing*scipy.special.logsumexp(((Ax-self.b)/self.max_smoothing)) + regularization
//  [python]
def _GMM_EM_diag(self, X, gmm):
        ll_new = None
        ll_old = None
        G = len(gmm)
        N = X.shape[1]
        
        psi = 0.01
        
        while ll_old is None or ll_new-ll_old>1e-6:
            ll_old = ll_new
            SJ = numpy.zeros((G, N))
            for g in range(G):
                SJ[g, :] = self._logpdf_GAU_ND_Opt(X, gmm[g][1], gmm[g][2]) + numpy.log(gmm[g][0])
            SM = numpy.log(numpy.sum(numpy.exp(SJ), axis=0))
            ll_new = SM.sum() / N
            P = numpy.exp(SJ - SM)
            
            gmm_new = []
            for g in range(G):
                gamma = P[g, :]
                Z = gamma.sum()
                F = (row(gamma)*X).sum(1)
                S = numpy.dot(X, (row(gamma)*X).T)
                w = Z/N
                mu = column(F/Z)
                sigma = S/Z - numpy.dot(mu, mu.T)
                #diagonalization
                sigma = sigma * numpy.eye(sigma.shape[0])
                
                #constraint
                U, s, _ = numpy.linalg.svd(sigma)
                s[s<psi] = psi
                sigma = numpy.dot(U, column(s)*U.T)
                
                gmm_new.append((w, mu, sigma))
            gmm = gmm_new
            #print(ll_new)
        #print(ll_new-ll_old)
        return gmm
//  [python]
def _GMM_EM_diag(self, X, gmm):
        ll_new = None
        ll_old = None
        G = len(gmm)
        N = X.shape[1]
        
        psi = 0.01
        
        while ll_old is None or ll_new-ll_old>1e-6:
            ll_old = ll_new
            SJ = numpy.zeros((G, N))
            for g in range(G):
                SJ[g, :] = self._logpdf_GAU_ND_Opt(X, gmm[g][1], gmm[g][2]) + numpy.log(gmm[g][0])
            SM = scipy.special.logsumexp(SJ, axis=0)
            ll_new = SM.sum() / N
            P = numpy.exp(SJ - SM)
            
            gmm_new = []
            for g in range(G):
                gamma = P[g, :]
                Z = gamma.sum()
                F = (row(gamma)*X).sum(1)
                S = numpy.dot(X, (row(gamma)*X).T)
                w = Z/N
                mu = column(F/Z)
                sigma = S/Z - numpy.dot(mu, mu.T)
                #diagonalization
                sigma = sigma * numpy.eye(sigma.shape[0])
                
                #constraint
                U, s, _ = numpy.linalg.svd(sigma)
                s[s<psi] = psi
                sigma = numpy.dot(U, column(s)*U.T)
                
                gmm_new.append((w, mu, sigma))
            gmm = gmm_new
            #print(ll_new)
        #print(ll_new-ll_old)
        return gmm
//  [python]
def _GMM_EM(self, X, gmm):
        ll_new = None
        ll_old = None
        G = len(gmm)
        N = X.shape[1]
        
        psi = 0.01
        
        while ll_old is None or ll_new-ll_old>1e-6:
            ll_old = ll_new
            SJ = numpy.zeros((G, N))
            for g in range(G):
                SJ[g, :] = self._logpdf_GAU_ND_Opt(X, gmm[g][1], gmm[g][2]) + numpy.log(gmm[g][0])
            SM = numpy.max(SJ, axis=0) + numpy.log(numpy.sum(numpy.exp(SJ - numpy.max(SJ, axis=0)), axis=0))
            ll_new = SM.sum() / N
            P = numpy.exp(SJ - SM)
            
            gmm_new = []
            for g in range(G):
                gamma = P[g, :]
                Z = gamma.sum()
                F = (row(gamma)*X).sum(1)
                S = numpy.dot(X, (row(gamma)*X).T)
                w = Z/N
                mu = column(F/Z)
                sigma = S/Z - numpy.dot(mu, mu.T)
                #constraint
                U, s, _ = numpy.linalg.svd(sigma)
                s[s<psi] = psi
                sigma = numpy.dot(U, column(s)*U.T)
                
                gmm_new.append((w, mu, sigma))
            gmm = gmm_new
            #print(ll_new)
        #print(ll_new-ll_old)
        return gmm
//  [python]
def _GMM_EM(self, X, gmm):
        ll_new = None
        ll_old = None
        G = len(gmm)
        N = X.shape[1]
        
        psi = 0.01
        
        while ll_old is None or ll_new-ll_old>1e-6:
            ll_old = ll_new
            SJ = numpy.zeros((G, N))
            for g in range(G):
                SJ[g, :] = self._logpdf_GAU_ND_Opt(X, gmm[g][1], gmm[g][2]) + numpy.log(gmm[g][0])
            SM = scipy.special.logsumexp(SJ, axis=0)
            ll_new = SM.sum() / N
            P = numpy.exp(SJ - SM)
            
            gmm_new = []
            for g in range(G):
                gamma = P[g, :]
                Z = gamma.sum()
                F = (row(gamma)*X).sum(1)
                S = numpy.dot(X, (row(gamma)*X).T)
                w = Z/N
                mu = column(F/Z)
                sigma = S/Z - numpy.dot(mu, mu.T)
                #constraint
                U, s, _ = numpy.linalg.svd(sigma)
                s[s<psi] = psi
                sigma = numpy.dot(U, column(s)*U.T)
                
                gmm_new.append((w, mu, sigma))
            gmm = gmm_new
            #print(ll_new)
        #print(ll_new-ll_old)
        return gmm
//  [python]
def ensemble_metrics(x,
                     y,
                     model,
                     log_likelihood_fn,
                     n_samples=1,
                     weight_files=None):
  """Evaluate metrics of an ensemble.

  Args:
    x: numpy array of inputs
    y: numpy array of labels
    model: tf.keras.Model.
    log_likelihood_fn: function which takes tuple of x, y and returns batched
      tuple output of the log prob and mean error.
    n_samples: number of Monte Carlo samples to draw per ensemble member (each
      weight file).
    weight_files: to draw samples from multiple weight sets, specify a list of
      weight files to load. These files must have been generated through
      keras's model.save_weights(...).

  Returns:
    metrics_dict: dictionary containing the metrics
  """
  if weight_files is None:
    ensemble_logprobs = [log_likelihood_fn([x, y])[0] for _ in range(n_samples)]
    ensemble_error = [log_likelihood_fn([x, y])[1] for _ in range(n_samples)]
  else:
    ensemble_logprobs = []
    ensemble_error = []
    for filename in weight_files:
      model.load_weights(filename)
      ensemble_logprobs.extend([
          log_likelihood_fn([x, y])[0] for _ in range(n_samples)])
      ensemble_error.extend([
          log_likelihood_fn([x, y])[1] for _ in range(n_samples)])

  results = {}
  ensemble_logprobs = np.array(ensemble_logprobs)
  results['log_likelihood'] = np.mean(ensemble_logprobs)
  results['mse'] = np.mean(np.square(ensemble_error))
  probabilistic_log_likelihood = np.mean(
      scipy.special.logsumexp(
          np.sum(ensemble_logprobs, axis=2)
          if len(ensemble_logprobs.shape) > 2 else ensemble_logprobs,
          b=1. / ensemble_logprobs.shape[0],
          axis=0),
      axis=0)
  results['probabilistic_log_likelihood'] = probabilistic_log_likelihood
  ensemble_error = np.stack([np.array(l) for l in ensemble_error])
  results['probabilistic_mse'] = np.mean(
      np.square(np.mean(ensemble_error, axis=0)))
  return results
//  [python]
def _GMM_EM_tied(self, X, gmm):
        ll_new = None
        ll_old = None
        G = len(gmm)
        N = X.shape[1]
        
        psi = 0.01
        
        while ll_old is None or ll_new-ll_old>1e-6:
            ll_old = ll_new
            SJ = numpy.zeros((G, N))
            for g in range(G):
                SJ[g, :] = self._logpdf_GAU_ND_Opt(X, gmm[g][1], gmm[g][2]) + numpy.log(gmm[g][0])
            SM = numpy.max(SJ, axis=0)
            SM += numpy.log(numpy.sum(numpy.exp(SJ - SM), axis=0))
            ll_new = SM.sum() / N
            P = numpy.exp(SJ - SM)
            
            gmm_new = []
            summatory = numpy.zeros((X.shape[0], X.shape[0]))
            for g in range(G):
                gamma = P[g, :]
                Z = gamma.sum()
                F = (row(gamma)*X).sum(1)
                S = numpy.dot(X, (row(gamma)*X).T)
                w = Z/N
                mu = column(F/Z)
                sigma = S/Z - numpy.dot(mu, mu.T)
                summatory += Z*sigma
                gmm_new.append((w, mu, sigma))
            #tying
            sigma = summatory / G
            #constraint
            U, s, _ = numpy.linalg.svd(sigma)
            s[s<psi] = psi
            sigma = numpy.dot(U, column(s)*U.T)
            gmm = gmm_new
            #print(ll_new)
        #print(ll_new-ll_old)
        return gmm
//  [python]
def _GMM_EM_tied(self, X, gmm):
        ll_new = None
        ll_old = None
        G = len(gmm)
        N = X.shape[1]
        
        psi = 0.01
        
        while ll_old is None or ll_new-ll_old>1e-6:
            ll_old = ll_new
            SJ = numpy.zeros((G, N))
            for g in range(G):
                SJ[g, :] = self._logpdf_GAU_ND_Opt(X, gmm[g][1], gmm[g][2]) + numpy.log(gmm[g][0])
            SM = scipy.special.logsumexp(SJ, axis=0)
            ll_new = SM.sum() / N
            P = numpy.exp(SJ - SM)
            
            gmm_new = []
            summatory = numpy.zeros((X.shape[0], X.shape[0]))
            for g in range(G):
                gamma = P[g, :]
                Z = gamma.sum()
                F = (row(gamma)*X).sum(1)
                S = numpy.dot(X, (row(gamma)*X).T)
                w = Z/N
                mu = column(F/Z)
                sigma = S/Z - numpy.dot(mu, mu.T)
                summatory += Z*sigma
                gmm_new.append((w, mu, sigma))
            #tying
            sigma = summatory / G
            #constraint
            U, s, _ = numpy.linalg.svd(sigma)
            s[s<psi] = psi
            sigma = numpy.dot(U, column(s)*U.T)
            gmm = gmm_new
            #print(ll_new)
        #print(ll_new-ll_old)
        return gmm
//  [python]
def _GMM_EM_diag_tied(self, X, gmm):
        ll_new = None
        ll_old = None
        G = len(gmm)
        N = X.shape[1]
        
        psi = 0.01
        
        while ll_old is None or ll_new-ll_old>1e-6:
            ll_old = ll_new
            SJ = numpy.zeros((G, N))
            for g in range(G):
                SJ[g, :] = self._logpdf_GAU_ND_Opt(X, gmm[g][1], gmm[g][2]) + numpy.log(gmm[g][0])
            SM = numpy.max(SJ, axis=0)
            SM = SM + numpy.log(numpy.sum(numpy.exp(SJ - SM), axis=0))
            ll_new = SM.sum() / N
            P = numpy.exp(SJ - SM)
            
            gmm_new = []
            summatory = numpy.zeros((X.shape[0], X.shape[0]))
            for g in range(G):
                gamma = P[g, :]
                Z = gamma.sum()
                F = (row(gamma)*X).sum(1)
                S = numpy.dot(X, (row(gamma)*X).T)
                w = Z/N
                mu = column(F/Z)
                sigma = S/Z - numpy.dot(mu, mu.T)
                #diagonalization
                sigma = sigma * numpy.eye(sigma.shape[0])
                summatory += Z*sigma
                gmm_new.append((w, mu, sigma))
            #tying
            sigma = summatory / G
            #constraint
            U, s, _ = numpy.linalg.svd(sigma)
            s[s<psi] = psi
            sigma = numpy.dot(U, column(s)*U.T)
            gmm = gmm_new
            #print(ll_new)
        #print(ll_new-ll_old)
        return gmm
//  [python]
def _GMM_EM_diag_tied(self, X, gmm):
        ll_new = None
        ll_old = None
        G = len(gmm)
        N = X.shape[1]
        
        psi = 0.01
        
        while ll_old is None or ll_new-ll_old>1e-6:
            ll_old = ll_new
            SJ = numpy.zeros((G, N))
            for g in range(G):
                SJ[g, :] = self._logpdf_GAU_ND_Opt(X, gmm[g][1], gmm[g][2]) + numpy.log(gmm[g][0])
            SM = scipy.special.logsumexp(SJ, axis=0)
            ll_new = SM.sum() / N
            P = numpy.exp(SJ - SM)
            
            gmm_new = []
            summatory = numpy.zeros((X.shape[0], X.shape[0]))
            for g in range(G):
                gamma = P[g, :]
                Z = gamma.sum()
                F = (row(gamma)*X).sum(1)
                S = numpy.dot(X, (row(gamma)*X).T)
                w = Z/N
                mu = column(F/Z)
                sigma = S/Z - numpy.dot(mu, mu.T)
                #diagonalization
                sigma = sigma * numpy.eye(sigma.shape[0])
                summatory += Z*sigma
                gmm_new.append((w, mu, sigma))
            #tying
            sigma = summatory / G
            #constraint
            U, s, _ = numpy.linalg.svd(sigma)
            s[s<psi] = psi
            sigma = numpy.dot(U, column(s)*U.T)
            gmm = gmm_new
            #print(ll_new)
        #print(ll_new-ll_old)
        return gmm
//  [python]
def __init__(self,
                 task_queue,

                 model_parameters,

                 optimize_doc_lambda,
                 # optimize_doc_nu_square,
                 optimize_doc_nu_square_in_log_space,

                 result_doc_parameter_queue,
                 result_log_likelihood_queue,
                 result_sufficient_statistics_queue,

                 diagonal_covariance_matrix=False,

                 parameter_iteration=10,
                 parameter_converge_threshold=1e-3):
        multiprocessing.Process.__init__(self);

        self._task_queue = task_queue;
        self._result_doc_parameter_queue = result_doc_parameter_queue;
        self._result_log_likelihood_queue = result_log_likelihood_queue;
        self._result_sufficient_statistics_queue = result_sufficient_statistics_queue;

        self._parameter_iteration = parameter_iteration;

        self._diagonal_covariance_matrix = diagonal_covariance_matrix;
        if self._diagonal_covariance_matrix:
            (self._E_log_eta, self._alpha_mu, self._alpha_sigma) = model_parameters;
        else:
            (self._E_log_eta, self._alpha_mu, self._alpha_sigma, self._alpha_sigma_inv) = model_parameters;
        (self._number_of_topics, self._number_of_types) = self._E_log_eta.shape;

        if result_sufficient_statistics_queue != None:
            self._E_log_prob_eta = self._E_log_eta - scipy.misc.logsumexp(self._E_log_eta, axis=1)[:, numpy.newaxis]

        self.optimize_doc_lambda = optimize_doc_lambda;
        # self.optimize_doc_nu_square = optimize_doc_nu_square;
        self.optimize_doc_nu_square_in_log_space = optimize_doc_nu_square_in_log_space;
//  [python]
def __init__(self,
                 task_queue,

                 model_parameters,

                 optimize_doc_lambda,
                 # optimize_doc_nu_square,
                 optimize_doc_nu_square_in_log_space,

                 result_doc_parameter_queue,
                 result_log_likelihood_queue,
                 result_sufficient_statistics_queue,

                 diagonal_covariance_matrix=False,

                 parameter_iteration=10,
                 parameter_converge_threshold=1e-3):
        multiprocessing.Process.__init__(self);

        self._task_queue = task_queue;
        self._result_doc_parameter_queue = result_doc_parameter_queue;
        self._result_log_likelihood_queue = result_log_likelihood_queue;
        self._result_sufficient_statistics_queue = result_sufficient_statistics_queue;

        self._parameter_iteration = parameter_iteration;

        self._diagonal_covariance_matrix = diagonal_covariance_matrix;
        if self._diagonal_covariance_matrix:
            (self._E_log_eta, self._alpha_mu, self._alpha_sigma) = model_parameters;
        else:
            (self._E_log_eta, self._alpha_mu, self._alpha_sigma, self._alpha_sigma_inv) = model_parameters;
        (self._number_of_topics, self._number_of_types) = self._E_log_eta.shape;

        if result_sufficient_statistics_queue != None:
            self._E_log_prob_eta = self._E_log_eta - scipy.special.logsumexp(self._E_log_eta, axis=1)[:, numpy.newaxis]

        self.optimize_doc_lambda = optimize_doc_lambda;
        # self.optimize_doc_nu_square = optimize_doc_nu_square;
        self.optimize_doc_nu_square_in_log_space = optimize_doc_nu_square_in_log_space;
//  [python]
def f_doc_lambda(self, doc_lambda, *args):
        (doc_nu_square, doc_zeta_factor, sum_phi, total_word_count) = args;

        assert doc_zeta_factor.shape == (self._number_of_topics, self._number_of_topics);
        assert sum_phi.shape == (self._number_of_topics,)
        # if doc_lambda.shape==(1, self._number_of_topics):
        # doc_lambda = doc_lambda[0, :];
        assert doc_lambda.shape == (self._number_of_topics,);

        exp_over_doc_zeta = scipy.misc.logsumexp(
            doc_zeta_factor - doc_lambda[:, numpy.newaxis] - 0.5 * doc_nu_square[:, numpy.newaxis], axis=1)
        exp_over_doc_zeta = numpy.exp(-exp_over_doc_zeta);

        function_doc_lambda = numpy.sum(sum_phi * doc_lambda);

        if self._diagonal_covariance_matrix:
            mean_adjustment = doc_lambda - self._alpha_mu
            assert mean_adjustment.shape == (self._number_of_topics,);
            function_doc_lambda += -0.5 * numpy.sum((mean_adjustment ** 2) / self._alpha_sigma)
        else:
            mean_adjustment = doc_lambda[numpy.newaxis, :] - self._alpha_mu;
            assert mean_adjustment.shape == (1, self._number_of_topics), (
            doc_lambda.shape, mean_adjustment.shape, self._alpha_mu.shape)
            function_doc_lambda += -0.5 * numpy.dot(numpy.dot(mean_adjustment, self._alpha_sigma_inv),
                                                    mean_adjustment.T)

        function_doc_lambda += -total_word_count * numpy.sum(exp_over_doc_zeta);

        return numpy.asscalar(-function_doc_lambda);
//  [python]
def f_doc_lambda(self, doc_lambda, *args):
        (doc_nu_square, doc_zeta_factor, sum_phi, total_word_count) = args;

        assert doc_zeta_factor.shape == (self._number_of_topics, self._number_of_topics);
        assert sum_phi.shape == (self._number_of_topics,)
        # if doc_lambda.shape==(1, self._number_of_topics):
        # doc_lambda = doc_lambda[0, :];
        assert doc_lambda.shape == (self._number_of_topics,);

        exp_over_doc_zeta = scipy.special.logsumexp(
            doc_zeta_factor - doc_lambda[:, numpy.newaxis] - 0.5 * doc_nu_square[:, numpy.newaxis], axis=1)
        exp_over_doc_zeta = numpy.exp(-exp_over_doc_zeta);

        function_doc_lambda = numpy.sum(sum_phi * doc_lambda);

        if self._diagonal_covariance_matrix:
            mean_adjustment = doc_lambda - self._alpha_mu
            assert mean_adjustment.shape == (self._number_of_topics,);
            function_doc_lambda += -0.5 * numpy.sum((mean_adjustment ** 2) / self._alpha_sigma)
        else:
            mean_adjustment = doc_lambda[numpy.newaxis, :] - self._alpha_mu;
            assert mean_adjustment.shape == (1, self._number_of_topics), (
            doc_lambda.shape, mean_adjustment.shape, self._alpha_mu.shape)
            function_doc_lambda += -0.5 * numpy.dot(numpy.dot(mean_adjustment, self._alpha_sigma_inv),
                                                    mean_adjustment.T)

        function_doc_lambda += -total_word_count * numpy.sum(exp_over_doc_zeta);

        return numpy.asscalar(-function_doc_lambda);
//  [python]
def f_prime_doc_lambda(self, doc_lambda, *args):
        (doc_nu_square, doc_zeta_factor, sum_phi, total_word_count) = args;

        assert doc_zeta_factor.shape == (self._number_of_topics, self._number_of_topics);
        assert sum_phi.shape == (self._number_of_topics,)

        exp_over_doc_zeta = scipy.misc.logsumexp(
            doc_zeta_factor - doc_lambda[:, numpy.newaxis] - 0.5 * doc_nu_square[:, numpy.newaxis], axis=1)
        exp_over_doc_zeta = numpy.exp(-exp_over_doc_zeta);
        assert exp_over_doc_zeta.shape == (self._number_of_topics,);

        if self._diagonal_covariance_matrix:
            function_prime_doc_lambda = (self._alpha_mu - doc_lambda) / self._alpha_sigma;
        else:
            function_prime_doc_lambda = numpy.dot((self._alpha_mu - doc_lambda[numpy.newaxis, :]),
                                                  self._alpha_sigma_inv)[0, :]

        function_prime_doc_lambda += sum_phi;
        function_prime_doc_lambda -= total_word_count * exp_over_doc_zeta;

        assert function_prime_doc_lambda.shape == (self._number_of_topics,);

        return numpy.asarray(-function_prime_doc_lambda);
//  [python]
def f_prime_doc_lambda(self, doc_lambda, *args):
        (doc_nu_square, doc_zeta_factor, sum_phi, total_word_count) = args;

        assert doc_zeta_factor.shape == (self._number_of_topics, self._number_of_topics);
        assert sum_phi.shape == (self._number_of_topics,)

        exp_over_doc_zeta = scipy.special.logsumexp(
            doc_zeta_factor - doc_lambda[:, numpy.newaxis] - 0.5 * doc_nu_square[:, numpy.newaxis], axis=1)
        exp_over_doc_zeta = numpy.exp(-exp_over_doc_zeta);
        assert exp_over_doc_zeta.shape == (self._number_of_topics,);

        if self._diagonal_covariance_matrix:
            function_prime_doc_lambda = (self._alpha_mu - doc_lambda) / self._alpha_sigma;
        else:
            function_prime_doc_lambda = numpy.dot((self._alpha_mu - doc_lambda[numpy.newaxis, :]),
                                                  self._alpha_sigma_inv)[0, :]

        function_prime_doc_lambda += sum_phi;
        function_prime_doc_lambda -= total_word_count * exp_over_doc_zeta;

        assert function_prime_doc_lambda.shape == (self._number_of_topics,);

        return numpy.asarray(-function_prime_doc_lambda);