jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def digraph_test_GMM(ref_points, test_points, algo_params={}):
    """Perform anomaly detection, with Gaussian Mixture Components according to my algorithm.
    Returns
    -------
    (list 1xM where M number of components) The weighted number of predicted inliers for each component
    """
    N_COMPONENTS = algo_params['n_components']
    DELTA = algo_params['delta']

    # Construct the cluster(s)
    gmm_model = mixture.GaussianMixture(n_components=N_COMPONENTS)
    gmm_model.fit(ref_points)
    weights = gmm_model.weights_
    means = gmm_model.means_
    covs = gmm_model.covariances_

    # Get the number of points for each cluster of trained data.
    train_labels = gmm_model.predict(ref_points)
    # stds = []
    # # Reshape covs and get stds
    # for c in gmm_model.covariances_:
    #     vec = np.sqrt(np.array([[c[i, i] for i in range(c.shape[0])]]))
    #     vec = np.array([[c[i, i] for i in range(c.shape[0])]])
    #     if stds == []:
    #         stds = vec
    #     else:
    #         stds = np.append(stds, vec, axis=0)

    # Compute score for each component
    score_comp = N_COMPONENTS * [0.]
    for m in range(N_COMPONENTS):

        # If a cluster has a limited number of points, ignore it
        if len([l for l in train_labels if l == m]) < 2:
            continue

        # Test each point
        _pass = 0.
        for test_point in test_points:

            # With malahnobis
            mlh = mahalanobis(test_point, means[m], np.linalg.inv(covs[m]))
            if mlh <= DELTA:
                _pass += 1

            # My first way:
            # flag = True
            # for dim, val in enumerate(test_point):
            #     if not (val >= means[m, dim] - DELTA * stds[m, dim] and val <= means[m, dim] + DELTA * stds[m, dim]):
            #         flag = False
            #         break
            # if flag is True:
            #     _pass += 1

        # Scale with the weight of component
        score_comp[m] += weights[m] * _pass

    # Return score
    return score_comp
//  [python]
def digraph_test_GMM(ref_points, test_points, algo_params={}):
    """Perform anomaly detection, with Gaussian Mixture Components according to my algorithm.
    Returns
    -------
    (list 1xM where M number of components) The weighted number of predicted inliers for each component
    """
    N_COMPONENTS = algo_params['n_components']
    DELTA = algo_params['delta']

    # Construct the cluster(s)
    gmm_model = GaussianMixture(n_components=N_COMPONENTS)
    gmm_model.fit(ref_points)
    weights = gmm_model.weights_
    means = gmm_model.means_
    covs = gmm_model.covariances_

    # Get the number of points for each cluster of trained data.
    train_labels = gmm_model.predict(ref_points)
    # stds = []
    # # Reshape covs and get stds
    # for c in gmm_model.covariances_:
    #     vec = np.sqrt(np.array([[c[i, i] for i in range(c.shape[0])]]))
    #     vec = np.array([[c[i, i] for i in range(c.shape[0])]])
    #     if stds == []:
    #         stds = vec
    #     else:
    #         stds = np.append(stds, vec, axis=0)

    # Compute score for each component
    score_comp = N_COMPONENTS * [0.]
    for m in range(N_COMPONENTS):

        # If a cluster has a limited number of points, ignore it
        if len([l for l in train_labels if l == m]) < 2:
            continue

        # Test each point
        _pass = 0.
        for test_point in test_points:

            # With malahnobis
            mlh = mahalanobis(test_point, means[m], np.linalg.inv(covs[m]))
            if mlh <= DELTA:
                _pass += 1

            # My first way:
            # flag = True
            # for dim, val in enumerate(test_point):
            #     if not (val >= means[m, dim] - DELTA * stds[m, dim] and val <= means[m, dim] + DELTA * stds[m, dim]):
            #         flag = False
            #         break
            # if flag is True:
            #     _pass += 1

        # Scale with the weight of component
        score_comp[m] += weights[m] * _pass

    # Return score
    return score_comp
//  [python]
def fit(  # type: ignore[override]
        self,
        x_ref: np.ndarray,
        tol: float = 1e-3,
        max_iter: int = 100,
        n_init: int = 1,
        init_params: str = 'kmeans',
        verbose: int = 0,
    ) -> Dict:
        """Fit the SKLearn GMM model`.

        Parameters
        ----------
        x_ref
            Reference data.
        tol
            Convergence threshold. EM iterations will stop when the lower bound average gain is below this threshold.
        max_iter
            Maximum number of EM iterations to perform.
        n_init
            Number of initializations to perform.
        init_params
            Method used to initialize the weights, the means and the precisions. Must be one of:
            'kmeans' : responsibilities are initialized using kmeans.
            'kmeans++' : responsibilities are initialized using kmeans++.
            'random' : responsibilities are initialized randomly.
            'random_from_data' : responsibilities are initialized randomly from the data.
        verbose
            Enable verbose output. If 1 then it prints the current initialization and each iteration step. If greater
            than 1 then it prints also the log probability and the time needed for each step.

        Returns
        -------
        Dictionary with fit results. The dictionary contains the following keys:
            - converged: bool indicating whether EM algorithm converged.
            - n_iter: number of EM iterations performed.
            - lower_bound: log-likelihood lower bound.
        """
        self.gmm = mixture.GaussianMixture(
            n_components=self.n_components,
            tol=tol,
            max_iter=max_iter,
            n_init=n_init,
            init_params=init_params,
            verbose=verbose,
        )
        self.gmm = self.gmm.fit(
            x_ref,
        )
        self._set_fitted()
        return {
            'converged': self.gmm.converged_,
            'n_iter': self.gmm.n_iter_,
            'lower_bound': self.gmm.lower_bound_
        }
//  [python]
def fit(  # type: ignore[override]
        self,
        x_ref: np.ndarray,
        tol: float = 1e-3,
        max_iter: int = 100,
        n_init: int = 1,
        init_params: str = 'kmeans',
        verbose: int = 0,
    ) -> Dict:
        """Fit the SKLearn GMM model`.

        Parameters
        ----------
        x_ref
            Reference data.
        tol
            Convergence threshold. EM iterations will stop when the lower bound average gain is below this threshold.
        max_iter
            Maximum number of EM iterations to perform.
        n_init
            Number of initializations to perform.
        init_params
            Method used to initialize the weights, the means and the precisions. Must be one of:
            'kmeans' : responsibilities are initialized using kmeans.
            'kmeans++' : responsibilities are initialized using kmeans++.
            'random' : responsibilities are initialized randomly.
            'random_from_data' : responsibilities are initialized randomly from the data.
        verbose
            Enable verbose output. If 1 then it prints the current initialization and each iteration step. If greater
            than 1 then it prints also the log probability and the time needed for each step.

        Returns
        -------
        Dictionary with fit results. The dictionary contains the following keys:
            - converged: bool indicating whether EM algorithm converged.
            - n_iter: number of EM iterations performed.
            - lower_bound: log-likelihood lower bound.
        """
        self.gmm = GaussianMixture(
            n_components=self.n_components,
            tol=tol,
            max_iter=max_iter,
            n_init=n_init,
            init_params=init_params,
            verbose=verbose,
        )
        self.gmm = self.gmm.fit(
            x_ref,
        )
        self._set_fitted()
        return {
            'converged': self.gmm.converged_,
            'n_iter': self.gmm.n_iter_,
            'lower_bound': self.gmm.lower_bound_
        }
//  [python]
def fit_gmm(self, train_loader, model, model_idx):    
        model.eval()

        losses = torch.zeros(len(train_loader.dataset))    
        with torch.no_grad():
            for batch_idx, (inputs, targets, index) in enumerate(train_loader):
                inputs, targets = inputs.cuda(), targets.cuda() 
                outputs = model(inputs) 
                loss = self.ce(outputs, targets)  
                for b in range(inputs.size(0)):
                    losses[index[b]] = loss[b]         
        losses = (losses - losses.min()) / (losses.max() - losses.min())    
        self.all_loss[model_idx].append(losses)

        history = torch.stack(self.all_loss[model_idx])
        input_loss = history[-5:].mean(0)
        input_loss = input_loss.reshape(-1,1)
        # input_loss = losses.reshape(-1,1)
        
        # fit a two-component GMM to the loss
        gmm = mixture.GaussianMixture(n_components=2, max_iter=10, tol=1e-2, reg_covar=5e-4)
        gmm.fit(input_loss)
        prob = gmm.predict_proba(input_loss) 
        prob = prob[:, gmm.means_.argmin()]       

        return prob
//  [python]
def fit_gmm(self, train_loader, model, model_idx):    
        model.eval()

        losses = torch.zeros(len(train_loader.dataset))    
        with torch.no_grad():
            for batch_idx, (inputs, targets, index) in enumerate(train_loader):
                inputs, targets = inputs.cuda(), targets.cuda() 
                outputs = model(inputs) 
                loss = self.ce(outputs, targets)  
                for b in range(inputs.size(0)):
                    losses[index[b]] = loss[b]         
        losses = (losses - losses.min()) / (losses.max() - losses.min())    
        self.all_loss[model_idx].append(losses)

        history = torch.stack(self.all_loss[model_idx])
        input_loss = history[-5:].mean(0)
        input_loss = input_loss.reshape(-1,1)
        # input_loss = losses.reshape(-1,1)
        
        # fit a two-component GMM to the loss
        gmm = GaussianMixture(n_components=2, max_iter=10, tol=1e-2, reg_covar=5e-4)
        gmm.fit(input_loss)
        prob = gmm.predict_proba(input_loss) 
        prob = prob[:, gmm.means_.argmin()]       

        return prob
//  [python]
def initialize_gmm(self, data):
        with torch.no_grad():
            mu, logvar = self.encoder(data)
            z = reparameterize(mu, logvar, self.training)
        z = z.cpu().detach().numpy()
        gmm = mixture.GaussianMixture(n_components=self.K, covariance_type='diag')
        gmm.fit(z)
        self.pi.data = torch.FloatTensor(gmm.weights_).to(device)
        self.mu.data = torch.FloatTensor(gmm.means_).to(device)
        self.logvar.data = torch.log(torch.FloatTensor(gmm.covariances_)).to(device)
//  [python]
def initialize_gmm(self, data):
        with torch.no_grad():
            mu, logvar = self.encoder(data)
            z = reparameterize(mu, logvar, self.training)
        z = z.cpu().detach().numpy()
        gmm = GaussianMixture(n_components=self.K, covariance_type='diag')
        gmm.fit(z)
        self.pi.data = torch.FloatTensor(gmm.weights_).to(device)
        self.mu.data = torch.FloatTensor(gmm.means_).to(device)
        self.logvar.data = torch.log(torch.FloatTensor(gmm.covariances_)).to(device)
//  [python]
def _fit_gmm(n_components: int, dataset: Dataset) -> GaussianMixture:
        """
        Estimate the density of the train part of the dataset using a GGM
        """

        print("Start estimate of GaussianMixture")
        start_time = time.time()

        # transform the data to numpy, keep only the feature to estimate the density
        X = np.asarray(dataset[DatasetColumnsEnum.mts.value]).reshape(len(dataset), -1)

        # compute the density
        GMM = mixture.GaussianMixture(
            n_components=n_components,
            covariance_type="full",
            init_params="k-means++",
            random_state=1,
        )

        res = GMM.fit(X)

        print(f"Finished fitting GaussianMixture, time took: {time.time()-start_time}")

        return res
//  [python]
def _fit_gmm(n_components: int, dataset: Dataset) -> GaussianMixture:
        """
        Estimate the density of the train part of the dataset using a GGM
        """

        print("Start estimate of GaussianMixture")
        start_time = time.time()

        # transform the data to numpy, keep only the feature to estimate the density
        X = np.asarray(dataset[DatasetColumnsEnum.mts.value]).reshape(len(dataset), -1)

        # compute the density
        GMM = GaussianMixture(
            n_components=n_components,
            covariance_type="full",
            init_params="k-means++",
            random_state=1,
        )

        res = GMM.fit(X)

        print(f"Finished fitting GaussianMixture, time took: {time.time()-start_time}")

        return res
//  [python]
def fit_GMM(X, max_clusters=21):
    # Number of clusters
    clusters = find_number_of_clusters(X, plot=False, max_clusters=10)
    print('Number of clusters with BIC : ', clusters)

    # Fit GMM model to sample
    gmm = mixture.GaussianMixture(n_components=clusters).fit(X)
    labels = gmm.predict(X)

    return clusters, labels
//  [python]
def fit_GMM(X, max_clusters=21):
    # Number of clusters
    clusters = find_number_of_clusters(X, plot=False, max_clusters=10)
    print('Number of clusters with BIC : ', clusters)

    # Fit GMM model to sample
    gmm = GaussianMixture(n_components=clusters).fit(X)
    labels = gmm.predict(X)

    return clusters, labels
//  [python]
def find_number_of_clusters(X, plot=True, max_clusters=21):
    # Find number of clusters
    n_components = np.arange(1, min(X.shape[0], max_clusters))
    models = [mixture.GaussianMixture(n, covariance_type='full',
                              random_state=0).fit(X)
              for n in n_components]

    if plot:
        plt.plot(n_components, [m.bic(X) for m in models], label='BIC')
        plt.plot(n_components, [m.aic(X) for m in models], label='AIC')
        plt.legend(loc='best')
        plt.xlabel('n_components')
        plt.show()

    # Getting number of clusters using Bayesian Information Control
    index = np.argmin([m.bic(X) for m in models])
    clusters = n_components[index]

    return clusters
//  [python]
def find_number_of_clusters(X, plot=True, max_clusters=21):
    # Find number of clusters
    n_components = np.arange(1, min(X.shape[0], max_clusters))
    models = [GaussianMixture(n, covariance_type='full',
                              random_state=0).fit(X)
              for n in n_components]

    if plot:
        plt.plot(n_components, [m.bic(X) for m in models], label='BIC')
        plt.plot(n_components, [m.aic(X) for m in models], label='AIC')
        plt.legend(loc='best')
        plt.xlabel('n_components')
        plt.show()

    # Getting number of clusters using Bayesian Information Control
    index = np.argmin([m.bic(X) for m in models])
    clusters = n_components[index]

    return clusters
//  [python]
def sklearn_fit_gmm(image, scrib, scribbles_bg_label, scribbles_fg_label, n_components):
    bg = mixture.GMM(n_components=n_components)
    bg.fit(image[scrib == scribbles_bg_label].reshape(-1, 1))

    fg = mixture.GMM(n_components=n_components)
    fg.fit(image[scrib == scribbles_fg_label].reshape(-1, 1))

    return bg, fg
//  [python]
def sklearn_fit_gmm(image, scrib, scribbles_bg_label, scribbles_fg_label, n_components):
    bg = GaussianMixture(n_components=n_components)
    bg.fit(image[scrib == scribbles_bg_label].reshape(-1, 1))

    fg = GaussianMixture(n_components=n_components)
    fg.fit(image[scrib == scribbles_fg_label].reshape(-1, 1))

    return bg, fg
//  [python]
def fitGMM(samples, maxComp=3, covType="full", useBic=True, gmmKwargs=None):
    """
    Fit a Gaussian Mixture Model to the posterior samples to derive an
    approximation of the posterior density.  Fit for the number of components
    by either minimizing the Bayesian Information Criterior (BIC) or via
    cross-validation.

    Parameters
    ----------
    samples : numpy array
        sampler.flatchain MCMC chain array of dimensions (nwalkers x nsteps, ndim)
    maxComp : int (optional)
        Maximum number of mixture model components to fit for.  Defaults to 3.
    covType : str (optional)
        GMM covariance type.  Defaults to "full".  See the documentation here:
        http://scikit-learn.org/stable/modules/generated/sklearn.mixture.GaussianMixture.html
        for more info
    useBic : bool (optional)
        Minimize the BIC to pick the number of GMM components or use 5-fold
        cross validation?  Defaults to True (aka, use the BIC)
    gmmKwargs : dict (optional)
        keyword arguments for sklearn.mixture.GaussianMixture. Defaults to
        None

    Returns
    -------
    GMM : sklearn.mixture.GaussianMixture
        fitted Gaussian mixture model
    """

    if gmmKwargs is None:
        gmmKwargs = dict()

    # Select optimal number of components via minimizing BIC
    if useBic:

        bic = None
        lowestBic = np.inf
        bestGMM = None
        gmm = mixture.GaussianMixture()

        for nComponents in range(1,maxComp+1):
            gmmKwargs["n_components"] = nComponents
            gmmKwargs["covariance_type"] = covType

            gmm.set_params(**gmmKwargs)
            gmm.fit(samples)
            bic = gmm.bic(samples)

            if bic < lowestBic:
                lowestBic = bic
                bestN = nComponents
                bestCovType = covType

        # Refit GMM with the lowest bic
        gmmKwargs["n_components"] = bestN
        gmmKwargs["covariance_type"] = bestCovType
        GMM = mixture.GaussianMixture(**gmmKwargs)
        GMM.fit(samples)

    # Select optimal number of components via 5 fold cross-validation
    else:
        hyperparams = {"n_components" : np.arange(maxComp+1)}
        gmm = GridSearchCV(mixture.GaussianMixture(covariance_type=covType),
                           hyperparams, cv=5)
        gmm.fit(samples)
        GMM = gmm.best_estimator_
        GMM.fit(samples)

    return GMM
//  [python]
def fitGMM(samples, maxComp=3, covType="full", useBic=True, gmmKwargs=None):
    """
    Fit a Gaussian Mixture Model to the posterior samples to derive an
    approximation of the posterior density.  Fit for the number of components
    by either minimizing the Bayesian Information Criterior (BIC) or via
    cross-validation.

    Parameters
    ----------
    samples : numpy array
        sampler.flatchain MCMC chain array of dimensions (nwalkers x nsteps, ndim)
    maxComp : int (optional)
        Maximum number of mixture model components to fit for.  Defaults to 3.
    covType : str (optional)
        GMM covariance type.  Defaults to "full".  See the documentation here:
        http://scikit-learn.org/stable/modules/generated/sklearn.mixture.GaussianMixture.html
        for more info
    useBic : bool (optional)
        Minimize the BIC to pick the number of GMM components or use 5-fold
        cross validation?  Defaults to True (aka, use the BIC)
    gmmKwargs : dict (optional)
        keyword arguments for sklearn.mixture.GaussianMixture. Defaults to
        None

    Returns
    -------
    GMM : sklearn.mixture.GaussianMixture
        fitted Gaussian mixture model
    """

    if gmmKwargs is None:
        gmmKwargs = dict()

    # Select optimal number of components via minimizing BIC
    if useBic:

        bic = None
        lowestBic = np.inf
        bestGMM = None
        gmm = GaussianMixture()

        for nComponents in range(1,maxComp+1):
            gmmKwargs["n_components"] = nComponents
            gmmKwargs["covariance_type"] = covType

            gmm.set_params(**gmmKwargs)
            gmm.fit(samples)
            bic = gmm.bic(samples)

            if bic < lowestBic:
                lowestBic = bic
                bestN = nComponents
                bestCovType = covType

        # Refit GMM with the lowest bic
        gmmKwargs["n_components"] = bestN
        gmmKwargs["covariance_type"] = bestCovType
        GMM = GaussianMixture(**gmmKwargs)
        GMM.fit(samples)

    # Select optimal number of components via 5 fold cross-validation
    else:
        hyperparams = {"n_components" : np.arange(maxComp+1)}
        gmm = GridSearchCV(GaussianMixture(covariance_type=covType),
                           hyperparams, cv=5)
        gmm.fit(samples)
        GMM = gmm.best_estimator_
        GMM.fit(samples)

    return GMM
//  [python]
def plot_eth(X, clusters, t):
    # Load reference image - ETH
    img_path = sys.argv[1] + '/datasets/UCY/students03/reference.png'
    img = plt.imread(img_path)

    # Homography matrix path
    H_path = sys.argv[1] + '/datasets/UCY/students03/H.txt'
    H = np.loadtxt(H_path)
    Hinv = np.linalg.inv(H)

    # Transform points to image space
    cat = np.vstack([X[:, 0], X[:, 1], np.ones_like(X[:, 0])]).T
    tCat = (Hinv @ cat.T).T

    # Get points in image
    x = tCat[:, 1] / tCat[:, 2]
    y = tCat[:, 0] / tCat[:, 2]

    # Fit GMM model to sample
    gmm = mixture.GMM(n_components=clusters).fit(X)
    labels = gmm.predict(X)
    plt.imshow(img)
    plt.scatter(x, y, c=labels,
                s=10, cmap='viridis')
    plt.title('Sample at t = {:.2f}'.format(t))
    plt.savefig(sys.argv[2] + '/clusters_t={:.2f}.png'.format(t))
    plt.close()
//  [python]
def plot_eth(X, clusters, t):
    # Load reference image - ETH
    img_path = sys.argv[1] + '/datasets/UCY/students03/reference.png'
    img = plt.imread(img_path)

    # Homography matrix path
    H_path = sys.argv[1] + '/datasets/UCY/students03/H.txt'
    H = np.loadtxt(H_path)
    Hinv = np.linalg.inv(H)

    # Transform points to image space
    cat = np.vstack([X[:, 0], X[:, 1], np.ones_like(X[:, 0])]).T
    tCat = (Hinv @ cat.T).T

    # Get points in image
    x = tCat[:, 1] / tCat[:, 2]
    y = tCat[:, 0] / tCat[:, 2]

    # Fit GMM model to sample
    gmm = GaussianMixture(n_components=clusters).fit(X)
    labels = gmm.predict(X)
    plt.imshow(img)
    plt.scatter(x, y, c=labels,
                s=10, cmap='viridis')
    plt.title('Sample at t = {:.2f}'.format(t))
    plt.savefig(sys.argv[2] + '/clusters_t={:.2f}.png'.format(t))
    plt.close()
//  [python]
def find_k(data,c_type,max_cluster_num=100):
    k_list = []
    inertia = []
    
    for k in range(1, max_cluster_num + 1):
    
        gmm = mixture.GaussianMixture(k,covariance_type=c_type).fit(data)
        k_list.append(k)
        inertia.append(gmm.bic(data))
    
    idx = np.argmin(inertia)
    final_k = k_list[idx]
    return final_k
//  [python]
def find_k(data,c_type,max_cluster_num=100):
    k_list = []
    inertia = []
    
    for k in range(1, max_cluster_num + 1):
    
        gmm = GaussianMixture(k,covariance_type=c_type).fit(data)
        k_list.append(k)
        inertia.append(gmm.bic(data))
    
    idx = np.argmin(inertia)
    final_k = k_list[idx]
    return final_k
//  [python]
def fit_gmm(
    latent_path,
    save_path=os.getcwd(),
    n_components=10,
):
    # Load latents
    z = np.load(latent_path).squeeze()

    # Fit a GMM model
    gmm = mixture.GaussianMixture(
        n_components=n_components, random_state=0, verbose=2, verbose_interval=1
    )
    gmm.fit(z)

    # Save the sklearn model on disk
    os.makedirs(save_path, exist_ok=True)
    s = dump(gmm, os.path.join(save_path, f"gmm_{n_components}.joblib"))
//  [python]
def fit_gmm(
    latent_path,
    save_path=os.getcwd(),
    n_components=10,
):
    # Load latents
    z = np.load(latent_path).squeeze()

    # Fit a GMM model
    gmm = GaussianMixture(
        n_components=n_components, random_state=0, verbose=2, verbose_interval=1
    )
    gmm.fit(z)

    # Save the sklearn model on disk
    os.makedirs(save_path, exist_ok=True)
    s = dump(gmm, os.path.join(save_path, f"gmm_{n_components}.joblib"))
//  [python]
def gmm(X, k):
    """
    function that calculates a GMM from a dataset
    """

    GMM = s.GaussianMixture(n_components=k)
    params = GMM.fit(X)
    pi = params.weights_
    m = params.means_
    S = params.covariances_
    clss = GMM.predict(X)
    bic = GMM.bic(X)

    return pi, m, S, clss, bic
//  [python]
def gmm(X, k):
    """
    function that calculates a GMM from a dataset
    """

    GMM = sklearn.mixture.GaussianMixture(n_components=k)
    params = GMM.fit(X)
    pi = params.weights_
    m = params.means_
    S = params.covariances_
    clss = GMM.predict(X)
    bic = GMM.bic(X)

    return pi, m, S, clss, bic
//  [python]
def scdv(train_text, embedding, clusters_num=60):
    '''
    input
    ---
    train_text(list): list of train text
    test_text(list): list of test text
    w2v_model(str): path of word2vec model
    embedding_dim(int): dimension of embedding
    fe_save_path(str): path of feature directory
    clusters_num(int): n_components of GMM
    '''
    train_corpus = [text_to_word_sequence(text) for text in train_text]
    model = KeyedVectors.load_word2vec_format(embedding, binary=True)
    word_vectors = model.wv.vectors

    gmm = mixture.GaussianMixture(n_components=clusters_num, covariance_type='tied', max_iter=50)
    gmm.fit(word_vectors)

    tfv = TfidfVectorizer(max_features=None, ngram_range=(1, 1))
    tfv.fit(list(train_text))

    idf_dic = dict(zip(tfv.get_feature_names(), tfv._tfidf.idf_))
    assign_dic = dict(zip(model.wv.index2word, gmm.predict(word_vectors)))
    soft_assign_dic = dict(zip(model.wv.index2word, gmm.predict_proba(word_vectors)))

    word_topic_vecs = {}
    for word in assign_dic:
        word_topic_vecs[word] = np.zeros(model.vector_size * clusters_num, dtype=np.float32)
        for i in range(0, clusters_num):
            try:
                word_topic_vecs[word][i * model.vector_size:(i + 1) * model.vector_size] = model.wv[word] * soft_assign_dic[word][
                    i] * idf_dic[word]
            except:
                continue

    result_col = ["scdv{}".format(i) for i in range(1, model.vector_size*clusters_num+1)]
    scdvs_train = _get_scdv_vector(train_corpus, word_topic_vecs, clusters_num, model.vector_size)
    X = pd.DataFrame(scdvs_train)
    X.columns = result_col

    return X
//  [python]
def scdv(train_text, embedding, clusters_num=60):
    '''
    input
    ---
    train_text(list): list of train text
    test_text(list): list of test text
    w2v_model(str): path of word2vec model
    embedding_dim(int): dimension of embedding
    fe_save_path(str): path of feature directory
    clusters_num(int): n_components of GMM
    '''
    train_corpus = [text_to_word_sequence(text) for text in train_text]
    model = KeyedVectors.load_word2vec_format(embedding, binary=True)
    word_vectors = model.wv.vectors

    gmm = GaussianMixture(n_components=clusters_num, covariance_type='tied', max_iter=50)
    gmm.fit(word_vectors)

    tfv = TfidfVectorizer(max_features=None, ngram_range=(1, 1))
    tfv.fit(list(train_text))

    idf_dic = dict(zip(tfv.get_feature_names(), tfv._tfidf.idf_))
    assign_dic = dict(zip(model.wv.index2word, gmm.predict(word_vectors)))
    soft_assign_dic = dict(zip(model.wv.index2word, gmm.predict_proba(word_vectors)))

    word_topic_vecs = {}
    for word in assign_dic:
        word_topic_vecs[word] = np.zeros(model.vector_size * clusters_num, dtype=np.float32)
        for i in range(0, clusters_num):
            try:
                word_topic_vecs[word][i * model.vector_size:(i + 1) * model.vector_size] = model.wv[word] * soft_assign_dic[word][
                    i] * idf_dic[word]
            except:
                continue

    result_col = ["scdv{}".format(i) for i in range(1, model.vector_size*clusters_num+1)]
    scdvs_train = _get_scdv_vector(train_corpus, word_topic_vecs, clusters_num, model.vector_size)
    X = pd.DataFrame(scdvs_train)
    X.columns = result_col

    return X
//  [python]
def FE_discretize_numeric_variables(train, bin_dict, test='', strategy='kmeans',verbose=0):
    """
    This handy function discretizes numeric variables into binned variables using kmeans algorithm.
    You need to provide the names of the variables and the numbers of bins for each variable in a dictionary.
    It will return the same dataframe with new binned variables that it has created.

    Inputs:
    ----------
    df : pandas dataframe - please ensure it is a dataframe. No arrays please.
    bin_dict: dictionary of names of variables and the bins that you want for each variable.
    strategy: default is 'kmeans': but you can choose: {'gauusian','uniform', 'quantile', 'kmeans'}

    Outputs:
    ----------
    df: pandas dataframe with new variables with names such as:  variable+'_discrete'
    """
    df = copy.deepcopy(train)
    test = copy.deepcopy(test)
    num_cols = len(bin_dict)
    nrows = int((num_cols/2)+0.5)
    #print('nrows',nrows)
    if verbose:
        fig = plt.figure(figsize=(10,3*num_cols))
    for i, (col, binvalue) in enumerate(bin_dict.items()):
        new_col = col+'_discrete'
        if strategy == 'gaussian':
            kbd = mixture.GaussianMixture(n_components=binvalue, random_state=99)
            df[new_col] = kbd.fit_predict(df[[col]]).astype(int)
            if not isinstance(test, str):
                test[new_col] = kbd.predict(test[[col]]).astype(int)
        else:
            kbd = KBinsDiscretizer(n_bins=binvalue, encode='ordinal', strategy=strategy)
            df[new_col] = kbd.fit_transform(df[[col]]).astype(int)
            if not isinstance(test, str):
                test[new_col] = kbd.transform(test[[col]]).astype(int)
        if verbose:
            ax1 = plt.subplot(nrows,2,i+1)
            ax1.scatter(df[col],df[new_col])
            ax1.set_title(new_col)
    if not isinstance(test, str):
        return df, test
    else:
        return df
//  [python]
def FE_discretize_numeric_variables(train, bin_dict, test='', strategy='kmeans',verbose=0):
    """
    This handy function discretizes numeric variables into binned variables using kmeans algorithm.
    You need to provide the names of the variables and the numbers of bins for each variable in a dictionary.
    It will return the same dataframe with new binned variables that it has created.

    Inputs:
    ----------
    df : pandas dataframe - please ensure it is a dataframe. No arrays please.
    bin_dict: dictionary of names of variables and the bins that you want for each variable.
    strategy: default is 'kmeans': but you can choose: {'gauusian','uniform', 'quantile', 'kmeans'}

    Outputs:
    ----------
    df: pandas dataframe with new variables with names such as:  variable+'_discrete'
    """
    df = copy.deepcopy(train)
    test = copy.deepcopy(test)
    num_cols = len(bin_dict)
    nrows = int((num_cols/2)+0.5)
    #print('nrows',nrows)
    if verbose:
        fig = plt.figure(figsize=(10,3*num_cols))
    for i, (col, binvalue) in enumerate(bin_dict.items()):
        new_col = col+'_discrete'
        if strategy == 'gaussian':
            kbd = GaussianMixture(n_components=binvalue, random_state=99)
            df[new_col] = kbd.fit_predict(df[[col]]).astype(int)
            if not isinstance(test, str):
                test[new_col] = kbd.predict(test[[col]]).astype(int)
        else:
            kbd = KBinsDiscretizer(n_bins=binvalue, encode='ordinal', strategy=strategy)
            df[new_col] = kbd.fit_transform(df[[col]]).astype(int)
            if not isinstance(test, str):
                test[new_col] = kbd.transform(test[[col]]).astype(int)
        if verbose:
            ax1 = plt.subplot(nrows,2,i+1)
            ax1.scatter(df[col],df[new_col])
            ax1.set_title(new_col)
    if not isinstance(test, str):
        return df, test
    else:
        return df
//  [python]
record_ngmm = pd.DataFrame(np.zeros([10,3]),columns = ['time','ll','n_component'])
record_dpgmm = pd.DataFrame(np.zeros([10,3]),columns = ['time','ll','n_component'])

record_gmm_fix_k = pd.DataFrame(np.zeros([10,3]),columns = ['time','ll','n_component'])
record_ngmm_fix_k = pd.DataFrame(np.zeros([10,3]),columns = ['time','ll','n_component'])



for t in range(10):
    
    print('CITE-sort.')
    start_time = time.time()
    rgmm = ReSplit(data,merge_cutoff) #sort(data,F_path,N,c_type,weight,rescan_cut,tol,fix_k,max_ndim)
    record_sort.iloc[t,0]  = time.time() - start_time
    # print("--- %s seconds ---" % (t1))
    trav = BTreeTraversal(rgmm)
    record_sort.iloc[t,1]  = trav.ll
    #record_sort.iloc[t,2]  = trav.bic
    record_sort.iloc[t,2] =  trav.n_components
    #print(rgmm_ll)
    
    print('full GMM.')
    start_time = time.time()
    final_k = find_k(data,'full', max_cluster_num)
    gmm = mixture.GaussianMixture(final_k).fit(data)
    record_gmm.iloc[t,0] = time.time() - start_time
    #print("--- %s seconds ---" % gmm_time)
    #record_gmm.iloc[t,2] = gmm.bic(data)
    record_gmm.iloc[t,1] = gmm.score(data)
    record_gmm.iloc[t,2] = final_k #trav.n_components
    
    
    print('full GMM wigh fix k.')
    start_time = time.time()
    gmm = mixture.GaussianMixture(trav.n_components).fit(data)
    record_gmm_fix_k.iloc[t,0] = time.time() - start_time
    #print("--- %s seconds ---" % gmm_time)
    #record_gmm.iloc[t,2] = gmm.bic(data)
    record_gmm_fix_k.iloc[t,1] = gmm.score(data)
    record_gmm_fix_k.iloc[t,2] = trav.n_components
    

    print('naive GMM.')
    start_time = time.time()
    final_k = find_k(data,'diag',max_cluster_num)
    ngmm = mixture.GaussianMixture(final_k,covariance_type='diag').fit(data)
    record_ngmm.iloc[t,0] = time.time() - start_time
    #print("--- %s seconds ---" % (t))
    #record_ngmm.iloc[t,2] = ngmm.bic(data)
    record_ngmm.iloc[t,1] = ngmm.score(data)
    record_ngmm.iloc[t,2] = final_k#trav.n_components
    #print(ngmm_ll)
    
    
    print('naive GMM with fix k.')
    start_time = time.time()
    ngmm = mixture.GaussianMixture(trav.n_components,covariance_type='diag').fit(data)
    record_ngmm_fix_k.iloc[t,0] = time.time() - start_time
    #print("--- %s seconds ---" % (t))
    #record_ngmm.iloc[t,2] = ngmm.bic(data)
    record_ngmm_fix_k.iloc[t,1] = ngmm.score(data)
    record_ngmm_fix_k.iloc[t,2] = trav.n_components
    #print(ngmm_ll)
    
    
    print('dpgmm.')
    start_time = time.time()
    dpgmm = mixture.BayesianGaussianMixture(n_components=max_cluster_num,max_iter=500).fit(data)
    record_dpgmm.iloc[t,0] = time.time() - start_time
    record_dpgmm.iloc[t,1] = dpgmm.score(data)
    record_dpgmm.iloc[t,2] = len(dpgmm.weights_)
//  [python]
record_ngmm = pd.DataFrame(np.zeros([10,3]),columns = ['time','ll','n_component'])
record_dpgmm = pd.DataFrame(np.zeros([10,3]),columns = ['time','ll','n_component'])

record_gmm_fix_k = pd.DataFrame(np.zeros([10,3]),columns = ['time','ll','n_component'])
record_ngmm_fix_k = pd.DataFrame(np.zeros([10,3]),columns = ['time','ll','n_component'])



for t in range(10):
    
    print('CITE-sort.')
    start_time = time.time()
    rgmm = ReSplit(data,merge_cutoff) #sort(data,F_path,N,c_type,weight,rescan_cut,tol,fix_k,max_ndim)
    record_sort.iloc[t,0]  = time.time() - start_time
    # print("--- %s seconds ---" % (t1))
    trav = BTreeTraversal(rgmm)
    record_sort.iloc[t,1]  = trav.ll
    #record_sort.iloc[t,2]  = trav.bic
    record_sort.iloc[t,2] =  trav.n_components
    #print(rgmm_ll)
    
    print('full GMM.')
    start_time = time.time()
    final_k = find_k(data,'full', max_cluster_num)
    gmm = GaussianMixture(final_k).fit(data)
    record_gmm.iloc[t,0] = time.time() - start_time
    #print("--- %s seconds ---" % gmm_time)
    #record_gmm.iloc[t,2] = gmm.bic(data)
    record_gmm.iloc[t,1] = gmm.score(data)
    record_gmm.iloc[t,2] = final_k #trav.n_components
    
    
    print('full GMM wigh fix k.')
    start_time = time.time()
    gmm = GaussianMixture(trav.n_components).fit(data)
    record_gmm_fix_k.iloc[t,0] = time.time() - start_time
    #print("--- %s seconds ---" % gmm_time)
    #record_gmm.iloc[t,2] = gmm.bic(data)
    record_gmm_fix_k.iloc[t,1] = gmm.score(data)
    record_gmm_fix_k.iloc[t,2] = trav.n_components
    

    print('naive GMM.')
    start_time = time.time()
    final_k = find_k(data,'diag',max_cluster_num)
    ngmm = GaussianMixture(final_k,covariance_type='diag').fit(data)
    record_ngmm.iloc[t,0] = time.time() - start_time
    #print("--- %s seconds ---" % (t))
    #record_ngmm.iloc[t,2] = ngmm.bic(data)
    record_ngmm.iloc[t,1] = ngmm.score(data)
    record_ngmm.iloc[t,2] = final_k#trav.n_components
    #print(ngmm_ll)
    
    
    print('naive GMM with fix k.')
    start_time = time.time()
    ngmm = GaussianMixture(trav.n_components,covariance_type='diag').fit(data)
    record_ngmm_fix_k.iloc[t,0] = time.time() - start_time
    #print("--- %s seconds ---" % (t))
    #record_ngmm.iloc[t,2] = ngmm.bic(data)
    record_ngmm_fix_k.iloc[t,1] = ngmm.score(data)
    record_ngmm_fix_k.iloc[t,2] = trav.n_components
    #print(ngmm_ll)
    
    
    print('dpgmm.')
    start_time = time.time()
    dpgmm = BayesianGaussianMixture(n_components=max_cluster_num,max_iter=500).fit(data)
    record_dpgmm.iloc[t,0] = time.time() - start_time
    record_dpgmm.iloc[t,1] = dpgmm.score(data)
    record_dpgmm.iloc[t,2] = len(dpgmm.weights_)
//  [python]
def sklearn_GaussianMixture(*args, **kwargs):
    return mixture.GaussianMixture(*args, **kwargs)
//  [python]
def create_gmm(site: SiteStr, n_components: int,
               date_range: tuple[datetime, datetime]) -> None:
    """Creates a custom GMM and saves in the ``gmms`` folder.

    Args:
        site: either 'caltech' or 'jpl'
        n_components: number of components of Gaussian mixture model
        date_range: a range of dates that falls inside 2018-11-01 and 2021-08-31.
    """
    # Get stations
    cn = site_str_to_site(site)
    n2i = {station_id: i for i, station_id in enumerate(cn.station_ids)}
    # Retrieve events and filter only claimed sessions
    df = get_real_events(date_range[0], date_range[1], site=site)
    df = df[df['claimed']]
    if len(df) == 0:
        print('Empty dataframe, abort GMM training. ')
        return
    # Get counts and station ids data
    num_days_total = (date_range[1] - date_range[0]).days + 1
    cnt = df.arrival.dt.date.value_counts().to_numpy()
    num_unseen_days = num_days_total - len(cnt)  # account for days when there are no EVs
    cnt = np.concatenate((cnt, np.zeros(num_unseen_days)))
    sid = station_id_cnts(df, n2i)
    # Preprocess DataFrame for GMM training
    df = preprocess(df)
    # Train and save
    print(f'Fitting GMM ({n_components} components, {len(df.columns)} '
          f'dimensions) on data from {site} site from '
          f'{date_range[0].strftime(DATE_FORMAT)} to '
          f'{date_range[1].strftime(DATE_FORMAT)}... ')
    gmm = mixture.GaussianMixture(n_components=n_components)
    gmm.fit(df)
    save_gmm_model(site, gmm, cnt, sid, *date_range, n_components)
//  [python]
def create_gmm(site: SiteStr, n_components: int,
               date_range: tuple[datetime, datetime]) -> None:
    """Creates a custom GMM and saves in the ``gmms`` folder.

    Args:
        site: either 'caltech' or 'jpl'
        n_components: number of components of Gaussian mixture model
        date_range: a range of dates that falls inside 2018-11-01 and 2021-08-31.
    """
    # Get stations
    cn = site_str_to_site(site)
    n2i = {station_id: i for i, station_id in enumerate(cn.station_ids)}
    # Retrieve events and filter only claimed sessions
    df = get_real_events(date_range[0], date_range[1], site=site)
    df = df[df['claimed']]
    if len(df) == 0:
        print('Empty dataframe, abort GMM training. ')
        return
    # Get counts and station ids data
    num_days_total = (date_range[1] - date_range[0]).days + 1
    cnt = df.arrival.dt.date.value_counts().to_numpy()
    num_unseen_days = num_days_total - len(cnt)  # account for days when there are no EVs
    cnt = np.concatenate((cnt, np.zeros(num_unseen_days)))
    sid = station_id_cnts(df, n2i)
    # Preprocess DataFrame for GMM training
    df = preprocess(df)
    # Train and save
    print(f'Fitting GMM ({n_components} components, {len(df.columns)} '
          f'dimensions) on data from {site} site from '
          f'{date_range[0].strftime(DATE_FORMAT)} to '
          f'{date_range[1].strftime(DATE_FORMAT)}... ')
    gmm = GaussianMixture(n_components=n_components)
    gmm.fit(df)
    save_gmm_model(site, gmm, cnt, sid, *date_range, n_components)
//  [python]
def get_states(self, maxdist=15):
        """
        Chromosome level estimation.
        """
        training_seqs = []
        whole_genome = np.r_[[]]
        for c in self.bin_cnv:
            arr = self.bin_cnv[c]
            nonzero = arr[arr > 0.1]
            idx = self.hampel_filter(np.log2(nonzero))
            filtered = nonzero[idx]
            whole_genome = np.r_[whole_genome, np.log2(filtered)]

            tmp = np.zeros(nonzero.size)
            tmp[idx] = filtered
            newarr = np.zeros(arr.size)
            newarr[arr > 0.1] = tmp

            self.bin_cnv[c] = newarr
            training_seqs.extend(self.pieces(newarr, scale='log'))
        
        X = whole_genome[:, np.newaxis]
        n_components_range = range(1, maxdist+1)
        _trace_gmm = {}
        aic_arr = []
        for n_components in n_components_range:
            gmm = mixture.GaussianMixture(n_components=n_components,
                                  covariance_type='diag',
                                  init_params='k-means++',
                                  n_init=10)
            gmm.fit(X)
            aic = gmm.aic(X)
            aic_arr.append(aic)
            _trace_gmm[n_components] = gmm
        
        aic_arr = np.r_[aic_arr]
        best_idx = aic_arr.argmin()
        gmm = _trace_gmm[best_idx+1]

        return training_seqs, gmm
//  [python]
def get_states(self, maxdist=15):
        """
        Chromosome level estimation.
        """
        training_seqs = []
        whole_genome = np.r_[[]]
        for c in self.bin_cnv:
            arr = self.bin_cnv[c]
            nonzero = arr[arr > 0.1]
            idx = self.hampel_filter(np.log2(nonzero))
            filtered = nonzero[idx]
            whole_genome = np.r_[whole_genome, np.log2(filtered)]

            tmp = np.zeros(nonzero.size)
            tmp[idx] = filtered
            newarr = np.zeros(arr.size)
            newarr[arr > 0.1] = tmp

            self.bin_cnv[c] = newarr
            training_seqs.extend(self.pieces(newarr, scale='log'))
        
        X = whole_genome[:, np.newaxis]
        n_components_range = range(1, maxdist+1)
        _trace_gmm = {}
        aic_arr = []
        for n_components in n_components_range:
            gmm = GaussianMixture(n_components=n_components,
                                  covariance_type='diag',
                                  init_params='k-means++',
                                  n_init=10)
            gmm.fit(X)
            aic = gmm.aic(X)
            aic_arr.append(aic)
            _trace_gmm[n_components] = gmm
        
        aic_arr = np.r_[aic_arr]
        best_idx = aic_arr.argmin()
        gmm = _trace_gmm[best_idx+1]

        return training_seqs, gmm
//  [python]
gm = mixture.GaussianMixture(n_components=4, random_state=0, init_params='kmeans').fit(latents) #plot a n_components v.s. Average negative log likelihood
print('Average negative log likelihood:', -1*gm.score(latents))
plot_gmm(gm, latents)

scores=[] #using elbow method to find out the best # of components
for i in range(1,8):
  gm = mixture.GaussianMixture(n_components=i, random_state=0, init_params='kmeans').fit(latents)
  print('Average negative log likelihood:', -1*gm.score(latents))
  scores.append(-1*gm.score(latents))
//  [python]
gm = GaussianMixture(n_components=4, random_state=0, init_params='kmeans').fit(latents) #plot a n_components v.s. Average negative log likelihood
print('Average negative log likelihood:', -1*gm.score(latents))
plot_gmm(gm, latents)

# Using elbow method to find out the best # of components, the lower the negative log likehood the better the model is, but too many cluster is trivial. just imagine you fit each individual data points with a Gaussian, in this case, you would have a very good model. but the such fitting is not very useful.

# In this case, the best number of cluster is either 4 or 5.

scores=[] #using elbow method to find out the best # of components
for i in range(1,8):
  gm = GaussianMixture(n_components=i, random_state=0, init_params='kmeans').fit(latents)
  print('Average negative log likelihood:', -1*gm.score(latents))
  scores.append(-1*gm.score(latents))
//  [python]
def fast_gmm(y_true, n_classes, eigenvectors):
    gmm_clf = mixture.GaussianMixture(n_components=n_classes, covariance_type='full', max_iter=500, random_state=42)
    gmm_clf.fit(eigenvectors)

    y_pred_brute = gmm_clf.predict(eigenvectors)
    y_pred_proba_brute = gmm_clf.predict_proba(eigenvectors)

    best_perm = find_best_perm(y_true, y_pred_brute)

    y_pred = np.vectorize(lambda x: best_perm[x])(y_pred_brute)
    d = {best_perm[i]:i for i in range(len(best_perm))}
    perm = np.array([ d[i] for i in range(len(d))])
    y_pred_proba = y_pred_proba_brute[:, perm]
    return y_pred, y_pred_proba
//  [python]
def fast_gmm(y_true, n_classes, eigenvectors):
    gmm_clf = GaussianMixture(n_components=n_classes, covariance_type='full', max_iter=500, random_state=42)
    gmm_clf.fit(eigenvectors)

    y_pred_brute = gmm_clf.predict(eigenvectors)
    y_pred_proba_brute = gmm_clf.predict_proba(eigenvectors)

    best_perm = find_best_perm(y_true, y_pred_brute)

    y_pred = np.vectorize(lambda x: best_perm[x])(y_pred_brute)
    d = {best_perm[i]:i for i in range(len(best_perm))}
    perm = np.array([ d[i] for i in range(len(d))])
    y_pred_proba = y_pred_proba_brute[:, perm]
    return y_pred, y_pred_proba
//  [python]
def gmm_clustering(data, n_gaussians):
    """Produces a GMM model with n_gaussians to cluster provided data."""
    gmm_ = mixture.GaussianMixture(n_components=n_gaussians).fit(data)
    return gmm_.predict(data), gmm_.aic(data)
//  [python]
def gmm_clustering(data, n_gaussians):
    """Produces a GMM model with n_gaussians to cluster provided data."""
    gmm_ = GaussianMixture(n_components=n_gaussians).fit(data)
    return gmm_.predict(data), gmm_.aic(data)
//  [python]
def get_continuous_imputers(df, features):
    cont_imputer = preprocessing.Imputer()
    cont_imputer.fit(df[features])
    print(cont_imputer.statistics_)
    return cont_imputer
//  [python]
def get_continuous_imputers(df, features):
    cont_imputer = impute.SimpleImputer()
    cont_imputer.fit(df[features])
    print(cont_imputer.statistics_)
    return cont_imputer
//  [python]
print("Score without the samples containing missing values = %.2f" % score)

# Estimate the score after imputation of the missing values
X_missing = X_full.copy()
X_missing[np.where(missing_samples)[0], missing_features] = 0
y_missing = y_full.copy()
estimator = Pipeline([("imputer", Imputer(missing_values=0,
                                          strategy="mean",
                                          axis=0)),
                      ("forest", RandomForestRegressor(random_state=0,
                                                       n_estimators=100))])
//  [python]
print("Score without the samples containing missing values = %.2f" % score)

# Estimate the score after imputation of the missing values
X_missing = X_full.copy()
X_missing[np.where(missing_samples)[0], missing_features] = 0
y_missing = y_full.copy()
estimator = Pipeline([("imputer", SimpleImputer(missing_values=0,
                                          strategy="mean")),
                      ("forest", RandomForestRegressor(random_state=0,
                                                       n_estimators=100))])
//  [python]
def test_cross_val_score_allow_nans():
    # Check that cross_val_score allows input data with NaNs
    X = np.arange(200, dtype=np.float64).reshape(10, -1)
    X[2, :] = np.nan
    y = np.repeat([0, 1], X.shape[0] / 2)
    p = Pipeline([
        ('imputer', Imputer(strategy='mean', missing_values='NaN')),
        ('classifier', MockClassifier()),
    ])
    cval.cross_val_score(p, X, y, cv=5)