cluster_inference.py

"""
we want to be able to assign a small user text entry to one of our clusters.
"""

import joblib
import pickle
from transformers import GPT2Tokenizer, GPT2Model
import torch

### inference demo

# load the GPT model
GPT_tokenizer = GPT2Tokenizer.from_pretrained('gpt2', padding=True)
GPT_tokenizer.pad_token = '[PAD]'
GPT_model = GPT2Model.from_pretrained('gpt2')

# set some user example
user_example = "we are looking to make some music! please point us to a lovely cluster where we can hear lovely sounds. I like the cranberries."

# tokenize the input
encoded_input = GPT_tokenizer(user_example, return_tensors="pt", padding=True, truncation=True)

# generate the embeddings
with torch.no_grad():

    # get outputs from GPT model
    outputs = GPT_model(**encoded_input)

    # get the [CLS] (classification) token for sequence representation
    cls_embedding = outputs.last_hidden_state[:, 0, :].numpy()

# load the kmeans model
kmeans_model = joblib.load('GPT_128k_means_model.joblib')

# do inference
example_cluster = kmeans_model.predict(cls_embedding)
print(example_cluster)



from collections import Counter
with open('mbid_GPT_128_clusters.pickle', 'rb') as f:
    mbid_clusters = pickle.load(f)


print(type(mbid_clusters))
# print(mbid_clusters)
sample_mbid = 'bd57a71ece2912664f5e267166a2a1fb'
cluster_assignment = mbid_clusters.get(sample_mbid)
print(cluster_assignment)

# cluster_distribution = Counter(mbid_clusters.values())
# print(cluster_distribution)













# # Load the KMeans model
# kmeans_model = joblib.load('GPT_512k_means_model.joblib')

# # Load the cluster assignments from the pickle file
# with open('mbid_GPT_512_clusters.pickle', 'rb') as f:
#     mbid_clusters = pickle.load(f)

# # Now you can access the KMeans model and cluster assignments
# # For example, to get the cluster assignments for a specific mbid:
# sample_mbid = '2a0a712b4b00f3df2d4fa50fe21f43cb'
# cluster_assignment = mbid_clusters.get(sample_mbid)

# # To get the distribution of clusters
# from collections import Counter
# cluster_distribution = Counter(mbid_clusters.values())
# # print(cluster_distribution)

# # To check if each article is assigned a cluster
# total_articles = len(mbid_clusters)
# articles_with_cluster = sum(1 for cluster in mbid_clusters.values() if cluster is not None)

# print(f"Total articles: {total_articles}")
# print(f"Articles with assigned clusters: {articles_with_cluster}")

# # To check different clusters
# # Replace 'cluster_number' with the cluster number you want to inspect
# cluster_number = 0
# articles_in_cluster = [mbid for mbid, cluster in mbid_clusters.items() if cluster == cluster_number]
# #print(f"Articles in cluster {cluster_number}: {articles_in_cluster}")
# # for cluster in mbid_clusters:
    





























# import joblib
# import numpy as np

# # vectorizer
# from sklearn.feature_extraction.text import HashingVectorizer

# # load cluster data pickle file, kmeans model, and vectorizer model
# clusters = joblib.load("clusters_data.pickle")
# vectorizer = joblib.load("vectorizer.joblib")
# kmeans = joblib.load("best_kmeans_model.joblib")

# # an example to try
# user_example = ["make me and my friends a cool song!"] 

# # vectorize user example 
# vectorized_example = vectorizer.transform(user_example)
# print(vectorized_example)

# # assign a cluster: result is cluster 497
# example_cluster = kmeans.predict(vectorized_example)
# print(example_cluster)

# # print(type(clusters[497]))
# # print(len(clusters[497]))
# # print(clusters[497][1])


# # Get the number of data points assigned to each cluster
# num_assigned = [len(cluster_data) for cluster_data in clusters.values()]

# # Compute mean and standard deviation of the number of data points per cluster
# mean_assigned = np.mean(num_assigned)
# std_assigned = np.std(num_assigned)

# print(f"Mean number of data points per cluster: {mean_assigned}")
# print(f"Standard deviation of number of data points per cluster: {std_assigned}")

# # Mean number of data points per cluster: 9.694656488549619
# # Standard deviation of number of data points per cluster: 21.820754225240147











# # get a view of some of the clusters
# num_samples = 3

# # # Print a short version of some clusters
# # for cluster_label, cluster_data in clusters.items():
# #     print(f"Cluster {cluster_label}:")
# #     for i, (mbid, text) in enumerate(cluster_data[:num_samples], 1):
# #         print(f"Sample {i}: {text[:100]}...")  # Print only the first 100 characters of each text
# #     print()  # Add a blank line between clusters