Update app.py

app.py

@@ -6,11 +6,10 @@ from datasets import load_dataset
 from transformers import AutoTokenizer, TFAutoModel
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.cluster import KMeans
-import matplotlib.pyplot as plt
 from sklearn.decomposition import PCA
 
 # Load the dataset
-dataset = load_dataset("sberhe/2023-3-software-release-notes")
+dataset = load_dataset("sberhe/2023-1000-software-release-notes")
 
 # Load a pre-trained model and tokenizer (TensorFlow version)
 model_name = "bert-base-uncased"
@@ -21,67 +20,56 @@ model = TFAutoModel.from_pretrained(model_name)
 def tokenize_function(examples):
     return tokenizer(examples["text"], padding="max_length", truncation=True, max_length=512)
 
-# Tokenize and preprocess in batches
-batch_size = 8  # You can adjust this based on your available memory
-tokenized_datasets = dataset.map(tokenize_function, batched=True, batch_size=batch_size)
+tokenized_datasets = dataset.map(tokenize_function, batched=True)
 
 # Function to extract embeddings
 def extract_embeddings(batch):
     inputs = {k: tf.convert_to_tensor(v) for k, v in batch.items() if k in tokenizer.model_input_names}
-    outputs = model(**inputs, output_hidden_states=True, return_dict=True)
-    embeddings = outputs.last_hidden_state
-    return {"embeddings": embeddings.numpy()} if "embeddings" in outputs else {}
+    outputs = model(**inputs)
+    # Use the embeddings of the [CLS] token ([0])
+    return {"embeddings": outputs.last_hidden_state[:, 0].numpy()}
 
 # Apply the function to extract embeddings in batches
-embeddings_dataset = tokenized_datasets.map(extract_embeddings, batched=True, batch_size=batch_size)
+embeddings_dataset = tokenized_datasets.map(extract_embeddings, batched=True)
 
-# Tokenize the text and convert to numerical values
-inputs = tokenizer(tokenized_datasets["train"]["text"], return_tensors="tf", padding=True, truncation=True, max_length=512)
-outputs = model(**inputs)
-
-# Access the embeddings
-#embeddings = np.vstack(embeddings_dataset["train"]["text"])
-embeddings = outputs.last_hidden_state
-
-# Access the embeddings
-# Debugging code to print dataset keys
-st.write("Dataset Keys:", embeddings_dataset.column_names)
-embeddings_numpy = embeddings.numpy()
-embeddings_reshaped = embeddings_numpy.reshape(-1, 1)  # Adjust the shape as needed
-
-# Reduce dimensionality using PCA
-pca = PCA(n_components=1)  # You can adjust the number of components as needed
-embeddings_2d_reduced = pca.fit_transform(embeddings_reshaped)
+# Flatten the embeddings and reduce dimensionality using PCA
+embeddings = np.vstack(embeddings_dataset['train']['embeddings'])
+pca = PCA(n_components=2)  # Using 2 components for better visualization
+embeddings_2d = pca.fit_transform(embeddings)
 
 # Perform unsupervised clustering (K-Means)
-num_clusters = 5  # You can adjust this based on your data
+num_clusters = 50
 kmeans = KMeans(n_clusters=num_clusters)
-cluster_labels = kmeans.fit_predict(embeddings_2d_reduced)
-
-# Create a DataFrame with cluster labels and PCA results
-# Adjust the number of components based on data shape
-num_components = min(embeddings_2d_reduced.shape[0], embeddings_2d_reduced.shape[1])
-pca = PCA(n_components=num_components)
-pca_result = pca.fit_transform(embeddings_2d_reduced)
-df = pd.DataFrame({'Cluster': cluster_labels, 'PCA1': pca_result[:, 0]})
-
-# Streamlit app
-st.title("Software Release Clustering")
-
-# Display the clusters on a scatter plot
-st.write("Cluster Visualization:")
-fig, ax = plt.subplots()
-scatter = ax.scatter(df['PCA1'], df['PCA1'], c=df['Cluster'], cmap='viridis')
-
-ax.set_xlabel('PCA1')
-ax.set_ylabel('PCA2')
-st.pyplot(fig)
-
-# Display the number of releases in each cluster
-st.write("Cluster Counts:")
-st.write(df['Cluster'].value_counts())
-
-# Display the details of a selected cluster
-selected_cluster = st.selectbox("Select a Cluster to Explore", sorted(df['Cluster'].unique()))
-st.write("Releases in Cluster", selected_cluster)
-st.write(dataset['train'][df['Cluster'] == selected_cluster])
+cluster_labels = kmeans.fit_predict(embeddings_2d)
+
+# Create a DataFrame with cluster labels and original texts
+original_texts = [example['text'] for example in dataset['train']]
+df = pd.DataFrame({'text': original_texts, 'Cluster': cluster_labels})
+
+# ...
+
+# TF-IDF calculation and finding representative terms for each cluster
+vectorizer = TfidfVectorizer(stop_words='english')
+X_tfidf = vectorizer.fit_transform(df['text'])
+feature_names = vectorizer.get_feature_names_out()
+
+cluster_names = []
+for i in range(num_clusters):
+    indices = df[df['Cluster'] == i].index
+    # Aggregate the TF-IDF scores for each feature in cluster i
+    aggregated_tfidf = np.mean(X_tfidf[indices], axis=0)
+    # Convert to array (if it's not already an array) and get the index of the max tf-idf score
+    aggregated_tfidf_array = np.array(aggregated_tfidf).flatten()
+    max_tfidf_index = aggregated_tfidf_array.argmax()
+    cluster_names.append(feature_names[max_tfidf_index])
+
+# Count the size of each cluster
+cluster_sizes = df['Cluster'].value_counts().sort_index()
+
+# Output cluster names and sizes using Streamlit
+for i in range(num_clusters):
+    cluster_name = cluster_names[i]
+    cluster_size = cluster_sizes.get(i, 0)  # Get size with a default of 0 if cluster is empty
+    print(f"Cluster {i+1} (Name: {cluster_name}, Size: {cluster_size})")
+
+# ...