turn the cluster into a cateogry, the r eqivalent of factor

app.py

@@ -40,6 +40,9 @@ columns = ['title', 'href', 'body']
 labels = cluster.KMeans(n_clusters=5).fit_predict(dimr[['umap1', 'umap2']])
 dimr['cluster'] = labels
 
+# Make the coloring easier on the eyes
+dimr['cluster'] = dimr['cluster'].astype('category')
+
 # Merge the data together
 dat = pd.concat([md.reset_index(), dimr.reset_index()], axis = 1)
 

flycheck_app.py

@@ -0,0 +1,84 @@
+import streamlit as st
+from duckduckgo_search import ddg
+import pandas as pd
+from sentence_transformers import SentenceTransformer
+import umap.umap_ as umap
+import numpy as np
+import sys
+import plotly.express as px
+import re
+import sklearn.cluster as cluster
+
+# The search bar
+keywords = st.text_input('Enter your search', 'How to use ChatGPT')
+
+# Set keywords as command line argument
+# print("searching for: " + ' '.join(sys.argv[1:]) + "...")
+# keywords = ' '.join(sys.argv[1:])
+
+to_display = 'body' # Sometimes this is title
+md = ddg(keywords, region='wt-wt', safesearch='Moderate', time='y', max_results=500)
+md = pd.DataFrame(md)
+
+# Load the model
+print("running sentence embeddings...")
+# model_name = 'all-mpnet-base-v2'
+model_name = 'all-MiniLM-L6-v2'
+model = SentenceTransformer(model_name)
+sentence_embeddings = model.encode(md['body'].tolist(), show_progress_bar = True)
+sentence_embeddings = pd.DataFrame(sentence_embeddings)
+
+# Reduce dimensionality
+print("reducing dimensionality...")
+reducer = umap.UMAP(metric = 'cosine')
+dimr = reducer.fit_transform(sentence_embeddings)
+dimr = pd.DataFrame(dimr, columns = ['umap1', 'umap2'])
+
+columns = ['title', 'href', 'body']
+
+# Clustering
+labels = cluster.KMeans(n_clusters=5).fit_predict(dimr[['umap1', 'umap2']])
+dimr['cluster'] = labels
+
+# Make the coloring easier on the eyes
+dimr['cluster'] = dimr['cluster'].astype('category')
+
+# Merge the data together
+dat = pd.concat([md.reset_index(), dimr.reset_index()], axis = 1)
+
+# handle duplicate index columns
+dat = dat.loc[:,~dat.columns.duplicated()]
+
+# Get it ready for plotting
+dat['title'] = dat.title.str.wrap(30).apply(lambda x: x.replace('\n', '<br>'))
+dat['body'] = dat.body.str.wrap(30).apply(lambda x: x.replace('\n', '<br>'))
+
+# Visualize
+fig = px.scatter(dat, x = 'umap1', y = 'umap2', hover_data = ['title', 'body'], color = 'cluster', title = 'Context similarity map of results')
+
+# Make the font a little bigger
+fig.update_layout(
+    hoverlabel=dict(
+        bgcolor="white",
+        font_size=16
+    )
+)
+
+# Show the figure
+st.plotly_chart(fig, use_container_width=True)
+
+# Remove <br> in the text for the table
+dat['title'] = [re.sub('<br>', ' ', i) for i in dat['title']]
+dat['body'] = [re.sub('<br>', ' ', i) for i in dat['body']]
+
+# Instructions
+st.caption('Click on the table and press ctrl+f (or command+f for mac) to search it')
+
+# Make the link clickable
+def make_clickable(val):
+    return f'<a target="_blank" href="{val}">{val}</a>'
+
+dat.style.format({'href': make_clickable})
+
+# Place a table under the plot
+st.dataframe(dat)