app.py

import streamlit as st
import pandas as pd
import bertopic
import plotly.express as px
import matplotlib as mp

st.set_page_config(page_title="Topic Modeling with Bertopic")

from datasets import load_dataset

st.markdown("""
https://github.com/pinecone-io/examples/tree/master/learn/algos-and-libraries/bertopic
""")

# data = load_dataset('jamescalam/python-reddit')
data = load_dataset("awacke1/LOINC-Panels-and-Forms")





from datasets import load_dataset

geo = load_dataset('jamescalam/world-cities-geo', split='train')
st.write(geo)






import plotly.express as px

palette = ['#1c17ff', '#faff00', '#8cf1ff', '#000000', '#030080', '#738fab']

fig = px.scatter_3d(
    x=geo['x'], y=geo['y'], z=geo['z'],
    color=geo['continent'],
    custom_data=[geo['country'], geo['city']],
    color_discrete_sequence=palette
)
fig.update_traces(
    hovertemplate="\n".join([
        "city: %{customdata[1]}",
        "country: %{customdata[0]}"
    ])
)


fig.write_html("umap-earth-3d.html", include_plotlyjs="cdn", full_html=False)


import numpy as np

geo_arr = np.asarray([geo['x'], geo['y'], geo['z']]).T
geo_arr = geo_arr / geo_arr.max()

st.markdown(geo_arr[:5])


import umap


colors = geo['continent']
c_map = {
    'Africa': '#8cf1ff',
    'Asia': '#1c17ff',
    'Europe': '#faff00',
    'North America': '#738fab',
    'Oceania': '#030080',
    'South America': '#000000'
}
for i in range(len(colors)):
    colors[i] = c_map[colors[i]]
colors[:5]



import matplotlib.pyplot as plt
import seaborn as sns
from tqdm.auto import tqdm

fig, ax = plt.subplots(3, 3, figsize=(14, 14))
nns = [2, 3, 4, 5, 10, 15, 30, 50, 100]
i, j = 0, 0
for n_neighbors in tqdm(nns):
    fit = umap.UMAP(n_neighbors=n_neighbors)
    u = fit.fit_transform(geo_arr)
    sns.scatterplot(x=u[:,0], y=u[:,1], c=colors, ax=ax[j, i])
    ax[j, i].set_title(f'n={n_neighbors}')
    if i < 2: i += 1
    else: i = 0; j += 1


target = geo['continent']
t_map = {
    'Africa': 0,
    'Asia': 1,
    'Europe': 2,
    'North America': 3,
    'Oceania': 4,
    'South America': 5
}
for i in range(len(target)):
    target[i] = t_map[target[i]]

fig, ax = plt.subplots(3, 3, figsize=(14, 14))
nns = [2, 3, 4, 5, 10, 15, 30, 50, 100]
i, j = 0, 0
for n_neighbors in tqdm(nns):
    fit = umap.UMAP(n_neighbors=n_neighbors)
    u = fit.fit_transform(geo_arr, y=target)
    sns.scatterplot(x=u[:,0], y=u[:,1], c=colors, ax=ax[j, i])
    ax[j, i].set_title(f'n={n_neighbors}')
    if i < 2: i += 1
    else: i = 0; j += 1

import umap

fit = umap.UMAP(n_neighbors=50, min_dist=0.5)
u = fit.fit_transform(geo_arr)

fig = px.scatter(
    x=u[:,0], y=u[:,1],
    color=geo['continent'],
    custom_data=[geo['country'], geo['city']],
    color_discrete_sequence=palette
)
fig.update_traces(
    hovertemplate="\n".join([
        "city: %{customdata[1]}",
        "country: %{customdata[0]}"
    ])
)

fig.write_html("umap-earth-2d.html", include_plotlyjs="cdn", full_html=False)

import pandas as pd

umapped = pd.DataFrame({
    'x': u[:,0],
    'y': u[:,1],
    'continent': geo['continent'],
    'country': geo['country'],
    'city': geo['city']
})

umapped.to_csv('umapped.csv', sep='|', index=False)

from sklearn.decomposition import PCA

pca = PCA(n_components=2)  # this means we will create 2-d space
p = pca.fit_transform(geo_arr)
fig = px.scatter(
    x=p[:,0], y=p[:,1],
    color=geo['continent'],
    custom_data=[geo['country'], geo['city']],
    color_discrete_sequence=palette
)
fig.update_traces(
    hovertemplate="\n".join([
        "city: %{customdata[1]}",
        "country: %{customdata[0]}"
    ])
)

fig.write_html("pca-earth-2d.html", include_plotlyjs="cdn", full_html=False)