{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "844c876f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from bertopic import BERTopic\n",
    "import pandas as pd \n",
    "import openai\n",
    "from bertopic.representation import OpenAI\n",
    "from umap import UMAP\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "560143ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df = pd.read_csv('mtu_merged_v3.csv')\n",
    "df = pd.read_csv('NikeTwitter.csv')\n",
    "\n",
    "docs = df['translated_text'].tolist()\n",
    "\n",
    "docs = [str(x).replace(\"#\",\" \") for x in docs]\n",
    "docs = [x for x in docs if len(x) > 30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a579594d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f40e914627074089906a1d35d06f55b4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/30 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-17 08:49:56,734 - BERTopic - Transformed documents to Embeddings\n",
      "2023-05-17 08:50:01,377 - BERTopic - Reduced dimensionality\n",
      "2023-05-17 08:50:01,415 - BERTopic - Clustered reduced embeddings\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "# we add this to remove stopwords\n",
    "vectorizer_model = CountVectorizer(ngram_range=(1, 2), stop_words=\"english\")\n",
    "\n",
    "model = BERTopic(\n",
    "    vectorizer_model=vectorizer_model,\n",
    "    language='english', calculate_probabilities=True,\n",
    "    verbose=True\n",
    ")\n",
    "topics, probs = model.fit_transform(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b4803af3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# freq = model.get_document_info(docs)\n",
    "# freq.head(10)\n",
    "from sentence_transformers import SentenceTransformer\n",
    "\n",
    "sentence_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n",
    "embeddings = sentence_model.encode(docs, show_progress_bar=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4aba61ad",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-17 08:50:09,299 - BERTopic - Reduced dimensionality\n",
      "2023-05-17 08:50:09,315 - BERTopic - Clustered reduced embeddings\n"
     ]
    }
   ],
   "source": [
    "# Train BERTopic\n",
    "topic_model = BERTopic().fit(docs, embeddings)\n",
    "\n",
    "# Run the visualization with the original embeddings\n",
    "topic_model.visualize_documents(docs, embeddings=embeddings)\n",
    "\n",
    "reduced_embeddings = UMAP(n_neighbors=10, n_components=2, min_dist=0.0, metric='cosine').fit_transform(embeddings)\n",
    "model.visualize_documents(docs,embeddings=embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "551db1f1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc8ece88",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create your representation model\n",
    "openai.api_key = \"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\"\n",
    "\n",
    "\n",
    "#prompt = \"I have the following documents: [DOCUMENTS] \\nThese documents are about the following topic: '\"\n",
    "#representation_model = OpenAI(prompt=prompt)\n",
    "\n",
    "# representation_model = OpenAI()\n",
    "\n",
    "\n",
    "\n",
    "seed_topic_list = [\n",
    "    ['nature','photography','trees','sunset','view'],\n",
    "    ['hiking','explore', 'running', 'biking','bike', 'walk', 'csodasmagyarorszag'],\n",
    "    ['zoo','lions','animals', 'aquarium', 'wildlife'],\n",
    "    ['wine', 'drinking', 'drink', 'winery'],\n",
    "    ['lake','balaton', 'sea'],\n",
    "    ['lake', 'velence']\n",
    "]\n",
    "\n",
    "\n",
    "# Use the representation model in BERTopic on top of the default pipeline\n",
    "topic_model = BERTopic(seed_topic_list=seed_topic_list, nr_topics=7)\n",
    "\n",
    "#topic_model.reduce_topics(docs, nr_topics=8, representation_model=representation_model)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97b97844",
   "metadata": {},
   "outputs": [],
   "source": [
    "topics, probs = topic_model.fit_transform(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c5d0a1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_model.get_document_info(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f29e0d00",
   "metadata": {},
   "outputs": [],
   "source": [
    "# topic_model.update_topics(docs, n_gram_range=(1, 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a828823b",
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_model.get_topics()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c02c597",
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_model.reduce_topics(docs, nr_topics=10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c031065",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d267dd67",
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_model.get_topics()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}