coppy 22.04

Home.py

@@ -0,0 +1,25 @@
+import gradio as gr
+from src.chatbot import chatbot, keyword_search
+
+# Adjust size of each block is not yet working
+output = gr.DataFrame(height=1000, show_label=True, scale=2)
+input = gr.Textbox(scale=1)
+
+with gr.Blocks() as App:
+    with gr.Tab("ChatBot"):
+        # Apply RAG using chatbut function from local file ChatBot.py
+        gr.ChatInterface(chatbot,
+                    title="PoliticsToYou",
+                    description= "This chatbot uses the infomation of speeches of the german parliament (since 2021) \
+                        to get insight on the view points of the german parties and the debate of the parliament.",
+                    examples=["Wie steht die CDU zur Cannabislegalisierung?","Was waren die wichtigsten Themen in der aktuellen Legislaturperiode?"], #change to meaningful examples
+                    cache_examples=False, #true increases the loading time
+                    )
+    with gr.Tab("KeyWordSearch"):
+        gr.Interface(fn=keyword_search, inputs=input, outputs=output)
+         
+        
+if __name__ == "__main__":
+    App.launch(share=True) 
+
+

README.md

@@ -5,7 +5,7 @@ colorFrom: pink
 colorTo: green
 sdk: gradio
 sdk_version: 4.26.0
-app_file: app.py
+app_file: Home.py
 pinned: false
 ---
 

requirements.txt

@@ -0,0 +1,7 @@
+pandas==2.1.3
+langchain==0.1.15
+transformers==4.35.2
+gradio==4.26.0
+sentence-transformers==2.6.1
+python-dotenv
+faiss-cpu

src/FAISS.ipynb

@@ -0,0 +1,46 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create vectorstore\n",
+    "import pandas as pd\n",
+    "from vectordatabase import load_documents\n",
+    "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
+    "from langchain_community.vectorstores import FAISS\n",
+    "\n",
+    "\n",
+    "df = pd.read_pickle(\"C:\\\\Users\\Tom\\SynologyDrive\\Programming\\\\NLP\\PoliticsToYou\\ChatBot\\Speeches\\speeches_1949_09_12.pkl\")\n",
+    "# Split speeches into documents\n",
+    "documents = load_documents(df)\n",
+    "embeddings = HuggingFaceEmbeddings(model_name=\"paraphrase-multilingual-MiniLM-L12-v2\")\n",
+    "db = FAISS.from_documents(documents, embeddings)\n",
+    "db.save_local(folder_path=\"ChatBot\\FAISS\", index_name=\"speeches_1949_09_12\")\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/FAISS/legislature20.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e748028f9f5a5424be29c57df6b2387e3c723b6490cc48d33b4ab75fdc813de
+size 82403373

src/FAISS/legislature20.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:614bbfa822757149ba2aa055d349b3e68828ba4eda24dfe1531ca523c446a3ea
+size 73684827

src/Speeches/querry.ipynb

@@ -0,0 +1,247 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import psycopg2\n",
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Pandas\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\Tom\\AppData\\Local\\Temp\\ipykernel_21040\\1041354989.py:12: UserWarning: pandas only supports SQLAlchemy connectable (engine/connection) or database string URI or sqlite3 DBAPI2 connection. Other DBAPI2 objects are not tested. Please consider using SQLAlchemy.\n",
+      "  df = pd.read_sql_query(\"\"\"SELECT s.id,s.speech_content,s.date,f.abbreviation AS party\n"
+     ]
+    }
+   ],
+   "source": [
+    "# db_connection -----------------------------------------------------------\n",
+    "con_details = {\n",
+    "    \"host\"      : \"localhost\",\n",
+    "    \"database\"  : \"next\",\n",
+    "    \"user\"      : \"postgres\",\n",
+    "    \"password\"  : \"postgres\",\n",
+    "    \"port\"      : \"5432\"\n",
+    "}\n",
+    "con = psycopg2.connect(**con_details)\n",
+    "\n",
+    "# get data tables ---------------------------------------------------------\n",
+    "df = pd.read_sql_query(\"\"\"SELECT s.id,s.speech_content,s.date,f.abbreviation AS party\n",
+    "                        FROM open_discourse.speeches AS s\n",
+    "                        INNER JOIN open_discourse.factions AS f ON\n",
+    "                        s.faction_id = f.id;\"\"\", con)\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Data Cleaning"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>id</th>\n",
+       "      <th>speech_content</th>\n",
+       "      <th>date</th>\n",
+       "      <th>party</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0</td>\n",
+       "      <td>Meine Damen und Herren! Ich eröffne die 2. Sit...</td>\n",
+       "      <td>1949-09-12</td>\n",
+       "      <td>not found</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>1</td>\n",
+       "      <td>Der Bundesrat ist versammelt, Herr Präsident.\\n</td>\n",
+       "      <td>1949-09-12</td>\n",
+       "      <td>not found</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>2</td>\n",
+       "      <td>Ich danke für diese Erklärung. Ich stelle dami...</td>\n",
+       "      <td>1949-09-12</td>\n",
+       "      <td>not found</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>3</td>\n",
+       "      <td>Ja, ich habe den Wunsch.\\n</td>\n",
+       "      <td>1949-09-12</td>\n",
+       "      <td>not found</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>4</td>\n",
+       "      <td>Ich erteile dem Herrn Bundespräsidenten das Wo...</td>\n",
+       "      <td>1949-09-12</td>\n",
+       "      <td>not found</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>930955</th>\n",
+       "      <td>1084268</td>\n",
+       "      <td>\\n\\nWir sind zwar Kollegen.</td>\n",
+       "      <td>2022-12-16</td>\n",
+       "      <td>not found</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>930956</th>\n",
+       "      <td>1084269</td>\n",
+       "      <td>\\n\\nLiebe, sehr geehrte Frau Präsidentin!</td>\n",
+       "      <td>2022-12-16</td>\n",
+       "      <td>CDU/CSU</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>930957</th>\n",
+       "      <td>1084270</td>\n",
+       "      <td>\\n\\nVielen Dank.</td>\n",
+       "      <td>2022-12-16</td>\n",
+       "      <td>not found</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>930958</th>\n",
+       "      <td>1084272</td>\n",
+       "      <td>\\n\\nDen Abschluss dieser Aktuellen Stunde bild...</td>\n",
+       "      <td>2022-12-16</td>\n",
+       "      <td>not found</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>930959</th>\n",
+       "      <td>1084273</td>\n",
+       "      <td>\\n\\nSehr geehrte Frau Präsidentin! Werte Kolle...</td>\n",
+       "      <td>2022-12-16</td>\n",
+       "      <td>SPD</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>930960 rows × 4 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "             id                                     speech_content  \\\n",
+       "0             0  Meine Damen und Herren! Ich eröffne die 2. Sit...   \n",
+       "1             1    Der Bundesrat ist versammelt, Herr Präsident.\\n   \n",
+       "2             2  Ich danke für diese Erklärung. Ich stelle dami...   \n",
+       "3             3                         Ja, ich habe den Wunsch.\\n   \n",
+       "4             4  Ich erteile dem Herrn Bundespräsidenten das Wo...   \n",
+       "...         ...                                                ...   \n",
+       "930955  1084268                        \\n\\nWir sind zwar Kollegen.   \n",
+       "930956  1084269          \\n\\nLiebe, sehr geehrte Frau Präsidentin!   \n",
+       "930957  1084270                                   \\n\\nVielen Dank.   \n",
+       "930958  1084272  \\n\\nDen Abschluss dieser Aktuellen Stunde bild...   \n",
+       "930959  1084273  \\n\\nSehr geehrte Frau Präsidentin! Werte Kolle...   \n",
+       "\n",
+       "              date      party  \n",
+       "0       1949-09-12  not found  \n",
+       "1       1949-09-12  not found  \n",
+       "2       1949-09-12  not found  \n",
+       "3       1949-09-12  not found  \n",
+       "4       1949-09-12  not found  \n",
+       "...            ...        ...  \n",
+       "930955  2022-12-16  not found  \n",
+       "930956  2022-12-16    CDU/CSU  \n",
+       "930957  2022-12-16  not found  \n",
+       "930958  2022-12-16  not found  \n",
+       "930959  2022-12-16        SPD  \n",
+       "\n",
+       "[930960 rows x 4 columns]"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df[\"speech_content\"].replace(\"\\({\\d+}\\)\", \"\", inplace=True, regex=True) #removing keys from interruptions\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df.to_pickle(\"speeches_1949_09_12\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/chatbot.py

@@ -0,0 +1,78 @@
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_community.llms.huggingface_hub import HuggingFaceHub
+from langchain_community.embeddings import HuggingFaceEmbeddings
+
+from src.vectordatabase import RAG, get_vectorstore
+import pandas as pd
+import os
+from dotenv import load_dotenv, find_dotenv
+
+#Load environmental variables from .env-file
+load_dotenv(find_dotenv())
+
+
+embeddings = HuggingFaceEmbeddings(model_name="paraphrase-multilingual-MiniLM-L12-v2")
+llm = HuggingFaceHub(
+    # Try different model here
+    # repo_id="mistralai/Mixtral-8x7B-Instruct-v0.1",
+    repo_id="CohereForAI/c4ai-command-r-v01",
+    task="text-generation",
+    model_kwargs={
+        "max_new_tokens": 512,
+        "top_k": 30,
+        "temperature": 0.1,
+        "repetition_penalty": 1.03,
+        }
+)
+# To Do: Experiment with different templates replying in german or english depending on the input language
+prompt1 = ChatPromptTemplate.from_template("""<s>[INST] 
+                    Instruction: Beantworte die folgende Frage auf deutsch und nur auf der Grundlage des angegebenen Kontexts:
+
+                    Context: {context}
+
+                    Question: {input}  
+                    [/INST]"""
+                    # Returns the answer in English!?
+) 
+
+prompt2 = ChatPromptTemplate.from_template("""Beantworte die folgende Frage auf deutsch und nur auf der Grundlage des angegebenen Kontexts:
+
+        <context>
+        {context}
+        </context>
+
+        Frage: {input}
+        """
+        # Returns the answer in German
+)
+
+ 
+folder_path = "./src/FAISS"
+#index_name = "speeches_1949_09_12"
+index_name = "legislature20"
+db = get_vectorstore(embeddings=embeddings, folder_path=folder_path, index_name=index_name)
+
+def chatbot(message, history, db=db, llm=llm, prompt=prompt2):
+    raw_response = RAG(llm=llm, prompt=prompt, db=db, question=message)
+    response = raw_response['answer'].split("Antwort: ")[1]
+    return response  
+
+# Retrieve speech contents based on keywords
+def keyword_search(query, db=db, embeddings=embeddings):
+    query_embedding = embeddings.embed_query(query)
+    results =  db.max_marginal_relevance_search_with_score_by_vector(query_embedding)
+    # Format vector store query results into dataframe
+    #print(results[0][0].metadata.keys())
+    
+    df_res = pd.DataFrame(columns=['Speech Content', 'Relevance']) # Add Date/Party/Politician
+    for doc in results:
+        speech_content = doc[0].page_content
+        #speech_date = doc[0].metadata["date"]
+        score = doc[1] # Relevance based on relevance search
+        df_res = pd.concat([df_res, pd.DataFrame({'Speech Content': [speech_content],
+                                                          #'Date': [speech_date],
+                                                          'Relevance': [score]})], ignore_index=True)
+    
+    df_res.sort_values('Relevance', inplace=True)
+    return df_res

src/vectordatabase.py

@@ -0,0 +1,53 @@
+from langchain_community.document_loaders import DataFrameLoader
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_community.vectorstores import FAISS
+from langchain_community.llms import HuggingFaceHub
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.chains.combine_documents import create_stuff_documents_chain
+from langchain.chains import create_retrieval_chain
+
+import os
+#from dotenv import load_dotenv
+
+#Load environmental variables from .env-file
+#load_dotenv()
+
+
+# Load documents to create a vectorstore later
+def load_documents(df):
+    # To Do: Create one initial vectore store loading all the documents with this function
+    #loader = CSVLoader(index_name, source_column="speech_content") #unprocessed csv file
+    loader = DataFrameLoader(data_frame=df, page_content_column='speech_content') #df
+    data = loader.load()
+    splitter = RecursiveCharacterTextSplitter(
+            chunk_size=1024,
+            chunk_overlap=32,
+            length_function=len,
+            is_separator_regex=False,
+        )
+    documents = splitter.split_documents(documents=data)
+    return documents
+
+def get_vectorstore(embeddings, folder_path, index_name): 
+    path = folder_path + "/" + index_name
+    print(path)
+    # To Do: Dynamicly update and merge verctorstores
+    #if os.path.exists(path):
+    db = FAISS.load_local(folder_path=folder_path, index_name=index_name,
+                                            embeddings=embeddings, allow_dangerous_deserialization=True)
+    #else:
+        #db = FAISS.from_documents(documents, embeddings)
+        #db.save_local(folder_path=folder_path, index_name=index_name)
+        #pass
+    return db
+
+# Apply RAG by providing the context and the question to the LLM using the predefined template
+def RAG(llm, prompt, db, question):       
+    document_chain = create_stuff_documents_chain(llm=llm, prompt=prompt)
+    retriever = db.as_retriever()
+    retrieval_chain = create_retrieval_chain(retriever, document_chain)
+
+    response = retrieval_chain.invoke({"input": question})
+    return response
+