Upload 6 files

conversation.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+import openai
+import sys
+import os
+
+import csv
+import json
+
+import lookup
+import gpt
+
+openai.api_key = "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"
+if not openai.api_key:
+    openai.api_key = input("Please enter your OpenAI API key: ")
+    print()
+
+program_name = sys.argv.pop(0)
+
+# CSV processing
+csv_file_path = "LIR.csv"  # Update with the correct path
+with open(csv_file_path, newline='', encoding='utf-8') as csvfile:
+    reader = csv.DictReader(csvfile)
+    rows = list(reader)
+
+# Configuration Parameters
+chunk_size = 4000
+overlap = 1000
+limit = 20  # Change to 3 to get the top 3 answers
+gpt.model = "gpt-3.5-turbo"
+
+# Chunking CSV text
+chunks = [row['texte'][i:i + chunk_size] for row in rows for i in range(0, len(row['texte']), chunk_size)]
+
+print("Chunking CSV...\n")
+
+def ask_question(question):
+    keywords = gpt.get_keywords(question)
+    matches = lookup.find_matches(chunks, keywords)
+    top_matches = list(matches.keys())[:limit]
+
+    responses = []
+
+    for i, chunk_id in enumerate(top_matches):
+        chunk = chunks[chunk_id]
+        response = gpt.answer_question(chunk, question)
+
+        if response.get("answer_found"):
+            matched_row = rows[chunk_id]
+            # Extract specific properties from the matched row
+            answer = response.get("response")
+
+            # Loop through the columns and add them to the JSON object
+            json_object = {"GPT_Response": answer}
+
+            for column_name, column_value in matched_row.items():
+                json_object[column_name] = column_value.encode("utf-8").decode("utf-8")
+
+
+            responses.append(json_object)
+
+    responses.append({"keywords:": keywords})
+
+    if not any(response.get("answer_found") for chunk_id in top_matches):
+        responses.append({"GPT_Response": "I'm sorry, but I can't find that information"})
+
+    return responses

gpt.py

@@ -0,0 +1,100 @@
+import openai
+import json
+
+model = "gpt-3.5-turbo"
+
+
+def get_keywords(question):
+    prompt = f""" Je souhaite trouver la réponse à la question suivante dans une colonne d'un fichier csv. Veuillez me fournir 10 mots-clés et synonymes que je peux utiliser pour trouver les informations du csv. Un seul mot par mot-clé. Utilisez uniquement des lettres minuscules.
+{question}"""
+
+
+    response = openai.ChatCompletion.create(
+        model=model,
+        messages=[
+            {
+                "role": "system",
+                "content": "Vous fournirez toujours 10 mots-clés incluant des synonymes pertinents et explicit des mots de la question d’origine. Les synonymes  doivent être des termes juridiques couramment utilisés dans les articles de loi canadienne",     },
+            {
+                "role": "user",
+                "content": prompt,
+            },
+        ],
+        functions=[
+            {
+                "name": "list_keywords",
+                "description": "Utilisez cette fonction pour donner à l'utilisateur une liste de mots-clés",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "list": {
+                            "type": "array",
+                            "items": {"type": "string", "description": "A keyword"},
+                            "description": "A list of keywords",
+                        }
+                    },
+                },
+                "required": ["list"],
+            }
+        ],
+        function_call={"name": "list_keywords", "arguments": ["list"]},
+    )
+
+    arguments = response["choices"][0]["message"]["function_call"]["arguments"].lower()
+    keywords = json.loads(arguments)["list"]
+
+    return " ".join(keywords).split(" ")
+
+
+def answer_question(chunk, question):
+    prompt = f"""```
+{chunk}
+```
+
+Sur la base des informations ci-dessus, quelle est la réponse à cette question?
+
+```
+{question}
+```"""
+
+    response = openai.ChatCompletion.create(
+        model=model,
+        messages=[
+            {
+                "role": "system",
+                "content": "Définissez toujours answer_found sur false si la réponse à la question n'a pas été trouvée dans les informations fournies.",
+            },
+            {
+                "role": "user",
+                "content": prompt,
+            },
+        ],
+        functions=[
+            {
+                "name": "give_response",
+                "description": "Utilisez cette fonction pour donner la réponse et si la réponse à la question a été trouvée ou non dans le texte.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "answer_found": {
+                            "type": "boolean",
+                            "description": "Définissez ceci sur true uniquement si le texte fourni inclut une réponse à la question",
+                        },
+                        "response": {
+                            "type": "string",
+                            "description": "La réponse complète à la question, si l'information était pertinente",
+                        },
+                    },
+                },
+                "required": ["answer_found"],
+            }
+        ],
+    )
+
+    try:
+        function_call = response["choices"][0]["message"]["function_call"]
+        arguments = function_call["arguments"].lower()
+        result = json.loads(arguments)
+        return result
+    except KeyError:
+        return {"answer_found": False, "response": ""}

lookup.py

@@ -0,0 +1,17 @@
+from sklearn.feature_extraction.text import TfidfVectorizer
+
+def find_matches(chunks, keywords, padding=500):
+    # Preprocess chunks
+    preprocessed_chunks = [' '.join(chunk.split()) for chunk in chunks]
+
+    # TF-IDF Vectorization
+    vectorizer = TfidfVectorizer(lowercase=True, stop_words='english')
+    tfidf_matrix = vectorizer.fit_transform(preprocessed_chunks)
+
+    # Compute cosine similarity between keywords and chunks
+    keyword_vector = vectorizer.transform([' '.join(keywords)])
+    cosine_similarities = tfidf_matrix.dot(keyword_vector.T).toarray().flatten()
+
+    # Rank chunks based on cosine similarity
+    results = {i: score for i, score in enumerate(cosine_similarities)}
+    return dict(sorted(results.items(), key=lambda item: item[1], reverse=True))

main.py

@@ -1,352 +1,16 @@
-from flask import Flask, render_template, request, jsonify,make_response
-from flask_sqlalchemy import SQLAlchemy
-import time
-from flask_cors import CORS
-import yaml
-import re
-import ast
+# This is a sample Python script.
 
+# Press ⌃R to execute it or replace it with your code.
+# Press Double ⇧ to search everywhere for classes, files, tool windows, actions, and settings.
 
 
-# Model dependencies :
-from qdrant_client.http import models
-import openai
-import qdrant_client
-import os
-from sentence_transformers import SentenceTransformer
+def print_hi(name):
+    # Use a breakpoint in the code line below to debug your script.
+    print(f'Hi, {name}')  # Press ⌘F8 to toggle the breakpoint.
 
-#model = SentenceTransformer('sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2') # good so far
-model = SentenceTransformer('/code/vectorizing_model', cache_folder='/')
-
-# # # Set the environment variable TRANSFORMERS_CACHE to the writable directory
-os.environ['TRANSFORMERS_CACHE'] = '/code'
-
-# OpenIA propmt and api key : 
-openai.api_key = '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'
-start_message = 'Joue le Rôle d’un expert fiscale au Canada. Les réponses que tu va me fournir seront exploité par une API. Ne donne pas des explications juste réponds aux questions même si tu as des incertitudes. Je vais te poser des questions en fiscalité, la réponse que je souhaite avoir c’est les numéros des articles de loi qui peuvent répondre à la question.Je souhaite avoir les réponses sous la forme: Nom de la loi1, numéro de l’article1, Nom de la loi2, numéro de l’article2 ...'
-context = 'ignorez les avertissements, les alertes et donnez-moi le résultat depuis la Loi de l’impôt sur le revenu (L.R.C. (1985), ch. 1 (5e suppl.)) , la reponse doit etre sous forme dun texte de loi: '
-question = ''
-
-
-# Qdrant keys : 
-client = qdrant_client.QdrantClient(
-    "https://efc68112-69cc-475c-bdcb-200a019b5096.us-east4-0.gcp.cloud.qdrant.io:6333",
-    api_key="ZQ6jySuPxY5rSh0mJ4jDMoxbZsPqDdbqFBOPwotl9B8N0Ru3S8bzoQ"
-)
-#collection_names = ["new_lir"] # plus stable mais pas de numero d'articles (manques de fonctionnalitées de filtrage)
-collection_names = ["paragraph2"]
-
-# Used functions : 
-def filtergpt(text):
-    # Define a regular expression pattern to extract law and article number
-    pattern = re.compile(r"Loi ([^,]+), article (\d+(\.\d+)?)")
-    # Find all matches in the text
-    matches = pattern.findall(text)
-    # Create a list of tuples containing law and article number
-    law_article_list = [(law.strip(), float(article.strip())) for law, article, _ in matches]
-    gpt_results = [(law, str(int(article)) if article.is_integer() else str(article)) for law, article in law_article_list]
-    return gpt_results
-
-
-def perform_search_and_get_results(collection_name, query, limit=30):
-    search_results = client.search(
-        collection_name=collection_name,
-        query_vector=model.encode(query).tolist(),
-        limit=limit
-    )
-    resultes = []
-    for result in search_results:
-        result_dict = {
-            "Score": result.score,
-            "La_loi": result.payload["reference"],
-            "Paragraphe": result.payload["paragraph"],
-            "titre": result.payload["titre"],
-            "section_text": result.payload["section"],
-            "section_label": result.payload["section_label"],
-            "source": result.payload["source"],
-            "numero_article": result.payload["numero_article"],
-            "collection": collection_name,
-            "hyperlink": ast.literal_eval(result.payload['hyperlink']),
-        }
-        resultes.append(result_dict)
-    return resultes
-
-def perform_search_and_get_results_with_filter(collection_name, query,reference_filter , limit=30):
-    search_results = client.search(
-        collection_name=collection_name,
-        query_filter=models.Filter(must=[models.FieldCondition(key="numero_article",match=models.MatchValue(value=reference_filter+"aymane",),)]),
-        query_vector=model.encode(query).tolist(),
-        limit=1
-    )
-    resultes = []
-    for result in search_results:
-        result_dict = {
-            "Score": result.score,
-            "La_loi": result.payload["reference"],
-            "Paragraphe": result.payload["paragraph"],
-            "titre": result.payload["titre"],
-            "section_text": result.payload["section"],
-            "section_label": result.payload["section_label"],
-            "source": result.payload["source"],
-            "numero_article": result.payload["numero_article"],
-            "collection": collection_name,
-            "hyperlink": ast.literal_eval(result.payload['hyperlink']),
-        }
-        resultes.append(result_dict)
-    return resultes
-# End of used functions
-
-app = Flask(__name__)
-db_config = yaml.safe_load(open('database.yaml'))
-app.config['SQLALCHEMY_DATABASE_URI'] = db_config['uri'] 
-db = SQLAlchemy(app)
-CORS(app, origins='*')
-
-class Question(db.Model):
-    __tablename__ = "questions"
-    id = db.Column(db.Integer, primary_key=True)
-    date = db.Column(db.String(255))
-    texte = db.Column(db.String(255))
-    
-    def __init__(self, date, texte):
-        self.date = date
-        self.texte = texte
-    
-    def __repr__(self):
-        return '%s/%s/%s' % (self.id, self.date, self.texte)
-
-
-@app.route('/')
-def index():
-    return render_template('home.html')
-
-@app.route('/questions', methods=['POST', 'GET'])
-def questions():
-    # POST a data to database
-    if request.method == 'POST':
-        body = request.json
-        date = body['date']
-        texte = body['texte']
-
-        data = Question(date, texte)
-        db.session.add(data)
-        db.session.commit()
-
-        return jsonify({
-            'status': 'Data is posted to PostgreSQL!',
-            'date': date,
-            'texte': texte
-        })
-    
-    # GET all data from database & sort by id
-    if request.method == 'GET':
-        # data = User.query.all()
-        data = Question.query.all()
-        print(data)
-        dataJson = []
-        for i in range(len(data)):
-            # print(str(data[i]).split('/'))
-            dataDict = {
-                'id': str(data[i]).split('/')[0],
-                'date': str(data[i]).split('/')[1],
-                'texte': str(data[i]).split('/')[2]
-            }
-            dataJson.append(dataDict)
-        return jsonify(dataJson)
-
-@app.route('/questions/<string:id>', methods=['GET', 'DELETE', 'PUT'])
-def onedata(id):
-
-    # GET a specific data by id
-    if request.method == 'GET':
-        data = Question.query.get(id)
-        print(data)
-        dataDict = {
-            'id': str(data).split('/')[0],
-            'date': str(data).split('/')[1],
-            'texte': str(data).split('/')[2]
-        }
-        return jsonify(dataDict)
-        
-    # DELETE a data
-    if request.method == 'DELETE':
-        delData = Question.query.filter_by(id=id).first()
-        db.session.delete(delData)
-        db.session.commit()
-        return jsonify({'status': 'Data '+id+' is deleted from PostgreSQL!'})
-
-    # UPDATE a data by id
-    if request.method == 'PUT':
-        body = request.json
-        newDate = body['date']
-        newTexte = body['texte']
-        editData = Question.query.filter_by(id=id).first()
-        editData.date = newDate
-        editData.texte = newTexte
-        db.session.commit()
-        return jsonify({'status': 'Data '+id+' is updated from PostgreSQL!'})
-
-@app.route('/chat', methods=['OPTIONS'])
-def options():
-    response = make_response()
-    response.headers.add("Access-Control-Allow-Origin", "*")
-    response.headers.add("Access-Control-Allow-Methods", "POST")
-    response.headers.add("Access-Control-Allow-Headers", "Content-Type, Authorization")
-    response.headers.add("Access-Control-Allow-Credentials", "true")
-    return response
-    
-@app.route('/chat', methods=['POST'])
-def chat():
-    try:
-        data = request.get_json()
-        messages = data.get('messages', [])
-
-        if messages:
-            results = []
-            # Update the model name to "text-davinci-003" (Ada)
-            prompt = "\n".join([f"{msg['role']}: {msg['content']}" for msg in messages])
-            response = openai.completions.create(
-                  model="gpt-3.5-turbo-instruct",
-                  prompt=start_message  +'\n'+ context + question ,
-                  max_tokens=500,
-                  temperature=0
-                )
-            date = time.ctime(time.time())
-            texte = prompt
-            data = Question(date, texte)
-            db.session.add(data)
-            db.session.commit()
-            question_id = data.id
-            resulta = response.choices[0].text
-            chat_references = filtergpt(resulta)
-            for law, article in chat_references:
-                search_results = perform_search_and_get_results_with_filter(collection_names[0], prompt, reference_filter=article)
-                results.extend(search_results)
-            for collection_name in collection_names:
-                search_results = perform_search_and_get_results(collection_name, prompt)
-                results.extend(search_results)
-            return jsonify({'question': {'id': question_id, 'date': date, 'texte': texte},'result_qdrant':results})
-        else:
-            return jsonify({'error': 'Invalid request'}), 400
-    except Exception as e:
-        return jsonify({'error': str(e)}), 500
-
-
-@app.route('/chatgrouped', methods=['OPTIONS'])
-def options_grouped():
-    response = make_response()
-    response.headers.add("Access-Control-Allow-Origin", "*")
-    response.headers.add("Access-Control-Allow-Methods", "POST")
-    response.headers.add("Access-Control-Allow-Headers", "Content-Type, Authorization")
-    response.headers.add("Access-Control-Allow-Credentials", "true")
-    return response
-    
-@app.route('/chatgrouped', methods=['POST'])
-def chat_grouped():
-    try:
-        data = request.get_json()
-        messages = data.get('messages', [])
-
-        if messages:
-            results = []
-            # Update the model name to "text-davinci-003" (Ada)
-            prompt = "\n".join([f"{msg['role']}: {msg['content']}" for msg in messages])
-            response = openai.completions.create(
-                  model="gpt-3.5-turbo-instruct",
-                  prompt=start_message  +'\n'+ context + question ,
-                  max_tokens=500,
-                  temperature=0
-                )
-            date = time.ctime(time.time())
-            texte = prompt
-            data = Question(date, texte)
-            db.session.add(data)
-            db.session.commit()
-            question_id = data.id
-            resulta = response.choices[0].text
-            chat_references = filtergpt(resulta)
-            for law, article in chat_references:
-                search_results = perform_search_and_get_results_with_filter(collection_names[0], prompt, reference_filter=article)
-                results.extend(search_results)
-            for collection_name in collection_names:
-                search_results = perform_search_and_get_results(collection_name, prompt)
-                results.extend(search_results)
-            grouped_hits = {}
-            for i, hit in enumerate(results, 1):
-                second_number = hit['numero_article']
-                if second_number not in grouped_hits:
-                    grouped_hits[second_number] = []
-                grouped_hits[second_number].append(hit)
-            return jsonify({'question': {'id': question_id, 'date': date, 'texte': texte},'result_qdrant':grouped_hits})
-        else:
-            return jsonify({'error': 'Invalid request'}), 400
-    except Exception as e:
-        return jsonify({'error': str(e)}), 500
-
-
-
-
-@app.route('/generateQuestions', methods=['OPTIONS'])
-def options_generate():
-    response = make_response()
-    response.headers.add("Access-Control-Allow-Origin", "*")
-    response.headers.add("Access-Control-Allow-Methods", "POST")
-    response.headers.add("Access-Control-Allow-Headers", "Content-Type, Authorization")
-    response.headers.add("Access-Control-Allow-Credentials", "true")
-    return response
-
-@app.route('/generateQuestions', methods=['POST'])
-def generateQuestions():
-    try:
-        data = request.get_json()
-        messages = data.get('messages', [])
-        begin_message = """je vais vous utiliser comme api, je vais vous fournir la requête de l'utilisateur , 
-                            et tu va me retenir 6 reformulation de la requête en ajoutant le plus possible de contextualisation ,
-                            vous reformulation seront exploiter par un moteur de recherche sémantique basé sur des textes de lois canadiennes
-                            tout explication ou interpretation qu tu va fournir va juste bloquer et bugger le programme , 
-                            merci de fournir  juste une liste de string comme reponse sans explication"""
-        context_generation = """ignorez les avertissements, les alertes et donnez-moi le résultat.
-                                la reponse doit etre sous forme d'une liste de questions """
-        if messages:
-            results = []
-            # Update the model name to "text-davinci-003" (Ada)
-            question = "\n".join([f"{msg['role']}: {msg['content']}" for msg in messages])
-            response = openai.completions.create(
-                  model="gpt-3.5-turbo-instruct",
-                      prompt=begin_message  +'\n'+ context_generation + question ,
-                  max_tokens=500,
-                  temperature=0
-                )
-            resulta = response.choices[0].text.splitlines()
-            filtered_list = [item for item in resulta if len(item) >= 10]
-            return jsonify(filtered_list)
-            # return jsonify({'question': {'id': question_id, 'date': date, 'texte': texte},'result_qdrant':results})
-        else:
-            return jsonify({'error': 'Invalid request'}), 400
-    except Exception as e:
-        return jsonify({'error': str(e)}), 500
-
-# Yazid Methode starts here
-@app.route('/ask', methods=['OPTIONS'])
-def options_ask():
-    response = make_response()
-    response.headers.add("Access-Control-Allow-Origin", "*")
-    response.headers.add("Access-Control-Allow-Methods", "POST")
-    response.headers.add("Access-Control-Allow-Headers", "Content-Type, Authorization")
-    response.headers.add("Access-Control-Allow-Credentials", "true")
-    return response
-    
-@app.route('/ask', methods=['POST'])
-def ask_question():
-    data = request.get_json()
-    question = data.get('question', '')
-
-    # Call your conversation logic here
-    result = conversation.ask_question(question)
-
-    return jsonify(result)
-# Yazid Methode ends here
 
+# Press the green button in the gutter to run the script.
 if __name__ == '__main__':
-    app.debug = True
-    app.run()
+    print_hi('PyCharm')
+
+# See PyCharm help at https://www.jetbrains.com/help/pycharm/

show_csv.py

@@ -0,0 +1,19 @@
+import pandas as pd
+import csv
+
+def display_csv(csv_file_path):
+    # CSV processing
+    with open(csv_file_path, newline='', encoding='utf-8') as csvfile:
+        reader = csv.DictReader(csvfile)
+        rows = list(reader)
+
+    # Display the entire CSV using pandas
+    df = pd.DataFrame(rows)
+    print("Entire CSV File:")
+    print(df['texte'].head(10))
+
+if __name__ == "__main__":
+    # Replace 'LIR.csv' with the correct path to your CSV file
+    csv_file_path = "LIR.csv"
+
+    display_csv("LIR.csv")