Initial commit

.gitignore

@@ -0,0 +1,7 @@
+.env
+.venv
+io/data/original-sample.txt
+**/*.pyc
+**/__pycache__
+io/generated/**
+!io/generated/.gitkeep

README.md

@@ -1,3 +1,3 @@
----
-license: cc-by-nc-4.0
----
+# Chatbot about Pele
+
+This is demo project.

commons/Configs.py

@@ -0,0 +1,21 @@
+import os
+
+
+class Configs:
+    def __init__(self):
+        # environment variables
+        os.environ["PROJECT_ROOT"] = os.getcwd()
+        # openai
+        self.OPENAI_KEY = ""
+        self.chatCompletionModel = "gpt-3.5-turbo"
+        self.embeddingsModel = "text-embedding-ada-002"
+        # generated files
+        self.generatedDatasetPath = f"{os.environ['PROJECT_ROOT']}/io/generated/dataset.json"
+        self.generatedEmbeddingsPath = f"{os.environ['PROJECT_ROOT']}/io/generated/embeddings.json"
+        # spacy
+        self.spacyModel = 'en_core_web_sm'
+        # model
+        self.generatedModelPath = f"{os.environ['PROJECT_ROOT']}/io/generated/model.sklearn"
+
+
+configs = Configs()

commons/File.py

@@ -0,0 +1,22 @@
+import json
+import os
+
+
+class File:
+    def readFile(self, file):
+        with open(file, 'r') as f:
+            return f.read()
+
+    def readJsonFile(self, file):
+        with open(file, 'r') as f:
+            return json.load(f)
+
+    def writeFile(self, outputFilePath, data):
+        with open(outputFilePath, 'w') as f:
+            f.write(json.dumps(data))
+
+    def exists(self, filePath):
+        return os.path.exists(filePath)
+
+
+file = File()

commons/Model.py

@@ -0,0 +1,26 @@
+from sklearn.linear_model import LogisticRegression
+import joblib
+from commons.Configs import configs
+from commons.File import file
+
+
+class Model:
+    def __init__(self, debug=False):
+        self.debug = debug
+
+    def train(self, x, y):
+        return LogisticRegression(solver='lbfgs', random_state=42).fit(x, y)
+
+    def save(self, clf):
+        # save model
+        joblib.dump(clf, configs.generatedModelPath)
+        print("Model saved to: ", configs.generatedModelPath)
+
+    def load(self):
+        if not file.exists(configs.generatedModelPath):
+            print("Model not found at: ", configs.generatedModelPath)
+            exit(1)
+        return joblib.load(configs.generatedModelPath)
+
+
+model = Model()

commons/OpenAIClient.py

@@ -0,0 +1,89 @@
+import os
+from commons.Configs import configs
+from commons.File import file
+import openai
+from openai.embeddings_utils import cosine_similarity
+import json
+
+
+class OpenAIClient:
+    def __init__(self, debug=False):
+        self.debug = debug
+        openai.api_key = configs.OPENAI_KEY
+        self.embeddingsModel = configs.embeddingsModel
+
+    def buildPrompt(self, name, variables):
+        # used by prepareutils.Dataset
+        promptFilePath = os.path.join(configs.promptsDir, f"{name}.prompt.txt")
+        prompt = file.readFile(promptFilePath)
+        for key, value in variables.items():
+            prompt = prompt.replace(f"{{{key}}}", value)
+        return prompt
+
+    def generateSyntheticQuestions(self, prompt, debugSentence=""):
+        # used by prepareutils.Dataset
+        """Use OpenAI completion API to generate synthetic questions for each sentence"""
+        # ----------------------------------------------
+        # generate questions (responseText)
+        # ----------------------------------------------
+        response = openai.ChatCompletion.create(
+            model=configs.chatCompletionModel,
+            messages=[{"role": "user", "content": prompt}]
+        )
+        responseText = response['choices'][0]['message']['content']
+        # ----------------------------------------------
+        # split questions and answers
+        # ----------------------------------------------
+        # make all question/answers to be on the same line
+        # and remove the response header
+        questionAnswers = responseText.replace("\n", "").split('(Q)', 1)[1]
+        # one line per question/answer
+        questionAnswers = questionAnswers.split('(Q)')
+        # split question and answers
+        questionAnswers = [x.split('(A)', 1) for x in questionAnswers]
+        # remove invalid rows and strip
+        questionAnswers = [[x[0].strip(), x[1].strip()]
+                           for x in questionAnswers if len(x) == 2]
+        jsonData = [{"question": x[0], "answer": x[1]}
+                    for x in questionAnswers]
+        # ----------------------------------------------
+        # debug
+        if self.debug:
+            print("Sentence: ", debugSentence)
+            print("Response text: ", responseText)
+            print("jsonData: ", json.dumps(jsonData, indent=4))
+        # ----------------------------------------------
+        return jsonData
+
+    def generateEmbeddings(self, sentences):
+        # used by prepareutils.Embeddings
+        response = openai.Embedding.create(
+            input=sentences,
+            model=self.embeddingsModel,
+        )
+        embeddings = []
+        for x in response['data']:
+            embeddings.append(x['embedding'])
+        assert len(embeddings) == len(sentences)
+        return embeddings
+
+    def searchBestEmbeddingIndex(self, embeddedQuestion, embeddingsToSearch):
+        # find the most similar sentence
+        # used by ask.py
+        """Search for the best embedding index"""
+        maxSimilarity = 0
+        maxSimilarityIndex = 0
+        for i, embedding in enumerate(embeddingsToSearch):
+            # similarity = cosineSimilarity(
+            #     np.array(questionEmbedding['data'][0]['embedding']), embedding)
+            similarity = cosine_similarity(embeddedQuestion, embedding)
+            if similarity > maxSimilarity:
+                maxSimilarity = similarity
+                maxSimilarityIndex = i
+        # return the most similar sentence index
+        return maxSimilarityIndex
+        # return the most similar embedding
+        # return df.iloc[maxSimilarityIndex].sentences
+
+
+openaiClient = OpenAIClient()

commons/SpacyUtils.py

@@ -0,0 +1,24 @@
+import spacy
+from commons.Configs import configs
+
+
+class SpacyUtils:
+    def __init__(self, debug=False):
+        self.debug = debug
+
+    # Receives a raw text and returns an array of sentences
+    def splitSentences(self, text):
+        """Split text into sentences"""
+        nlp = self.spacyLoad()
+        doc = nlp(text)
+        return [str(sent.text).replace('"', '') for sent in doc.sents]
+
+    # Returns a spacy.load() model
+    def spacyLoad(self):
+        """Load spacy model"""
+        if not hasattr(self, 'spacyInstance'):
+            self.spacyInstance = spacy.load(configs.spacyModel)
+        return self.spacyInstance
+
+
+spacyUtils = SpacyUtils()

prepareutils/Dataset.py

@@ -0,0 +1,56 @@
+from tqdm import tqdm
+from commons.Configs import configs
+from commons.File import file
+from commons.OpenAIClient import openaiClient
+from commons.SpacyUtils import spacyUtils
+
+
+class Dataset:
+    def __init__(self, debug=False):
+        self.debug = debug
+
+    # Receives an <inputFile>
+    # generate synthetic questions and answers
+    # save to <outputFile>
+    def generateDatasetFromFile(self, inputFile):
+        outputFile = configs.generatedDatasetPath
+        # allQaRows is an array where each item is a dict with {"question","answer"} keys
+        # ? should I use a list of tuples instead?
+        allQaRows = []
+        print("Reading input file: ", inputFile)
+        text = file.readFile(inputFile)
+        # split text into sentences and augment each sentence with synthetic questions and answers
+        print("Generating questions and answers for each sentence")
+        for sent in tqdm(spacyUtils.splitSentences(text)):
+            prompt = openaiClient.buildPrompt("generateQuestionsPerson", {
+                'NAME': configs.PROMPT_PERSON_NAME,
+                'SOCIALNAME': configs.PROMPT_PERSON_SOCIALNAME,
+                'TITLE': configs.PROMPT_PERSON_TITLE,
+                'HESHEIT': configs.PROMPT_PERSON_HESHEIT,
+                'BIRTHDAY': configs.PROMPT_PERSON_BIRTHDAY,
+                'DEATHDAY': configs.PROMPT_PERSON_DEATHDAY,
+                'BIRTHPLACE': configs.PROMPT_PERSON_BIRTHPLACE,
+                'DEATHPLACE': configs.PROMPT_PERSON_DEATHPLACE,
+                'NUMBER_OF_QUESTIONS': configs.PROMPT_PERSON_NUMBER_OF_QUESTIONS,
+                'SENTENCE': sent
+            })
+            genq = openaiClient.generateSyntheticQuestions(
+                prompt, debugSentence=sent)
+            allQaRows.extend(genq)
+            # debug
+            if self.debug:
+                for x in genq:
+                    print("Sentence: ", sent)
+                    print("Q: ", x['question'])
+                    print("A: ", x['answer'])
+        # save all the generated questions and answers in a generated dataset file
+        # Default: io/generated/dataset.json
+        print("Writing dataset to file: ", outputFile)
+        file.writeFile(outputFile, allQaRows)
+
+    def loadDataset(self):
+        inputFilePath = configs.generatedDatasetPath
+        return file.readJsonFile(inputFilePath)
+
+
+dataset = Dataset()

prepareutils/Embeddings.py

@@ -0,0 +1,44 @@
+from tqdm import tqdm
+import numpy as np
+from commons.Configs import configs
+from commons.File import file
+from commons.OpenAIClient import openaiClient
+
+
+class Embeddings:
+    def __init__(self, debug=False):
+        self.debug = debug
+
+    def generateEmbeddings(self):
+        inputFilePath = configs.generatedDatasetPath
+        outputFilePath = configs.generatedEmbeddingsPath
+        dataset = file.readJsonFile(inputFilePath)
+        embeddings = []
+        print("")
+        # for each sentence
+        for i, qa in enumerate(tqdm(dataset)):
+            sentences = [qa['question'], qa['answer']]
+            emb = openaiClient.generateEmbeddings(sentences)
+            embjson = {'question': emb[0], 'answer': emb[1], 'label': i}
+            print("Sentence: ", i, sentences)
+            embeddings.append(embjson)
+        # save all the generated embeddings
+        # Default: io/generated/embeddings.json
+        print("Writing embeddings to file: ", outputFilePath)
+        file.writeFile(outputFilePath, embeddings)
+
+    def loadEmbeddings(self):
+        inputFilePath = configs.generatedEmbeddingsPath
+        embeddings = file.readJsonFile(inputFilePath)
+        questionEmbeddings = [x['question'] for x in embeddings]
+        answerEmbeddings = [x['answer'] for x in embeddings]
+        labels = [x['label'] for x in embeddings]
+        # i would use float16, but I've had issues with GPU
+        # I know I'm not using GPU now, but I might in the future
+        return \
+            np.array(questionEmbeddings, dtype=np.float32), \
+            np.array(answerEmbeddings, dtype=np.float32), \
+            np.array(labels, dtype=np.int32)
+
+
+embeddings = Embeddings()

requirements.txt

@@ -0,0 +1,15 @@
+# openai
+openai==0.27.2
+scipy
+
+# spacy       https://pypi.org/project/spacy/
+setuptools
+wheel
+spacy
+
+# model
+scikit-learn
+numpy
+
+# demo webserver
+gradio

serve.py

@@ -0,0 +1,43 @@
+import gradio as gr
+from commons.Model import model
+from commons.Configs import configs
+from commons.OpenAIClient import openaiClient
+from prepareutils.Dataset import dataset
+import numpy as np
+import openai
+
+# load the model once
+clf = model.load()
+# load dataset
+qaDataset = dataset.loadDataset()
+
+
+def predict(question, openaiKey):
+    # set openaiKey
+    configs.OPENAI_KEY = openaiKey
+    openai.api_key = openaiKey
+    # embed question
+    questionEmbedding = openaiClient.generateEmbeddings([question])[0]
+    # predict answer index
+    answerIndex = clf.predict([questionEmbedding]).item()
+    # get answer
+    bestAnswer = qaDataset[answerIndex]
+    return bestAnswer["answer"]
+
+
+def randomExamples(numberOfExamples=15):
+    # create random indexes in the range between 0 and len(qaDataset)
+    randomIndexes = np.random.randint(0, len(qaDataset), numberOfExamples)
+    examples = []
+    for index in randomIndexes:
+        question = qaDataset[index]["question"]
+        examples.append([question])
+    return examples
+
+
+gr.Interface(
+    fn=predict,
+    inputs=["text", "text"],
+    outputs="text",
+    examples=randomExamples(),
+).launch()