Upload deployment.py

deployment.py

@@ -0,0 +1,466 @@
+# -*- coding: utf-8 -*-
+"""Deployment.ipynb
+
+Automatically generated by Colaboratory.
+
+Original file is located at
+    https://colab.research.google.com/drive/1RtXMnveLECPLSum0IJcSGtQTk1pGRjNE
+
+# Proof of Concept:
+
+Breakdown:
+
+1. One must first load the dataset that our group created on Mockaroo based on the guidelines given to us by the client. This dataset models a food delivery business that has 4 tables: Driver, Customer, Orders and Customer support. Each table has various types of data spanning from strings, ints to unique ids. Tables are linked by ids as well. 
+
+2. Using the textblob library, we run spell checking on the user input in order to avoid any query generation issues due to misspelt words. 
+
+3. We use spacy in order to run named entity recognition; these entities will be used in step 4.
+
+4. Using the named entities and a list of unique values from the dataset, we use tensorflow embeddings and cosine similarity to find the column value most likely being referenced in the user's query. For instance, an input of San Francisco Jail would have a strong cosine similarity with the actual value from the client's column: San Francisco Penitentiary. After the correct name has been found we use regex to substitute the corrected name in place of the user input. 
+
+5. Finally, we do the actual query translation from plain text. We first input the formatted query and send it to openai that has already been fed the schema for the query. We then receive the SQL query and call our own hand-crafted SQL-to-MongoDB method that converts into a final MongoDB query. 
+
+### User Instructions
+
+For the code to function, you need to load the four datasets (driver_data, cust_data, order_data, cust_service_data) from the github repo into your google drive as outlined in the following cells. 
+
+Our main method first asks the user for their openai key. Then we have some test cases that may contain noun spelling issues, name spelling issues, etc.
+"""
+
+pip install openai
+
+pip install gradio
+
+#imports
+import pandas as pd
+from google.colab import drive
+from textblob import TextBlob
+import spacy
+import tensorflow_hub as hub
+from scipy.spatial import distance
+from numpy.core.fromnumeric import argmax
+import os
+import openai
+import re
+import gradio as gr
+
+"""# Loading Mockaroo dataset from drive"""
+
+drive.mount('/content/drive')
+
+"""### **Attention**: Upload all four datasets into your MyDrive directory in google drive"""
+
+driver = pd.read_csv('/content/drive/MyDrive/driver_data.csv')
+customer = pd.read_csv('/content/drive/MyDrive/customer_data.csv')
+order = pd.read_csv('/content/drive/MyDrive/order_data.csv')
+service = pd.read_csv('/content/drive/MyDrive/cust_service_data.csv')
+
+"""# Spelling correction"""
+
+def correctSpelling(sentence):
+  return str(TextBlob(sentence).correct())
+
+"""# Entity Extraction"""
+
+# extract entities, label, label definition from natural language questions and append to dataframe
+nlp = spacy.load("en_core_web_sm")
+def EntityExtraction(text:str):
+  # print(text)
+  entities = []
+  entities_label = []
+  label_explanation = {}
+  doc = nlp(text)
+  for entity in doc.ents:
+    entities.append(entity.text)
+    entities_label.append(entity.label_)
+    label_explanation[entity.label_] = spacy.explain(entity.label_)
+  return entities, entities_label
+
+"""# Column Cosine Similarity"""
+
+#creating a dictionary of unique values in the dataset
+#Used for cosine similarity
+unique_values = {}
+
+for column in driver:
+    unique_values[column] = driver[column].unique()
+
+for column in customer:
+    unique_values[column] = customer[column].unique()
+
+for column in order:
+    if column in ['cust_id', 'driver_id']:
+        unique_values[column] = order[column].unique()
+
+unique_values['sales_id'] = service['sales_id'].unique()
+
+embed = hub.load("https://tfhub.dev/google/universal-sentence-encoder/4")
+
+# Uses TF word embeddings to find the word/phrase in words[1:] most related
+# to words[0] 
+def ClosestSimilarity(words):
+  embeddings = embed(words)
+
+  similarities = [1 - distance.cosine(embeddings[0],x) for x in embeddings[1:]]
+
+  return max(similarities), argmax(similarities)
+
+def find_column(item, array = unique_values):
+
+    best_similarity = 0
+    best_item = None
+    best_key = None
+
+    for key in array:
+        values = [str(x) for x in unique_values[key]]
+        values = [item] + values
+        max_similarity, item_similar = ClosestSimilarity(values)
+        if not best_similarity or max_similarity > best_similarity:
+
+            best_similarity = max_similarity
+            best_item = unique_values[key][item_similar]
+            best_key = key
+
+    if best_similarity < 0.2:
+
+        return best_key, item
+    return best_key, best_item
+
+"""# Query to SQL to MongoDB"""
+
+def query_to_SQL_to_MongoDB(query, key, organization):
+
+    
+    openai.api_key = key # put in the unique key
+    openai.organization = organization # sets the specific parameters of the openai var
+
+    response = openai.Completion.create( # use the appropriate SQL model and set the parameters accordingly 
+        model="text-davinci-003",
+        prompt="### Postgres SQL tables, with their properties:\n#\n# Customer_Support(sales_id, order_id, date)\n# Driver(driver_id, driver_name, driver_address, driver_experience)\n# Customer(cust_id, cust_name, cust_address)\n# Orders(order_id, cust_id, driver_id, date, amount)\n#\n### A query to " + query + ".\nSELECT",
+        temperature=0,
+        max_tokens=150,
+        top_p=1.0,
+        frequency_penalty=0.0,
+        presence_penalty=0.0,
+        stop=["#", ";"]
+    )
+
+    SQL = response['choices'][0]['text'] # extract the outputted SQL Query
+    return complex_SQL_to_MongoDB(SQL)
+
+#Example: 
+
+'''
+db.Customer.aggregate(
+
+    {
+        
+        $lookup:
+
+        {
+
+            from: "Orders",
+            localField: "cust_id",
+            foreignField: "cust_id",
+            as: "Customer"
+
+        }
+    
+    }, 
+
+    {
+
+        $group:
+
+        {
+
+            _id: "cust_name",
+            count: {$count : {}}
+
+        }
+
+    }, 
+
+    {
+
+        $sort:{count : -1}
+
+    }, 
+
+    {
+
+        $limit: 1
+
+    }
+
+)
+
+
+'''
+
+'''
+db.Customer.aggregate( 
+    {
+        $lookup: 
+        { 
+            from : "Orders", 
+            localField: "cust_id", 
+            foreignField: "cust_id", 
+            as: "Customer"
+            
+        } 
+    },
+    {
+        $lookup: 
+        { 
+            from : "Driver", 
+            localField: "driver_id", 
+            foreignField: "driver_id", 
+            as: "Customer"
+        } 
+    },
+        
+    {
+        $match:
+        {
+
+            $group: 
+            {
+                 _id: "c.cust_name", 
+                 count: {$count: {order_id}
+            } 
+         }
+         
+    }, 
+    { 
+        $sort: {count : -1}
+    }, 
+    { $limit : 1 }
+    
+    )
+'''
+
+keywords = {'INNER', 'FROM', 'WHERE', 'GROUP', 'BY', 'ON', 'SELECT', 'BETWEEN', 'LIMIT', 'AND', 'ORDER'}
+
+mapper = {} # maps SQL symbols to MongoDB functions
+
+mapper['<'] = '$lt'
+mapper['>'] = '$gt'
+mapper['!='] = '$ne'
+
+def complex_SQL_to_MongoDB(query):
+
+    query = re.split(r' |\n', query) # split the query on spaces and turn in to array
+    query = [ x for x in query if len(x) > 0]
+
+    if len(query[0]) > 3 and (query[0][:3] == 'MAX' or query[0][:3] == 'MIN'):
+        
+        query += ['ORDER', 'BY', query[0][4:-1], 'DESC' if query[0][:3] == 'MAX' else 'ASC', 'LIMIT', '1']
+
+    count_str = ''
+
+    if len(query[0]) > 5 and query[0][:5] == 'COUNT': 
+
+        count_str += ' {$count : ' 
+        if query[0][6] == '*':
+
+            count_str += '{} }'
+
+        else:
+
+            count_str += query[0][6:-1] + ' }'
+
+    print(query)
+    fields = ''
+    i = 0 
+    while query[i] != 'FROM':
+
+        fields += ' ' + query[i] + ' : 1,'
+        i += 1
+
+    fields = fields[:-1]
+    i = i +1
+    collection = query[i]
+    i = i + 1
+    if i < len(query) and query[i] not in keywords:
+
+        i += 1
+    answer = 'db.' + collection + ".aggregate( " # MongoDB function for aggregation
+
+    while i < len(query) and query[i] == 'INNER':
+
+        i = i + 2
+        lookup = '{$lookup: { from : "'
+        lookup += query[i] + '", localField: "'
+        if query[i+1] not in keywords:
+            i += 1
+        i = i + 2
+        lookup += query[i].split('.')[1] + '", foreignField: "'
+        i = i+2
+        lookup += query[i].split('.')[1] + '", as: "' + collection + '"} },'
+        i = i + 1
+        answer += lookup
+
+
+    if i < len(query) and query[i] == 'WHERE':
+
+        where = '{$match:'
+        count = 0 
+
+        while i < len(query) and (query[i] == 'WHERE' or query[i] == 'AND'):
+
+            count += 1
+            i = i+1 
+            conditions = ''
+            print(query[i])
+            conditions = '{' + (query[i].split('.')[1] if len(query[i].split('.')) > 1 else query[i] )  + " : "
+            if query[i+1] == '=':
+
+                conditions += query[i+2] 
+                i = i + 3
+
+            elif query[i+1] == 'BETWEEN':
+
+                conditions += '{$gt: ISODate(' + query[i+2] + '), $lt: ISODate(' + query[i+4] + ')}'
+                i+= 5
+                
+            else:
+                
+                conditions += '{ ' + mapper[query[i+1]] + ' : ' + query[i+2] + ' }'
+                i = i+3
+
+            conditions += '},'
+
+        if count > 1:
+
+            where += '{ $and: [' + conditions[:-1] + ']}}'
+
+        else:
+            
+            where += conditions[:-1] + '}, '
+
+        answer += where
+
+
+    if i < len(query) and (query[i] == 'GROUP' or query[i] == 'ORDER'):
+
+        i = i + 2
+        group = '{$group: { _id: "' + query[i] + '"'
+        i += 1
+        i -= 3 if query[i -3 ] == 'ORDER' else 0
+        if query[i] == 'ORDER' and len(query[i+2]) > 5 and query[i+2][0:5] == 'COUNT':
+             
+            group += ', count: {$count: ' + ('{}' if query[i+2].split('(')[1][:-1] == '*' else ('{' + query[i+2].split('(')[1][:-1].split('.')[1] + '}') ) + '} }}, { $sort: {count : ' + ('1' if query[i+3] == 'ASC' else '-1') + '}}, '
+             
+        else:
+
+            group += '} }, { $sort: {' + query[i+2] + ' : ' + ('1' if query[i+3] == 'ASC' else '-1') + '}},'
+
+        i += 4
+
+        answer += group
+
+    if i < len(query) and query[i] == 'LIMIT':
+
+        answer += '{ $limit : ' + query[i+1] + ' }, '
+
+    answer += count_str
+    answer += ')'
+
+    return answer
+
+
+def simple_SQL_to_MongoDB(query): #ignore function as it is replaced by new complex version
+
+    query = query.split(' ') # split the query on spaces and turn in to array
+    query = query[1:] # remove the initial space
+    answer = 'db.collection.find' # MongoDB function for selection
+    fields = ''
+    i = 0 
+    while query[i] != 'FROM':
+
+        fields += ' ' + query[i] + ' : 1,'
+        i += 1
+
+    fields = fields[:-1]
+    while query[i] != 'WHERE':
+
+        i += 1
+    
+    i += 1
+    conditions = ''
+    while i+2 < len(query):
+
+        print(i)
+
+        conditions += ' ' + query[i] + ' : '
+        if query[i+1] == '=':
+
+            conditions += query[i+2] 
+
+        elif query[i+1] == 'BETWEEN':
+
+            conditions += '{$gt: ISODate(' + query[i+2] + '), $lt: ISODate(' + query[i+4] + ')}'
+            i+= 6
+            conditions += ','
+            continue
+
+        else:
+
+            conditions += '{ ' + mapper[query[i+1]] + ' : ' + query[i+2] + ' }'
+
+        conditions += ','
+
+        i+= 4
+
+    
+
+        
+
+    conditions = conditions[:-1]
+
+
+    answer += '({' + conditions + '}, {' + fields + '})'
+
+    return answer
+
+"""# Main method"""
+
+def query_creator(key, organization, plain_query):
+  # find named entities in text, e.g. names, addresses, etc.
+  plain_query = correctSpelling(plain_query)
+  entities, entities_label = EntityExtraction(plain_query)
+  modified_query = plain_query
+  
+  #print(entities)
+  #print(entities_label)
+  #For each named entity in the query
+  for i in range(len(entities)):
+
+    if entities_label[i] in ['ORDINAL', 'CARDINAL']:
+        continue
+    #Use cosine similarity on each entity to find closest matching string from tables.
+    col, best_match = find_column(entities[i])
+    #substitute table string in place of partial match found in previous step
+    modified_query = re.sub(entities[i],best_match,modified_query)
+
+  print("Modified input: ", modified_query)
+  #Convert adjusted plain text query to SQL, then MongoDB
+  MongoDB_query = query_to_SQL_to_MongoDB(modified_query, key, organization)
+  return MongoDB_query
+
+"""#Testers of query creator"""
+
+tests = ["giv me number of orders from the driver elizbeth", "name of driver with maximum ordres", "first two orders with the highest order amount", "address of customer with lowest ordr amount",\
+           "id of customer with most complints", "date of customer support with sales id 21695-828", "number of drivers with order amount 20", "numbser of orders by customer martha", "order amount of most recent customer support",\
+           "amount of the highest order by customber Federica"]
+
+#for test in tests:
+
+   # print(query_creator(api_key, org_key, test)) # put in your api and org keys to use the tester
+
+"""# UI"""
+
+iface = gr.Interface(fn=query_creator, inputs= [gr.Textbox(label = "API Key"), gr.Textbox(label = "Organization Key"),  gr.Textbox(label = "Plain Text Query")], outputs=gr.Textbox(label = "MongoDB Query"), )
+iface.launch(share = True, debug = True)
+