deployment.py

# -*- coding: utf-8 -*-
"""Deployment.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1RtXMnveLECPLSum0IJcSGtQTk1pGRjNE

# Proof of Concept:

Breakdown:

1. One must first load the dataset that our group created on Mockaroo based on the guidelines given to us by the client. This dataset models a food delivery business that has 4 tables: Driver, Customer, Orders and Customer support. Each table has various types of data spanning from strings, ints to unique ids. Tables are linked by ids as well. 

2. Using the textblob library, we run spell checking on the user input in order to avoid any query generation issues due to misspelt words. 

3. We use spacy in order to run named entity recognition; these entities will be used in step 4.

4. Using the named entities and a list of unique values from the dataset, we use tensorflow embeddings and cosine similarity to find the column value most likely being referenced in the user's query. For instance, an input of San Francisco Jail would have a strong cosine similarity with the actual value from the client's column: San Francisco Penitentiary. After the correct name has been found we use regex to substitute the corrected name in place of the user input. 

5. Finally, we do the actual query translation from plain text. We first input the formatted query and send it to openai that has already been fed the schema for the query. We then receive the SQL query and call our own hand-crafted SQL-to-MongoDB method that converts into a final MongoDB query. 

### User Instructions

For the code to function, you need to load the four datasets (driver_data, cust_data, order_data, cust_service_data) from the github repo into your google drive as outlined in the following cells. 

Our main method first asks the user for their openai key. Then we have some test cases that may contain noun spelling issues, name spelling issues, etc.
"""

pip install openai

pip install gradio

#imports
import pandas as pd
from google.colab import drive
from textblob import TextBlob
import spacy
import tensorflow_hub as hub
from scipy.spatial import distance
from numpy.core.fromnumeric import argmax
import os
import openai
import re
import gradio as gr

"""# Loading Mockaroo dataset from drive"""

drive.mount('/content/drive')

"""### **Attention**: Upload all four datasets into your MyDrive directory in google drive"""

driver = pd.read_csv('/content/drive/MyDrive/driver_data.csv')
customer = pd.read_csv('/content/drive/MyDrive/customer_data.csv')
order = pd.read_csv('/content/drive/MyDrive/order_data.csv')
service = pd.read_csv('/content/drive/MyDrive/cust_service_data.csv')

"""# Spelling correction"""

def correctSpelling(sentence):
  return str(TextBlob(sentence).correct())

"""# Entity Extraction"""

# extract entities, label, label definition from natural language questions and append to dataframe
nlp = spacy.load("en_core_web_sm")
def EntityExtraction(text:str):
  # print(text)
  entities = []
  entities_label = []
  label_explanation = {}
  doc = nlp(text)
  for entity in doc.ents:
    entities.append(entity.text)
    entities_label.append(entity.label_)
    label_explanation[entity.label_] = spacy.explain(entity.label_)
  return entities, entities_label

"""# Column Cosine Similarity"""

#creating a dictionary of unique values in the dataset
#Used for cosine similarity
unique_values = {}

for column in driver:
    unique_values[column] = driver[column].unique()

for column in customer:
    unique_values[column] = customer[column].unique()

for column in order:
    if column in ['cust_id', 'driver_id']:
        unique_values[column] = order[column].unique()

unique_values['sales_id'] = service['sales_id'].unique()

embed = hub.load("https://tfhub.dev/google/universal-sentence-encoder/4")

# Uses TF word embeddings to find the word/phrase in words[1:] most related
# to words[0] 
def ClosestSimilarity(words):
  embeddings = embed(words)

  similarities = [1 - distance.cosine(embeddings[0],x) for x in embeddings[1:]]

  return max(similarities), argmax(similarities)

def find_column(item, array = unique_values):

    best_similarity = 0
    best_item = None
    best_key = None

    for key in array:
        values = [str(x) for x in unique_values[key]]
        values = [item] + values
        max_similarity, item_similar = ClosestSimilarity(values)
        if not best_similarity or max_similarity > best_similarity:

            best_similarity = max_similarity
            best_item = unique_values[key][item_similar]
            best_key = key

    if best_similarity < 0.2:

        return best_key, item
    return best_key, best_item

"""# Query to SQL to MongoDB"""

def query_to_SQL_to_MongoDB(query, key, organization):

    
    openai.api_key = key # put in the unique key
    openai.organization = organization # sets the specific parameters of the openai var

    response = openai.Completion.create( # use the appropriate SQL model and set the parameters accordingly 
        model="text-davinci-003",
        prompt="### Postgres SQL tables, with their properties:\n#\n# Customer_Support(sales_id, order_id, date)\n# Driver(driver_id, driver_name, driver_address, driver_experience)\n# Customer(cust_id, cust_name, cust_address)\n# Orders(order_id, cust_id, driver_id, date, amount)\n#\n### A query to " + query + ".\nSELECT",
        temperature=0,
        max_tokens=150,
        top_p=1.0,
        frequency_penalty=0.0,
        presence_penalty=0.0,
        stop=["#", ";"]
    )

    SQL = response['choices'][0]['text'] # extract the outputted SQL Query
    return complex_SQL_to_MongoDB(SQL)

#Example: 

'''
db.Customer.aggregate(

    {
        
        $lookup:

        {

            from: "Orders",
            localField: "cust_id",
            foreignField: "cust_id",
            as: "Customer"

        }
    
    }, 

    {

        $group:

        {

            _id: "cust_name",
            count: {$count : {}}

        }

    }, 

    {

        $sort:{count : -1}

    }, 

    {

        $limit: 1

    }

)


'''

'''
db.Customer.aggregate( 
    {
        $lookup: 
        { 
            from : "Orders", 
            localField: "cust_id", 
            foreignField: "cust_id", 
            as: "Customer"
            
        } 
    },
    {
        $lookup: 
        { 
            from : "Driver", 
            localField: "driver_id", 
            foreignField: "driver_id", 
            as: "Customer"
        } 
    },
        
    {
        $match:
        {

            $group: 
            {
                 _id: "c.cust_name", 
                 count: {$count: {order_id}
            } 
         }
         
    }, 
    { 
        $sort: {count : -1}
    }, 
    { $limit : 1 }
    
    )
'''

keywords = {'INNER', 'FROM', 'WHERE', 'GROUP', 'BY', 'ON', 'SELECT', 'BETWEEN', 'LIMIT', 'AND', 'ORDER'}

mapper = {} # maps SQL symbols to MongoDB functions

mapper['<'] = '$lt'
mapper['>'] = '$gt'
mapper['!='] = '$ne'

def complex_SQL_to_MongoDB(query):

    query = re.split(r' |\n', query) # split the query on spaces and turn in to array
    query = [ x for x in query if len(x) > 0]

    if len(query[0]) > 3 and (query[0][:3] == 'MAX' or query[0][:3] == 'MIN'):
        
        query += ['ORDER', 'BY', query[0][4:-1], 'DESC' if query[0][:3] == 'MAX' else 'ASC', 'LIMIT', '1']

    count_str = ''

    if len(query[0]) > 5 and query[0][:5] == 'COUNT': 

        count_str += ' {$count : ' 
        if query[0][6] == '*':

            count_str += '{} }'

        else:

            count_str += query[0][6:-1] + ' }'

    print(query)
    fields = ''
    i = 0 
    while query[i] != 'FROM':

        fields += ' ' + query[i] + ' : 1,'
        i += 1

    fields = fields[:-1]
    i = i +1
    collection = query[i]
    i = i + 1
    if i < len(query) and query[i] not in keywords:

        i += 1
    answer = 'db.' + collection + ".aggregate( " # MongoDB function for aggregation

    while i < len(query) and query[i] == 'INNER':

        i = i + 2
        lookup = '{$lookup: { from : "'
        lookup += query[i] + '", localField: "'
        if query[i+1] not in keywords:
            i += 1
        i = i + 2
        lookup += query[i].split('.')[1] + '", foreignField: "'
        i = i+2
        lookup += query[i].split('.')[1] + '", as: "' + collection + '"} },'
        i = i + 1
        answer += lookup


    if i < len(query) and query[i] == 'WHERE':

        where = '{$match:'
        count = 0 

        while i < len(query) and (query[i] == 'WHERE' or query[i] == 'AND'):

            count += 1
            i = i+1 
            conditions = ''
            print(query[i])
            conditions = '{' + (query[i].split('.')[1] if len(query[i].split('.')) > 1 else query[i] )  + " : "
            if query[i+1] == '=':

                conditions += query[i+2] 
                i = i + 3

            elif query[i+1] == 'BETWEEN':

                conditions += '{$gt: ISODate(' + query[i+2] + '), $lt: ISODate(' + query[i+4] + ')}'
                i+= 5
                
            else:
                
                conditions += '{ ' + mapper[query[i+1]] + ' : ' + query[i+2] + ' }'
                i = i+3

            conditions += '},'

        if count > 1:

            where += '{ $and: [' + conditions[:-1] + ']}}'

        else:
            
            where += conditions[:-1] + '}, '

        answer += where


    if i < len(query) and (query[i] == 'GROUP' or query[i] == 'ORDER'):

        i = i + 2
        group = '{$group: { _id: "' + query[i] + '"'
        i += 1
        i -= 3 if query[i -3 ] == 'ORDER' else 0
        if query[i] == 'ORDER' and len(query[i+2]) > 5 and query[i+2][0:5] == 'COUNT':
             
            group += ', count: {$count: ' + ('{}' if query[i+2].split('(')[1][:-1] == '*' else ('{' + query[i+2].split('(')[1][:-1].split('.')[1] + '}') ) + '} }}, { $sort: {count : ' + ('1' if query[i+3] == 'ASC' else '-1') + '}}, '
             
        else:

            group += '} }, { $sort: {' + query[i+2] + ' : ' + ('1' if query[i+3] == 'ASC' else '-1') + '}},'

        i += 4

        answer += group

    if i < len(query) and query[i] == 'LIMIT':

        answer += '{ $limit : ' + query[i+1] + ' }, '

    answer += count_str
    answer += ')'

    return answer


def simple_SQL_to_MongoDB(query): #ignore function as it is replaced by new complex version

    query = query.split(' ') # split the query on spaces and turn in to array
    query = query[1:] # remove the initial space
    answer = 'db.collection.find' # MongoDB function for selection
    fields = ''
    i = 0 
    while query[i] != 'FROM':

        fields += ' ' + query[i] + ' : 1,'
        i += 1

    fields = fields[:-1]
    while query[i] != 'WHERE':

        i += 1
    
    i += 1
    conditions = ''
    while i+2 < len(query):

        print(i)

        conditions += ' ' + query[i] + ' : '
        if query[i+1] == '=':

            conditions += query[i+2] 

        elif query[i+1] == 'BETWEEN':

            conditions += '{$gt: ISODate(' + query[i+2] + '), $lt: ISODate(' + query[i+4] + ')}'
            i+= 6
            conditions += ','
            continue

        else:

            conditions += '{ ' + mapper[query[i+1]] + ' : ' + query[i+2] + ' }'

        conditions += ','

        i+= 4

    

        

    conditions = conditions[:-1]


    answer += '({' + conditions + '}, {' + fields + '})'

    return answer

"""# Main method"""

def query_creator(key, organization, plain_query):
  # find named entities in text, e.g. names, addresses, etc.
  plain_query = correctSpelling(plain_query)
  entities, entities_label = EntityExtraction(plain_query)
  modified_query = plain_query
  
  #print(entities)
  #print(entities_label)
  #For each named entity in the query
  for i in range(len(entities)):

    if entities_label[i] in ['ORDINAL', 'CARDINAL']:
        continue
    #Use cosine similarity on each entity to find closest matching string from tables.
    col, best_match = find_column(entities[i])
    #substitute table string in place of partial match found in previous step
    modified_query = re.sub(entities[i],best_match,modified_query)

  print("Modified input: ", modified_query)
  #Convert adjusted plain text query to SQL, then MongoDB
  MongoDB_query = query_to_SQL_to_MongoDB(modified_query, key, organization)
  return MongoDB_query

"""#Testers of query creator"""

tests = ["giv me number of orders from the driver elizbeth", "name of driver with maximum ordres", "first two orders with the highest order amount", "address of customer with lowest ordr amount",\
           "id of customer with most complints", "date of customer support with sales id 21695-828", "number of drivers with order amount 20", "numbser of orders by customer martha", "order amount of most recent customer support",\
           "amount of the highest order by customber Federica"]

#for test in tests:

   # print(query_creator(api_key, org_key, test)) # put in your api and org keys to use the tester

"""# UI"""

iface = gr.Interface(fn=query_creator, inputs= [gr.Textbox(label = "API Key"), gr.Textbox(label = "Organization Key"),  gr.Textbox(label = "Plain Text Query")], outputs=gr.Textbox(label = "MongoDB Query"), )
iface.launch(share = True, debug = True)