Climate_site/python_scripts/patent_functions_own_details.py

##packages code
from shapely.geometry import Point
import pandas as pd
from tqdm import tqdm
import streamlit as st
import numpy as np
import json, requests 
import pandas as pd
#from pandas.io.json import json_normalize

import matplotlib.pyplot as plt
import seaborn as sns
from math import radians, cos, sin, asin, sqrt

#import spacy
#!python -m spacy download en_core_web_lg


from sentence_transformers import SentenceTransformer, util


path = "Climate_site/python_scripts/"


@st.cache_resource
def model_nlp():
    model = SentenceTransformer('all-MiniLM-L6-v2')
    return model

import unicodedata

from metaphone import doublemetaphone
from fuzzywuzzy import fuzz
from difflib import SequenceMatcher
import re

import geopandas as gpd
from geopandas import GeoDataFrame


@st.cache_data  # 👈 Add the caching decorator
def load_data():
    url = path + "institutions.tsv"
    dic = pd.read_csv(url, delimiter = "\t" , index_col = 0).to_dict('index')
    return dic

dic_institutions = load_data()


#################### General Functions #############################

def URL(base_URL , entity_type , filters):
    url = base_URL + entity_type + filters 
    return url


def get_data(url):
    url = requests.get(url)
    text = url.text
    import json
    data = json.loads(text)
    return data

    
## calculate efficiently the dot product between two vectors

def norm(vector):
    return np.sqrt(sum(x * x for x in vector))    

def cosine_similarity2(vec_a, vec_b):
        norm_a = norm(vec_a)
        norm_b = norm(vec_b)
        dot = sum(a * b for a, b in zip(vec_a, vec_b))
        return dot / (norm_a * norm_b)




#################### Patent Functions #############################


def related_patents(research_words , display):
    

    dic_patents = {}

    max_count = 0
    base_URL_PV = "https://api.patentsview.org/"
    filter_works = "patents/query?"
    filter_PV = "q={%22_and%22:[{%22_text_all%22:{%22patent_abstract%22:%22" 
    filter_PV += research_words

    filter_PV += "%22}}]}&f=[%22patent_number%22,%22patent_title%22,%22assignee_country%22,%22patent_date%22,%22assignee_organization%22,%22inventor_longitude%22,%22inventor_latitude%22,%22inventor_last_name%22,%22inventor_id%22,%22inventor_first_name%22,%22cpc_subsection_title%22,%22assignee_city%22,%22patent_abstract%22,%22patent_kind%22,%22cpc_group_id%22,%22assignee_organization%22,%22citedby_patent_number%22]"

    filter_PV = filter_PV.replace(" " , "%20")

        

    url = URL(base_URL_PV , filter_works, filter_PV) 
    data = get_data(url)


    if display == True:
        print(  data["total_patent_count"] , elem[-1] )
        print(url)



    
    for i in range(data["count"]):
        dic_patents[ "US-" + data["patents"][i]["patent_number"]] = {}
        dic_patents[ "US-" + data["patents"][i]["patent_number"]]["title"] = data["patents"][i]["patent_title"]
        dic_patents["US-" + data["patents"][i]["patent_number"]]["abstract"] = data["patents"][i]["patent_abstract"]

        dic_patents[ "US-" + data["patents"][i]["patent_number"]]["assignee"] = str(data["patents"][i]["assignees"][0]["assignee_organization"])
        dic_patents["US-" + data["patents"][i]["patent_number"]]["assignee_city"] = str(data["patents"][i]["assignees"][0]["assignee_city"])
        dic_patents["US-" + data["patents"][i]["patent_number"]]["assignee_country"] = str(data["patents"][i]["assignees"][0]["assignee_country"])
        for j in range(1, len(data["patents"][i]["assignees"])):
            dic_patents[ "US-" + data["patents"][i]["patent_number"]]["assignee"] +=  ", " + str(data["patents"][i]["assignees"][j]["assignee_organization"])
            dic_patents[ "US-" + data["patents"][i]["patent_number"]]["assignee_city"] +=  ", " + str(data["patents"][i]["assignees"][j]["assignee_city"])
            dic_patents["US-" + data["patents"][i]["patent_number"]]["assignee_country"] +=  ", " + str(data["patents"][i]["assignees"][j]["assignee_country"])

        dic_patents[ "US-" + data["patents"][i]["patent_number"]]["list_inventors"] = data["patents"][i]["inventors"]

        dic_patents[ "US-" + data["patents"][i]["patent_number"]]["inventors"] = str(data["patents"][i]["inventors"][0]["inventor_first_name"]) + " " + str(data["patents"][i]["inventors"][0]["inventor_last_name"])
        for j in range(1, len(data["patents"][i]["inventors"])):
            dic_patents[ "US-" + data["patents"][i]["patent_number"]]["inventors"] +=  ", " + str(data["patents"][i]["inventors"][j]["inventor_first_name"]) + " " + str(data["patents"][i]["inventors"][j]["inventor_last_name"])


        dic_patents["US-" + data["patents"][i]["patent_number"]]["date"] = data["patents"][i]["patent_date"]
        dic_patents["US-" + data["patents"][i]["patent_number"]]["number_citations"] = len(data["patents"][i]["citedby_patents"])


    if display == True:
        print(" ")
            
            
    return dic_patents
  
  
  
  
  
def ranking_patents( research_words, details):
    
    model = model_nlp()
    
    dic_patents = related_patents( research_words , False)
    

    dic_scores = {}


    
    reference_text = details

    encoded_text = model.encode(reference_text, convert_to_tensor=False).tolist()
        
        
    if len(dic_patents ) == 0:
        return "Select other key words"
        
    else:
        for ids in list(dic_patents.keys()):

            dic_scores[ids] = {}

            encoded_title = model.encode(dic_patents[ids]["title"], convert_to_tensor=False).tolist()
            score_title = cosine_similarity2(encoded_title, encoded_text)


            if dic_patents[ids]["abstract"] != None:
                encoded_abstract = model.encode(dic_patents[ids]["abstract"], convert_to_tensor=False).tolist()
                score_abstract = cosine_similarity2(encoded_abstract, encoded_text)
            else:
                score_abstract = None

            dic_scores[ids]["title comparision"] = score_title
            dic_scores[ids]["abstract comparison"] = score_abstract
            dic_scores[ids]["title"] = dic_patents[ids]["title"]
            dic_scores[ids]["citations"] = dic_patents[ids]["number_citations"]
            dic_scores[ids]["date"] = dic_patents[ids]["date"][:4]
            dic_scores[ids]["assignee"] = dic_patents[ids]["assignee"]
            dic_scores[ids]["inventors"] = dic_patents[ids]["inventors"]
            dic_scores[ids]["number of co-inventors"] = len(dic_patents[ids]["inventors"].split(","))
            
    return dic_patents , dic_scores




def get_ranking_own_research(research_key_words,  details  , size):
    
    dic , dic_scores_papers = ranking_patents(research_key_words,  details )
    
    if dic_scores_papers == {}:
        return "No paper found"
    
    elif type(dic_scores_papers) == str:
        return dic_scores_papers
    
    else:
        return pd.DataFrame(dic_scores_papers).T.sort_values(by="abstract comparison" , ascending = False).head(size)
     
 

def extract_quantitative_data_patent(patent_id):
    

    patent_id = patent_id[3:]
        
    url = "https://api.patentsview.org/patents/query?q={%22patent_id%22:%22" + str(patent_id) + "%22}&f=[%22patent_number%22,%22patent_title%22,%22patent_abstract%22,%22patent_date%22,%22inventor_last_name%22,%22inventor_first_name%22,%22assignee_organization%22]"
    url_google = "https://patents.google.com/patent/US" + str(patent_id)
    
    data = get_data(url)["patents"][0]
    title = data["patent_title"]
    abstract = data["patent_abstract"]
    co_inventors = ", ".join([ data["inventors"][i]["inventor_first_name"] + " " + data["inventors"][i]["inventor_last_name"] for i in range(len(data["inventors"])) ]) 
    assignees =  ", ".join([ str(data["assignees"][i]["assignee_organization"]) for i in range(len(data["assignees"])) ] )

    return url_google , title , abstract , data["patent_date"] , co_inventors , assignees




    


## Mapping the authors

#merge the nobiliary particles with the last name
#ln_suff file can be modified if more or less nobiliary particles want to be suppressed

ln_suff = ['oster',
 'nordre',
 'vaster',
 'aust',
 'vesle',
 'da',
 'van t',
 'af',
 'al',
 'setya',
 'zu',
 'la',
 'na',
 'mic',
 'ofver',
 'el',
 'vetle',
 'van het',
 'dos',
 'ui',
 'vest',
 'ab',
 'vste',
 'nord',
 'van der',
 'bin',
 'ibn',
 'war',
 'fitz',
 'alam',
 'di',
 'erch',
 'fetch',
 'nga',
 'ka',
 'soder',
 'lille',
 'upp',
 'ua',
 'te',
 'ni',
 'bint',
 'von und zu',
 'vast',
 'vestre',
 'over',
 'syd',
 'mac',
 'nin',
 'nic',
 'putri',
 'bet',
 'verch',
 'norr',
 'bath',
 'della',
 'van',
 'ben',
 'du',
 'stor',
 'das',
 'neder',
 'abu',
 'degli',
 'vre',
 'ait',
 'ny',
 'opp',
 'pour',
 'kil',
 'der',
 'oz',
 'von',
 'at',
 'nedre',
 'van den',
 'setia',
 'ap',
 'gil',
 'myljom',
 'van de',
 'stre',
 'dele',
 'mck',
 'de',
 'mellom',
 'mhic',
 'binti',
 'ath',
 'binte',
 'snder',
 'sre',
 'ned',
 'ter',
 'bar',
 'le',
 'mala',
 'ost',
 'syndre',
 'sr',
 'bat',
 'sndre',
 'austre',
 'putra',
 'putera',
 'av',
 'lu',
 'vetch',
 'ver',
 'puteri',
 'mc',
 'tre',
 'st']

#suppress all the unwanted suffixes from a string. 
#name_del file can be modified if more or less suffixes want to be suppressed 

name_del = ['2nd', '3rd', 'Jr', 'Jr.', 'Junior', 'Sr', 'Sr.', 'Senior']


def name_delete(string):
    for elmt in name_del:
        if f" {elmt}" in string:
            return string.replace(f" {elmt}","")
    return string

def ln_suff_merge(string):
    for suff in ln_suff:
        if f" {suff} " in string or string.startswith(f"{suff} "):
            return string.replace(f"{suff} ",suff.replace(" ",""))
    return string

#normalize a string dat that represents often a name. 

def normalize(data):
    normal = unicodedata.normalize('NFKD', data).encode('ASCII', 'ignore')
    val = normal.decode("utf-8")
    # delete unwanted elmt
    val = name_delete(val)
    # lower full name in upper
    val = re.sub(r"[A-Z]{3,}", lambda x: x.group().lower(), val)
    # add space in front of upper case
    val = re.sub(r"(\w)([A-Z])", r"\1 \2", val)
    # Lower case
    val = val.lower()
    # remove special characters
    val = re.sub('[^A-Za-z0-9 ]+', ' ', val)
    # remove multiple spaces
    val = re.sub(' +', ' ', val)
    # remove trailing spaces
    val = val.strip()
    # suffix merge
    val = ln_suff_merge(val)

    return val


def main_inventors( research_key_words , details , size):
    
    dic_patents , dic_ranked = ranking_patents(research_key_words , details)
    dic_patents_co_inventors = {}
    

    for patent in list(dic_ranked.keys())[:size]:
        for k in range(len(dic_patents[patent]["list_inventors"])):
            
            inventor_id = dic_patents[patent]["list_inventors"][k]["inventor_id"]
            inventor_name = dic_patents[patent]["list_inventors"][k]["inventor_first_name"] + " " + dic_patents[patent]["list_inventors"][k]["inventor_last_name"]
            inventor_name_norm = normalize(inventor_name).split()
            inventor_name_norm = inventor_name_norm[0] + " " + inventor_name_norm[-1]

            if inventor_name_norm not in dic_patents_co_inventors:
                dic_patents_co_inventors[inventor_name_norm] = {}
                dic_patents_co_inventors[inventor_name_norm]["Inventor's name"] = inventor_name 
                dic_patents_co_inventors[inventor_name_norm]["PatentsView inventor's id"] =  inventor_id 
                dic_patents_co_inventors[inventor_name_norm]["Number of occurence"] = 1
                dic_patents_co_inventors[inventor_name_norm]["Number of related citations"] = dic_patents[patent]["number_citations"]

            else:
                if inventor_id not in dic_patents_co_inventors[inventor_name_norm]["PatentsView inventor's id"] :
                    dic_patents_co_inventors[inventor_name_norm]["PatentsView inventor's id"] += ", " + inventor_id
                if inventor_name not in dic_patents_co_inventors[inventor_name_norm]["Inventor's name"] :
                    dic_patents_co_inventors[inventor_name_norm]["Inventor's name"] += ", " + inventor_name
                dic_patents_co_inventors[inventor_name_norm]["Number of occurence"] += 1
                dic_patents_co_inventors[inventor_name_norm]["Number of related citations"] += dic_patents[patent]["number_citations"]


        dic_patents_co_inventors = {k: v for k, v in sorted(dic_patents_co_inventors.items(), key=lambda item: item[1]["Number of occurence"] , reverse = True)}
    
    if dic_patents_co_inventors == {}:
    
        
        return "No patent, select other key words"
    else:
        
        for inventor_name_norm in list(dic_patents_co_inventors.keys()):
            list_inventors = dic_patents_co_inventors[inventor_name_norm]["PatentsView inventor's id"].split(", ")
            work_count = 0
            cited_by_count = 0
            
            for elem in list_inventors:
                url = "https://api.patentsview.org/inventors/query?q={%22inventor_id%22:[%22" + elem + "%22]}&f=[%22inventor_total_num_patents%22,%22patent_num_cited_by_us_patents%22]"
                data = get_data(url)["inventors"][0]
                work_count += int(data["inventor_total_num_patents"])
                for k in range(len(data["patents"])):
                    cited_by_count += int(data["patents"][k]["patent_num_cited_by_us_patents"])
                
            dic_patents_co_inventors[inventor_name_norm]["Number of patents"] = work_count
            dic_patents_co_inventors[inventor_name_norm]["Number of US patents citations"] = cited_by_count
            
        
                
                
        return pd.DataFrame(dic_patents_co_inventors , index = ["Inventor's name", "PatentsView inventor's id", "Number of occurence"  , "Number of patents" ,"Number of US patents citations" ,  "Number of related citations"]).T.style.hide(axis="index")


    
def map_inventors(research_key_words , details , size):
    display = False
    dic_patents , dic_ranked = ranking_patents(research_key_words , details)
    dic_patents_co_inventors = {}
    count = 0
    
    for patent in list(dic_ranked.keys())[:size]:
        for k in range(len(dic_patents[patent]["list_inventors"])):
            
            dic_patents_co_inventors[count] = {}
            
            dic_patents_co_inventors[count]["latitude"] = dic_patents[patent]["list_inventors"][k]["inventor_latitude"]
            dic_patents_co_inventors[count]["longitude"] = dic_patents[patent]["list_inventors"][k]["inventor_longitude"]
            dic_patents_co_inventors[count]["longitude"] = dic_patents[patent]["list_inventors"][k]["inventor_longitude"]
            dic_patents_co_inventors[count]["inventor_name"]  = str(dic_patents[patent]["list_inventors"][k]["inventor_first_name"]) + " " + str(dic_patents[patent]["list_inventors"][k]["inventor_last_name"])
            dic_patents_co_inventors[count]["patent_date"]  = dic_patents[patent]["date"]
            count += 1
                
    if dic_patents_co_inventors == {}:
        return "No patent, select other key words"
    
    
    map_df = pd.DataFrame(dic_patents_co_inventors).T
    map_df["longitude"]=map_df['longitude'].astype(float)
    map_df['latitude']=map_df['latitude'].astype(float)
    map_df = map_df[map_df["latitude"].notnull()]

    return map_df