|
|
|
|
|
|
|
|
|
|
|
import pandas as pd |
|
|
import streamlit as st |
|
|
import numpy as np |
|
|
import json, requests |
|
|
|
|
|
from tqdm import tqdm |
|
|
|
|
|
path = "Climate_site/python_scripts/" |
|
|
|
|
|
@st.cache_data |
|
|
def load_data(): |
|
|
url = path + "institutions.tsv" |
|
|
dic = pd.read_csv(url, delimiter = "\t" , index_col = 1).to_dict('index') |
|
|
return dic |
|
|
|
|
|
dic_institutions = load_data() |
|
|
|
|
|
|
|
|
dic_country_codes = {'Europe': ['AD', |
|
|
'AL', |
|
|
'AT', |
|
|
'BE', |
|
|
'BG', |
|
|
'BY', |
|
|
'CZ', |
|
|
'DE', |
|
|
'DK', |
|
|
'EE', |
|
|
'FI', |
|
|
'FR', |
|
|
'GR', |
|
|
'HU', |
|
|
'IE', |
|
|
'IS', |
|
|
'IT', |
|
|
'LI', |
|
|
'LT', |
|
|
'LU', |
|
|
'LV', |
|
|
'MK', |
|
|
'MT', |
|
|
'NL', |
|
|
'NO', |
|
|
'PL', |
|
|
'PT', |
|
|
'RO', |
|
|
'RU', |
|
|
'SE', |
|
|
'SI', |
|
|
'SK', |
|
|
'SM', |
|
|
'UA', |
|
|
'VA', |
|
|
'BA', |
|
|
'HR', |
|
|
'MD', |
|
|
'MC', |
|
|
'ME', |
|
|
'RS', |
|
|
'ES', |
|
|
'CH', |
|
|
'GB'], |
|
|
'Asia': ['AF', |
|
|
'AM', |
|
|
'AZ', |
|
|
'BD', |
|
|
'BH', |
|
|
'BN', |
|
|
'BT', |
|
|
'CN', |
|
|
'CY', |
|
|
'GE', |
|
|
'ID', |
|
|
'IL', |
|
|
'IN', |
|
|
'IQ', |
|
|
'IR', |
|
|
'JO', |
|
|
'JP', |
|
|
'KG', |
|
|
'KP', |
|
|
'KR', |
|
|
'KW', |
|
|
'LB', |
|
|
'MM', |
|
|
'MN', |
|
|
'MV', |
|
|
'MY', |
|
|
'NP', |
|
|
'OM', |
|
|
'PH', |
|
|
'PK', |
|
|
'QA', |
|
|
'SA', |
|
|
'SG', |
|
|
'SY', |
|
|
'TH', |
|
|
'TJ', |
|
|
'TM', |
|
|
'TR', |
|
|
'UZ', |
|
|
'VN', |
|
|
'YE', |
|
|
'KH', |
|
|
'TL', |
|
|
'KZ', |
|
|
'LA', |
|
|
'LK', |
|
|
'AE'], |
|
|
'North America': ['AG', |
|
|
'BB', |
|
|
'BS', |
|
|
'BZ', |
|
|
'CA', |
|
|
'CR', |
|
|
'CU', |
|
|
'DM', |
|
|
'DO', |
|
|
'GT', |
|
|
'GT', |
|
|
'HN', |
|
|
'JM', |
|
|
'MX', |
|
|
'NI', |
|
|
'PA', |
|
|
'TT', |
|
|
'US', |
|
|
'SV', |
|
|
'GD', |
|
|
'KN', |
|
|
'LC', |
|
|
'VC'], |
|
|
'Africa': ['AO', |
|
|
'BF', |
|
|
'BI', |
|
|
'BJ', |
|
|
'BW', |
|
|
'CD', |
|
|
'CG', |
|
|
'CI', |
|
|
'CM', |
|
|
'CV', |
|
|
'DJ', |
|
|
'EG', |
|
|
'ER', |
|
|
'ET', |
|
|
'GA', |
|
|
'GH', |
|
|
'GM', |
|
|
'GN', |
|
|
'GW', |
|
|
'KE', |
|
|
'LR', |
|
|
'LS', |
|
|
'LY', |
|
|
'MG', |
|
|
'ML', |
|
|
'MR', |
|
|
'MU', |
|
|
'MW', |
|
|
'MZ', |
|
|
'NA', |
|
|
'NE', |
|
|
'NG', |
|
|
'RW', |
|
|
'SC', |
|
|
'SD', |
|
|
'SL', |
|
|
'SN', |
|
|
'SO', |
|
|
'ST', |
|
|
'TG', |
|
|
'TN', |
|
|
'TZ', |
|
|
'UG', |
|
|
'ZM', |
|
|
'ZW', |
|
|
'DZ', |
|
|
'CF', |
|
|
'TD', |
|
|
'KM', |
|
|
'GQ', |
|
|
'MA', |
|
|
'ZA', |
|
|
'SZ'], |
|
|
'South America': ['AR', |
|
|
'BO', |
|
|
'BR', |
|
|
'CL', |
|
|
'CO', |
|
|
'EC', |
|
|
'GY', |
|
|
'PE', |
|
|
'PY', |
|
|
'SR', |
|
|
'UY', |
|
|
'VE'], |
|
|
'Oceania': ['AU', |
|
|
'FJ', |
|
|
'KI', |
|
|
'MH', |
|
|
'NR', |
|
|
'NZ', |
|
|
'PG', |
|
|
'PW', |
|
|
'SB', |
|
|
'TO', |
|
|
'TV', |
|
|
'VU', |
|
|
'FM', |
|
|
'WS']} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def URL(base_URL , entity_type , filters): |
|
|
url = base_URL + entity_type + filters |
|
|
return url |
|
|
|
|
|
|
|
|
def get_data(url): |
|
|
url = requests.get(url) |
|
|
text = url.text |
|
|
import json |
|
|
data = json.loads(text) |
|
|
return data |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def author_id_from_name(name): |
|
|
|
|
|
dic_names = {} |
|
|
|
|
|
name = name.title() |
|
|
|
|
|
url = "https://api.openalex.org/authors?search=" + name + "&per_page=200&mailto=emma_scharfmann@berkeley.edu" |
|
|
|
|
|
data = get_data(url)["results"] |
|
|
|
|
|
for k in range(len(data)): |
|
|
author_id = data[k]["id"][21:] |
|
|
dic_names[author_id] = {} |
|
|
dic_names[author_id]["author_name"] = data[k]["display_name"] |
|
|
|
|
|
dic_names[author_id]["number_of_works"] = data[k]["works_count"] |
|
|
dic_names[author_id]["number_of_citations"] = data[k]["cited_by_count"] |
|
|
|
|
|
dic_names[author_id]["field_of_study"] = ", ".join([ data[k]["x_concepts"][j]["display_name"] for j in range(min(5,len(data[k]["x_concepts"])))]) |
|
|
|
|
|
dic_names[author_id]["last_known_institution"] = None |
|
|
dic_names[author_id]["country_code"] = None |
|
|
|
|
|
if data[k]["last_known_institutions"] != None and len(data[k]["last_known_institutions"]) > 0: |
|
|
dic_names[author_id]["last_known_institution"] = data[k]["last_known_institutions"][0]["display_name"] |
|
|
dic_names[author_id]["country_code"] = data[k]["last_known_institutions"][0]["country_code"] |
|
|
|
|
|
dic_names[author_id]["orcid"] = data[k]["orcid"] |
|
|
|
|
|
|
|
|
return pd.DataFrame(dic_names).T , [ ( k , ", ".join([ elem["author_name"] , str(elem["last_known_institution"]) , str(elem["field_of_study"].split(", ")[0]) , "Number of works: " + str(elem["number_of_works"]) ] ) ) for k , elem in dic_names.items()] |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def from_author_id(main_author_ids, year, country_code , size): |
|
|
|
|
|
dic_main_workers = {} |
|
|
|
|
|
|
|
|
for main_author_id in main_author_ids: |
|
|
|
|
|
url = "https://api.openalex.org/works?filter=author.id:" + main_author_id + ",publication_year:>" + str(year) + "&per_page=200&mailto=emma_scharfmann@berkeley.edu" |
|
|
|
|
|
try: |
|
|
data = get_data(url)["results"] |
|
|
|
|
|
for k in tqdm(range(len(data))): |
|
|
work_id = data[k]["id"][21:] |
|
|
for j in range(len(data[k]["authorships"])): |
|
|
author_id = data[k]["authorships"][j]["author"]["id"][21:] |
|
|
|
|
|
if author_id != main_author_id: |
|
|
if author_id not in dic_main_workers: |
|
|
dic_main_workers[author_id] = {} |
|
|
dic_main_workers[author_id]["author_name"] = data[k]["authorships"][j]["author"]["display_name"] |
|
|
dic_main_workers[author_id]["co_authors"] = 0 |
|
|
dic_main_workers[author_id]["citations"] = 0 |
|
|
dic_main_workers[author_id]["institution"] = None |
|
|
dic_main_workers[author_id]["country_code"] = None |
|
|
dic_main_workers[author_id]["id"] = None |
|
|
dic_main_workers[author_id]["longitude"] = None |
|
|
dic_main_workers[author_id]["latitude"] = None |
|
|
|
|
|
if dic_main_workers[author_id]["institution"] == None and data[k]["authorships"][j]["institutions"] != [] and "display_name" in data[k]["authorships"][j]["institutions"][0]: |
|
|
dic_main_workers[author_id]["institution"] = data[k]["authorships"][j]["institutions"][0]["display_name"] |
|
|
if dic_main_workers[author_id]["country_code"] == None and data[k]["authorships"][j]["institutions"] != [] and "country_code" in data[k]["authorships"][j]["institutions"][0]: |
|
|
dic_main_workers[author_id]["country_code"] = data[k]["authorships"][j]["institutions"][0]["country_code"] |
|
|
if dic_main_workers[author_id]["id"] == None and data[k]["authorships"][j]["institutions"] != [] and "id" in data[k]["authorships"][j]["institutions"][0] and data[k]["authorships"][j]["institutions"][0]["id"] != None: |
|
|
dic_main_workers[author_id]["id"] = data[k]["authorships"][j]["institutions"][0]["id"][21:] |
|
|
|
|
|
|
|
|
|
|
|
dic_main_workers[author_id]["co_authors"] += 1 |
|
|
|
|
|
url = "https://api.openalex.org/works?filter=referenced_works:" + work_id + ",publication_year:>" + str(year) + "&per_page=200&mailto=emma_scharfmann@berkeley.edu" |
|
|
|
|
|
|
|
|
try: |
|
|
citations_data = get_data(url)["results"] |
|
|
for i in range(len(citations_data)): |
|
|
citing_work_id = citations_data[i]["id"][21:] |
|
|
for j in range(len(citations_data[i]["authorships"])): |
|
|
citing_author_id = citations_data[i]["authorships"][j]["author"]["id"][21:] |
|
|
|
|
|
if citing_author_id != main_author_id: |
|
|
if citing_author_id not in dic_main_workers: |
|
|
dic_main_workers[citing_author_id] = {} |
|
|
dic_main_workers[citing_author_id]["author_name"] = citations_data[i]["authorships"][j]["author"]["display_name"] |
|
|
dic_main_workers[citing_author_id]["co_authors"] = 0 |
|
|
dic_main_workers[citing_author_id]["citations"] = 0 |
|
|
dic_main_workers[citing_author_id]["institution"] = None |
|
|
dic_main_workers[citing_author_id]["country_code"] = None |
|
|
dic_main_workers[citing_author_id]["id"] = None |
|
|
dic_main_workers[citing_author_id]["longitude"] = None |
|
|
dic_main_workers[citing_author_id]["latitude"] = None |
|
|
|
|
|
if dic_main_workers[citing_author_id]["institution"] == None and (citations_data[i]["authorships"][j]["institutions"] != [] and "display_name" in citations_data[i]["authorships"][j]["institutions"][0]): |
|
|
dic_main_workers[citing_author_id]["institution"] = citations_data[i]["authorships"][j]["institutions"][0]["display_name"] |
|
|
if dic_main_workers[citing_author_id]["country_code"] == None and (citations_data[i]["authorships"][j]["institutions"] != [] and "country_code" in citations_data[i]["authorships"][j]["institutions"][0] ): |
|
|
dic_main_workers[citing_author_id]["country_code"] = citations_data[i]["authorships"][j]["institutions"][0]["country_code"] |
|
|
if dic_main_workers[citing_author_id]["id"] == None and ( citations_data[i]["authorships"][j]["institutions"] != [] and "id" in citations_data[i]["authorships"][j]["institutions"][0] and citations_data[i]["authorships"][j]["institutions"][0]["id"] != None): |
|
|
dic_main_workers[citing_author_id]["id"] = citations_data[i]["authorships"][j]["institutions"][0]["id"][21:] |
|
|
|
|
|
dic_main_workers[citing_author_id]["citations"] += 1 |
|
|
|
|
|
except: |
|
|
pass |
|
|
except: |
|
|
pass |
|
|
|
|
|
for author_id in dic_main_workers: |
|
|
if dic_main_workers[author_id]["id"] != None: |
|
|
institution_id = dic_main_workers[author_id]["id"] |
|
|
if institution_id in dic_institutions: |
|
|
geo_data = dic_institutions[institution_id] |
|
|
dic_main_workers[author_id]["longitude"] = geo_data["longitude"] |
|
|
dic_main_workers[author_id]["latitude"] = geo_data["latitude"] |
|
|
|
|
|
|
|
|
res = pd.DataFrame(dic_main_workers).T.sort_values("citations" , ascending = False)[['author_name', 'co_authors', 'citations', 'institution', 'country_code']] |
|
|
|
|
|
res_geo = pd.DataFrame(dic_main_workers).T.sort_values("citations" , ascending = False)[['author_name', 'co_authors', 'citations', 'institution', 'country_code','longitude', 'latitude']] |
|
|
res_geo = res_geo[res_geo["longitude"].notnull()] |
|
|
if country_code != False: |
|
|
|
|
|
if country_code in [ "Europe" , "North America" , "Asia" , "South America", "Oceania" , "Africa" ]: |
|
|
|
|
|
return res[res["country_code"].isin(dic_country_codes[country_code])].head(size) , res_geo[res_geo["country_code"].isin(dic_country_codes[country_code])].head(size) |
|
|
|
|
|
else: |
|
|
return res[res["country_code"] == country_code].head(size) , res_geo[res_geo["country_code"] == country_code].head(size) |
|
|
else: |
|
|
return res.head(size), res_geo.head(size) |
|
|
|
|
|
|
|
|
|
