Add remaining app files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/nlp_papers_field_diversity.csv filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,10 +1,12 @@
 ---
-title: Field Time Diversity Docker
-emoji: 🐨
-colorFrom: gray
-colorTo: blue
-sdk: docker
-pinned: false
+title: Field Diversity
+emoji: 🌈
+colorFrom: pink
+colorTo: indigo
+sdk: gradio
+sdk_version: 3.29.0
+app_file: app.py
+pinned: true
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

aclanthology.py

@@ -0,0 +1,203 @@
+# Copyright 2023 by Jan Philip Wahle, https://jpwahle.com/
+# All rights reserved.
+
+import asyncio
+import json
+
+import aiohttp
+import requests
+from bs4 import BeautifulSoup
+
+
+async def fetch(session, url):
+    """Asynchronous function to fetch a URL using aiohttp."""
+    async with session.get(url) as response:
+        return await response.text()
+
+
+async def async_match_acl_id_to_s2_paper(acl_id):
+    """
+    Fetches the paper information from the Semantic Scholar API for the given ACL ID.
+
+    Args:
+        acl_id (str): The ACL ID of the paper to fetch.
+
+    Returns:
+        dict: A dictionary containing the paper information.
+    """
+    url = f"https://api.semanticscholar.org/graph/v1/paper/ACL:{acl_id}"
+    async with aiohttp.ClientSession() as session:
+        res_text = await fetch(session, url)
+    return json.loads(res_text)
+
+
+def extract_paper_info(paper_url):
+    """
+    Extracts information about a paper from its ACL Anthology URL.
+
+    Args:
+        paper_url (str): The URL of the paper on the ACL Anthology website.
+
+    Returns:
+        dict: A dictionary containing the title, authors, and ACL Anthology ID of the paper.
+    """
+    html_doc = requests.get(paper_url, timeout=10).text
+    soup = BeautifulSoup(html_doc, "html.parser")
+
+    title = soup.find("h2", id="title").text.strip()
+    authors = [
+        a.text
+        for a in soup.find_all("a")
+        if a.parent.name == "p" and a.parent["class"] == ["lead"]
+    ]
+    acl_id = paper_url.split("/")[-2]
+
+    return {"title": title, "authors": authors, "acl_id": acl_id}
+
+
+def extract_author_info(author_url):
+    """
+    Extracts author information from the given author URL.
+
+    Args:
+        author_url (str): The URL of the author's page on ACL Anthology.
+
+    Returns:
+        dict: A dictionary containing the author's name and a list of their papers.
+              Each paper is represented as a dictionary with keys "title" and "url".
+    """
+    html_doc = requests.get(author_url, timeout=10).text
+    soup = BeautifulSoup(html_doc, "html.parser")
+
+    author_name = soup.find("h2", id="title").text.strip()
+    paper_elements = soup.find_all("p")
+    papers = []
+    for paper in paper_elements:
+        links = paper.find_all("a")
+        # Filter out a with text pdf and bib
+        links = [
+            l for l in links if l.text.strip() not in ["pdf", "bib", "abs"]
+        ]
+        if not links:
+            continue
+        title = links[0].text.strip()
+        url = "https://aclanthology.org" + links[0]["href"]
+        papers.append({"title": title, "url": url})
+
+    return {"author": author_name, "papers": papers}
+
+
+def extract_venue_info(venue_url):
+    """
+    Extracts venue information from the given URL.
+
+    Args:
+        venue_url (str): The URL of the venue to extract information from.
+
+    Returns:
+        dict: A dictionary containing the venue name and a list of papers with their titles and URLs.
+    """
+    html_doc = requests.get(venue_url, timeout=10).text
+    soup = BeautifulSoup(html_doc, "html.parser")
+
+    venue_name = soup.find("h2", id="title").text.strip()
+    paper_elements = soup.find_all("p")
+    papers = []
+    for paper in paper_elements:
+        links = paper.find_all("a")
+        # Filter out a with text pdf and bib
+        links = [
+            l for l in links if l.text.strip() not in ["pdf", "bib", "abs"]
+        ]
+        if not links:
+            continue
+        title = links[0].text.strip()
+        url = "https://aclanthology.org" + links[0]["href"]
+        papers.append({"title": title, "url": url})
+
+    return {"venue": venue_name, "papers": papers}
+
+
+def determine_page_type(url):
+    """
+    Determine the type of ACL Anthology page given its URL.
+
+    Args:
+        url (str): The URL to be checked.
+
+    Returns:
+        str: "paper", "author", or "venue". Returns None if the type can't be determined.
+    """
+    # Extract last segments from the URL
+    segments = [segment for segment in url.split("/") if segment]
+
+    # Check if the URL points to an event (venue)
+    if "events" in url or "volumes" in url:
+        return "venue"
+
+    # If URL ends in a pattern like "2023.acl-long.1" it's a paper
+    if len(segments) > 1 and segments[-2].isnumeric() and "." in segments[-1]:
+        return "paper"
+
+    if "people" in url:
+        return "author"
+
+        # If none of the above rules apply, fetch the page and check its content
+    try:
+        html_doc = requests.get(url, timeout=10).text
+        soup = BeautifulSoup(html_doc, "html.parser")
+
+        # Check for unique elements specific to each page type
+        if soup.find("h2", id="title"):
+            return (
+                "author"
+                if soup.find("a", href=True, text="Google Scholar")
+                else "paper"
+            )
+        elif soup.find("h1", text="Anthology Volume"):
+            return "venue"
+    except Exception as e:
+        print(f"Error determining page type: {e}")
+
+    return None
+
+
+if __name__ == "__main__":
+    loop = asyncio.get_event_loop()
+
+    urls = [
+        "https://aclanthology.org/2023.acl-long.1/",
+        "https://aclanthology.org/people/a/anna-rogers/",
+        "https://aclanthology.org/events/acl-2022/",
+    ]
+
+    for url in urls:
+        if determine_page_type(url) == "paper":
+            print(f"Paper: {url}")
+            res = extract_paper_info(url)
+            paper = loop.run_until_complete(
+                async_match_acl_id_to_s2_paper(res["acl_id"])
+            )
+            print(paper)
+
+        elif determine_page_type(url) == "author":
+            print(f"Author: {url}")
+            res = extract_author_info(url)
+            tasks = [
+                async_match_acl_id_to_s2_paper(paper["url"].split("/")[-2])
+                for paper in res["papers"]
+            ]
+            s2_ids = loop.run_until_complete(asyncio.gather(*tasks))
+            for paper, s2_id in zip(res["papers"], s2_ids):
+                print(paper["paperId"])
+
+        elif determine_page_type(url) == "venue":
+            print(f"Venue: {url}")
+            res = extract_venue_info(url)
+            tasks = [
+                async_match_acl_id_to_s2_paper(paper["url"].split("/")[-2])
+                for paper in res["papers"]
+            ]
+            s2_ids = loop.run_until_complete(asyncio.gather(*tasks))
+            for paper, s2_id in zip(res["papers"], s2_ids):
+                print(paper["paperId"])

data/nlp_papers_field_diversity.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d9eae4b2ad51e66fa4601af3b05ce130f42b3febe8982d1e3dcb0eef53afa15
+size 20053659

main.py

@@ -0,0 +1,275 @@
+# Copyright 2023 by Jan Philip Wahle, https://jpwahle.com/
+# All rights reserved.
+# Thanks to Mukund Rungta for inspiration on early versions of this demo https://huggingface.co/spaces/mrungta8/CitationalAmnesia
+
+
+import asyncio
+
+import gradio as gr
+
+from aclanthology import determine_page_type
+from plots import generate_cfdi_plot, generate_maoc_plot
+from s2 import (
+    check_s2_id_type,
+    compute_stats_for_acl_author,
+    compute_stats_for_acl_paper,
+    compute_stats_for_acl_venue,
+    compute_stats_for_s2_author,
+    compute_stats_for_s2_paper,
+)
+
+
+def return_clear():
+    """Clearing all demo inputs
+
+    Returns:
+        None
+    """
+    return None, None, None, None, None, None, None, None, None, None, None
+
+
+def create_compute_stats(submit_type=None):
+    def compute_stats(s2_id=None, pdf_file=None, acl_link=None):
+        if submit_type == "s2_id" and s2_id:
+            # Check if s2_id is a paper id or an author id
+            id_type, author_name = check_s2_id_type(s2_id)
+            if id_type == "paper":
+                results = compute_stats_for_s2_paper(s2_id)
+                return plot_and_return_stats(*results)
+            if id_type == "author":
+                results = compute_stats_for_s2_author(s2_id, author_name)
+                return plot_and_return_stats(*results)
+        if submit_type == "acl_link" and acl_link:
+            # Crawl all papers for the author or venue or just the paper if it is a paper link
+            url_type = determine_page_type(acl_link)
+            if url_type == "paper":
+                results = compute_stats_for_acl_paper(acl_link)
+                return plot_and_return_stats(*results)
+            if url_type == "author":
+                results = compute_stats_for_acl_author(acl_link)
+                return plot_and_return_stats(*results)
+            if url_type == "venue":
+                results = compute_stats_for_acl_venue(acl_link)
+                return plot_and_return_stats(*results)
+        # if submit_type == "pdf_file" and pdf_file:
+        #     # Compute the citation field diversity index and citation age diversity index
+        #     pass
+        return None, None, None, None, None, None, None, None
+
+    return compute_stats
+
+
+def plot_and_return_stats(
+    title_authors,
+    num_references,
+    field_counts,
+    year_title_dict,
+    cfdi,
+    cadi,
+    maoc,
+):
+    """
+    Plots the data and returns statistics.
+
+    Args:
+        title_authors (str): The title and authors of the paper.
+        num_references (int): The number of references in the paper.
+        field_counts (dict): A dictionary containing the count of each field.
+        year_title_dict (dict): A dictionary containing the year and title of each paper.
+        cfdi (list): A list of tuples containing the citation field and the number of papers in that field.
+        cadi (list): A list of tuples containing the citation author and the number of papers by that author.
+        maoc (list): A list of tuples containing the main author and the number of papers by that author.
+
+    Returns:
+        tuple: A tuple containing the title and authors of the paper, the number of references, the top 3 most cited fields,
+        the most common oldest papers, the cfdi, cadi, and the plots for cfdi and maoc.
+    """
+    # Generate cfdi plot
+    plot_cfdi = generate_cfdi_plot(cfdi)
+
+    # Generate cadi plot
+    plot_maoc = generate_maoc_plot(maoc)
+
+    # Get top 3 most cited fields
+    top_fields_text = "\n".join(
+        [
+            f"{field}: {count}"
+            for field, count in sorted(
+                field_counts.items(), reverse=True, key=lambda x: x[1]
+            )[:3]
+        ]
+    )
+
+    # Get most common oldest papers
+    oldest_paper_text = "".join(
+        f"[{str(year)}] {title}" + "\n"
+        for year, title in sorted(year_title_dict.items())[:3]
+    )
+
+    return (
+        title_authors,
+        num_references,
+        top_fields_text,
+        oldest_paper_text,
+        cfdi,
+        cadi,
+        plot_cfdi,
+        plot_maoc,
+    )
+
+
+with gr.Blocks(
+    theme=gr.themes.Soft(), css=".gradio-container {background-color: white}"
+) as demo:
+    with gr.Row():
+        gr.Markdown(
+            """
+            # Citation Age and Field Diversity Calculator
+
+            <div align="center">
+                <img src="https://onedrive.live.com/embed?resid=684CB5200DB6B388%21673358&authkey=%21AOrhvc6YI9lKFSc&width=1582&height=385" />
+            </div>
+
+            Welcome to this interactive demo to analyze various aspects of your citational diversity. This tool will enable you to reflect on two critical aspects:
+
+            - By whom am I influenced? Which fields heavily inform and shape the research trajectory of my works?
+            - How far back in time do I cite? What are critical works (present and past) that shape my research?
+
+            In addition, you will be able to analyze how the above compares to the average paper or author. The results you will receive can not be categorized into “good” or “bad”. Instead, they are meant to raise self-awareness about one’s citational diversity and reflect on it. The results might bring you to further questions, such as:
+
+            - Am I reading widely across fields and time?
+            - Should I expand my literature search to include works from other fields?
+            - Are there ideas rooted in the past that can be used in an innovative way?
+
+            Using citations as a tangible marker of influence, our demo provides empirical insights into the influence of papers across fields and time.
+
+            ## What is Citation Field Diversity?
+
+            Field diversity is a measure of the variety of research Fields that a paper or an author draws upon. A high field diversity indicates that the work draws from various distinct research fields, demonstrating a multidisciplinary influence on that work or author.
+
+            ## What is Citation Age Diversity?
+
+            Citation age is a measure of how far back in time a paper cites other papers. A high citation age shows that the work draws from past works, while a low citation age indicates that mostly recent work has influenced that paper.
+
+            """
+        )
+        gr.Markdown(
+            """
+            ## What are the Citation Field Diversity Index (CFDI) and Citation Age Diversity Index (CADI) and how are they calculated?
+
+            The calculation of Field Diversity involves extracting all the references of a paper, categorizing them into distinct study fields, and determining the proportion of each study field over all the references. The Citation Field Diversity Index (CFDI) is then computed by applying the Gini Index on these proportions.
+            Calculating CADI is similar to CFDI but instead of determining the proportion of each study field, we determine the proportion of citation ages. If we take a paper from 2020 that cites two papers, one from 2010 and one from 1990, the citation ages are 10 and 30, respectively. The CADI is then computed by applying the Gini Index on these ages.
+            For more details, please refer to Eq. 3 in [this paper](https://aclanthology.org/2023.acl-long.341/) and Eq. 4 in [this paper](https://arxiv.org/).
+            
+            ## How do I Interpret CFDI and CADI?
+
+            For both indices, higher values indicate a greater diversity of a NLP paper (in terms of how far back it cites and in the fields it cites). On the other hand, lower values signify a lower diversity, indicating that citations are more concentrated in specific fields and time ranges. 
+            
+            ## How can I use this demo?
+
+            There are three ways how you to compute the field and age diversity for papers:
+            1. **Semantic Scholar ID**: Enter the Semantic Scholar ID of a **paper** or **author** and click the *"Compute"* button.
+            2. **ACL Anthology Link**: Paste the ACL Anthology link of a **paper**, **venue**, or **author** and click the *"Compute"* button.
+            3. **PDF File**: Upload your **paper** PDF and click the *"Compute"* button.
+            
+            To retrieve the **Semantic Scholar ID** for a paper such as "The Elephant in the Room: Analyzing the Presence of Big Tech in Natural Language Processing Research," search the paper on Semantic Scholar [here](https://www.semanticscholar.org/paper/The-Elephant-in-the-Room%3A-Analyzing-the-Presence-of-Abdalla-Wahle/587ffdfd7229e8e0dbc5250b44df5fad6251f6ad) and use the last part of the URL. The Semantic Scholar ID (SSID) for this paper is: **587ffdfd7229e8e0dbc5250b44df5fad6251f6ad**.        
+
+            To get an ACL Anthology link, you can go to any ACL Anthology paper, author or proceedings page and just copy and paste the url. For example:
+            - https://aclanthology.org/2023.acl-long.1/
+            - https://aclanthology.org/people/a/anna-rogers/
+            - https://aclanthology.org/events/acl-2002/
+            """
+        )
+
+    with gr.Row():
+        with gr.Tabs():
+            with gr.TabItem("Semantic Scholar ID"):
+                s2_id = gr.Textbox(
+                    label="Semantic Scholar ID",
+                    placeholder=(
+                        "Enter the Semantic Scholar ID here and press enter..."
+                    ),
+                    # value="587ffdfd7229e8e0dbc5250b44df5fad6251f6ad",
+                )
+                with gr.Row():
+                    s2_submit_btn = gr.Button("Compute")
+            with gr.TabItem("ACL Anthology Link"):
+                acl_link = gr.Textbox(
+                    label="ACL Anthology Link",
+                    placeholder="Paste the ACL Anthology link here...",
+                )
+                with gr.Row():
+                    acl_submit_btn = gr.Button("Compute")
+            with gr.TabItem("PDF File"):
+                pdf_file = gr.File(
+                    file_types=[".pdf"], label="Upload your paper PDF"
+                )
+                with gr.Row():
+                    file_submit_btn = gr.Button("Compute")
+    with gr.Row():
+        title = gr.Textbox(
+            label="Title / Author Name / Venue Name:", lines=2
+        )  # Can be either paper title, author name, or proceedings title
+    with gr.Row():
+        num_ref = gr.Textbox(label="Number of references", lines=3)
+        top_field_list = gr.Textbox(label="Top 3 fields cited:", lines=3)
+        top_age_list = gr.Textbox(label="Top 3 oldest papers cited:", lines=3)
+    with gr.Row():
+        cfdi = gr.Textbox(label="CFDI")
+        cadi = gr.Textbox(label="CADI")
+    with gr.Row():
+        cfdi_plot = gr.Plot(label="Citation Field Diversity")
+        cadi_plot = gr.Plot(label="Citation Age Diversity")
+    with gr.Row():
+        clear_btn = gr.Button("Clear")
+
+    submit_args = dict(
+        inputs=[s2_id, pdf_file, acl_link],
+        outputs=[
+            title,
+            num_ref,
+            top_field_list,
+            top_age_list,
+            cfdi,
+            cadi,
+            cfdi_plot,
+            cadi_plot,
+        ],
+    )
+
+    s2_submit_args = submit_args.copy()
+    s2_submit_args["fn"] = create_compute_stats(submit_type="s2_id")
+
+    acl_submit_args = submit_args.copy()
+    acl_submit_args["fn"] = create_compute_stats(submit_type="acl_link")
+
+    file_submit_args = submit_args.copy()
+    file_submit_args["fn"] = create_compute_stats(submit_type="pdf_file")
+
+    s2_id.submit(**s2_submit_args)
+    acl_link.submit(**acl_submit_args)
+
+    acl_submit_btn.click(**acl_submit_args)
+    s2_submit_btn.click(**s2_submit_args)
+    file_submit_btn.click(**file_submit_args)
+
+    clear_btn.click(
+        fn=return_clear,
+        inputs=[],
+        outputs=[
+            title,
+            num_ref,
+            top_field_list,
+            top_age_list,
+            cfdi,
+            cadi,
+            cfdi_plot,
+            cadi_plot,
+            s2_id,
+            acl_link,
+            pdf_file,
+        ],
+    )
+
+demo.queue(concurrency_count=3)
+demo.launch(server_port=7860)

metrics.py

@@ -0,0 +1,27 @@
+# Copyright 2023 by Jan Philip Wahle, https://jpwahle.com/
+# All rights reserved.
+
+import numpy as np
+
+
+def calculate_gini_simpson(dictionary):
+    """
+    Function to Calculate Gini Simpson's Diversity Index
+    """
+    total = sum(dictionary.values())
+    sum_squares = sum((n / total) ** 2 for n in dictionary.values())
+    return 1 - sum_squares
+
+
+def calculate_gini(frequencies):
+    """
+    Function to Calculate Gini's Diversity Index
+    """
+    frequencies = np.array(frequencies)
+    if len(frequencies) == 0 or np.mean(frequencies) == 0:
+        return None
+    total = sum(
+        np.sum(np.abs(xi - frequencies[i:]))
+        for i, xi in enumerate(frequencies[:-1], 1)
+    )
+    return total / (len(frequencies) ** 2 * np.mean(frequencies))

plots.py

@@ -0,0 +1,122 @@
+# Copyright 2023 by Jan Philip Wahle, https://jpwahle.com/
+# All rights reserved.
+
+import os
+
+import numpy as np
+import pandas as pd
+import seaborn as sns
+from matplotlib import pyplot as plt
+from scipy.stats import gaussian_kde
+
+dirname = os.path.dirname(__file__)
+
+# Load the csv file into a pandas DataFrame
+papers_df = pd.read_csv(
+    os.path.join(dirname, "data/nlp_papers_field_diversity.csv")
+)
+
+# Compute the mean CFDI
+mean_cfdi = papers_df["incoming_diversity"].mean()
+
+# Compute the mean CADI
+mean_citation_ages = []
+
+# Open the file and read the content in a list
+with open(
+    os.path.join(dirname, "data/nlp_papers_citation_age.txt"),
+    "r",
+    encoding="utf-8",
+) as filehandle:
+    for line in filehandle:
+        temp = float(line[:-1])
+        mean_citation_ages.append(temp)
+
+
+def generate_cfdi_plot(input_cfdi):
+    """
+    Function to generate a plot for CFDI
+    """
+    # Using kdeplot to fill the distribution curve
+    sns.set(font_scale=1.3, style="whitegrid")
+
+    data = papers_df[papers_df["incoming_diversity"] > 0]["incoming_diversity"]
+    kde = gaussian_kde(data)
+    x_vals = np.linspace(data.min(), data.max(), 1000)
+    y_vals = kde.evaluate(x_vals)
+
+    fig, ax = plt.subplots()  # create a new figure and axis
+
+    ax.fill_between(x_vals, y_vals, color="skyblue", alpha=0.3)
+    ax.plot(x_vals, y_vals, color="skyblue", linewidth=2, label="Distribution")
+
+    interpolated_y_cfdi = np.interp(input_cfdi, x_vals, y_vals)
+    ax.scatter(
+        input_cfdi,
+        interpolated_y_cfdi,
+        c="r",
+        marker="*",
+        linewidths=1,
+        zorder=2,
+    )
+    ax.vlines(
+        input_cfdi, 0, interpolated_y_cfdi, color="tomato", ls="--", lw=1.5
+    )
+    epsilon = 0.005
+    # ax.text(
+    #     input_cfdi + epsilon,
+    #     interpolated_y_cfdi + epsilon,
+    #     "Your paper",
+    #     {"color": "#DC143C", "fontsize": 13},
+    #     ha="left",  # Horizontal alignment
+    # )
+
+    ax.set_xlabel("Citation Field Diversity Index (CFDI)", fontsize=15)
+    ax.set_ylabel("Density", fontsize=15)
+    sns.despine(left=True, bottom=True, right=True, top=True)
+
+    return fig
+
+
+def generate_maoc_plot(input_maoc):
+    """
+    Function to generate a plot for CFDI
+    """
+    # Using kdeplot to fill the distribution curve
+    sns.set(font_scale=1.3, style="whitegrid")
+
+    data = pd.DataFrame(mean_citation_ages)[0]
+    kde = gaussian_kde(data)
+    x_vals = np.linspace(data.min(), data.max(), 1000)
+    y_vals = kde.evaluate(x_vals)
+
+    fig, ax = plt.subplots()  # create a new figure and axis
+    ax.fill_between(x_vals, y_vals, color="skyblue", alpha=0.3)
+    ax.plot(x_vals, y_vals, color="skyblue", linewidth=2, label="Distribution")
+
+    interpolated_y_cfdi = np.interp(input_maoc, x_vals, y_vals)
+    ax.scatter(
+        input_maoc,
+        interpolated_y_cfdi,
+        c="r",
+        marker="*",
+        linewidths=1,
+        zorder=2,
+    )
+    ax.vlines(
+        input_maoc, 0, interpolated_y_cfdi, color="tomato", ls="--", lw=1.5
+    )
+    epsilon = 0.005
+    # ax.text(
+    #     input_maoc + epsilon,
+    #     interpolated_y_cfdi + epsilon,
+    #     "Your paper",
+    #     {"color": "#DC143C", "fontsize": 13},
+    #     ha="left",  # Horizontal alignment
+    # )
+
+    ax.set_xlabel("Mean Age of Citation (mAoC)", fontsize=15)
+    ax.set_ylabel("Density", fontsize=15)
+    sns.despine(left=True, bottom=True, right=True, top=True)
+
+    return fig

requirements.txt

@@ -0,0 +1,10 @@
+gradio==3.44.3
+numpy==1.25.2
+matplotlib==3.8.0
+requests==2.31.0
+futures
+seaborn==0.12.2
+scipy==1.11.2
+beautifulsoup4==4.12.2
+aiohttp==3.8.5
+asyncio

s2.py

@@ -0,0 +1,354 @@
+# Copyright 2023 by Jan Philip Wahle, https://jpwahle.com/
+# All rights reserved.
+
+import asyncio
+import os
+from collections import Counter
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+import requests
+
+from aclanthology import (
+    async_match_acl_id_to_s2_paper,
+    extract_author_info,
+    extract_paper_info,
+    extract_venue_info,
+)
+from metrics import calculate_gini, calculate_gini_simpson
+
+
+def get_or_create_eventloop():
+    try:
+        return asyncio.get_event_loop()
+    except RuntimeError as ex:
+        if "There is no current event loop in thread" in str(ex):
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+            return asyncio.get_event_loop()
+
+
+def send_s2_request(request_url):
+    """
+    Sends a GET request to the specified URL with the S2 API key in the headers.
+
+    Args:
+        request_url (str): The URL to send the request to.
+
+    Returns:
+        requests.Response: The response object returned by the request.
+    """
+    return requests.get(
+        request_url,
+        headers={"x-api-key": os.environ["s2apikey"]},
+        timeout=10,
+    )
+
+
+def check_s2_id_type(semantic_scholar_id):
+    """
+    Check whether a given Semantic Scholar ID is valid for a paper or an author.
+
+    Args:
+        semantic_scholar_id (str): The Semantic Scholar ID to check.
+
+    Returns:
+        tuple: A tuple containing the type of the ID ("paper" or "author") and
+        the name of the author (if the ID is valid for an author), or "invalid"
+        if the ID is not valid for either a paper or an author.
+    """
+    # Define the base URL for Semantic Scholar API
+    base_url = "https://api.semanticscholar.org/v1/"
+
+    # First, check if it's a paper ID
+    paper_response = requests.get(
+        f"{base_url}paper/{semantic_scholar_id}", timeout=5
+    )
+
+    # If the response status code is 200, it means the ID is valid for a paper
+    if paper_response.status_code == 200:
+        return "paper", None
+
+    # Next, check if it's an author ID
+    author_response = requests.get(
+        f"{base_url}author/{semantic_scholar_id}", timeout=5
+    )
+
+    # If the response status code is 200, it means the ID is valid for an author
+    return (
+        "author",
+        author_response.json()["name"]
+        if author_response.status_code == 200
+        else "invalid",
+    )
+
+
+def get_papers_from_author(ssid_author_id):
+    """Retrieves all papers for a given author
+
+    Args:
+        ssid_author_id (str): semantic scholar id
+
+    Returns:
+        list: a list of all papers for the given author
+    """
+    # Create request URL for an author
+    request_url = f"https://api.semanticscholar.org/graph/v1/author/{ssid_author_id}?fields=papers"
+
+    r = send_s2_request(request_url)
+    if r.status_code == 200:
+        papers = r.json().get("papers", [])
+        return [paper["paperId"] for paper in papers]
+    return []
+
+
+def compute_stats_for_s2_paper(ssid_paper_id):
+    """
+    Computes statistics for a given paper ID using the Semantic Scholar API.
+
+    Args:
+        ssid_paper_id (str): The Semantic Scholar ID of the paper to compute statistics for.
+
+    Returns:
+        Tuple containing the following statistics:
+        - title_authors (str): The title and authors of the paper.
+        - num_references (int): The number of references in the paper.
+        - fields_of_study_counts (dict): A dictionary containing the count of each field of study in the paper's references.
+        - year_to_title_dict (dict): A dictionary mapping the year of each reference to its title.
+        - cfdi (float): The CFDI (Cumulative Field Diversity Index) of the paper's references.
+        - cadi (float): The CADI (Citation Age Diversity Index) of the paper's references.
+        - output_maoc (float): The MAOC (Mean Age of Citation) of the paper's references.
+    """
+    # Get the paper and its references
+    request_url = f"https://api.semanticscholar.org/graph/v1/paper/{ssid_paper_id}?fields=references,title,year,authors"
+    r = send_s2_request(request_url)
+    if r.status_code == 200:  # if successful request
+        result = r.json()
+        if not result.get("references") or result.get("references") == []:
+            return None, None, None, None, None, None, None, None
+        s2_ref_paper_keys = [
+            reference_paper_tuple["paperId"]
+            for reference_paper_tuple in r.json()["references"]
+        ]
+        filtered_s2_ref_paper_keys = [
+            s2_ref_paper_key
+            for s2_ref_paper_key in s2_ref_paper_keys
+            if s2_ref_paper_key is not None
+        ]
+        title, year, authors = (
+            result["title"],
+            result["year"],
+            result["authors"],
+        )
+        title_authors = (
+            title + "\n" + ", ".join([author["name"] for author in authors])
+        )
+
+        # Go over the references of the paper
+        reference_year_list = []
+        reference_title_list = []
+        reference_fos_list = []
+        with ThreadPoolExecutor() as executor:
+            request_url_refs = [
+                f"https://api.semanticscholar.org/graph/v1/paper/{ref_paper_key}?fields=title,year,s2FieldsOfStudy"
+                for ref_paper_key in filtered_s2_ref_paper_keys
+            ]
+            futures = [
+                executor.submit(send_s2_request, request_url_ref)
+                for request_url_ref in request_url_refs
+            ]
+            for future in as_completed(futures):
+                r_ref = future.result()
+                if r_ref.status_code == 200:
+                    result_ref = r_ref.json()
+                    (title_ref, year_ref, fields_ref) = (
+                        result_ref["title"],
+                        result_ref["year"],
+                        result_ref["s2FieldsOfStudy"],
+                    )
+                    reference_year_list.append(year_ref)
+                    reference_title_list.append(title_ref)
+                    reference_fos_list.extend(
+                        field["category"]
+                        for field in fields_ref
+                        if field["source"] == "s2-fos-model"
+                    )
+                else:
+                    print(
+                        f"Error retrieving reference {r_ref.status_code} for"
+                        f" paper {ssid_paper_id}"
+                    )
+
+        # Remove all None from reference_year_list and reference_title_list
+        reference_year_list = [
+            year_ref
+            for year_ref in reference_year_list
+            if year_ref is not None
+        ]
+        reference_title_list = [
+            title_ref
+            for title_ref in reference_title_list
+            if title_ref is not None
+        ]
+
+        # Count references
+        num_references = len(reference_year_list)
+
+        # Flatten list and count occurrences
+        fields_of_study_counts = dict(
+            Counter(
+                [
+                    field
+                    for field in reference_fos_list
+                    if "Computer Science" not in field
+                ]
+            )
+        )
+
+        # Citation age list
+        aoc_list = [
+            year - year_ref
+            for year_ref in reference_year_list
+            if year_ref and year
+        ]
+        if not aoc_list:
+            return None, None, None, None, None, None, None, None
+
+        # Compute citation age
+        output_maoc = sum(aoc_list) / len(aoc_list)
+        cadi = calculate_gini(aoc_list)
+
+        # Create a dictionary of year to title
+        year_to_title_dict = dict(
+            zip(reference_year_list, reference_title_list)
+        )
+
+        # Compute CFDI
+        cfdi = calculate_gini_simpson(fields_of_study_counts)
+
+        # Return the results
+        return (
+            title_authors,
+            num_references,
+            fields_of_study_counts,
+            year_to_title_dict,
+            cfdi,
+            cadi,
+            output_maoc,
+        )
+
+
+def compute_stats_for_s2_author(ssid_author_id, author_name):
+    """
+    Computes statistics for an author based on their papers in the Semantic Scholar database.
+
+    Args:
+        ssid_author_id (str): The Semantic Scholar author ID.
+        author_name (str): The name of the author.
+
+    Returns:
+        dict: A dictionary containing statistics for the author, or None if no papers were found.
+    """
+    if papers := get_papers_from_author(ssid_author_id):
+        return compute_stats_for_multiple_s2_papers(papers, author_name)
+    return None
+
+
+def compute_stats_for_acl_paper(url):
+    """
+    Computes statistics for a paper based on its ACL Anthology URL.
+
+    Args:
+        url (str): The URL of the paper on the ACL Anthology website.
+
+    Returns:
+        dict: A dictionary containing statistics for the paper, or None if the paper was not found.
+    """
+    if paper_info := extract_paper_info(url):
+        loop = get_or_create_eventloop()
+        # Match paper ID to Semantic Scholar ID
+        s2_paper = loop.run_until_complete(
+            async_match_acl_id_to_s2_paper(paper_info["acl_id"])
+        )
+        return compute_stats_for_s2_paper(s2_paper["paperId"])
+    return None
+
+
+import asyncio
+
+
+def compute_stats_for_acl_author(url):
+    """
+    Computes statistics for an author's papers in the ACL anthology.
+
+    Args:
+        url (str): The URL of the author's page on the ACL anthology website.
+
+    Returns:
+        dict: A dictionary containing statistics for the author's papers, including
+        the number of papers, the number of citations, and the h-index.
+        Returns None if the author's page cannot be accessed or no papers are found.
+    """
+    if paper_info := extract_author_info(url):
+        loop = get_or_create_eventloop()
+        tasks = [
+            async_match_acl_id_to_s2_paper(paper["url"].split("/")[-2])
+            for paper in paper_info["papers"]
+        ]
+        papers = loop.run_until_complete(asyncio.gather(*tasks))
+        return compute_stats_for_multiple_s2_papers(
+            [paper["paperId"] for paper in papers if "paperId" in paper],
+            paper_info["author"],
+        )
+    return None
+
+
+def compute_stats_for_acl_venue(url):
+    if paper_info := extract_venue_info(url):
+        loop = get_or_create_eventloop()
+        tasks = [
+            async_match_acl_id_to_s2_paper(paper["url"].split("/")[-2])
+            for paper in paper_info["papers"]
+        ]
+        papers = loop.run_until_complete(asyncio.gather(*tasks))
+        return compute_stats_for_multiple_s2_papers(
+            [paper["paperId"] for paper in papers if "paperId" in paper],
+            paper_info["venue"],
+        )
+    return None
+
+
+def compute_stats_for_multiple_s2_papers(papers, title):
+    num_references = 0
+    top_fields = {}
+    oldest_paper_dict = {}
+    cfdi = 0
+    cadi = 0
+    output_maoc = 0
+
+    def process_paper(paper):
+        return compute_stats_for_s2_paper(paper)
+
+    with ThreadPoolExecutor() as executor:
+        results_list = list(executor.map(process_paper, papers))
+
+    for results in results_list:
+        if not results or results[0] is None:
+            continue
+        num_references += results[1]
+        for field, count in results[2].items():
+            top_fields[field] = top_fields.get(field, 0) + count
+        for year, title in results[3].items():
+            oldest_paper_dict[year] = title
+        cfdi += results[4]
+        cadi += results[5]
+        output_maoc += results[6]
+
+    return (
+        title,
+        num_references,
+        top_fields,
+        oldest_paper_dict,
+        cfdi / len(papers),
+        cadi / len(papers),
+        output_maoc / len(papers),
+    )