Spaces:
Running
Running
| # Copyright 2023 by Jan Philip Wahle, https://jpwahle.com/ | |
| # All rights reserved. | |
| import asyncio | |
| import json | |
| import aiohttp | |
| import requests | |
| from bs4 import BeautifulSoup | |
| async def fetch(session, url): | |
| """Asynchronous function to fetch a URL using aiohttp.""" | |
| async with session.get(url) as response: | |
| return await response.text() | |
| async def async_match_acl_id_to_s2_paper(acl_id): | |
| """ | |
| Fetches the paper information from the Semantic Scholar API for the given ACL ID. | |
| Args: | |
| acl_id (str): The ACL ID of the paper to fetch. | |
| Returns: | |
| dict: A dictionary containing the paper information. | |
| """ | |
| url = f"https://api.semanticscholar.org/graph/v1/paper/ACL:{acl_id}" | |
| async with aiohttp.ClientSession() as session: | |
| res_text = await fetch(session, url) | |
| return json.loads(res_text) | |
| def extract_paper_info(paper_url): | |
| """ | |
| Extracts information about a paper from its ACL Anthology URL. | |
| Args: | |
| paper_url (str): The URL of the paper on the ACL Anthology website. | |
| Returns: | |
| dict: A dictionary containing the title, authors, and ACL Anthology ID of the paper. | |
| """ | |
| html_doc = requests.get(paper_url, timeout=10).text | |
| soup = BeautifulSoup(html_doc, "html.parser") | |
| title = soup.find("h2", id="title").text.strip() | |
| authors = [ | |
| a.text | |
| for a in soup.find_all("a") | |
| if a.parent.name == "p" and a.parent["class"] == ["lead"] | |
| ] | |
| acl_id = paper_url.split("/")[-2] | |
| return {"title": title, "authors": authors, "acl_id": acl_id} | |
| def extract_author_info(author_url): | |
| """ | |
| Extracts author information from the given author URL. | |
| Args: | |
| author_url (str): The URL of the author's page on ACL Anthology. | |
| Returns: | |
| dict: A dictionary containing the author's name and a list of their papers. | |
| Each paper is represented as a dictionary with keys "title" and "url". | |
| """ | |
| html_doc = requests.get(author_url, timeout=10).text | |
| soup = BeautifulSoup(html_doc, "html.parser") | |
| author_name = soup.find("h2", id="title").text.strip() | |
| paper_elements = soup.find_all("p") | |
| papers = [] | |
| for paper in paper_elements: | |
| links = paper.find_all("a") | |
| # Filter out a with text pdf and bib | |
| links = [ | |
| l for l in links if l.text.strip() not in ["pdf", "bib", "abs"] | |
| ] | |
| if not links: | |
| continue | |
| title = links[0].text.strip() | |
| url = "https://aclanthology.org" + links[0]["href"] | |
| papers.append({"title": title, "url": url}) | |
| return {"author": author_name, "papers": papers} | |
| def extract_venue_info(venue_url): | |
| """ | |
| Extracts venue information from the given URL. | |
| Args: | |
| venue_url (str): The URL of the venue to extract information from. | |
| Returns: | |
| dict: A dictionary containing the venue name and a list of papers with their titles and URLs. | |
| """ | |
| html_doc = requests.get(venue_url, timeout=10).text | |
| soup = BeautifulSoup(html_doc, "html.parser") | |
| venue_name = soup.find("h2", id="title").text.strip() | |
| paper_elements = soup.find_all("p") | |
| papers = [] | |
| for paper in paper_elements: | |
| links = paper.find_all("a") | |
| # Filter out a with text pdf and bib | |
| links = [ | |
| l for l in links if l.text.strip() not in ["pdf", "bib", "abs"] | |
| ] | |
| if not links: | |
| continue | |
| title = links[0].text.strip() | |
| url = "https://aclanthology.org" + links[0]["href"] | |
| papers.append({"title": title, "url": url}) | |
| return {"venue": venue_name, "papers": papers} | |
| def determine_page_type(url): | |
| """ | |
| Determine the type of ACL Anthology page given its URL. | |
| Args: | |
| url (str): The URL to be checked. | |
| Returns: | |
| str: "paper", "author", or "venue". Returns None if the type can't be determined. | |
| """ | |
| # Extract last segments from the URL | |
| segments = [segment for segment in url.split("/") if segment] | |
| # Check if the URL points to an event (venue) | |
| if "events" in url or "volumes" in url: | |
| return "venue" | |
| # If URL ends in a pattern like "2023.acl-long.1" it's a paper | |
| if len(segments) > 1 and segments[-2].isnumeric() and "." in segments[-1]: | |
| return "paper" | |
| if "people" in url: | |
| return "author" | |
| # If none of the above rules apply, fetch the page and check its content | |
| try: | |
| html_doc = requests.get(url, timeout=10).text | |
| soup = BeautifulSoup(html_doc, "html.parser") | |
| # Check for unique elements specific to each page type | |
| if soup.find("h2", id="title"): | |
| return ( | |
| "author" | |
| if soup.find("a", href=True, text="Google Scholar") | |
| else "paper" | |
| ) | |
| elif soup.find("h1", text="Anthology Volume"): | |
| return "venue" | |
| except Exception as e: | |
| print(f"Error determining page type: {e}") | |
| return None | |
| if __name__ == "__main__": | |
| loop = asyncio.get_event_loop() | |
| urls = [ | |
| "https://aclanthology.org/2023.acl-long.1/", | |
| "https://aclanthology.org/people/a/anna-rogers/", | |
| "https://aclanthology.org/events/acl-2022/", | |
| ] | |
| for url in urls: | |
| if determine_page_type(url) == "paper": | |
| print(f"Paper: {url}") | |
| res = extract_paper_info(url) | |
| paper = loop.run_until_complete( | |
| async_match_acl_id_to_s2_paper(res["acl_id"]) | |
| ) | |
| print(paper) | |
| elif determine_page_type(url) == "author": | |
| print(f"Author: {url}") | |
| res = extract_author_info(url) | |
| tasks = [ | |
| async_match_acl_id_to_s2_paper(paper["url"].split("/")[-2]) | |
| for paper in res["papers"] | |
| ] | |
| s2_ids = loop.run_until_complete(asyncio.gather(*tasks)) | |
| for paper, s2_id in zip(res["papers"], s2_ids): | |
| print(paper["paperId"]) | |
| elif determine_page_type(url) == "venue": | |
| print(f"Venue: {url}") | |
| res = extract_venue_info(url) | |
| tasks = [ | |
| async_match_acl_id_to_s2_paper(paper["url"].split("/")[-2]) | |
| for paper in res["papers"] | |
| ] | |
| s2_ids = loop.run_until_complete(asyncio.gather(*tasks)) | |
| for paper, s2_id in zip(res["papers"], s2_ids): | |
| print(paper["paperId"]) | |