Spaces:
Sleeping
Sleeping
File size: 3,445 Bytes
9aba82f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 |
from datetime import datetime, timedelta
import json
import time
from typing import List, Dict
from bs4 import BeautifulSoup
import requests
def get_hf_daily_papers_info(date: str) -> List[Dict]:
url = 'https://huggingface.co/papers?date=' + date
response = requests.get(url)
if response.status_code == 200:
soup = BeautifulSoup(response.text, 'html.parser')
papers = []
articles = soup.find_all('article', class_='relative flex flex-col overflow-hidden rounded-xl border')
for article in articles:
title_tag = article.find('h3')
title = title_tag.get_text(strip=True)
link = title_tag.find('a')['href']
authors = [author_tag['title'] for author_tag in article.find_all('li', title=True)]
submitter_tag = article.find('div', class_='pointer-events-none absolute right-2 top-56 -mt-8 flex h-6 items-center gap-1 self-end whitespace-nowrap rounded-md border bg-white px-2 text-xs leading-none text-gray-700 shadow-sm dark:bg-gray-900 dark:text-gray-400 sm:text-sm md:top-64')
submitter = submitter_tag.get_text(strip=True).replace('Submitted by', '').strip()
paper_info = {
'title': title,
'link': "https://huggingface.co" + link,
'authors': authors,
'submitter': submitter,
'date': date
}
papers.append(paper_info)
return papers
else:
print(f"No papers, date: {date}")
return []
def get_abstract(url: str) -> str:
response = requests.get(url)
if response.status_code != 200:
raise Exception(f"Failed to load page {url}")
soup = BeautifulSoup(response.text, 'html.parser')
abstract_section = soup.find('div', class_='pb-8 pr-4 md:pr-16')
if not abstract_section:
raise Exception("Abstract section not found")
abstract_header = abstract_section.find('h2', string='Abstract')
if not abstract_header:
raise Exception("Abstract header not found")
abstract_paragraph = abstract_header.find_next('p')
if not abstract_paragraph:
raise Exception("Abstract paragraph not found")
return abstract_paragraph.get_text(strip=True).replace('\n', ' ')
def fetch_and_save_papers(start_date: str, end_date: str, output_file: str):
start_date = datetime.strptime(start_date, "%Y-%m-%d")
end_date = datetime.strptime(end_date, "%Y-%m-%d")
date_list = [(start_date + timedelta(days=x)).strftime("%Y-%m-%d") for x in range((end_date - start_date).days + 1)]
all_papers = []
for date in date_list:
papers = get_hf_daily_papers_info(date)
for paper in papers:
try:
paper['abstract'] = get_abstract(paper['link'])
except Exception as e:
print(f"Failed to get abstract for {paper['link']}: {e}")
all_papers.append(paper)
time.sleep(1.0) # sleep for 1 second to safe access to the website
with open(output_file, 'w', encoding='utf-8') as json_file:
json.dump(all_papers, json_file, ensure_ascii=False, indent=4)
print(f"Data saved to {output_file}")
if __name__ == '__main__':
# Example usage
args = {
'start_date': '2023-05-04',
'end_date': '2024-06-27',
'output_file': 'hf_daily_papers_2023-05-04_2024-06-27.json'
}
fetch_and_save_papers(**args) |