import requests
from datetime import datetime,timedelta
from bs4 import BeautifulSoup
import pandas as pd

class NatureBrief:
    def __init__(self) -> None:
        
        self.start_date = datetime.now() # datetime.strptime('2023-06-06', '%Y-%m-%d')
        
        pass
    
    def fetch(self, url) -> str:
        headers = {
            'accept':
            'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
            'user-agent': 'application/json, text/javascript, */*; q=0.01',
        }
        r = requests.get(url, headers=headers)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    
    def get_today_articles(self, count: int = 3):
        # 通过日期计算获取的文章数
        current_date = datetime.now()

        for i in range(300):
            elapsed_time = current_date - timedelta(days=i)
            
        article_count = elapsed_time
        # elapsed_time = current_date - self.start_date
        # article_count = (elapsed_time.days + 1) * count
            
        # 获取对应数量的文章
        total_page = self.get_total_page()
        article_list = []
        
        page_i = 1
        while page_i < total_page and len(article_list) < article_count:
            if page_i + 1 > total_page:
                print('「NATURE BRIEF」历史文章已全部获取。无法获取更多...')
                break
            
            article_list += self.get_page(page_i)
            page_i += 1
            if len(article_list) >= article_count:
                break
            
        print(f'「NATURE BRIEF」共计获取 {len(article_list)} 篇文章')
        
        # 抽取对应的文章
        article_list = article_list[:article_count]
        # 倒着获取 3 篇
        article_list.reverse()
        
        articles = []
        count = 1
        for article in article_list:
            if count > count:
                break
            print(f'「NATURE BRIEF」正在获取第 {count} 篇文章： {article["title"]}')
            article['text'] = self.get_article(article['link'])
            articles.append(article)
            count += 1
            
        # 反转
        articles.reverse()
            
        return articles

    def get_total_page(self):
        print('「NATURE BRIEF」正在获取总页数...')
        url = 'https://www.nature.com/nature/articles?type=news-in-brief'
        html = self.fetch(url)
        soup = BeautifulSoup(html, 'html.parser')
        pagination_items = soup.find_all('li', class_='c-pagination__item')
        total_page = None
        for item in pagination_items:
            data_page = item.get('data-page')
            if data_page and data_page.isdigit():
                total_page = data_page
        print(f'「NATURE BRIEF」共计 {total_page} 页')
        return int(total_page)

    def get_page(self, page: int = 1):
        url = f'https://www.nature.com/nature/articles?type=news-in-brief&page={page}'
        print('「NATURE BRIEF」正在获取网页...')
        html = self.fetch(url)
        
        print('「NATURE BRIEF」正在解析...')
        # 使用 bs4 解析
        soup = BeautifulSoup(html, 'html.parser')
        
        # 获取文章列表
        articles = soup.find_all('article')
        articles_arr = []
        for article in articles:
            title = article.find('h3', class_='c-card__title').text.strip()
            # 提取链接
            link = article.find('a', class_='c-card__link')['href']
            
            description_element = article.find('div', class_='c-card__summary')
            description = description_element.text.strip() if description_element else None
            
            author_element = article.find('span', itemprop='name')
            author = author_element.text.strip() if author_element else None
            
            date_published = article.find('time', itemprop='datePublished')['datetime']
            
            articles_arr.append({
                'id': link.split('/')[-1],
                'title': title,
                'description': description,
                'author': author,
                'link': f"https://www.nature.com{link}",
                'date_published': date_published
            })
            
        print(f'「NATURE BRIEF」解析完成！第 {page} 页，共计 {len(articles_arr)} 篇文章')
        
        return articles_arr
    
    def get_article(self, url: str):
        print('「NATURE BRIEF」正在获取文章...')
        html = self.fetch(url)
        soup = BeautifulSoup(html, 'html.parser')
        
        main_content = soup.find('div', class_='main-content')
        
        text = main_content.get_text(strip=True)
        return text
    
    def exportToExcel(self, news):
        data = {
            # "date": getDailyNews().formatted_date,
             "ID": [],
             "title": [],
             # "副标题": [],
             "input_content": [],
             # "图片 URL": [],
             # "图片来源": [],
             # "图片作者": []
        }

        for article in news:
            data["ID"].append(article['id'])
            data["title"].append(article['title'])
            # data["副标题"].append(article["subtitle"])
            data["input_content"].append(article['text'])

                # pictures = article["pictures”]
                # if pictures:
                #     picture = pictures[0]
                #     data[“图片 URL”].append(picture[“url”])
                #     data[“图片来源”].append(picture[“source”])
                #     data[“图片作者”].append(picture[“author”])
                # else:
                #     data[“图片 URL”].append(“”)
                #     data[“图片来源”].append(“”)
                #     data[“图片作者”].append(“”)

        df = pd.DataFrame(data)
        df.to_excel("c.xlsx", index=False)
        

naturebrief_spider = NatureBrief()
    
if __name__ == '__main__':
    spider = NatureBrief()
    article = naturebrief_spider.get_today_articles()
    spider.exportToExcel(article)
    print('「NATURE BRIEF」获取成功')
    print(article)
如何获取每天对应的文章