import requests
from datetime import datetime,timedelta
from bs4 import BeautifulSoup
import pandas as pd

class NatureBrief:
    def __init__(self) -> None:
        
        self.start_date = datetime.now() # datetime.strptime('2023-06-06', '%Y-%m-%d')
        
        pass
    
    def fetch(self, url) -> str:
        headers = {
            'accept':
            'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
            'user-agent': 'application/json, text/javascript, */*; q=0.01',
        }
        r = requests.get(url, headers=headers)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    
    def get_today_articles(self, count: int = 3000):
        # 通过日期计算获取的文章数
        current_date = datetime.now()

        # for i in range(300):
        #     elapsed_time = current_date - timedelta(days=i)
        #     print(elapsed_time)
        # article_count = elapsed_time
        elapsed_time = current_date - self.start_date
        article_count = (elapsed_time.days + 1) * count
        print(article_count)
            
        # 获取对应数量的文章
        total_page = self.get_total_page()
        article_list = []
        
        page_i = 1
        while page_i < total_page and len(article_list) < article_count:
            if page_i + 1 > total_page:
                print('「NATURE BRIEF」历史文章已全部获取。无法获取更多...')
                break
            
            article_list += self.get_page(page_i)
            page_i += 1
            if len(article_list) >= article_count:
                break
            
        print(f'「NATURE BRIEF」共计获取 {len(article_list)} 篇文章')
        
        # 抽取对应的文章
        article_list = article_list[:article_count]
        # 倒着获取 3 篇
        article_list.reverse()
        
        articles = []
        count = 1
        for article in article_list:
            if count > count:
                break
            print(f'「NATURE BRIEF」正在获取第 {count} 篇文章： {article["title"]}')
            article['text'] = self.get_article(article['link'])
            articles.append(article)
            count += 1
            
        # 反转
        articles.reverse()
            
        return articles

    def get_total_page(self):
        print('「NATURE BRIEF」正在获取总页数...')
        url = 'https://www.nature.com/nature/articles?type=news-in-brief'
        html = self.fetch(url)
        soup = BeautifulSoup(html, 'html.parser')
        pagination_items = soup.find_all('li', class_='c-pagination__item')
        total_page = None
        for item in pagination_items:
            data_page = item.get('data-page')
            if data_page and data_page.isdigit():
                total_page = data_page
        print(f'「NATURE BRIEF」共计 {total_page} 页')
        return int(total_page)

    def get_page(self, page: int = 1):
        url = f'https://www.nature.com/nature/articles?type=news-in-brief&page={page}'
        print('「NATURE BRIEF」正在获取网页...')
        html = self.fetch(url)
        
        print('「NATURE BRIEF」正在解析...')
        # 使用 bs4 解析
        soup = BeautifulSoup(html, 'html.parser')
        
        # 获取文章列表
        articles = soup.find_all('article')
        articles_arr = []
        for article in articles:
            title = article.find('h3', class_='c-card__title').text.strip()
            # 提取链接
            link = article.find('a', class_='c-card__link')['href']
            
            description_element = article.find('div', class_='c-card__summary')
            description = description_element.text.strip() if description_element else None
            
            author_element = article.find('span', itemprop='name')
            author = author_element.text.strip() if author_element else None
            
            date_published = article.find('time', itemprop='datePublished')['datetime']
            
            articles_arr.append({
                'id': link.split('/')[-1],
                'title': title,
                'description': description,
                'author': author,
                'link': f"https://www.nature.com{link}",
                'date_published': date_published
            })
            
        print(f'「NATURE BRIEF」解析完成！第 {page} 页，共计 {len(articles_arr)} 篇文章')
        
        return articles_arr
    
    def get_article(self, url: str):
        print('「NATURE BRIEF」正在获取文章...')
        html = self.fetch(url)
        soup = BeautifulSoup(html, 'html.parser')
        
        main_content = soup.find('div', class_='main-content')
        
        text = main_content.get_text(strip=True)
        return text
    
    def exportToExcel(self, news):
        data = {
            # "date": getDailyNews().formatted_date,
             "ID": [],
             "title": [],
             # "副标题": [],
             "input_content": [],
             # "图片 URL": [],
             # "图片来源": [],
             # "图片作者": []
        }

        for article in news:
            data["ID"].append(article['id'])
            data["title"].append(article['title'])
            # data["副标题"].append(article["subtitle"])
            data["input_content"].append(article['text'])

                # pictures = article["pictures”]
                # if pictures:
                #     picture = pictures[0]
                #     data[“图片 URL”].append(picture[“url”])
                #     data[“图片来源”].append(picture[“source”])
                #     data[“图片作者”].append(picture[“author”])
                # else:
                #     data[“图片 URL”].append(“”)
                #     data[“图片来源”].append(“”)
                #     data[“图片作者”].append(“”)

        df = pd.DataFrame(data)
        df.to_excel("c.xlsx", index=False)
        

naturebrief_spider = NatureBrief()
    
if __name__ == '__main__':
    spider = NatureBrief()
    article = naturebrief_spider.get_today_articles()
    spider.exportToExcel(article)
    print('「NATURE BRIEF」获取成功')
    print(article)
NATURE BRIEF」解析完成！第 79 页，共计 20 篇文章
「NATURE BRIEF」正在获取网页...
「NATURE BRIEF」正在解析...
「NATURE BRIEF」解析完成！第 80 页，共计 20 篇文章
「NATURE BRIEF」正在获取网页...
「NATURE BRIEF」正在解析...
「NATURE BRIEF」解析完成！第 81 页，共计 20 篇文章
「NATURE BRIEF」正在获取网页...
「NATURE BRIEF」正在解析...
「NATURE BRIEF」解析完成！第 82 页，共计 20 篇文章
「NATURE BRIEF」正在获取网页...
「NATURE BRIEF」正在解析...
「NATURE BRIEF」解析完成！第 83 页，共计 20 篇文章
「NATURE BRIEF」正在获取网页...
「NATURE BRIEF」正在解析...
「NATURE BRIEF」解析完成！第 84 页，共计 20 篇文章
「NATURE BRIEF」正在获取网页...
「NATURE BRIEF」正在解析...
「NATURE BRIEF」解析完成！第 85 页，共计 20 篇文章
「NATURE BRIEF」正在获取网页...
「NATURE BRIEF」正在解析...
「NATURE BRIEF」解析完成！第 86 页，共计 20 篇文章
「NATURE BRIEF」正在获取网页...
「NATURE BRIEF」正在解析...
「NATURE BRIEF」解析完成！第 87 页，共计 20 篇文章
「NATURE BRIEF」共计获取 1740 篇文章
「NATURE BRIEF」正在获取第 1 篇文章： News in Brief
「NATURE BRIEF」正在获取文章...
「NATURE BRIEF」正在获取第 2 篇文章： A refuge for scientists' writings
「NATURE BRIEF」正在获取文章...
「NATURE BRIEF」正在获取第 3 篇文章： Environmental research in Northern Finland
「NATURE BRIEF」正在获取文章...
Traceback (most recent call last):
  File "e:\桌面\spider\nature.py", line 172, in <module>
    article = naturebrief_spider.get_today_articles()
  File "e:\桌面\spider\nature.py", line 64, in get_today_articles
    article['text'] = self.get_article(article['link'])
  File "e:\桌面\spider\nature.py", line 132, in get_article
    text = main_content.get_text()
AttributeError: 'NoneType' object has no attribute 'get_text'
怎么解决问题