import requests
from bs4 import BeautifulSoup
import csv
import time

# 設定搜尋關鍵字
KEYWORD = "台積電"

# 自由時報搜尋 URL 範本
SEARCH_URL = "https://news.ltn.com.tw/search?keyword={}&page={}"

# 設定 HTTP 標頭，以模擬瀏覽器請求
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)\
     Chrome/58.0.3029.110 Safari/537.3"
}

def fetch_news(page):
    """
    抓取特定頁數的新聞列表
    """
    url = SEARCH_URL.format(KEYWORD, page)
    response = requests.get(url, headers=HEADERS)
    
    if response.status_code != 200:
        print(f"無法抓取第 {page} 頁")
        return []
    
    soup = BeautifulSoup(response.text, "html.parser")
    
    # 找到所有具有 class "tit" 的 <a> 標籤
    article_links = soup.find_all("a", class_="tit")
    
    news_list = []
    for link in article_links:
        title = link.get_text(strip=True)
        href = link.get('href')
        description = link.get('data-desc', '').split("  ")[0]  # 擷取 data-desc 中的描述
        full_link = href if href.startswith('http') else f"https://news.ltn.com.tw{href}"
        
        # 抓取文章的發布日期
        # 有些頁面可能不包含日期，需要額外請求文章頁面來取得
        date = fetch_article_date(full_link)
        
        news_list.append({
            "title": title,
            "link": full_link,
            "description": description,
            "date": date
        })
    
    return news_list

def fetch_article_date(article_url):
    """
    抓取單篇文章的發布日期
    """
    try:
        response = requests.get(article_url, headers=HEADERS)
        if response.status_code != 200:
            return "未知日期"
        
        soup = BeautifulSoup(response.text, "html.parser")
        # 一般文章頁面，日期在 <time> 標籤內或特定的 class
        time_tag = soup.find("time")
        if time_tag:
            return time_tag.get_text(strip=True)
        
        # 如果沒有 <time> 標籤，嘗試其他方法
        meta_tag = soup.find("meta", {"property": "article:published_time"})
        if meta_tag and meta_tag.get("content"):
            return meta_tag.get("content")
        
        return "未知日期"
    except Exception as e:
        print(f"抓取文章日期時出錯: {e}")
        return "未知日期"

def save_to_csv(news_data, filename="tsmc_news.csv"):
    """
    將新聞資料儲存到 CSV 檔案
    """
    if not news_data:
        print("沒有新聞資料可儲存。")
        return
    
    keys = news_data[0].keys()
    with open(filename, "w", newline='', encoding='utf-8-sig') as output_file:
        dict_writer = csv.DictWriter(output_file, fieldnames=keys)
        dict_writer.writeheader()
        dict_writer.writerows(news_data)

def main():
    all_news = []
    total_pages = 5  # 設定要抓取的頁數

    for page in range(1, total_pages + 1):
        print(f"正在抓取第 {page} 頁...")
        news = fetch_news(page)
        if not news:
            break
        all_news.extend(news)
        time.sleep(1)  # 等待1秒以避免過於頻繁的請求

    if all_news:
        save_to_csv(all_news)
        print(f"已儲存 {len(all_news)} 則新聞到 tsmc_news.csv")
    else:
        print("沒有抓取到新聞資料。")

if __name__ == "__main__":
        main()


##date我只要11/01
###<a href="https://search.ltn.com.tw/list?keyword=台積電&amp;start_time=20041201&amp;end_time=20241117&amp;sort=date&amp;type=all&amp;page=2" class="p_num" data-desc="2" title="第2頁">2</a>


##跳頁按鈕長這樣