import requests
import pandas as pd
import time
import logging
import os
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from tqdm import tqdm

# 设置日志
def setup_logger():
    if not os.path.exists('logs'):
        os.makedirs('logs')
    
    log_filename = f"logs/cninfo_crawler_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log"
    
    logging.basicConfig(
        level=logging.INFO,
        format='%(asctime)s - %(levelname)s - %(message)s',
        handlers=[
            logging.FileHandler(log_filename),
            logging.StreamHandler()
        ]
    )
    return logging.getLogger(__name__)

# 生成日期段
def generate_date_chunks(start_date, end_date, years_per_chunk=3):
    start = datetime.strptime(start_date, "%Y-%m-%d")
    end = datetime.strptime(end_date, "%Y-%m-%d")
    
    chunks = []
    current = start
    
    while current < end:
        chunk_end = min(current + relativedelta(years=years_per_chunk), end)
        chunks.append((
            current.strftime("%Y-%m-%d"),
            chunk_end.strftime("%Y-%m-%d")
        ))
        current = chunk_end + timedelta(days=1)
    
    return chunks

# 爬取数据
def fetch_announcements(keyword, start_date, end_date, logger):
    logger.info(f"开始爬取关键词 '{keyword}' 从 {start_date} 到 {end_date} 的数据")
    
    url = "http://www.cninfo.com.cn/new/hisAnnouncement/query"
    
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
        "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
        "Accept": "application/json, text/javascript, */*; q=0.01",
        "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
        "Referer": "http://www.cninfo.com.cn/new/disclosure",
        "X-Requested-With": "XMLHttpRequest"
    }
    
    page_num = 1
    total_pages = 1  # 初始值，会在第一次请求后更新
    all_announcements = []
    
    # 使用tqdm创建进度条
    pbar = tqdm(total=1, desc=f"爬取 '{keyword}' {start_date}~{end_date}")
    
    while page_num <= total_pages:
        payload = {
            "pageNum": page_num,
            "pageSize": 30,
            "column": "szse",
            "tabName": "fulltext",
            "plate": "",
            "stock": "",
            "searchkey": keyword,
            "secid": "",
            "category": "",
            "trade": "",
            "seDate": f"{start_date}~{end_date}",
            "sortName": "",
            "sortType": "",
            "isHLtitle": "true"
        }
        
        retry_count = 0
        max_retries = 3
        
        while retry_count < max_retries:
            try:
                response = requests.post(url, headers=headers, data=payload, timeout=30)
                data = response.json()
                
                # 更新总页数和进度条
                if page_num == 1:
                    total_pages = data.get("totalpages", 0)
                    # 有时候返回的totalpages比实际少一页
                    total_pages += 1  
                    pbar.total = total_pages
                    pbar.refresh()
                
                announcements = data.get("announcements", [])
                if announcements:
                    all_announcements.extend(announcements)
                
                # 如果当前页没有数据但已经超过了获取的总页数，则退出
                if not announcements and page_num > data.get("totalpages", 0):
                    break
                    
                break  # 成功获取数据，退出重试循环
            
            except Exception as e:
                retry_count += 1
                logger.warning(f"请求失败，正在进行第 {retry_count} 次重试: {str(e)}")
                time.sleep(5)  # 失败后等待5秒再重试
                
                if retry_count == max_retries:
                    logger.error(f"达到最大重试次数，跳过此页: {page_num}")
        
        pbar.update(1)
        page_num += 1
        time.sleep(1)  # 避免请求过快
    
    pbar.close()
    logger.info(f"关键词 '{keyword}' 从 {start_date} 到 {end_date} 爬取完成，共获取 {len(all_announcements)} 条公告")
    
    return all_announcements

# 主函数
def main():
    logger = setup_logger()
    logger.info("开始爬取公告数据")
    
    # 设置参数
    keywords = ["战略合作", "战略联盟", "企业联盟", "企业合作"]
    start_date = "2000-01-01"
    end_date = "2025-03-29"
    output_file = "all_announcements_complete.xlsx"
    
    # 每个关键词的所有公告
    all_data = []
    
    # 遍历关键词
    for keyword in keywords:
        logger.info(f"开始处理关键词: {keyword}")
        
        # 生成日期段
        date_chunks = generate_date_chunks(start_date, end_date, years_per_chunk=3)
        logger.info(f"将日期范围分为 {len(date_chunks)} 个子区间")
        
        # 遍历日期段
        for chunk_start, chunk_end in date_chunks:
            announcements = fetch_announcements(keyword, chunk_start, chunk_end, logger)
            
            # 将当前区间的数据添加到总数据中
            for announcement in announcements:
                # 添加关键词信息
                announcement['keyword'] = keyword
                all_data.append(announcement)
            
            # 每个区间爬取完后保存一次数据，防止意外中断导致数据丢失
            temp_df = pd.DataFrame(all_data)
            temp_df.to_excel(f"temp_{output_file}", index=False)
            logger.info(f"临时数据已保存，当前共 {len(all_data)} 条公告")
            
            # 避免请求过快被封
            time.sleep(3)
    
    # 处理最终数据
    logger.info("开始处理最终数据...")
    
    # 转换为DataFrame并做必要的处理
    df = pd.DataFrame(all_data)
    
    # 转换时间戳为可读日期
    if 'announcementTime' in df.columns:
        df['announcementDate'] = pd.to_datetime(df['announcementTime'], unit='ms').dt.strftime('%Y-%m-%d')
    
    # 生成PDF下载链接
    if 'adjunctUrl' in df.columns:
        df['pdfLink'] = df['adjunctUrl'].apply(lambda x: f"http://static.cninfo.com.cn/{x}" if pd.notna(x) else None)
    
    # 保存结果
    df.to_excel(output_file, index=False)
    logger.info(f"所有数据已保存到 {output_file}，共 {len(df)} 条公告")
    
    # 删除临时文件
    if os.path.exists(f"temp_{output_file}"):
        os.remove(f"temp_{output_file}")
        logger.info("临时文件已删除")
    
    logger.info("爬取任务完成")

if __name__ == "__main__":
    main()
我想只爬取公告，不下载任何东西，并且我发现填写stock之后 pyload显示是stock
600000,gssh0600000，如何更改代码