Skill: nighmat1220/ai-news-workflow (version 0.1.3)

# AI News Workflow This skill collects AI industry news (RSS + web pages), filters/deduplicates/classifies them, optionally generates Chinese titles & ~80-char summaries(recommended 60-100,hard cap ~110) via Doubao (Volcengine Ark), and exports Excel tables + a Word briefing. ## How to run - Entry: `skill_entry.py` - Init (optional): `python skill_entry.py --init` - Run: `python skill_entry.py` ## Inputs Put these files under `./input/`: - 企业名单.xlsx - source_config.xlsx - web_source_config.xlsx (optional) ## Outputs Generated under `./output/`: - Excel: `./output/excel/` - Word: `./output/word/` ## Optional: Ark API Provide API key via environment variable: - `ARK_API_KEY` Model can be selected at runtime: - `python skill_entry.py --model <MODEL_NAME>`

path: app/core/constants.py
content: CATEGORY_LEVEL_1 = ["全球AI产业动态", "名单企业动态"]

CATEGORY_LEVEL_2 = [
    "企业重大动态",
    "行业大事件",
    "政策动态",
    "融资",
    "投资/并购",
    "成立新企业",
    "产品发布",
    "获得认证",
    "其他",
]

IMPORTANCE_LEVELS = ["高", "中", "低"]

ENTRY_STATUS = ["pending", "included", "manual_review", "excluded"]

path: app/core/database.py
content: from __future__ import annotations

import sqlite3
from pathlib import Path
from typing import Optional

from app.core.settings import Settings


class DatabaseManager:
    """
    SQLite 数据库管理器。
    """

    def __init__(self, settings: Settings) -> None:
        db_file = settings.get("paths", "db_file")
        if not db_file:
            raise ValueError("config.yaml 中未配置 db_file")

        self.db_path = Path(db_file)
        self.db_path.parent.mkdir(parents=True, exist_ok=True)

    def get_connection(self) -> sqlite3.Connection:
        conn = sqlite3.connect(self.db_path)
        conn.row_factory = sqlite3.Row
        return conn

    def init_db(self) -> None:
        """
        初始化数据库表结构。
        """
        with self.get_connection() as conn:
            cursor = conn.cursor()

            cursor.execute(
                """
                CREATE TABLE IF NOT EXISTS news_history (
                    id INTEGER PRIMARY KEY AUTOINCREMENT,
                    title TEXT NOT NULL,
                    normalized_title TEXT,
                    source_name TEXT,
                    source_url TEXT UNIQUE,
                    publish_time TEXT,
                    related_companies TEXT,
                    category_level_1 TEXT,
                    category_level_2 TEXT,
                    content_hash TEXT,
                    importance TEXT,
                    created_at TEXT DEFAULT CURRENT_TIMESTAMP
                )
                """
            )

            cursor.execute(
                """
                CREATE TABLE IF NOT EXISTS run_log (
                    id INTEGER PRIMARY KEY AUTOINCREMENT,
                    run_date TEXT,
                    start_time TEXT,
                    end_time TEXT,
                    fetched_count INTEGER DEFAULT 0,
                    valid_count INTEGER DEFAULT 0,
                    deduped_count INTEGER DEFAULT 0,
                    excel_path TEXT,
                    word_path TEXT,
                    status TEXT,
                    error_message TEXT,
                    created_at TEXT DEFAULT CURRENT_TIMESTAMP
                )
                """
            )

            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_news_publish_time ON news_history(publish_time)"
            )
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_news_title ON news_history(title)"
            )
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_news_content_hash ON news_history(content_hash)"
            )
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_news_normalized_title ON news_history(normalized_title)"
            )
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_news_source_url ON news_history(source_url)"
            )

            conn.commit()

path: app/core/logger.py
content: from __future__ import annotations

from pathlib import Path
from typing import Optional

from loguru import logger

from app.core.settings import Settings


def setup_logger(settings: Settings, log_name: Optional[str] = None) -> None:
    """
    初始化日志系统。
    """
    logger.remove()

    log_dir = Path(settings.get("paths", "log_dir"))
    log_dir.mkdir(parents=True, exist_ok=True)

    log_level = settings.get("logging", "level", default="INFO")
    rotation = settings.get("logging", "rotation", default="10 MB")
    retention = settings.get("logging", "retention", default="30 days")
    encoding = settings.get("logging", "encoding", default="utf-8")

    if not log_name:
        log_name = "workflow.log"

    log_file = log_dir / log_name

    logger.add(
        sink=lambda msg: print(msg, end=""),
        level=log_level,
        colorize=True,
        enqueue=False,
    )

    logger.add(
        str(log_file),
        level=log_level,
        rotation=rotation,
        retention=retention,
        encoding=encoding,
        enqueue=False,
    )


def get_logger():
    return logger

path: app/core/settings.py
content: from __future__ import annotations

from pathlib import Path
from typing import Any, Dict, Optional

import yaml


class Settings:
    """
    全局配置读取器。
    相对路径一律按 config.yaml 所在目录解析。
    """

    def __init__(self, config_path: str = "config.yaml") -> None:
        self.config_path = Path(config_path).resolve()
        if not self.config_path.exists():
            raise FileNotFoundError(f"配置文件不存在: {self.config_path}")

        self.base_dir = self.config_path.parent

        with self.config_path.open("r", encoding="utf-8") as f:
            self._data: Dict[str, Any] = yaml.safe_load(f) or {}

        self._normalize_paths()
        self._ensure_directories()

    def get(self, *keys: str, default: Optional[Any] = None) -> Any:
        data: Any = self._data
        for key in keys:
            if not isinstance(data, dict):
                return default
            data = data.get(key)
            if data is None:
                return default
        return data

    @property
    def data(self) -> Dict[str, Any]:
        return self._data

    def _normalize_paths(self) -> None:
        """
        将 config.yaml 中的相对路径统一转换为绝对路径。
        """
        path_fields = [
            "company_file",
            "source_file",
            "output_excel_dir",
            "output_word_dir",
            "log_dir",
            "db_file",
            "cache_dir",
            "temp_dir",
            "excel_template",
            "word_template",
        ]

        paths = self._data.get("paths", {})
        if not isinstance(paths, dict):
            return

        for field in path_fields:
            value = paths.get(field)
            if not value:
                continue

            p = Path(value)
            if not p.is_absolute():
                paths[field] = str((self.base_dir / p).resolve())
            else:
                paths[field] = str(p.resolve())

    def _ensure_directories(self) -> None:
        path_keys = [
            ("paths", "output_excel_dir"),
            ("paths", "output_word_dir"),
            ("paths", "log_dir"),
            ("paths", "cache_dir"),
            ("paths", "temp_dir"),
        ]

        for keys in path_keys:
            path_value = self.get(*keys)
            if path_value:
                Path(path_value).mkdir(parents=True, exist_ok=True)

        db_file = self.get("paths", "db_file")
        if db_file:
            Path(db_file).parent.mkdir(parents=True, exist_ok=True)


_settings_instance: Optional[Settings] = None


def get_settings(config_path: str = "config.yaml") -> Settings:
    global _settings_instance
    if _settings_instance is None:
        _settings_instance = Settings(config_path=config_path)
    return _settings_instance

path: app/core/time_window.py
content: from __future__ import annotations

from dataclasses import dataclass
from datetime import datetime, timedelta
from zoneinfo import ZoneInfo


DEFAULT_TZ = "Asia/Shanghai"


@dataclass
class TimeWindow:
    start_time: datetime
    end_time: datetime

    def contains(self, dt: datetime) -> bool:
        dt = normalize_to_timezone(
            dt,
            self.start_time.tzinfo.key if hasattr(self.start_time.tzinfo, "key") else DEFAULT_TZ,
        )
        return self.start_time <= dt < self.end_time

    def to_text(self) -> str:
        return f"{self.start_time.strftime('%Y-%m-%d %H:%M:%S')} ~ {self.end_time.strftime('%Y-%m-%d %H:%M:%S')}"


def normalize_to_timezone(dt: datetime | None, timezone_str: str = DEFAULT_TZ) -> datetime | None:
    if dt is None:
        return None

    tz = ZoneInfo(timezone_str)

    if dt.tzinfo is None:
        return dt.replace(tzinfo=tz)

    return dt.astimezone(tz)

def build_time_window(
    mode: str = "previous_day",
    timezone_str: str = DEFAULT_TZ,
    start_hour: int = 0,
    end_hour: int = 0,
    lookback_days: int = 3,
    now: datetime | None = None,
) -> TimeWindow:
    """
    支持两种模式：
    1) previous_day：前一天 start_hour:00:00（含）到当天 end_hour:00:00（不含）
       - 允许 start_hour != end_hour（例如 23->23）
       - end_time 以“今天 end_hour:00:00”为锚点
    2) rolling_days：当前时间往前 lookback_days 天（含 now）
    """
    tz = ZoneInfo(timezone_str)

    # 统一 now 为带时区的时间
    if now is None:
        now = datetime.now(tz)
    else:
        now = normalize_to_timezone(now, timezone_str)

    mode = (mode or "previous_day").strip().lower()

    if mode in {"rolling_days", "rolling", "lookback"}:
        end_time = now
        start_time = now - timedelta(days=int(lookback_days))
        return TimeWindow(start_time=start_time, end_time=end_time)

    # previous_day：以“今天 end_hour:00:00”为锚点
    # 如果当前时间早于今天 end_hour:00:00，则锚点仍是今天 end_hour
    # 如果当前时间晚于今天 end_hour:00:00，锚点仍是今天 end_hour（符合你“统计口径到当天 end_hour”）
    anchor_end = now.replace(hour=int(end_hour), minute=0, second=0, microsecond=0)

    # start_time 是“锚点往前一天”，再设置 start_hour
    start_time = (anchor_end - timedelta(days=1)).replace(hour=int(start_hour), minute=0, second=0, microsecond=0)
    end_time = anchor_end

    # 防御：如果 start_time >= end_time（极少见但可能来自非法参数），则兜底为前一天0点到今天0点
    if start_time >= end_time:
        end_time = now.replace(hour=0, minute=0, second=0, microsecond=0)
        start_time = (end_time - timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0)

    return TimeWindow(start_time=start_time, end_time=end_time)

def get_event_date(dt: datetime | None, timezone_str: str = DEFAULT_TZ) -> str:
    if dt is None:
        return ""
    dt = normalize_to_timezone(dt, timezone_str)
    return dt.strftime("%Y-%m-%d")

def normalize_to_shanghai(dt: datetime | None) -> datetime | None:
    """
    向后兼容旧代码。
    等价于 normalize_to_timezone(dt, "Asia/Shanghai")
    """
    return normalize_to_timezone(dt, "Asia/Shanghai")

path: app/crawlers/rss_crawler.py
content: from __future__ import annotations

from datetime import datetime, timedelta
from typing import List, Optional
from zoneinfo import ZoneInfo

import feedparser

from app.models.news_item import NewsItem
from app.models.source import Source
from app.parsers.content_cleaner import ContentCleaner
from app.utils.hash_utils import build_content_hash


SHANGHAI_TZ = ZoneInfo("Asia/Shanghai")


class RSSCrawler:
    """
    RSS 抓取器。
    支持：
    - 抓取 RSS
    - 只保留近 N 天内的资讯
    """

    def __init__(
        self,
        request_timeout: int = 20,
        lookback_days: int = 3,
    ) -> None:
        self.request_timeout = request_timeout
        self.lookback_days = lookback_days
        self.cleaner = ContentCleaner()

    def fetch(self, source: Source) -> List[NewsItem]:
        feed = feedparser.parse(source.url)

        items: List[NewsItem] = []
        now_dt = datetime.now(SHANGHAI_TZ)

        for entry in feed.entries:
            title = (getattr(entry, "title", "") or "").strip()
            link = (getattr(entry, "link", "") or "").strip()

            raw_summary = (
                getattr(entry, "summary", "")
                or getattr(entry, "description", "")
                or ""
            )
            raw_summary = str(raw_summary).strip()

            publish_time = self._extract_publish_time(entry)

            # 只保留近 N 天内的资讯
            if not self._is_within_lookback(publish_time, now_dt):
                continue

            summary = self.cleaner.clean_summary(raw_summary, max_length=300)
            cleaned_content = self.cleaner.clean_text(raw_summary)

            item = NewsItem(
                title=title if title else "Untitled",
                summary=summary,
                source_name=source.source_name,
                source_url=link if link else source.url,
                source_type=source.source_type,
                source_region_scope=getattr(source, "region_scope", None),
                is_official=(source.source_type in {"official", "government"}),
                publish_time=publish_time,
                crawl_time=now_dt,
                raw_content=raw_summary,
                cleaned_content=cleaned_content,
                content_hash=build_content_hash(
                    title=title,
                    summary=summary,
                    content=cleaned_content,
                ),
                normalized_title=(title or "").strip().lower(),
                language=None,
                entry_status="pending",
                dedup_status="unique",
            )
            items.append(item)

        return items

    def _extract_publish_time(self, entry) -> Optional[datetime]:
        """
        从 RSS entry 中提取发布时间，优先 published_parsed / updated_parsed。
        """
        time_struct = None

        if hasattr(entry, "published_parsed") and entry.published_parsed:
            time_struct = entry.published_parsed
        elif hasattr(entry, "updated_parsed") and entry.updated_parsed:
            time_struct = entry.updated_parsed

        if not time_struct:
            return None

        try:
            dt = datetime(
                year=time_struct.tm_year,
                month=time_struct.tm_mon,
                day=time_struct.tm_mday,
                hour=time_struct.tm_hour,
                minute=time_struct.tm_min,
                second=time_struct.tm_sec,
                tzinfo=ZoneInfo("UTC"),
            )
            return dt.astimezone(SHANGHAI_TZ)
        except Exception:
            return None

    def _is_within_lookback(
        self,
        publish_time: Optional[datetime],
        now_dt: datetime,
    ) -> bool:
        """
        判断是否在近 N 天内。
        - publish_time 为空时，默认保留，避免误漏抓
        """
        if publish_time is None:
            return True

        threshold = now_dt - timedelta(days=self.lookback_days)
        return publish_time >= threshold

path: app/crawlers/web_crawler.py
content: from __future__ import annotations

from datetime import datetime, timedelta
from typing import List, Optional
from urllib.parse import urljoin
from zoneinfo import ZoneInfo

import requests
from bs4 import BeautifulSoup

from app.models.news_item import NewsItem
from app.models.source import Source
from app.parsers.html_extractors import extract_article_by_selectors
from app.utils.hash_utils import build_content_hash


SHANGHAI_TZ = ZoneInfo("Asia/Shanghai")


class WebCrawler:
    """
    网页爬虫：
    - 列表页抓链接
    - 详情页抽取标题/时间/正文
    """

    def __init__(
        self,
        request_timeout: int = 20,
        lookback_days: int = 3,
        user_agent: str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
    ) -> None:
        self.request_timeout = request_timeout
        self.lookback_days = lookback_days
        self.session = requests.Session()
        self.session.headers.update({"User-Agent": user_agent})

    def fetch(self, source: Source) -> List[NewsItem]:
        now_dt = datetime.now(SHANGHAI_TZ)
        threshold = now_dt - timedelta(days=self.lookback_days)

        list_url = getattr(source, "list_url", None) or source.url
        link_selector = getattr(source, "link_selector", None)
        url_prefix = getattr(source, "url_prefix", None) or list_url
        max_items = int(getattr(source, "max_items_per_run", None) or 30)

        title_selector = getattr(source, "title_selector", None)
        date_selector = getattr(source, "date_selector", None)
        content_selector = getattr(source, "content_selector", None)

        html = self._get_text(list_url)
        if not html:
            return []

        links = self._extract_links(html, link_selector=link_selector, base_url=url_prefix)
        links = links[:max_items]

        items: List[NewsItem] = []
        for link in links:
            detail_html = self._get_text(link)
            if not detail_html:
                continue

            article = extract_article_by_selectors(
                detail_html,
                title_selector=title_selector,
                date_selector=date_selector,
                content_selector=content_selector,
            )

            # 近 N 天过滤（发布时间缺失则先保留，后面由统计窗口再筛）
            if article.publish_time and article.publish_time < threshold:
                continue

            content_text = article.content_text

            item = NewsItem(
                title=article.title,
                summary="",  # 后续由豆包生成更好的摘要
                source_name=source.source_name,
                source_url=link,
                source_type=source.source_type,
                source_region_scope=getattr(source, "region_scope", None),
                is_official=(source.source_type in {"official", "government"}),
                publish_time=article.publish_time,
                crawl_time=now_dt,
                raw_content=content_text,
                cleaned_content=content_text,
                content_hash=build_content_hash(
                    title=article.title,
                    summary="",
                    content=content_text,
                ),
                normalized_title=(article.title or "").strip().lower(),
                entry_status="pending",
                dedup_status="unique",
            )

            # 把“详细正文”保存在 original_cleaned_content（供豆包总结用）
            item.original_cleaned_content = content_text
            item.original_title = article.title

            items.append(item)

        return items

    def _get_text(self, url: str) -> str:
        r = self.session.get(url, timeout=self.request_timeout)
        r.raise_for_status()
        r.encoding = r.apparent_encoding or "utf-8"
        return r.text

    @staticmethod
    def _extract_links(html: str, link_selector: Optional[str], base_url: str) -> List[str]:
        soup = BeautifulSoup(html, "lxml")

        anchors = []
        if link_selector:
            anchors = soup.select(link_selector)
        else:
            anchors = soup.select("a")

        urls = []
        for a in anchors:
            href = a.get("href")
            if not href:
                continue
            full = urljoin(base_url, href)
            if full.startswith("http"):
                urls.append(full)

        # 去重保持顺序
        seen = set()
        uniq = []
        for u in urls:
            if u not in seen:
                uniq.append(u)
                seen.add(u)
        return uniq

path: app/exporters/excel_exporter.py
content: from __future__ import annotations

from pathlib import Path
from typing import List, Optional

import pandas as pd
from openpyxl.styles import Alignment, Font, PatternFill
from openpyxl.utils import get_column_letter

from app.models.news_item import NewsItem


class ExcelExporter:
    """
    Excel 导出器（增强显示版）。
    支持：
    - summary 概览 sheet
    - 主结果 sheet
    - 重复项 sheet
    - 无效项 sheet
    - 排序增强
    - 自动筛选 / 冻结首行 / 列宽调整 / 重要性着色 / 超链接样式
    """

    SUMMARY_SHEET_NAME = "summary"
    MAIN_SHEET_NAME = "news_table"
    DUP_SHEET_NAME = "duplicate_items"
    INVALID_SHEET_NAME = "invalid_items"

    def __init__(self, output_dir: str, filename_prefix: str = "AI资讯信息表") -> None:
        self.output_dir = Path(output_dir)
        self.output_dir.mkdir(parents=True, exist_ok=True)
        self.filename_prefix = filename_prefix

    def export(
        self,
        items: List[NewsItem],
        run_date: str,
        filename: Optional[str] = None,
    ) -> Path:
        return self.export_full(
            valid_items=items,
            duplicate_items=[],
            invalid_items=[],
            run_date=run_date,
            filename=filename,
        )

    def export_full(
        self,
        valid_items: List[NewsItem],
        duplicate_items: List[NewsItem],
        invalid_items: List[NewsItem],
        run_date: str,
        filename: Optional[str] = None,
    ) -> Path:
        if filename:
            file_path = self.output_dir / filename
        else:
            safe_date = run_date.replace("-", "")
            file_path = self.output_dir / f"{self.filename_prefix}_{safe_date}.xlsx"

        main_df = self._build_dataframe(valid_items)
        dup_df = self._build_dataframe(duplicate_items)
        invalid_df = self._build_dataframe(invalid_items)

        main_df = self._sort_main_df(main_df)
        dup_df = self._sort_secondary_df(dup_df)
        invalid_df = self._sort_secondary_df(invalid_df)

        summary_df = self._build_summary_dataframe(
            valid_df=main_df,
            duplicate_df=dup_df,
            invalid_df=invalid_df,
            run_date=run_date,
        )

        with pd.ExcelWriter(file_path, engine="openpyxl") as writer:
            summary_df.to_excel(writer, index=False, sheet_name=self.SUMMARY_SHEET_NAME)
            main_df.to_excel(writer, index=False, sheet_name=self.MAIN_SHEET_NAME)
            dup_df.to_excel(writer, index=False, sheet_name=self.DUP_SHEET_NAME)
            invalid_df.to_excel(writer, index=False, sheet_name=self.INVALID_SHEET_NAME)

            self._format_summary_sheet(writer.sheets[self.SUMMARY_SHEET_NAME], summary_df)
            self._format_sheet(writer.sheets[self.MAIN_SHEET_NAME], main_df)
            self._format_sheet(writer.sheets[self.DUP_SHEET_NAME], dup_df)
            self._format_sheet(writer.sheets[self.INVALID_SHEET_NAME], invalid_df)

        return file_path

    def _build_dataframe(self, items: List[NewsItem]) -> pd.DataFrame:
        rows = [item.to_excel_row() for item in items]
        df = pd.DataFrame(rows)

        if df.empty:
            df = pd.DataFrame(columns=self._default_columns())

        return df

    @staticmethod
    def _default_columns() -> List[str]:
        return [
            "资讯日期",
            "抓取日期",
            "时间窗口",
            "一级类别",
            "二级类别",
            "关联企业",
            "国家/地区",
            "标题",
            "摘要",
            "事件要点",
            "来源名称",
            "来源链接",
            "发布时间",
            "抓取时间",
            "是否官方来源",
            "重要性等级",
            "去重标记",
            "入表状态",
            "备注",
        ]

    def _sort_main_df(self, df: pd.DataFrame) -> pd.DataFrame:
        if df.empty:
            return df

        df = df.copy()

        importance_order = {"高": 0, "中": 1, "低": 2, "": 3}
        df["_importance_sort"] = df["重要性等级"].map(lambda x: importance_order.get(str(x), 3))
        df["_publish_sort"] = pd.to_datetime(df["发布时间"], errors="coerce")

        sort_cols = ["_importance_sort", "一级类别", "二级类别", "_publish_sort", "标题"]
        ascending = [True, True, True, False, True]

        df = df.sort_values(by=sort_cols, ascending=ascending, na_position="last")
        df = df.drop(columns=["_importance_sort", "_publish_sort"])
        df = df.reset_index(drop=True)
        return df

    def _sort_secondary_df(self, df: pd.DataFrame) -> pd.DataFrame:
        if df.empty:
            return df

        df = df.copy()
        if "发布时间" in df.columns:
            df["_publish_sort"] = pd.to_datetime(df["发布时间"], errors="coerce")
            df = df.sort_values(by=["_publish_sort", "标题"], ascending=[False, True], na_position="last")
            df = df.drop(columns=["_publish_sort"])
        else:
            df = df.sort_values(by=["标题"], ascending=[True])

        df = df.reset_index(drop=True)
        return df

    def _build_summary_dataframe(
        self,
        valid_df: pd.DataFrame,
        duplicate_df: pd.DataFrame,
        invalid_df: pd.DataFrame,
        run_date: str,
    ) -> pd.DataFrame:
        time_window = ""
        if not valid_df.empty and "时间窗口" in valid_df.columns:
            vals = [v for v in valid_df["时间窗口"].astype(str).tolist() if v and v != "nan"]
            if vals:
                time_window = vals[0]
        elif not invalid_df.empty and "时间窗口" in invalid_df.columns:
            vals = [v for v in invalid_df["时间窗口"].astype(str).tolist() if v and v != "nan"]
            if vals:
                time_window = vals[0]

        total_count = len(valid_df) + len(duplicate_df) + len(invalid_df)
        high_count = self._count_value(valid_df, "重要性等级", "高")
        medium_count = self._count_value(valid_df, "重要性等级", "中")
        low_count = self._count_value(valid_df, "重要性等级", "低")

        rows = [
            {"模块": "运行概览", "指标": "运行日期", "值": run_date},
            {"模块": "运行概览", "指标": "统计窗口", "值": time_window},
            {"模块": "运行概览", "指标": "总处理条数", "值": total_count},
            {"模块": "运行概览", "指标": "有效资讯条数", "值": len(valid_df)},
            {"模块": "运行概览", "指标": "重复资讯条数", "值": len(duplicate_df)},
            {"模块": "运行概览", "指标": "无效资讯条数", "值": len(invalid_df)},
            {"模块": "重要性分布", "指标": "高", "值": high_count},
            {"模块": "重要性分布", "指标": "中", "值": medium_count},
            {"模块": "重要性分布", "指标": "低", "值": low_count},
        ]

        if not valid_df.empty and "一级类别" in valid_df.columns:
            level1_counts = valid_df["一级类别"].fillna("").astype(str).value_counts()
            for k, v in level1_counts.items():
                if k:
                    rows.append({"模块": "一级分类分布", "指标": k, "值": int(v)})

        if not valid_df.empty and "二级类别" in valid_df.columns:
            level2_counts = valid_df["二级类别"].fillna("").astype(str).value_counts()
            for k, v in level2_counts.items():
                if k:
                    rows.append({"模块": "二级分类分布", "指标": k, "值": int(v)})

        return pd.DataFrame(rows)

    @staticmethod
    def _count_value(df: pd.DataFrame, col: str, value: str) -> int:
        if df.empty or col not in df.columns:
            return 0
        return int((df[col].fillna("").astype(str) == value).sum())

    def _format_sheet(self, ws, df: pd.DataFrame) -> None:
        self._freeze_header(ws)
        self._apply_auto_filter(ws)
        self._style_header(ws)
        self._adjust_column_width(ws, df)
        self._apply_wrap_text(ws)
        self._highlight_importance(ws)
        self._style_hyperlinks(ws)

    def _format_summary_sheet(self, ws, df: pd.DataFrame) -> None:
        self._freeze_header(ws)
        self._apply_auto_filter(ws)
        self._style_header(ws)
        self._adjust_column_width(ws, df)
        self._apply_wrap_text(ws)

        module_fill_map = {
            "运行概览": PatternFill(fill_type="solid", fgColor="D9EAF7"),
            "重要性分布": PatternFill(fill_type="solid", fgColor="FCE4D6"),
            "一级分类分布": PatternFill(fill_type="solid", fgColor="E2F0D9"),
            "二级分类分布": PatternFill(fill_type="solid", fgColor="FFF2CC"),
        }

        for row in range(2, ws.max_row + 1):
            module_value = str(ws.cell(row=row, column=1).value or "").strip()
            fill = module_fill_map.get(module_value)
            if fill:
                for col in range(1, ws.max_column + 1):
                    ws.cell(row=row, column=col).fill = fill

    @staticmethod
    def _freeze_header(ws) -> None:
        ws.freeze_panes = "A2"

    @staticmethod
    def _apply_auto_filter(ws) -> None:
        ws.auto_filter.ref = ws.dimensions

    @staticmethod
    def _style_header(ws) -> None:
        header_fill = PatternFill(fill_type="solid", fgColor="D9EAF7")
        header_font = Font(bold=True)

        for cell in ws[1]:
            cell.fill = header_fill
            cell.font = header_font
            cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)

    @staticmethod
    def _adjust_column_width(ws, df: pd.DataFrame) -> None:
        for idx, col in enumerate(df.columns, start=1):
            max_len = len(str(col))

            if not df.empty:
                series_max = df[col].astype(str).map(len).max()
                max_len = max(max_len, int(series_max))

            if col in {"标题"}:
                width = min(max(max_len + 2, 20), 50)
            elif col in {"摘要", "事件要点", "备注", "值"}:
                width = min(max(max_len + 2, 18), 60)
            elif col in {"来源链接"}:
                width = 50
            elif col in {"模块", "指标"}:
                width = min(max(max_len + 2, 12), 24)
            else:
                width = min(max_len + 2, 24)

            col_letter = get_column_letter(idx)
            ws.column_dimensions[col_letter].width = width

    @staticmethod
    def _apply_wrap_text(ws) -> None:
        for row in ws.iter_rows(min_row=2):
            for cell in row:
                cell.alignment = Alignment(vertical="top", wrap_text=True)

    @staticmethod
    def _highlight_importance(ws) -> None:
        header_map = {}
        for idx, cell in enumerate(ws[1], start=1):
            header_map[cell.value] = idx

        importance_col_idx = header_map.get("重要性等级")
        if not importance_col_idx:
            return

        high_fill = PatternFill(fill_type="solid", fgColor="FCE4D6")
        medium_fill = PatternFill(fill_type="solid", fgColor="FFF2CC")
        low_fill = PatternFill(fill_type="solid", fgColor="E2F0D9")

        for row in range(2, ws.max_row + 1):
            cell = ws.cell(row=row, column=importance_col_idx)
            value = str(cell.value or "").strip()

            if value == "高":
                cell.fill = high_fill
            elif value == "中":
                cell.fill = medium_fill
            elif value == "低":
                cell.fill = low_fill

    @staticmethod
    def _style_hyperlinks(ws) -> None:
        header_map = {}
        for idx, cell in enumerate(ws[1], start=1):
            header_map[cell.value] = idx

        link_col_idx = header_map.get("来源链接")
        if not link_col_idx:
            return

        for row in range(2, ws.max_row + 1):
            cell = ws.cell(row=row, column=link_col_idx)
            value = str(cell.value or "").strip()

            if value.startswith("http://") or value.startswith("https://"):
                cell.hyperlink = value
                cell.style = "Hyperlink"

path: app/exporters/simple_excel_exporter.py
content: from __future__ import annotations

from pathlib import Path
from typing import List, Optional

import pandas as pd
from openpyxl.styles import Alignment, Font, PatternFill, Border, Side
from openpyxl.utils import get_column_letter

from app.models.news_item import NewsItem


class SimpleExcelExporter:
    """
    简表版 Excel 导出器：
    - 仅导出有效数据
    - 仅保留一个 sheet
    - 仅保留指定表头
    - 按一级分类排序
    - 名单企业动态：浅蓝底
    - 全球AI产业动态：浅绿底
    - 增加完整边框线
    """

    SHEET_NAME = "valid_items"

    def __init__(self, output_dir: str, filename_prefix: str = "AI资讯有效数据简表") -> None:
        self.output_dir = Path(output_dir)
        self.output_dir.mkdir(parents=True, exist_ok=True)
        self.filename_prefix = filename_prefix

    def export(
        self,
        valid_items: List[NewsItem],
        run_date: str,
        filename: Optional[str] = None,
    ) -> Path:
        if filename:
            file_path = self.output_dir / filename
        else:
            safe_date = run_date.replace("-", "")
            file_path = self.output_dir / f"{self.filename_prefix}_{safe_date}.xlsx"

        df = self._build_dataframe(valid_items)
        df = self._sort_dataframe(df)

        with pd.ExcelWriter(file_path, engine="openpyxl") as writer:
            df.to_excel(writer, index=False, sheet_name=self.SHEET_NAME)
            ws = writer.sheets[self.SHEET_NAME]

            self._freeze_header(ws)
            self._style_header(ws)
            self._apply_wrap_text(ws)
            self._adjust_column_width(ws, df)
            self._style_hyperlinks(ws)
            self._fill_rows_by_category(ws)
            self._apply_borders(ws)

        return file_path

    def _build_dataframe(self, items: List[NewsItem]) -> pd.DataFrame:
        rows = []
        for item in items:
            rows.append({
                "资讯日期": item.event_date or "",
                "一级类别": item.category_level_1 or "",
                "关联企业": "；".join(item.related_companies) if item.related_companies else "",
                "标题": item.title_zh or item.title or "",
                "摘要": item.summary_zh or item.summary or "",
                "来源名称": item.source_name or "",
                "发布时间": item.format_dt(item.publish_time),
                "来源链接": item.source_url or "",
                "全球动态机构归属": item.global_region_label or "",
            })

        df = pd.DataFrame(rows)
        if df.empty:
            df = pd.DataFrame(columns=[
                "资讯日期", "一级类别", "关联企业", "标题", "摘要", "来源名称", "发布时间", "来源链接"
            ])
        return df

    @staticmethod
    def _sort_dataframe(df: pd.DataFrame) -> pd.DataFrame:
        if df.empty:
            return df

        df = df.copy()

        def level1_sort(row):
            level1 = str(row.get("一级类别", "")).strip()
            region = str(row.get("全球动态机构归属", "")).strip()

            if level1 == "名单企业动态":
                return 0
            if level1 == "全球AI产业动态" and region == "国内":
                return 1
            if level1 == "全球AI产业动态" and region == "国际":
                return 2
            if level1 == "全球AI产业动态":
                return 3
            return 9

        df["_level1_sort"] = df.apply(level1_sort, axis=1)
        df["_publish_sort"] = pd.to_datetime(df["发布时间"], errors="coerce")

        df = df.sort_values(
            by=["_level1_sort", "_publish_sort", "标题"],
            ascending=[True, False, True],
            na_position="last",
        ).drop(columns=["_level1_sort", "_publish_sort", "全球动态机构归属"])

        df = df.reset_index(drop=True)
        return df

    @staticmethod
    def _freeze_header(ws) -> None:
        ws.freeze_panes = "A2"

    @staticmethod
    def _style_header(ws) -> None:
        header_fill = PatternFill(fill_type="solid", fgColor="5B9BD5")
        header_font = Font(bold=True, color="FFFFFF")

        for cell in ws[1]:
            cell.fill = header_fill
            cell.font = header_font
            cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)

    @staticmethod
    def _apply_wrap_text(ws) -> None:
        for row in ws.iter_rows(min_row=2):
            for cell in row:
                cell.alignment = Alignment(vertical="top", wrap_text=True)

    @staticmethod
    def _adjust_column_width(ws, df: pd.DataFrame) -> None:
        for idx, col in enumerate(df.columns, start=1):
            max_len = len(str(col))
            if not df.empty:
                series_max = df[col].astype(str).map(len).max()
                max_len = max(max_len, int(series_max))

            if col == "标题":
                width = min(max(max_len + 2, 18), 45)
            elif col == "摘要":
                width = min(max(max_len + 2, 20), 60)
            elif col == "来源链接":
                width = 50
            else:
                width = min(max_len + 2, 24)

            ws.column_dimensions[get_column_letter(idx)].width = width

    @staticmethod
    def _style_hyperlinks(ws) -> None:
        header_map = {cell.value: idx for idx, cell in enumerate(ws[1], start=1)}
        link_col_idx = header_map.get("来源链接")
        if not link_col_idx:
            return

        for row in range(2, ws.max_row + 1):
            cell = ws.cell(row=row, column=link_col_idx)
            value = str(cell.value or "").strip()
            if value.startswith("http://") or value.startswith("https://"):
                cell.hyperlink = value
                cell.style = "Hyperlink"

    @staticmethod
    def _fill_rows_by_category(ws) -> None:
        header_map = {cell.value: idx for idx, cell in enumerate(ws[1], start=1)}
        level1_col_idx = header_map.get("一级类别")
        if not level1_col_idx:
            return

        company_fill = PatternFill(fill_type="solid", fgColor="DDEBF7")  # 浅蓝
        global_fill = PatternFill(fill_type="solid", fgColor="E2F0D9")   # 浅绿

        for row in range(2, ws.max_row + 1):
            level1_value = str(ws.cell(row=row, column=level1_col_idx).value or "").strip()

            if level1_value == "名单企业动态":
                fill = company_fill
            elif level1_value == "全球AI产业动态":
                fill = global_fill
            else:
                continue

            for col in range(1, ws.max_column + 1):
                ws.cell(row=row, column=col).fill = fill

    @staticmethod
    def _apply_borders(ws) -> None:
        thin = Side(style="thin", color="000000")
        border = Border(left=thin, right=thin, top=thin, bottom=thin)

        for row in ws.iter_rows():
            for cell in row:
                cell.border = border

path: app/exporters/word_exporter.py
content: from __future__ import annotations

from pathlib import Path
from typing import Dict, List, Optional

from docx import Document
from docx.enum.text import WD_ALIGN_PARAGRAPH, WD_LINE_SPACING
from docx.oxml import OxmlElement
from docx.oxml.ns import qn
from docx.shared import Mm, Pt


class WordExporter:
    """
    Word 导出器（按排版规范适配版）：
    只保留两大部分，且顺序固定：
    1. 名单企业动态
    2. 全球AI产业动态
    """

    def __init__(self, output_dir: str, filename_prefix: str = "AI资讯简报") -> None:
        self.output_dir = Path(output_dir)
        self.output_dir.mkdir(parents=True, exist_ok=True)
        self.filename_prefix = filename_prefix

    def export(self, briefing_data: Dict, run_date: str, filename: Optional[str] = None) -> Path:
        if filename:
            file_path = self.output_dir / filename
        else:
            safe_date = run_date.replace("-", "")
            file_path = self.output_dir / f"{self.filename_prefix}_{safe_date}.docx"

        doc = Document()
        self._set_page_layout(doc)
        self._set_default_style(doc)
        self._add_page_number(doc)

        self._add_title(doc, briefing_data)
        self._add_section(doc, "一、“一带”AI领军企业动态", briefing_data.get("company_items", []))
        self._add_section(doc, "二、全球AI产业动态", briefing_data.get("global_items", []))

        doc.save(file_path)
        return file_path

    @staticmethod
    def _set_page_layout(doc: Document) -> None:
        section = doc.sections[0]
        section.top_margin = Mm(37)
        section.bottom_margin = Mm(35)
        section.left_margin = Mm(28)
        section.right_margin = Mm(26)

    @staticmethod
    def _set_default_style(doc: Document) -> None:
        style = doc.styles["Normal"]
        style.font.name = "仿宋_GB2312"
        style._element.rPr.rFonts.set(qn("w:eastAsia"), "仿宋_GB2312")
        style.font.size = Pt(16)

        p_format = style.paragraph_format
        p_format.first_line_indent = Pt(32)
        p_format.space_before = Pt(0)
        p_format.space_after = Pt(0)
        p_format.line_spacing_rule = WD_LINE_SPACING.EXACTLY
        p_format.line_spacing = Pt(28)

    def _add_title(self, doc: Document, briefing_data: Dict) -> None:
        title = briefing_data.get("title", "每日AI产业资讯简报")
        run_date = briefing_data.get("run_date", "")
        time_window = briefing_data.get("time_window", "")

        #doc.add_paragraph("")
        #doc.add_paragraph("")

        p = doc.add_paragraph()
        p.alignment = WD_ALIGN_PARAGRAPH.CENTER
        p.paragraph_format.first_line_indent = Pt(0)
        p.paragraph_format.space_before = Pt(0)
        p.paragraph_format.space_after = Pt(0)
        p.paragraph_format.line_spacing_rule = WD_LINE_SPACING.EXACTLY
        p.paragraph_format.line_spacing = Pt(28)

        run = p.add_run(title)
        run.bold = False
        run.font.name = "方正小标宋简体"
        run._element.rPr.rFonts.set(qn("w:eastAsia"), "方正小标宋简体")
        run.font.size = Pt(22)

        if run_date:
            p2 = doc.add_paragraph()
            p2.alignment = WD_ALIGN_PARAGRAPH.CENTER
            p.paragraph_format.first_line_indent = Pt(0)
            p.paragraph_format.space_before = Pt(0)
            p.paragraph_format.space_after = Pt(0)
            p.paragraph_format.line_spacing_rule = WD_LINE_SPACING.EXACTLY
            p.paragraph_format.line_spacing = Pt(28)

            r2 = p2.add_run(f"日期：{run_date}")
            r2.font.name = "仿宋_GB2312"
            r2._element.rPr.rFonts.set(qn("w:eastAsia"), "仿宋_GB2312")
            r2.font.size = Pt(16)

        if time_window:
            p3 = doc.add_paragraph()
            p3.alignment = WD_ALIGN_PARAGRAPH.CENTER
            p.paragraph_format.first_line_indent = Pt(0)
            p.paragraph_format.space_before = Pt(0)
            p.paragraph_format.space_after = Pt(0)
            p.paragraph_format.line_spacing_rule = WD_LINE_SPACING.EXACTLY
            p.paragraph_format.line_spacing = Pt(28)

            r3 = p3.add_run(f"统计口径：{time_window}")
            r3.font.name = "仿宋_GB2312"
            r3._element.rPr.rFonts.set(qn("w:eastAsia"), "仿宋_GB2312")
            r3.font.size = Pt(16)

    def _add_section(self, doc: Document, section_title: str, items: List[Dict]) -> None:
        p = doc.add_paragraph()
        r = p.add_run(section_title)
        r.bold = False
        r.font.name = "黑体"
        r._element.rPr.rFonts.set(qn("w:eastAsia"), "黑体")
        r.font.size = Pt(16)

        if not items:
            self._add_body_paragraph(doc, "暂无相关资讯。")
            return

        for idx, item in enumerate(items, start=1):
            title = item.get("title", "")
            summary = item.get("summary", "")
            source_name = item.get("source_name", "")
            source_url = item.get("source_url", "")

            p_title = doc.add_paragraph()
            r1 = p_title.add_run(f"{idx}. ")
            r1.bold = True
            r1.font.name = "仿宋_GB2312"
            r1._element.rPr.rFonts.set(qn("w:eastAsia"), "仿宋_GB2312")
            r1.font.size = Pt(16)

            r2 = p_title.add_run(title)
            r2.bold = True
            r2.font.name = "仿宋_GB2312"
            r2._element.rPr.rFonts.set(qn("w:eastAsia"), "仿宋_GB2312")
            r2.font.size = Pt(16)

            if summary:
                self._add_body_paragraph(doc, summary)

            if source_name:
                self._add_body_paragraph(doc, f"来源：{source_name}")

    @staticmethod
    def _add_body_paragraph(doc: Document, text: str) -> None:
        p = doc.add_paragraph()
        p.alignment = WD_ALIGN_PARAGRAPH.JUSTIFY

        fmt = p.paragraph_format
        fmt.first_line_indent = Pt(32)
        fmt.space_before = Pt(0)
        fmt.space_after = Pt(0)
        fmt.line_spacing_rule = WD_LINE_SPACING.EXACTLY
        fmt.line_spacing = Pt(28)

        run = p.add_run(text)
        run.font.name = "仿宋_GB2312"
        run._element.rPr.rFonts.set(qn("w:eastAsia"), "仿宋_GB2312")
        run.font.size = Pt(16)

    def _add_page_number(self, doc: Document) -> None:
        """
        页脚居中页码，样式为：-1-
        """
        section = doc.sections[0]
        footer = section.footer

        p = footer.paragraphs[0] if footer.paragraphs else footer.add_paragraph()
        p.alignment = WD_ALIGN_PARAGRAPH.CENTER
        p.clear()

        run_prefix = p.add_run("-")
        run_prefix.font.name = "Times New Roman"
        run_prefix._element.rPr.rFonts.set(qn("w:eastAsia"), "Times New Roman")
        run_prefix.font.size = Pt(14)

        self._add_page_field(p)

        run_suffix = p.add_run("-")
        run_suffix.font.name = "Times New Roman"
        run_suffix._element.rPr.rFonts.set(qn("w:eastAsia"), "Times New Roman")
        run_suffix.font.size = Pt(14)

    @staticmethod
    def _add_page_field(paragraph) -> None:
        run = paragraph.add_run()
        run.font.name = "Times New Roman"
        run._element.rPr.rFonts.set(qn("w:eastAsia"), "Times New Roman")
        run.font.size = Pt(14)

        fld_char_begin = OxmlElement("w:fldChar")
        fld_char_begin.set(qn("w:fldCharType"), "begin")

        instr_text = OxmlElement("w:instrText")
        instr_text.set(qn("xml:space"), "preserve")
        instr_text.text = " PAGE "

        fld_char_separate = OxmlElement("w:fldChar")
        fld_char_separate.set(qn("w:fldCharType"), "separate")

        text = OxmlElement("w:t")
        text.text = "1"

        fld_char_end = OxmlElement("w:fldChar")
        fld_char_end.set(qn("w:fldCharType"), "end")

        run._r.append(fld_char_begin)
        run._r.append(instr_text)
        run._r.append(fld_char_separate)
        run._r.append(text)
        run._r.append(fld_char_end)

path: app/loaders/company_loader.py
content: from __future__ import annotations

from pathlib import Path
from typing import List

import pandas as pd

from app.models.company import Company


class CompanyLoader:
    """
    企业名单加载器。
    """

    def __init__(self, file_path: str, sheet_name: str = "company_list") -> None:
        self.file_path = Path(file_path)
        self.sheet_name = sheet_name

        if not self.file_path.exists():
            raise FileNotFoundError(f"企业名单文件不存在: {self.file_path}")

    def load(self) -> List[Company]:
        """
        读取 Excel 并转换为 Company 列表。
        """
        df = pd.read_excel(self.file_path, sheet_name=self.sheet_name)
        df = df.fillna("")

        companies: List[Company] = []

        for _, row in df.iterrows():
            company_cn = self._get_value(row, ["企业中文名", "公司中文名", "中文名"])
            if not company_cn:
                continue

            company = Company(
                company_cn=company_cn,
                company_en=self._get_value(row, ["企业英文名", "公司英文名", "英文名"]),
                short_name=self._get_value(row, ["企业简称", "简称"]),
                aliases=self._parse_aliases(self._get_value(row, ["常见别名", "别名"])),
                parent_company=self._get_value(row, ["母公司"]),
                country_region=self._get_value(row, ["国家/地区", "国家地区"]),
                is_key=self._parse_bool(self._get_value(row, ["重点关注", "是否重点关注"])),
                remarks=self._get_value(row, ["备注"]),
            )
            companies.append(company)

        return companies

    @staticmethod
    def _get_value(row: pd.Series, candidate_columns: List[str]) -> str:
        """
        按候选列名依次读取。
        """
        for col in candidate_columns:
            if col in row.index:
                value = str(row[col]).strip()
                if value and value.lower() != "nan":
                    return value
        return ""

    @staticmethod
    def _parse_aliases(alias_text: str) -> List[str]:
        """
        将别名字段解析为列表。
        支持分隔符：; ； , ， / |
        """
        if not alias_text:
            return []

        separators = [";", "；", ",", "，", "/", "|"]
        text = alias_text
        for sep in separators[1:]:
            text = text.replace(sep, separators[0])

        aliases = [item.strip() for item in text.split(separators[0]) if item.strip()]
        return aliases

    @staticmethod
    def _parse_bool(value: str) -> bool:
        """
        将重点关注字段转为布尔值。
        """
        if not value:
            return False

        normalized = value.strip().lower()
        true_values = {"是", "true", "yes", "y", "1"}
        return normalized in true_values

path: app/loaders/source_loader.py
content: from __future__ import annotations

from pathlib import Path
from typing import List

import pandas as pd

from app.models.source import Source


class SourceLoader:
    """
    资讯来源配置加载器。
    """

    def __init__(self, file_path: str, sheet_name: str = "sources") -> None:
        self.file_path = Path(file_path)
        self.sheet_name = sheet_name

        if not self.file_path.exists():
            raise FileNotFoundError(f"来源配置文件不存在: {self.file_path}")

    def load(self, enabled_only: bool = True) -> List[Source]:
        """
        读取 Excel 并转换为 Source 列表。
        """
        df = pd.read_excel(self.file_path, sheet_name=self.sheet_name)
        df = df.fillna("")

        sources: List[Source] = []

        for _, row in df.iterrows():
            source_name = self._get_value(row, ["来源名称", "source_name"])
            url = self._get_value(row, ["网址/入口", "网址", "url"])

            if not source_name or not url:
                continue

            source = Source(
                source_name=source_name,
                source_type=self._get_value(row, ["来源类型", "source_type"]) or "other",
                url=url,
                coverage=self._get_value(row, ["覆盖范围", "coverage"]),
                priority=self._parse_int(self._get_value(row, ["优先级", "priority"]), default=5),
                enabled=self._parse_bool(self._get_value(row, ["是否启用", "enabled"]), default=True),
                fetch_method=self._get_value(row, ["抓取方式", "fetch_method"]) or "webpage",
                region_scope=self._get_value(row, ["地区", "region_scope"]),
            )
            sources.append(source)

        if enabled_only:
            sources = [s for s in sources if s.enabled]

        sources.sort(key=lambda x: x.priority)
        return sources

    @staticmethod
    def _get_value(row: pd.Series, candidate_columns: List[str]) -> str:
        for col in candidate_columns:
            if col in row.index:
                value = str(row[col]).strip()
                if value and value.lower() != "nan":
                    return value
        return ""

    @staticmethod
    def _parse_bool(value: str, default: bool = False) -> bool:
        if not value:
            return default

        normalized = value.strip().lower()
        true_values = {"是", "true", "yes", "y", "1"}
        false_values = {"否", "false", "no", "n", "0"}

        if normalized in true_values:
            return True
        if normalized in false_values:
            return False
        return default

    @staticmethod
    def _parse_int(value: str, default: int = 0) -> int:
        try:
            return int(float(str(value).strip()))
        except Exception:
            return default

path: app/models/company.py
content: from __future__ import annotations

from typing import List, Set

from pydantic import BaseModel, Field, field_validator


class Company(BaseModel):
    """
    企业基础信息模型。
    """

    company_cn: str = Field(..., description="企业中文名")
    company_en: str = Field(default="", description="企业英文名")
    short_name: str = Field(default="", description="企业简称")
    aliases: List[str] = Field(default_factory=list, description="企业别名列表")
    parent_company: str = Field(default="", description="母公司")
    country_region: str = Field(default="", description="国家/地区")
    is_key: bool = Field(default=True, description="是否重点关注")
    remarks: str = Field(default="", description="备注")

    @field_validator("company_cn")
    @classmethod
    def validate_company_cn(cls, value: str) -> str:
        value = value.strip()
        if not value:
            raise ValueError("company_cn 不能为空")
        return value

    def all_names(self) -> List[str]:
        """
        返回企业所有名称字段。
        """
        values = [
            self.company_cn,
            self.company_en,
            self.short_name,
            self.parent_company,
            *self.aliases,
        ]
        return [v.strip() for v in values if isinstance(v, str) and v.strip()]

    def keyword_set(self) -> Set[str]:
        """
        返回企业匹配检索词集合。
        """
        return set(self.all_names())

path: app/models/news_item.py
content: from __future__ import annotations

from datetime import datetime
from typing import List, Optional

from pydantic import BaseModel, Field, field_validator


class NewsItem(BaseModel):
    """
    表示一条经过抓取、清洗、翻译、筛选后的资讯记录。
    """

    # 基础标识
    item_id: Optional[str] = Field(default=None, description="资讯唯一ID，可由程序生成")
    run_date: Optional[str] = Field(default=None, description="本次任务执行日期，格式 YYYY-MM-DD")

    # 时间字段
    event_date: Optional[str] = Field(default=None, description="资讯归属日期，格式 YYYY-MM-DD")
    publish_time: Optional[datetime] = Field(default=None, description="来源页面发布时间")
    crawl_time: Optional[datetime] = Field(default=None, description="程序抓取时间")

    # 时间窗口记录
    window_start: Optional[datetime] = Field(default=None, description="统计窗口起始时间")
    window_end: Optional[datetime] = Field(default=None, description="统计窗口结束时间")

    # 分类字段
    category_level_1: Optional[str] = Field(default=None, description="一级类别")
    category_level_2: Optional[str] = Field(default=None, description="二级类别")
    global_region_label: Optional[str] = Field(default=None, description="全球AI产业动态机构归属:国内/国际/不适用")
    global_key_event_flag: Optional[str] = Field(default=None, description="全球AI产业动态是否为重点事件：是/否/不适用")
    global_key_event_score: Optional[int] = Field(default=None, description="全球AI产业动态重点事件评分，0-100")
    global_key_event_reason: Optional[str] = Field(default=None, description="全球AI产业动态重点事件判断理由")
    
    # 关联信息
    related_companies: List[str] = Field(default_factory=list, description="关联企业列表")
    country_region: Optional[str] = Field(default=None, description="国家或地区")

    # 主内容字段（当前主流程默认使用中文字段覆盖写入）
    title: str = Field(..., description="当前主标题（建议为中文）")
    summary: Optional[str] = Field(default=None, description="当前主摘要（建议为中文）")
    key_points: Optional[str] = Field(default=None, description="事件要点")
    raw_content: Optional[str] = Field(default=None, description="原始正文")
    cleaned_content: Optional[str] = Field(default=None, description="清洗后的正文（建议为中文）")

    # 原文字段
    original_title: Optional[str] = Field(default=None, description="原始标题")
    original_summary: Optional[str] = Field(default=None, description="原始摘要")
    original_cleaned_content: Optional[str] = Field(default=None, description="原始清洗后正文")
    original_language: Optional[str] = Field(default=None, description="原始语言，如 zh/en")

    # 中文标准字段
    title_zh: Optional[str] = Field(default=None, description="中文标题")
    summary_zh: Optional[str] = Field(default=None, description="中文摘要")
    cleaned_content_zh: Optional[str] = Field(default=None, description="中文正文")

    # 翻译标记
    is_machine_translated: bool = Field(default=False, description="是否机器翻译")
    translation_backend: Optional[str] = Field(default=None, description="翻译后端标识")

    # 来源字段
    source_name: str = Field(..., description="来源名称")
    source_url: str = Field(..., description="来源链接")
    source_type: Optional[str] = Field(default=None, description="来源类型，如 official/media/government")
    source_region_scope: Optional[str] = Field(default=None, description="来源地区范围，如 国内/国际")
    is_official: bool = Field(default=False, description="是否官方来源")

    # 处理标记
    importance: Optional[str] = Field(default=None, description="重要性等级：高/中/低")
    dedup_status: Optional[str] = Field(default="unique", description="去重状态")
    entry_status: Optional[str] = Field(default="pending", description="入表状态：pending/included/manual_review/excluded")
    language: Optional[str] = Field(default=None, description="兼容旧字段")
    remarks: Optional[str] = Field(default=None, description="备注")

    # 技术字段
    normalized_title: Optional[str] = Field(default=None, description="规范化标题（建议为中文）")
    content_hash: Optional[str] = Field(default=None, description="正文哈希（建议基于中文字段）")
    matched_keywords: List[str] = Field(default_factory=list, description="命中的关键词")
    extra_data: dict = Field(default_factory=dict, description="扩展字段")

    @field_validator("title")
    @classmethod
    def validate_title(cls, value: str) -> str:
        value = value.strip()
        if not value:
            raise ValueError("title 不能为空")
        return value

    @field_validator("source_name")
    @classmethod
    def validate_source_name(cls, value: str) -> str:
        value = value.strip()
        if not value:
            raise ValueError("source_name 不能为空")
        return value

    @field_validator("source_url")
    @classmethod
    def validate_source_url(cls, value: str) -> str:
        value = value.strip()
        if not value:
            raise ValueError("source_url 不能为空")
        if not (value.startswith("http://") or value.startswith("https://")):
            raise ValueError("source_url 必须以 http:// 或 https:// 开头")
        return value

    @field_validator("importance")
    @classmethod
    def validate_importance(cls, value: Optional[str]) -> Optional[str]:
        if value is None:
            return value
        allowed = {"高", "中", "低"}
        if value not in allowed:
            raise ValueError(f"importance 必须属于 {allowed}")
        return value

    @field_validator("entry_status")
    @classmethod
    def validate_entry_status(cls, value: Optional[str]) -> Optional[str]:
        if value is None:
            return value
        allowed = {"pending", "included", "manual_review", "excluded"}
        if value not in allowed:
            raise ValueError(f"entry_status 必须属于 {allowed}")
        return value

    def to_excel_row(self) -> dict:
        """
        转换为 Excel 输出所需的一行字典。
        默认优先展示中文字段。
        """
        display_title = self.title_zh or self.title or ""
        display_summary = self.summary_zh or self.summary or ""
        display_content = self.cleaned_content_zh or self.cleaned_content or ""
        display_key_points = self.key_points or display_summary

        return {
            "资讯日期": self.event_date or "",
            "抓取日期": self.run_date or "",
            "时间窗口": self.get_window_text(),
            "一级类别": self.category_level_1 or "",
            "二级类别": self.category_level_2 or "",
            "关联企业": "；".join(self.related_companies) if self.related_companies else "",
            "国家/地区": self.country_region or "",
            "标题": display_title,
            "摘要": display_summary,
            "事件要点": display_key_points,
            "来源名称": self.source_name,
            "来源链接": self.source_url,
            "发布时间": self.format_dt(self.publish_time),
            "抓取时间": self.format_dt(self.crawl_time),
            "是否官方来源": "是" if self.is_official else "否",
            "重要性等级": self.importance or "",
            "去重标记": self.dedup_status or "",
            "入表状态": self.entry_status or "",
            "备注": self.remarks or "",
            "原始语言": self.original_language or "",
            "原始标题": self.original_title or "",
            "中文标题": self.title_zh or "",
            "中文摘要": self.summary_zh or "",
            "是否机器翻译": "是" if self.is_machine_translated else "否",
            "翻译后端": self.translation_backend or "",
        }

    def get_window_text(self) -> str:
        start_text = self.format_dt(self.window_start)
        end_text = self.format_dt(self.window_end)
        if start_text and end_text:
            return f"{start_text} ~ {end_text}"
        return ""

    @staticmethod
    def format_dt(value: Optional[datetime]) -> str:
        if value is None:
            return ""
        return value.strftime("%Y-%m-%d %H:%M:%S")

path: app/models/source.py
content: from __future__ import annotations

from pydantic import BaseModel, Field, field_validator


class Source(BaseModel):
    """
    资讯来源配置模型。
    """

    source_name: str = Field(..., description="来源名称")
    source_type: str = Field(default="other", description="来源类型，如 official/media/government/rss")
    url: str = Field(..., description="来源网址或入口")
    coverage: str = Field(default="", description="覆盖范围")
    priority: int = Field(default=5, description="优先级，数字越小越高")
    enabled: bool = Field(default=True, description="是否启用")
    fetch_method: str = Field(default="webpage", description="抓取方式，如 rss/webpage/api")
    region_scope: str = Field(default="", description="来源地区范围，如 国内/国际")

    @field_validator("source_name")
    @classmethod
    def validate_source_name(cls, value: str) -> str:
        value = value.strip()
        if not value:
            raise ValueError("source_name 不能为空")
        return value

    @field_validator("url")
    @classmethod
    def validate_url(cls, value: str) -> str:
        value = value.strip()
        if not value:
            raise ValueError("url 不能为空")
        return value

    @field_validator("priority")
    @classmethod
    def validate_priority(cls, value: int) -> int:
        if value < 1:
            return 1
        return value

path: app/parsers/content_cleaner.py
content: from __future__ import annotations

import html
import re
from typing import Optional

from bs4 import BeautifulSoup


class ContentCleaner:
    """
    内容清洗器。
    适用于 RSS summary、网页正文摘要等。
    """

    def __init__(self) -> None:
        self.noise_patterns = [
            r"click here to read more",
            r"read more\.?",
            r"continue reading\.?",
            r"本文来自.*",
            r"责任编辑[:：].*",
            r"来源[:：].*",
        ]

    def clean_text(self, text: Optional[str]) -> str:
        if not text:
            return ""

        text = str(text)

        # 1. HTML实体解码
        text = html.unescape(text)

        # 2. 去HTML标签
        text = self._strip_html(text)

        # 3. 去噪音
        text = self._remove_noise(text)

        # 4. 统一空白
        text = self._normalize_whitespace(text)

        return text.strip()

    def clean_summary(self, text: Optional[str], max_length: int = 300) -> str:
        """
        清理摘要并按最大长度截断。
        """
        text = self.clean_text(text)
        if len(text) > max_length:
            return text[: max_length - 1].rstrip() + "…"
        return text

    @staticmethod
    def _strip_html(text: str) -> str:
        soup = BeautifulSoup(text, "lxml")
        return soup.get_text(separator=" ", strip=True)

    def _remove_noise(self, text: str) -> str:
        cleaned = text
        for pattern in self.noise_patterns:
            cleaned = re.sub(pattern, "", cleaned, flags=re.IGNORECASE)
        return cleaned

    @staticmethod
    def _normalize_whitespace(text: str) -> str:
        # 把各种空白统一为单个空格
        text = re.sub(r"\s+", " ", text)
        # 去掉中英文标点前多余空格
        text = re.sub(r"\s+([,.;:!?，。；：！？])", r"\1", text)
        return text

path: app/parsers/html_extractors.py
content: from __future__ import annotations

import re
from dataclasses import dataclass
from datetime import datetime
from typing import Optional
from zoneinfo import ZoneInfo

from bs4 import BeautifulSoup


SHANGHAI_TZ = ZoneInfo("Asia/Shanghai")


@dataclass
class ExtractedArticle:
    title: str
    publish_time: Optional[datetime]
    content_text: str


def clean_text(text: str) -> str:
    text = re.sub(r"\s+", " ", (text or "").strip())
    return text.strip()


def parse_datetime_cn(text: str) -> Optional[datetime]:
    """
    尝试解析中文常见日期格式：
    2026-03-03
    2026/03/03
    2026年03月03日
    可按需继续扩展
    """
    if not text:
        return None
    t = text.strip()

    patterns = [
        r"(\d{4})[-/\.](\d{1,2})[-/\.](\d{1,2})",
        r"(\d{4})年(\d{1,2})月(\d{1,2})日",
    ]
    for p in patterns:
        m = re.search(p, t)
        if m:
            y, mo, d = int(m.group(1)), int(m.group(2)), int(m.group(3))
            return datetime(y, mo, d, 0, 0, 0, tzinfo=SHANGHAI_TZ)
    return None


def extract_article_by_selectors(
    html: str,
    title_selector: str | None,
    date_selector: str | None,
    content_selector: str | None,
) -> ExtractedArticle:
    soup = BeautifulSoup(html, "lxml")

    # 标题
    title = ""
    if title_selector:
        node = soup.select_one(title_selector)
        title = clean_text(node.get_text(" ", strip=True) if node else "")
    if not title:
        # fallback
        h1 = soup.find("h1")
        title = clean_text(h1.get_text(" ", strip=True) if h1 else "")

    # 时间
    publish_time = None
    date_text = ""
    if date_selector:
        node = soup.select_one(date_selector)
        date_text = clean_text(node.get_text(" ", strip=True) if node else "")
    if date_text:
        publish_time = parse_datetime_cn(date_text)

    # 正文
    content_text = ""
    if content_selector:
        node = soup.select_one(content_selector)
        content_text = clean_text(node.get_text("\n", strip=True) if node else "")
    if not content_text:
        # fallback：取 body 里较大段落（粗略）
        body = soup.find("body")
        content_text = clean_text(body.get_text("\n", strip=True) if body else "")

    return ExtractedArticle(
        title=title or "Untitled",
        publish_time=publish_time,
        content_text=content_text,
    )

path: app/repositories/news_repository.py
content: from __future__ import annotations

from typing import List, Optional

from rapidfuzz import fuzz

from app.core.database import DatabaseManager
from app.models.news_item import NewsItem
from app.utils.text_utils import normalize_title
from app.utils.url_utils import normalize_url


class NewsRepository:
    """
    资讯历史库访问层（增强版：支持数据库级近似标题查重）。
    """

    def __init__(self, db_manager: DatabaseManager) -> None:
        self.db_manager = db_manager

    def exists_by_url(self, source_url: str) -> bool:
        normalized_url = normalize_url(source_url)
        if not normalized_url:
            return False

        with self.db_manager.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                "SELECT 1 FROM news_history WHERE source_url = ? LIMIT 1",
                (normalized_url,),
            )
            return cursor.fetchone() is not None

    def exists_by_hash(self, content_hash: str | None) -> bool:
        if not content_hash:
            return False

        with self.db_manager.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                "SELECT 1 FROM news_history WHERE content_hash = ? LIMIT 1",
                (content_hash,),
            )
            return cursor.fetchone() is not None

    def exists_by_normalized_title(self, normalized_title: str | None) -> bool:
        if not normalized_title:
            return False

        with self.db_manager.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                "SELECT 1 FROM news_history WHERE normalized_title = ? LIMIT 1",
                (normalized_title,),
            )
            return cursor.fetchone() is not None

    def find_similar_title_in_db(
        self,
        normalized_title: str | None,
        threshold: int = 92,
        recent_days: int = 30,
        category_level_2: str | None = None,
    ) -> Optional[dict]:
        """
        在数据库中查找近似标题。
        返回匹配到的历史记录 dict；找不到则返回 None。
        """
        if not normalized_title:
            return None

        with self.db_manager.get_connection() as conn:
            cursor = conn.cursor()

            if category_level_2:
                cursor.execute(
                    """
                    SELECT id, title, normalized_title, source_url, category_level_2, created_at
                    FROM news_history
                    WHERE created_at >= datetime('now', ?)
                      AND category_level_2 = ?
                      AND normalized_title IS NOT NULL
                      AND normalized_title != ''
                    ORDER BY created_at DESC
                    """,
                    (f"-{recent_days} days", category_level_2),
                )
            else:
                cursor.execute(
                    """
                    SELECT id, title, normalized_title, source_url, category_level_2, created_at
                    FROM news_history
                    WHERE created_at >= datetime('now', ?)
                      AND normalized_title IS NOT NULL
                      AND normalized_title != ''
                    ORDER BY created_at DESC
                    """,
                    (f"-{recent_days} days",),
                )

            rows = cursor.fetchall()

        best_match = None
        best_score = 0

        for row in rows:
            hist_title = row["normalized_title"] or ""
            if not hist_title:
                continue

            score = fuzz.ratio(normalized_title, hist_title)
            if score >= threshold and score > best_score:
                best_score = score
                best_match = {
                    "id": row["id"],
                    "title": row["title"],
                    "normalized_title": row["normalized_title"],
                    "source_url": row["source_url"],
                    "category_level_2": row["category_level_2"],
                    "created_at": row["created_at"],
                    "score": score,
                }

        return best_match

    def save_item(self, item: NewsItem) -> None:
        normalized_url = normalize_url(item.source_url)
        normalized_title = item.normalized_title or normalize_title(item.title)

        with self.db_manager.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                """
                INSERT OR IGNORE INTO news_history (
                    title,
                    normalized_title,
                    source_name,
                    source_url,
                    publish_time,
                    related_companies,
                    category_level_1,
                    category_level_2,
                    content_hash,
                    importance
                )
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                """,
                (
                    item.title,
                    normalized_title,
                    item.source_name,
                    normalized_url,
                    item.format_dt(item.publish_time),
                    "；".join(item.related_companies) if item.related_companies else "",
                    item.category_level_1,
                    item.category_level_2,
                    item.content_hash,
                    item.importance,
                ),
            )
            conn.commit()

    def save_items(self, items: List[NewsItem]) -> None:
        for item in items:
            self.save_item_prefer_earliest(item)

    def filter_existing(self, items: List[NewsItem]) -> List[NewsItem]:
        """
        兼容旧接口：只按 URL 过滤。
        """
        result: List[NewsItem] = []

        for item in items:
            if self.exists_by_url(item.source_url):
                item.entry_status = "excluded"
                item.dedup_status = "duplicate_url"
                item.remarks = self._append_remark(item.remarks, "数据库已存在同URL记录")
                continue
            result.append(item)

        return result

    def filter_existing_advanced(
        self,
        items: List[NewsItem],
        similar_title_threshold: int = 92,
        similar_title_recent_days: int = 30,
    ) -> List[NewsItem]:
        """
        按 URL + content_hash + normalized_title + 近似标题 多条件过滤历史重复。
        """
        result: List[NewsItem] = []

        for item in items:
            normalized_title = item.normalized_title or normalize_title(item.title)
            item.normalized_title = normalized_title

            if self.exists_by_url(item.source_url):
                item.entry_status = "excluded"
                item.dedup_status = "duplicate_db_url"
                item.remarks = self._append_remark(item.remarks, "数据库已存在同URL记录")
                continue

            if self.exists_by_hash(item.content_hash):
                item.entry_status = "excluded"
                item.dedup_status = "duplicate_db_hash"
                item.remarks = self._append_remark(item.remarks, "数据库已存在同内容哈希记录")
                continue

            if self.exists_by_normalized_title(normalized_title):
                item.entry_status = "excluded"
                item.dedup_status = "duplicate_db_title"
                item.remarks = self._append_remark(item.remarks, "数据库已存在同规范化标题记录")
                continue

            similar_record = self.find_similar_title_in_db(
                normalized_title=normalized_title,
                threshold=similar_title_threshold,
                recent_days=similar_title_recent_days,
                category_level_2=item.category_level_2,
            )
            if similar_record:
                item.entry_status = "excluded"
                item.dedup_status = "duplicate_db_similar_title"
                item.remarks = self._append_remark(
                    item.remarks,
                    f"数据库近似标题重复（相似度={similar_record['score']}，参考标题：{similar_record['title'][:80]}）"
                )
                continue

            result.append(item)

        return result

    def save_item_prefer_earliest(self, item: NewsItem) -> None:
        normalized_url = normalize_url(item.source_url)
        normalized_title = item.normalized_title or normalize_title(item.title)

        with self.db_manager.get_connection() as conn:
            cursor = conn.cursor()

            # 优先按 source_url 查
            existing = None
            if normalized_url:
                cursor.execute(
                    """
                    SELECT id, publish_time
                    FROM news_history
                    WHERE source_url = ?
                    LIMIT 1
                    """,
                    (normalized_url,),
                )
                existing = cursor.fetchone()

            if existing:
                existing_publish_time = existing["publish_time"] or ""
                current_publish_time = item.format_dt(item.publish_time)

                # 当前更早，则更新
                if current_publish_time and (not existing_publish_time or current_publish_time < existing_publish_time):
                    cursor.execute(
                        """
                        UPDATE news_history
                        SET title = ?, normalized_title = ?, source_name = ?, source_url = ?,
                            publish_time = ?, related_companies = ?, category_level_1 = ?,
                            category_level_2 = ?, content_hash = ?, importance = ?
                        WHERE id = ?
                        """,
                        (
                            item.title,
                            normalized_title,
                            item.source_name,
                            normalized_url,
                            current_publish_time,
                            "；".join(item.related_companies) if item.related_companies else "",
                            item.category_level_1,
                            item.category_level_2,
                            item.content_hash,
                            item.importance,
                            existing["id"],
                        ),
                    )
                conn.commit()
                return

            # 不存在则插入
            cursor.execute(
                """
                INSERT INTO news_history (
                    title,
                    normalized_title,
                    source_name,
                    source_url,
                    publish_time,
                    related_companies,
                    category_level_1,
                    category_level_2,
                    content_hash,
                    importance
                )
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                """,
                (
                    item.title,
                    normalized_title,
                    item.source_name,
                    normalized_url,
                    item.format_dt(item.publish_time),
                    "；".join(item.related_companies) if item.related_companies else "",
                    item.category_level_1,
                    item.category_level_2,
                    item.content_hash,
                    item.importance,
                ),
            )
            conn.commit()


    @staticmethod
    def _append_remark(existing: str | None, new_text: str) -> str:
        if not existing:
            return new_text
        return f"{existing}；{new_text}"

path: app/services/ai_summary_service.py
content: from __future__ import annotations

import json
import os
import time
from typing import List, Optional

from openai import OpenAI

from app.models.news_item import NewsItem


class AISummaryService:
    """
    使用豆包（火山方舟）对最终有效资讯进行标题与摘要标准化，并补充全球AI产业动态的归属/重点事件判断。

    建议放在：
    数据库去重后 -> 调用本服务 -> 导出 Excel / Word
    """

    def __init__(
        self,
        api_key: str | None = None,
        model: str = "doubao-seed-2-0-mini-260215",
        base_url: str = "https://ark.cn-beijing.volces.com/api/v3",
        max_retries: int = 2,
        retry_sleep_seconds: float = 1.5,
    ) -> None:
        self.[REDACTED_SECRET] or os.getenv("ARK_API_KEY")
        if not self.api_key:
            raise ValueError("未提供方舟 API Key，请配置 ARK_API_KEY 环境变量或传入 api_key")

        self.client = OpenAI(
            base_url=base_url,
            [REDACTED_SECRET],
        )
        self.model = model
        self.max_retries = max_retries
        self.retry_sleep_seconds = retry_sleep_seconds

    def process_items(self, items: List[NewsItem]) -> List[NewsItem]:
        result: List[NewsItem] = []
        for idx, item in enumerate(items, start=1):
            print(f"[AISummaryService] processing {idx}/{len(items)}")
            result.append(self.process_item(item))
        return result

    def process_item(self, item: NewsItem) -> NewsItem:
        prompt_text = self._build_input_content(item)
        if not prompt_text.strip():
            item.remarks = self._append_remark(item.remarks, "AI摘要跳过：缺少可用正文")
            self._fill_default_global_fields(item)
            return item

        data = self._call_model_with_retry(prompt_text)

        if not data:
            item.remarks = self._append_remark(item.remarks, "AI摘要失败，保留原标题摘要")
            self._fill_default_global_fields(item)
            return item

        ai_title = self._safe_str(data.get("title"))
        ai_summary = self._safe_str(data.get("summary"))
        ai_region_label = self._safe_str(data.get("global_region_label"))
        ai_key_flag = self._safe_str(data.get("global_key_event_flag"))
        ai_key_reason = self._safe_str(data.get("global_key_event_reason"))

        try:
            ai_key_score = int(data.get("global_key_event_score", 0))
        except Exception:
            ai_key_score = 0

        # 双保险：摘要最长100字
        if ai_summary and len(ai_summary) > 120:
            ai_summary = ai_summary[:120].rstrip()

        # 写回标题、摘要
        if ai_title:
            item.title_zh = ai_title
            item.title = ai_title

        if ai_summary:
            item.summary_zh = ai_summary
            item.summary = ai_summary

        # 注意：不覆盖 cleaned_content / original_cleaned_content
        # 这样能保留抓到的详细正文，供后续排查或再次处理使用

        # 写回全球AI产业动态的归属判断
        if ai_region_label in {"国内", "国际", "不适用"}:
            item.global_region_label = ai_region_label
        else:
            item.global_region_label = "不适用"

        # 写回重点事件判断
        if ai_key_flag in {"是", "否", "不适用"}:
            item.global_key_event_flag = ai_key_flag
        else:
            item.global_key_event_flag = "不适用"

        item.global_key_event_score = max(0, min(ai_key_score, 100))
        item.global_key_event_reason = ai_key_reason if ai_key_reason else "不适用"

        item.remarks = self._append_remark(item.remarks, "已使用豆包重写标题与摘要")
        return item

    def _call_model_with_retry(self, prompt_text: str) -> Optional[dict]:
        last_error: Optional[Exception] = None

        for attempt in range(self.max_retries + 1):
            try:
                response = self.client.responses.create(
                    model=self.model,
                    input=[
                        {
                            "role": "user",
                            "content": [
                                {
                                    "type": "input_text",
                                    "text": prompt_text,
                                }
                            ],
                        }
                    ],
                )

                text = getattr(response, "output_text", "") or ""
                text = text.strip()

                if text.startswith("```"):
                    text = text.strip("`")
                    text = text.replace("json", "", 1).strip()

                data = json.loads(text)

                if not isinstance(data, dict):
                    raise ValueError("模型返回不是对象")

                required_fields = {
                    "title",
                    "summary",
                    "global_region_label",
                    "global_key_event_flag",
                    "global_key_event_score",
                    "global_key_event_reason",
                }
                missing = required_fields - set(data.keys())
                if missing:
                    raise ValueError(f"模型返回缺少字段: {sorted(missing)}")

                return data

            except Exception as e:
                last_error = e
                print(f"[AISummaryService] attempt={attempt + 1} failed: {repr(e)}")
                if attempt < self.max_retries:
                    time.sleep(self.retry_sleep_seconds)

        print(f"[AISummaryService] all retries failed: {repr(last_error)}")
        return None

    @staticmethod
    def _build_input_content(item: NewsItem) -> str:
        company_text = "、".join(item.related_companies) if item.related_companies else "未识别"
        title_text = item.original_title or item.title or ""
        summary_text = item.original_summary or item.summary or ""

        # 优先选择更长、更完整的正文
        detail_candidates = [
            item.original_cleaned_content or "",
            item.raw_content or "",
            item.cleaned_content_zh or "",
            item.cleaned_content or "",
            item.summary_zh or "",
            item.summary or "",
        ]
        detail_text = max(detail_candidates, key=lambda x: len(str(x or "").strip()))
        detail_text = str(detail_text or "").strip()

        return (
            "请严格依据以下资讯内容生成结果，不得补充外部信息，不得猜测。\n"
            "请只返回 JSON，不要返回其他说明文字。\n"
            'JSON格式：{"title":"中文标题","summary":"中文摘要（目标80字左右）","global_region_label":"国内/国际/不适用","global_key_event_flag":"是/否/不适用","global_key_event_score":0,"global_key_event_reason":"简短理由"}\n'
            "要求：\n"
            "1. 优先依据“详细资讯内容”总结，不要只改写原标题或原摘要；\n"
            "2. title：中文标题，准确、简洁，只聚焦一个核心事件；\n"
            "3. summary：中文摘要，目标80个汉字左右，允许范围60-110个汉字，要把事情讲清楚，包含核心事实要素（谁/做了什么/关键结果），要把句子说全，以句号结束，禁止空泛套话（如“引发关注”“意义重大”），不要罗列无关背景；\n"
            "4. summary:中文摘要请体现本条资讯与关联企业有什么关系；\n"
            "5. global_region_label：仅当一级类别为“全球AI产业动态”时判断该资讯主体属于“国内”还是“国际”；\n"
            "6. 若主体属于中国境内机构、国内企业、国内高校、国内科研院所、国内政府部门，则标记为“国内”；\n"
            "7. 若主体属于国外企业、国外机构、国际组织、外国政府部门，则标记为“国际”；\n"
            "8. 若该条资讯不是“全球AI产业动态”，则 global_region_label 填“不适用”；\n"
            "9. 对于“全球AI产业动态”，请再从“机构全球知名度”“事件产业影响力”两个维度综合判断，是否属于重点事件；\n"
            "10. global_key_event_flag：若是全球AI产业动态中的重点事件，填“是”，否则填“否”；若不是全球AI产业动态，填“不适用”；\n"
            "11. global_key_event_score：0-100分，分数越高表示越值得列为重点事件；若不是全球AI产业动态，填0；\n"
            "12. global_key_event_reason：用一句简短中文说明判断理由；若不是全球AI产业动态，填“不适用”；\n"
            "13. 重点事件优先考虑：全球知名企业/组织/高校/院所的重要动作，或对AI产业链、资本市场、政策监管、技术路线产生明显影响的事件；\n"
            "14. 只能依据我提供的内容。\n\n"
            f"关联企业：{company_text}\n"
            f"一级类别：{item.category_level_1 or ''}\n"
            f"二级类别：{item.category_level_2 or ''}\n"
            f"原标题：{title_text}\n"
            f"原摘要：{summary_text}\n"
            f"详细资讯内容：{detail_text}"
        ).strip()

    @staticmethod
    def _fill_default_global_fields(item: NewsItem) -> None:
        if not item.global_region_label:
            item.global_region_label = "不适用"
        if not item.global_key_event_flag:
            item.global_key_event_flag = "不适用"
        if item.global_key_event_score is None:
            item.global_key_event_score = 0
        if not item.global_key_event_reason:
            item.global_key_event_reason = "不适用"

    @staticmethod
    def _safe_str(value) -> str:
        if value is None:
            return ""
        return str(value).strip()

    @staticmethod
    def _append_remark(existing: str | None, new_text: str) -> str:
        if not existing:
            return new_text
        return f"{existing}；{new_text}"

path: app/services/briefing_service.py
content: from __future__ import annotations

from typing import Dict, List, Optional

from app.models.news_item import NewsItem


class BriefingService:
    """
    简报生成服务（Word用）：
    只输出两大部分：
    1. “一带”AI领军企业动态
    2. 全球AI产业动态

    特点：
    - 不再对传入 items 做二次有效性筛选
    - 支持 global_items_override：可传入“全球AI产业动态重点事件Top5”
    - 全球AI产业动态内部默认按 国内 -> 国际 排序
    """

    def __init__(self, max_items_per_section: int = 999999) -> None:
        self.max_items_per_section = max_items_per_section

    def build_briefing(
        self,
        items: List[NewsItem],
        run_date: str,
        time_window: str = "",
        global_items_override: Optional[List[NewsItem]] = None,
    ) -> Dict:
        valid_items = list(items)
        valid_items = self._sort_items(valid_items)

        company_items: List[NewsItem] = []
        global_items: List[NewsItem] = []

        for item in valid_items:
            level1 = (item.category_level_1 or "").strip()

            if level1 == "名单企业动态":
                company_items.append(item)
            elif level1 == "全球AI产业动态":
                global_items.append(item)
            else:
                # 兜底：按国家/地区归类，防止一级类别脏值导致 Word 漏项
                country_region = (item.country_region or "").strip()
                if country_region == "中国":
                    company_items.append(item)
                else:
                    global_items.append(item)

        # 全球AI产业动态部分：
        # 若传入重点事件覆盖列表，则只用它；否则用全部全球AI产业动态
        if global_items_override is not None:
            global_items = list(global_items_override)

        company_items = self._sort_company_items(company_items)[: self.max_items_per_section]
        global_items = self._sort_global_items(global_items)[: self.max_items_per_section]

        return {
            "title": "每日AI产业资讯简报",
            "run_date": run_date,
            "time_window": time_window,
            "company_items": self._build_items(company_items),
            "global_items": self._build_items(global_items),
        }

    def _build_items(self, items: List[NewsItem]) -> List[Dict]:
        result: List[Dict] = []

        for item in items:
            result.append({
                "title": item.title_zh or item.title or "",
                "summary": self._item_summary_text(item),
                "source_name": item.source_name or "",
                "source_url": item.source_url or "",
                "global_region_label": item.global_region_label or "",
                "global_key_event_flag": item.global_key_event_flag or "",
                "global_key_event_score": item.global_key_event_score or 0,
                "global_key_event_reason": item.global_key_event_reason or "",
            })

        return result

    @staticmethod
    def _item_summary_text(item: NewsItem) -> str:
        parts: List[str] = []

        if item.related_companies:
            parts.append(f"关联企业：{'、'.join(item.related_companies)}。")

        # 全球AI产业动态可在摘要里带上国内/国际归属，帮助阅读(不再显示)
        #if (item.category_level_1 or "").strip() == "全球AI产业动态":
        #    if (item.global_region_label or "").strip() in {"国内", "国际"}:
        #        parts.append(f"机构归属：{item.global_region_label}。")

        summary_text = (item.summary_zh or item.summary or "").strip()
        if summary_text:
            parts.append(summary_text)

#        if item.source_name:
#            parts.append(f"来源：{item.source_name}。")

        return " ".join(parts).strip()

    def _sort_company_items(self, items: List[NewsItem]) -> List[NewsItem]:
        importance_order = {"高": 0, "中": 1, "低": 2, None: 3, "": 3}

        def sort_key(x: NewsItem):
            return (
                importance_order.get(x.importance, 3),
                self._publish_sort_value(x),
                x.title_zh or x.title or "",
            )

        return sorted(items, key=sort_key)

    def _sort_global_items(self, items: List[NewsItem]) -> List[NewsItem]:
        importance_order = {"高": 0, "中": 1, "低": 2, None: 3, "": 3}

        def region_sort(x: NewsItem) -> int:
            label = (x.global_region_label or "").strip()
            if label == "国内":
                return 0
            if label == "国际":
                return 1
            return 2

        def key_event_sort(x: NewsItem) -> int:
            # 重点事件优先
            return 0 if (x.global_key_event_flag or "").strip() == "是" else 1

        def score_sort(x: NewsItem) -> int:
            # 分数高的排前面，所以这里取负值
            score = x.global_key_event_score or 0
            return -score

        def sort_key(x: NewsItem):
            return (
                region_sort(x),                     # 国内在前，国际在后
                key_event_sort(x),                  # 标记为重点事件的在前
                score_sort(x),                      # 分数高的在前
                importance_order.get(x.importance, 3),
                self._publish_sort_value(x),
                x.title_zh or x.title or "",
            )

        return sorted(items, key=sort_key)

    @staticmethod
    def _publish_sort_value(item: NewsItem):
        """
        让更早的发布时间在前（与你现在“保留最早”逻辑一致）。
        若无发布时间，则排后。
        """
        return item.publish_time or "9999-12-31 23:59:59"

    @staticmethod
    def _sort_items(items: List[NewsItem]) -> List[NewsItem]:
        """
        总体排序：
        1. 名单企业动态在前
        2. 全球AI产业动态在后
        """
        def level1_sort(x: NewsItem) -> int:
            level1 = (x.category_level_1 or "").strip()
            if level1 == "名单企业动态":
                return 0
            if level1 == "全球AI产业动态":
                return 1
            return 2

        def sort_key(x: NewsItem):
            return (
                level1_sort(x),
                x.title_zh or x.title or "",
            )

        return sorted(items, key=sort_key)

path: app/services/classify_service.py
content: from __future__ import annotations

from typing import Dict, List

from app.models.news_item import NewsItem


class ClassifyService:
    """
    资讯分类服务。
    """

    def __init__(self) -> None:
        self.category_rules: Dict[str, List[str]] = {
            "融资": [
                "融资", "募资", "完成a轮", "完成b轮", "完成c轮", "天使轮", "种子轮",
                "pre-a", "series a", "series b", "series c", "funding", "raised",
                "financing", "investment round"
            ],
            "投资/并购": [
                "投资", "并购", "收购", "入股", "控股", "增资",
                "acquisition", "acquire", "investment", "invests", "merger", "m&a"
            ],
            "成立新企业": [
                "成立", "新设", "注册", "设立子公司", "成立子公司", "新公司",
                "subsidiary", "incorporated", "established", "set up"
            ],
            "产品发布": [
                "发布", "推出", "上线", "开源", "模型", "平台", "系统", "芯片",
                "launch", "launched", "unveiled", "introduced", "released", "model"
            ],
            "获得认证": [
                "认证", "批准", "获批", "通过认证", "ce", "fda", "iso", "approved",
                "certified", "approval", "clearance"
            ],
            "政策动态": [
                "政策", "法规", "监管", "法案", "战略", "指导意见", "框架",
                "policy", "regulation", "act", "framework", "guideline", "government"
            ],
            "行业大事件": [
                "行业", "大会", "峰会", "联盟", "基础设施", "算力", "数据中心",
                "industry", "summit", "alliance", "infrastructure", "datacenter"
            ],
        }

    def classify_item(self, item: NewsItem) -> NewsItem:
        """
        对单条资讯进行一级、二级分类。
        """
        text = self._build_text(item)

        level_2 = self._classify_level_2(text)
        level_1 = self._classify_level_1(item, level_2)

        item.category_level_1 = level_1
        item.category_level_2 = level_2

        return item

    def classify_items(self, items: List[NewsItem]) -> List[NewsItem]:
        """
        批量分类。
        """
        for item in items:
            self.classify_item(item)
        return items

    def _classify_level_2(self, text: str) -> str:
        """
        二级分类优先按关键词命中。
        """
        for category, keywords in self.category_rules.items():
            for keyword in keywords:
                if keyword.lower() in text:
                    return category

        return "企业重大动态"

    @staticmethod
    def _classify_level_1(item: NewsItem, level_2: str) -> str:
        """
        一级分类规则：
        1. 若 match_service 已根据企业名单国家/地区赋值，则直接保留
        2. 否则按兜底逻辑判断
        """
        if item.category_level_1:
            return item.category_level_1

        if level_2 in {"政策动态", "行业大事件"}:
            return "全球AI产业动态"

        if item.related_companies:
            return "名单企业动态"

        return "全球AI产业动态"

    @staticmethod
    def _build_text(item: NewsItem) -> str:
        parts = [
            item.title_zh or item.title or "",
            item.summary_zh or item.summary or "",
            item.cleaned_content_zh or item.cleaned_content or "",
            item.raw_content or "",
            item.source_name or "",
        ]
        return "\n".join(parts).lower()

path: app/services/company_event_split_service.py
content: from __future__ import annotations

import copy
import re
from typing import Dict, List

from app.models.company import Company
from app.models.news_item import NewsItem


class CompanyEventSplitService:
    """
    企业级事件拆分服务。

    目标：
    - 一条资讯只讲一家企业
    - 如果原资讯命中多家企业，则按企业拆分成多条子事件
    - 每条子事件仅保留与该企业直接相关的句子
    """

    def __init__(self, companies: List[Company]) -> None:
        self.companies = companies
        self.company_map: Dict[str, Company] = {c.company_cn: c for c in companies}

    def split_items(self, items: List[NewsItem]) -> List[NewsItem]:
        result: List[NewsItem] = []

        for item in items:
            split_result = self.split_item(item)
            result.extend(split_result)

        return result

    def split_item(self, item: NewsItem) -> List[NewsItem]:
        matched_companies = item.related_companies or []

        # 只命中 0/1 家企业，不拆
        if len(matched_companies) <= 1:
            return [item]

        split_items: List[NewsItem] = []

        for company_name in matched_companies:
            company = self.company_map.get(company_name)
            if not company:
                continue

            sub_item = self._build_company_sub_item(item, company)

            # 若拆不出该企业明确相关的内容，则不生成子项
            if not sub_item:
                continue

            split_items.append(sub_item)

        # 如果一个都没拆出来，回退保留原记录，避免误丢
        if not split_items:
            return [item]

        return split_items

    def _build_company_sub_item(self, item: NewsItem, company: Company) -> NewsItem | None:
        keywords = self._build_company_keywords(company)

        title_source = item.title_zh or item.title or ""
        summary_source = item.summary_zh or item.summary or ""
        content_source = item.cleaned_content_zh or item.cleaned_content or ""

        title_parts = self._split_text_to_segments(title_source, is_title=True)
        summary_sentences = self._split_text_to_segments(summary_source, is_title=False)
        content_sentences = self._split_text_to_segments(content_source, is_title=False)

        matched_title_parts = self._extract_related_segments(title_parts, keywords)
        matched_summary_sentences = self._extract_related_segments(summary_sentences, keywords)
        matched_content_sentences = self._extract_related_segments(content_sentences, keywords)

        # 如果摘要和正文都没有明确提及该企业，不拆这一家
        if not matched_summary_sentences and not matched_content_sentences and not matched_title_parts:
            return None

        new_item = copy.deepcopy(item)

        # 只保留当前企业
        new_item.related_companies = [company.company_cn]

        # 按企业名单国家/地区重新写一级类别和地区
        if self._is_china_company(company):
            new_item.country_region = "中国"
            new_item.category_level_1 = "名单企业动态"
        else:
            new_item.country_region = "国际"
            new_item.category_level_1 = "全球AI产业动态"

        # 重写标题
        new_title = self._build_title(
            company_name=company.company_cn,
            matched_title_parts=matched_title_parts,
            matched_summary_sentences=matched_summary_sentences,
            matched_content_sentences=matched_content_sentences,
        )

        # 重写摘要
        new_summary = self._build_summary(
            matched_summary_sentences=matched_summary_sentences,
            matched_content_sentences=matched_content_sentences,
        )

        # 没有摘要时，用标题兜底
        if not new_summary:
            new_summary = new_title

        new_item.title = new_title
        new_item.summary = new_summary
        new_item.cleaned_content = new_summary

        # 中文字段同步
        new_item.title_zh = new_title
        new_item.summary_zh = new_summary
        new_item.cleaned_content_zh = new_summary

        # 原文标题/摘要不改，便于追溯
        new_item.normalized_title = self._normalize_simple(new_title)

        # 给备注加标记，方便排查
        new_item.remarks = self._append_remark(new_item.remarks, f"已按企业拆分：{company.company_cn}")

        return new_item

    def _build_title(
        self,
        company_name: str,
        matched_title_parts: List[str],
        matched_summary_sentences: List[str],
        matched_content_sentences: List[str],
    ) -> str:
        # 1. 优先用原标题中命中的片段
        if matched_title_parts:
            candidate = self._clean_segment(matched_title_parts[0])
            if candidate:
                # 若片段里没公司名，则补上
                if company_name not in candidate:
                    return f"{company_name}{candidate}"
                return candidate

        # 2. 其次用摘要中第一句
        if matched_summary_sentences:
            sentence = self._clean_segment(matched_summary_sentences[0])
            if sentence:
                return self._compress_to_title(sentence, company_name)

        # 3. 再次用正文中第一句
        if matched_content_sentences:
            sentence = self._clean_segment(matched_content_sentences[0])
            if sentence:
                return self._compress_to_title(sentence, company_name)

        # 4. 最后兜底
        return f"{company_name}相关动态"

    def _build_summary(
        self,
        matched_summary_sentences: List[str],
        matched_content_sentences: List[str],
    ) -> str:
        collected: List[str] = []

        for s in matched_summary_sentences:
            s = self._clean_segment(s)
            if s and s not in collected:
                collected.append(s)

        for s in matched_content_sentences:
            s = self._clean_segment(s)
            if s and s not in collected:
                collected.append(s)

        # 最多保留前2句
        collected = collected[:2]

        if not collected:
            return ""

        summary = " ".join(collected).strip()
        if len(summary) > 220:
            summary = summary[:219].rstrip() + "…"
        return summary

    @staticmethod
    def _split_text_to_segments(text: str, is_title: bool = False) -> List[str]:
        if not text:
            return []

        text = str(text).strip()
        if not text:
            return []

        if is_title:
            # 标题允许按中文顿号/逗号/分号/破折号等切
            parts = re.split(r"[；;｜|/、，,：:]", text)
        else:
            parts = re.split(r"[。！？!?；;\n]", text)

        return [p.strip() for p in parts if p and p.strip()]

    @staticmethod
    def _extract_related_segments(segments: List[str], keywords: List[str]) -> List[str]:
        matched = []

        for seg in segments:
            seg_lower = seg.lower()
            for kw in keywords:
                if kw and kw.lower() in seg_lower:
                    matched.append(seg)
                    break

        return matched

    def _build_company_keywords(self, company: Company) -> List[str]:
        keywords = []

        # 强关键词优先
        if company.company_cn:
            keywords.append(company.company_cn.strip())
        if company.company_en:
            keywords.append(company.company_en.strip())

        # 安全简称 / 别名
        candidates = [company.short_name, *company.aliases]
        for kw in candidates:
            kw = (kw or "").strip()
            if not kw:
                continue

            if self._contains_chinese(kw):
                if len(kw) >= 3:
                    keywords.append(kw)
            else:
                if len(kw) >= 4:
                    keywords.append(kw)

        # 去重
        return list(dict.fromkeys(keywords))

    @staticmethod
    def _contains_chinese(text: str) -> bool:
        return any("\u4e00" <= ch <= "\u9fff" for ch in text)

    @staticmethod
    def _compress_to_title(sentence: str, company_name: str) -> str:
        sentence = sentence.strip()
        if not sentence:
            return f"{company_name}相关动态"

        # 去尾部句号
        sentence = re.sub(r"[。；;,.，]+$", "", sentence)

        if company_name not in sentence:
            sentence = f"{company_name}{sentence}"

        # 标题太长时截断
        if len(sentence) > 60:
            sentence = sentence[:59].rstrip() + "…"

        return sentence

    @staticmethod
    def _clean_segment(text: str) -> str:
        text = (text or "").strip()
        text = re.sub(r"\s+", " ", text)
        text = re.sub(r"^[：:；;，,、\-\|]+", "", text)
        text = re.sub(r"[：:；;，,、\-\|]+$", "", text)
        return text.strip()

    @staticmethod
    def _normalize_simple(text: str) -> str:
        return " ".join((text or "").strip().lower().split())

    @staticmethod
    def _is_china_company(company: Company) -> bool:
        value = (company.country_region or "").strip().lower()
        return value in {"中国", "china", "cn", "中国大陆"}

    @staticmethod
    def _append_remark(existing: str | None, new_text: str) -> str:
        if not existing:
            return new_text
        return f"{existing}；{new_text}"

path: app/services/dedup_service.py
content: from __future__ import annotations

from datetime import datetime
from typing import Dict, List, Optional, Set, Tuple

from rapidfuzz import fuzz

from app.models.news_item import NewsItem
from app.utils.text_utils import (
    contains_financing_signal,
    contains_policy_signal,
    contains_product_signal,
    jaccard_similarity,
    normalize_title,
    tokenize_text,
)
from app.utils.url_utils import normalize_url


class DedupService:
    """
    资讯去重服务（保留最早发布时间版）。
    去重顺序：
    1. URL
    2. 内容哈希
    3. 标题相似度
    4. 事件相似度

    规则：
    - 若两条重复，保留 publish_time 更早的那条
    """

    def __init__(
        self,
        title_similarity_threshold: int = 90,
        event_similarity_threshold: float = 0.72,
    ) -> None:
        self.title_similarity_threshold = title_similarity_threshold
        self.event_similarity_threshold = event_similarity_threshold

    def dedup_items(self, items: List[NewsItem]) -> Tuple[List[NewsItem], List[NewsItem]]:
        if not items:
            return [], []

        # 先统一标题
        for item in items:
            item.normalized_title = normalize_title(item.title)

        # 先按发布时间升序，尽量让最早的先进入
        items_sorted = sorted(items, key=self._sort_key_by_time)

        unique_items: List[NewsItem] = []
        duplicate_items: List[NewsItem] = []

        kept_items: List[NewsItem] = []
        url_map: Dict[str, NewsItem] = {}
        hash_map: Dict[str, NewsItem] = {}
        title_map: Dict[str, NewsItem] = {}

        for item in items_sorted:
            replaced = False

            # 1) URL 去重
            url = normalize_url(item.source_url)
            if url:
                kept = url_map.get(url)
                if kept:
                    winner, loser = self._pick_earlier_item(kept, item)
                    if winner is item:
                        self._replace_kept_item(kept_items, kept, item)
                        url_map[url] = item
                        self._mark_duplicate(loser, "duplicate_url", f"URL重复，保留更早发布时间：{item.title[:80]}")
                        duplicate_items.append(loser)
                    else:
                        self._mark_duplicate(item, "duplicate_url", f"URL重复，保留更早发布时间：{kept.title[:80]}")
                        duplicate_items.append(item)
                    continue

            # 2) 内容哈希去重
            if item.content_hash:
                kept = hash_map.get(item.content_hash)
                if kept:
                    winner, loser = self._pick_earlier_item(kept, item)
                    if winner is item:
                        self._replace_kept_item(kept_items, kept, item)
                        hash_map[item.content_hash] = item
                        self._mark_duplicate(loser, "duplicate_hash", f"内容哈希重复，保留更早发布时间：{item.title[:80]}")
                        duplicate_items.append(loser)
                    else:
                        self._mark_duplicate(item, "duplicate_hash", f"内容哈希重复，保留更早发布时间：{kept.title[:80]}")
                        duplicate_items.append(item)
                    continue

            # 3) 标题相似度去重
            dup_title = self._find_similar_title(item, title_map)
            if dup_title is not None:
                winner, loser = self._pick_earlier_item(dup_title, item)
                if winner is item:
                    self._replace_kept_item(kept_items, dup_title, item)
                    # title_map 里把旧标题删掉，再写新标题
                    old_key = dup_title.normalized_title or ""
                    if old_key in title_map:
                        del title_map[old_key]
                    title_map[item.normalized_title or ""] = item
                    self._mark_duplicate(loser, "duplicate_title", f"标题相似重复，保留更早发布时间：{item.title[:80]}")
                    duplicate_items.append(loser)
                else:
                    self._mark_duplicate(item, "duplicate_title", f"标题相似重复，保留更早发布时间：{dup_title.title[:80]}")
                    duplicate_items.append(item)
                continue

            # 4) 事件相似度去重
            dup_event = self._find_similar_event(item, kept_items)
            if dup_event is not None:
                winner, loser = self._pick_earlier_item(dup_event, item)
                if winner is item:
                    self._replace_kept_item(kept_items, dup_event, item)
                    # 可能需要同步更新 url/hash/title map
                    self._rebuild_maps(kept_items, url_map, hash_map, title_map)
                    self._mark_duplicate(loser, "duplicate_event", f"疑似同事件重复，保留更早发布时间：{item.title[:80]}")
                    duplicate_items.append(loser)
                else:
                    self._mark_duplicate(item, "duplicate_event", f"疑似同事件重复，保留更早发布时间：{dup_event.title[:80]}")
                    duplicate_items.append(item)
                continue

            # 保留
            item.dedup_status = "unique"
            unique_items.append(item)
            kept_items.append(item)

            if url:
                url_map[url] = item
            if item.content_hash:
                hash_map[item.content_hash] = item
            if item.normalized_title:
                title_map[item.normalized_title] = item

        # unique_items 重新对齐 kept_items
        unique_items = kept_items
        return unique_items, duplicate_items

    def _find_similar_title(self, item: NewsItem, title_map: Dict[str, NewsItem]) -> Optional[NewsItem]:
        current_title = item.normalized_title or ""
        if not current_title:
            return None

        for kept_title, kept_item in title_map.items():
            if not kept_title:
                continue

            score = fuzz.ratio(current_title, kept_title)
            if score >= self.title_similarity_threshold:
                return kept_item

        return None

    def _find_similar_event(self, item: NewsItem, kept_items: List[NewsItem]) -> Optional[NewsItem]:
        current_text = self._build_event_text(item)
        current_tokens = tokenize_text(current_text)

        if not current_tokens:
            return None

        for kept_item in kept_items:
            company_overlap = self._has_company_overlap(item, kept_item)
            same_category = (
                (item.category_level_2 or "") == (kept_item.category_level_2 or "")
                and bool(item.category_level_2)
            )

            kept_text = self._build_event_text(kept_item)
            kept_tokens = tokenize_text(kept_text)
            sim = jaccard_similarity(current_tokens, kept_tokens)

            if company_overlap and same_category and sim >= self.event_similarity_threshold:
                return kept_item

            if company_overlap and self._both_financing(item, kept_item) and sim >= 0.55:
                return kept_item

            if company_overlap and self._both_product(item, kept_item) and sim >= 0.58:
                return kept_item

            if self._both_policy(item, kept_item) and sim >= 0.68:
                return kept_item

        return None

    @staticmethod
    def _build_event_text(item: NewsItem) -> str:
        parts = [
            item.title or "",
            item.summary or "",
            item.category_level_2 or "",
            " ".join(item.related_companies) if item.related_companies else "",
        ]
        return " ".join(parts)

    @staticmethod
    def _has_company_overlap(a: NewsItem, b: NewsItem) -> bool:
        set_a = set(a.related_companies or [])
        set_b = set(b.related_companies or [])
        if not set_a or not set_b:
            return False
        return bool(set_a & set_b)

    @staticmethod
    def _both_financing(a: NewsItem, b: NewsItem) -> bool:
        ta = f"{a.title or ''} {a.summary or ''}"
        tb = f"{b.title or ''} {b.summary or ''}"
        return contains_financing_signal(ta) and contains_financing_signal(tb)

    @staticmethod
    def _both_product(a: NewsItem, b: NewsItem) -> bool:
        ta = f"{a.title or ''} {a.summary or ''}"
        tb = f"{b.title or ''} {b.summary or ''}"
        return contains_product_signal(ta) and contains_product_signal(tb)

    @staticmethod
    def _both_policy(a: NewsItem, b: NewsItem) -> bool:
        ta = f"{a.title or ''} {a.summary or ''}"
        tb = f"{b.title or ''} {b.summary or ''}"
        return contains_policy_signal(ta) and contains_policy_signal(tb)

    @staticmethod
    def _sort_key_by_time(item: NewsItem):
        dt = item.publish_time
        if dt is None:
            return (1, datetime.max.replace(year=2099))
        return (0, dt)

    @staticmethod
    def _pick_earlier_item(a: NewsItem, b: NewsItem) -> Tuple[NewsItem, NewsItem]:
        a_time = a.publish_time
        b_time = b.publish_time

        if a_time and b_time:
            return (a, b) if a_time <= b_time else (b, a)

        if a_time and not b_time:
            return a, b

        if b_time and not a_time:
            return b, a

        # 都没有发布时间，则保留原来的 a
        return a, b

    @staticmethod
    def _replace_kept_item(kept_items: List[NewsItem], old_item: NewsItem, new_item: NewsItem) -> None:
        for idx, item in enumerate(kept_items):
            if item is old_item:
                kept_items[idx] = new_item
                return

    @staticmethod
    def _mark_duplicate(item: NewsItem, status: str, remark: str) -> None:
        item.dedup_status = status
        item.entry_status = "excluded"
        item.remarks = remark

    @staticmethod
    def _rebuild_maps(
        kept_items: List[NewsItem],
        url_map: Dict[str, NewsItem],
        hash_map: Dict[str, NewsItem],
        title_map: Dict[str, NewsItem],
    ) -> None:
        url_map.clear()
        hash_map.clear()
        title_map.clear()

        for item in kept_items:
            url = normalize_url(item.source_url)
            if url:
                url_map[url] = item
            if item.content_hash:
                hash_map[item.content_hash] = item
            if item.normalized_title:
                title_map[item.normalized_title] = item

path: app/services/filter_service.py
content: from __future__ import annotations

from typing import List, Tuple

from app.core.time_window import TimeWindow, get_event_date, normalize_to_shanghai
from app.models.news_item import NewsItem


class FilterService:
    """
    资讯过滤服务。
    """

    def __init__(self, min_title_length: int = 8, require_source_url: bool = True) -> None:
        self.min_title_length = min_title_length
        self.require_source_url = require_source_url

    def filter_items(
        self, items: List[NewsItem], window: TimeWindow
    ) -> Tuple[List[NewsItem], List[NewsItem]]:
        """
        返回：
        - valid_items: 通过过滤的资讯
        - invalid_items: 未通过过滤的资讯
        """
        valid_items: List[NewsItem] = []
        invalid_items: List[NewsItem] = []
        item.publish_time = normalize_to_shanghai(item.publish_time) if item.publish_time else item.publish_time
        for item in items:
            passed, reason = self.is_valid(item, window)

            item.window_start = window.start_time
            item.window_end = window.end_time

            if item.publish_time:
                item.publish_time = normalize_to_shanghai(item.publish_time)
                item.event_date = get_event_date(item.publish_time)

            if passed:
                item.entry_status = "included"
                if not item.remarks:
                    item.remarks = ""
                valid_items.append(item)
            else:
                item.entry_status = "excluded"
                item.remarks = reason
                invalid_items.append(item)

        return valid_items, invalid_items

    def is_valid(self, item: NewsItem, window: TimeWindow) -> Tuple[bool, str]:
        """
        判断单条资讯是否有效。
        """
        if not item.title or len(item.title.strip()) < self.min_title_length:
            return False, "标题过短或为空"

        if self.require_source_url and not item.source_url:
            return False, "缺少来源链接"

        if not item.publish_time:
            return False, "缺少发布时间"

        publish_time = normalize_to_shanghai(item.publish_time)
        if publish_time is None:
            return False, "发布时间无法解析"

        if not window.contains(publish_time):
            return False, "发布时间不在统计窗口内"

        return True, "有效"

path: app/services/global_key_event_service.py
content: from __future__ import annotations

from typing import List

from app.models.news_item import NewsItem


class GlobalKeyEventService:
    """
    从全球AI产业动态中筛选重点事件 Top N。
    """

    def __init__(self, top_n: int = 5) -> None:
        self.top_n = top_n

    def select_top_events(self, items: List[NewsItem]) -> List[NewsItem]:
        global_items = [
            x for x in items
            if (x.category_level_1 or "").strip() == "全球AI产业动态"
        ]

        if not global_items:
            return []

        def sort_key(x: NewsItem):
            flag = (x.global_key_event_flag or "").strip()
            score = x.global_key_event_score or 0
            # 先按“是/否”，再按分数降序，再按发布时间升序（更早优先）
            flag_sort = 0 if flag == "是" else 1
            publish_time = x.publish_time
            return (flag_sort, -score, publish_time or 0)

        selected = sorted(global_items, key=sort_key)

        # 如果模型标“是”的不足5条，仍按分数补足到5条
        return selected[: self.top_n]

path: app/services/importance_service.py
content: from __future__ import annotations

from typing import Dict, List

from app.models.news_item import NewsItem


class ImportanceService:
    """
    资讯重要性评分服务。
    """

    def __init__(self) -> None:
        self.high_keywords: List[str] = [
            "融资", "并购", "收购", "重大", "发布", "新模型", "法案", "监管", "战略",
            "funding", "raised", "acquisition", "merger", "launch", "released",
            "regulation", "policy", "approval"
        ]

        self.medium_keywords: List[str] = [
            "合作", "认证", "设立", "成立子公司", "投资", "产品", "平台",
            "partnership", "certification", "approved", "investment", "introduced"
        ]

        self.high_categories = {"融资", "投资/并购", "政策动态"}
        self.medium_categories = {"成立新企业", "产品发布", "获得认证", "企业重大动态"}

    def score_item(self, item: NewsItem) -> NewsItem:
        text = self._build_text(item)

        # 一级规则：按二级分类先给基础等级
        if item.category_level_2 in self.high_categories:
            item.importance = "高"
            return item

        if item.category_level_2 in self.medium_categories:
            item.importance = "中"
            return item

        # 二级规则：按关键词命中
        if self._contains_any(text, self.high_keywords):
            item.importance = "高"
            return item

        if self._contains_any(text, self.medium_keywords):
            item.importance = "中"
            return item

        item.importance = "低"
        return item

    def score_items(self, items: List[NewsItem]) -> List[NewsItem]:
        for item in items:
            self.score_item(item)
        return items

    @staticmethod
    def _contains_any(text: str, keywords: List[str]) -> bool:
        for keyword in keywords:
            if keyword.lower() in text:
                return True
        return False

    @staticmethod
    def _build_text(item: NewsItem) -> str:
        parts = [
            item.title or "",
            item.summary or "",
            item.cleaned_content or "",
            item.raw_content or "",
            item.category_level_1 or "",
            item.category_level_2 or "",
        ]
        return "\n".join(parts).lower()

path: app/services/match_service.py
content: from __future__ import annotations

import re
from typing import Dict, List, Set

from app.models.company import Company
from app.models.news_item import NewsItem


class MatchService:
    """
    企业匹配服务（增强版）。
    特点：
    1. 中文全称/英文全称优先
    2. 过滤高风险短别名
    3. 英文使用单词边界匹配，避免子串误伤
    4. 根据企业名单国家/地区，给 item 写入：
       - related_companies
       - country_region
       - category_level_1
    """

    def __init__(self, companies: List[Company]) -> None:
        self.companies = companies
        self.company_map: Dict[str, Company] = {c.company_cn: c for c in companies}

    def match_item(self, item: NewsItem) -> NewsItem:
        matched_companies = self.find_company_objects(item)
        matched_names = [c.company_cn for c in matched_companies]

        item.related_companies = matched_names

        if matched_companies:
            if any(self._is_china_company(c) for c in matched_companies):
                item.country_region = "中国"
                item.category_level_1 = "名单企业动态"
            else:
                item.country_region = "国际"
                item.category_level_1 = "全球AI产业动态"

        return item

    def match_items(self, items: List[NewsItem]) -> List[NewsItem]:
        for item in items:
            self.match_item(item)
        return items

    def find_company_objects(self, item: NewsItem) -> List[Company]:
        text_parts = [
            item.title_zh or item.title or "",
            item.summary_zh or item.summary or "",
            item.cleaned_content_zh or item.cleaned_content or "",
            item.raw_content or "",
        ]
        full_text = "\n".join(text_parts)
        full_text_lower = full_text.lower()

        matched_companies: List[Company] = []
        seen: Set[str] = set()

        for company in self.companies:
            if self._match_company(full_text, full_text_lower, company):
                if company.company_cn not in seen:
                    matched_companies.append(company)
                    seen.add(company.company_cn)

        return matched_companies

    def _match_company(self, full_text: str, full_text_lower: str, company: Company) -> bool:
        """
        匹配顺序：
        1. 中文全称
        2. 英文全称
        3. 中文简称/别名（过滤短词）
        4. 英文简称/别名（单词边界）
        """

        # 1) 中文全称：强匹配
        if company.company_cn and self._match_chinese_phrase(full_text, company.company_cn):
            return True

        # 2) 英文全称：强匹配
        if company.company_en and self._match_english_phrase(full_text_lower, company.company_en.lower()):
            return True

        # 3) 中文简称 + 中文别名：中匹配（过滤短词）
        zh_candidates = self._get_chinese_aliases(company)
        for kw in zh_candidates:
            if self._is_safe_chinese_keyword(kw) and self._match_chinese_phrase(full_text, kw):
                return True

        # 4) 英文简称 + 英文别名：中匹配（过滤短词 + 单词边界）
        en_candidates = self._get_english_aliases(company)
        for kw in en_candidates:
            kw_lower = kw.lower()
            if self._is_safe_english_keyword(kw_lower) and self._match_english_phrase(full_text_lower, kw_lower):
                return True

        return False

    @staticmethod
    def _get_chinese_aliases(company: Company) -> List[str]:
        values = [company.short_name, *company.aliases]
        result = []
        for v in values:
            v = (v or "").strip()
            if v and MatchService._contains_chinese(v):
                result.append(v)
        return list(dict.fromkeys(result))

    @staticmethod
    def _get_english_aliases(company: Company) -> List[str]:
        values = [company.short_name, *company.aliases, company.parent_company]
        result = []
        for v in values:
            v = (v or "").strip()
            if v and not MatchService._contains_chinese(v):
                result.append(v)
        return list(dict.fromkeys(result))

    @staticmethod
    def _contains_chinese(text: str) -> bool:
        return any("\u4e00" <= ch <= "\u9fff" for ch in text)

    @staticmethod
    def _is_safe_chinese_keyword(keyword: str) -> bool:
        """
        中文关键词安全阈值：
        - 长度至少 3
        - 避免单字/双字通用词误伤
        """
        keyword = keyword.strip()
        if len(keyword) < 3:
            return False
        return True

    @staticmethod
    def _is_safe_english_keyword(keyword: str) -> bool:
        """
        英文关键词安全阈值：
        - 长度至少 4
        """
        keyword = keyword.strip()
        if len(keyword) < 4:
            return False
        return True

    @staticmethod
    def _match_chinese_phrase(text: str, keyword: str) -> bool:
        """
        中文短语匹配：
        当前先做严格包含，但前提是 keyword 已通过长度过滤。
        """
        keyword = keyword.strip()
        if not keyword:
            return False
        return keyword in text

    @staticmethod
    def _match_english_phrase(text_lower: str, keyword_lower: str) -> bool:
        """
        英文短语匹配：使用单词边界，避免子串误伤
        """
        keyword_lower = keyword_lower.strip()
        if not keyword_lower:
            return False

        pattern = r"\b" + re.escape(keyword_lower) + r"\b"
        return re.search(pattern, text_lower) is not None

    @staticmethod
    def _is_china_company(company: Company) -> bool:
        value = (company.country_region or "").strip().lower()
        return value in {"中国", "china", "cn", "中国大陆"}

path: app/services/relevance_service.py
content: from __future__ import annotations

from typing import List, Tuple

from app.models.news_item import NewsItem


class RelevanceService:
    """
    相关性筛选服务：
    过滤掉中文综合媒体中的无关资讯。
    """

    def __init__(self) -> None:
        self.ai_keywords = {
            "人工智能", "ai", "大模型", "生成式ai", "生成式人工智能", "机器学习", "深度学习",
            "多模态", "智能体", "算力", "智算", "模型训练", "ai芯片", "智能驾驶",
            "llm", "foundation model", "model", "agent"
        }

        self.event_keywords = {
            "融资", "募资", "天使轮", "种子轮", "a轮", "b轮", "c轮",
            "投资", "并购", "收购", "入股", "增资",
            "成立", "新设", "注册", "子公司", "设立",
            "发布", "推出", "上线", "开源", "模型", "平台", "系统", "芯片",
            "认证", "获批", "批准", "通过认证", "ce", "fda", "iso",
            "政策", "法规", "监管", "法案", "战略", "框架", "guideline",
            "funding", "raised", "investment", "acquisition", "launch",
            "released", "approved", "policy", "regulation"
        }

        self.important_ai_companies = {
            "openai", "anthropic", "英伟达", "nvidia", "google", "deepmind",
            "meta", "xai", "mistral", "cohere", "microsoft", "微软",
            "亚马逊", "amazon", "oracle", "amd", "intel"
        }

    def filter_items(self, items: List[NewsItem]) -> Tuple[List[NewsItem], List[NewsItem]]:
        relevant_items = []
        irrelevant_items = []

        for item in items:
            passed, reason = self.is_relevant(item)
            if passed:
                relevant_items.append(item)
            else:
                item.entry_status = "excluded"
                item.remarks = self._append_remark(item.remarks, reason)
                irrelevant_items.append(item)

        return relevant_items, irrelevant_items

    def is_relevant(self, item: NewsItem) -> Tuple[bool, str]:
        text = self._build_text(item)

        has_ai_signal = self._contains_any(text, self.ai_keywords)
        has_event_signal = self._contains_any(text, self.event_keywords)
        has_target_company = bool(item.related_companies)
        has_important_ai_company = self._contains_any(text, self.important_ai_companies)

        # 规则1：名单企业 + 事件
        if has_target_company and has_event_signal:
            return True, "名单企业相关"

        # 规则2：AI主题 + 事件
        if has_ai_signal and has_event_signal:
            return True, "AI主题相关"

        # 规则3：重点AI公司 + 事件
        if has_important_ai_company and has_event_signal:
            return True, "重点AI企业相关"

        # 规则4：政策动态直接放行（前提是分类已判定）
        if item.category_level_2 == "政策动态":
            return True, "政策动态"

        # 规则5：行业大事件直接放行（前提是分类已判定）
        if item.category_level_2 == "行业大事件":
            return True, "行业大事件"

        return False, "与AI关注范围不匹配"

    @staticmethod
    def _build_text(item: NewsItem) -> str:
        parts = [
            item.title_zh or item.title or "",
            item.summary_zh or item.summary or "",
            item.cleaned_content_zh or item.cleaned_content or "",
            item.category_level_1 or "",
            item.category_level_2 or "",
            " ".join(item.related_companies) if item.related_companies else "",
        ]
        return "\n".join(parts).lower()

    @staticmethod
    def _contains_any(text: str, keywords: set[str]) -> bool:
        for kw in keywords:
            if kw.lower() in text:
                return True
        return False

    @staticmethod
    def _append_remark(existing: str | None, new_text: str) -> str:
        if not existing:
            return new_text
        return f"{existing}；{new_text}"

path: app/services/source_scope_service.py
content: from __future__ import annotations

from typing import Dict, List, Tuple

from app.models.company import Company
from app.models.news_item import NewsItem


class SourceScopeService:
    """
    来源地区约束服务。

    规则：
    - 若来源地区 = 国内
      只保留命中“企业名单中国家/地区=中国”的企业资讯
    - 若来源地区 = 国际
      不做这个限定
    """

    def __init__(self, companies: List[Company]) -> None:
        self.company_map: Dict[str, Company] = {c.company_cn: c for c in companies}

    def filter_items(self, items: List[NewsItem]) -> Tuple[List[NewsItem], List[NewsItem]]:
        kept_items: List[NewsItem] = []
        dropped_items: List[NewsItem] = []

        for item in items:
            passed, reason = self.is_allowed(item)

            if passed:
                kept_items.append(item)
            else:
                item.entry_status = "excluded"
                item.remarks = self._append_remark(item.remarks, reason)
                dropped_items.append(item)

        return kept_items, dropped_items

    def is_allowed(self, item: NewsItem) -> Tuple[bool, str]:
        source_region = (item.source_region_scope or "").strip()

        # 国际来源：不做“中国企业限定”
        if source_region != "国内":
            return True, "国际来源不过滤"

        # 国内来源：必须命中中国企业
        if not item.related_companies:
            return False, "国内来源未命中中国企业"

        matched_companies = [
            self.company_map[name]
            for name in item.related_companies
            if name in self.company_map
        ]

        if any(self._is_china_company(c.country_region) for c in matched_companies):
            return True, "国内来源且命中中国企业"

        return False, "国内来源仅保留中国企业匹配资讯"

    @staticmethod
    def _is_china_company(country_region: str | None) -> bool:
        value = (country_region or "").strip().lower()
        return value in {"中国", "china", "cn", "中国大陆"}

    @staticmethod
    def _append_remark(existing: str | None, new_text: str) -> str:
        if not existing:
            return new_text
        return f"{existing}；{new_text}"

path: app/services/translation_service.py
content: from __future__ import annotations

from dataclasses import dataclass
from typing import List, Optional

from app.models.news_item import NewsItem
from app.parsers.content_cleaner import ContentCleaner
from app.utils.hash_utils import build_content_hash
from app.utils.text_utils import normalize_title


@dataclass
class TranslationResult:
    text: str
    translated: bool
    backend: str


class BaseTranslatorBackend:
    """
    翻译后端抽象类。
    """

    name = "base"

    def translate_to_zh(self, text: str) -> TranslationResult:
        raise NotImplementedError


class PassthroughTranslatorBackend(BaseTranslatorBackend):
    """
    占位后端：
    - 中文原样返回
    - 非中文也原样返回
    适合在未接入真实翻译服务时保证流程可运行。
    """

    name = "passthrough"

    def translate_to_zh(self, text: str) -> TranslationResult:
        return TranslationResult(text=text, translated=False, backend=self.name)


class DeepTranslatorBackend(BaseTranslatorBackend):
    """
    可选翻译后端。
    依赖:
        pip install deep-translator

    说明：
    - 若环境中未安装 deep_translator，会在初始化时报错
    - 这里只做 en -> zh-CN / auto -> zh-CN 的简单调用
    """

    name = "deep_translator"

    def __init__(self, source: str = "auto", target: str = "zh-CN") -> None:
        from deep_translator import GoogleTranslator  # type: ignore
        self._translator = GoogleTranslator(source=source, target=target)

    def translate_to_zh(self, text: str) -> TranslationResult:
        if not text or not text.strip():
            return TranslationResult(text="", translated=False, backend=self.name)

        translated_text = self._translator.translate(text)
        return TranslationResult(
            text=(translated_text or "").strip(),
            translated=True,
            backend=self.name,
        )


class TranslationService:
    """
    统一翻译服务。

    处理策略：
    - 保留原文字段
    - 生成中文字段
    - 再把 item.title / item.summary / item.cleaned_content 覆盖为中文版本
      以兼容当前已有的匹配、分类、去重逻辑
    """

    def __init__(
        self,
        backend: Optional[BaseTranslatorBackend] = None,
    ) -> None:
        self.backend = backend or self._build_default_backend()
        self.cleaner = ContentCleaner()

    def process_items(self, items: List[NewsItem]) -> List[NewsItem]:
        for item in items:
            self.process_item(item)
        return items

    def process_item(self, item: NewsItem) -> NewsItem:
        # 1. 先保留原文
        item.original_title = item.title or ""
        item.original_summary = item.summary or ""
        item.original_cleaned_content = item.cleaned_content or item.raw_content or ""

        # 2. 识别语言（优先看标题+摘要）
        language_sample = " ".join([
            item.original_title or "",
            item.original_summary or "",
        ]).strip()

        item.original_language = self.detect_language(language_sample)

        # 3. 生成中文字段
        if item.original_language == "zh":
            item.title_zh = self.cleaner.clean_text(item.original_title)
            item.summary_zh = self.cleaner.clean_summary(item.original_summary, max_length=300)
            item.cleaned_content_zh = self.cleaner.clean_text(item.original_cleaned_content)
            item.is_machine_translated = False
            item.translation_backend = "none"
        else:
            item.title_zh = self._translate_text(item.original_title, max_length=None)
            item.summary_zh = self._translate_text(item.original_summary, max_length=300)
            item.cleaned_content_zh = self._translate_text(item.original_cleaned_content, max_length=None)
            item.is_machine_translated = True
            item.translation_backend = self.backend.name

        # 4. 中文字段兜底
        item.title_zh = item.title_zh or self.cleaner.clean_text(item.original_title)
        item.summary_zh = item.summary_zh or self.cleaner.clean_summary(item.original_summary, max_length=300)
        item.cleaned_content_zh = item.cleaned_content_zh or self.cleaner.clean_text(item.original_cleaned_content)

        # 5. 为兼容现有链路，把当前主字段覆盖为中文
        item.title = item.title_zh or item.title or ""
        item.summary = item.summary_zh or item.summary or ""
        item.cleaned_content = item.cleaned_content_zh or item.cleaned_content or ""

        # 6. 生成中文去重字段
        item.normalized_title = normalize_title(item.title_zh or item.title or "")
        item.content_hash = build_content_hash(
            title=item.title_zh or item.title,
            summary=item.summary_zh or item.summary,
            content=item.cleaned_content_zh or item.cleaned_content,
        )

        return item

    def _translate_text(self, text: str, max_length: Optional[int] = None) -> str:
        cleaned = self.cleaner.clean_text(text)
        if not cleaned:
            return ""

        try:
            result = self.backend.translate_to_zh(cleaned)
            translated = self.cleaner.clean_text(result.text)
        except Exception:
            translated = cleaned

        if max_length and len(translated) > max_length:
            translated = translated[: max_length - 1].rstrip() + "…"

        return translated

    @staticmethod
    def detect_language(text: str) -> str:
        """
        简单语言识别：
        - 中文字符占比高 -> zh
        - 英文字母占比高 -> en
        - 否则 unknown
        """
        if not text or not text.strip():
            return "unknown"

        text = text.strip()

        zh_count = sum(1 for ch in text if "\u4e00" <= ch <= "\u9fff")
        en_count = sum(1 for ch in text if ("a" <= ch.lower() <= "z"))

        if zh_count >= 2 and zh_count >= en_count:
            return "zh"
        if en_count >= 3:
            return "en"
        return "unknown"

    @staticmethod
    def _build_default_backend() -> BaseTranslatorBackend:
        """
        默认后端：
        - 优先尝试 deep_translator
        - 不可用则回退为 passthrough
        """
        try:
            return DeepTranslatorBackend()
        except Exception:
            return PassthroughTranslatorBackend()

path: app/utils/hash_utils.py
content: from __future__ import annotations

import hashlib
from typing import Optional


def md5_text(text: Optional[str]) -> str:
    """
    对文本生成 MD5 哈希。
    """
    value = (text or "").strip()
    return hashlib.md5(value.encode("utf-8")).hexdigest()


def build_content_hash(
    title: Optional[str],
    summary: Optional[str] = None,
    content: Optional[str] = None,
) -> str:
    """
    组合标题/摘要/正文生成稳定内容哈希。
    """
    combined = " || ".join([
        (title or "").strip().lower(),
        (summary or "").strip().lower(),
        (content or "").strip().lower(),
    ])
    return md5_text(combined)

path: app/utils/text_utils.py
content: from __future__ import annotations

import re
from typing import Iterable


STOPWORDS = {
    "the", "a", "an", "and", "or", "to", "of", "for", "in", "on", "at", "by",
    "is", "are", "was", "were", "be", "been", "with", "from", "as", "that",
    "this", "it", "its", "their", "our", "will", "has", "have", "had",
    "宣布", "发布", "推出", "上线", "完成", "获得", "进行", "相关", "最新", "正式",
    "公司", "企业", "方面", "此次", "其中", "表示", "消息", "今日", "近日"
}


def normalize_text(text: str | None) -> str:
    """
    基础文本归一化：
    - 小写
    - 去HTML残留空白
    - 去多余标点
    - 空白折叠
    """
    if not text:
        return ""

    text = text.lower().strip()

    # 全角空格 -> 半角空格
    text = text.replace("\u3000", " ")

    # 常见连接符统一
    text = text.replace("—", "-").replace("–", "-").replace("_", " ")

    # 去 URL
    text = re.sub(r"https?://\S+", " ", text)

    # 保留中英文、数字、空格；其余多转为空格
    text = re.sub(r"[^0-9a-z\u4e00-\u9fff\s]", " ", text)

    # 空白折叠
    text = re.sub(r"\s+", " ", text).strip()
    return text


def normalize_title(title: str | None) -> str:
    """
    标题归一化，适合做去重比对。
    """
    text = normalize_text(title)

    # 去常见媒体前后缀噪音
    patterns = [
        r"^快讯\s+",
        r"^独家\s+",
        r"^重磅\s+",
        r"\s+全文$",
        r"\s+更新$",
    ]
    for p in patterns:
        text = re.sub(p, "", text)

    text = re.sub(r"\s+", " ", text).strip()
    return text


def tokenize_text(text: str | None) -> list[str]:
    """
    简单分词：
    - 英文按空格切
    - 中文连续文本先按空格切；对未分开的中文保留整段
    """
    normalized = normalize_text(text)
    if not normalized:
        return []

    tokens = normalized.split()

    result: list[str] = []
    for token in tokens:
        [REDACTED_SECRET])
        if not [REDACTED_SECRET]
        if token in STOPWORDS:
            continue
        if len(token) == 1 and not token.isdigit():
            continue
        result.append(token)

    return result


def jaccard_similarity(tokens_a: Iterable[str], tokens_b: Iterable[str]) -> float:
    set_a = set(tokens_a)
    set_b = set(tokens_b)

    if not set_a or not set_b:
        return 0.0

    inter = set_a & set_b
    union = set_a | set_b
    return len(inter) / len(union)


def contains_financing_signal(text: str | None) -> bool:
    t = normalize_text(text)
    signals = [
        "融资", "募资", "天使轮", "种子轮", "a轮", "b轮", "c轮",
        "funding", "raised", "series a", "series b", "series c", "financing"
    ]
    return any(s in t for s in signals)


def contains_product_signal(text: str | None) -> bool:
    t = normalize_text(text)
    signals = [
        "发布", "推出", "上线", "开源", "模型", "平台", "系统",
        "launch", "launched", "released", "unveiled", "model", "platform"
    ]
    return any(s in t for s in signals)


def contains_policy_signal(text: str | None) -> bool:
    t = normalize_text(text)
    signals = [
        "政策", "法规", "监管", "法案", "战略",
        "policy", "regulation", "act", "framework", "guideline"
    ]
    return any(s in t for s in signals)

path: app/utils/url_utils.py
content: from __future__ import annotations

from urllib.parse import urlparse, urlunparse, parse_qsl, urlencode


TRACKING_PARAMS = {
    "utm_source",
    "utm_medium",
    "utm_campaign",
    "utm_term",
    "utm_content",
    "utm_id",
    "gclid",
    "fbclid",
    "mc_cid",
    "mc_eid",
    "spm",
    "from",
    "ref",
    "ref_src",
}


def normalize_url(url: str | None) -> str:
    """
    归一化 URL，尽量把“同一链接不同写法”收敛成一个形式。
    """
    if not url:
        return ""

    url = url.strip()
    if not url:
        return ""

    try:
        parsed = urlparse(url)

        scheme = (parsed.scheme or "https").lower()
        netloc = parsed.netloc.lower()

        # 去掉默认端口
        if netloc.endswith(":80") and scheme == "http":
            netloc = netloc[:-3]
        if netloc.endswith(":443") and scheme == "https":
            netloc = netloc[:-4]

        # path 去尾部斜杠（根路径除外）
        path = parsed.path or ""
        if path != "/" and path.endswith("/"):
            path = path[:-1]

        # 去 tracking query 参数，并按 key 排序
        query_pairs = parse_qsl(parsed.query, keep_blank_values=True)
        filtered_pairs = [
            (k, v) for k, v in query_pairs
            if k.lower() not in TRACKING_PARAMS
        ]
        filtered_pairs.sort(key=lambda x: (x[0], x[1]))
        query = urlencode(filtered_pairs, doseq=True)

        # fragment 去掉
        fragment = ""

        normalized = urlunparse((scheme, netloc, path, "", query, fragment))
        return normalized
    except Exception:
        return url

path: config.default.yaml
content: project:
  name: "ai_news_workflow"
  version: "0.1.1"
  timezone: "Asia/Shanghai"
  language: "zh"

paths:
  company_file: "./input/企业名单.xlsx"
  source_file: "./input/source_config.xlsx"
  output_excel_dir: "./output/excel"
  output_word_dir: "./output/word"
  log_dir: "./logs"
  db_file: "./data/history.db"
  cache_dir: "./data/cache"
  temp_dir: "./data/temp"
  excel_template: "./templates/excel_template.xlsx"
  word_template: "./templates/briefing_template.docx"
  web_source_file: "./input/web_source_config.xlsx"

schedule:
  enabled: true
  run_hour: 23
  run_minute: 0

time_window:
  mode: "previous_day"
  start_inclusive: false
  end_inclusive: true
  fallback_if_publish_time_missing: "manual_review"

output:
  excel_filename_prefix: "AI资讯信息表"
  word_filename_prefix: "AI资讯简报"
  date_format: "%Y%m%d"
  datetime_format: "%Y-%m-%d %H:%M:%S"

features:
  enable_global_ai: true
  enable_company_news: true
  enable_excel_export: true
  enable_word_export: true
  enable_dedup: true
  enable_summary: true
  enable_importance_score: true

crawler:
  request_timeout_seconds: 20
  max_retries: 2
  retry_interval_seconds: 2
  user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"
  verify_ssl: true

filter_rules:
  min_title_length: 8
  min_content_length: 50
  require_source_url: true
  allowed_languages:
    - "zh"
    - "en"

dedup:
  title_similarity_threshold: 90
  content_hash_enabled: true
  url_normalization_enabled: true
  keep_priority_order:
    - "official"
    - "government"
    - "tier1_media"
    - "industry_media"
    - "other"

importance_rules:
  high_keywords:
    - "融资"
    - "并购"
    - "收购"
    - "重大发布"
    - "新模型"
    - "国家战略"
    - "监管"
    - "法案"
    - "funding"
    - "acquisition"
    - "investment"
    - "launch"
    - "regulation"
    - "policy"
  medium_keywords:
    - "合作"
    - "成立子公司"
    - "认证"
    - "合作协议"
    - "partnership"
    - "certification"
    - "approval"

categories:
  level_1:
    - "全球AI产业动态"
    - "名单企业动态"

  level_2:
    - "企业重大动态"
    - "行业大事件"
    - "政策动态"
    - "融资"
    - "投资/并购"
    - "成立新企业"
    - "产品发布"
    - "获得认证"
    - "其他"

source_priority:
  official: 1
  government: 2
  tier1_media: 3
  industry_media: 4
  other: 5

briefing:
  max_highlights: 5
  max_items_per_section: 10
  include_source_name: true
  include_judgement_section: true

logging:
  level: "INFO"
  rotation: "10 MB"
  retention: "30 days"
  encoding: "utf-8"

openai:
  api_key: ""
  summary_model: "gpt-4.1-mini"

ark:
  api_key: ""
  base_url: "https://ark.cn-beijing.volces.com/api/v3"
  summary_model: ""

crawler:
  request_timeout_seconds: 20
  rss_lookback_days: 3
  web_lookback_days: 30

time_window:
  mode: "previous_day"          # previous_day / rolling_days
  timezone: "Asia/Shanghai"

  # mode = previous_day 时使用
  start_hour: 0
  end_hour: 0
  lookback_days: 3

  # mode = rolling_days 时使用
  #lookback_days: 3

path: README.md
content: # AI资讯自动化工作流（ClawHub Skill）

本项目用于自动抓取 AI 产业资讯（RSS + 网页），完成筛选、企业匹配与拆分、分类、去重（保留最早）、豆包（火山方舟）标题/摘要生成与重点事件筛选，并输出 Excel 信息表与 Word 简报。

**入口脚本：** `skill_entry.py`  
**输出目录：** `./output/excel`、`./output/word`

---

## 1. 功能概览

### 1) 资讯来源
- **RSS 来源**：从 `input/source_config.xlsx` 配置读取（`fetch_method=rss`）
- **网页来源**：从 `input/web_source_config.xlsx` 配置读取（`fetch_method=webpage/web`），适用于无 RSS 的机构/平台网站

### 2) 处理流程（概念）
1. 抓取（RSS + Web）
2. 企业匹配（基于企业名单）
3. 企业级拆分：一条资讯只对应一家企业
4. 来源地区约束过滤（国内来源只匹配中国企业等）
5. 分类 / 重要性 / 相关性（按你的服务实现）
6. 去重：批次内去重 + 数据库去重（保留更早发布时间）
7. 豆包生成中文标题 + 摘要（可选，字数80字左右，上限约120）
   - 全球AI产业动态：判断国内/国际
   - 全球AI产业动态：重点事件打分与标记（供 Word Top5 使用）
8. 输出：Excel（完整版+简版） + Word

### 3) 输出文件
- **完整 Excel**：多 sheet（有效、重复、无效等）
- **有效数据简表 Excel**：仅保留关键列 + 颜色/冻结表头
- **Word 简报**：两大部分（企业动态 + 全球动态；全球动态可只写 Top5 重点事件）

---

## 2. 目录结构

运行后目录大致如下：
ai_news_workflow/
app/ # 核心代码
input/ # 输入文件（需准备）
企业名单.xlsx
source_config.xlsx
web_source_config.xlsx
output/ # 输出文件（运行后生成）
excel/
word/
data/ # sqlite 数据库（运行后生成）
logs/ # 日志（运行后生成）
templates/ # 可选：放模板文件
input_templates/
config.yaml # 当前运行配置（不存在时可由默认配置生成）
config.default.yaml # 默认配置（相对路径版，推荐）
requirements.txt
skill.yaml
skill_entry.py # Skill 入口（推荐运行它）
run.py # 主流程

---

## 3. 环境要求

- Python 3.10+（推荐 3.11）
- 能访问资讯来源站点（RSS/网页）
- 若启用豆包摘要：需要火山方舟 API Key（环境变量 `ARK_API_KEY`）

---

## 4. 安装依赖

在项目目录下执行：

```bash
pip install -r requirements.txt

## 5. 配置文件说明（config.yaml）
推荐做法
保留 config.default.yaml（相对路径）
若根目录下没有 config.yaml，运行 skill_entry.py 时会从 config.default.yaml 复制生成

关键配置项（常用）
paths.company_file：企业名单路径
paths.source_file：RSS 来源配置路径
paths.web_source_file：网页来源配置路径
crawler.rss_lookback_days：RSS 抓取回看天数（建议 3）
crawler.web_lookback_days：网页抓取回看天数（建议 3）
time_window.*：最终统计窗口（日报口径）
ark.*：豆包调用参数（建议 API Key 用环境变量）

##6. 输入文件准备（必须）

把以下 3 个文件放到 ./input/：
企业名单.xlsx
至少包含：企业名称、国家/地区等你代码里会用到的字段
source_config.xlsx（RSS 来源）
含 RSS 源 URL 及启用状态等
web_source_config.xlsx（网页来源）
用于配置无 RSS 的网站抓取规则（列表页 URL、链接过滤、选择器等）
网页来源（web_source_config.xlsx）建议字段
（列名需与你的 SourceLoader 匹配；如果你已跑通测试，就按你现有表头继续）
来源名称
是否启用（TRUE/1）
抓取方式（webpage/web）
地区（国内/国际）
来源类型（official/media/other）
url（建议填写，与 list_url 一致也可）
list_url
url_prefix
link_selector（可空）
link_href_contains（强烈推荐，用于过滤“查看详情”链接）
title_selector/date_selector/content_selector（可空，先用 fallback）

##7. 启用豆包（火山方舟）标题/摘要生成（可选但推荐）

推荐用环境变量，不要把 Key 写进配置文件。
Windows PowerShell：
$env:ARK_API_KEY="你的真实方舟APIKey"
Linux/macOS bash：
export ARK_API_KEY="你的真实方舟APIKey"
注意：环境变量里只放纯 Key，不要加 “Bearer ” 前缀，不要包含中文。

##8. 运行方式
方式 A：初始化（只创建目录与生成默认 config.yaml）
python skill_entry.py --init
会做：
创建 input/output/data/logs/templates 等目录（如不存在）
若没有 config.yaml，从 config.default.yaml 复制生成

方式 B：运行一次完整流程（推荐）
python skill_entry.py
成功后输出：
./output/excel/ 下生成 Excel
./output/word/ 下生成 Word

方式 C：指定配置文件运行
python skill_entry.py --config config.yaml

##9. 输出说明
1) Excel（完整）
一般包含：
news_table（有效数据）
duplicate_items（重复数据）
invalid_items（无效数据）
以及其它辅助 sheet（视你的 exporter 实现）

2) Excel（有效数据简表）
表头蓝底 + 冻结首行
按一级分类排序
行底色区分不同一级类别
可扩展：全球AI产业动态内按 “国内/国际” 排序

3) Word 简报
仅保留两部分：
一、“一带”AI领军企业动态
二、全球AI产业动态
可选逻辑：
全球AI产业动态只输出 5 个重点事件（由豆包综合“机构全球知名度 + 事件产业影响力”打分选出 Top5）
Excel 不变

##10. 常见问题排查
Q1：运行秒结束，没有输出文件
确认你运行的是 skill_entry.py（而不是只 import 的文件）
skill_entry.py 必须包含：
if __name__ == "__main__": ...
并且在里面调用 main(...)

Q2：web_sources=0（网页来源没被读到）
检查：
config.yaml 是否配置了 paths.web_source_file
web_source_config.xlsx 的 sheet 名是否为 sources（或与你 loader 默认一致）
“是否启用”是否为 TRUE/1
“抓取方式”是否为 webpage/web

Q3：网页来源抓到 0 条
检查：
list_url 是否可访问
link_href_contains 是否写对（建议用来过滤详情页链接）
网站是否强前端渲染（需要 Playwright 才能抓）——若是这种站，需要单独升级 crawler

Q4：豆包调用失败/被跳过
检查：
是否设置了 ARK_API_KEY
Key 是否包含中文、空格、或 “Bearer ” 前缀
网络是否可访问火山方舟接口
是否有调用额度/配额限制

Q5：Word 比有效数据简表少
常见原因：
Word 的全球AI产业动态只输出 Top5 重点事件（这是设计要求）
或你的 briefing_service.py 做了筛选/截断（建议对齐逻辑）

11. 生产化建议（定时任务）
Windows：任务计划程序（Task Scheduler）
Linux：crontab
建议每天定时执行：
python skill_entry.py

并将：
time_window 设置为日报口径（例如前一天 00:00 ~ 当天 00:00）
抓取回看天数设置为 3 天以上（防止来源延迟更新）

## 运行参数（覆盖 config.yaml）

参数优先级：命令行参数 > config.yaml > 默认值

### 抓取回看范围（Lookback Days）
- `--rss-lookback-days N`：RSS 抓取回看 N 天
- `--web-lookback-days N`：网页抓取回看 N 天

示例：抓取最近 7 天
```bash
python skill_entry.py --rss-lookback-days 7 --web-lookback-days 7

12. 版本信息
v0.1.1：支持 RSS+网页抓取、企业匹配与拆分、去重、豆包摘要与重点事件、Excel+Word 输出、统计时间自定义


path: requirements.txt
content: pandas>=2.2.0
openpyxl>=3.1.2
python-docx>=1.1.0
requests>=2.31.0
beautifulsoup4>=4.12.3
lxml>=5.1.0
feedparser>=6.0.11
PyYAML>=6.0.1
loguru>=0.7.2
python-dateutil>=2.9.0
pytz>=2024.1
pydantic>=2.6.1
rapidfuzz>=3.6.1
deep-translator>=1.11.4
openai>=2.24.0

path: skill.yaml
content: name: ai_news_workflow
display_name: AI资讯自动化工作流
version: 0.1.1
description: 抓取RSS+网页资讯，筛选去重分类，生成Excel与Word简报，支持豆包摘要与重点事件筛选
language: python
entrypoint: skill_entry.py:main

# 可选：暴露命令（如果 ClawHub 支持）
commands:
  - name: run
    description: 运行一次资讯采集与产出
    args:
      - name: config
        type: string
        required: false
        default: config.yaml
      - name: init
        type: boolean
        required: false
        default: false

path: skill_entry.py
content: from __future__ import annotations

import argparse
import shutil
from pathlib import Path
import sys

PROJECT_ROOT = Path(__file__).resolve().parent
sys.path.insert(0, str(PROJECT_ROOT))

from run import main as run_main


def _ensure_dirs(root: Path) -> None:
    for d in ["input", "output", "data", "logs", "templates"]:
        (root / d).mkdir(parents=True, exist_ok=True)
    (root / "output" / "excel").mkdir(parents=True, exist_ok=True)
    (root / "output" / "word").mkdir(parents=True, exist_ok=True)


def _ensure_config(root: Path, config_name_or_path: str) -> Path:
    """
    确保配置文件存在：
    - 如果用户指定的 config 存在，直接用
    - 否则尝试从 config.default.yaml 复制到 root/config.yaml
    """
    cfg = Path(config_name_or_path)
    if not cfg.is_absolute():
        cfg = (root / cfg).resolve()

    if cfg.exists():
        return cfg

    default_cfg = root / "config.default.yaml"
    target_cfg = root / "config.yaml"
    if default_cfg.exists():
        shutil.copy(default_cfg, target_cfg)
        return target_cfg

    raise FileNotFoundError(f"未找到配置文件: {cfg}，且不存在 {default_cfg}")


def main() -> str:
    parser = argparse.ArgumentParser()
    parser.add_argument("--config", default="config.yaml", help="配置文件名或路径（默认 config.yaml）")
    parser.add_argument("--init", action="store_true", help="仅初始化目录与默认配置，不执行主流程")

    # 模型可选
    parser.add_argument("--model", default="", help="LLM模型名（例如 doubao-seed-2-0-mini-260215）。留空则使用配置。")

    # 抓取回看天数（覆盖 config）
    parser.add_argument("--rss-lookback-days", type=int, default=0, help="RSS抓取回看天数（>0 生效，覆盖配置）")
    parser.add_argument("--web-lookback-days", type=int, default=0, help="网页抓取回看天数（>0 生效，覆盖配置）")

    # 统计窗口（覆盖 config）
    parser.add_argument("--window-mode", default="", help="统计窗口模式：previous_day 或 rolling_days（留空则用配置）")
    parser.add_argument("--start-hour", type=int, default=-1, help="previous_day 模式：起始小时（0-23，-1表示不覆盖）")
    parser.add_argument("--end-hour", type=int, default=-1, help="previous_day 模式：结束小时（0-23，-1表示不覆盖）")
    parser.add_argument("--window-lookback-days", type=int, default=0, help="rolling_days 模式：回看天数（>0 生效，覆盖配置）")

    args = parser.parse_args()

    root = PROJECT_ROOT
    print("[skill] start")
    print("[skill] PROJECT_ROOT =", root)

    _ensure_dirs(root)
    cfg_path = _ensure_config(root, args.config)
    print("[skill] config =", cfg_path)

    if args.init:
        print("[skill] init only, done")
        return f"初始化完成：{cfg_path}"

    print("[skill] running workflow...")

    # 只传 run.py 能接收的参数；init 不传给 run_main
    run_main(
        config_path=str(cfg_path),
        model=(args.model.strip() or None),
        rss_lookback_days=(args.rss_lookback_days if args.rss_lookback_days > 0 else None),
        web_lookback_days=(args.web_lookback_days if args.web_lookback_days > 0 else None),
        window_mode=(args.window_mode.strip() or None),
        start_hour=(args.start_hour if args.start_hour >= 0 else None),
        end_hour=(args.end_hour if args.end_hour >= 0 else None),
        window_lookback_days=(args.window_lookback_days if args.window_lookback_days > 0 else None),
    )

    print("[skill] workflow done")
    return "运行完成"


if __name__ == "__main__":
    main()

Security scan verdicts — ClawScan: clean | static analysis: clean | VirusTotal: clean | SkillSpector: suspicious (HIGH)