Upload 2 files

app.py

@@ -7,20 +7,14 @@ import json
 import chardet
 from sklearn.metrics import precision_score, recall_score, f1_score
 import time
-from nlp_pipeline import process_text
-
 # ======================== 数据库模块 ========================
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 from contextlib import contextmanager
 import logging
 
-# 日志配置
-logger = logging.getLogger(__name__)
-logging.basicConfig(
-    level=logging.INFO,
-    format="%(asctime)s - %(levelname)s - %(message)s"
-)
+# 配置日志
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
 
 # 使用SQLAlchemy的连接池来管理数据库连接
 DATABASE_URL = "mysql+pymysql://user:password@host/dbname"  # 请根据实际情况修改连接字符串
@@ -415,41 +409,31 @@ def process_text(text, model_type="bert"):
 
 
 def process_file(file, model_type="bert"):
-    # 读取二进制
-    with open(file.name, 'rb') as f:
-        raw = f.read()
-    if len(raw) > 5 * 1024 * 1024:
-        return "❌ 文件太大", "", "", ""
-
-    detect = chardet.detect(raw)
-    encoding = detect.get('encoding')
-    logger.info(f"chardet 猜测编码：{encoding} (置信度 {detect.get('confidence'):.2f})")
-
-    text = None
-    if encoding:
-        try:
-            text = raw.decode(encoding)
-        except UnicodeDecodeError:
-            logger.warning(f"{encoding} 解码失败，尝试其他编码")
+    try:
+        with open(file.name, 'rb') as f:
+            content = f.read()
 
-    if text is None:
-        for enc in ['utf-8','gb18030','utf-16','big5','shift_jis','iso-8859-1']:
-            try:
-                text = raw.decode(enc)
-                logger.info(f"成功使用 {enc} 解码")
-                break
-            except UnicodeDecodeError:
-                continue
+        if len(content) > 5 * 1024 * 1024:
+            return "❌ 文件太大", "", "", ""
 
-    if text is None:
+        # 检测编码
         try:
-            text = raw.decode('utf-8', errors='replace')
-            logger.warning("所有严格解码失败，使用 utf-8+replace 模式")
-        except Exception:
-            return "❌ 编码解析失败（所有尝试均失败）", "", "", ""
-
-    return process_text(text, model_type)
+            encoding = chardet.detect(content)['encoding'] or 'utf-8'
+            text = content.decode(encoding)
+        except UnicodeDecodeError:
+            # 尝试常见中文编码
+            for enc in ['gb18030', 'utf-16', 'big5'] :
+                try:
+                    text = content.decode(enc)
+                    break
+                except:
+                    continue
+            else:
+                return "❌ 编码解析失败", "", "", ""
 
+        return process_text(text, model_type)
+    except Exception as e:
+        return f"❌ 文件处理错误: {str(e)}", "", "", ""
 
 
 

requirements.txt

@@ -1,28 +1,15 @@
-# ========== 深度学习 & 加速 ==========
-torch>=2.1.0,<3.0.0
-transformers==4.39.3
-accelerate>=0.27.0,<1.0.0
-sentencepiece>=0.2.0,<0.3.0
-cpm-kernels>=1.0.11,<2.0.0
-
-# ========== 自然语言处理 ==========
-networkx>=3.0,<4.0
-scikit-learn>=1.3.0,<2.0.0
-chardet>=5.2.0,<6.0.0
-protobuf==3.20.3
-
-# ========== 数据 & 可视化 ==========
-pandas>=2.1.0,<3.0.0
-pyvis>=0.3.2,<0.4.0
-
-# ========== Web 服务 ==========
 gradio==3.50.2
-uvicorn[standard]>=0.22.0,<1.0.0     # Gradio 部署时可选
-gunicorn>=20.1.0,<21.0.0             # 生产环境 WSGI
-
-# ========== 配置管理 ==========
-python-dotenv>=1.0.0,<2.0.0
-
-# ========== 数据库 ==========
-sqlalchemy>=1.4,<2.0.0
+transformers==4.39.3
+torch>=2.1.0,<3.0.0
+networkx>=3.0
+python-dotenv>=1.0.0
+sentencepiece>=0.2.0
+cpm-kernels>=1.0.11 
+accelerate>=0.27.0
+scikit-learn>=1.3.0
+chardet>=5.2.0
+pandas>=2.1.0
+pyvis>=0.3.2
 pymysql==1.1.0
+protobuf==3.20.3  # 避免与新版transformers冲突
+sqlalchemy>=1.4