MergedUserSearchWithSupplierAlgorithm

app.py

@@ -16,7 +16,6 @@ from typing import Any, Optional
 
 import torch
 from flask import Flask, jsonify, request
-from sentence_transformers import SentenceTransformer
 
 # Импорт экстракторов
 from extractors import (
@@ -31,7 +30,6 @@ from extractors import (
 HF_TOKEN = os.getenv("HF_TOKEN")
 
 DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
-_MODEL: Optional[SentenceTransformer] = None
 _WHISPER_MODEL: Optional[Any] = None
 _WHISPER_PROCESSOR: Optional[Any] = None
 
@@ -139,16 +137,6 @@ TEST_PHRASES = [
 ]
 
 
-def get_embedding_model() -> SentenceTransformer:
-    """Возвращает модель эмбеддингов (ленивая загрузка)."""
-    global _MODEL
-
-    if _MODEL is None:
-        _MODEL = SentenceTransformer("Qwen/Qwen3-Embedding-0.6B", device=DEVICE)
-
-    return _MODEL
-
-
 def get_whisper_pipeline() -> Any:
     """Возвращает Whisper pipeline (ленивая загрузка)."""
     global _WHISPER_MODEL, _WHISPER_PROCESSOR
@@ -160,7 +148,7 @@ def get_whisper_pipeline() -> Any:
 
         model = AutoModelForSpeechSeq2Seq.from_pretrained(
             model_id,
-            torch_dtype=torch.float32,
+            dtype=torch.float32,
             low_cpu_mem_usage=True,
             use_safetensors=True,
         )
@@ -204,60 +192,49 @@ class ExpenseTextExtractor:
         self.date_extractor = ExpenseDateExtractor()
         self.supplier_extractor = ExpenseSupplierExtractor(suppliers=suppliers)
         self.amount_extractor = ExpenseAmountExtractor(suppliers=suppliers)
-        self.user_extractor = ExpenseUserExtractor(
-            users=users, 
-            suppliers=suppliers, 
-            model=get_embedding_model()
-        )
+        self.user_extractor = ExpenseUserExtractor(users=users, suppliers=suppliers)
 
     def extract(
-        self, 
-        text: str, 
-        reference_date: str | date | None = None, 
-        debug_supplier: bool = False
+        self,
+        text: str,
+        reference_date: str | date | None = None,
+        debug: bool = False,
     ) -> dict[str, Any]:
-        """
-        Извлекает все данные из текста.
-        
-        Args:
-            text: Текст для анализа
-            reference_date: Базовая дата
-            debug_supplier: Включить отладку поставщиков
-            
-        Returns:
-            Словарь со всеми извлечёнными данными
-        """
-        timings = {}
-        
+        """Извлекает все данные из текста."""
+        timings: dict[str, float] = {}
+
         t0 = time.time()
-        date_info = self.date_extractor.extract(text, reference_date=reference_date)
+        date_info = self.date_extractor.extract(text, reference_date=reference_date, debug=debug)
         timings["date_extractor"] = round(time.time() - t0, 3)
-        
+
         t0 = time.time()
         supplier_info = self.supplier_extractor.extract(
             text,
             date_phrase=date_info.get("matched_date_phrase"),
-            debug=debug_supplier,
+            debug=debug,
         )
         timings["supplier_extractor"] = round(time.time() - t0, 3)
-        
+
         t0 = time.time()
         user_info = self.user_extractor.extract(
             text,
             supplier_phrase=supplier_info.get("matched_supplier_phrase"),
             date_phrase=date_info.get("matched_date_phrase"),
+            debug=debug,
         )
         timings["user_extractor"] = round(time.time() - t0, 3)
-        
+
         t0 = time.time()
         amount_info = self.amount_extractor.extract(
             text,
             matched_date_phrase=date_info["matched_date_phrase"],
             matched_supplier_phrase=supplier_info["matched_supplier_phrase"],
+            debug=debug,
         )
         timings["amount_extractor"] = round(time.time() - t0, 3)
-        
-        print(f"[TIMINGS] {timings}")
+
+        if debug:
+            print(f"[TIMINGS] {timings}")
 
         result = {
             "text": text,
@@ -267,8 +244,16 @@ class ExpenseTextExtractor:
             "date": date_info["date"],
             "date_iso": date_info["date_iso"],
         }
-        if debug_supplier and "supplier_debug" in supplier_info:
-            result["supplier_debug"] = supplier_info["supplier_debug"]
+
+        if debug:
+            result["debug"] = {
+                "timings": timings,
+                "date": date_info.get("date_debug"),
+                "supplier": supplier_info.get("supplier_debug"),
+                "user": user_info.get("user_debug"),
+                "amount": amount_info.get("amount_debug"),
+            }
+
         return result
 
 
@@ -345,10 +330,10 @@ def transcribe_audio_text(audio_path: str, suppliers: list[str] | None = None, u
     raise RuntimeError("Speech-to-text backend is unavailable.")
 
 
-def process_voice_request(audio_path: str, mode: str, payload: dict[str, Any]) -> dict[str, Any]:
+def process_voice_request(audio_path: str, mode: str, payload: dict[str, Any], debug: bool = False) -> dict[str, Any]:
     """Обрабатывает голосовой запрос."""
     total_start = time.time()
-    
+
     context = payload.get("context", {}) if isinstance(payload, dict) else {}
     supplier_names = extract_names(context.get("suppliers"))
     user_names = extract_names(context.get("users"))
@@ -382,12 +367,12 @@ def process_voice_request(audio_path: str, mode: str, payload: dict[str, Any]) -
     pipeline_init_time = round(time.time() - t0, 3)
     print(f"[TIMINGS] pipeline_init: {pipeline_init_time}s")
     
-    extracted = extractor.extract(transcript, reference_date=date.today().isoformat())
-    
+    extracted = extractor.extract(transcript, reference_date=date.today().isoformat(), debug=debug)
+
     total_time = round(time.time() - total_start, 3)
     print(f"[TIMINGS] TOTAL: {total_time}s (whisper: {whisper_time}s)")
 
-    return {
+    payload = {
         "status": "ok",
         "text": transcript,
         "notes": polish_notes_text(extracted.get("text") or transcript),
@@ -396,6 +381,9 @@ def process_voice_request(audio_path: str, mode: str, payload: dict[str, Any]) -
         "date": extracted.get("date_iso") or extracted.get("date"),
         "sum": extracted.get("amount"),
     }
+    if debug and extracted.get("debug"):
+        payload["debug"] = extracted.get("debug")
+    return payload
 
 
 def require_auth():
@@ -455,7 +443,7 @@ def health():
 @app.get("/test-data")
 def test_data():
     """Тестирует извлечение данных из текста без использования Whisper."""
-    debug_supplier = (request.args.get("debug") or "").strip().lower() in {"1", "true", "yes"}
+    debug = (request.args.get("debug") or "").strip().lower() in {"1", "true", "yes"}
     extractor = build_default_pipeline(suppliers=TEST_SUPPLIERS, users=TEST_USERS)
 
     started = time.time()
@@ -466,9 +454,9 @@ def test_data():
         extracted = extractor.extract(
             phrase,
             reference_date=date.today().isoformat(),
-            debug_supplier=debug_supplier,
+            debug=debug,
         )
-        results.append({
+        row = {
             "text": phrase,
             "user": extracted.get("user"),
             "supplier": extracted.get("supplier"),
@@ -476,8 +464,10 @@ def test_data():
             "date": extracted.get("date"),
             "date_iso": extracted.get("date_iso"),
             "processing_time": round(time.time() - item_started, 3),
-            **({"supplier_debug": extracted.get("supplier_debug")} if debug_supplier and extracted.get("supplier_debug") else {}),
-        })
+        }
+        if debug and extracted.get("debug"):
+            row["debug"] = extracted.get("debug")
+        results.append(row)
 
     return jsonify({
         "status": "ok",
@@ -500,6 +490,7 @@ def process_audio():
 
     audio = request.files.get("audio")
     mode = (request.form.get("mode") or "expense").strip()
+    debug = ((request.form.get("debug") or request.args.get("debug") or "").strip().lower() in {"1", "true", "yes"})
     context = parse_context(request.form.get("context"))
 
     if audio is None:
@@ -513,7 +504,7 @@ def process_audio():
             temp_path = temp_file.name
             audio.save(temp_file)
 
-        result = process_voice_request(audio_path=temp_path, mode=mode, payload={"context": context})
+        result = process_voice_request(audio_path=temp_path, mode=mode, payload={"context": context}, debug=debug)
         return jsonify(result)
     except Exception as exception:
         return jsonify({"status": "error", "message": str(exception)}), 422

extractors/amount_extractor.py

@@ -42,21 +42,47 @@ class ExpenseAmountExtractor:
         text: str,
         matched_date_phrase: Optional[str] = None,
         matched_supplier_phrase: Optional[str] = None,
+        debug: bool = False,
     ) -> dict[str, Any]:
         date_span = self.phrase_span(text, matched_date_phrase)
         supplier_span = self.phrase_span(text, matched_supplier_phrase)
+        candidates: list[dict[str, Any]] = []
 
         for match in AMOUNT_PATTERN.finditer(text):
             span = match.span()
+            overlaps_date = self.overlaps(span, date_span)
+            overlaps_supplier = self.overlaps(span, supplier_span)
+            amount_text = match.group(0)
 
-            if self.overlaps(span, date_span):
-                continue
-            if self.overlaps(span, supplier_span):
+            if debug:
+                candidates.append({
+                    "value": amount_text,
+                    "span": [span[0], span[1]],
+                    "overlaps_date": overlaps_date,
+                    "overlaps_supplier": overlaps_supplier,
+                })
+
+            if overlaps_date or overlaps_supplier:
                 continue
 
-            amount_text = match.group(0)
             amount = self.to_float(amount_text)
             if amount is not None:
-                return {"amount": amount, "amount_text": amount_text}
+                payload = {"amount": amount, "amount_text": amount_text}
+                if debug:
+                    payload["amount_debug"] = {
+                        "date_span": list(date_span) if date_span else None,
+                        "supplier_span": list(supplier_span) if supplier_span else None,
+                        "candidates": candidates,
+                        "selected": amount_text,
+                    }
+                return payload
 
-        return {"amount": None, "amount_text": None}
+        payload = {"amount": None, "amount_text": None}
+        if debug:
+            payload["amount_debug"] = {
+                "date_span": list(date_span) if date_span else None,
+                "supplier_span": list(supplier_span) if supplier_span else None,
+                "candidates": candidates,
+                "selected": None,
+            }
+        return payload

extractors/date_extractor.py

@@ -492,26 +492,37 @@ class ExpenseDateExtractor:
     def __init__(self) -> None:
         self.parser = UniversalDateParser()
 
-    def extract(self, text: str, reference_date: str | date | None = None) -> dict[str, Any]:
+    def extract(self, text: str, reference_date: str | date | None = None, debug: bool = False) -> dict[str, Any]:
         """
         Извлекает дату из текста.
-        
+
         Args:
             text: Текст для анализа
             reference_date: Базовая дата (по умолчанию сегодня)
-            
+            debug: Включить отладочную информацию
+
         Returns:
             Словарь с date, date_iso, matched_date_phrase
         """
         ref_date = self.to_date(reference_date or date.today().isoformat())
         parsed = self.parser.parse(text=text, reference_date=ref_date)
 
-        return {
+        payload = {
             "date": datetime.strptime(parsed.date_iso, "%Y-%m-%d").strftime("%d.%m.%Y") if parsed else None,
             "date_iso": parsed.date_iso if parsed else None,
             "matched_date_phrase": parsed.matched_expression if parsed else None,
         }
 
+        if debug:
+            payload["date_debug"] = {
+                "reference_date": ref_date.isoformat(),
+                "input_text": text,
+                "matched_date_phrase": payload["matched_date_phrase"],
+                "date_iso": payload["date_iso"],
+            }
+
+        return payload
+
     @staticmethod
     def to_date(value: str | date) -> date:
         """Преобразует строку или date в date."""

extractors/supplier_extractor.py

@@ -143,7 +143,7 @@ class ExpenseSupplierExtractor:
         self.lexical_token_cache: dict[str, float] = {}
         self.phrase_support_cache: dict[str, float] = {}
         self.noise_terms = {
-            "за", "на", "из", "для", "под", "над", "при", "без", "и", "или",
+            "для", "под", "над", "при", "без", "или",
             "купил", "купила", "купили", "покупка", "заказал", "заказала", "заказали",
             "оплатил", "оплатила", "оплатили", "заплатил", "заплатила", "заплатили",
             "был", "была", "было", "были", "утром", "днем", "днём", "вечером", "ночью",
@@ -290,15 +290,22 @@ class ExpenseSupplierExtractor:
                 best = {"supplier": self.suppliers[i], "score": local, "phrase": phrase, "variant": local_variant}
         return best
 
-    def extract(self, text: str, date_phrase: str | None = None, debug: bool = False) -> dict[str, Any]:
+    def extract(
+        self,
+        text: str,
+        date_phrase: str | None = None,
+        excluded_phrases: list[str] | None = None,
+        debug: bool = False,
+    ) -> dict[str, Any]:
         """
         Извлекает поставщика из текста.
-        
+
         Args:
             text: Текст для анализа
             date_phrase: Фраза даты для исключения
+            excluded_phrases: Дополнительные фразы для исключения
             debug: Включить отладочную информацию
-            
+
         Returns:
             Словарь с supplier, supplier_score, matched_supplier_phrase
         """
@@ -306,6 +313,10 @@ class ExpenseSupplierExtractor:
         excluded_tokens: set[str] = set()
         if date_phrase:
             excluded_tokens.update(normalize_text(date_phrase).split())
+        if excluded_phrases:
+            for phrase in excluded_phrases:
+                if phrase:
+                    excluded_tokens.update(normalize_text(phrase).split())
         excluded_tokens.update(self.noise_terms)
 
         raw_tokens = normalize_text(text).split()
@@ -396,6 +407,7 @@ class ExpenseSupplierExtractor:
             payload["supplier_debug"] = {
                 "tokens": tokens,
                 "phrases_count": len(phrases),
+                "excluded_tokens": sorted(excluded_tokens)[:80],
                 "top_candidates": top_candidates,
             }
 

extractors/user_extractor.py

@@ -1,152 +1,72 @@
-"""
-Экстрактор пользователей из текста.
-
-Использует семантические эмбеддинги для поиска пользователей.
-"""
+"""Экстрактор пользователей на той же логике, что и поиск поставщика."""
 
 from __future__ import annotations
 
 import re
-import unicodedata
 from typing import Any
 
-import torch
-from pymorphy3 import MorphAnalyzer
-from sentence_transformers import SentenceTransformer
-
-
-MORPH = MorphAnalyzer()
-
-
-def normalize_text(text: str) -> str:
-    """Нормализует текст: lowercase, удаление диакритики и пунктуации."""
-    text = unicodedata.normalize("NFKD", text.lower())
-    text = "".join(ch for ch in text if not unicodedata.combining(ch))
-    return re.sub(r"[^\w\s]", "", text).strip()
-
-
-def tokenize_text(text: str) -> list[str]:
-    """Токенизирует текст."""
-    return normalize_text(text).split()
-
-
-def lemmatize_word(word: str) -> str:
-    """Возвращает лемму слова."""
-    return MORPH.parse(word)[0].normal_form if re.fullmatch(r"[а-я]+", word) else word
-
-
-def lemmatize_text(text: str) -> list[str]:
-    """Лемматизирует текст."""
-    return [lemmatize_word(word) for word in tokenize_text(text)]
+from extractors.supplier_extractor import ExpenseSupplierExtractor, normalize_text
 
 
 class ExpenseUserExtractor:
-    """
-    Экстрактор пользователей из текста.
-
-    Сначала использует точное/текстовое совпадение имени, а затем
-    один батч эмбеддингов для оставшихся кандидатов.
-    """
+    """Ищет пользователя тем же fuzzy-matcher, что и поставщика."""
 
     def __init__(
         self,
         users: list[str],
         suppliers: list[str],
-        model: SentenceTransformer,
-        threshold: float = 0.6
+        model: Any = None,
+        threshold: float = 0.5,
     ) -> None:
         self.users = users
-        self.model = model
         self.threshold = threshold
         self.supplier_terms = {normalize_text(supplier) for supplier in suppliers}
-        self.user_terms = [normalize_text(user) for user in users]
-        self.user_embeddings = model.encode(
-            [f"passage: {user}" for user in self.user_terms],
-            convert_to_tensor=True,
-            normalize_embeddings=True,
-            show_progress_bar=False,
-        )
-
-    @staticmethod
-    def _contains_whole_phrase(text: str, phrase: str) -> bool:
-        if not phrase:
-            return False
-        return re.search(rf"(?<!\w){re.escape(phrase)}(?!\w)", text) is not None
-
-    def _extract_candidates(self, text: str, excluded_tokens: set[str]) -> list[str]:
-        candidates: list[str] = []
-        seen: set[str] = set()
-
-        for word in lemmatize_text(text):
-            if len(word) < 3:
-                continue
-            if word in excluded_tokens or word in self.supplier_terms or word in seen:
-                continue
-            seen.add(word)
-            candidates.append(word)
-
-        return candidates
+        self.user_matcher = ExpenseSupplierExtractor(suppliers=users)
 
     def extract(
         self,
         text: str,
         supplier_phrase: str | None = None,
-        date_phrase: str | None = None
+        date_phrase: str | None = None,
+        debug: bool = False,
     ) -> dict[str, Any]:
-        excluded_tokens: set[str] = set()
-        if supplier_phrase:
-            excluded_tokens.update(normalize_text(supplier_phrase).split())
-        if date_phrase:
-            excluded_tokens.update(normalize_text(date_phrase).split())
-
         normalized_text = normalize_text(text)
 
-        for user_term, original_user in zip(self.user_terms, self.users):
-            if user_term and self._contains_whole_phrase(normalized_text, user_term):
-                return {
-                    "user": original_user,
-                    "user_score": 1.0,
-                    "matched_user_phrase": user_term,
-                }
-
         if re.search(r"(?<!\S)я(?!\S)", normalized_text, re.IGNORECASE):
-            return {
+            payload = {
                 "user": "Я",
                 "user_score": 1.0,
                 "matched_user_phrase": "я",
             }
+            if debug:
+                payload["user_debug"] = {
+                    "mode": "direct-pronoun",
+                    "normalized_text": normalized_text,
+                }
+            return payload
 
-        candidates = self._extract_candidates(text, excluded_tokens)
-        if not candidates:
-            return {
-                "user": None,
-                "user_score": None,
-                "matched_user_phrase": None,
-            }
-
-        query_embeddings = self.model.encode(
-            [f"query: {word}" for word in candidates],
-            convert_to_tensor=True,
-            normalize_embeddings=True,
-            show_progress_bar=False,
-            batch_size=max(1, min(32, len(candidates))),
+        match = self.user_matcher.extract(
+            text=text,
+            date_phrase=date_phrase,
+            excluded_phrases=[supplier_phrase] if supplier_phrase else None,
+            debug=debug,
         )
 
-        similarity_matrix = torch.matmul(query_embeddings, self.user_embeddings.T)
-        flat_index = int(torch.argmax(similarity_matrix))
-        candidate_index = flat_index // len(self.users)
-        user_index = flat_index % len(self.users)
-        score = similarity_matrix[candidate_index, user_index].item()
+        score = match.get("supplier_score")
+        payload = {
+            "user": match.get("supplier") if score is not None and score >= self.threshold else None,
+            "user_score": score if score is not None and score >= self.threshold else None,
+            "matched_user_phrase": match.get("matched_supplier_phrase") if score is not None and score >= self.threshold else None,
+        }
 
-        if score >= self.threshold:
-            return {
-                "user": self.users[user_index],
-                "user_score": round(score, 4),
-                "matched_user_phrase": candidates[candidate_index],
+        if debug:
+            payload["user_debug"] = {
+                "mode": "supplier-matcher",
+                "threshold": self.threshold,
+                "excluded_supplier_phrase": supplier_phrase,
+                "normalized_text": normalized_text,
+                "matcher_debug": match.get("supplier_debug"),
             }
 
-        return {
-            "user": None,
-            "user_score": None,
-            "matched_user_phrase": None,
-        }
+        return payload
+