SolvedCommitConflicts

app.py

@@ -62,937 +62,6 @@ def get_whisper_model() -> Any:
     return _WHISPER_MODEL
 
 
-<<<<<<< HEAD
-=======
-def normalize_text(text: str) -> str:
-    text = unicodedata.normalize("NFKD", text.lower())
-    text = "".join(ch for ch in text if not unicodedata.combining(ch))
-    return re.sub(r"[^\w\s]", "", text).strip()
-
-
-def tokenize_text(text: str) -> list[str]:
-    return normalize_text(text).split()
-
-
-def lemmatize_word(word: str) -> str:
-    return MORPH.parse(word)[0].normal_form if re.fullmatch(r"[а-я]+", word) else word
-
-
-def lemmatize_text(text: str) -> list[str]:
-    return [lemmatize_word(word) for word in tokenize_text(text)]
-
-
-def variants(text: str) -> list[str]:
-    base = normalize_text(text)
-    result = [base]
-
-    for schema in (iuliia.WIKIPEDIA, iuliia.MOSMETRO, iuliia.ALA_LC):
-        try:
-            v = normalize_text(schema.translate(base))
-            if v and v not in result:
-                result.append(v)
-        except Exception:
-            pass
-
-    for v in list(result):
-        core = " ".join(w for w in v.split() if len(w) > 1 and any(ch.isalpha() for ch in w))
-        core = normalize_text(core)
-        if core and core not in result:
-            result.insert(0, core)
-
-    return result
-
-
-def token_alignment_score(phrase_variant: str, candidate_tokens: list[str]) -> float:
-    phrase_tokens = [t for t in phrase_variant.split() if len(t) > 2]
-    if not phrase_tokens or not candidate_tokens:
-        return 0.0
-    best_scores = []
-    for pt in phrase_tokens:
-        best = 0.0
-        for ct in candidate_tokens:
-            sim = Levenshtein.normalized_similarity(pt, ct)
-            if sim > best:
-                best = sim
-        best_scores.append(best)
-    return sum(best_scores) / len(best_scores)
-
-
-def length_penalty(phrase_len: int, candidate_len: int) -> float:
-    if phrase_len == 0 or candidate_len == 0:
-        return 0.0
-    ratio = min(phrase_len, candidate_len) / max(phrase_len, candidate_len)
-    if ratio >= 0.80:
-        return 1.0
-    if ratio >= 0.60:
-        return 0.90
-    if ratio >= 0.40:
-        return 0.70
-    return 0.50
-
-
-def canonicalize_for_similarity(text: str) -> str:
-    t = normalize_text(text).replace(" ", "")
-    replacements = (
-        ("sch", "sh"),
-        ("tch", "ch"),
-        ("dzh", "j"),
-        ("zh", "j"),
-        ("sh", "s"),
-        ("ch", "c"),
-        ("kh", "h"),
-        ("ph", "f"),
-        ("ck", "k"),
-        ("qu", "k"),
-        ("q", "k"),
-        ("w", "v"),
-        ("x", "ks"),
-        ("ts", "z"),
-        ("tz", "z"),
-    )
-    for src, dst in replacements:
-        t = t.replace(src, dst)
-    return re.sub(r"(.)\1+", r"\1", t)
-
-
-def phonetic_similarity(left: str, right: str) -> float:
-    l = canonicalize_for_similarity(left)
-    r = canonicalize_for_similarity(right)
-    if not l or not r:
-        return 0.0
-    char = fuzz.ratio(l, r) / 100.0
-    lev = Levenshtein.normalized_similarity(l, r)
-    return 0.50 * char + 0.50 * lev
-
-
-@dataclass(frozen=True)
-class ParsedDate:
-    date_iso: str
-    matched_expression: Optional[str]
-
-
-@dataclass(frozen=True)
-class Token:
-    original: str
-    normalized: str
-    raw_lemma: str
-    lemma: str
-    lemma_correction: Optional[str]
-    start: int
-    end: int
-    lemma_start: int
-    lemma_end: int
-
-
-WORD_RE = re.compile(r"[0-9]+(?:[./-][0-9]+)*|[а-яё]+", re.IGNORECASE)
-
-
-class UniversalDateParser:
-    MONTHS = {
-        "январь": 1, "февраль": 2, "март": 3, "апрель": 4, "май": 5, "июнь": 6,
-        "июль": 7, "август": 8, "сентябрь": 9, "октябрь": 10, "ноябрь": 11, "декабрь": 12,
-    }
-    WEEKDAYS = {
-        "понедельник": 0, "вторник": 1, "среда": 2, "четверг": 3,
-        "пятница": 4, "суббота": 5, "воскресенье": 6,
-    }
-    DIRECT_RELATIVE = {"послезавтра": 2, "позавчера": -2, "сегодня": 0, "вчера": -1, "завтра": 1}
-    ORDINAL_DAYS = {
-        "первый": 1, "второй": 2, "третий": 3, "четвертый": 4, "пятый": 5, "шестой": 6,
-        "седьмой": 7, "восьмой": 8, "девятый": 9, "десятый": 10, "одиннадцатый": 11,
-        "двенадцатый": 12, "тринадцатый": 13, "четырнадцатый": 14, "пятнадцатый": 15,
-        "шестнадцатый": 16, "семнадцатый": 17, "восемнадцатый": 18, "девятнадцатый": 19,
-        "двадцатый": 20, "двадцать первый": 21, "двадцать второй": 22, "двадцать третий": 23,
-        "двадцать четвертый": 24, "двадцать пятый": 25, "двадцать шестой": 26,
-        "двадцать седьмой": 27, "двадцать восьмой": 28, "двадцать девятый": 29,
-        "тридцатый": 30, "тридцать п��рвый": 31,
-    }
-    NUMBER_WORDS = {
-        "ноль": 0, "один": 1, "два": 2, "три": 3, "четыре": 4, "пять": 5, "шесть": 6,
-        "семь": 7, "восемь": 8, "девять": 9, "десять": 10, "одиннадцать": 11,
-        "двенадцать": 12, "тринадцать": 13, "четырнадцать": 14, "пятнадцать": 15,
-        "шестнадцать": 16, "семнадцать": 17, "восемнадцать": 18, "девятнадцать": 19,
-        "двадцать": 20, "тридцать": 30,
-    }
-    FUTURE_HINTS = ("завтра", "послезавтра", "через", "быть", "заплатить", "следующий", "последующий")
-    PAST_HINTS = ("вчера", "позавчера", "назад", "прошлый", "предыдущий", "оплатить", "купить", "заказать")
-
-    DIRECT_RELATIVE_RE = re.compile(r"(?<!\S)(послезавтра|позавчера|сегодня|вчера|завтра)(?!\S)")
-    WEEK_RELATIVE_RE = re.compile(
-        r"(?<!\S)на (?P<which>следующий|последующий|прошлый|предыдущий|этот) неделя"
-        r"(?: (?P<prep>в|во|на) (?P<weekday>понедельник|вторник|среда|четверг|пятница|суббота|воскресенье))?(?!\S)"
-    )
-    QUANTITY_RELATIVE_RE = re.compile(
-        r"(?<!\S)(?P<number>\d+|[а-яё]+(?: [а-яё]+)?) "
-        r"(?P<unit>месяц|неделя|день) "
-        r"(?P<ago>назад)"
-        r"(?: (?P<prep>в|во|на) (?P<weekday>понедельник|вторник|среда|четверг|пятница|суббота|воскресенье))?(?!\S)",
-        re.IGNORECASE,
-    )
-    FORWARD_QUANTITY_RE = re.compile(
-        r"(?<!\S)(?P<through>через) "
-        r"(?P<number>\d+|[а-яё]+(?: [а-яё]+)?) "
-        r"(?P<unit>месяц|неделя|день)"
-        r"(?: (?P<prep>в|во|на) (?P<weekday>понедельник|вторник|среда|четверг|пятница|суббота|воскресенье))?(?!\S)",
-        re.IGNORECASE,
-    )
-    FORWARD_SINGLE_UNIT_RE = re.compile(
-        r"(?<!\S)(?P<through>через) "
-        r"(?P<unit>месяц|неделя|день)"
-        r"(?: (?P<prep>в|во|на) (?P<weekday>понедельник|вторник|среда|четверг|пятница|суббота|воскресенье))?(?!\S)",
-        re.IGNORECASE,
-    )
-    TEXTUAL_ABSOLUTE_RE = re.compile(
-        r"(?<!\S)(?P<day>\d{1,2}|[а-яё]+(?: [а-яё]+)?) "
-        r"(?P<month>январь|февраль|март|апрель|май|июнь|июль|август|сентябрь|октябрь|ноябрь|декабрь)"
-        r"(?: (?P<year>\d{4}))?(?!\S)",
-        re.IGNORECASE,
-    )
-    PERIOD_EDGE_RE = re.compile(
-        r"(?<!\S)(?:в )?(?P<edge>начало|конец) (?P<which>этот|следующий|последующий|прошлый|предыдущий) (?P<unit>неделя|месяц)(?!\S)",
-        re.IGNORECASE,
-    )
-
-    @classmethod
-    def temporal_vocabulary(cls) -> set[str]:
-        vocab: set[str] = set()
-        vocab.update(cls.MONTHS)
-        vocab.update(cls.WEEKDAYS)
-        vocab.update(cls.DIRECT_RELATIVE)
-        vocab.update(cls.ORDINAL_DAYS)
-        vocab.update(cls.NUMBER_WORDS)
-        vocab.update({
-            "неделя", "месяц", "день", "назад", "через", "начало", "конец", "на", "в", "во",
-            "этот", "прошлый", "предыдущий", "следующий", "последующий",
-        })
-        return vocab
-
-    @staticmethod
-    def similarity(left: str, right: str) -> float:
-        return difflib.SequenceMatcher(None, left, right).ratio()
-
-    @classmethod
-    def pick_temporal_correction(cls, normalized: str, raw_lemma: str) -> tuple[str, Optional[str]]:
-        vocab = cls.temporal_vocabulary()
-        if raw_lemma in vocab or not normalized.isalpha() or len(normalized) < 5:
-            return raw_lemma, None
-
-        candidates = list(difflib.get_close_matches(normalized, list(vocab), n=4, cutoff=0.74))
-        candidates.extend(difflib.get_close_matches(raw_lemma, list(vocab), n=4, cutoff=0.74))
-        candidates = list(dict.fromkeys(candidates))
-        if not candidates:
-            return raw_lemma, None
-
-        best = max(candidates, key=lambda item: max(cls.similarity(normalized, item), cls.similarity(raw_lemma, item)))
-        best_score = max(cls.similarity(normalized, best), cls.similarity(raw_lemma, best))
-        return (best, f"{raw_lemma}->{best}") if best_score >= 0.80 else (raw_lemma, None)
-
-    @staticmethod
-    def normalize_word(word: str) -> str:
-        return word.lower().replace("ё", "е")
-
-    @classmethod
-    def lemmatize(cls, word: str) -> str:
-        return MORPH.parse(word)[0].normal_form if word.isalpha() else word
-
-    @classmethod
-    def tokenize(cls, text: str) -> list[Token]:
-        tokens: list[Token] = []
-        lemma_cursor = 0
-
-        for match in WORD_RE.finditer(text):
-            original = match.group(0)
-            normalized = cls.normalize_word(original)
-            raw_lemma = cls.lemmatize(normalized)
-            lemma, correction = cls.pick_temporal_correction(normalized, raw_lemma)
-            lemma_start = lemma_cursor
-            lemma_end = lemma_start + len(lemma)
-            tokens.append(Token(original, normalized, raw_lemma, lemma, correction, match.start(), match.end(), lemma_start, lemma_end))
-            lemma_cursor = lemma_end + 1
-
-        return tokens
-
-    @staticmethod
-    def lemma_text(tokens: list[Token]) -> str:
-        return " ".join(token.lemma for token in tokens)
-
-    @staticmethod
-    def surface_text(text: str, tokens: list[Token], start_idx: int, end_idx: int) -> str:
-        return text[tokens[start_idx].start:tokens[end_idx].end].strip() if tokens else ""
-
-    @staticmethod
-    def lemma_span_to_token_range(tokens: list[Token], span: tuple[int, int]) -> Optional[tuple[int, int]]:
-        start_char, end_char = span
-        start_idx = end_idx = None
-
-        for idx, token in enumerate(tokens):
-            if start_idx is None and token.lemma_start <= start_char < token.lemma_end:
-                start_idx = idx
-            if token.lemma_start < end_char <= token.lemma_end:
-                end_idx = idx
-                break
-
-        return (start_idx, end_idx) if start_idx is not None and end_idx is not None else None
-
-    @classmethod
-    def make_parsed_date(cls, text: str, tokens: list[Token], match, parsed_date: date) -> Optional[ParsedDate]:
-        token_span = cls.lemma_span_to_token_range(tokens, match.span())
-        if token_span is None:
-            return None
-        return ParsedDate(parsed_date.isoformat(), cls.surface_text(text, tokens, token_span[0], token_span[1]))
-
-    @classmethod
-    def parse_number_phrase(cls, phrase: str) -> Optional[int]:
-        phrase = phrase.strip()
-        if not phrase:
-            return None
-        if phrase.isdigit():
-            return int(phrase)
-
-        parts = phrase.split()
-        if len(parts) == 1:
-            return cls.NUMBER_WORDS.get(parts[0])
-        if len(parts) == 2 and parts[0] in {"двадцать", "тридцать"}:
-            base = cls.NUMBER_WORDS.get(parts[0])
-            addon = cls.NUMBER_WORDS.get(parts[1])
-            if base is not None and addon is not None and 1 <= addon <= 9:
-                return base + addon
-        return None
-
-    @classmethod
-    def parse_day_phrase(cls, phrase: str) -> Optional[int]:
-        if phrase.isdigit():
-            value = int(phrase)
-            return value if 1 <= value <= 31 else None
-        return cls.ORDINAL_DAYS.get(phrase.strip())
-
-    @staticmethod
-    def shift_months(value: date, months: int) -> date:
-        month_index = value.month - 1 + months
-        year = value.year + month_index // 12
-        month = month_index % 12 + 1
-        day = min(value.day, calendar.monthrange(year, month)[1])
-        return date(year, month, day)
-
-    @staticmethod
-    def parse_numeric_absolute(tokens: list[Token]) -> Optional[ParsedDate]:
-        for token in tokens:
-            separator = "." if "." in token.original else "-" if "-" in token.original else "/" if "/" in token.original else None
-            if separator is None:
-                continue
-
-            parts = token.original.split(separator)
-            if len(parts) != 3 or not all(part.isdigit() for part in parts):
-                continue
-
-            try:
-                if len(parts[0]) == 4:
-                    parsed = date(int(parts[0]), int(parts[1]), int(parts[2]))
-                elif len(parts[2]) == 4:
-                    parsed = date(int(parts[2]), int(parts[1]), int(parts[0]))
-                else:
-                    continue
-                return ParsedDate(parsed.isoformat(), token.original)
-            except ValueError:
-                continue
-
-        return None
-
-    @classmethod
-    def parse_textual_absolute(cls, text: str, tokens: list[Token], reference_date: date) -> Optional[ParsedDate]:
-        lemma_text = cls.lemma_text(tokens)
-        for match in cls.TEXTUAL_ABSOLUTE_RE.finditer(lemma_text):
-            day = cls.parse_day_phrase(match.group("day"))
-            month = cls.MONTHS.get(match.group("month"))
-            if day is None or month is None:
-                continue
-
-            year = int(match.group("year")) if match.group("year") else reference_date.year
-            try:
-                parsed = date(year, month, day)
-            except ValueError:
-                continue
-
-            result = cls.make_parsed_date(text, tokens, match, parsed)
-            if result is not None:
-                return result
-
-        return None
-
-    @classmethod
-    def parse_direct_relative(cls, text: str, tokens: list[Token], reference_date: date) -> Optional[ParsedDate]:
-        lemma_text = cls.lemma_text(tokens)
-        match = cls.DIRECT_RELATIVE_RE.search(lemma_text)
-        if not match:
-            return None
-
-        parsed = reference_date + timedelta(days=cls.DIRECT_RELATIVE[match.group(1)])
-        return cls.make_parsed_date(text, tokens, match, parsed)
-
-    @staticmethod
-    def week_monday(value: date) -> date:
-        return value - timedelta(days=value.weekday())
-
-    @classmethod
-    def parse_week_relative(cls, text: str, tokens: list[Token], reference_date: date) -> Optional[ParsedDate]:
-        lemma_text = cls.lemma_text(tokens)
-        match = cls.WEEK_RELATIVE_RE.search(lemma_text)
-        if not match:
-            return None
-
-        offsets = {"следующий": 7, "последующий": 7, "прошлый": -7, "предыдущий": -7, "этот": 0}
-        anchor = reference_date + timedelta(days=offsets[match.group("which")])
-
-        if match.group("weekday"):
-            anchor = cls.week_monday(anchor) + timedelta(days=cls.WEEKDAYS[match.group("weekday")])
-
-        return cls.make_parsed_date(text, tokens, match, anchor)
-
-    @classmethod
-    def parse_period_edge(cls, text: str, tokens: list[Token], reference_date: date) -> Optional[ParsedDate]:
-        lemma_text = cls.lemma_text(tokens)
-        match = cls.PERIOD_EDGE_RE.search(lemma_text)
-        if not match:
-            return None
-
-        edge, which, unit = match.group("edge"), match.group("which"), match.group("unit")
-
-        if unit == "неделя":
-            offsets = {"прошлый": -7, "предыдущий": -7, "этот": 0, "следующий": 7, "последующий": 7}
-            monday = cls.week_monday(reference_date + timedelta(days=offsets[which]))
-            parsed_date = monday if edge == "начало" else monday + timedelta(days=6)
-        else:
-            month_offset = {"прошлый": -1, "предыдущий": -1, "этот": 0, "следующий": 1, "последующий": 1}[which]
-            shifted = cls.shift_months(date(reference_date.year, reference_date.month, 1), month_offset)
-            parsed_date = shifted if edge == "начало" else date(shifted.year, shifted.month, calendar.monthrange(shifted.year, shifted.month)[1])
-
-        return cls.make_parsed_date(text, tokens, match, parsed_date)
-
-    @classmethod
-    def parse_quantity_relative(cls, text: str, tokens: list[Token], reference_date: date) -> Optional[ParsedDate]:
-        lemma_text = cls.lemma_text(tokens)
-
-        for regex, direction in ((cls.QUANTITY_RELATIVE_RE, -1), (cls.FORWARD_QUANTITY_RE, 1)):
-            for match in regex.finditer(lemma_text):
-                number = cls.parse_number_phrase(match.group("number"))
-                if number is None:
-                    continue
-
-                unit = match.group("unit")
-                if unit == "месяц":
-                    anchor = cls.shift_months(reference_date, direction * number)
-                else:
-                    days = number * 7 if unit == "неделя" else number
-                    anchor = reference_date + timedelta(days=direction * days)
-
-                if match.group("weekday"):
-                    anchor = cls.week_monday(anchor) + timedelta(days=cls.WEEKDAYS[match.group("weekday")])
-
-                result = cls.make_parsed_date(text, tokens, match, anchor)
-                if result is not None:
-                    return result
-
-        for match in cls.FORWARD_SINGLE_UNIT_RE.finditer(lemma_text):
-            unit = match.group("unit")
-            if unit == "месяц":
-                anchor = cls.shift_months(reference_date, 1)
-            else:
-                days = 7 if unit == "неделя" else 1
-                anchor = reference_date + timedelta(days=days)
-
-            if match.group("weekday"):
-                anchor = cls.week_monday(anchor) + timedelta(days=cls.WEEKDAYS[match.group("weekday")])
-
-            result = cls.make_parsed_date(text, tokens, match, anchor)
-            if result is not None:
-                return result
-
-        return None
-
-    @classmethod
-    def preference_for_text(cls, tokens: list[Token]) -> str:
-        lemmas = [token.lemma for token in tokens]
-        future = sum(1 for hint in cls.FUTURE_HINTS if hint in lemmas)
-        past = sum(1 for hint in cls.PAST_HINTS if hint in lemmas)
-        return "future" if future > past else "past"
-
-    @staticmethod
-    def choose_best(matches: list[tuple[str, datetime]]) -> tuple[str, datetime]:
-        return sorted(matches, key=lambda item: (len(item[0]), -item[1].timestamp()), reverse=True)[0]
-
-    def parse(self, text: str, reference_date: date) -> Optional[ParsedDate]:
-        tokens = self.tokenize(text)
-
-        for parser in (
-            lambda: self.parse_numeric_absolute(tokens),
-            lambda: self.parse_textual_absolute(text, tokens, reference_date),
-            lambda: self.parse_direct_relative(text, tokens, reference_date),
-            lambda: self.parse_week_relative(text, tokens, reference_date),
-            lambda: self.parse_period_edge(text, tokens, reference_date),
-            lambda: self.parse_quantity_relative(text, tokens, reference_date),
-        ):
-            parsed = parser()
-            if parsed is not None:
-                return parsed
-
-        normalized = " ".join(token.normalized for token in tokens)
-        relative_base = datetime.combine(reference_date, datetime.min.time()).replace(hour=12)
-        result = search_dates(
-            normalized,
-            languages=["ru"],
-            settings={
-                "RELATIVE_BASE": relative_base,
-                "PREFER_DATES_FROM": self.preference_for_text(tokens),
-                "STRICT_PARSING": False,
-                "REQUIRE_PARTS": [],
-                "NORMALIZE": True,
-                "RETURN_AS_TIMEZONE_AWARE": False,
-                "DATE_ORDER": "DMY",
-            },
-        )
-
-        filtered: list[tuple[str, datetime]] = []
-        for matched, value in result or []:
-            if isinstance(value, datetime) and not matched.strip().isdigit() and 2020 <= value.year <= 2100:
-                filtered.append((matched.strip(), value))
-
-        if not filtered:
-            return None
-
-        matched_expression, value = self.choose_best(filtered)
-        return ParsedDate(date_iso=value.date().isoformat(), matched_expression=matched_expression)
-
-
-class ExpenseDateExtractor:
-    def __init__(self) -> None:
-        self.parser = UniversalDateParser()
-
-    def extract(self, text: str, reference_date: str | date | None = None) -> dict[str, Any]:
-        ref_date = self.to_date(reference_date or date.today().isoformat())
-        parsed = self.parser.parse(text=text, reference_date=ref_date)
-
-        return {
-            "date": datetime.strptime(parsed.date_iso, "%Y-%m-%d").strftime("%d.%m.%Y") if parsed else None,
-            "date_iso": parsed.date_iso if parsed else None,
-            "matched_date_phrase": parsed.matched_expression if parsed else None,
-        }
-
-    @staticmethod
-    def to_date(value: str | date) -> date:
-        return value if isinstance(value, date) else datetime.strptime(value, "%Y-%m-%d").date()
-
-
-# Парсер дат: "natasha" (рекомендуется) или "legacy"
-DATE_PARSER_MODE = os.getenv("DATE_PARSER_MODE", "natasha")
-
-def get_date_extractor():
-    """
-    Возвращает экстрактор дат.
-    - natasha: Лучший для русского языка (по умолчанию)
-    - legacy: Старый код ExpenseDateExtractor
-    """
-    if DATE_PARSER_MODE == "natasha":
-        return NatashaDateExtractor()
-    return ExpenseDateExtractor()
-
-
-class ExpenseUserExtractor:
-    def __init__(self, users: list[str], suppliers: list[str], model: SentenceTransformer, threshold: float = 0.6) -> None:
-        self.users = users
-        self.model = model
-        self.threshold = threshold
-        self.supplier_terms = {normalize_text(supplier) for supplier in suppliers}
-        self.user_terms = [normalize_text(user) for user in users]
-        self.user_embeddings = model.encode(
-            [f"passage: {user}" for user in self.user_terms],
-            convert_to_tensor=True,
-            normalize_embeddings=True,
-        )
-
-    def extract(self, text: str, supplier_phrase: str | None = None, date_phrase: str | None = None) -> dict[str, Any]:
-        excluded_tokens: set[str] = set()
-        if supplier_phrase:
-            excluded_tokens.update(normalize_text(supplier_phrase).split())
-        if date_phrase:
-            excluded_tokens.update(normalize_text(date_phrase).split())
-
-        best_user = None
-        best_score = -1.0
-        best_phrase = None
-
-        for word in lemmatize_text(text):
-            if len(word) < 3:
-                continue
-            if word in excluded_tokens or word in self.supplier_terms:
-                continue
-
-            query_emb = self.model.encode(
-                f"query: {word}",
-                convert_to_tensor=True,
-                normalize_embeddings=True,
-            )
-            similarities = torch.cosine_similarity(query_emb.unsqueeze(0), self.user_embeddings, dim=1)
-            idx = int(torch.argmax(similarities))
-            score = similarities[idx].item()
-
-            if score > best_score:
-                best_score = score
-                best_user = self.users[idx]
-                best_phrase = word
-
-        if best_score >= self.threshold:
-            return {
-                "user": best_user,
-                "user_score": round(best_score, 4),
-                "matched_user_phrase": best_phrase,
-            }
-
-        if re.search(r"(?<!\S)я(?!\S)", normalize_text(text), re.IGNORECASE):
-            return {
-                "user": "Я",
-                "user_score": 1.0,
-                "matched_user_phrase": "я",
-            }
-
-        return {
-            "user": None,
-            "user_score": None,
-            "matched_user_phrase": None,
-        }
-
-
-class ExpenseSupplierExtractor:
-    def __init__(self, suppliers: list[str]) -> None:
-        self.suppliers = suppliers
-        self.sup_norm = [normalize_text(s) for s in suppliers]
-        self.sup_tokens = [s.split() for s in self.sup_norm]
-        self.sup_num_sets = [self.numeric_tokens(s) for s in self.sup_norm]
-        self.sup_number_tokens = {token for supplier in self.sup_tokens for token in supplier if token.isdigit()}
-        self.supplier_lexicon = [
-            token
-            for token in sorted({tok for tokens in self.sup_tokens for tok in tokens})
-            if token and not token.isdigit()
-        ]
-        self.tfidf = TfidfVectorizer(analyzer="char_wb", ngram_range=(3, 5))
-        self.sup_mat = self.tfidf.fit_transform(self.sup_norm)
-        self.max_words = max(len(s.split()) for s in self.sup_norm)
-        self.variant_cache: dict[str, list[str]] = {}
-        self.lexical_token_cache: dict[str, float] = {}
-        self.phrase_support_cache: dict[str, float] = {}
-        self.noise_terms = {
-            "за", "на", "из", "для", "под", "над", "при", "без", "и", "или",
-            "купил", "купила", "купили", "покупка", "заказал", "заказала", "заказали",
-            "оплатил", "оплатила", "оплатили", "заплатил", "заплатила", "заплатили",
-            "был", "была", "было", "были", "утром", "днем", "днём", "вечером", "ночью",
-            "товар", "товары", "продукт", "продукты", "десерт", "еда",
-            "лей", "лея", "леи", "целых", "сотых", "сом", "сомов", "руб", "рублей", "грн", "usd", "eur",
-        }
-        self.noise_terms.update(UniversalDateParser.temporal_vocabulary())
-
-    @staticmethod
-    def numeric_tokens(text: str) -> set[str]:
-        return set(re.findall(r"\d+", text))
-
-    def cached_variants(self, text: str) -> list[str]:
-        key = normalize_text(text)
-        cached = self.variant_cache.get(key)
-        if cached is None:
-            cached = variants(key)
-            self.variant_cache[key] = cached
-        return cached
-
-    @staticmethod
-    def split_words(text: str) -> list[str]:
-        return [w for w in normalize_text(text).split() if w]
-
-    @classmethod
-    def is_supplier_extension(cls, base_supplier: str, extended_supplier: str) -> bool:
-        base_tokens = cls.split_words(base_supplier)
-        extended_tokens = cls.split_words(extended_supplier)
-        return len(base_tokens) < len(extended_tokens) and extended_tokens[:len(base_tokens)] == base_tokens
-
-    @classmethod
-    def phrase_token_count(cls, phrase: str | None) -> int:
-        return len(cls.split_words(phrase or ""))
-
-    @classmethod
-    def resolve_overlapping_suppliers(cls, ranking: list[dict[str, Any]]) -> dict[str, Any]:
-        if not ranking:
-            return {"supplier": None, "score": -1.0, "phrase": None}
-
-        best = ranking[0]
-        best_combined = float(best.get("combined", best.get("score", -1.0)))
-        best_phrase_len = cls.phrase_token_count(best.get("phrase"))
-
-        for alt in ranking[1:]:
-            if not cls.is_supplier_extension(str(best.get("supplier") or ""), str(alt.get("supplier") or "")):
-                continue
-
-            alt_combined = float(alt.get("combined", alt.get("score", -1.0)))
-            alt_phrase_len = cls.phrase_token_count(alt.get("phrase"))
-
-            if alt_phrase_len > best_phrase_len and alt_combined >= best_combined - 0.15:
-                best = alt
-                best_combined = alt_combined
-                best_phrase_len = alt_phrase_len
-
-        return best
-
-    @staticmethod
-    def numeric_compatibility_multiplier(phrase_nums: set[str], candidate_nums: set[str]) -> float:
-        if not phrase_nums and not candidate_nums:
-            return 1.0
-        if phrase_nums == candidate_nums:
-            return 1.08
-        if phrase_nums and candidate_nums:
-            return 1.03 if phrase_nums & candidate_nums else 0.80
-        return 0.82
-
-    def lexical_support(self, phrase: str) -> float:
-        tokens = [token for token in normalize_text(phrase).split() if token and not token.isdigit()]
-        if not tokens or not self.supplier_lexicon:
-            return 0.0
-
-        support_scores: list[float] = []
-        for token in tokens:
-            cached = self.lexical_token_cache.get(token)
-            if cached is not None:
-                support_scores.append(cached)
-                continue
-
-            best = 0.0
-            for token_variant in self.cached_variants(token):
-                for lex in self.supplier_lexicon:
-                    lev = Levenshtein.normalized_similarity(token_variant, lex)
-                    phon = phonetic_similarity(token_variant, lex)
-                    sim = max(lev, phon)
-                    if sim > best:
-                        best = sim
-
-            self.lexical_token_cache[token] = best
-            support_scores.append(best)
-
-        return sum(support_scores) / len(support_scores)
-
-    def score_phrase(self, phrase: str) -> dict[str, Any]:
-        vs = self.cached_variants(phrase)
-        q = self.tfidf.transform(vs)
-        tf = cosine_similarity(q, self.sup_mat)
-
-        best: dict[str, Any] = {"supplier": None, "score": -1.0, "phrase": phrase, "variant": ""}
-        for i, cand in enumerate(self.sup_norm):
-            local = -1.0
-            local_variant = ""
-            candidate_nums = self.sup_num_sets[i]
-            for j, v in enumerate(vs):
-                char = fuzz.ratio(v, cand) / 100.0
-                tf_val = float(tf[j, i])
-                penalty = length_penalty(len(v), len(cand))
-                phon = phonetic_similarity(v, cand)
-                phrase_nums = self.numeric_tokens(v)
-
-                if len(v.split()) == 1 and len(cand.split()) == 1:
-                    lev = Levenshtein.normalized_similarity(v, cand)
-                    val = (0.45 * lev + 0.25 * char + 0.10 * tf_val + 0.20 * phon) * penalty
-                else:
-                    align = token_alignment_score(v, self.sup_tokens[i])
-                    tok = fuzz.token_set_ratio(v, cand) / 100.0
-                    val = (0.30 * char + 0.20 * tok + 0.10 * tf_val + 0.20 * align + 0.20 * phon) * penalty
-
-                    compact_v = v.replace(" ", "")
-                    compact_cand = cand.replace(" ", "")
-                    compact_char = fuzz.ratio(compact_v, compact_cand) / 100.0
-                    compact_lev = Levenshtein.normalized_similarity(compact_v, compact_cand)
-                    compact_phon = phonetic_similarity(compact_v, compact_cand)
-                    compact = max(compact_char, compact_lev, compact_phon)
-                    if compact > 0.55:
-                        val = max(val, compact * penalty)
-
-                val *= self.numeric_compatibility_multiplier(phrase_nums, candidate_nums)
-
-                if val > local:
-                    local = val
-                    local_variant = v
-
-            if local > best["score"]:
-                best = {"supplier": self.suppliers[i], "score": local, "phrase": phrase, "variant": local_variant}
-        return best
-
-    def extract(self, text: str, date_phrase: str | None = None, debug: bool = False) -> dict[str, Any]:
-        threshold = 0.50
-        excluded_tokens: set[str] = set()
-        if date_phrase:
-            excluded_tokens.update(normalize_text(date_phrase).split())
-        excluded_tokens.update(self.noise_terms)
-
-        raw_tokens = normalize_text(text).split()
-        tokens: list[str] = []
-        for token in raw_tokens:
-            if token in excluded_tokens:
-                continue
-
-            if token.isdigit():
-                if token in self.sup_number_tokens:
-                    tokens.append(token)
-
-                if tokens and len(token) <= 3 and len(tokens[-1]) >= 4 and tokens[-1].isalpha():
-                    tokens.append(f"{tokens[-1]}{token}")
-                continue
-
-            if len(token) > 1:
-                tokens.append(token)
-
-        tokens = [t for t in tokens if len(t) > 1 and t not in excluded_tokens]
-
-        phrases: list[str] = []
-        seen: set[str] = set()
-        for i in range(len(tokens)):
-            for j in range(i + 1, min(i + 1 + self.max_words, len(tokens) + 1)):
-                p = " ".join(tokens[i:j])
-                if p not in seen:
-                    seen.add(p)
-                    phrases.append(p)
-
-        results = [self.score_phrase(p) for p in phrases]
-        candidate_rows: list[dict[str, Any]] = []
-        best_by_supplier: dict[str, dict[str, Any]] = {}
-        for row in results:
-            supplier = row["supplier"]
-            score = float(row.get("score", -1.0))
-            phrase = str(row.get("phrase") or "")
-            support = self.phrase_support_cache.get(phrase)
-            if support is None:
-                support = self.lexical_support(phrase)
-                self.phrase_support_cache[phrase] = support
-            combined = 0.75 * score + 0.25 * support
-
-            if debug:
-                candidate_rows.append({
-                    "supplier": supplier,
-                    "phrase": phrase,
-                    "score": round(score, 4),
-                    "support": round(support, 4),
-                    "combined": round(combined, 4),
-                })
-
-            enriched = {**row, "combined": combined}
-            passes = score >= threshold or combined >= 0.48
-            if passes and (supplier not in best_by_supplier or combined > float(best_by_supplier[supplier].get("combined", -1.0))):
-                best_by_supplier[supplier] = enriched
-
-        if not best_by_supplier and results:
-            def support_for_phrase(phrase: str) -> float:
-                cached_support = self.phrase_support_cache.get(phrase)
-                if cached_support is None:
-                    cached_support = self.lexical_support(phrase)
-                    self.phrase_support_cache[phrase] = cached_support
-                return cached_support
-
-            fallback = max(
-                results,
-                key=lambda item: 0.75 * float(item.get("score", -1.0)) + 0.25 * support_for_phrase(str(item.get("phrase") or "")),
-            )
-            fallback_score = float(fallback.get("score", -1.0))
-            fallback_phrase = str(fallback.get("phrase") or "")
-            fallback_support = support_for_phrase(fallback_phrase)
-            fallback_combined = 0.75 * fallback_score + 0.25 * fallback_support
-            if fallback_score >= 0.40 and fallback_support >= 0.43 and fallback_combined >= 0.43:
-                best_by_supplier[fallback["supplier"]] = {**fallback, "combined": fallback_combined}
-
-        supplier_ranking = sorted(best_by_supplier.values(), key=lambda x: float(x.get("combined", x["score"])), reverse=True)
-        best = self.resolve_overlapping_suppliers(supplier_ranking)
-
-        payload = {
-            "supplier": best["supplier"],
-            "supplier_score": round(best["score"], 4) if best["score"] >= 0 else None,
-            "matched_supplier_phrase": best.get("phrase"),
-        }
-
-        if debug:
-            top_candidates = sorted(candidate_rows, key=lambda item: item["combined"], reverse=True)[:8]
-            payload["supplier_debug"] = {
-                "tokens": tokens,
-                "phrases_count": len(phrases),
-                "top_candidates": top_candidates,
-            }
-
-        return payload
-
-
-class ExpenseAmountExtractor:
-    def __init__(self, suppliers: list[str]) -> None:
-        self.model = get_amount_model()
-
-    @staticmethod
-    def to_float(value: str) -> Optional[float]:
-        cleaned = value.replace(" ", "").replace("\u00A0", "")
-        match = re.search(r"\d+(?:[,]\d{1,2})?", cleaned)
-        if not match:
-            return None
-        try:
-            return float(match.group(0).replace(",", "."))
-        except ValueError:
-            return None
-
-    @staticmethod
-    def phrase_span(text: str, phrase: Optional[str]) -> Optional[tuple[int, int]]:
-        if not phrase:
-            return None
-        idx = text.lower().find(phrase.lower())
-        if idx == -1:
-            return None
-        return idx, idx + len(phrase)
-
-    @staticmethod
-    def overlaps(span1: tuple[int, int], span2: Optional[tuple[int, int]]) -> bool:
-        if span2 is None:
-            return False
-        return span1[0] < span2[1] and span2[0] < span1[1]
-
-    @staticmethod
-    def expand_amount_text(text: str, start: int, end: int) -> tuple[str, tuple[int, int]]:
-        suffix = re.match(r",\d{1,2}", text[end:])
-        if suffix:
-            new_end = end + len(suffix.group(0))
-            return text[start:new_end].strip(), (start, new_end)
-
-        prefix = re.search(r"(\d{1,3}(?:\s*\d{3})*),", text[:start])
-        if prefix:
-            new_start = prefix.start(1)
-            return text[new_start:end].strip(), (new_start, end)
-
-        return text[start:end].strip(), (start, end)
-
-    def extract(
-        self,
-        text: str,
-        matched_date_phrase: Optional[str] = None,
-        matched_supplier_phrase: Optional[str] = None,
-    ) -> dict[str, Any]:
-        if self.model is None:
-            return {"amount": None, "amount_text": None}
-
-        date_span = self.phrase_span(text, matched_date_phrase)
-        supplier_span = self.phrase_span(text, matched_supplier_phrase)
-        entities = self.model.predict_entities(text, ["money"], threshold=0.3)
-
-        for ent in sorted(entities, key=lambda item: float(item.get("score", 0.0)), reverse=True):
-            raw_span = (int(ent.get("start", 0)), int(ent.get("end", 0)))
-            amount_text, span = self.expand_amount_text(text, raw_span[0], raw_span[1])
-            amount = self.to_float(amount_text)
-            overlaps_date = self.overlaps(span, date_span)
-            overlaps_supplier = self.overlaps(span, supplier_span)
-
-            if amount is None:
-                continue
-            if overlaps_date or overlaps_supplier:
-                continue
-            return {"amount": amount, "amount_text": amount_text}
-
-        return {"amount": None, "amount_text": None}
-
-
->>>>>>> acab9140c760dd9ad71b1a76a9ae5c130efa1829
 class ExpenseTextExtractor:
     """
     Главный экстрактор данных о расходах.