# Standard library imports import logging import re from typing import List, Dict, Set, Tuple, Optional, Union, Any from functools import lru_cache # Configure logging logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) class LanguageDetector: """ A language detection system that provides balanced detection across multiple languages using an enhanced statistical approach. """ def __init__(self): """Initialize the language detector with statistical language models""" logger.info("Initializing language detector with statistical models") # Initialize language indicators dictionary for statistical detection self._init_language_indicators() # Set thresholds for language detection confidence self.single_lang_confidence = 65 # Minimum score to consider a language detected self.secondary_lang_threshold = 0.75 # Secondary language must be at least this fraction of primary score def _init_language_indicators(self): """Initialize language indicators for statistical detection with historical markers""" # Define indicators for all supported languages with equal detail level # Each language has: # - Distinctive characters # - Common words (including historical forms) # - N-grams (character sequences) # - Historical markers specific to older forms of the language self.language_indicators = { "English": { "chars": [], # English uses basic Latin alphabet without special chars "words": ['the', 'and', 'of', 'to', 'in', 'a', 'is', 'that', 'for', 'it', 'with', 'as', 'be', 'on', 'by', 'at', 'this', 'have', 'from', 'or', 'an', 'but', 'not', 'what', 'all', 'were', 'when', 'we', 'there', 'can', 'would', 'who', 'you', 'been', 'one', 'their', 'has', 'more', 'if', 'no'], "ngrams": ['th', 'he', 'in', 'er', 'an', 're', 'on', 'at', 'en', 'nd', 'ti', 'es', 'or', 'ing', 'tion', 'the', 'and', 'tha', 'ent', 'ion'], "historical": { "chars": ['þ', 'ȝ', 'æ', 'ſ'], # Thorn, yogh, ash, long s "words": ['thou', 'thee', 'thy', 'thine', 'hath', 'doth', 'ere', 'whilom', 'betwixt', 'ye', 'art', 'wast', 'dost', 'hast', 'shalt', 'mayst', 'verily'], "patterns": ['eth$', '^y[^a-z]', 'ck$', 'aught', 'ought'] # -eth endings, y- prefixes } }, "French": { "chars": ['é', 'è', 'ê', 'à', 'ç', 'ù', 'â', 'î', 'ô', 'û', 'ë', 'ï', 'ü'], "words": ['le', 'la', 'les', 'et', 'en', 'de', 'du', 'des', 'un', 'une', 'ce', 'cette', 'ces', 'dans', 'par', 'pour', 'sur', 'qui', 'que', 'quoi', 'où', 'quand', 'comment', 'est', 'sont', 'ont', 'nous', 'vous', 'ils', 'elles', 'avec', 'sans', 'mais', 'ou'], "ngrams": ['es', 'le', 'de', 'en', 'on', 'nt', 'qu', 'ai', 'an', 'ou', 'ur', 're', 'me', 'les', 'ent', 'que', 'des', 'ons', 'ant', 'ion'], "historical": { "chars": ['ſ', 'æ', 'œ'], # Long s and ligatures "words": ['aultre', 'avecq', 'icelluy', 'oncques', 'moult', 'estre', 'mesme', 'ceste', 'ledict', 'celuy', 'ceulx', 'aulcun', 'ainſi', 'touſiours', 'eſtre', 'eſt', 'meſme', 'felon', 'auec', 'iufques', 'chofe', 'fcience'], "patterns": ['oi[ts]$', 'oi[re]$', 'f[^aeiou]', 'ff', 'ſ', 'auoit', 'eſtoit', 'ſi', 'ſur', 'ſa', 'cy', 'ayant', 'oy', 'uſ', 'auſ'] }, }, "German": { "chars": ['ä', 'ö', 'ü', 'ß'], "words": ['der', 'die', 'das', 'und', 'in', 'zu', 'den', 'ein', 'eine', 'mit', 'ist', 'von', 'des', 'sich', 'auf', 'für', 'als', 'auch', 'werden', 'bei', 'durch', 'aus', 'sind', 'nicht', 'nur', 'wurde', 'wie', 'wenn', 'aber', 'noch', 'nach', 'so', 'sein', 'über'], "ngrams": ['en', 'er', 'ch', 'de', 'ei', 'in', 'te', 'nd', 'ie', 'ge', 'un', 'sch', 'ich', 'den', 'die', 'und', 'der', 'ein', 'ung', 'cht'], "historical": { "chars": ['ſ', 'ů', 'ė', 'ÿ'], "words": ['vnnd', 'vnnd', 'vnter', 'vnd', 'seyn', 'thun', 'auff', 'auß', 'deß', 'diß'], "patterns": ['^v[nd]', 'th', 'vnter', 'ſch'] } }, "Spanish": { "chars": ['á', 'é', 'í', 'ó', 'ú', 'ñ', 'ü', '¿', '¡'], "words": ['el', 'la', 'los', 'las', 'de', 'en', 'y', 'a', 'que', 'por', 'un', 'una', 'no', 'es', 'con', 'para', 'su', 'al', 'se', 'del', 'como', 'más', 'pero', 'lo', 'mi', 'si', 'ya', 'todo', 'esta', 'cuando', 'hay', 'muy', 'bien', 'sin', 'así'], "ngrams": ['de', 'en', 'os', 'es', 'la', 'ar', 'el', 'er', 'ra', 'as', 'an', 'do', 'or', 'que', 'nte', 'los', 'ado', 'con', 'ent', 'ien'], "historical": { "chars": ['ſ', 'ç', 'ñ'], "words": ['facer', 'fijo', 'fermoso', 'agora', 'asaz', 'aver', 'caſa', 'deſde', 'eſte', 'eſta', 'eſto', 'deſto', 'deſta', 'eſſo', 'muger', 'dixo', 'fazer'], "patterns": ['^f[aei]', 'ſſ', 'ſc', '^deſ', 'xo$', 'xe$'] }, }, "Italian": { "chars": ['à', 'è', 'é', 'ì', 'í', 'ò', 'ó', 'ù', 'ú'], "words": ['il', 'la', 'i', 'le', 'e', 'di', 'a', 'in', 'che', 'non', 'per', 'con', 'un', 'una', 'del', 'della', 'è', 'sono', 'da', 'si', 'come', 'anche', 'più', 'ma', 'ci', 'se', 'ha', 'mi', 'lo', 'ti', 'al', 'tu', 'questo', 'questi'], "ngrams": ['di', 'la', 'er', 'to', 're', 'co', 'de', 'in', 'ra', 'on', 'li', 'no', 'ri', 'che', 'ent', 'con', 'per', 'ion', 'ato', 'lla'] }, "Portuguese": { "chars": ['á', 'â', 'ã', 'à', 'é', 'ê', 'í', 'ó', 'ô', 'õ', 'ú', 'ç'], "words": ['o', 'a', 'os', 'as', 'de', 'em', 'e', 'do', 'da', 'dos', 'das', 'no', 'na', 'para', 'que', 'um', 'uma', 'por', 'com', 'se', 'não', 'mais', 'como', 'mas', 'você', 'eu', 'este', 'isso', 'ele', 'seu', 'sua', 'ou', 'já', 'me'], "ngrams": ['de', 'os', 'em', 'ar', 'es', 'ra', 'do', 'da', 'en', 'co', 'nt', 'ad', 'to', 'que', 'nto', 'ent', 'com', 'ção', 'ado', 'ment'] }, "Dutch": { "chars": ['ë', 'ï', 'ö', 'ü', 'é', 'è', 'ê', 'ç', 'á', 'à', 'ä', 'ó', 'ô', 'ú', 'ù', 'û', 'ij'], "words": ['de', 'het', 'een', 'en', 'van', 'in', 'is', 'dat', 'op', 'te', 'zijn', 'met', 'voor', 'niet', 'aan', 'er', 'die', 'maar', 'dan', 'ik', 'je', 'hij', 'zij', 'we', 'kunnen', 'wordt', 'nog', 'door', 'over', 'als', 'uit', 'bij', 'om', 'ook'], "ngrams": ['en', 'de', 'er', 'ee', 'ge', 'an', 'aa', 'in', 'te', 'et', 'ng', 'ee', 'or', 'van', 'het', 'een', 'ing', 'ver', 'den', 'sch'] }, "Russian": { # Russian (Cyrillic alphabet) characters "chars": ['а', 'б', 'в', 'г', 'д', 'е', 'ё', 'ж', 'з', 'и', 'й', 'к', 'л', 'м', 'н', 'о', 'п', 'р', 'с', 'т', 'у', 'ф', 'х', 'ц', 'ч', 'ш', 'щ', 'ъ', 'ы', 'ь', 'э', 'ю', 'я'], "words": ['и', 'в', 'не', 'на', 'что', 'я', 'с', 'а', 'то', 'он', 'как', 'этот', 'по', 'но', 'из', 'к', 'у', 'за', 'вы', 'все', 'так', 'же', 'от', 'для', 'о', 'его', 'мы', 'было', 'она', 'бы', 'мне', 'еще', 'есть', 'быть', 'был'], "ngrams": ['о', 'е', 'а', 'н', 'и', 'т', 'р', 'с', 'в', 'л', 'к', 'м', 'д', 'ст', 'но', 'то', 'ни', 'на', 'по', 'ет'] }, "Chinese": { "chars": ['的', '是', '不', '了', '在', '和', '有', '我', '们', '人', '这', '上', '中', '个', '大', '来', '到', '国', '时', '要', '地', '出', '会', '可', '也', '就', '年', '生', '对', '能', '自', '那', '都', '得', '说', '过', '子', '家', '后', '多'], # Chinese doesn't have "words" in the same way as alphabetic languages "words": ['的', '是', '不', '了', '在', '和', '有', '我', '们', '人', '这', '上', '中', '个', '大', '来', '到', '国', '时', '要', '地', '出', '会', '可', '也', '就'], "ngrams": ['的', '是', '不', '了', '在', '我', '有', '和', '人', '这', '中', '大', '来', '上', '国', '个', '到', '说', '们', '为'] }, "Japanese": { # A mix of hiragana, katakana, and common kanji "chars": ['あ', 'い', 'う', 'え', 'お', 'か', 'き', 'く', 'け', 'こ', 'さ', 'し', 'す', 'せ', 'そ', 'ア', 'イ', 'ウ', 'エ', 'オ', 'カ', 'キ', 'ク', 'ケ', 'コ', 'サ', 'シ', 'ス', 'セ', 'ソ', '日', '本', '人', '大', '小', '中', '山', '川', '田', '子', '女', '男', '月', '火', '水'], "words": ['は', 'を', 'に', 'の', 'が', 'で', 'へ', 'から', 'より', 'まで', 'だ', 'です', 'した', 'ます', 'ません', 'です', 'これ', 'それ', 'あれ', 'この', 'その', 'あの', 'わたし'], "ngrams": ['の', 'は', 'た', 'が', 'を', 'に', 'て', 'で', 'と', 'し', 'か', 'ま', 'こ', 'い', 'する', 'いる', 'れる', 'なる', 'れて', 'した'] }, "Korean": { "chars": ['가', '나', '다', '라', '마', '바', '사', '아', '자', '차', '카', '타', '파', '하', '그', '는', '을', '이', '에', '에서', '로', '으로', '와', '과', '또는', '하지만'], "words": ['이', '그', '저', '나', '너', '우리', '그들', '이것', '그것', '저것', '은', '는', '이', '가', '을', '를', '에', '에서', '으로', '로', '와', '과', '의', '하다', '되다'], "ngrams": ['이', '다', '는', '에', '하', '고', '지', '서', '의', '가', '을', '로', '을', '으', '니다', '습니', '하는', '이다', '에서', '하고'] }, "Arabic": { "chars": ['ا', 'ب', 'ت', 'ث', 'ج', 'ح', 'خ', 'د', 'ذ', 'ر', 'ز', 'س', 'ش', 'ص', 'ض', 'ط', 'ظ', 'ع', 'غ', 'ف', 'ق', 'ك', 'ل', 'م', 'ن', 'ه', 'و', 'ي', 'ء', 'ة', 'ى'], "words": ['في', 'من', 'على', 'إلى', 'هذا', 'هذه', 'ذلك', 'تلك', 'هو', 'هي', 'هم', 'أنا', 'أنت', 'نحن', 'كان', 'كانت', 'يكون', 'لا', 'لم', 'ما', 'أن', 'و', 'أو', 'ثم', 'بعد'], "ngrams": ['ال', 'ان', 'في', 'من', 'ون', 'ين', 'ات', 'ار', 'ور', 'ما', 'لا', 'ها', 'ان', 'الم', 'لان', 'علا', 'الح', 'الس', 'الع', 'الت'] }, "Hindi": { "chars": ['अ', 'आ', 'इ', 'ई', 'उ', 'ऊ', 'ए', 'ऐ', 'ओ', 'औ', 'क', 'ख', 'ग', 'घ', 'ङ', 'च', 'छ', 'ज', 'झ', 'ञ', 'ट', 'ठ', 'ड', 'ढ', 'ण', 'त', 'थ', 'द', 'ध', 'न', 'प', 'फ', 'ब', 'भ', 'म', 'य', 'र', 'ल', 'व', 'श', 'ष', 'स', 'ह', 'ा', 'ि', 'ी', 'ु', 'ू', 'े', 'ै', 'ो', 'ौ', '्', 'ं', 'ः'], "words": ['और', 'का', 'के', 'की', 'एक', 'में', 'है', 'यह', 'हैं', 'से', 'को', 'पर', 'इस', 'हो', 'गया', 'कर', 'मैं', 'या', 'हुआ', 'था', 'वह', 'अपने', 'सकता', 'ने', 'बहुत'], "ngrams": ['का', 'के', 'की', 'है', 'ने', 'से', 'मे', 'को', 'पर', 'हा', 'रा', 'ता', 'या', 'ार', 'ान', 'कार', 'राज', 'ारा', 'जाए', 'ेजा'] }, "Latin": { "chars": [], # Latin uses basic Latin alphabet "words": ['et', 'in', 'ad', 'est', 'sunt', 'non', 'cum', 'sed', 'qui', 'quod', 'ut', 'si', 'nec', 'ex', 'per', 'quam', 'pro', 'iam', 'hoc', 'aut', 'esse', 'enim', 'de', 'atque', 'ac', 'ante', 'post', 'sub', 'ab'], "ngrams": ['us', 'is', 'um', 'er', 'it', 'nt', 'am', 'em', 're', 'at', 'ti', 'es', 'ur', 'tur', 'que', 'ere', 'ent', 'ius', 'rum', 'tus'] }, "Greek": { "chars": ['α', 'β', 'γ', 'δ', 'ε', 'ζ', 'η', 'θ', 'ι', 'κ', 'λ', 'μ', 'ν', 'ξ', 'ο', 'π', 'ρ', 'σ', 'ς', 'τ', 'υ', 'φ', 'χ', 'ψ', 'ω', 'ά', 'έ', 'ή', 'ί', 'ό', 'ύ', 'ώ'], "words": ['και', 'του', 'της', 'των', 'στο', 'στη', 'με', 'από', 'για', 'είναι', 'να', 'ότι', 'δεν', 'στον', 'μια', 'που', 'ένα', 'έχει', 'θα', 'το', 'ο', 'η', 'τον'], "ngrams": ['αι', 'τα', 'ου', 'τη', 'οι', 'το', 'ης', 'αν', 'ος', 'ον', 'ις', 'ει', 'ερ', 'και', 'την', 'τον', 'ους', 'νου', 'εντ', 'μεν'] } } def detect_languages(self, text: str, filename: str = None, current_languages: List[str] = None) -> List[str]: """ Detect languages in text using an enhanced statistical approach Args: text: Text to analyze filename: Optional filename to provide additional context current_languages: Optional list of languages already detected Returns: List of detected languages """ logger = logging.getLogger("language_detector") # If no text provided, return current languages or default if not text or len(text.strip()) < 10: return current_languages if current_languages else ["English"] # If we already have detected languages, use them if current_languages and len(current_languages) > 0: logger.info(f"Using already detected languages: {current_languages}") return current_languages # Use enhanced statistical detection detected_languages = self._detect_statistically(text, filename) logger.info(f"Statistical language detection results: {detected_languages}") return detected_languages def _detect_statistically(self, text: str, filename: str = None) -> List[str]: """ Detect languages using enhanced statistical analysis with historical language indicators Args: text: Text to analyze filename: Optional filename for additional context Returns: List of detected languages """ logger = logging.getLogger("language_detector") # Normalize text to lowercase for consistent analysis text_lower = text.lower() words = re.findall(r'\b\w+\b', text_lower) # Extract words # Score each language based on characters, words, n-grams, and historical markers language_scores = {} historical_bonus = {} # PHASE 1: Special character analysis # Count special characters for each language special_char_counts = {} total_special_chars = 0 for language, indicators in self.language_indicators.items(): chars = indicators["chars"] count = 0 for char in chars: if char in text_lower: count += text_lower.count(char) special_char_counts[language] = count total_special_chars += count # Normalize character scores (0-30 points) for language, count in special_char_counts.items(): if total_special_chars > 0: # Scale score to 0-30 range (reduced from 35 to make room for historical) normalized_score = (count / total_special_chars) * 30 language_scores[language] = normalized_score else: language_scores[language] = 0 # PHASE 2: Word analysis (0-30 points) # Count common words for each language for language, indicators in self.language_indicators.items(): word_list = indicators["words"] word_matches = sum(1 for word in words if word in word_list) # Normalize word score based on text length and word list size word_score_factor = min(1.0, word_matches / (len(words) * 0.1)) # Max 1.0 if 10% match language_scores[language] = language_scores.get(language, 0) + (word_score_factor * 30) # PHASE 3: N-gram analysis (0-20 points) for language, indicators in self.language_indicators.items(): ngram_list = indicators["ngrams"] ngram_matches = 0 # Count ngram occurrences for ngram in ngram_list: ngram_matches += text_lower.count(ngram) # Normalize ngram score based on text length if len(text_lower) > 0: ngram_score_factor = min(1.0, ngram_matches / (len(text_lower) * 0.05)) # Max 1.0 if 5% match language_scores[language] = language_scores.get(language, 0) + (ngram_score_factor * 20) # PHASE 4: Historical language markers (0-20 points) for language, indicators in self.language_indicators.items(): if "historical" in indicators: historical_indicators = indicators["historical"] historical_score = 0 # Check for historical chars if "chars" in historical_indicators: for char in historical_indicators["chars"]: if char in text_lower: historical_score += text_lower.count(char) * 0.5 # Check for historical words if "words" in historical_indicators: hist_words = historical_indicators["words"] hist_word_matches = sum(1 for word in words if word in hist_words) if hist_word_matches > 0: # Historical words are strong indicators historical_score += min(10, hist_word_matches * 2) # Check for historical patterns if "patterns" in historical_indicators: for pattern in historical_indicators["patterns"]: matches = len(re.findall(pattern, text_lower)) if matches > 0: historical_score += min(5, matches * 0.5) # Cap historical score at 20 points historical_score = min(20, historical_score) historical_bonus[language] = historical_score # Apply historical bonus language_scores[language] += historical_score # Apply language-specific exclusivity multiplier if present if "exclusivity" in indicators: exclusivity = indicators["exclusivity"] language_scores[language] *= exclusivity logger.info(f"Applied exclusivity multiplier {exclusivity} to {language}") # Print historical bonus for debugging for language, bonus in historical_bonus.items(): if bonus > 0: logger.info(f"Historical language bonus for {language}: {bonus} points") # Final language selection with more stringent criteria # Get languages with scores above threshold threshold = self.single_lang_confidence # Higher minimum score candidates = [(lang, score) for lang, score in language_scores.items() if score >= threshold] candidates.sort(key=lambda x: x[1], reverse=True) logger.info(f"Language candidates: {candidates}") # If we have candidate languages, return top 1-2 with higher threshold for secondary if candidates: # Always take top language result = [candidates[0][0]] # Add second language only if it's significantly strong compared to primary # and doesn't have a historical/exclusivity conflict if len(candidates) > 1: primary_lang = candidates[0][0] secondary_lang = candidates[1][0] primary_score = candidates[0][1] secondary_score = candidates[1][1] # Only add secondary if it meets threshold and doesn't conflict ratio = secondary_score / primary_score # Check for French and Spanish conflict (historical French often gets misidentified) historical_conflict = False if (primary_lang == "French" and secondary_lang == "Spanish" and historical_bonus.get("French", 0) > 5): historical_conflict = True logger.info("Historical French markers detected, suppressing Spanish detection") if ratio >= self.secondary_lang_threshold and not historical_conflict: result.append(secondary_lang) logger.info(f"Added secondary language {secondary_lang} (score ratio: {ratio:.2f})") else: logger.info(f"Rejected secondary language {secondary_lang} (score ratio: {ratio:.2f})") return result # Default to English if no clear signals