import re import logging from typing import Set, Tuple, List SENTENCE_SPLITTER = re.compile(r'[^\.?!…]+[\.?!…]*["»“]*') LAST_WORD_PATTERN = re.compile(r'(?:\b|\d)([a-zа-я]+)\.$', re.IGNORECASE) FIRST_WORD_PATTERN = re.compile(r'^\W*(\w+)') ENDS_WITH_ONE_LETTER_LAT_AND_DOT_PATTERN = re.compile(r'(\d|\W|\b)([a-zA-Z])\.$') HAS_DOT_INSIDE_PATTERN = re.compile(r'[\w]+\.[\w]+\.$', re.IGNORECASE) INITIALS_PATTERN = re.compile(r'(\W|\b)([A-ZА-Я]{1})\.$') ONLY_RUS_CONSONANTS_PATTERN = re.compile(r'^[бвгджзйклмнпрстфхцчшщ]{1,4}$', re.IGNORECASE) STARTS_WITH_EMPTYNESS_PATTERN = re.compile(r'^\s+') ENDS_WITH_EMOTION_PATTERN = re.compile(r'[!?…]|\.{2,}\s?[)"«»,“]?$') STARTS_WITH_LOWER_PATTERN = re.compile(r'^\s*[–-—-("«]?\s*[a-zа-я]') STARTS_WITH_DIGIT_PATTERN = re.compile(r'^\s*\d') NUMERATION_PATTERN = re.compile(r'^\W*[IVXMCL\d]+\.$') PAIRED_SHORTENING_IN_THE_END_PATTERN = re.compile(r'\b(\w+)\. (\w+)\.\W*$') JOIN = 0 MAYBE = 1 SPLIT = 2 JOINING_SHORTENINGS = { 'mr', 'mrs', 'ms', 'dr', 'vs', 'англ', 'итал', 'греч', 'евр', 'араб', 'яп', 'слав', 'кит', 'тел', 'св', 'ул', 'устар', 'им', 'г', 'см', 'д', 'стр', 'корп', 'пл', 'пер', 'сокр', 'рис' } SHORTENINGS = { 'co', 'corp', 'inc', 'авт', 'адм', 'барр', 'внутр', 'га', 'дифф', 'дол', 'долл', 'зав', 'зам', 'искл', 'коп', 'корп', 'куб', 'лат', 'мин', 'о', 'обл', 'обр', 'прим', 'проц', 'р', 'ред', 'руб', 'рус', 'русск', 'сан', 'сек', 'тыс', 'эт', 'яз', 'гос', 'мн', 'жен', 'муж', 'накл', 'повел', 'букв', 'шутл', 'ед' } PAIRED_SHORTENINGS = {('и', 'о'), ('т', 'е'), ('т', 'п'), ('у', 'е'), ('н', 'э')} def split_sentences(text: str) -> List[str]: return [x.strip() for x in SENTENCE_SPLITTER.findall(text)] def is_sentence_end(left: str, right: str, shortenings: Set[str], joining_shortenings: Set[str], paired_shortenings: Set[Tuple[str, str]]) -> int: if not STARTS_WITH_EMPTYNESS_PATTERN.match(right): return JOIN if HAS_DOT_INSIDE_PATTERN.search(left): return JOIN left_last_word = LAST_WORD_PATTERN.search(left) lw = ' ' if left_last_word: lw = left_last_word.group(1) if lw.lower() in joining_shortenings: return JOIN if ONLY_RUS_CONSONANTS_PATTERN.search(lw) and lw[-1].islower(): return MAYBE pse = PAIRED_SHORTENING_IN_THE_END_PATTERN.search(left) if pse: s1, s2 = pse.groups() if (s1, s2) in paired_shortenings: return MAYBE right_first_word = FIRST_WORD_PATTERN.match(right) if right_first_word: rw = right_first_word.group(1) if (lw, rw) in paired_shortenings: return MAYBE if ENDS_WITH_EMOTION_PATTERN.search(left) and STARTS_WITH_LOWER_PATTERN.match(right): return JOIN initials = INITIALS_PATTERN.search(left) if initials: border, _ = initials.groups() if (border or ' ') not in "°'": return JOIN if lw.lower() in shortenings: return MAYBE last_letter = ENDS_WITH_ONE_LETTER_LAT_AND_DOT_PATTERN.search(left) if last_letter: border, _ = last_letter.groups() if (border or ' ') not in "°'": return MAYBE if NUMERATION_PATTERN.match(left): return JOIN return SPLIT def split_by_sentences(text: str, shortenings: Set[str] = SHORTENINGS, joining_shortenings: Set[str] = JOINING_SHORTENINGS, paired_shortenings: Set[Tuple[str, str]] = PAIRED_SHORTENINGS) -> List[str]: sentences = [] sents = split_sentences(text) si = 0 processed_index = 0 sent_start = 0 while si < len(sents): s = sents[si] span_start = text[processed_index:].index(s) + processed_index span_end = span_start + len(s) processed_index += len(s) si += 1 send = is_sentence_end(text[sent_start: span_end], text[span_end:], shortenings, joining_shortenings, paired_shortenings) if send == JOIN: continue if send == MAYBE: if STARTS_WITH_LOWER_PATTERN.match(text[span_end:]): continue if STARTS_WITH_DIGIT_PATTERN.match(text[span_end:]): continue if not text[sent_start: span_end].strip(): print(text) sentences.append(text[sent_start: span_end].strip()) sent_start = span_end processed_index = span_end if sent_start != len(text): if text[sent_start:].strip(): sentences.append(text[sent_start:].strip()) return sentences