init

.devcontainer/devcontainer.json

@@ -0,0 +1,33 @@
+{
+  "name": "Python 3",
+  // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+  "image": "mcr.microsoft.com/devcontainers/python:1-3.11-bullseye",
+  "customizations": {
+    "codespaces": {
+      "openFiles": [
+        "README.md",
+        "main.py"
+      ]
+    },
+    "vscode": {
+      "settings": {},
+      "extensions": [
+        "ms-python.python",
+        "ms-python.vscode-pylance"
+      ]
+    }
+  },
+  "updateContentCommand": "[ -f packages.txt ] && sudo apt update && sudo apt upgrade -y && sudo xargs apt install -y <packages.txt; [ -f requirements.txt ] && pip3 install --user -r requirements.txt; pip3 install --user streamlit; echo '✅ Packages installed and Requirements met'",
+  "postAttachCommand": {
+    "server": "streamlit run main.py --server.enableCORS false --server.enableXsrfProtection false"
+  },
+  "portsAttributes": {
+    "8501": {
+      "label": "Application",
+      "onAutoForward": "openPreview"
+    }
+  },
+  "forwardPorts": [
+    8501
+  ]
+}

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,12 @@
+.venv/
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.huggingface/
+.cache/
+local_llms/.cache/
+local_llms/.locks/
+local_llms/locks/
+local_llms/instances/
+local_llms/models--*/

.idea/Tilmash_Translator.iml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module version="4">
+  <component name="PyDocumentationSettings">
+    <option name="format" value="PLAIN" />
+    <option name="myDocStringFormat" value="Plain" />
+  </component>
+</module>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Black">
+    <option name="sdkName" value="Python 3.13 (Tilmash_Translator)" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.11 (Tilmash_Translator)" project-jdk-type="Python SDK" />
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

Dockerfile

@@ -0,0 +1,50 @@
+<<<<<<< HEAD
+FROM python:3.9-slim
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    curl \
+    software-properties-common \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt ./
+COPY src/ ./src/
+
+RUN pip3 install -r requirements.txt
+
+EXPOSE 8501
+
+HEALTHCHECK CMD curl --fail http://localhost:8501/_stcore/health
+
+ENTRYPOINT ["streamlit", "run", "src/streamlit_app.py", "--server.port=8501", "--server.address=0.0.0.0"]
+=======
+FROM python:3.11-slim
+
+# Базовые системные зависимости (для сборки/запуска)
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential git libglib2.0-0 libgl1 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Установка CUDA поддержки (для PyTorch)
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_NO_CACHE_DIR=1
+
+WORKDIR /app
+
+# Устанавливаем Python-зависимости
+COPY requirements.txt /app/requirements.txt
+ENV PIP_NO_CACHE_DIR=1 \
+    HF_HOME=/data/.cache/huggingface \
+    PORT=7860
+RUN pip install -U pip && pip install -r requirements.txt
+
+# Копируем весь проект
+COPY . /app
+
+# Запуск Streamlit внутри контейнера
+CMD ["streamlit", "run", "main.py", "--server.port=7860", "--server.address=0.0.0.0"]
+>>>>>>> 805a119 (Initial commit for HF Space)

LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2011-2025 The Bootstrap Authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

README.md

@@ -0,0 +1,115 @@
+<<<<<<< HEAD
+---
+title: Translate Tl
+emoji: 🚀
+colorFrom: red
+colorTo: red
+sdk: docker
+app_port: 8501
+tags:
+- streamlit
+pinned: false
+short_description: Streamlit template space
+---
+
+# Welcome to Streamlit!
+
+Edit `/src/streamlit_app.py` to customize this app to your heart's desire. :heart:
+
+If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community
+forums](https://discuss.streamlit.io).
+=======
+$yaml = @"
+---
+title: Tilmash Translator
+sdk: streamlit
+app_file: main.py
+python_version: "3.11"
+pinned: false
+---
+"@
+$orig = Get-Content -Raw README.md
+Set-Content README.md $yaml -Encoding UTF8
+Add-Content README.md $orig
+
+# Tilmash Translator
+
+**Tilmash Translator** is an offline‑first, privacy‑preserving translation and readability toolkit for Russian, English and Kazakh.
+
+It ships as a Streamlit web‑app and offers two core capabilities:
+
+1. **Neural Machine Translation**  
+   • Primary model — [ISSAI/tilmash](https://huggingface.co/issai/tilmash) (Seq2Seq) for RU ↔ EN ↔ KK  
+   • Smart chunking & streaming make multi‑page documents feel snappy
+2. **Readability Analysis**  
+   • Calculates Flesch Reading Ease, Flesch‑Kincaid, Gunning Fog and SMOG  
+   • Highlights complex words and supports RU/EN/KK
+
+
+---
+
+## Quick Start
+
+```bash
+# 1. Clone & create a virtual environment
+$ git clone https://github.com/medetshatayev/Tilmash_Translator.git
+$ cd Tilmash_Translator
+$ python3 -m venv .venv && source .venv/bin/activate
+
+# 2. Install dependencies
+$ pip install -r requirements.txt
+
+# 3. (optional) authenticate once to download the Tilmash weights
+$ echo "HF_TOKEN=🪄your_huggingface_token" > .env
+
+# 4. Launch the Streamlit app
+$ streamlit run main.py
+```
+
+💡 The helper script `start.sh` automates the above and sets safe memory limits for `llama‑cpp-python`.
+
+### GPU Off‑loading (Gemma‑3)
+
+Set `GEMMA_GPU_LAYERS=<num_layers>` in your environment (defaults to **48**) to off‑load those layers to Metal/CUDA.
+
+---
+
+## Project Layout
+
+```
+.
+├── main.py               # Streamlit UI
+├── utils/                # Translation & analysis helpers
+│   ├── tilmash_translation.py
+│   ├── gemma_translation.py
+│   ├── readability_indices.py
+│   └── ...
+├── models/               # Extra resources (NLTK, etc.)
+├── config.py             # Default env‑vars
+├── start.sh              # Convenience launcher
+└── requirements.txt      # Python deps
+```
+
+## Configuration Keys
+
+| Variable               | Default | Purpose                                   |
+|------------------------|---------|-------------------------------------------|
+| `GEMMA_GPU_LAYERS`     | 48      | Layers to move to GPU (0 = CPU‑only)      |
+| `GEMMA_CONTEXT_SIZE`   | 8192    | Context window for Gemma‑3                |
+| `MAX_PARALLEL_MODELS`  | 4       | Concurrency guard                         |
+| `MAX_TOKENS`           | 4096    | Generation cap per request                |
+| `CHUNK_SIZE`           | 3000    | Token threshold before auto‑chunking      |
+
+Override any of these via the environment or edit **config.py**.
+
+---
+
+## How It Works
+
+1. **File ingestion** — `.txt`, `.docx`, `.pdf` loaded via `utils/file_readers.py`  
+2. **Language detection** — `langdetect` (auto‑detect option in UI)  
+3. **Translation pipeline** — <3000 tokens translate directly; longer texts are chunked (`utils/chunking.py`) and streamed through Tilmash or Gemma‑3  
+4. **Readability analysis** — scores computed in `utils/readability_indices.py` and color‑coded in the app.
+
+---
+>>>>>>> 805a119 (Initial commit for HF Space)

check_gpu.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+
+"""
+Этот скрипт проверяет наличие GPU и совместимость с PyTorch.
+Запустите его, чтобы проверить конфигурацию вашей системы:
+
+python check_gpu.py
+"""
+
+import platform
+import os
+import sys
+
+def check_system():
+    print(f"Операционная система: {platform.system()} {platform.release()}")
+    print(f"Python версия: {sys.version}")
+
+    try:
+        import torch
+        print(f"\nПроверка PyTorch:")
+        print(f"Версия PyTorch: {torch.__version__}")
+
+        # Проверка CUDA
+        if hasattr(torch.version, "cuda"):
+            print(f"CUDA версия: {torch.version.cuda}")
+        else:
+            print("CUDA версия: не найдена")
+
+        # Проверка доступности CUDA
+        print(f"CUDA доступен: {torch.cuda.is_available()}")
+
+        if torch.cuda.is_available():
+            print(f"Обнаружено GPU: {torch.cuda.get_device_name(0)}")
+            print(f"Количество GPU: {torch.cuda.device_count()}")
+            for i in range(torch.cuda.device_count()):
+                print(f"  GPU {i}: {torch.cuda.get_device_name(i)}")
+
+        # Проверка MPS (Apple Silicon)
+        if hasattr(torch.backends, "mps"):
+            print(f"\nApple MPS доступен: {torch.backends.mps.is_available()}")
+            if torch.backends.mps.is_available():
+                print("Обнаружен Apple Silicon GPU (M1/M2/M3)")
+
+    except ImportError:
+        print("PyTorch не установлен.")
+        print("Установите PyTorch командой: pip install torch")
+    except Exception as e:
+        print(f"Ошибка при проверке PyTorch: {str(e)}")
+
+if __name__ == "__main__":
+    print("===== Диагностика GPU для Tilmash =====")
+    check_system()
+
+    print("\n===== Готовность системы =====")
+    try:
+        import torch
+        if torch.cuda.is_available():
+            print("✅ GPU CUDA обнаружен и готов к использованию")
+        elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+            print("✅ Apple Silicon GPU (MPS) обнаружен и готов к использованию")
+        else:
+            print("⚠️  GPU не обнаружен. Tilmash будет работать на CPU (медленнее)")
+    except ImportError:
+        print("❌ PyTorch не установлен. Установите его командой: pip install torch")
+
+    print("\nСовет: Если у вас есть GPU, но он не обнаружен, проверьте:\n"
+          "1. Драйверы NVIDIA (для CUDA)\n"
+          "2. Правильную версию PyTorch для вашей системы\n"
+          "3. Переустановите PyTorch с поддержкой CUDA: pip install torch --upgrade")

config.py

@@ -0,0 +1,22 @@
+"""
+Configuration defaults for the Translator application.
+This file contains the default values for environment variables.
+These are only used if the actual environment variables are not set.
+"""
+
+# Default model configuration
+DEFAULT_CONFIG = {
+    "MAX_PARALLEL_MODELS": 4,
+    "SESSION_TIMEOUT": 1800,
+    "MODEL_INSTANCE_TIMEOUT": 1800,
+    "ALLOW_GPU": True,   # Разрешить использование GPU если доступно
+    "LOGLEVEL": "INFO",
+    "MAX_TOKENS": 4096,
+    "CHUNK_SIZE": 3000
+}
+
+# Convert boolean and integer values to strings for environment variables
+ENV_DEFAULTS = {
+    key: str(value).lower() if isinstance(value, bool) else str(value)
+    for key, value in DEFAULT_CONFIG.items()
+}

main.py

@@ -0,0 +1,345 @@
+# main.py
+
+import os
+import streamlit as st
+import logging
+from dotenv import load_dotenv
+
+# Load environment variables first, before any other code
+load_dotenv()
+
+# Import configuration defaults (after loading .env to prioritize environment variables)
+from config import ENV_DEFAULTS, DEFAULT_CONFIG
+
+# Configure logging based on configuration
+log_level = os.environ.get('LOGLEVEL', DEFAULT_CONFIG['LOGLEVEL']).upper()
+logging.basicConfig(
+    level=getattr(logging, log_level),
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        # Only log to console if level is INFO or higher
+        logging.StreamHandler() if log_level != 'WARNING' else logging.NullHandler()
+    ]
+)
+
+# Configure app
+st.set_page_config(page_title="Translator & Readability", layout="wide")
+
+# Check for missing environment variables and use defaults from config
+for var, default in ENV_DEFAULTS.items():
+    if var not in os.environ:
+        logging.debug(f"Environment variable {var} not found, using default: {default}")
+        os.environ[var] = default
+
+# Model configuration from default config
+MODEL_CONFIG = {
+    "max_parallel_models": DEFAULT_CONFIG["MAX_PARALLEL_MODELS"],
+    "session_timeout": DEFAULT_CONFIG["SESSION_TIMEOUT"],
+    "allow_gpu": DEFAULT_CONFIG["ALLOW_GPU"]
+}
+
+# Initialize model semaphore for limiting concurrent model usage
+import threading
+model_semaphore = threading.Semaphore(MODEL_CONFIG["max_parallel_models"])
+
+import tempfile
+import io
+from docx import Document
+import uuid
+import traceback
+
+from models.nltk_resources import setup_nltk
+from utils.file_readers import read_file
+from utils.text_processing import detect_language
+from utils.readability_indices import (
+    flesch_reading_ease,
+    flesch_kincaid_grade_level,
+    gunning_fog_index,
+    smog_index,
+    highlight_complex_text
+)
+from utils.formatting import color_code_index
+from utils.tilmash_translation import tilmash_translate, display_tilmash_streaming_translation
+
+# Initialize session state for user identification
+if 'session_id' not in st.session_state:
+    st.session_state.session_id = str(uuid.uuid4())
+    
+if 'translation_lock' not in st.session_state:
+    st.session_state.translation_lock = False
+
+def handle_translation():
+    st.header("Перевод (Kazakh, Russian, English)")
+    
+    # Show session ID in sidebar for debugging
+    with st.sidebar.expander("Session Info", expanded=False):
+        st.write(f"Session ID: {st.session_state.session_id}")
+
+        # Add GPU usage option
+        if MODEL_CONFIG["allow_gpu"]:
+            st.session_state.use_gpu = st.checkbox("Использовать GPU (быстрее)", value=True)
+            if st.session_state.use_gpu:
+                try:
+                    import torch
+                    if torch.cuda.is_available():
+                        gpu_info = f"CUDA: {torch.cuda.get_device_name(0)}"
+                        st.success(f"Доступен GPU: {gpu_info}")
+                    elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+                        st.success("Доступен Apple Silicon GPU (MPS)")
+                    else:
+                        st.warning("GPU не обнаружен, будет использован CPU")
+                        st.session_state.use_gpu = False
+                except ImportError:
+                    st.warning("PyTorch не установлен, будет использован CPU")
+                    st.session_state.use_gpu = False
+        else:
+            st.session_state.use_gpu = False
+            st.write("GPU отключен в конфигурации")
+    
+    translate_input_method = st.radio("Способ ввода текста:", ["Загрузить файл", "Вставить текст"])
+    input_text = ""
+
+    if translate_input_method == "Загрузить файл":
+        uploaded_file = st.file_uploader("Выберите файл (.txt, .docx, .pdf)", type=["txt", "docx", "pdf"])
+        if uploaded_file is not None:
+            suffix = os.path.splitext(uploaded_file.name)[1]
+            with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp_file:
+                tmp_file.write(uploaded_file.getbuffer())
+                temp_file_path = tmp_file.name
+            input_text = read_file(temp_file_path)
+            os.remove(temp_file_path)
+            st.write("**Содержимое файла:**")
+            st.write(input_text)
+    else:
+        input_text = st.text_area("Вставьте ваш текст здесь", height=200)
+
+    if input_text:
+        auto_detect = st.checkbox("Автоматически определить язык", value=True)
+        src_lang = None
+        if auto_detect:
+            detected_lang = detect_language(input_text)
+            if detected_lang in ['ru','en','kk']:
+                st.info(f"Определён язык: {detected_lang}")
+                src_lang = detected_lang
+            else:
+                st.warning("Не удалось определить язык. Выберите вручную.")
+                src_lang = st.selectbox("Язык текста", ["ru", "en", "kk"])
+        else:
+            src_lang = st.selectbox("Язык текста", ["ru", "en", "kk"])
+
+        if src_lang == "ru":
+            tgt_options = ["en","kk"]
+        elif src_lang == "en":
+            tgt_options = ["ru","kk"]
+        else:
+            tgt_options = ["ru","en"]
+
+        tgt_lang = st.selectbox("Перевод на:", tgt_options)
+        
+        if st.button("Перевести"):
+            # Prevent multiple concurrent translations from same session
+            if st.session_state.translation_lock:
+                st.warning("Перевод уже выполняется. Пожалуйста, дождитесь завершения.")
+                return
+                
+            # Set translation lock
+            st.session_state.translation_lock = True
+            
+            try:
+                # Use the model semaphore to limit concurrent model access
+                acquired = model_semaphore.acquire(blocking=False)
+                if not acquired:
+                    st.warning("Максимальное количество параллельных моделей достигнуто. Пожалуйста, попробуйте позже.")
+                    st.session_state.translation_lock = False
+                    return
+                    
+                try:
+                    st.subheader("Результат перевода:")
+                    # Get the approximate size of the text to determine if chunking is needed
+                    approx_text_size = len(input_text) / 4  # rough approximation (4 chars ≈ 1 token)
+                    needs_chunking = approx_text_size > 500  # If text is likely over 500 tokens
+                    
+                    # Display appropriate spinner message
+                    spinner_message = "Processing text in chunks..." if needs_chunking else "Processing translation..."
+                    
+                    # Create a dedicated translator instance for this session
+                    from utils.tilmash_translation import TilmashTranslator
+                    # Используем GPU если включено в настройках
+                    use_gpu = getattr(st.session_state, 'use_gpu', False)
+                    translator = TilmashTranslator(use_gpu=use_gpu)
+                    
+                    with st.spinner(spinner_message):
+                        try:
+                            # Use direct streaming approach with session-specific translator
+                            result = ""
+                            translation_placeholder = st.empty()
+                            
+                            # Stream translation
+                            for chunk in translator.translate_streaming(input_text, src_lang, tgt_lang):
+                                result += chunk
+                                translation_placeholder.markdown(result)
+                                
+                        except Exception as e:
+                            st.error(f"Translation error: {str(e)}")
+                            logging.error(f"Tilmash translation error: {traceback.format_exc()}")
+                            result = None
+                    
+                    if result:
+                        # Prepare download capability
+                        doc = Document()
+                        doc.add_paragraph(result)
+                        doc_io = io.BytesIO()
+                        doc.save(doc_io)
+                        doc_io.seek(0)
+
+                        st.download_button(
+                            label="Скачать переведённый текст (.docx)",
+                            data=doc_io,
+                            file_name="translated_text.docx",
+                            mime="application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+                        )
+                    else:
+                        st.warning("Не удалось выполнить перевод.")
+                        
+                    # Unload Tilmash model after use
+                    try:
+                        if translator.initialized:
+                            translator.unload_model()
+                    except Exception as unload_error:
+                        logging.error(f"Error unloading Tilmash model: {str(unload_error)}")
+                except Exception as tilmash_error:
+                    st.error(f"Tilmash model error: {str(tilmash_error)}")
+                    logging.error(f"Tilmash model error: {traceback.format_exc()}")
+                finally:
+                    # Release the semaphore
+                    model_semaphore.release()
+            except Exception as outer_error:
+                st.error(f"Unexpected error: {str(outer_error)}")
+                logging.error(f"Unexpected error: {traceback.format_exc()}")
+            finally:
+                # Release translation lock
+                st.session_state.translation_lock = False
+
+def handle_readability_analysis():
+    st.header("Анализ удобочитаемости текста")
+    input_method = st.radio("Способ ввода текста:", ["Загрузить файл", "Вставить текст"])
+    text = ""
+
+    if input_method == "Загрузить файл":
+        uploaded_file = st.file_uploader("Выберите файл (.txt, .docx, .pdf)", type=["txt", "docx", "pdf"])
+        if uploaded_file is not None:
+            suffix = os.path.splitext(uploaded_file.name)[1]
+            with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp_file:
+                tmp_file.write(uploaded_file.getbuffer())
+                temp_file_path = tmp_file.name
+            text = read_file(temp_file_path)
+            os.remove(temp_file_path)
+            st.write("**Содержимое файла:**")
+            st.write(text)
+    else:
+        text = st.text_area("Вставьте ваш текст здесь", height=200)
+
+    if text:
+        auto_detect = st.checkbox("Определить язык автоматически", value=True)
+        if auto_detect:
+            detected_lang = detect_language(text)
+            st.info(f"Определён язык: {detected_lang}")
+            lang_code = detected_lang if detected_lang in ['ru','en','kk'] else 'en'
+        else:
+            lang_code = st.selectbox("Язык текста", ["ru", "en", "kk"])
+
+        if st.button("Анализировать"):
+            # Prevent multiple concurrent analyses
+            if 'analysis_lock' in st.session_state and st.session_state.analysis_lock:
+                st.warning("Анализ уже выполняется. Пожалуйста, дождитесь завершения.")
+                return
+                
+            # Set analysis lock
+            st.session_state.analysis_lock = True
+            
+            try:
+                # Use the model semaphore for consistency with translation
+                acquired = model_semaphore.acquire(blocking=False)
+                if not acquired:
+                    st.warning("Система загружена. Пожалуйста, попробуйте позже.")
+                    st.session_state.analysis_lock = False
+                    return
+                    
+                try:
+                    with st.spinner("Выполняется анализ..."):
+                        fre = flesch_reading_ease(text, lang_code)
+                        fkgl = flesch_kincaid_grade_level(text, lang_code)
+                        fog = gunning_fog_index(text, lang_code)
+                        smog = smog_index(text, lang_code)
+                        highlighted_text, complex_words_list = highlight_complex_text(text, lang_code)
+
+                    st.subheader("Результаты удобочитаемости")
+                    st.markdown(
+                        f"**Индекс удобочитаемости Флеша:** {color_code_index('Flesch Reading Ease', fre)}",
+                        unsafe_allow_html=True
+                    )
+                    st.markdown(
+                        f"**Индекс Флеша-Кинкейда:** {color_code_index('Flesch-Kincaid Grade Level', fkgl)}",
+                        unsafe_allow_html=True
+                    )
+                    st.markdown(
+                        f"**Индекс тумана Ганнинга:** {color_code_index('Gunning Fog Index', fog)}",
+                        unsafe_allow_html=True
+                    )
+                    st.markdown(
+                        f"**Индекс SMOG:** {color_code_index('SMOG Index', smog)}",
+                        unsafe_allow_html=True
+                    )
+                    
+                    st.subheader("Сложные слова")
+                    st.write(", ".join(set(complex_words_list)))
+                finally:
+                    # Release the semaphore
+                    model_semaphore.release()
+            finally:
+                # Release analysis lock
+                st.session_state.analysis_lock = False
+
+def main():
+    setup_nltk()
+    
+    # Log the model configuration only once per session
+    if 'model_config_logged' not in st.session_state:
+        logging.info(f"Using model configuration: {MODEL_CONFIG}")
+        st.session_state.model_config_logged = True
+
+        # Проверка доступности GPU при запуске
+        try:
+            import torch
+            if torch.cuda.is_available():
+                gpu_name = torch.cuda.get_device_name(0)
+                cuda_ver = torch.version.cuda if hasattr(torch.version, "cuda") else "N/A"
+                logging.info(f"Обнаружен GPU: {gpu_name}, CUDA {cuda_ver}")
+            elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+                logging.info("Обнаружен Apple Silicon GPU (MPS)")
+            else:
+                logging.warning("GPU не обнаружен. Устанавливаем устройство на CPU")
+                if not torch.cuda.is_available():
+                    # Вывод диагностической информации
+                    logging.warning("Диагностика CUDA:")
+                    logging.warning(f"torch.__version__: {torch.__version__}")
+                    if hasattr(torch.version, "cuda"):
+                        logging.warning(f"torch.version.cuda: {torch.version.cuda}")
+                    if hasattr(torch.cuda, "is_available"):
+                        logging.warning(f"torch.cuda.is_available(): {torch.cuda.is_available()}")
+        except ImportError:
+            logging.warning("PyTorch не установлен, будет использован CPU")
+        except Exception as e:
+            logging.warning(f"Ошибка при проверке GPU: {str(e)}")
+
+    st.title("Translation & Readability Analysis")
+    st.sidebar.header("Функциональность")
+    functionality = st.sidebar.radio("Выберите режим:", ["Перевод", "Анализ удобочитаемости"])
+
+    if functionality == "Перевод":
+        handle_translation()
+    elif functionality == "Анализ удобочитаемости":
+        handle_readability_analysis()
+
+if __name__ == "__main__":
+    main()

models/nltk_resources.py

@@ -0,0 +1,21 @@
+# models/nltk_resources.py
+
+import nltk
+import logging
+
+def setup_nltk():
+    nltk_data_dir = 'nltk_data'
+
+    # Add the nltk_data directory to the NLTK data path
+    if nltk_data_dir not in nltk.data.path:
+        nltk.data.path.insert(0, nltk_data_dir)
+
+    # Define the required package
+    required_package = 'punkt_tab'
+
+    # Check if the package is installed locally
+    try:
+        nltk.data.find('tokenizers/punkt_tab')
+    except LookupError:
+        logging.info(f"Downloading NLTK package: {required_package}")
+        nltk.download(required_package, download_dir=nltk_data_dir, quiet=True)

nltk_data/tokenizers/punkt_tab.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e57f64187974277726a3417ca6f181ec5403676c717672eef6a748a7b20e0106
+size 4319076

nltk_data/tokenizers/punkt_tab/README

@@ -0,0 +1,98 @@
+Pretrained Punkt Models -- Jan Strunk (New version trained after issues 313 and 514 had been corrected)
+
+Most models were prepared using the test corpora from Kiss and Strunk (2006). Additional models have
+been contributed by various people using NLTK for sentence boundary detection.
+
+For information about how to use these models, please confer the tokenization HOWTO:
+http://nltk.googlecode.com/svn/trunk/doc/howto/tokenize.html
+and chapter 3.8 of the NLTK book:
+http://nltk.googlecode.com/svn/trunk/doc/book/ch03.html#sec-segmentation
+
+There are pretrained tokenizers for the following languages:
+
+File                Language            Source                             Contents                Size of training corpus(in tokens)           Model contributed by
+=======================================================================================================================================================================
+czech.pickle        Czech               Multilingual Corpus 1 (ECI)        Lidove Noviny                   ~345,000                             Jan Strunk / Tibor Kiss
+                                                                           Literarni Noviny
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+danish.pickle       Danish              Avisdata CD-Rom Ver. 1.1. 1995     Berlingske Tidende              ~550,000                             Jan Strunk / Tibor Kiss
+                                        (Berlingske Avisdata, Copenhagen)  Weekend Avisen
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+dutch.pickle        Dutch               Multilingual Corpus 1 (ECI)        De Limburger                    ~340,000                             Jan Strunk / Tibor Kiss
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+english.pickle      English             Penn Treebank (LDC)                Wall Street Journal             ~469,000                             Jan Strunk / Tibor Kiss
+                    (American)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+estonian.pickle     Estonian            University of Tartu, Estonia       Eesti Ekspress                  ~359,000                             Jan Strunk / Tibor Kiss
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+finnish.pickle      Finnish             Finnish Parole Corpus, Finnish     Books and major national        ~364,000                             Jan Strunk / Tibor Kiss
+                                        Text Bank (Suomen Kielen           newspapers
+                                        Tekstipankki)
+                                        Finnish Center for IT Science
+                                        (CSC)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+french.pickle       French              Multilingual Corpus 1 (ECI)        Le Monde                        ~370,000                             Jan Strunk / Tibor Kiss
+                    (European)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+german.pickle       German              Neue Zürcher Zeitung AG            Neue Zürcher Zeitung            ~847,000                             Jan Strunk / Tibor Kiss
+                    (Switzerland)       CD-ROM
+                    (Uses "ss"
+                     instead of "ß")
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+greek.pickle        Greek               Efstathios Stamatatos              To Vima (TO BHMA)               ~227,000                             Jan Strunk / Tibor Kiss
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+italian.pickle      Italian             Multilingual Corpus 1 (ECI)        La Stampa, Il Mattino           ~312,000                             Jan Strunk / Tibor Kiss
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+norwegian.pickle    Norwegian           Centre for Humanities              Bergens Tidende                 ~479,000                             Jan Strunk / Tibor Kiss
+                    (Bokmål and         Information Technologies,
+                     Nynorsk)           Bergen
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+polish.pickle       Polish              Polish National Corpus             Literature, newspapers, etc.  ~1,000,000                             Krzysztof Langner
+                                        (http://www.nkjp.pl/)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+portuguese.pickle   Portuguese          CETENFolha Corpus                  Folha de São Paulo              ~321,000                             Jan Strunk / Tibor Kiss
+                    (Brazilian)         (Linguateca)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+slovene.pickle      Slovene             TRACTOR                            Delo                            ~354,000                             Jan Strunk / Tibor Kiss
+                                        Slovene Academy for Arts
+                                        and Sciences
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+spanish.pickle      Spanish             Multilingual Corpus 1 (ECI)        Sur                             ~353,000                             Jan Strunk / Tibor Kiss
+                    (European)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+swedish.pickle      Swedish             Multilingual Corpus 1 (ECI)        Dagens Nyheter                  ~339,000                             Jan Strunk / Tibor Kiss
+                                                                           (and some other texts)
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+turkish.pickle      Turkish             METU Turkish Corpus                Milliyet                        ~333,000                             Jan Strunk / Tibor Kiss
+                                        (Türkçe Derlem Projesi)
+                                        University of Ankara
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+
+The corpora contained about 400,000 tokens on average and mostly consisted of newspaper text converted to
+Unicode using the codecs module.
+
+Kiss, Tibor and Strunk, Jan (2006): Unsupervised Multilingual Sentence Boundary Detection.
+Computational Linguistics 32: 485-525.
+
+---- Training Code ----
+
+# import punkt
+import nltk.tokenize.punkt
+
+# Make a new Tokenizer
+tokenizer = nltk.tokenize.punkt.PunktSentenceTokenizer()
+
+# Read in training corpus (one example: Slovene)
+import codecs
+text = codecs.open("slovene.plain","Ur","iso-8859-2").read()
+
+# Train tokenizer
+tokenizer.train(text)
+
+# Dump pickled tokenizer
+import pickle
+out = open("slovene.pickle","wb")
+pickle.dump(tokenizer, out)
+out.close()
+
+---------

nltk_data/tokenizers/punkt_tab/czech/abbrev_types.txt

@@ -0,0 +1,118 @@
+t
+množ
+např
+j.h
+man
+ú
+jug
+dr
+bl
+ml
+okr
+st
+uh
+šp
+judr
+u.s.a
+p
+arg
+žitě
+st.celsia
+etc
+p.s
+t.r
+lok
+mil
+ict
+n
+tl
+min
+č
+d
+al
+ravenně
+mj
+nar
+plk
+s.p
+a.g
+roč
+b
+zdi
+r.s.c
+přek
+m
+gen
+csc
+mudr
+vic
+š
+sb
+resp
+tzn
+iv
+s.r.o
+mar
+w
+čs
+vi
+tzv
+ul
+pen
+zv
+str
+čp
+org
+rak
+sv
+pplk
+u.s
+prof
+c.k
+op
+g
+vii
+kr
+ing
+j.o
+drsc
+m3
+l
+tr
+ceo
+ch
+fuk
+vl
+viii
+líp
+hl.m
+t.zv
+phdr
+o.k
+tis
+doc
+kl
+ard
+čkd
+pok
+apod
+r
+př
+a.s
+j
+jr
+i.m
+e
+kupř
+f
+tř
+xvi
+mir
+atď
+vr
+r.i.v
+hl
+kv
+t.j
+y
+q.p.r

nltk_data/tokenizers/punkt_tab/czech/collocations.tab

@@ -0,0 +1,96 @@
+i	dejmala
+##number##	prosince
+h	steina
+##number##	listopadu
+a	dvořák
+v	klaus
+i	čnhl
+##number##	wladyslawowo
+##number##	letech
+a	jiráska
+a	dubček
+##number##	štrasburk
+##number##	juniorské
+##number##	století
+##number##	kola
+##number##	pád
+##number##	května
+##number##	týdne
+v	dlouhý
+k	design
+##number##	červenec
+i	ligy
+##number##	kolo
+z	svěrák
+##number##	mája
+##number##	šimková
+a	bělého
+a	bradáč
+##number##	ročníku
+##number##	dubna
+a	vivaldiho
+v	mečiara
+c	carrićre
+##number##	sjezd
+##number##	výroční
+##number##	kole
+##number##	narozenin
+k	maleevová
+i	čnfl
+##number##	pádě
+##number##	září
+##number##	výročí
+a	dvořáka
+h	g.
+##number##	ledna
+a	dvorský
+h	měsíc
+##number##	srpna
+##number##	tř.
+a	mozarta
+##number##	sudetoněmeckých
+o	sokolov
+k	škrach
+v	benda
+##number##	symfonie
+##number##	července
+x	šalda
+c	abrahama
+a	tichý
+##number##	místo
+k	bielecki
+v	havel
+##number##	etapu
+a	dubčeka
+i	liga
+##number##	světový
+v	klausem
+##number##	ženy
+##number##	létech
+##number##	minutě
+##number##	listopadem
+##number##	místě
+o	vlček
+k	peteraje
+i	sponzor
+##number##	června
+##number##	min.
+##number##	oprávněnou
+##number##	květnu
+##number##	aktu
+##number##	květnem
+##number##	října
+i	rynda
+##number##	února
+i	snfl
+a	mozart
+z	košler
+a	dvorskému
+v	marhoul
+v	mečiar
+##number##	ročník
+##number##	máje
+v	havla
+k	gott
+s	bacha
+##number##	ad

nltk_data/tokenizers/punkt_tab/czech/sent_starters.txt

@@ -0,0 +1,54 @@
+já
+milena
+tomáš
+oznámila
+podle
+my
+vyplývá
+hlavní
+jelikož
+musíme
+kdyby
+foto
+rozptylové
+snad
+zároveň
+jaroslav
+po
+v
+kromě
+pokud
+toto
+jenže
+oba
+jak
+zatímco
+ten
+myslím
+navíc
+dušan
+zdá
+dnes
+přesto
+tato
+ti
+bratislava
+ale
+když
+nicméně
+tento
+mirka
+přitom
+dokud
+jan
+bohužel
+ta
+díky
+prohlásil
+praha
+jestliže
+jde
+vždyť
+moskva
+proto
+to

nltk_data/tokenizers/punkt_tab/danish/abbrev_types.txt

@@ -0,0 +1,211 @@
+t
+tlf
+b.p
+evt
+j.h
+lenz
+mht
+gl
+bl
+stud.polit
+e.j
+st
+o
+dec
+mag
+h.b
+p
+adm
+el.lign
+e.s
+saalba
+styrt
+nr
+m.a.s.h
+etc
+pharm
+hg
+j.j
+dj
+mountainb
+f.kr
+h.r
+cand.jur
+sp
+osv
+s.g
+ndr
+inc
+b.i.g
+dk-sver
+sl
+v.s.o.d
+cand.mag
+d.v.s
+v.i
+bøddel
+fr
+ø«
+dr.phil
+chr
+p.d
+bj
+fhv
+tilskudsforhold
+m.a
+sek
+p.g.a
+int
+pokalf
+ik
+dir
+em-lodtrækn
+a.h
+o.lign
+p.t
+m.v
+n.j
+m.h.t
+m.m
+a.p
+pers
+4-bakketurn
+dr.med
+w.ø
+polit
+fremsættes
+techn
+tidl
+o.g
+i.c.i
+mill
+skt
+m.fl
+cand.merc
+kbh
+indiv
+stk
+dk-maked
+memorandum
+mestersk
+mag.art
+kitzb
+h
+lic
+fig
+dressurst
+sportsg
+r.e.m
+d.u.m
+sct
+kld
+bl.a
+hf
+g.a
+corp
+w
+konk
+zoeterm
+b.t
+a.d
+l.b
+jf
+s.b
+kgl
+ill
+beck
+tosset
+afd
+johs
+pct
+k.b
+sv
+verbalt
+kgs
+l.m.k
+j.l
+aus
+superl
+t.v
+mia
+kr
+pr
+præmien
+j.b.s
+j.o
+o.s.v
+edb-oplysninger
+o.m.a
+ca
+1b
+f.eks
+rens
+ch
+mr
+schw
+d.c
+utraditionelt
+idrætsgym
+hhv
+e.l
+s.s
+eks
+f.o.m
+dk-storbrit
+dk-jugo
+n.z
+derivater
+c
+pt
+vm-kval
+kl
+hr
+cand
+jur
+sav
+h.c
+arab.-danm
+d.a.d
+fl
+o.a
+a.s
+cand.polit
+grundejerform
+j
+faglærte
+cr
+a.a
+mou
+f.r.i
+årh
+o.m.m
+sve
+c.a
+engl
+sikkerhedssystemerne
+m.f
+j.k
+phil
+f
+vet
+mio
+k.e
+m.k
+atla
+idrætsg
+n.n
+4-bakketur
+dvs
+sdr
+s.j
+hol
+s.h
+pei
+kbhvn
+aa
+m.g.i
+fvt
+i«
+b.c
+th
+lrs

nltk_data/tokenizers/punkt_tab/danish/collocations.tab

@@ -0,0 +1,101 @@
+##number##	skak
+##number##	speedway
+##number##	rally
+##number##	april
+##number##	dm-fin
+##number##	viceformand
+m	jensen
+##number##	kano/kajak
+##number##	bowling
+##number##	dm-finale
+##number##	årh.
+##number##	januar
+##number##	august
+##number##	marathon
+##number##	kamp
+##number##	skihop
+##number##	etage
+##number##	tennis
+##number##	cykling
+e	andersen
+##number##	december
+g	h.
+##number##	neb
+##number##	sektion
+##number##	afd.
+##number##	klasse
+##number##	trampolin
+##number##	bordtennis
+##number##	formel
+##number##	århundredes
+##number##	dm-semifin
+##number##	heks
+##number##	taekwondo
+##number##	galop
+##number##	basketball
+##number##	dm
+m	skræl
+##number##	trav
+##number##	provins
+##number##	triathlon
+k	axel
+##number##	rugby
+s	h.
+##number##	klaverkoncert
+a	p.
+e	løgstrup
+k	telefax
+##number##	gyldendal
+##number##	fodbold
+e	rosenfeldt
+##number##	oktober
+k	o.
+##number##	september
+##number##	dec.
+##number##	juledag
+##number##	badminton
+##number##	sejlsport
+##number##	håndbold
+r	førsund
+e	jørgensen
+d	##number##
+k	e
+##number##	alp.ski
+##number##	judo
+##number##	roning
+##number##	november
+##number##	atletik
+##number##	århundrede
+##number##	ridning
+##number##	marts
+m	andersen
+d	roosevelt
+##number##	brydning
+s	kr.
+##number##	runde
+##number##	division
+##number##	sal
+##number##	boksning
+##number##	minut
+##number##	golf
+##number##	juni
+##number##	symfoni
+##number##	hurtigløb
+k	jørgensen
+##number##	jörgen
+##number##	klasses
+e	jacobsen
+k	jensen
+##number##	februar
+k	nielsen
+##number##	volleyball
+##number##	maj
+##number##	verdenskrig
+##number##	juli
+##number##	ishockey
+##number##	kunstskøjteløb
+b	jørgensen
+##number##	gymnastik
+##number##	svømning
+##number##	tw
+i	pedersens

nltk_data/tokenizers/punkt_tab/danish/sent_starters.txt

@@ -0,0 +1,64 @@
+kronik
+alligevel
+de
+først
+derfor
+vi
+selv
+hertil
+sådan
+dette
+sport
+man
+foto
+begge
+tag
+dertil
+reuter
+efter
+endelig
+ifølge
+lad
+når
+det
+desuden
+nu
+reuters
+årsagen
+tænk
+samtidig
+udover
+men
+endvidere
+rør
+rb
+udstillingen
+faktabox
+reception
+blandt
+hvad
+skær
+lilot
+derudover
+da
+tilsæt
+denne
+afp
+her
+hvis
+hæld
+problemet
+dermed
+jeg
+grafik
+anmeldelse
+den
+ebbe
+resultatet
+tværtimod
+hans
+måske
+feature
+tillæg
+hun
+han

nltk_data/tokenizers/punkt_tab/dutch/abbrev_types.txt

@@ -0,0 +1,99 @@
+m.j
+t
+ph
+j.h
+p.a.m
+j.m
+dr
+st
+j.b.m
+p
+nr
+h.s
+e.d
+t.e
+a.v
+esb
+s.z
+drs
+b.b
+m.o
+inc
+n
+pensioenfonds
+s.v.p
+bod
+fr
+pk
+r.p
+c.p.j
+v.l.n.r
+chr
+m.v.d
+int
+o.m
+j.v.d
+u.o.m
+f.c
+k
+bijgebracht
+ontwaakte
+m
+j.w
+a.l
+a.v.d
+s.v
+s
+j.d
+binnengekomen
+ds
+schouwburg
+b.v
+h
+a
+j.a
+aanvielen
+h.g
+p.f
+j.l
+mgr
+c.j
+blz
+l.e.h
+w.k
+g
+m.g
+r.v.d
+ing
+v.d
+c.q
+l
+h.p
+mr
+gesch
+e.l
+p.j
+mm
+j.g
+j.f
+c
+f.m
+jl
+r
+o.a
+a.s
+ir
+v
+j
+jr
+e
+m.i.v
+l.a
+f.v.d
+aansluit
+c.c
+a.m
+f.o.j
+m.b
+y
+th

nltk_data/tokenizers/punkt_tab/dutch/collocations.tab

@@ -0,0 +1,37 @@
+##number##	sotelo
+##number##	clas
+##number##	buckler
+##number##	carrera
+##number##	rmo
+##number##	orioli
+w	baron
+##number##	morales
+##number##	snotselelaank
+##number##	arcarons
+##number##	cavandoli
+##number##	pdm
+##number##	helvetia
+##number##	panasonic
+##number##	motorola
+w	bruinsma
+##number##	heer
+##number##	lotus
+##number##	banesto
+##number##	magnaldi
+w	jense
+w	heuvelmans
+w	spatje
+##number##	telekom
+f	kennedy
+##number##	gatorade
+##number##	mg-gb
+##number##	once
+##number##	peterhansel
+##number##	ariostea
+##number##	tvm
+##number##	höl
+##number##	castorama
+##number##	tulip
+b	situatie
+##number##	mas
+##number##	lotto

nltk_data/tokenizers/punkt_tab/dutch/sent_starters.txt

@@ -0,0 +1,54 @@
+het
+daardoor
+de
+er
+hoewel
+wat
+urlings
+na
+ze
+alleen
+dat
+ik
+pijls
+wie
+daarna
+foto
+als
+boer
+hammes
+verder
+ook
+evers
+vandaar
+toen
+we
+langenberg
+naast
+want
+in
+wij
+zo
+hendrikx
+daar
+crouzen
+dit
+daarnaast
+anp
+zij
+behalve
+waarom
+daarom
+bovendien
+hij
+daarbij
+nee
+volgens
+daarmee
+bukkems
+dvnl
+eén
+pas
+tijdens
+vooral
+maar

nltk_data/tokenizers/punkt_tab/english/abbrev_types.txt

@@ -0,0 +1,156 @@
+ct
+m.j
+t
+a.c
+n.h
+ms
+p.a.m
+dr
+pa
+p.m
+u.k
+st
+dec
+u.s.a
+lt
+g.k
+adm
+p
+h.m
+ga
+tenn
+yr
+sen
+n.c
+j.j
+d.h
+s.g
+inc
+vs
+s.p.a
+a.t
+n
+feb
+sr
+jan
+s.a.y
+n.y
+col
+g.f
+c.o.m.b
+d
+ft
+va
+r.k
+e.f
+chg
+r.i
+a.g
+minn
+a.h
+k
+n.j
+m
+l.f
+f.j
+gen
+i.m.s
+s.a
+aug
+j.p
+okla
+m.d.c
+ltd
+oct
+s
+vt
+r.a
+j.c
+ariz
+w.w
+b.v
+ore
+h
+w.r
+e.h
+mrs
+cie
+corp
+w
+n.v
+a.d
+r.j
+ok
+. . 
+e.m
+w.c
+ill
+nov
+u.s
+prof
+conn
+u.s.s.r
+mg
+f.g
+ph.d
+g
+calif
+messrs
+h.f
+wash
+tues
+sw
+bros
+u.n
+l
+wis
+mr
+sep
+d.c
+ave
+e.l
+co
+s.s
+reps
+c
+r.t
+h.c
+r
+wed
+a.s
+v
+fla
+jr
+r.h
+c.v
+m.b.a
+rep
+a.a
+e
+c.i.t
+l.a
+b.f
+j.b
+d.w
+j.k
+ala
+f
+w.va
+sept
+mich
+n.m
+j.r
+l.p
+s.c
+colo
+fri
+a.m
+g.d
+kan
+maj
+ky
+a.m.e
+n.d
+t.j
+cos
+nev

nltk_data/tokenizers/punkt_tab/english/collocations.tab

@@ -0,0 +1,37 @@
+##number##	international
+##number##	rj
+##number##	commodities
+##number##	cooper
+b	stewart
+##number##	genentech
+##number##	wedgestone
+i	toussie
+##number##	pepper
+j	fialka
+o	ludcke
+##number##	insider
+##number##	aes
+i	magnin
+##number##	credit
+##number##	corrections
+##number##	financing
+##number##	henley
+##number##	business
+##number##	pay-fone
+b	wigton
+b	edelman
+b	levine
+##number##	leisure
+b	smith
+j	walter
+##number##	pegasus
+##number##	dividend
+j	aron
+##number##	review
+##number##	abreast
+##number##	who
+##number##	letters
+##number##	colgate
+##number##	cbot
+##number##	notable
+##number##	zimmer

nltk_data/tokenizers/punkt_tab/english/sent_starters.txt

@@ -0,0 +1,39 @@
+most
+he
+since
+so
+both
+these
+it
+nevertheless
+this
+indeed
+however
+instead
+under
+similarly
+some
+though
+while
+when
+in
+despite
+although
+nonetheless
+thus
+there
+if
+the
+nor
+separately
+moreover
+but
+they
+yet
+many
+according
+sales
+among
+meanwhile
+even
+i

nltk_data/tokenizers/punkt_tab/estonian/abbrev_types.txt

@@ -0,0 +1,48 @@
+eos
+c
+a.d
+t.a.s.s
+e.t
+päevapiltnikud
+c.h
+b.p
+amm
+ameerika-mees
+n.-ö
+cm
+b
+mhm
+a.s
+m.e
+j.l
+j
+u.t
+vm
+g.u.n
+hajutada
+p.s
+a.b
+c.h.-r
+i.q
+gr
+fido
+pankurit
+s.v
+l.l
+c.-h
+m.h
+h.l
+m.k
+j.r
+t.k
+k.h
+89/90
+h
+a
+dost
+v.k
+e.q
+t.j
+m.b
+d
+p.k

nltk_data/tokenizers/punkt_tab/estonian/collocations.tab

@@ -0,0 +1,100 @@
+##number##	juuni
+##number##	novembril
+##number##	juulilt
+r	järve-vomm
+##number##	mida
+n	liidu
+##number##	milliseid
+##number##	oktoobri
+##number##	iidol
+m	e
+##number##	klassist
+##number##	millest
+##number##	august
+##number##	pariis
+##number##	septembrist
+##number##	oktoober
+##number##	märtsini
+##number##	kust
+k	mägi
+##number##	detsembrist
+##number##	jaanuari
+##number##	epee
+##number##	nimetage
+##number##	novembrini
+##number##	eluaasta
+s	mill
+##number##	helsingi
+##number##	jaanuarini
+##number##	aastail
+##number##	augustil
+##number##	millise
+##number##	juulist
+##number##	mai
+##number##	novembri
+##number##	oktoobrist
+##number##	juunini
+##number##	septembriks
+##number##	detsembril
+p	s
+##number##	jaanuar
+##number##	aastate
+##number##	milline
+##number##	kelle
+##number##	jaanuaril
+s	stadnikov
+##number##	aastaks
+##number##	stockholm
+##number##	suurim
+##number##	aasta
+##number##	sajandi
+##number##	millega
+##number##	aastast
+##number##	aastal
+##number##	kumb
+##number##	septembril
+##number##	korruselt
+##number##	septembri
+##number##	veebruarini
+##number##	london
+##number##	aastatel
+##number##	september
+##number##	veebruari
+##number##	oktoobrini
+##number##	mail
+m	kassovitz
+##number##	action-film
+##number##	mis
+k	herkül
+n	n
+##number##	detsembrini
+##number##	imre
+t	jõgeda
+##number##	casino
+##number##	septembrit
+##number##	augustini
+##number##	juulil
+##number##	november
+##number##	kuupäeval
+##number##	taevas
+##number##	septembrini
+##number##	detsember
+##number##	detsembri
+##number##	juunil
+##number##	augustist
+n	jurist
+##number##	missugust
+##number##	aastatesse
+##number##	aprillil
+##number##	augusti
+##number##	oktoobril
+##number##	märtsil
+##number##	a
+##number##	the
+##number##	sajandil
+##number##	aastani
+##number##	juuli
+##number##	septembrile
+##number##	millist
+##number##	millised
+##number##	veebruaril

nltk_data/tokenizers/punkt_tab/estonian/sent_starters.txt

@@ -0,0 +1,89 @@
+kalad
+õnneks
+selle
+maimu
+teisipäeval
+ma
+skorpion
+aga
+lisaks
+selleks
+maitse
+esiteks
+erinevalt
+pealegi
+praegu
+kas
+tegelikult
+neitsi
+nädalavahetus
+tema
+kui
+seega
+täna
+lugupidamisega
+miks
+teiseks
+väldi
+pohlak
+osades
+sõnn
+samas
+nimelt
+juhtkiri
+krimi
+nädalavahetusel
+näiteks
+kuidas
+ambur
+telgmaa
+laupäeval
+seetõttu
+rezhissöör
+kahjuks
+ent
+samuti
+ehkki
+veevalaja
+seepärast
+muidugi
+kuna
+tänaseks
+mina
+loomulikult
+ometi
+arvamus
+lõvi
+ee
+niisiis
+mul
+kaksikud
+tõsi
+hinnete
+sestap
+tõenäoliselt
+samal
+see
+paraku
+jäär
+kokkuvõttes
+küllap
+muide
+nüüd
+kolmapäeval
+võibolla
+kuid
+nädalavahetuse
+kuigi
+võid
+lõpuks
+kaalud
+areen
+kirjad
+vähk
+esmaspäeval
+nii
+need
+uue
+ta
+minu

nltk_data/tokenizers/punkt_tab/finnish/abbrev_types.txt

@@ -0,0 +1,81 @@
+t
+suom
+dr
+st
+970125090.jtun
+p
+sis
+t.h
+961221327.jtun
+a.i
+milj
+ski
+kp
+970131067.jtun
+970124030.jtun
+nk
+va
+pan
+yhteystiedot
+ruots
+jne
+t.a
+l.-g
+k
+j.w
+p2
+oik
+970102248.jtun
+hj
+s
+vt
+muistelmia
+o.s
+elo
+h
+ortod
+o.l
+w
+tms
+970120219.jtun
+pj
+ok
+toissapäiväinen
+28.t1
+pelintekijä
+970111011.jtun
+op
+os
+ns
+m.g
+o.-i
+m3
+pros
+mr
+970102171.jtun
+waller
+hels
+rotary-järjestössä
+ins
+esim
+apul
+fil
+id
+ym
+j
+rf
+v.o
+lis
+c.a
+em
+kand
+r.y
+valt
+dipl
+ö
+970111092.jtun
+ponteva
+y
+kapakoista
+970130160.jtun
+th

nltk_data/tokenizers/punkt_tab/finnish/collocations.tab

@@ -0,0 +1,167 @@
+##number##	sm
+##number##	ohjelmassa
+##number##	a3
+##number##	rc3
+##number##	rxd4
+##number##	hxg4
+o	stenberg
+##number##	lg5
+##number##	tallitontun
+##number##	lähetysohjeet
+##number##	uimakoulu
+##number##	jaana
+##number##	alustuksen
+##number##	uppo-nallen
+##number##	anne
+##number##	rxf3
+a	sjögren
+##number##	kamarikuoro
+##number##	vetäjänä
+##number##	pääsymaksu
+##number##	kerros
+##number##	kurssi
+##number##	kuori
+##number##	g4
+##number##	h3
+##number##	tiede-teatterissa
+##number##	kh2
+##number##	kausimaksu
+##number##	tia
+##number##	gxf5
+##number##	täky-galleria
+##number##	le2
+##number##	te8+
+##number##	la4
+##number##	keitä
+##number##	huhtikuuta
+##number##	menotiedoissa
+##number##	valmista
+##number##	txb5
+##number##	maskeerauskurssin
+##number##	rd2
+##number##	re2
+##number##	solisteina
+##number##	esitelmä
+##number##	puupiirrossarja
+##number##	ta1
+##number##	vaahdota
+##number##	h4
+##number##	kesäkuuta
+##number##	liikkeitä
+##number##	tuolloin
+##number##	viikko
+##number##	mittaa
+a	sjögrenin
+##number##	exf6
+##number##	rc6+
+##number##	viimeistele
+##number##	ld1
+##number##	elokuuta
+##number##	dh5+
+##number##	syyskuuta
+##number##	opettajina
+##number##	b3
+##number##	rauhankatu
+c	clarke
+##number##	saakka
+##number##	elokuvat
+b	huggins
+g	gahmberg
+##number##	luento
+##number##	lf3
+##number##	tammikuuta
+##number##	ryömä
+##number##	meller
+##number##	jäsenkortti
+##number##	esiintyjinä
+##number##	maria
+##number##	lf4
+##number##	siirto
+##number##	aurinko
+##number##	lxg6
+##number##	marraskuuta
+##number##	harjoituksissa
+##number##	romantika-yhtye
+##number##	g3
+##number##	heinäkuuta
+##number##	rxd5
+##number##	kuumenna
+e	hämäläisen
+##number##	bxc4
+##number##	te1
+##number##	kg2
+##number##	osallistumismaksu
+##number##	re5
+##number##	ohjelma
+##number##	varapuheenjohtajaksi
+##number##	raisa
+##number##	päivään
+##number##	luokan
+##number##	sulata
+##number##	levitä
+##number##	kaustinen
+##number##	kuoroa
+##number##	df3
+v	helsingistä
+##number##	mieskuoro
+##number##	lokakuuta
+##number##	kerho
+##number##	helmikuuta
+##number##	kokkola
+##number##	suuruusluokan
+v	kaupungista
+##number##	krs
+##number##	tekstit
+##number##	menyy
+##number##	rf3
+##number##	ulkoasiainministeriön
+##number##	kaada
+##number##	cxd5
+##number##	ilmailumuseo
+e	waris
+##number##	kierros
+##number##	tunnille
+##number##	kh3
+##number##	ohjaus
+a	t.
+##number##	postimaksu
+##number##	pane
+##number##	th3
+##number##	joulukuuta
+##number##	vatkaa
+##number##	kokeessa
+l	j.
+##number##	asti
+##number##	opastajana
+##number##	kirsi
+##number##	lc2
+##number##	lh2
+##number##	e4
+##number##	sairaankuljetukset
+##number##	sekoita
+##number##	mervi
+##number##	de2
+a	pietilän
+##number##	kf1
+##number##	toukokuuta
+##number##	maaliskuuta
+##number##	leikkaa
+##number##	ryhmänäytökset
+v	maaseudulta
+##number##	de3-e1
+##number##	c4
+##number##	ta1-b1
+##number##	d5
+##number##	pia
+##number##	lxd6
+##number##	d4
+##number##	f3-f4
+##number##	dxg6+
+##number##	sari
+##number##	pelkkään
+##number##	ld3
+##number##	perkaa
+##number##	lg3
+##number##	kg3
+##number##	kvm
+##number##	tb1xb6

nltk_data/tokenizers/punkt_tab/finnish/sent_starters.txt

@@ -0,0 +1,86 @@
+siinä
+lämpötila
+viiden
+he
+vapaa
+viime
+useimmat
+kansallisooppera
+rooleissa
+näin
+odotettavissa
+tiedustelut
+kansallisteatterin
+sen
+musiikki
+monet
+uusi
+avoinna
+pakkasta
+freeze
+tämä
+lämpö
+lautakunta
+vastaväittäjänä
+päivällä
+tällä
+esimerkiksi
+varoituksia
+merenkurkku
+meriennuste
+näyttelyssä
+kun
+pilvistä
+silloin
+selkämeren
+suurin
+se
+jos
+vaihtelevaa
+vastaväittäjinä
+sivu
+kaupunginteatterin
+pilvisyys
+siellä
+siksi
+kurssimaksu
+tämän
+kotimaa
+näiden
+teatteri
+kaikki
+puolipilvistä
+niiden
+maksimilämpötila
+lisäksi
+kaupunginhallitus
+helsingin
+nyt
+samalla
+hänen
+olen
+kaupunkikierros
+vastaväittäjä
+ne
+tästä
+enimmäkseen
+poika
+niinpä
+viirus
+me
+poliisi
+liput
+ilmoittautuminen
+tarjoa
+hän
+molemmat
+ulkomaat
+rock
+lääketieteen
+tanssi
+sainks
+näyttely
+lisätietoja
+ulkomaiden
+näyttelyn
+palo

nltk_data/tokenizers/punkt_tab/french/abbrev_types.txt

@@ -0,0 +1,61 @@
+p.o.l
+pds
+3o
+inscr
+suè
+z
+abst
+g.-b
+tél
+r
+ed
+o
+b
+esp
+j.l
+v
+k
+e.p
+aus
+jap
+r.e
+gb-bel
+p
+aut
+usx
+arg
+g
+e
+etc
+fra
+p.s
+j.-l
+blu
+e.-u
+f.b
+msf
+e.d
+shi
+can
+j.b
+s.a
+f.o
+you
+mir
+inc
+ital
+expr
+tch
+g-b-bel
+cid
+c.u
+ctk
+j.-m.g
+bta
+p.-b
+cie
+ita
+equ
+corp
+vot
+w

nltk_data/tokenizers/punkt_tab/french/collocations.tab

@@ -0,0 +1,18 @@
+##number##	shinozuka-magne
+##number##	ambrosino-baumgartner
+c	tanvier
+f	b.
+##number##	waldegaard-fenouil
+##number##	fermé
+a	dechaume
+i	demongeot
+s	motos
+##number##	rahier
+##number##	magnaldi
+##number##	orioli
+f	tél.
+##number##	cowan-delferrier
+##number##	vatanen-berglund
+##number##	picco
+##number##	masuoka-oligo
+##number##	medardo

nltk_data/tokenizers/punkt_tab/french/sent_starters.txt

@@ -0,0 +1,48 @@
+c
+depuis
+la
+enfin
+certains
+selon
+cet
+car
+ces
+il
+cependant
+pour
+j
+alors
+un
+certes
+les
+nous
+dans
+le
+une
+si
+mais
+en
+dès
+or
+tout
+ils
+l
+mr
+malgré
+elles
+né
+je
+on
+quand
+pourtant
+cela
+a
+après
+puis
+ce
+elle
+voilà
+cette
+comment
+quant
+ainsi

nltk_data/tokenizers/punkt_tab/german/abbrev_types.txt

@@ -0,0 +1,71 @@
+rfr
+t
+c
+a.d
+dk
+he
+mjm
+inkl
+bt
+69f
+crz
+dr
+st
+ib
+liv
+mrd
+n.r
+rg
+v
+vgl
+mgr
+cs
+prof
+j
+kfr
+bd
+fre
+gfh
+fon
+m
+rp
+nr
+chr
+etc
+hg
+sx
+rz
+48f
+kmu
+abs
+nkm
+z.b
+usw
+f
+d.h
+lz
+sc
+usf
+gir
+hag
+ff
+mio
+zr
+k
+h
+mey
+bst
+ne
+u.a
+fem
+bzw
+bü
+med
+u
+lts
+fr
+s.o.s
+w
+lib
+k.a
+th

nltk_data/tokenizers/punkt_tab/german/collocations.tab

@@ -0,0 +1,28 @@
+##number##	oktober
+##number##	jahrhunderts
+##number##	geburtstag
+##number##	juni
+s	##number##
+##number##	september
+##number##	mai
+##number##	dezember
+##number##	april
+##number##	ahv-revision
+##number##	revision
+##number##	jahrhundert
+##number##	landwirtschaftsbericht
+##number##	altersjahr
+##number##	februar
+a	schumpeter
+##number##	freiheit
+##number##	august
+##number##	januar
+##number##	märz
+a	meyers
+##number##	november
+##number##	bauetappe
+##number##	ahv-
+##number##	eu-richtlinie
+##number##	juli
+a	meyer
+##number##	säule

nltk_data/tokenizers/punkt_tab/german/sent_starters.txt

@@ -0,0 +1,107 @@
+das
+man
+es
+wir
+dabei
+ferner
+ähnliches
+während
+entscheidend
+ausserdem
+ein
+in
+der
+daraus
+obschon
+beide
+hier
+all
+neben
+solche
+hingegen
+selbstverständlich
+daneben
+hinzu
+vielmehr
+sie
+natürlich
+obwohl
+nun
+doch
+ob
+abgesehen
+überdies
+im
+zweitens
+darin
+erstens
+dieses
+nach
+wer
+da
+interessant
+seit
+zudem
+darüber
+umgekehrt
+ähnlich
+aber
+was
+nachdem
+insbesondere
+statt
+angesichts
+gefragt
+gleiches
+solange
+wenn
+dies
+dass
+wie
+damit
+allerdings
+denn
+letztere
+eine
+selbst
+gleichzeitig
+wo
+weder
+gerade
+unter
+problematischer
+wieso
+dennoch
+bei
+deshalb
+davon
+andernfalls
+er
+die
+anders
+auch
+ebenso
+so
+inzwischen
+sonst
+immerhin
+entsprechend
+danach
+am
+trotz
+trotzdem
+worum
+damals
+dafür
+schliesslich
+gemäss
+demgegenüber
+warum
+letzteres
+mit
+dazu
+anderseits
+ganz
+zwar
+dieser
+diese

nltk_data/tokenizers/punkt_tab/greek/abbrev_types.txt

@@ -0,0 +1,100 @@
+κλ
+δημ
+χλμ
+σ.τ.ε
+ό.π
+δρχ
+κων
+χρ
+π.α
+ριχ
+π.χρ
+υγ
+tel
+ζ
+ο.π
+βασ
+γλ
+n.c
+d.j
+σωκ
+π
+ιω
+αχ
+βα
+γερ
+εκδ
+κλπ
+φ
+ελ
+οσ
+α
+σελ
+ευ
+ε.έ
+ρ
+ε.τ.α
+λ
+εβ
+θρ
+ν
+βλ
+ηλ
+γ
+αρ
+π.χ
+ε.μ
+κ.μ
+α.ε
+μιχ
+δισ
+ολ
+μ
+κ.ά
+κ
+δηλ
+ε.α.χ
+πρ
+αγ
+μac
+κ.ο.κ
+λ.χ
+θ
+αδσ
+εκατ
+δρη
+εμμ
+δ
+δεκ
+σ.σ
+55ο
+κκ
+αδ
+τ.μ
+ε.ε
+μ.χ
+ν.μ
+κτλ
+δολ
+κ.ά.π
+αγγ
+μ.κ
+δ.σ
+μπ
+έκδ
+ι
+v
+χαρ
+γρ
+μ.μ.ε
+σχ
+λεκ
+σπ
+πλι
+αθ
+χ
+τζ
+τρισ
+στ
+ευθ
+μ.μ