first release

.gitignore

@@ -0,0 +1,195 @@
+# Created by https://www.toptal.com/developers/gitignore/api/python,visualstudiocode
+# Edit at https://www.toptal.com/developers/gitignore?templates=python,visualstudiocode
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+### Python Patch ###
+# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
+poetry.toml
+
+# ruff
+.ruff_cache/
+
+# LSP config files
+pyrightconfig.json
+
+### VisualStudioCode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+!.vscode/*.code-snippets
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
+
+# End of https://www.toptal.com/developers/gitignore/api/python,visualstudiocode

pyproject.toml

@@ -0,0 +1,29 @@
+[tool.poetry]
+name = "python-learning-bot"
+version = "0.1.0"
+description = ""
+authors = ["Vladislav.Novikov <mb1te.comcis@gmail.com>"]
+readme = "README.md"
+packages = [{include = "python_learning_bot"}]
+
+[tool.poetry.dependencies]
+python = "^3.11"
+llama-cpp-python = "^0.1.63"
+pydantic = {extras = ["dotenv"], version = "^1.10.9"}
+aiogram = {url = "https://github.com/aiogram/aiogram/archive/refs/tags/v3.0.0b7.zip"}
+redis = "^4.5.5"
+beautifulsoup4 = "^4.12.2"
+lxml = "^4.9.2"
+requests = "^2.31.0"
+sqlalchemy = "^2.0.16"
+
+
+[tool.poetry.group.dev.dependencies]
+mypy = "^1.3.0"
+pre-commit = "^3.3.3"
+ruff = "^0.0.272"
+isort = "^5.12.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

src/bot.py

@@ -0,0 +1,22 @@
+import asyncio
+import logging
+import sys
+
+from aiogram import Bot, Dispatcher
+from config import settings
+from router import router
+from storage import SQLiteStorage
+
+
+async def main():
+    bot = Bot(token=settings.TELEGRAM_TOKEN)
+    storage = SQLiteStorage()
+    dp = Dispatcher(storage=storage)
+    dp.include_router(router)
+
+    await dp.start_polling(bot)
+
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
+    asyncio.run(main())

src/config.py

@@ -0,0 +1,13 @@
+from pydantic import BaseSettings
+
+
+class Settings(BaseSettings):
+    TELEGRAM_TOKEN: str
+    MODEL_PATH: str
+    DB_PATH: str
+
+    class Config:
+        env_file = ".env"
+
+
+settings = Settings()

src/db/repositories/question.py

@@ -0,0 +1,22 @@
+from typing import Optional
+from sqlalchemy import select, update
+from sqlalchemy.orm.session import Session
+
+from db.tables.question import QuestionModel
+
+
+class QuestionRepository:
+    def __init__(self, session: Session):
+        self.session = session
+    
+    def get(self, **kwargs) -> Optional[QuestionModel]:
+        query = select(QuestionModel).filter_by(**kwargs)
+        return self.session.execute(query).unique().scalar_one_or_none()
+
+    def get_all(self, **kwargs) -> list[QuestionModel]:
+        query = select(QuestionModel).filter_by(**kwargs)
+        return self.session.execute(query).unique().scalars()
+
+    def update(self, question_id: int, **kwargs):
+        query = update(QuestionModel).filter_by(id=question_id).values(**kwargs)
+        self.session.execute(query)

src/db/repositories/user.py

@@ -0,0 +1,23 @@
+from typing import Optional
+from sqlalchemy import select, update
+from sqlalchemy.orm.session import Session
+
+from db.tables.user import UserModel
+
+
+class UserRepository:
+    def __init__(self, session: Session):
+        self.session = session
+    
+    def get(self, **kwargs) -> Optional[UserModel]:
+        query = select(UserModel).filter_by(**kwargs)
+        return self.session.execute(query).scalar_one_or_none()
+
+    def create(self, **kwargs) -> UserModel:
+        obj = UserModel(**kwargs)
+        self.session.add(obj)
+        return obj
+
+    def update(self, user_id: int, **kwargs) -> UserModel:
+        query = update(UserModel).filter_by(user_id=user_id).values(**kwargs)
+        self.session.execute(query)

src/db/session.py

@@ -0,0 +1,7 @@
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from config import settings
+
+engine = create_engine(settings.DB_PATH)
+create_session = sessionmaker(autocommit=False, autoflush=False, bind=engine)

src/db/tables/answer.py

@@ -0,0 +1,14 @@
+from typing import Literal
+from sqlalchemy import ForeignKey
+from sqlalchemy.orm import Mapped, mapped_column
+
+from db.tables.base import BaseModel
+
+
+class AnswerModel(BaseModel):
+    __tablename__ = "answer"
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    answer_text: Mapped[str] = mapped_column(nullable=False)
+    question_id: Mapped[int] = mapped_column(ForeignKey("question.id"), nullable=False)
+    is_correct: Mapped[int] = mapped_column(nullable=False)

src/db/tables/base.py

@@ -0,0 +1,3 @@
+from sqlalchemy.orm import declarative_base
+
+BaseModel = declarative_base()

src/db/tables/question.py

@@ -0,0 +1,17 @@
+from typing import TYPE_CHECKING
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from db.tables.base import BaseModel
+
+from db.tables.answer import AnswerModel
+
+
+class QuestionModel(BaseModel):
+    __tablename__ = "question"
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    rating: Mapped[int] = mapped_column(nullable=False)
+    question: Mapped[str] = mapped_column(nullable=False)
+    explanation: Mapped[str] = mapped_column(nullable=False)
+
+    answers: Mapped[list[AnswerModel]] = relationship(lazy="joined")

src/db/tables/user.py

@@ -0,0 +1,18 @@
+from typing import TYPE_CHECKING, Optional
+from sqlalchemy import ForeignKey
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from db.tables.base import BaseModel
+
+if TYPE_CHECKING:
+    from db.tables.question import QuestionModel
+
+class UserModel(BaseModel):
+    __tablename__ = "user"
+
+    user_id: Mapped[int] = mapped_column(primary_key=True)
+    rating: Mapped[int] = mapped_column(nullable=False)
+    state: Mapped[str] = mapped_column(nullable=False)
+    last_question_id: Mapped[Optional[int]] = mapped_column(ForeignKey("question.id"))
+
+    # last_question: Mapped[Optional["QuestionModel"]] = relationship(lazy="joined")

src/dependencies.py

@@ -0,0 +1,38 @@
+from contextlib import contextmanager
+import logging
+from db.repositories.user import UserRepository
+from db.repositories.question import QuestionRepository
+
+from db.session import create_session
+
+
+logger = logging.getLogger(__name__)
+
+
+@contextmanager
+def get_db_session():
+    try:
+        session = create_session()
+        yield session
+    except Exception as e:
+        logger.exception(e)
+    finally:
+        session.close()
+
+
+@contextmanager
+def get_user_repository():
+    try:
+        with get_db_session() as session:
+            yield UserRepository(session=session)
+    except Exception as e:
+        logger.exception(e)
+
+
+@contextmanager
+def get_question_repository():
+    try:
+        with get_db_session() as session:
+            yield QuestionRepository(session=session)
+    except Exception as e:
+        logger.exception(e)

src/find_rating.py

@@ -0,0 +1,14 @@
+from saiga.llama_cpu import LlamaCpu
+
+model = LlamaCpu()
+prompt = (
+    "I have python question. Detect complexity of it as Elo rating. "
+    "Return only one number.\n"
+    "Question: 1. Who developed Python Programming Language?\n"
+    "a) Wick van Rossum\n"
+    "b) Rasmus Lerdorf\n"
+    "c) Guido van Rossum\n"
+    "d) Niene Stom\n"
+    "Answer: c) Guido van Rossum"
+)
+print(model.answer(prompt))

src/keyboard.py

@@ -0,0 +1,13 @@
+from aiogram.types import KeyboardButton
+
+START_MENU_BUTTON = "Главное меню"
+QUIZ_BUTTON = "Ответить на вопрос"
+QUESTION_BUTTON = "Задать вопрос"
+
+BOT_KEYBOARD = [
+    [
+        KeyboardButton(text=START_MENU_BUTTON),
+        KeyboardButton(text=QUIZ_BUTTON),
+        KeyboardButton(text=QUESTION_BUTTON)
+    ]
+]

src/router.py

@@ -0,0 +1,108 @@
+from aiogram import F, Router
+from aiogram.filters import Command
+from aiogram.fsm.context import FSMContext
+from aiogram.types import KeyboardButton, Message, ReplyKeyboardMarkup
+from keyboard import BOT_KEYBOARD, QUESTION_BUTTON, QUIZ_BUTTON
+from dependencies import get_question_repository, get_user_repository
+
+import random
+from saiga.llama_cpu import LlamaCpu
+
+from state import UserState
+from utils import escape_all, update_elo
+
+router = Router()
+llm = LlamaCpu()
+
+
+@router.message(Command("start"))
+@router.message(UserState.start_menu)
+async def command_start(message: Message, state: FSMContext):
+    await state.set_state(UserState.await_tap)
+    await message.answer(
+        "Привет! Для использования моих функций используй следущие команды:",
+        reply_markup=ReplyKeyboardMarkup(keyboard=BOT_KEYBOARD, resize_keyboard=True)
+    )
+
+
+@router.message(UserState.await_tap, F.text == QUESTION_BUTTON)
+async def get_user_question(message: Message, state: FSMContext):
+    await state.set_state(UserState.question)
+    await message.answer("Задайте ваш вопрос")
+
+
+@router.message(UserState.question)
+async def answer_user_question(message: Message, state: FSMContext):
+    await state.set_state(UserState.await_tap)
+    answer = llm.answer(message.text)
+    return await message.answer(
+        answer,
+        reply_markup=ReplyKeyboardMarkup(keyboard=BOT_KEYBOARD, resize_keyboard=True)
+    )
+
+
+@router.message(UserState.await_tap, F.text == QUIZ_BUTTON)
+async def send_quiz(message: Message, state: FSMContext):
+    with get_user_repository() as user_repo, get_question_repository() as question_repo:
+        user = user_repo.get(user_id=state.key.user_id)
+        questions = question_repo.get_all()
+        questions = sorted(questions, key=lambda question: abs(question.rating - user.rating))
+        min_abs = abs(questions[0].rating - user.rating)
+        questions = [q for q in questions if abs(q.rating - user.rating) == min_abs]
+        selected_question = random.choice(questions)
+        user_repo.update(user.user_id, last_question_id=selected_question.id)
+        user_repo.session.commit()
+
+        await state.set_state(UserState.quiz)
+        return await message.answer(
+            selected_question.question,
+            # parse_mode="MarkdownV2",
+            reply_markup=ReplyKeyboardMarkup(
+                keyboard=[
+                    [KeyboardButton(text=answer.answer_text)]
+                    for answer in selected_question.answers
+                ],
+                resize_keyboard=True
+            )
+        )
+
+
+@router.message(UserState.quiz)
+async def check_answer(message: Message, state: FSMContext):
+    with get_user_repository() as user_repo, get_question_repository() as question_repo:
+        user = user_repo.get(user_id=state.key.user_id)
+        question = question_repo.get(id=user.last_question_id)
+        correct_answers = [
+            answer.answer_text.strip()
+            for answer in question.answers
+            if answer.is_correct == 1
+        ]
+
+        is_correct = int(message.text.strip() in correct_answers)
+        old_user_rating, old_question_rating = user.rating, question.rating
+        new_user_rating, new_question_rating = update_elo(
+            rating_user=old_user_rating,
+            rating_task=old_question_rating,
+            is_correct=is_correct
+        )
+
+        user_repo.update(user.user_id, rating=new_user_rating)
+        user_repo.session.commit()
+        question_repo.update(question.id, rating=new_question_rating)
+        question_repo.session.commit()
+
+        await state.set_state(UserState.await_tap)
+        answer = (
+            f"{question.explanation}\n"
+            f"{is_correct=}\n"
+            f"answer={message.text.strip()}\n"
+            f"{correct_answers=}\n"
+            f"User rating: {old_user_rating} => {new_user_rating}\n"
+            f"Question rating: {old_question_rating} => {new_question_rating}"
+        )
+
+        return await message.answer(
+            answer,
+            # parse_mode="MarkdownV2",
+            reply_markup=ReplyKeyboardMarkup(keyboard=BOT_KEYBOARD, resize_keyboard=True)
+        )

src/saiga/bin/saiga_7b_lora_q41.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fbf7c4d1c6986082325fcdb54eb446bb3e99b20bfaf9a5ca87ca9967a4db31d
+size 4212859520

src/saiga/llama_cpu.py

@@ -0,0 +1,85 @@
+import asyncio
+from concurrent.futures import ThreadPoolExecutor, TimeoutError
+
+from llama_cpp import Llama
+
+from config import settings
+
+
+class LlamaCpu:
+    def __init__(
+        self, n_ctx=2000, top_k=30, top_p=0.9, temperature=0.2, repeat_penalty=1.1
+    ):
+        self.SYSTEM_PROMPT = (
+            "Твоя задача отвечать на вопросы, связанные с языком "
+            "программирования Python."
+        )
+        self.SYSTEM_TOKEN = 1788
+        self.USER_TOKEN = 1404
+        self.BOT_TOKEN = 9225
+        self.LINEBREAK_TOKEN = 13
+
+        self.ROLE_TOKENS = {
+            "user": self.USER_TOKEN,
+            "bot": self.BOT_TOKEN,
+            "system": self.SYSTEM_TOKEN,
+        }
+
+        self.top_k = top_k
+        self.top_p = top_p
+        self.temperature = temperature
+        self.repeat_penalty = repeat_penalty
+
+        self.model = Llama(
+            model_path=settings.MODEL_PATH,
+            n_ctx=n_ctx,
+            n_parts=1,
+        )
+
+        self.system_tokens = self.get_system_tokens()
+        self.model.eval(self.system_tokens)
+
+    def get_message_tokens(self, role, content):
+        message_tokens = self.model.tokenize(content.encode("utf-8"))
+        message_tokens.insert(1, self.ROLE_TOKENS[role])
+        message_tokens.insert(2, self.LINEBREAK_TOKEN)
+        message_tokens.append(self.model.token_eos())
+        return message_tokens
+
+    def get_system_tokens(self):
+        return self.get_message_tokens(
+            role="system",
+            content=self.SYSTEM_PROMPT,
+        )
+
+    def answer(self, question: str) -> str:
+        message_tokens = self.get_message_tokens(role="user", content=question)
+        role_tokens = [self.model.token_bos(), self.BOT_TOKEN, self.LINEBREAK_TOKEN]
+        cur_tokens = self.system_tokens + message_tokens + role_tokens
+        generator = self.model.generate(
+            cur_tokens,
+            top_k=self.top_k,
+            top_p=self.top_p,
+            temp=self.temperature,
+            repeat_penalty=self.repeat_penalty,
+        )
+
+        answer = ""
+
+        for token in generator:
+            token_str = self.model.detokenize([token]).decode("utf-8")
+            if token == self.model.token_eos():
+                break
+
+            answer += token_str
+
+        return answer
+
+    async def async_answer(self, question: str) -> str:
+        loop = asyncio.get_event_loop()
+        with ThreadPoolExecutor() as executor:
+            future = loop.run_in_executor(executor, self.answer, question)
+            try:
+                return await asyncio.wait_for(future, timeout=settings.MODEL_TIMEOUT)
+            except TimeoutError:
+                return "Простите, мне удалось сочинить ответ за 60 секунд :("

src/state.py

@@ -0,0 +1,8 @@
+from aiogram.fsm.state import State, StatesGroup
+
+
+class UserState(StatesGroup):
+    start_menu = State()
+    await_tap = State()
+    quiz = State()
+    question = State()

src/storage.py

@@ -0,0 +1,34 @@
+from typing import Any, Dict, Optional
+from aiogram import Bot
+from aiogram.fsm.storage.base import BaseStorage, StateType, StorageKey
+
+from dependencies import get_user_repository
+from state import UserState
+
+
+class SQLiteStorage(BaseStorage):
+    async def set_state(self, bot: Bot, key: StorageKey, state: StateType = None):
+        with get_user_repository() as user_repo:
+            user_repo.update(key.user_id, state=state.state)
+            user_repo.session.commit()
+
+    async def get_state(self, bot: Bot, key: StorageKey) -> Optional[str]:
+        with get_user_repository() as user_repo:
+            user = user_repo.get(user_id=key.user_id)
+            if user is None:
+                user = user_repo.create(
+                    user_id=key.user_id,
+                    rating=0,
+                    state=UserState.start_menu.state
+                )
+                user_repo.session.commit()
+            return user.state
+
+    async def set_data(self, bot: Bot, key: StorageKey, data: Dict[str, Any]):
+        pass
+
+    async def get_data(self, bot: Bot, key: StorageKey) -> Dict[str, Any]:
+        pass
+
+    async def close(self) -> None:
+        pass

src/utils.py

@@ -0,0 +1,18 @@
+import re
+
+
+def update_elo(rating_user, rating_task, is_correct, K=20):
+    E_user = 1 / (1 + 10 ** ((rating_task - rating_user) / 400))
+    E_task = 1 / (1 + 10 ** ((rating_user - rating_task) / 400))
+
+    S_user = 1 if is_correct else 0
+    S_task = 0 if is_correct else 1
+
+    rating_user += K * (S_user - E_user)
+    rating_task += K * (S_task - E_task)
+    
+    return rating_user, rating_task
+
+
+def escape_all(message: str) -> str:
+    return re.sub(r"([\.\-\=\<\>\(\)])", r"\\\1", message)