Update app

pages/2_👨‍🏫_Начало_работы.py

@@ -4,7 +4,6 @@ from utilities_database.user_database_utils import load_user_tasks_data
 from utilities_database.user_database_utils import save_data_in_database
 from utilities_database.user_database_widgets import user_save_text_table
 from utilities_database.user_database_utils import load_users_particular_task
-from streamlit_extras.no_default_selectbox import selectbox as custom_select_box
 
 # Interface
 if st.session_state.get('-LOGGED_IN_BOOL-'):
@@ -15,52 +14,38 @@ if st.session_state.get('-LOGGED_IN_BOOL-'):
         '\n\n**_:red[СОЗДАНИЕ ЗАДАНИЙ]_**'
         '\n\nПосле выбора данного режима работы появится форма, которую необходимо заполнить:'
         '\n\n1. Придумайте **название** для файла с заданиями. '
-        'Вы можете оставить это поле пустым - именем по умолчанию служит текущая дата и первые 20 символов '
-        'введенного Вами текста.'
-        '\n\n2. Введите **текст** или выберите **текстовый файл** с исходным текстом, на основе которого Вы хотите '
-        'создать задания. '
+        'Вы можете оставить это поле пустым - именем по умолчанию служит текущая дата и первые 20 символов'
+        ' введенного Вами текста.'
+        '\n\n2. Введите **текст** или выберите **текстовый файл** с исходным текстом, на основе которого Вы хотите'
+        ' создать задания. '
         '\n\n3. Укажите *способ выбора целевых слов*:'
         '\n\t* *:green[Автоматически]*: программа сама выберет подходящие по сложности целевые слова.'
-        '\n\t* *:blue[Самостоятельно]*: введите в соответствующее поле целевые слова через запятую в той форме, '
-        'в которой они встречаются в тексте. В этом случае *:orange[языковой уровень]* можно не указывать, но тогда '
-        'дистракторы будут полностью случайными и несоотнесёнными с уровнем.'
-        '\n4. Если Вы выбрали *:green[автоматический поиск целевых слов]*, **_:red[обязательно]_** укажите '
-        '*:orange[языковой уровень]*. Данный параметр отвечает за выбор лексического минимума, использующегося при '
-        'подборе дистракторов.'
-        '\n5. Если Вы выбрали *:blue[самостоятельный ввод целевых слов]*, проверьте, что заполнили соответствующее '
-        'поле. ️ ❗ **:red[Введите слова в той форме, в которой они встречаются в тексте]**.'
-        '\n6. Укажите число дистракторов - неправильных вариантов ответа. Если указано _более четырех_ '
-        'дистракторов, возможно, что в некоторых заданиях будет выведено _меньшее количество, но не менее четырех_ '
-        'вариантов. Данное обстоятельство связано с проверкой наличия дистракторов в лексических минимумах.'
+        '\n\t* *:blue[Самостоятельно]*: введите в соответствующее поле целевые слова через запятую в той форме,'
+        ' в которой они встречаются в тексте. В этом случае *:orange[языковой уровень]* можно не указывать, но тогда'
+        ' дистракторы будут полностью случайными и несоотнесёнными с уровнем.'
+        '\n4. Если Вы выбрали *:green[автоматический поиск целевых слов]*, **_:red[обязательно]_** укажите'
+        ' *:orange[языковой уровень]*. Данный параметр отвечает за выбор лексического минимума, использующегося при'
+        ' подборе дистракторов.'
+        '\n5. Если Вы выбрали *:blue[самостоятельный ввод целевых слов]*, проверьте, что заполнили соответствующее'
+        ' поле. ️ ❗ **:red[Введите слова в той форме, в которой они встречаются в тексте]**.'
+        '\n6. Укажите число дистракторов - неправильных вариантов ответа. Если указано _более четырех_'
+        ' дистракторов, возможно, что в некоторых заданиях будет выведено _меньшее количество, но не менее четырех_'
+        ' вариантов. Данное обстоятельство связано с проверкой наличия дистракторов в лексических минимумах.'
         '\n7. Выберите **способы вывода** готовых материалов.'
-        '\n8. Для начала работы нажмите на кнопку **"Запуск"**. Если все поля заполнены верно, '
-        'начнется процесс генерации заданий. Прогресс будет отображаться на экране.'
+        '\n8. Для начала работы нажмите на кнопку **"Запуск"**. Если все поля заполнены верно,'
+        ' начнется процесс генерации заданий. Прогресс будет отображаться на экране.'
         '\n9. По окончании процесса генерации заданий будет выведено **_:green[соответсвующее сообщение]_**. '
         'Затем Вы можете перейти на вкладки **просмотра и 📥 сохранения** заданий, а так же 📝**онлайн-теста**.'
         '\n\n**_:red[ЗАГРУЗКА ИЗ АРХИВА]_**'
-        '\n\nПосле выбора данного режима работы появится таблица, в которой перечислены названия заданий, '
-        'которые Вы сохранили, языковой уровень и дата их создания.'
+        '\n\nПосле выбора данного режима работы появится таблица, в которой перечислены названия заданий,'
+        ' которые Вы сохранили, языковой уровень и дата их создания.'
         ' Для загрузки определенного файла с заданиями:'
         '\n1. Введите (или скопируйте из таблицы) название.'
         '\n2. Укажите соответсвующий языковой уровень.'
         '\n3. Нажмите на кнопку **"Загрузить"**.'
         '\n4. Если все поля заполнены верно, Вы увидите сообщение о том, что **:green[задания успешно загружены]**.'
-        '\n\n\nДля того, чтобы свернуть/развернуть блоки **Инструкций** или **Важной информации**, '
-        'кликните по заголовку этого блока или по стрелке (ᐯ / ᐱ), располагающейся в его правом верхнем углу.')
-    ANNOUNCES = st.expander('**ВАЖНАЯ ИНФОРМАЦИЯ**', expanded=False)
-    ANNOUNCES.success(
-        '**Уважаемые пользователи, пожалуйста, после генерации заданий перейдите на вкладку "📝Онлайн-тест" '
-        'и заполните там опросник. Таким образом Вы очень поможете в улучшении качества заданий! Спасибо!🤗**')
-    ANNOUNCES.warning(
-        '**Сейчас генератор проходит завершающую настройку и отладку, для которой необходимо большое количество '
-        'данных об уместности выбранных целевых слов и дистракторов к ним. Поэтому просим Вас отнестись с пониманием '
-        'к излишне большому количеству заданий.**')
-    ANNOUNCES.warning(
-        '**❗️ㅤУбедительно просим Вас дожидаться окончания генерации или загрузки и не переходить на '
-        'другие вкладки до выведения соответствующего сообщения.**')
-    ANNOUNCES.warning(
-        '**❗ㅤВ случае появления красных сообщений об ошибке, как правило, проблема решается '
-        'повторными нажатиями на нужный Вам элемент. Приносим извинения за неудобства.**')
+        '\n\n\nДля того, чтоб�� свернуть/развернуть блоки **Инструкций** или **Важной информации**,'
+        ' кликните по заголовку этого блока или по стрелке (ᐯ / ᐱ), располагающейся в его правом верхнем углу.')
     WHAT_TO_DO = st.radio(
         label='**Выберите режим работы**',
         options=[
@@ -79,10 +64,11 @@ if st.session_state.get('-LOGGED_IN_BOOL-'):
         COL1, COL2 = LOAD_FORM.columns([1, 1])
         UPLOAD_CLOUD_FILE_NAME = COL1.text_input('Введите название заданий', placeholder='Жду название')
         with COL2:
-            UPLOAD_CLOUD_CEFR_LEVEL = custom_select_box(
-                'Выберите языковой уровень',
-                ['A1', 'A2', 'B1', 'B2', 'Без уровня'],
-                no_selection_label='-Выберите языковой уровень-')
+            UPLOAD_CLOUD_CEFR_LEVEL = st.selectbox(
+                label='Выберите языковой уровень',
+                options=['A1', 'A2', 'B1', 'B2', 'C1', 'Без уровня'],
+                index=None,
+                placeholder='-Выберите языковой уровень-')
             st.session_state['-UPLOAD_CLOUD_CEFR_LEVEL-'] = UPLOAD_CLOUD_CEFR_LEVEL
         LOAD_BUTTON = LOAD_FORM.form_submit_button('Загрузить')
         if LOAD_BUTTON:
@@ -132,22 +118,23 @@ if st.session_state.get('-LOGGED_IN_BOOL-'):
             key='-DISTRACTOR_MODEL_MODE-', horizontal=True)
         CEFR_NUM_DISTRACTORS_COL, UTW_COL = FORM.columns([2, 2])
         with CEFR_NUM_DISTRACTORS_COL:
-            CEFR_TEXT_LEVEL = custom_select_box(
-                'Укажите уровень по CEFR:',
-                ['Без уровня', 'A1', 'A2', 'B1', 'B2'],
-                no_selection_label='-Выберите языковой уровень-')
+            CEFR_TEXT_LEVEL = st.selectbox(
+                label='Выберите языковой уровень',
+                options=['A1', 'A2', 'B1', 'B2', 'Без уровня'],
+                index=None,
+                placeholder='-Выберите языковой уровень-')
             st.session_state['-CEFR_TEXT_LEVEL-'] = CEFR_TEXT_LEVEL
         NUMBER_DISTRACTORS = CEFR_NUM_DISTRACTORS_COL.number_input(
             label='**Выберите количество дистракторов в задании:**',
             min_value=1,
             max_value=9,
-            value=2,
+            value=3,
             key='-NUM_DISTRACTORS-')
         TARGET_WORDS = UTW_COL.text_area(
             label='**Если "Самостоятельно", введите целевые слова:**',
             value='',
             height=120,
-            placeholder='Через запятую и пробел',
+            placeholder='Через запятую',
             key='-INPUT_TARGET_WORDS-')
         FORM.markdown('**Выберите формат(-ы) вывода:**')
         col1, col2, col3 = FORM.columns(3)
@@ -166,47 +153,44 @@ if st.session_state.get('-LOGGED_IN_BOOL-'):
 
         START_COL, RERUN_COL, EXIT_COL = FORM.columns([1, 1, 1])
         START_BUTTON = START_COL.form_submit_button(
-            label='**Запуск**')
+            label='**Запуск**',
+            use_container_width=True)
         RERUN_BUTTON = RERUN_COL.form_submit_button(
-            label='**Перезагрузка**')
+            label='**Перезагрузка**',
+            use_container_width=True)
         EXIT_BUTTON = EXIT_COL.form_submit_button(
-            label='**Выход**')
+            label='**Выход**',
+            use_container_width=True)
 
         if START_BUTTON:
-            LOGS = st.empty()
-            LOGS.success('**Готовые задания или онлайн-тест появятся на отдельных страницах**')
-
             # Initiate interface structure
-            PROGRESS_BAR = st.progress(0)
-            LOGS_D = st.empty()
-            PROGRESS_BAR_S = st.progress(0)
+            LOGS = st.status(label='Прогресс выполнения', expanded=True)
+
+            PROGRESS_BAR = LOGS.progress(0)
+            PROGRESS_BAR_DISTRACTORS = LOGS.progress(0)
 
             # Start generation process. Everything happens inside main_workflow func
             if DISTRACTOR_MODEL == 'Модель-3':
-                from utilities_language_bert.rus_main_workflow_bert import main_workflow_bert
-                __TASK_DATA__ = main_workflow_bert(
+                from utilities_language_bert.rus_main_workflow_bert import main_workflow
+                __TASK_DATA__ = main_workflow(
                     file=UPLOAD_FILE,
                     text=UPLOAD_TEXT,
                     logs=LOGS,
-                    logs_d=LOGS_D,
                     progress=PROGRESS_BAR,
-                    progress_s=PROGRESS_BAR_S,
+                    progress_d=PROGRESS_BAR_DISTRACTORS,
                     level=CEFR_TEXT_LEVEL,
                     tw_mode_automatic_mode=TARGET_WORDS_MODE,
                     target_words=TARGET_WORDS,
                     num_distractors=NUMBER_DISTRACTORS,
                     save_name=USER__SAVE_IN_CLOUD_FILE_NAME)
             else:
-                PROGRESS_BAR_D = st.progress(0)
-                from utilities_language_w2v.rus_main_workflow_w2v import main_workflow_w2v
-                __TASK_DATA__ = main_workflow_w2v(
+                from utilities_language_w2v.rus_main_workflow_w2v import main_workflow
+                __TASK_DATA__ = main_workflow(
                     file=UPLOAD_FILE,
                     text=UPLOAD_TEXT,
                     logs=LOGS,
-                    logs_d=LOGS_D,
                     progress=PROGRESS_BAR,
-                    progress_d=PROGRESS_BAR_D,
-                    progress_s=PROGRESS_BAR_S,
+                    progress_d=PROGRESS_BAR_DISTRACTORS,
                     level=CEFR_TEXT_LEVEL,
                     tw_mode_automatic_mode=TARGET_WORDS_MODE,
                     target_words=TARGET_WORDS,
@@ -225,28 +209,18 @@ if st.session_state.get('-LOGGED_IN_BOOL-'):
             st.session_state['-UPLOAD_CLOUD_FILE_NAME-'] = USER__SAVE_IN_CLOUD_FILE_NAME
 
             PROGRESS_BAR.progress(100)
-            PROGRESS_BAR_S.progress(100)
-            if SAVE_IN_CLOUD:
-                save_data_in_database(
-                    user_task_database=user_save_text_table,
-                    save_type='download',
-                    save_name=USER__SAVE_IN_CLOUD_FILE_NAME,
-                    cefr_level=CEFR_TEXT_LEVEL,
-                    created_at=str(datetime.datetime.now())[:-7],
-                    creator_name=st.session_state.get('-USER_NAME-'),
-                    generated_result=__TASK_DATA__,
-                    distractor_model=DISTRACTOR_MODEL)
-                LOGS.success('**Все готово! Сохраняем задания в облако! Чтобы скачать задания перейдите в '
-                             'соответсвующую вкладку**')
-            else:
-                LOGS.success('**Все готово! Готовые задания и/или онлайн-тест доступны в соответствующих вкладках.**')
-
-            # if TARGET_WORDS_MODE == 'Самостоятельно':
-            #     st.error('По разным причинам не смогли не смогли придумать задания со словами: ' +
-            #              ', '.join(__TASK_DATA__['BAD_USER_TARGET_WORDS']))
-            PROGRESS_BAR = ''
-            PROGRESS_BAR_S = ''
-            LOGS_D = ''
+            PROGRESS_BAR_DISTRACTORS.progress(100)
+            LOGS.update(label='**Все готово! Готовые задания и/или онлайн-тест доступны в соответствующих вкладках.**',
+                        state='complete', expanded=False)
+            save_data_in_database(
+                user_task_database=user_save_text_table,
+                save_type='download',
+                save_name=USER__SAVE_IN_CLOUD_FILE_NAME,
+                cefr_level=CEFR_TEXT_LEVEL,
+                created_at=str(datetime.datetime.now())[:-7],
+                creator_name=st.session_state.get('-USER_NAME-'),
+                generated_result=__TASK_DATA__,
+                distractor_model=DISTRACTOR_MODEL, allow=SAVE_IN_CLOUD)
 
         if EXIT_BUTTON:
             for key in st.session_state:
@@ -260,12 +234,6 @@ if st.session_state.get('-LOGGED_IN_BOOL-'):
             st.error('Что-то пошло не так?! Перезагружаюсь!')
             st.session_state["START_GENERATION"] = False
             st.stop()
-            st.experimental_rerun()
-
-    # LABEL
-    # st.markdown('*Автор-разработчик: А.В.Белый, кафедра математической лингвистики, филологический факультет СПбГУ,'
-    #             ' 4 курс, бакалавриат, "Прикладная, компьютерная и математическая лингвистика (английский язык)"*'
-    #             '\n\n*Научный руководитель: канд. филол. наук, доц. О.А.Митрофанова*')
-    # st.markdown('*E-mail: st087202@student.spbu.ru*')
+            st.rerun()
 else:
     st.warning('**Войдите или зарегистрируйтесь**')

pages/3_📥_Скачать.py

@@ -1,7 +1,7 @@
 import streamlit as st
 from utilities_ui.custom_download_button import download_button as d_button
 
-st.set_page_config(page_title='Скачать', layout="wide", page_icon=':ru:')
+st.set_page_config(page_title='Скачать', layout="wide", page_icon=':ru:', initial_sidebar_state='collapsed')
 if st.session_state.get('-LOGGED_IN_BOOL-') and (st.session_state.get('-DISPLAY_READY-')
                                                  or st.session_state.get('-DOWNLOAD_VERSION-')):
     result = st.session_state.get('RESULT')
@@ -10,7 +10,7 @@ if st.session_state.get('-LOGGED_IN_BOOL-') and (st.session_state.get('-DISPLAY_
         st.stop()
     # Download buttons
     if st.session_state.get('-DOWNLOAD_VERSION-'):
-        invite, tasks_col, tasks_with_answers_col, full_coll, rest = st.columns([1, 1, 2, 3, 1])
+        invite, tasks_col, tasks_with_answers_col, keys_only_col, full_coll, rest = st.columns([1, 1, 2, 1, 3, 1])
         invite.write('Скачать:')
         with tasks_col:
             d_button(
@@ -22,6 +22,11 @@ if st.session_state.get('-LOGGED_IN_BOOL-') and (st.session_state.get('-DISPLAY_
                 label='Задания+Ключи',
                 data=result['TEACHER_OUT'],
                 file_name=f'{result["name"]}_tasks_and_keys.txt')
+        with keys_only_col:
+            d_button(
+                label='Ключи',
+                data=result['KEYS_ONLY'],
+                file_name=f'{result["name"]}_keys.txt')
         with full_coll:
             d_button(
                 label='Исходник+Задания+Ключи',

pages/4_📝_Онлайн-тест (эксперимент).py

@@ -4,19 +4,19 @@ import streamlit as st
 from utilities_database.user_database_utils import save_data_in_database
 from utilities_database.user_database_widgets import user_save_text_table
 
-st.set_page_config(page_title='Онлайн-тест', layout="wide", page_icon=':ru:')
+st.set_page_config(page_title='Онлайн-тест', layout="wide", page_icon=':ru:', initial_sidebar_state='collapsed')
 if st.session_state.get('-ONLINE_TEST_READY-') and st.session_state.get('-LOGGED_IN_BOOL-'):
-    INSTRUCTION = st.expander(label='**ИНСТРУКЦИЯ**', expanded=True)
+    INSTRUCTION = st.expander(label='**ИНСТРУКЦИЯ**', expanded=False)
     INSTRUCTION.markdown(
         'Уважаемые пользователи, предлагаем Вам заполнить опросник по оценке качества созданных заданий. '
         '\n\nНиже находится анкета с заданиями в таблице.'
         '\n\n- В **первом столбце** приводится ответ - слово, удаленное из оригинального текста.'
         '\n\n- Отметьте во **втором столбце**, уместно ли создавать задание с данным словом.'
         '\n\n- В **третьем столбце** приведены подобранные программой дистракторы.'
-        '\n\n- Введите в **четвертый столбец** дистракторы (целиком или букву), которые, по Вашему мнению, '
-        '**:red[не уместны]**. '
-        '\n\n**:green[Уместными дистракторами]** мы предлагаем считать те, которые одновременно удовлетворяют '
-        'следующим условиям в рамках языкового уровня, для которого они созданы:'
+        '\n\n- Введите в **четвертый столбец** дистракторы (целиком или букву), которые, по Вашему мнению,'
+        ' **:red[не уместны]**. '
+        '\n\n**:green[Уместными дистракторами]** мы предлагаем считать те, которые одновременно удовлетворяют'
+        ' следующим условиям в рамках языкового уровня, для которого они созданы:'
         '\n\n1. не слишком очевидно являются неправильными вариантами (*варить суп/стол*);'
         '\n\n2. и при этом не могут быть полноценной заменой удаленного слова (*варить суп/кашу*)'
     )
@@ -31,25 +31,26 @@ if st.session_state.get('-ONLINE_TEST_READY-') and st.session_state.get('-LOGGED
     ONLINE_TEST = st.form('Онлайн тест')
     ONLINE_TEST.write(result['TEXT_WITH_GAPS'].replace('_', '\_'))
     BAD_DISTRACTORS_AND_ANSWERS_temp = ONLINE_TEST.data_editor(
-        pd.DataFrame([{"Задание №": i+1,
+        pd.DataFrame([{"Задание №": i + 1,
                        "Ответ": [answers[i][1]],
                        "Задание уместно": False,
                        "Дистракторы": tasks[i][1],
                        "Неуместные дистракторы": ''}
                       for i in range(len(tasks))]),
         num_rows="fixed",
-        height=45*len_answers,
+        height=40 * len_answers,
+        hide_index=True,
         use_container_width=True)
-    COMMENTS = ONLINE_TEST.text_input(label='**Прокомментировать**',
-                                      placeholder='Напишите комментарий')
-    SUBMIT = ONLINE_TEST.form_submit_button('READY')
+    COMMENTS = ONLINE_TEST.text_area(label='**Прокомментировать**',
+                                     placeholder='Напишите комментарий')
+    SUBMIT = ONLINE_TEST.form_submit_button('ГОТОВО')
     if SUBMIT:
         points = test_mark = 'Teacher'
         appropriate_tasks = BAD_DISTRACTORS_AND_ANSWERS_temp["Задание уместно"].values.tolist()
         inappropriate_distractors = BAD_DISTRACTORS_AND_ANSWERS_temp["Неуместные дистракторы"].values.tolist()
         RETURN_TEST_DATA = [{'ANSWER': answers[i],
                              'APPROPRIATE_TASK': appropriate_tasks[i],
-                            'INAPPROPRIATE_DISTRACTORS': inappropriate_distractors[i]} for i in range(len_answers)]
+                             'INAPPROPRIATE_DISTRACTORS': inappropriate_distractors[i]} for i in range(len_answers)]
         save_data_in_database(user_task_database=user_save_text_table,
                               save_type='online_test',
                               save_name=st.session_state['-UPLOAD_CLOUD_FILE_NAME-'],
@@ -57,8 +58,8 @@ if st.session_state.get('-ONLINE_TEST_READY-') and st.session_state.get('-LOGGED
                               created_at=str(datetime.datetime.now())[:-7],
                               creator_name=st.session_state.get('-USER_NAME-'),
                               test_taker_name=st.session_state.get('-USER_NAME-'),
-                              test_taker_answers=RETURN_TEST_DATA,
                               generated_result=result,
+                              test_taker_answers=RETURN_TEST_DATA,
                               test_taker_result={'Баллов': points, 'Всего': len_answers, 'Оценка': test_mark},
                               comments=COMMENTS)
 elif st.session_state.get('-LOGGED_IN_BOOL-'):

requirements.txt

@@ -1,15 +1,17 @@
-supabase>=2.4.0
-nltk>=3.8.1
-spacy>=3.7.2
-torch>=2.1.0
-gensim>=4.3.2
-pandas>=2.2.0
-requests>=2.31.0
-pymorphy2>=0.9.1
-trycourier>=5.0.0
-streamlit==1.30.0
-argon2-cffi>=21.3.0
-cryptography>=42.0.3
-transformers>=4.37.2
-streamlit-extras>=0.4.0
+nltk==3.8.1
+spacy==3.7.2
+torch==2.1.0
+gensim==4.3.2
+pandas==2.2.0
+catboost==1.2.2
+supabase==2.4.0
+requests==2.31.0
+pymorphy2==0.9.1
+trycourier==5.0.0
+streamlit==1.32.2
+argon2-cffi==21.1.0
+cryptography==42.0.3
+transformers==4.38.2
+streamlit-extras==0.4.0
+bert-extractive-summarizer==0.10.1
 ru_core_news_lg @ https://github.com/explosion/spacy-models/releases/download/ru_core_news_lg-3.7.0/ru_core_news_lg-3.7.0-py3-none-any.whl

utilities_cookies/cookie_manager.py

@@ -1,10 +1,8 @@
 import streamlit as st
 from pathlib import Path
-from typing import Mapping
-from datetime import datetime
-from datetime import timedelta
 from urllib.parse import unquote
-from typing import MutableMapping
+from datetime import datetime, timedelta
+from typing import Mapping, MutableMapping
 from streamlit.components.v1 import components
 
 

utilities_cookies/encrypted_cookie_manager.py

@@ -1,12 +1,10 @@
 import os
 import base64
 import streamlit as st
-from typing import Tuple
-from typing import Optional
 from cryptography import fernet
-from typing import MutableMapping
 from cryptography.fernet import Fernet
 from cryptography.hazmat.primitives import hashes
+from typing import Tuple, Optional, MutableMapping
 from utilities_cookies.cookie_manager import CookieManager
 from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
 

utilities_database/user_database_utils.py

@@ -1,9 +1,9 @@
-import re
-import json
-import secrets
 import pandas as pd
 import streamlit as st
+from json import loads
+from re import search, compile
 from trycourier import Courier
+from secrets import token_urlsafe
 from argon2 import PasswordHasher
 from argon2.exceptions import VerifyMismatchError
 
@@ -37,7 +37,7 @@ def check_valid_name(name_sign_up: str) -> bool:
     name_regex_eng = r'^[A-Za-z_]\w *'
     name_regex_rus = r'^[А-Яа-я_][А-Яа-я0-9_] *'
 
-    if re.search(name_regex_eng, name_sign_up) or re.search(name_regex_rus, name_sign_up):
+    if search(name_regex_eng, name_sign_up) or search(name_regex_rus, name_sign_up):
         return True
     return False
 
@@ -46,7 +46,7 @@ def check_valid_email(email_sign_up: str) -> bool:
     """
     Checks if the user entered a valid email while creating the account.
     """
-    regex = re.compile(r'([A-Za-z0-9]+[.-_])*[A-Za-z0-9]+@[A-Za-z0-9-]+(\.[A-Z|a-z]{2,})+')
+    regex = compile(r'([A-Za-z0-9]+[.-_])*[A-Za-z0-9]+@[A-Za-z0-9-]+(\.[A-Z|a-z]{2,})+')
     return True
     
     # if re.fullmatch(regex, email_sign_up):
@@ -133,7 +133,7 @@ def generate_random_passwd() -> str:
     Generates a random password to be sent in email.
     """
     password_length = 10
-    return secrets.token_urlsafe(password_length)
+    return token_urlsafe(password_length)
 
 
 def send_passwd_in_email(auth_token: str, user_name_forgot_passwd: str, email_forgot_passwd: str, company_name: str,
@@ -266,5 +266,5 @@ def load_users_particular_task(user_task_database, load_mode, creator_name, save
                                                 .eq('save_name', save_name)\
                                                 .eq('save_type', load_mode)\
                                                 .eq('cefr_level',cefr_level).execute().data[0]['generated_result']
-    return_data = json.loads(return_data.replace("'", '"'), strict=False)
+    return_data = loads(return_data.replace("'", '"'), strict=False)
     return return_data

utilities_database/user_database_widgets.py

@@ -3,7 +3,6 @@ from datetime import datetime
 from supabase import create_client, Client
 from utilities_option_menu.option_menu import option_menu
 import utilities_database.user_database_utils as db_utils
-from utilities_database.user_database_utils import check_usr_pass
 from utilities_cookies.encrypted_cookie_manager import EncryptedCookieManager
 
 DB_URL = st.secrets['SUPABASE_URL']
@@ -91,9 +90,9 @@ class LogIn:
                 login_submit_button = st.form_submit_button(label='Войти')
 
                 if login_submit_button:
-                    authenticate_user_check = check_usr_pass(user_log_in_database=user_login_table,
-                                                             user_name=user_name,
-                                                             password=password)
+                    authenticate_user_check = db_utils.check_usr_pass(user_log_in_database=user_login_table,
+                                                                      user_name=user_name,
+                                                                      password=password)
 
                     if not authenticate_user_check:
                         st.error("Неверное имя пользователя или пароль!")

utilities_language_bert/rus_main_workflow_bert.py

@@ -1,28 +1,20 @@
 import datetime
 from io import StringIO
+from typing import Union
 from random import sample
 from collections import defaultdict
-from streamlit import progress as st_progress
-from utilities_language_general.rus_constants import st
-from streamlit.elements import WIDGETS as ST_WIDGETS
-from utilities_language_bert.rus_sentence_bert import TASK
-from utilities_language_general.rus_constants import load_bert
-from utilities_language_general.rus_utils import prepare_tasks
-from utilities_language_bert.rus_sentence_bert import SENTENCE
-import utilities_language_general.rus_constants as esp_constants
-from utilities_language_general.rus_utils import prepare_target_words
-from utilities_language_general.rus_utils import compute_frequency_dict
 from streamlit.runtime.uploaded_file_manager import UploadedFile
-from utilities_language_general.rus_constants import BAD_USER_TARGET_WORDS
+from utilities_language_bert.rus_sentence_bert import TASK, SENTENCE
+from utilities_language_general.rus_utils import compute_frequency_dict, prepare_tasks, prepare_target_words
+from utilities_language_general.rus_constants import st, load_bert, load_classifiers, nlp, summarization, BAD_USER_TARGET_WORDS, MINIMUM_SETS
 
 
-def main_workflow_bert(
-        file: UploadedFile or None,
+def main_workflow(
+        file: Union[UploadedFile, None],
         text: str,
-        logs: ST_WIDGETS,
-        logs_d: ST_WIDGETS,
-        progress: st_progress,
-        progress_s: st_progress,
+        logs,
+        progress,
+        progress_d,
         level: str,
         tw_mode_automatic_mode: str,
         target_words: str,
@@ -31,13 +23,15 @@ def main_workflow_bert(
         global_bad_target_words=BAD_USER_TARGET_WORDS):
 
     # Clear bad target_words each time
-    global_bad_target_words = []
+    if global_bad_target_words:
+        global_bad_target_words = []
 
     # Define main global variables
-    logs.write()
     GLOBAL_DISTRACTORS = set()
     MAX_FREQUENCY = 0
 
+    logs.update(label='Загружаем языковые модели и другие данные', state='running')
+    pos_dict, scaler, classifier = load_classifiers('model3')
     mask_filler = load_bert()
 
     # Get input text
@@ -47,15 +41,15 @@ def main_workflow_bert(
     elif text != '':
         current_text = text
     else:
-        esp_constants.st.warning('Вы и текст не вставили, и файл не выбрали 😢')
+        st.warning('Вы ни текст не вставили, ни файл не выбрали 😢')
         current_text = ''
-        esp_constants.st.stop()
+        st.stop()
 
     # Process target words
     if tw_mode_automatic_mode == 'Самостоятельно':
         if target_words == '':
-            esp_constants.st.warning('Вы не ввели целевые слова')
-            esp_constants.st.stop()
+            st.warning('Вы не ввели целевые слова')
+            st.stop()
         # Cannot make up paradigm, so only USER_TARGET_WORDS is used
         USER_TARGET_WORDS = prepare_target_words(target_words)
         tw_mode_automatic_mode = False
@@ -68,8 +62,8 @@ def main_workflow_bert(
     current_text = current_text.replace('.', '. ').replace('. . .', '...').replace('  ', ' ').replace('…', '...') \
         .replace('…', '...').replace('—', '-').replace('\u2014', '-').replace('—', '-').replace('-\n', '') \
         .replace('\n', '%^&*')
-    current_text_sentences = [sent.text.strip() for sent in esp_constants.nlp(current_text).sents]
-    logs.success('Получили Ваш текст!')
+    current_text_sentences = [sent.text.strip() for sent in nlp(current_text).sents]
+    logs.update(label='Получили Ваш текст!', state='running')
     progress.progress(10)
 
     # Compute frequency dict
@@ -81,31 +75,12 @@ def main_workflow_bert(
         if j < len(FREQ_DICT) * _frequency_barrier_percent:
             MAX_FREQUENCY = tp[1]
     MAX_FREQUENCY = 3 if MAX_FREQUENCY < 3 else MAX_FREQUENCY
-    logs.success("Посчитали немного статистики!")
+    logs.update(label="Посчитали немного статистики!", state='running')
     progress.progress(15)
 
     # Choose necessary language minimum according to user's input
-    if level == 'A1':
-        target_minimum = esp_constants.a1_target_set
-        distractor_minimum = esp_constants.a1_distractor_set
-    elif level == 'A2':
-        target_minimum = esp_constants.a2_target_set
-        distractor_minimum = esp_constants.a2_distractor_set
-    elif level == 'B1':
-        target_minimum = esp_constants.b1_target_set
-        distractor_minimum = esp_constants.b1_distractor_set
-    elif level == 'B2':
-        target_minimum = esp_constants.b2_target_set
-        distractor_minimum = esp_constants.b2_distractor_set
-    elif level == 'C1':
-        target_minimum = esp_constants.c1_target_set
-        distractor_minimum = esp_constants.c1_distractor_set
-    elif level == 'C2':
-        target_minimum = esp_constants.c2_target_set
-        distractor_minimum = esp_constants.c2_distractor_set
-    elif level == 'Без уровня':
-        target_minimum = None
-        distractor_minimum = None
+    if level:
+        target_minimum, distractor_minimum = MINIMUM_SETS[level]
     else:
         target_minimum = None
         distractor_minimum = None
@@ -115,24 +90,41 @@ def main_workflow_bert(
     # Start generation process
     workflow = [SENTENCE(original=sent.strip(), n_sentence=num, max_num_distractors=num_distractors)
                 for num, sent in enumerate(current_text_sentences)]
-    logs.success("Запускаем процесс генерации заданий!")
+    logs.update(label="Запускаем процесс генерации заданий!", state='running')
     progress.progress(20)
 
+    # Define summary length
+    text_length = len(current_text_sentences)
+    if text_length <= 15:
+        summary_length = text_length
+    elif text_length <= 25:
+        summary_length = 15
+    else:
+        n = (text_length - 20) // 5
+        summary_length = 15 + 2 * n
+    round_summary_length = summary_length - (summary_length % - 10)
+
+    # Get summary. May choose between round_summary_length and summary_length
+    SUMMARY = summarization(current_text, num_sentences=round_summary_length)
+    logs.update('Нашли интересные предложения. Пригодятся!')
+    progress.progress(25)
+
     for sentence in workflow:
         sentence.lemmatize_sentence()
 
     for sentence in workflow:
         sentence.bind_phrases()
-    logs.success("Подготовили предложения для дальнейшей работы!")
+    logs.update(label="Подготовили предложения для дальнейшей работы!", state='running')
     progress.progress(30)
 
     for j, sentence in enumerate(workflow):
         sentence.search_target_words(target_words_automatic_mode=tw_mode_automatic_mode,
                                      target_minimum=target_minimum,
                                      user_target_words=USER_TARGET_WORDS,
-                                     frequency_dict=FREQ_DICT)
+                                     frequency_dict=FREQ_DICT, 
+                                     summary=SUMMARY)
         progress.progress(int(30 + (j * (20 / len(workflow)))))
-    progress_s.progress(50)
+    progress.progress(50)
     DUPLICATE_TARGET_WORDS = defaultdict(list)
     for sentence in workflow:
         for target_word in sentence.target_words:
@@ -145,8 +137,8 @@ def main_workflow_bert(
             if target_word not in RESULT_TW:
                 global_bad_target_words.append(target_word['original_text'])
                 sentence.target_words.remove(target_word)
-    progress_s.progress(55)
-    logs.success('Выбрали слова-пропуски!')
+    progress.progress(55)
+    logs.update(label='Выбрали слова-пропуски!', state='running')
 
     for sentence in workflow:
         for i, target_word in enumerate(sentence.target_words):
@@ -157,7 +149,7 @@ def main_workflow_bert(
 
     for sentence in workflow:
         sentence.filter_target_words(target_words_automatic_mode=tw_mode_automatic_mode)
-    progress_s.progress(60)
+    progress.progress(60)
 
     RESULT_TASKS = []
     for sentence in workflow:
@@ -166,25 +158,29 @@ def main_workflow_bert(
             RESULT_TASKS.append(task)
 
     for num, task in enumerate(RESULT_TASKS):
-        task.attach_distractors_to_target_word(model=mask_filler, level_name=level,
+        task.attach_distractors_to_target_word(model=mask_filler, 
+                                               scaler=scaler,
+                                               classifier=classifier,
+                                               pos_dict=pos_dict,
+                                               level_name=level,
                                                global_distractors=GLOBAL_DISTRACTORS,
                                                distractor_minimum=distractor_minimum,
                                                max_frequency=MAX_FREQUENCY)
-        logs_d.success(
-            f'Обработали {num}/{len(RESULT_TASKS)} целевых слов!')
-    logs_d.success(
-        f'Обработали {len(RESULT_TASKS)}/{len(RESULT_TASKS)} целевых слов!')
-    progress_s.progress(65)
-    logs.success('Подобрали неправильные варианты!')
+        progress_d.progress(num / len(RESULT_TASKS))
+        logs.update(label=f'Обработали {num}/{len(RESULT_TASKS)} целевых слов!', state='running')
+    logs.update(label=f'Обработали {len(RESULT_TASKS)}/{len(RESULT_TASKS)} целевых слов!', state='running')
+    progress_d.progress(100)
+    progress.progress(70)
+    logs.update(label='Подобрали неправильные варианты!', state='running')
 
     for task in RESULT_TASKS:
         task.inflect_distractors()
-    progress_s.progress(70)
-    logs.success('П��осклоняли и проспрягали неправильные варианты!')
+    progress.progress(80)
+    logs.update(label='Просклоняли и проспрягали неправильные варианты!', state='running')
 
     for task in RESULT_TASKS:
         task.sample_distractors(num_distractors=num_distractors)
-    progress_s.progress(75)
+    progress.progress(85)
     RESULT_TASKS = list(filter(lambda t: not t.bad_target_word, RESULT_TASKS))
 
     for task in RESULT_TASKS[::-1]:
@@ -202,13 +198,18 @@ def main_workflow_bert(
                 NUMBER_TASKS = 10
             else:
                 NUMBER_TASKS = len(RESULT_TASKS)
-    RESULT_TASKS = sample(RESULT_TASKS, NUMBER_TASKS)
+    RESULT_TASKS_in_summary = list(filter(lambda task: task.in_summary, RESULT_TASKS))
+    RESULT_TASTS_not_in_summary = list(filter(lambda task: not task.in_summary, RESULT_TASKS))
+    if len(RESULT_TASKS_in_summary) >= NUMBER_TASKS:
+        RESULT_TASKS = RESULT_TASKS_in_summary
+    else:
+        RESULT_TASKS = RESULT_TASKS_in_summary + sample(RESULT_TASTS_not_in_summary, NUMBER_TASKS - len(RESULT_TASKS_in_summary))
     RESULT_TASKS = sorted(RESULT_TASKS, key=lambda t: (t.sentence_number, t.position_in_sentence))
 
     for task in RESULT_TASKS:
         task.compile_task(max_num_distractors=num_distractors)
-    progress_s.progress(85)
-    logs.success('Отобрали лучшие задания!')
+    progress.progress(90)
+    logs.update(label='Отобрали лучшие задания!', state='running')
 
     TEXT_WITH_GAPS = []
     VARIANTS = []
@@ -222,9 +223,6 @@ def main_workflow_bert(
         TEXT_WITH_GAPS.append(sentence)
     del RESULT_TASKS
 
-    logs.success('Сейчас все будет готово!')
-    progress_s.progress(90)
-
     TEXT_WITH_GAPS = ' '.join([sentence for sentence in TEXT_WITH_GAPS]).replace('%^&*', '\n')
     PREPARED_TASKS = prepare_tasks(VARIANTS)
     STUDENT_OUT = f'{TEXT_WITH_GAPS}\n\n{"=" * 70}\n\n{PREPARED_TASKS["TASKS_STUDENT"]}'
@@ -232,8 +230,8 @@ def main_workflow_bert(
                   f'{PREPARED_TASKS["KEYS_ONLY"]}'
     TOTAL_OUT = f'{original_text}\n\n{"$" * 70}\n\n{STUDENT_OUT}\n\n{"=" * 70}\n\n{PREPARED_TASKS["TASKS_TEACHER"]}' \
                 f'\n\n{"$" * 70}\n\n{PREPARED_TASKS["KEYS_ONLY"]}'
-    logs.success('Сейчас все будет готово!')
-    progress_s.progress(90)
+    logs.update(label='Сейчас все будет готово!', state='running')
+    progress.progress(95)
     save_name = save_name if save_name != '' else f'{str(datetime.datetime.now())[:-7]}_{original_text[:20]}'
     out = {
         'name': save_name,

utilities_language_bert/rus_sentence_bert.py

@@ -1,15 +1,7 @@
-import copy
 import string
-from random import random
-from random import sample
-from utilities_language_general.rus_constants import nlp
-from utilities_language_general.rus_utils import get_tags
-from utilities_language_general.rus_constants import PHRASES
-from utilities_language_general.rus_utils import define_gender
-from utilities_language_general.rus_utils import make_inflection
-from utilities_language_general.rus_utils import check_token_bert
-from utilities_language_general.rus_constants import BAD_USER_TARGET_WORDS
-from utilities_language_general.rus_utils import get_distractors_from_model_bert
+from random import random, sample
+from utilities_language_general.rus_constants import nlp, PHRASES, BAD_USER_TARGET_WORDS
+from utilities_language_general.rus_utils import get_tags, define_gender, make_inflection, check_token_bert, get_distractors_from_model_bert
 
 
 class SENTENCE:
@@ -47,11 +39,10 @@ class SENTENCE:
                 if not previous_was_phrase:
                     self.sentence_phrases.append(self.sentence_lemma_pos[i][1])
                 previous_was_phrase = False
+        self.sentence_phrases.append(self.sentence_lemma_pos[-1][1])
 
-    def search_target_words_automatically(self, target_minimum: set, frequency_dict: dict = None):
+    def search_target_words_automatically(self, target_minimum: set, frequency_dict: dict = None, summary: list=None):
         for token in self.sentence_phrases:
-            # TODO: Still do not have w2v model with phrases
-            #  therefore cannot come up with the criteria
             if isinstance(token, list):  # if token is a phrase
                 original_token1 = token[1]['original_token1']
                 original_token2 = token[1]['original_token2']
@@ -71,7 +62,8 @@ class SENTENCE:
                     'tags': tags,
                     'position_in_sentence': self.original.find(original_token1.text),
                     'not_named_entity': not_ner,
-                    'frequency_in_text': 0
+                    'frequency_in_text': 0,
+                    'in_summary': self.original in summary
                 }
                 self.target_words.append(target_word)
             else:  # if token is just a spacy.nlp token
@@ -89,10 +81,11 @@ class SENTENCE:
                         'position_in_sentence': self.original.find(token.text),
                         'not_named_entity': True if token.ent_type == 0 else False,
                         'frequency_in_text': frequency_dict.get(token.lemma_, 1),
+                        'in_summary': self.original in summary
                     }
                     self.target_words.append(target_word)
 
-    def search_user_target_words(self, user_target_words: set = None, frequency_dict: dict = None):
+    def search_user_target_words(self, user_target_words: set = None, frequency_dict: dict = None, summary: list=None):
         for _utw in user_target_words:
             if _utw in self.original:
                 parse_utw = nlp(_utw)
@@ -118,19 +111,20 @@ class SENTENCE:
                     'tags': user_target_word_tags,
                     'position_in_sentence': self.original.find(_utw),
                     'not_named_entity': not_ner,
-                    'frequency_in_text': frequency_dict.get(user_target_word_lemma, 1)
+                    'frequency_in_text': frequency_dict.get(user_target_word_lemma, 1),
+                    'in_summary': self.original in summary
                 }
                 self.target_words.append(target_word)
 
     def search_target_words(self, target_words_automatic_mode: bool, target_minimum,
                             user_target_words: set = None,
-                            frequency_dict: dict = None):
+                            frequency_dict: dict = None, summary: list=None):
         if target_words_automatic_mode:
             self.search_target_words_automatically(target_minimum=target_minimum,
-                                                   frequency_dict=frequency_dict)
+                                                   frequency_dict=frequency_dict, summary=summary)
         else:
             self.search_user_target_words(user_target_words=user_target_words,
-                                          frequency_dict=frequency_dict)
+                                          frequency_dict=frequency_dict, summary=summary)
 
     def filter_target_words(self, target_words_automatic_mode):
         c_position = 0
@@ -170,17 +164,11 @@ class TASK:
     def __repr__(self):
         return '\n'.join([f'{key}\t=\t{value}' for key, value in self.__dict__.items()])
 
-    def attach_distractors_to_target_word(self, model, global_distractors, distractor_minimum,
-                                          level_name, max_frequency):
+    def attach_distractors_to_target_word(self, model, scaler, classifier, pos_dict, 
+                                          global_distractors, distractor_minimum, level_name, max_frequency):
         pos = self.pos[0] if self.pos[0] == 'phrase' else self.pos[1]
-        # distractors_full_text = get_distractors_from_model_bert(model=model, lemma=self.lemma, pos=pos,
-        #                                                         gender=self.gender, level_name=level_name,
-        #                                                         text_with_masked_task=self.text_with_masked_task,
-        #                                                         global_distractors=global_distractors,
-        #                                                         distractor_minimum=distractor_minimum,
-        #                                                         max_num_distractors=self.max_num_distractors)
-        distractors_sentence = get_distractors_from_model_bert(model=model, lemma=self.lemma, pos=pos,
-                                                               gender=self.gender, level_name=level_name,
+        distractors_sentence = get_distractors_from_model_bert(model=model, scaler=scaler, classifier=classifier, pos_dict=pos_dict,
+                                                               level_name=level_name, lemma=self.lemma, pos=pos, gender=self.gender, 
                                                                text_with_masked_task=self.masked_sentence,
                                                                global_distractors=global_distractors,
                                                                distractor_minimum=distractor_minimum,
@@ -201,17 +189,6 @@ class TASK:
             inflected = make_inflection(text=distractor_lemma, pos=self.pos[1], tags=self.tags)
             if inflected is not None:
                 inflected_distractors.append(inflected)
-            else:
-                new_tags = copy.deepcopy(self.tags)
-                if 'NOUN' in self.tags and 'inan' in self.tags:
-                    new_tags.discard('inan')
-                    new_tags.add('anim')
-                elif 'NOUN' in self.tags and 'anim' in self.tags:
-                    new_tags.discard('anim')
-                    new_tags.add('inan')
-                inflected = make_inflection(text=distractor_lemma, pos=self.pos[1], tags=new_tags)
-                if inflected is not None:
-                    inflected_distractors.append(inflected)
         num_distractors = min(4, self.max_num_distractors) if self.max_num_distractors >= 4 \
             else self.max_num_distractors
         if len(inflected_distractors) < num_distractors:

utilities_language_general/rus_constants.py

@@ -3,7 +3,9 @@ import spacy
 import gensim
 import pymorphy2
 import streamlit as st
+from pickle import load
 from transformers import pipeline
+from summarizer import Summarizer
 
 
 @st.cache_resource
@@ -13,24 +15,53 @@ def load_morph():
 
 
 @st.cache_resource
-def load_w2v(model_path):
-    _w2v_model = gensim.models.KeyedVectors.load_word2vec_format(model_path, binary=True)
-    return _w2v_model
+def load_w2v(model):
+    with st.spinner('Загружаю языковую модель'):
+        if model == 'model1':
+            model_path = r'language_data/model1.gz'
+        else:
+            model_path = r'language_data/model2.gz'
+    return gensim.models.KeyedVectors.load_word2vec_format(model_path, binary=True)
 
 
 @st.cache_resource
 def load_spacy():
-    _nlp = spacy.load('ru_core_news_lg')
+    with st.spinner('Загружаю морфо-синтаксический парсер'):
+        _nlp = spacy.load('ru_core_news_lg')
     return _nlp
 
 
 @st.cache_resource
 def load_bert():
-    return pipeline("fill-mask", model="a-v-white/ruBert-base-finetuned-russian-moshkov-child-corpus-pro")
+    with st.spinner('Загружаю языковую модель'):
+        _pipeline = pipeline(task="fill-mask", model="a-v-white/bert-base-spanish-wwm-cased-finetuned-literature-pro")
+    return _pipeline
 
 
+@st.cache_resource
+def load_summarizer():
+    return Summarizer()
+
+@st.cache_resource
+def load_classifiers(model):
+    if model == 'model1':
+        scaler_path = 'language_data/model1_no_wn_minmaxscaler.pickle'
+        classifier_path = 'language_data/model1_no_wn_catboost_classifier.pickle'
+    elif model == 'model2':
+        scaler_path = 'language_data/model2_no_wn_minmaxscaler.pickle'
+        classifier_path = 'language_data/model2_no_wn_catboost_classifier.pickle'
+    else:
+        scaler_path = 'language_data/model3_no_wn_minmaxscaler.pickle'
+        classifier_path = 'language_data/model3_no_wn_catboost_classifier.pickle'
+    with (open(scaler_path, 'rb') as f1, open(classifier_path, 'rb') as f2, open('language_data/pos_dict.pickle', 'rb') as f3):
+        scaler = load(f1)
+        classifier = load(f2)
+        pos_dict = load(f3)
+    return pos_dict, scaler, classifier
+
 nlp = load_spacy()
 morph = load_morph()
+summarization = load_summarizer()
 w2v_model1_path = r'model1.gz'
 w2v_model2_path = r'model2.gz'
 
@@ -47,24 +78,49 @@ b1_path, b1_target_set = r'language_data/B1_MINIMUM.txt', set()
 b2_path, b2_target_set = r'language_data/B2_MINIMUM.txt', set()
 c1_path, c1_target_set = r'language_data/C1_MINIMUM.txt', set()
 c2_path, c2_target_set = r'language_data/C2_MINIMUM.txt', set()
-minimums_paths = (a1_path, a2_path, b1_path, b2_path)
+
+minimums_paths = (a1_path, a2_path, b1_path, b2_path, c1_path, c2_path)
 minimums_sets = (a1_target_set, a2_target_set, b1_target_set, b2_target_set, c1_target_set, c2_target_set)
 for i in range(len(minimums_paths)):
     with open(minimums_paths[i], 'r', encoding='utf-8') as read_file:
         for line in read_file:
             minimums_sets[i].add(line.strip())
 
-a1_distractor_set = a1_target_set
-a2_distractor_set = a2_target_set.union(a1_target_set)
-b1_distractor_set = b1_target_set.union(a2_target_set)
-b2_distractor_set = b2_target_set.union(b1_target_set)
-c1_distractor_set = c1_target_set.union(b2_target_set)
-c2_distractor_set = c2_target_set.union(c1_target_set)
+MINIMUM_SETS = {
+    'A1': (a1_target_set, a1_target_set),
+    'A2': (a2_target_set, a2_target_set.union(a1_target_set)),
+    'B1': (b1_target_set, b1_target_set.union(a2_target_set)),
+    'B2': (b2_target_set, b2_target_set.union(b1_target_set)),
+    'C1': (c1_target_set, c1_target_set.union(b2_target_set)),
+    'C2': (c2_target_set, c2_target_set.union(c1_target_set)),
+    'Без уровня': (None, None)
+}
+
+LEVEL_NUMBERS = {'A1': 1, 'A2': 2, 'B1': 3, 'B2': 4, 'C1': 5, 'C2': 6}
 
 with open('language_data/phrases.json', 'r', encoding='utf-8') as f:
     PHRASES = set(json.load(f)['PHRASES'])
 
-SIMILARITY_VALUES_w2v = {'A1': 1.0, 'A2': 1.0, 'B1': 1.0, 'B2': 1.0, 'C1': 1.0, 'C2': 1.0, 'Без уровня': 1.0}
-SIMILARITY_VALUES_bert = {'A1': 1.0, 'A2': 1.0, 'B1': 1.0, 'B2': 1.0, 'C1': 1.0, 'C2': 1.0, 'Без уровня': 1.0}
-
 BAD_USER_TARGET_WORDS = []
+
+
+COMBINE_POS = {
+    'simple':
+    {
+        'A1': {'VERB': ['AUX']},
+        'A2': {'VERB': ['AUX']},
+        'B1': {'VERB': ['AUX']},
+        'B2': {'VERB': ['AUX']},
+        'C1': {'VERB': ['AUX']},
+        'C2': {'VERB': ['AUX']},
+    },
+    'phrase': 
+    {
+        'A1': {'VERB': ['AUX']},
+        'A2': {'VERB': ['AUX']},
+        'B1': {'VERB': ['AUX']},
+        'B2': {'VERB': ['AUX']},
+        'C1': {'VERB': ['AUX']},
+        'C2': {'VERB': ['AUX']},
+    },
+}

utilities_language_general/rus_utils.py

@@ -1,10 +1,8 @@
+from random import randint
 from nltk import edit_distance
 from utilities.utils import answer_letter
-from utilities_language_general.rus_constants import nlp
-from utilities_language_general.rus_constants import morph
-from utilities_language_general.rus_constants import stop_list
-from utilities_language_general.rus_constants import SIMILARITY_VALUES_w2v
-from utilities_language_general.rus_constants import SIMILARITY_VALUES_bert
+from utilities_language_general.similarity_measures import make_decision
+from utilities_language_general.rus_constants import nlp, morph, stop_list, COMBINE_POS
 
 
 def prepare_target_words(target_words):
@@ -73,7 +71,7 @@ def get_tags(token: str):
     return set(parts), method
 
 
-def make_inflection(text: str, pos: str or list, tags: set) -> str or None:
+def make_inflection(text: str, pos: str or list, tags: set, level: str) -> str or None:
     if isinstance(pos, list):
         if set(pos).issubset({'NOUN', 'ADJ', 'PROPN'}):
             noun_adjective_phrase_tags = {'nomn', 'gent', 'datv', 'accs', 'ablt', 'loct', 'voct',
@@ -101,6 +99,75 @@ def make_inflection(text: str, pos: str or list, tags: set) -> str or None:
             return None
     else:
         word_form = morph.parse(text)[0].inflect(tags)
+        rnd = randint(0,5)
+        if pos == 'VERB' and 'impf' in tags and level in ('A1', 'A2'):
+            tags.discard('impf')
+            tags.add('perf')
+            word_form = morph.parse(text)[0].inflect(tags)
+            if word_form is not None:
+                return word_form.word
+        elif pos == 'VERB' and 'perf' in tags and level in ('A1', 'A2'):
+            tags.discard('perf')
+            tags.add('impf')
+            word_form = morph.parse(text)[0].inflect(tags)
+            if word_form is not None:
+                return word_form.word
+        if pos == 'NOUN' and level in ('A1', 'A2'):
+            if word_form is None:
+                if 'inan' in tags:
+                    tags.discard('inan')
+                    tags.add('anim')
+                elif 'anim' in tags:
+                    tags.discard('anim')
+                    tags.add('inan')            
+        if pos in ('NOUN', 'ADJ') and level in ('A1, A2') and rnd == 0:
+            if 'sing' in tags:
+                tags.discard('sing')
+                tags.add('plur')
+            elif 'plur' in tags:
+                tags.discard('plur')
+                tags.add('sing')
+        elif pos in ('NOUN', 'ADJ')  and level in ('A1', 'A2') and rnd == 1:
+            if 'masc' in tags:
+                tags.remove('masc')
+                tags.add('femn')
+                word_form = morph.parse(text)[0].inflect(tags)
+                if word_form is not None:
+                    return word_form.word
+                else:
+                    tags.remove('femn')
+                    tags.add('neut')
+                    word_form = morph.parse(text)[0].inflect(tags)
+                    if word_form is not None:
+                        return word_form.word
+            if 'femn' in tags:
+                tags.remove('femn')
+                tags.add('masc')
+                word_form = morph.parse(text)[0].inflect(tags)
+                if word_form is not None:
+                    return word_form.word
+                else:
+                    tags.remove('masc')
+                    tags.add('neut')
+                    word_form = morph.parse(text)[0].inflect(tags)
+                    if word_form is not None:
+                        return word_form.word
+            if 'neut' in tags:
+                tags.remove('neut')
+                tags.add('masc')
+                word_form = morph.parse(text)[0].inflect(tags)
+                if word_form is not None:
+                    return word_form.word
+                else:
+                    tags.remove('masc')
+                    tags.add('femn')
+                    word_form = morph.parse(text)[0].inflect(tags)
+                    if word_form is not None:
+                        return word_form.word
+        else:
+            word_form = morph.parse(text)[0].inflect(tags)
+            return word_form.word if word_form is not None else None
+        word_form = morph.parse(text)[0].inflect(tags)
         return word_form.word if word_form is not None else None
 
 
@@ -184,8 +251,8 @@ def check_token_bert(token, current_minimum: set = None, stop_words=stop_list,
         return False
 
 
-def get_distractors_from_model(model, lemma: str, pos: str, gender: str or None, global_distractors: set,
-                               distractor_minimum: set, level_name: str, max_num_distractors: int,
+def get_distractors_from_model(doc, model, scaler, classifier, pos_dict:dict, target_text:str, lemma: str, pos: str, gender: str, 
+                               lemma_index:int, global_distractors: set, distractor_minimum: set, level_name: str, max_num_distractors: int, 
                                max_length_ratio=5, min_edit_distance_ratio=0.5):
     distractors = []
     query = lemma if '_' in lemma else f'{lemma}_{pos}'
@@ -205,12 +272,16 @@ def get_distractors_from_model(model, lemma: str, pos: str, gender: str or None,
             distractor_similarity = candidate[1]
             candidate_gender = define_gender(distractor_lemma)
             length_ratio = abs(len(lemma) - len(distractor_lemma))
+            decision = make_decision(doc, model_type='w2v', model=model, scaler=scaler, classifier=classifier, pos_dict=pos_dict, 
+                                     level=level_name, target_lemma=query, target_text=target_text, target_pos=pos, target_position=lemma_index, 
+                                     substitute_lemma=distractor_lemma, substitute_pos=distractor_pos)
             condition = ((distractor_pos == pos
-                          or (distractor_pos in ('VERB', 'ADJ', 'phrase') and pos in ('VERB', 'ADJ', 'phrase')))
+                          or (COMBINE_POS['simple'][level_name].get(pos) is not None and COMBINE_POS['simple'][level_name].get(distractor_pos) is not None
+                              and distractor_pos in COMBINE_POS['simple'][level_name][pos] and pos in COMBINE_POS['simple'][level_name][distractor_pos]))
+                         and decision
                          and distractor_lemma != lemma
                          and len(distractors) < 100
-                         and distractor_similarity < SIMILARITY_VALUES_w2v[level_name]
-                         and candidate_gender == gender
+                         and (candidate_gender == gender and level_name in ('B1', 'B2', 'C1', 'C2'))
                          and length_ratio <= max_length_ratio
                          and distractor_lemma not in global_distractors
                          and edit_distance(lemma, distractor_lemma) / ((len(lemma) + len(distractor_lemma)) / 2) >
@@ -228,16 +299,22 @@ def get_distractors_from_model(model, lemma: str, pos: str, gender: str or None,
                 continue
             d1_lemma, d1_pos, d2_lemma, d2_pos = candidate[0].split('_')
             distractor_lemma = f'{d1_lemma}_{d2_lemma}'
+            d_pos = f'{d1_pos}_{d2_pos}'
             distractor_similarity = candidate[1]
+            decision = make_decision(doc, model_type='w2v', model=model, scaler=scaler, classifier=classifier, pos_dict=pos_dict, 
+                                     level=level_name, target_lemma=query, target_text=target_text, target_pos=pos, target_position=lemma_index, 
+                                     substitute_lemma=candidate[0], substitute_pos=d_pos)
             condition = (((d1_pos == pos or d2_pos == pos)
+                          or (COMBINE_POS['simple'][level_name].get(pos) is not None and COMBINE_POS['simple'][level_name].get(distractor_pos) is not None
+                              and distractor_pos in COMBINE_POS['simple'][level_name][pos] and pos in COMBINE_POS['simple'][level_name][distractor_pos])
                           or (d1_pos in ('VERB', 'AUX', 'SCONJ', 'ADP')
                               and pos in ('phrase', 'VERB', 'AUX', 'SCONJ', 'ADP'))
                           or (d2_pos in ('VERB', 'AUX', 'SCONJ', 'ADP')
                               and pos in ('phrase', 'VERB', 'AUX', 'SCONJ', 'ADP')))
+                         and decision
                          and candidate[0] != lemma
                          and distractor_lemma != lemma
                          and len(distractors) < 100
-                         and distractor_similarity < SIMILARITY_VALUES_w2v[level_name]
                          and distractor_lemma not in global_distractors)
             if condition:
                 if distractor_minimum is not None:
@@ -255,8 +332,8 @@ def get_distractors_from_model(model, lemma: str, pos: str, gender: str or None,
         return None
 
 
-def get_distractors_from_model_bert(model, text_with_masked_task: str, lemma: str, pos: str, gender: str or None,
-                                    global_distractors: set, distractor_minimum: set, level_name: str,
+def get_distractors_from_model_bert(model, scaler, classifier, pos_dict:dict, level_name: str, lemma: str, pos: str, gender: str, 
+                                    text_with_masked_task: str, global_distractors: set, distractor_minimum: set, 
                                     max_num_distractors: int, max_length_ratio=5, min_edit_distance_ratio=0.5):
     _distractors = []
     try:
@@ -283,12 +360,16 @@ def get_distractors_from_model_bert(model, text_with_masked_task: str, lemma: st
         distractor_similarity = candidate_distractor[1]
         candidate_gender = define_gender(distractor_lemma)
         length_ratio = abs(len(lemma) - len(distractor_lemma))
+        decision = make_decision(doc=None, model_type='bert', scaler=scaler, classifier=classifier, pos_dict=pos_dict, level=level_name, 
+                                 target_lemma=lemma, target_text=None, target_pos=pos, target_position=None, 
+                                 substitute_lemma=distractor_lemma, substitute_pos=distractor_pos, bert_score=distractor_similarity)
         if (((distractor_pos == pos)
-             or (pos in ('VERB', 'ADJ', 'phrase') and distractor_pos in ('VERB', 'ADJ', 'phrase')))
+             or (COMBINE_POS['phrase'][level_name].get(pos) is not None and COMBINE_POS['phrase'][level_name].get(distractor_pos) is not None
+                and distractor_pos in COMBINE_POS['phrase'][level_name][pos] and pos in COMBINE_POS['phrase'][level_name][distractor_pos]))
+                and decision
                 and distractor_lemma != lemma
                 and (len(_distractors) < max_num_distractors + 10)
-                and (distractor_similarity < SIMILARITY_VALUES_bert[level_name])
-                and (candidate_gender == gender)
+                and (candidate_gender == gender and level_name in ('B1', 'B2', 'C1', 'C2'))
                 and (length_ratio <= max_length_ratio)  # May be changed if case of phrases
                 and (distractor_lemma not in global_distractors)
                 and (edit_distance(lemma, distractor_lemma)  # May be changed if case of phrases

utilities_language_general/similarity_measures.py

@@ -0,0 +1,255 @@
+import numpy as np
+from math import pow
+from nltk.corpus import wordnet as wn
+from utilities_language_general.rus_constants import nlp, PHRASES, LEVEL_NUMBERS
+
+def eucledian_distance(x, y):   
+    return np.sqrt(np.sum((x - y) ** 2))
+
+def cosine_similarity(x, y):
+    out = np.dot(x, y) / (np.sqrt(np.dot(x, x)) * np.sqrt(np.dot(y, y)))
+    if str(out) != 'nan':
+        return out
+    return None
+
+def get_vector_for_token(model, token):
+    vector = None
+    
+    splitted = token.split('_')
+    token_list = [f'{splitted[i]}_{splitted[i+1]}' for i in range(len(splitted)-1)]
+    
+    if model.has_index_for(token):
+        vector = model.get_vector(token)
+    else:
+        try:
+            vector = model.get_mean_vector(token_list)
+        except ValueError:
+            return None
+    return vector
+
+def compute_metric(func, vector1, vector2):
+    if vector1 is not None and vector2 is not None:
+        return func(vector1, vector2)
+    else:
+        return None
+
+def compute_positive_cos(x, y):
+    cos_sim = cosine_similarity(x, y)
+    if cos_sim:
+        return (cos_sim + 1) / 2
+    else:
+        return None
+
+def addition_metric(substitute, target, context):
+    substitute_target_cos = compute_metric(cosine_similarity, substitute, target)
+    if not substitute_target_cos:
+        return None
+    if not context:
+        return None
+
+    context_vectors = []
+    for context_tk in context:
+        substitute_context_cos = compute_metric(cosine_similarity, substitute, context_tk)
+        if substitute_context_cos:
+            context_vectors.append(substitute_context_cos)
+    sum_of_context_vectors = np.sum(context_vectors)
+    
+    metric = (substitute_target_cos + sum_of_context_vectors) / (len(context) + 1)
+    return metric
+
+def balanced_addition_metric(substitute, target, context):
+    substitute_target_cos = compute_metric(cosine_similarity, substitute, target)
+    if not substitute_target_cos:
+        return None
+    if not context:
+        return None
+
+    context_vectors = []
+    for context_tk in context:
+        substitute_context_cos = compute_metric(cosine_similarity, substitute, context_tk)
+        if substitute_context_cos:
+            context_vectors.append(substitute_context_cos)
+    sum_of_context_vectors = np.sum(context_vectors)
+
+    context_len = len(context)
+    metric = (context_len * substitute_target_cos + sum_of_context_vectors) / (2 * context_len)
+    return metric
+
+def multiplication_metric(substitute, target, context):
+    substitute_target_cos = compute_metric(compute_positive_cos, substitute, target)
+    if not substitute_target_cos:
+        return None
+    if not context:
+        return None
+        
+    context_vectors = []
+    for context_tk in context:
+        substitute_context_positive_cos = compute_metric(compute_positive_cos, substitute, context_tk)
+        if substitute_context_positive_cos:
+            context_vectors.append(substitute_context_positive_cos)
+    prod_of_context_vectors = np.prod(context_vectors)
+    try:
+        metric = pow((substitute_target_cos + prod_of_context_vectors), 1 / (len(context) + 1))
+    except ValueError:
+        return None
+    return metric
+
+def balanced_multiplication_metric(substitute, target, context):
+    substitute_target_cos = compute_metric(compute_positive_cos, substitute, target)
+    if not substitute_target_cos:
+        return None
+    if not context:
+        return None
+        
+    context_vectors = []
+    for context_tk in context:
+        substitute_context_positive_cos = compute_metric(compute_positive_cos, substitute, context_tk)
+        if substitute_context_positive_cos:
+            context_vectors.append(substitute_context_positive_cos)
+    prod_of_context_vectors = np.prod(context_vectors)
+    
+    context_len = len(context)
+    try:
+        metric = pow((pow(substitute_target_cos, context_len) + prod_of_context_vectors), 1 / (2 * context_len))
+    except ValueError:
+        return None
+    return metric
+
+def bind_phrases(context_list):
+    context = []
+    previous_was_phrase = False
+    for i in range(len(context_list)-1):
+        phrase_candidate = f'{context_list[i]}_{context_list[i+1]}'
+        if phrase_candidate in PHRASES and not previous_was_phrase:
+            context.append(phrase_candidate)
+            previous_was_phrase = True
+        else:
+            if not previous_was_phrase:
+                context.append(context_list[i])
+            previous_was_phrase = False
+    if context_list:
+        if not context:
+            context.append(context_list[-1])
+        elif not context_list[-1] in context[-1]:
+            context.append(context_list[-1])
+    return context
+
+def get_context_windows(doc, target_text, window_size):
+    sentence_str = doc.text
+    sentence_masked = sentence_str.lower().replace(target_text.lower().strip(), ' [MASK] ')
+    alpha_tokens_lemma_pos = [f'{tk.lemma_.lower()}_{tk.pos_}' for tk in nlp(sentence_masked) if tk.text.isalpha()]
+    alpha_tokens_lemma_pos_no_stop = [f'{tk.lemma_.lower()}_{tk.pos_}' for tk in nlp(sentence_masked) if tk.text.isalpha() and not tk.is_stop]
+    try:
+        mask_token_index = alpha_tokens_lemma_pos.index('mask_PROPN')
+        mask_token_index_no_stop = alpha_tokens_lemma_pos_no_stop.index('mask_PROPN')
+    except ValueError:
+        return None
+
+    left_border = max(mask_token_index-window_size, 0)
+    right_border = min(mask_token_index+window_size, len(alpha_tokens_lemma_pos))
+    l_context = alpha_tokens_lemma_pos[left_border:mask_token_index]
+    r_context = alpha_tokens_lemma_pos[mask_token_index+1:right_border+1]
+
+    left_border_no_stop = max(mask_token_index_no_stop-window_size, 0)
+    right_border_no_stop = min(mask_token_index_no_stop+window_size, len(alpha_tokens_lemma_pos_no_stop))
+    l_context_no_stop = alpha_tokens_lemma_pos_no_stop[left_border_no_stop:mask_token_index_no_stop]
+    r_context_no_stop = alpha_tokens_lemma_pos_no_stop[mask_token_index_no_stop+1:right_border_no_stop+1]
+    return (bind_phrases(l_context) + bind_phrases(r_context), bind_phrases(l_context_no_stop) + bind_phrases(r_context_no_stop))
+
+def get_context_linked_words(doc, target_position, target_text):
+    answer_list = target_text.split(' ')
+    context_words = []
+    for tk in doc:
+        if tk.text.isalpha():
+            if (tk.text in answer_list and abs(target_position - tk.idx) <= sum([len(t) for t in answer_list])):
+                context_words.extend([t for t in tk.subtree if t.text.isalpha() and not t.is_stop])
+                context_words.extend([t for t in tk.children if t.text.isalpha() and not t.is_stop])
+                context_words.extend([t for t in tk.ancestors if t.text.isalpha() and not t.is_stop])
+    context_words = [(tk, f'{tk.lemma_}_{tk.pos_}') for tk in sorted(set(context_words), key=lambda tk: tk.i) if tk.text not in answer_list]
+    context = []
+    previous_was_phrase = False
+    for i in range(len(context_words)-1):
+        phrase_candidate = f'{context_words[i][1]}_{context_words[i+1][1]}'
+        if phrase_candidate in PHRASES and not previous_was_phrase and abs(context_words[i][0].i - context_words[i+1][0].i) <=1:
+            context.append(phrase_candidate)
+            previous_was_phrase = True
+        else:
+            if not previous_was_phrase:
+                context.append(context_words[i][1])
+    if context and context_words:
+        if not context_words[-1][1] in context[-1]:
+            context.append(context_words[-1][1])
+    elif context_words:
+        context.append(context_words[-1][1])
+    return context
+ 
+
+def compute_all_necessary_metrics(target_lemma, target_text, target_position, substitute_lemma, doc, model_type:str, model=None):
+
+    target_vector = get_vector_for_token(model, target_lemma)
+    substitute_vector = get_vector_for_token(model, substitute_lemma)
+
+    cosimilarity = compute_metric(cosine_similarity, substitute_vector, target_vector)
+    eucledian_similarity = compute_metric(eucledian_distance, substitute_vector, target_vector)
+
+    context_window3, context_window3_no_stop = get_context_windows(doc=doc, target_text=target_text, window_size=3)
+    context_window5, context_window5_no_stop = get_context_windows(doc=doc, target_text=target_text, window_size=5)
+    context_window_synt = get_context_linked_words(doc, target_position, target_text)
+
+    context_window3 = [get_vector_for_token(model, token) for token in context_window3]
+    context_window3_no_stop = [get_vector_for_token(model, token) for token in context_window3_no_stop]
+    context_window5 = [get_vector_for_token(model, token) for token in context_window5]
+    context_window5_no_stop = [get_vector_for_token(model, token) for token in context_window5_no_stop]
+    context_window_synt = [get_vector_for_token(model, token) for token in context_window_synt]
+
+    add_metric_window3 = addition_metric(target_vector, substitute_vector, context_window3)
+    bal_add_metric_window3 = balanced_addition_metric(target_vector, substitute_vector, context_window3)
+    add_metric_window3_no_stop = addition_metric(target_vector, substitute_vector, context_window3_no_stop)
+    bal_add_metric_window3_no_stop = balanced_addition_metric(target_vector, substitute_vector, context_window3_no_stop)
+
+    mult_metric_window3 = multiplication_metric(target_vector, substitute_vector, context_window3)
+    bal_mult_metric_window3 = balanced_multiplication_metric(target_vector, substitute_vector, context_window3)
+    mult_metric_window3_no_stop = multiplication_metric(target_vector, substitute_vector, context_window3_no_stop)
+    bal_mult_metric_window3_no_stop = balanced_multiplication_metric(target_vector, substitute_vector, context_window3_no_stop)
+
+    add_metric_window5 = addition_metric(target_vector, substitute_vector, context_window5)
+    bal_add_metric_window5 = balanced_addition_metric(target_vector, substitute_vector, context_window5)
+    add_metric_window5_no_stop = addition_metric(target_vector, substitute_vector, context_window5_no_stop)
+    bal_add_metric_window5_no_stop = balanced_addition_metric(target_vector, substitute_vector, context_window5_no_stop)
+
+    mult_metric_window5 = multiplication_metric(target_vector, substitute_vector, context_window5)
+    bal_mult_metric_window5 = balanced_multiplication_metric(target_vector, substitute_vector, context_window5)
+    mult_metric_window5_no_stop = multiplication_metric(target_vector, substitute_vector, context_window5_no_stop)
+    bal_mult_metric_window5_no_stop = balanced_multiplication_metric(target_vector, substitute_vector, context_window5_no_stop)
+
+    add_metric_synt = addition_metric(target_vector, substitute_vector, context_window_synt)
+    bal_add_metric_synt = balanced_addition_metric(target_vector, substitute_vector, context_window_synt)
+
+    mult_metric_synt = multiplication_metric(target_vector, substitute_vector, context_window_synt)
+    bal_mult_metric_synt = balanced_multiplication_metric(target_vector, substitute_vector, context_window_synt)
+
+    return (cosimilarity, eucledian_similarity, 
+            add_metric_window3, bal_add_metric_window3, 
+            mult_metric_window3, bal_mult_metric_window3, 
+            add_metric_window3_no_stop, bal_add_metric_window3_no_stop,
+            mult_metric_window3_no_stop, bal_mult_metric_window3_no_stop,
+            add_metric_window5, bal_add_metric_window5, 
+            mult_metric_window5, bal_mult_metric_window5, 
+            add_metric_window5_no_stop, bal_add_metric_window5_no_stop,         
+            mult_metric_window5_no_stop, bal_mult_metric_window5_no_stop,
+            add_metric_synt, bal_add_metric_synt, 
+            mult_metric_synt, bal_mult_metric_synt)
+
+def make_decision(doc, model_type, scaler, classifier, pos_dict, level, target_lemma, target_text, target_pos, target_position, 
+                  substitute_lemma, substitute_pos, model=None, bert_score=None):
+    # return True
+    metrics = compute_all_necessary_metrics(target_lemma=target_lemma, target_text=target_text, target_position=target_position, 
+                                            substitute_lemma=substitute_lemma, doc=doc, model_type=model_type, model=model)
+    target_multiword, substitute_multiword = target_lemma.count('_') > 2, substitute_lemma.count('_') > 2
+    data = [LEVEL_NUMBERS.get(level), pos_dict.get(target_pos), target_multiword, pos_dict.get(substitute_pos), substitute_multiword] + scaler.transform([metrics]).tolist()[0]
+    if model_type == 'bert':
+        data = [LEVEL_NUMBERS.get(level), pos_dict.get(target_pos), target_multiword, pos_dict.get(substitute_pos), substitute_multiword, bert_score]
+    predict = classifier.predict(data)
+    return bool(predict)
+
+

utilities_language_w2v/rus_main_workflow_w2v.py

@@ -1,31 +1,20 @@
 import datetime
 from io import StringIO
+from typing import Union
 from random import sample
 from collections import defaultdict
-from streamlit import progress as st_progress
-from streamlit.elements import WIDGETS as ST_WIDGETS
-from utilities_language_general.rus_constants import st
-from utilities_language_w2v.rus_sentence_w2v import TASK
-from utilities_language_general.rus_constants import load_w2v
-from utilities_language_general.rus_utils import prepare_tasks
-from utilities_language_w2v.rus_sentence_w2v import SENTENCE
-import utilities_language_general.rus_constants as esp_constants
-from utilities_language_general.rus_utils import prepare_target_words
-from utilities_language_general.rus_constants import w2v_model1_path
-from utilities_language_general.rus_constants import w2v_model2_path
-from utilities_language_general.rus_utils import compute_frequency_dict
 from streamlit.runtime.uploaded_file_manager import UploadedFile
-from utilities_language_general.rus_constants import BAD_USER_TARGET_WORDS
+from utilities_language_w2v.rus_sentence_w2v import SENTENCE, TASK
+from utilities_language_general.rus_utils import compute_frequency_dict, prepare_target_words, prepare_tasks
+from utilities_language_general.rus_constants import st, load_w2v, load_classifiers, nlp, summarization, BAD_USER_TARGET_WORDS, MINIMUM_SETS
 
 
-def main_workflow_w2v(
-        file: UploadedFile or None,
+def main_workflow(
+        file: Union[UploadedFile, None],
         text: str,
-        logs: ST_WIDGETS,
-        logs_d: ST_WIDGETS,
-        progress: st_progress,
-        progress_d: st_progress,
-        progress_s: st_progress,
+        logs,
+        progress,
+        progress_d,
         level: str,
         tw_mode_automatic_mode: str,
         target_words: str,
@@ -56,19 +45,13 @@ def main_workflow_w2v(
              student_out, teacher_out, total_out, original_text
     """
     # Clear bad target_words each time
-    global_bad_target_words = []
+    if global_bad_target_words:
+        global_bad_target_words = []
 
     # Define main global variables
-    logs.write()
     GLOBAL_DISTRACTORS = set()
     MAX_FREQUENCY = 0
 
-    # Define which model is used for distractor generation
-    if model_name == 'Модель-1':
-        mask_filler = load_w2v(w2v_model1_path)
-    else:
-        mask_filler = load_w2v(w2v_model2_path)
-
     # Get input text
     if file is not None:
         stringio = StringIO(file.getvalue().decode("utf-8"))
@@ -76,15 +59,15 @@ def main_workflow_w2v(
     elif text != '':
         current_text = text
     else:
-        esp_constants.st.warning('Вы и текст не вставили, и файл не выбрали 😢')
+        st.warning('Вы и текст не вставили, и файл не выбрали 😢')
         current_text = ''
-        esp_constants.st.stop()
+        st.stop()
 
     # Process target words
     if tw_mode_automatic_mode == 'Самостоятельно':
         if target_words == '':
-            esp_constants.st.warning('Вы не ввели целевые слова')
-            esp_constants.st.stop()
+            st.warning('Вы не ввели целевые слова')
+            st.stop()
         # Cannot make up paradigm, so only USER_TARGET_WORDS is used
         USER_TARGET_WORDS = prepare_target_words(target_words)
         tw_mode_automatic_mode = False
@@ -97,8 +80,8 @@ def main_workflow_w2v(
     current_text = current_text.replace('.', '. ').replace('. . .', '...').replace('  ', ' ').replace('…', '...') \
         .replace('…', '...').replace('—', '-').replace('\u2014', '-').replace('—', '-').replace('-\n', '') \
         .replace('\n', '%^&*')
-    current_text_sentences = [sent.text.strip() for sent in esp_constants.nlp(current_text).sents]
-    logs.success('Получили Ваш текст!')
+    current_text_sentences = [sent.text.strip() for sent in nlp(current_text).sents]
+    logs.update(label='Получили Ваш текст!', state='running')
     progress.progress(10)
 
     # Compute frequency dict
@@ -110,49 +93,55 @@ def main_workflow_w2v(
         if j < len(FREQ_DICT) * _frequency_barrier_percent:
             MAX_FREQUENCY = tp[1]
     MAX_FREQUENCY = 3 if MAX_FREQUENCY < 3 else MAX_FREQUENCY
-    logs.success("Посчитали немного статистики!")
+    logs.update(label="Посчитали немного статистики!", state='running')
     progress.progress(15)
 
     # Choose necessary language minimum according to user's input
-    if level == 'A1':
-        target_minimum = esp_constants.a1_target_set
-        distractor_minimum = esp_constants.a1_distractor_set
-    elif level == 'A2':
-        target_minimum = esp_constants.a2_target_set
-        distractor_minimum = esp_constants.a2_distractor_set
-    elif level == 'B1':
-        target_minimum = esp_constants.b1_target_set
-        distractor_minimum = esp_constants.b1_distractor_set
-    elif level == 'B2':
-        target_minimum = esp_constants.b2_target_set
-        distractor_minimum = esp_constants.b2_distractor_set
-    elif level == 'C1':
-        target_minimum = esp_constants.c1_target_set
-        distractor_minimum = esp_constants.c1_distractor_set
-    elif level == 'C2':
-        target_minimum = esp_constants.c2_target_set
-        distractor_minimum = esp_constants.c2_distractor_set
-    elif level == 'Без уровня':
-        target_minimum = None
-        distractor_minimum = None
+    if level:
+        target_minimum, distractor_minimum = MINIMUM_SETS[level]
     else:
         target_minimum = None
         distractor_minimum = None
         logs.error('Вы не выбрали языковой уровень!')
         st.stop()
 
+    # Define which model is used for distractor generation
+    logs.update(label='Загружаем языковые модели и другие данные', state='running')
+    if model_name == 'Модель-1':
+        mask_filler = load_w2v('model1')
+        pos_dict, scaler, classifier = load_classifiers('model1')
+    else:
+        mask_filler = load_w2v('model2')
+        pos_dict, scaler, classifier = load_classifiers('model1')
+
     # Start generation process
     workflow = [SENTENCE(original=sent.strip(), n_sentence=num, max_num_distractors=num_distractors)
                 for num, sent in enumerate(current_text_sentences)]
-    logs.success("Запускаем процесс генерации заданий!")
+    logs.update(label="Запускаем процесс генерации заданий!", state='running')
     progress.progress(20)
 
+    # Define summary length
+    text_length = len(current_text_sentences)
+    if text_length <= 15:
+        summary_length = text_length
+    elif text_length <= 25:
+        summary_length = 15
+    else:
+        n = (text_length - 20) // 5
+        summary_length = 15 + 2 * n
+    round_summary_length = summary_length - (summary_length % - 10)
+
+    # Get summary. May choose between round_summary_length and summary_length
+    SUMMARY = summarization(current_text, num_sentences=round_summary_length)
+    logs.success('Нашли интересные предложения. Пригодятся!')
+    progress.progress(25)
+
     for sentence in workflow:
         sentence.lemmatize_sentence()
 
     for sentence in workflow:
         sentence.bind_phrases()
-    logs.success("Подготовили предложения для дальнейшей работы!")
+    logs.update(label="Подготовили предложения для дальнейшей работы!", state='running')
     progress.progress(30)
 
     for j, sentence in enumerate(workflow):
@@ -160,9 +149,10 @@ def main_workflow_w2v(
                                      target_words_automatic_mode=tw_mode_automatic_mode,
                                      target_minimum=target_minimum,
                                      user_target_words=USER_TARGET_WORDS,
-                                     frequency_dict=FREQ_DICT)
+                                     frequency_dict=FREQ_DICT,
+                                     summary=SUMMARY)
         progress.progress(int(30 + (j * (30 / len(workflow)))))
-    progress_s.progress(60)
+    progress.progress(60)
     DUPLICATE_TARGET_WORDS = defaultdict(list)
     for sentence in workflow:
         for target_word in sentence.target_words:
@@ -175,31 +165,33 @@ def main_workflow_w2v(
             if target_word not in RESULT_TW:
                 global_bad_target_words.append(target_word['original_text'])
                 sentence.target_words.remove(target_word)
-    progress_s.progress(65)
-    logs.success('Выбрали слова-пропуски!')
+    progress.progress(65)
+    logs.update(label='Выбрали слова-пропуски!', state='running')
 
     for sentence in workflow:
         sentence.attach_distractors_to_target_word(model=mask_filler,
+                                                   scaler=scaler,
+                                                   classifier=classifier,
+                                                   pos_dict=pos_dict,
                                                    global_distractors=GLOBAL_DISTRACTORS,
                                                    distractor_minimum=distractor_minimum,
                                                    level_name=level,
                                                    max_frequency=MAX_FREQUENCY,
-                                                   progress=progress_d,
-                                                   logs=logs_d)
-    progress_s.progress(70)
-    logs.success('Подобрали неправильные варианты!')
+                                                   logs=logs, progress=progress_d)
+    progress.progress(70)
+    logs.update(label='Подобрали неправильные варианты!', state='running')
     for sentence in workflow:
-        sentence.inflect_distractors()
-    progress_s.progress(80)
-    logs.success('Просклоняли и проспрягали неправильн��е варианты!')
+        sentence.inflect_distractors(level_name=level)
+    progress.progress(80)
+    logs.update(label='Просклоняли и проспрягали неправильные варианты!', state='running')
 
     for sentence in workflow:
         sentence.filter_target_words(target_words_automatic_mode=tw_mode_automatic_mode)
 
     for sentence in workflow:
         sentence.sample_distractors(num_distractors=num_distractors)
-    progress_s.progress(90)
-    logs.success('Отобрали лучшие задания!')
+    progress.progress(90)
+    logs.update(label='Отобрали лучшие задания!', state='running')
 
     RESULT_TASKS = []
     for sentence in workflow:
@@ -219,7 +211,12 @@ def main_workflow_w2v(
                 NUMBER_TASKS = 10
             else:
                 NUMBER_TASKS = len(RESULT_TASKS)
-    RESULT_TASKS = sample(RESULT_TASKS, NUMBER_TASKS)
+    RESULT_TASKS_in_summary = list(filter(lambda task: task.in_summary, RESULT_TASKS))
+    RESULT_TASTS_not_in_summary = list(filter(lambda task: not task.in_summary, RESULT_TASKS))
+    if len(RESULT_TASKS_in_summary) >= NUMBER_TASKS:
+        RESULT_TASKS = RESULT_TASKS_in_summary
+    else:
+        RESULT_TASKS = RESULT_TASKS_in_summary + sample(RESULT_TASTS_not_in_summary, NUMBER_TASKS - len(RESULT_TASKS_in_summary))
     RESULT_TASKS = sorted(RESULT_TASKS, key=lambda t: (t.sentence_number, t.position_in_sentence))
 
     for task in RESULT_TASKS:
@@ -243,8 +240,8 @@ def main_workflow_w2v(
                   f'{PREPARED_TASKS["KEYS_ONLY"]}'
     TOTAL_OUT = f'{original_text}\n\n{"$" * 70}\n\n{STUDENT_OUT}\n\n{"=" * 70}\n\n{PREPARED_TASKS["TASKS_TEACHER"]}' \
                 f'\n\n{"$" * 70}\n\n{PREPARED_TASKS["KEYS_ONLY"]}'
-    logs.success('Сейчас все будет готово!')
-    progress_s.progress(90)
+    logs.update(label='Сейчас все будет готово!', state='running')
+    progress.progress(90)
     save_name = save_name if save_name != '' else f'{str(datetime.datetime.now())[:-7]}_{original_text[:20]}'
     out = {
         'name': save_name,

utilities_language_w2v/rus_sentence_w2v.py

@@ -1,16 +1,7 @@
-import copy
 import string
-from random import random
-from random import sample
-from utilities_language_general.rus_constants import nlp
-from utilities_language_general.rus_utils import get_tags
-from utilities_language_general.rus_utils import check_token
-from utilities_language_general.rus_constants import PHRASES
-from utilities_language_general.rus_utils import define_gender
-from utilities_language_general.rus_utils import convert_gender
-from utilities_language_general.rus_utils import make_inflection
-from utilities_language_general.rus_constants import BAD_USER_TARGET_WORDS
-from utilities_language_general.rus_utils import get_distractors_from_model
+from random import random, sample
+from utilities_language_general.rus_constants import nlp, PHRASES, BAD_USER_TARGET_WORDS
+from utilities_language_general.rus_utils import get_tags, check_token, define_gender, convert_gender, make_inflection, get_distractors_from_model
 
 
 class SENTENCE:
@@ -48,7 +39,7 @@ class SENTENCE:
                     self.sentence_phrases.append(self.sentence_lemma_pos[i][1])
                 previous_was_phrase = False
 
-    def search_target_words_automatically(self, model, target_minimum: set, frequency_dict: dict = None):
+    def search_target_words_automatically(self, model, target_minimum: set, frequency_dict: dict = None, summary:list=None):
         for token in self.sentence_phrases:
             # TODO: Still do not have w2v model with phrases
             #  therefore cannot come up with the criteria
@@ -69,7 +60,8 @@ class SENTENCE:
                     'tags': tags,
                     'position_in_sentence': self.original.find(original_token1.text),
                     'not_named_entity': not_ner,
-                    'frequency_in_text': 0
+                    'frequency_in_text': 0,
+                    'in_summary': self.original in summary
                 }
                 self.target_words.append(target_word)
             else:  # if token is just a spacy.nlp token
@@ -86,10 +78,11 @@ class SENTENCE:
                         'position_in_sentence': self.original.find(token.text),
                         'not_named_entity': True if token.ent_type == 0 else False,
                         'frequency_in_text': frequency_dict.get(token.lemma_, 1),
+                        'in_summary': self.original in summary
                     }
                     self.target_words.append(target_word)
 
-    def search_user_target_words(self, model, user_target_words: set = None, frequency_dict: dict = None):
+    def search_user_target_words(self, model, user_target_words: set = None, frequency_dict: dict = None, summary:list=None):
         for _utw in user_target_words:
             if _utw in self.original:
                 parse_utw = nlp(_utw)
@@ -114,7 +107,8 @@ class SENTENCE:
                     'tags': user_target_word_tags,
                     'position_in_sentence': self.original.find(_utw),
                     'not_named_entity': not_ner,
-                    'frequency_in_text': frequency_dict.get(user_target_word_lemma, 1)
+                    'frequency_in_text': frequency_dict.get(user_target_word_lemma, 1),
+                    'in_summary': self.original in summary
                 }
                 if not (model.has_index_for(user_target_word_lemma)
                         or model.has_index_for(f'{user_target_word_lemma}_{user_target_word_pos[1]}')):
@@ -124,25 +118,24 @@ class SENTENCE:
 
     def search_target_words(self, model, target_words_automatic_mode: bool, target_minimum,
                             user_target_words: set = None,
-                            frequency_dict: dict = None):
+                            frequency_dict: dict = None, summary: list=None):
         if target_words_automatic_mode:
             self.search_target_words_automatically(model=model, target_minimum=target_minimum,
-                                                   frequency_dict=frequency_dict)
+                                                   frequency_dict=frequency_dict, summary=summary)
         else:
             self.search_user_target_words(model=model, user_target_words=user_target_words,
-                                          frequency_dict=frequency_dict)
+                                          frequency_dict=frequency_dict, summary=summary)
 
-    def attach_distractors_to_target_word(self, model, global_distractors, distractor_minimum, level_name,
-                                          max_frequency,
-                                          progress, logs):
+    def attach_distractors_to_target_word(self, model, scaler, classifier, pos_dict, global_distractors, 
+                                          distractor_minimum, level_name, max_frequency, logs, progress):
         n_target_words = len(self.target_words)
         bad_target_words = []
         for i, target_word in enumerate(self.target_words):
             pos = target_word['pos'][0] if target_word['pos'][0] == 'phrase' else target_word['pos'][1]
-            distractors = get_distractors_from_model(model, lemma=target_word['lemma'], pos=pos,
-                                                     gender=target_word['gender'], level_name=level_name,
-                                                     global_distractors=global_distractors,
-                                                     distractor_minimum=distractor_minimum,
+            distractors = get_distractors_from_model(doc=self.parsed, model=model, scaler=scaler, classifier=classifier, pos_dict=pos_dict,
+                                                     target_text=target_word['original_text'], lemma=target_word['lemma'],
+                                                     pos=pos, gender=target_word['gender'], lemma_index=target_word['position_in_sentence'],
+                                                     global_distractors=global_distractors, distractor_minimum=distractor_minimum, level_name=level_name, 
                                                      max_num_distractors=self.max_num_distractors)
             if distractors is None or target_word['frequency_in_text'] > max_frequency:
                 target_word['distractors'] = distractors
@@ -150,39 +143,29 @@ class SENTENCE:
             target_word['distractors'] = distractors
             target_word['distractors_number'] = len(distractors) if distractors is not None else 0
             progress.progress(i / n_target_words)
-            logs.success(f'Обработали {i}/{n_target_words} слов в {self.n_sentence + 1}-м предложении')
+            logs.update(label=f'Обработали {i}/{n_target_words} слов в {self.n_sentence + 1}-м предложении',
+                        state='running')
         for btw in bad_target_words:
             BAD_USER_TARGET_WORDS.append(btw['original_text'])
             self.target_words.remove(btw)
         progress.progress(100)
-        logs.success(
-            f'Обработали {n_target_words}/{n_target_words} слов в {self.n_sentence + 1}-м предложении')
+        logs.update(label=f'Обработали {n_target_words}/{n_target_words} слов в {self.n_sentence + 1}-м предложении',
+                    state='running')
 
-    def inflect_distractors(self):
+    def inflect_distractors(self, level_name):
         bad_target_words = []
         for target_word in self.target_words:
             inflected_distractors = []
             for distractor_lemma, distractor_similarity in target_word['distractors']:
                 if distractor_lemma.count('_') > 1:
                     # TODO The same. Has to train model and test this code
-                    inflected = make_inflection(text=distractor_lemma,
+                    inflected = make_inflection(text=distractor_lemma, level=level_name,
                                                 pos=target_word['pos'][1], tags=target_word['tags'])
                 else:
-                    inflected = make_inflection(text=distractor_lemma,
+                    inflected = make_inflection(text=distractor_lemma, level=level_name,
                                                 pos=target_word['pos'][1], tags=target_word['tags'])
                 if inflected is not None:
                     inflected_distractors.append(inflected)
-                else:
-                    new_tags = copy.deepcopy(target_word['tags'])
-                    if 'NOUN' in target_word['tags'] and 'inan' in target_word['tags']:
-                        new_tags.discard('inan')
-                        new_tags.add('anim')
-                    elif 'NOUN' in target_word['tags'] and 'anim' in target_word['tags']:
-                        new_tags.discard('anim')
-                        new_tags.add('inan')
-                    inflected = make_inflection(text=distractor_lemma, pos=target_word['pos'][1], tags=new_tags)
-                    if inflected is not None:
-                        inflected_distractors.append(inflected)
             num_distractors = min(4, self.max_num_distractors) if self.max_num_distractors >= 4 \
                 else self.max_num_distractors
             if len(inflected_distractors) < num_distractors: