Update backend

utilities_database/user_database_widgets.py

@@ -123,7 +123,7 @@ class LogIn:
                                                              email_sign_up=email_sign_up)
 
             user_name_sign_up = st.text_input("Имя пользователя *",
-                                              placeholder='Введите имя пользователя')
+                                              placeholder='Введите имя пользователя (латинские буквы и символы)')
             unique_user_name_check = db_utils.check_unique_usr(user_log_in_database=db,
                                                                user_name_sign_up=user_name_sign_up)
 
@@ -206,7 +206,7 @@ class LogIn:
             new_passwd = st.text_input("Новый пароль", placeholder='Введите новый пароль',
                                        type='password')
 
-            new_passwd_1 = st.text_input("Повторите новый пароль", placeholder='Введите повторите пароль',
+            new_passwd_1 = st.text_input("Повторите новый пароль", placeholder='Повторите пароль',
                                          type='password')
 
             reset_passwd_submit_button = st.form_submit_button(label='Изменить пароль')

utilities_language_bert/esp_main_workflow_bert.py

@@ -20,9 +20,8 @@ def main_workflow(
         file: UploadedFile or None,
         text: str,
         logs: ST_WIDGETS,
-        logs_d: ST_WIDGETS,
         progress: st_progress,
-        progress_s: st_progress,
+        progress_d: st_progress,
         level: str,
         tw_mode_automatic_mode: str,
         target_words: str,
@@ -37,26 +36,26 @@ def main_workflow(
     :param file: user's file to generate tasks in
     :param text: user's text input to generate tasks in
     :param logs: widget to output logs to
-    :param logs_d: show how many distractors already processed
     :param progress: progress bar
-    :param progress_s: sentences progress bar
+    :param progress_d: sentences progress bar
     :param target_words: how target words are chosen: by user or automatically
     :param tw_mode_automatic_mode:
     :param level: user's specification of CEFR level of text
     :param num_distractors: how many distractors does the user want the task to contain
     :param save_name: user specifies name to save file in cloud
-    :param global_bad_target_words:global_bad_target_words
+    :param global_bad_target_words: global bad target words
     :return: Dictionary with output data: filename, amount_mode, text_with_gaps, tasks_as_list, correct_answers,
              student_out, teacher_out, total_out, original_text
     """
     # Clear bad target_words each time
-    global_bad_target_words = []
+    if global_bad_target_words:
+        global_bad_target_words = []
 
     # Define main global variables
-    logs.write()
     GLOBAL_DISTRACTORS = set()
     MAX_FREQUENCY = 0
 
+    logs.update(label='Загружаем языковые модели и другие данные', state='running')
     mask_filler = load_bert()
 
     # Get input text
@@ -84,11 +83,12 @@ def main_workflow(
 
     # Text preprocessing
     original_text = current_text
-    current_text = current_text.replace('.', '. ').replace('. . .', '...').replace('  ', ' ').replace('…', '...') \
-        .replace('…', '...').replace('—', '-').replace('\u2014', '-').replace('—', '-').replace('-\n', '') \
-        .replace('\n', '%^&*')
+    current_text = (current_text.replace('.', '. ').replace('. . .', '...')
+                    .replace('  ', ' ').replace('…', '...').replace('…', '...')
+                    .replace('—', '-').replace('\u2014', '-').replace('—', '-')
+                    .replace('-\n', '').replace('\n', '%^&*'))
     current_text_sentences = [sent.text.strip() for sent in esp_constants.nlp(current_text).sents]
-    logs.success('Получили Ваш текст!')
+    logs.update(label='Получили Ваш текст!', state='running')
     progress.progress(10)
 
     # Compute frequency dict
@@ -100,7 +100,7 @@ def main_workflow(
         if j < len(FREQ_DICT) * _frequency_barrier_percent:
             MAX_FREQUENCY = tp[1]
     MAX_FREQUENCY = 3 if MAX_FREQUENCY < 3 else MAX_FREQUENCY
-    logs.success("Посчитали немного статистики!")
+    logs.update(label="Посчитали немного статистики!", state='running')
     progress.progress(15)
 
     # Choose necessary language minimum according to user's input
@@ -134,7 +134,7 @@ def main_workflow(
     # Start generation process
     workflow = [SENTENCE(original=sent.strip(), n_sentence=num, max_num_distractors=num_distractors)
                 for num, sent in enumerate(current_text_sentences)]
-    logs.success("Запускаем процесс генерации заданий!")
+    logs.update(label="Запускаем процесс генерации заданий!", state='running')
     progress.progress(20)
 
     for sentence in workflow:
@@ -142,7 +142,7 @@ def main_workflow(
 
     for sentence in workflow:
         sentence.bind_phrases()
-    logs.success("Подготовили предложения для дальнейшей работы!")
+    logs.update(label="Подготовили предложения для дальнейшей работы!", state='running')
     progress.progress(30)
 
     for j, sentence in enumerate(workflow):
@@ -151,7 +151,7 @@ def main_workflow(
                                      user_target_words=USER_TARGET_WORDS,
                                      frequency_dict=FREQ_DICT)
         progress.progress(int(30 + (j * (20 / len(workflow)))))
-    progress_s.progress(50)
+    progress.progress(50)
     DUPLICATE_TARGET_WORDS = defaultdict(list)
     for sentence in workflow:
         for target_word in sentence.target_words:
@@ -164,8 +164,8 @@ def main_workflow(
             if target_word not in RESULT_TW:
                 global_bad_target_words.append(target_word['original_text'])
                 sentence.target_words.remove(target_word)
-    progress_s.progress(55)
-    logs.success('Выбрали слова-пропуски!')
+    progress.progress(55)
+    logs.update(label='Выбрали слова-пропуски!', state='running')
 
     for sentence in workflow:
         for i, target_word in enumerate(sentence.target_words):
@@ -176,7 +176,7 @@ def main_workflow(
 
     for sentence in workflow:
         sentence.filter_target_words(target_words_automatic_mode=tw_mode_automatic_mode)
-    progress_s.progress(60)
+    progress.progress(60)
 
     RESULT_TASKS = []
     for sentence in workflow:
@@ -189,21 +189,21 @@ def main_workflow(
                                                global_distractors=GLOBAL_DISTRACTORS,
                                                distractor_minimum=distractor_minimum,
                                                max_frequency=MAX_FREQUENCY)
-        logs_d.success(
-            f'Обработали {num}/{len(RESULT_TASKS)} целевых слов!')
-    logs_d.success(
-        f'Обработали {len(RESULT_TASKS)}/{len(RESULT_TASKS)} целевых слов!')
-    progress_s.progress(65)
-    logs.success('Подобрали неправильные варианты!')
+        progress_d.progress(num / len(RESULT_TASKS))
+        logs.update(label=f'Обработали {num}/{len(RESULT_TASKS)} целевых слов!', state='running')
+    logs.update(label=f'Обработали {len(RESULT_TASKS)}/{len(RESULT_TASKS)} целевых слов!', state='running')
+    progress_d.progress(100)
+    progress.progress(70)
+    logs.update(label='Подобрали неправильные варианты!', state='running')
 
     for task in RESULT_TASKS:
         task.inflect_distractors()
-    progress_s.progress(70)
-    logs.success('Просклоняли и проспрягали неправильные варианты!')
+    progress.progress(80)
+    logs.update(label='Просклоняли и проспрягали неправильные варианты!', state='running')
 
     for task in RESULT_TASKS:
         task.sample_distractors(num_distractors=num_distractors)
-    progress_s.progress(75)
+    progress.progress(85)
     RESULT_TASKS = list(filter(lambda t: not t.bad_target_word, RESULT_TASKS))
 
     for task in RESULT_TASKS[::-1]:
@@ -226,8 +226,8 @@ def main_workflow(
 
     for task in RESULT_TASKS:
         task.compile_task(max_num_distractors=num_distractors)
-    progress_s.progress(85)
-    logs.success('Отобрали лучшие задания!')
+    progress.progress(90)
+    logs.update(label='Отобрали лучшие задания!', state='running')
 
     TEXT_WITH_GAPS = []
     VARIANTS = []
@@ -241,9 +241,6 @@ def main_workflow(
         TEXT_WITH_GAPS.append(sentence)
     del RESULT_TASKS
 
-    logs.success('Сейчас все будет готово!')
-    progress_s.progress(90)
-
     TEXT_WITH_GAPS = ' '.join([sentence for sentence in TEXT_WITH_GAPS]).replace('%^&*', '\n')
     PREPARED_TASKS = prepare_tasks(VARIANTS)
     STUDENT_OUT = f'{TEXT_WITH_GAPS}\n\n{"=" * 70}\n\n{PREPARED_TASKS["TASKS_STUDENT"]}'
@@ -251,8 +248,8 @@ def main_workflow(
                   f'{PREPARED_TASKS["KEYS_ONLY"]}'
     TOTAL_OUT = f'{original_text}\n\n{"$" * 70}\n\n{STUDENT_OUT}\n\n{"=" * 70}\n\n{PREPARED_TASKS["TASKS_TEACHER"]}' \
                 f'\n\n{"$" * 70}\n\n{PREPARED_TASKS["KEYS_ONLY"]}'
-    logs.success('Сейчас все будет готово!')
-    progress_s.progress(90)
+    logs.update(label='Сейчас все будет готово!', state='running')
+    progress.progress(95)
     save_name = save_name if save_name != '' else f'{str(datetime.datetime.now())[:-7]}_{original_text[:20]}'
     out = {
         'name': save_name,

utilities_language_bert/esp_sentence_bert.py

@@ -208,7 +208,7 @@ class TASK:
             self.bad_target_word = True
             self.distractors = None
         else:
-            self.distractors = [d[0] for i, d in enumerate(distractors_sentence) if i < 15]
+            self.distractors = [d[0] for i, d in enumerate(distractors_sentence) if i < 30]
             self.distractors_number = len(distractors_sentence) if distractors_sentence is not None else 0
 
     def inflect_distractors(self):
@@ -238,8 +238,7 @@ class TASK:
     def sample_distractors(self, num_distractors):
         if not self.bad_target_word:
             num_distractors = min(self.distractors_number, num_distractors) if num_distractors >= 4 else num_distractors
-            self.inflected_distractors = sample(self.inflected_distractors[:min(self.distractors_number, 10)],
-                                                num_distractors)
+            self.inflected_distractors = sample(self.inflected_distractors, num_distractors)
 
     def compile_task(self, max_num_distractors):
         len_distractors = len(self.inflected_distractors)
@@ -248,7 +247,8 @@ class TASK:
         letters = (f'({letter})' for letter in string.ascii_lowercase[:len_variants + 1])
         try:
             distractors = sample(self.inflected_distractors, len_variants) + [self.original_text, ]
-        except ValueError:
+        except ValueError as e:
+            print(f'{e}\n{len_distractors=}\n{len_variants=}')
             distractors = self.inflected_distractors + [self.original_text, ]
         tmp_vars = [f'{item[0]} {item[1].replace("_", " ")}'.lower()
                     for item in zip(letters, sorted(distractors, key=lambda _: random()))]

utilities_language_general/esp_constants.py

@@ -7,19 +7,23 @@ from transformers import pipeline
 
 @st.cache_resource
 def load_w2v(model_path):
-    _w2v_model = gensim.models.KeyedVectors.load_word2vec_format(model_path, binary=True)
+    with st.spinner('Загружаю языковую модель'):
+        _w2v_model = gensim.models.KeyedVectors.load_word2vec_format(model_path, binary=True)
     return _w2v_model
 
 
 @st.cache_resource
 def load_spacy():
-    _nlp = spacy.load('es_core_news_lg')
+    with st.spinner('Загружаю морфо-синтаксический парсер'):
+        _nlp = spacy.load('es_core_news_lg')
     return _nlp
 
 
 @st.cache_resource
 def load_bert():
-    return pipeline("fill-mask", model="a-v-white/bert-base-spanish-wwm-cased-finetuned-literature-pro")
+    with st.spinner('Загружаю языковую модель'):
+        _pipeline = pipeline(task="fill-mask", model="a-v-white/bert-base-spanish-wwm-cased-finetuned-literature-pro")
+    return _pipeline
 
 
 nlp = load_spacy()

utilities_language_general/esp_utils.py

@@ -132,7 +132,6 @@ def get_distractors_from_model(model, lemma: str, pos: str, gender: str or None,
             condition = ((distractor_pos == pos
                           or (distractor_pos in ('VERB', 'ADJ', 'phrase') and pos in ('VERB', 'ADJ', 'phrase')))
                          and distractor_lemma != lemma
-                         and len(distractors) < 100
                          and distractor_similarity < SIMILARITY_VALUES[level_name]
                          and candidate_gender == gender
                          and length_ratio <= max_length_ratio
@@ -160,7 +159,6 @@ def get_distractors_from_model(model, lemma: str, pos: str, gender: str or None,
                               and pos in ('phrase', 'VERB', 'AUX', 'SCONJ', 'ADP')))
                          and candidate[0] != lemma
                          and distractor_lemma != lemma
-                         and len(distractors) < 100
                          and distractor_similarity < SIMILARITY_VALUES[level_name]
                          and distractor_lemma not in global_distractors)
             if condition:
@@ -173,10 +171,10 @@ def get_distractors_from_model(model, lemma: str, pos: str, gender: str or None,
                     distractors.append((candidate[0], distractor_similarity))
                     global_distractors.add(distractor_lemma)
     max_num_distractors = min(4, max_num_distractors) if max_num_distractors >= 4 else max_num_distractors
-    if len(distractors) >= max_num_distractors:
-        return distractors
-    else:
+    if len(distractors) < max_num_distractors:
         return None
+    else:
+        return distractors
 
 
 def get_distractors_from_model_bert(model, text_with_masked_task: str, lemma: str, pos: str, gender: str or None,
@@ -207,7 +205,7 @@ def get_distractors_from_model_bert(model, text_with_masked_task: str, lemma: st
         if (((distractor_pos == pos)
              or (pos in ('VERB', 'ADJ', 'phrase') and distractor_pos in ('VERB', 'ADJ', 'phrase')))
                 and distractor_lemma != lemma
-                and (len(_distractors) < max_num_distractors+10)
+                and (len(_distractors) < max_num_distractors+100)
                 and (distractor_similarity < SIMILARITY_VALUES_bert[level_name])
                 and (candidate_gender == gender)
                 and (length_ratio <= max_length_ratio)  # May be changed if case of phrases

utilities_language_w2v/esp_main_workflow_w2v.py

@@ -22,10 +22,8 @@ def main_workflow(
         file: UploadedFile or None,
         text: str,
         logs: ST_WIDGETS,
-        logs_d: ST_WIDGETS,
         progress: st_progress,
         progress_d: st_progress,
-        progress_s: st_progress,
         level: str,
         tw_mode_automatic_mode: str,
         target_words: str,
@@ -41,10 +39,8 @@ def main_workflow(
     :param file: user's file to generate tasks in
     :param text: user's text input to generate tasks in
     :param logs: widget to output logs to
-    :param logs_d: show how many distractors already processed
     :param progress: progress bar
     :param progress_d: distractors progress bar
-    :param progress_s: sentences progress bar
     :param target_words: how target words are chosen: by user or automatically
     :param tw_mode_automatic_mode:
     :param level: user's specification of CEFR level of text
@@ -55,11 +51,12 @@ def main_workflow(
     :return: Dictionary with output data: filename, amount_mode, text_with_gaps, tasks_as_list, correct_answers,
              student_out, teacher_out, total_out, original_text
     """
+
     # Clear bad target_words each time
-    global_bad_target_words = []
+    if global_bad_target_words:
+        global_bad_target_words = []
 
     # Define main global variables
-    logs.write()
     GLOBAL_DISTRACTORS = set()
     MAX_FREQUENCY = 0
 
@@ -88,11 +85,12 @@ def main_workflow(
 
     # Text preprocessing
     original_text = current_text
-    current_text = current_text.replace('.', '. ').replace('. . .', '...').replace('  ', ' ').replace('…', '...') \
-        .replace('…', '...').replace('—', '-').replace('\u2014', '-').replace('—', '-').replace('-\n', '') \
-        .replace('\n', '%^&*')
+    current_text = (current_text.replace('.', '. ').replace('. . .', '...')
+                    .replace('  ', ' ').replace('…', '...').replace('…', '...')
+                    .replace('—', '-').replace('\u2014', '-').replace('—', '-')
+                    .replace('-\n', '').replace('\n', '%^&*'))
     current_text_sentences = [sent.text.strip() for sent in esp_constants.nlp(current_text).sents]
-    logs.success('Получили Ваш текст!')
+    logs.update(label='Получили Ваш текст!', state='running')
     progress.progress(10)
 
     # Compute frequency dict
@@ -104,7 +102,7 @@ def main_workflow(
         if j < len(FREQ_DICT) * _frequency_barrier_percent:
             MAX_FREQUENCY = tp[1]
     MAX_FREQUENCY = 3 if MAX_FREQUENCY < 3 else MAX_FREQUENCY
-    logs.success("Посчитали немного статистики!")
+    logs.update(label="Посчитали немного статистики!", state='running')
     progress.progress(15)
 
     # Choose necessary language minimum according to user's input
@@ -136,7 +134,7 @@ def main_workflow(
         st.stop()
 
     # Define which model is used for distractor generation
-
+    logs.update(label='Загружаем языковые модели и другие данные', state='running')
     if model_name == 'Модель-1':
         mask_filler = load_w2v(w2v_model_1_path)
     else:
@@ -145,7 +143,7 @@ def main_workflow(
     # Start generation process
     workflow = [SENTENCE(original=sent.strip(), n_sentence=num, max_num_distractors=num_distractors)
                 for num, sent in enumerate(current_text_sentences)]
-    logs.success("Запускаем процесс генерации заданий!")
+    logs.update(label="Запускаем процесс генерации заданий!", state='running')
     progress.progress(20)
 
     for sentence in workflow:
@@ -153,7 +151,7 @@ def main_workflow(
 
     for sentence in workflow:
         sentence.bind_phrases()
-    logs.success("Подготовили предложения для дальнейшей работы!")
+    logs.update(label="Подготовили предложения для дальнейшей работы!", state='running')
     progress.progress(30)
 
     for j, sentence in enumerate(workflow):
@@ -163,7 +161,7 @@ def main_workflow(
                                      user_target_words=USER_TARGET_WORDS,
                                      frequency_dict=FREQ_DICT)
         progress.progress(int(30 + (j * (30 / len(workflow)))))
-    progress_s.progress(60)
+    progress.progress(60)
     DUPLICATE_TARGET_WORDS = defaultdict(list)
     for sentence in workflow:
         for target_word in sentence.target_words:
@@ -176,8 +174,8 @@ def main_workflow(
             if target_word not in RESULT_TW:
                 global_bad_target_words.append(target_word['original_text'])
                 sentence.target_words.remove(target_word)
-    progress_s.progress(65)
-    logs.success('Выбрали слова-пропуски!')
+    progress.progress(65)
+    logs.update(label='Выбрали слова-пропуски!', state='running')
 
     for sentence in workflow:
         sentence.attach_distractors_to_target_word(model=mask_filler,
@@ -185,22 +183,21 @@ def main_workflow(
                                                    distractor_minimum=distractor_minimum,
                                                    level_name=level,
                                                    max_frequency=MAX_FREQUENCY,
-                                                   progress=progress_d,
-                                                   logs=logs_d)
-    progress_s.progress(70)
-    logs.success('Подобрали неправильные варианты!')
+                                                   logs=logs, progress=progress_d)
+    progress.progress(70)
+    logs.update(label='Подобрали неправильные варианты!', state='running')
     for sentence in workflow:
         sentence.inflect_distractors()
-    progress_s.progress(80)
-    logs.success('Просклоняли и проспрягали неправильные варианты!')
+    progress.progress(80)
+    logs.update(label='Просклоняли и проспрягали неправильные варианты!', state='running')
 
     for sentence in workflow:
         sentence.filter_target_words(target_words_automatic_mode=tw_mode_automatic_mode)
 
     for sentence in workflow:
         sentence.sample_distractors(num_distractors=num_distractors)
-    progress_s.progress(90)
-    logs.success('Отобрали лучшие задания!')
+    progress.progress(90)
+    logs.update(label='Отобрали лучшие задания!', state='running')
 
     RESULT_TASKS = []
     for sentence in workflow:
@@ -244,8 +241,8 @@ def main_workflow(
                   f'{PREPARED_TASKS["KEYS_ONLY"]}'
     TOTAL_OUT = f'{original_text}\n\n{"$" * 70}\n\n{STUDENT_OUT}\n\n{"=" * 70}\n\n{PREPARED_TASKS["TASKS_TEACHER"]}' \
                 f'\n\n{"$" * 70}\n\n{PREPARED_TASKS["KEYS_ONLY"]}'
-    logs.success('Сейчас все будет готово!')
-    progress_s.progress(90)
+    logs.update(label='Сейчас все будет готово!', state='running')
+    progress.progress(90)
     save_name = save_name if save_name != '' else f'{str(datetime.datetime.now())[:-7]}_{original_text[:20]}'
     out = {
         'name': save_name,

utilities_language_w2v/esp_sentence_w2v.py

@@ -150,7 +150,7 @@ class SENTENCE:
                                           frequency_dict=frequency_dict)
 
     def attach_distractors_to_target_word(self, model, global_distractors, distractor_minimum, level_name,
-                                          max_frequency, progress, logs):
+                                          max_frequency, logs, progress):
         n_target_words = len(self.target_words)
         bad_target_words = []
         for i, target_word in enumerate(self.target_words):
@@ -165,13 +165,14 @@ class SENTENCE:
             target_word['distractors'] = distractors
             target_word['distractors_number'] = len(distractors) if distractors is not None else 0
             progress.progress(i / n_target_words)
-            logs.success(f'Обработали {i}/{n_target_words} слов в {self.n_sentence + 1}-м предложении')
+            logs.update(label=f'Обработали {i}/{n_target_words} слов в {self.n_sentence + 1}-м предложении',
+                        state='running')
+        progress.progress(100)
         for btw in bad_target_words:
             BAD_USER_TARGET_WORDS.append(btw['original_text'])
             self.target_words.remove(btw)
-        progress.progress(100)
-        logs.success(
-            f'Обработали {n_target_words}/{n_target_words} слов в {self.n_sentence + 1}-м предложении')
+        logs.update(label=f'Обработали {n_target_words}/{n_target_words} слов в {self.n_sentence + 1}-м предложении',
+                    state='running')
 
     def inflect_distractors(self):
         bad_target_words = []
@@ -184,7 +185,7 @@ class SENTENCE:
                         inflected = inflect(lemma=distractor_lemma, target_pos=target_word['pos'],
                                             target_tags=target_word['tags'])
                     else:
-                        continue
+                        continue  # TODO
                 else:
                     inflected = inflect(lemma=distractor_lemma, target_pos=target_word['pos'],
                                         target_tags=target_word['tags'])
@@ -217,8 +218,7 @@ class SENTENCE:
             len_inflected_distractors = len(target_word['inflected_distractors'])
             num_distractors = min(len_inflected_distractors, num_distractors) \
                 if num_distractors >= 4 else num_distractors
-            target_word['inflected_distractors'] = sample(target_word['inflected_distractors'][:min(
-                len_inflected_distractors, 10)], num_distractors)
+            target_word['inflected_distractors'] = sample(target_word['inflected_distractors'], num_distractors)
 
 
 class TASK:
@@ -240,12 +240,13 @@ class TASK:
 
     def compile_task(self, max_num_distractors):
         len_distractors = len(self.inflected_distractors)
-        len_variants = min(len_distractors, max_num_distractors) if max_num_distractors > 4 \
+        len_variants = min(len_distractors, max_num_distractors) if max_num_distractors >= 4 \
             else max_num_distractors
         letters = (f'({letter})' for letter in string.ascii_lowercase[:len_variants + 1])
         try:
             distractors = sample(self.inflected_distractors, len_variants) + [self.original_text, ]
-        except ValueError:
+        except ValueError as e:
+            print(f'{e}\n{len_distractors=}\n{len_variants=}')
             distractors = self.inflected_distractors + [self.original_text, ]
         self.variants.append(
             (self.original_text, [f'{item[0]} {item[1].replace("_", " ").lower()}'.lower()

utilities_ui/custom_download_button.py

@@ -4,7 +4,7 @@ import uuid
 import base64
 import streamlit as st
 from typing import Optional, Union
-from streamlit.elements.button import DownloadButtonDataType
+from streamlit.elements.widgets.button import DownloadButtonDataType
 
 DownloadButtonDataType = Union[DownloadButtonDataType, "pd.DataFrame", "Styler"]