Spaces:
Runtime error
Runtime error
File size: 11,519 Bytes
2609fac 85c4b9b 2609fac 85c4b9b 2609fac 85c4b9b 2609fac 41e198b 2609fac 41e198b 2609fac 41e198b 2609fac 41e198b 2609fac 41e198b 2609fac 9efc4ef 2609fac 85c4b9b 41e198b 85c4b9b 2609fac 41e198b 2609fac 41e198b 2609fac 41e198b 2609fac 41e198b 2609fac 41e198b 2609fac 41e198b 2609fac 41e198b 2609fac 41e198b 2609fac |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 |
import datetime
from io import StringIO
from random import sample
from collections import defaultdict
from streamlit import progress as st_progress
from streamlit.elements import WIDGETS as ST_WIDGETS
from utilities_language_general.esp_constants import st
from utilities_language_w2v.esp_sentence_w2v import TASK
from utilities_language_w2v.esp_sentence_w2v import SENTENCE
from utilities_language_general.esp_constants import load_w2v
from utilities_language_general.esp_utils import prepare_tasks
from streamlit.runtime.uploaded_file_manager import UploadedFile
import utilities_language_general.esp_constants as esp_constants
from utilities_language_general.esp_constants import w2v_model_1_path
from utilities_language_general.esp_constants import w2v_model_2_path
from utilities_language_general.esp_utils import prepare_target_words
from utilities_language_general.esp_utils import compute_frequency_dict
from utilities_language_general.esp_constants import BAD_USER_TARGET_WORDS
def main_workflow(
file: UploadedFile or None,
text: str,
logs: ST_WIDGETS,
progress: st_progress,
progress_d: st_progress,
level: str,
tw_mode_automatic_mode: str,
target_words: str,
num_distractors: int,
save_name: str,
model_name: str,
global_bad_target_words=BAD_USER_TARGET_WORDS):
"""
This is the main course of the program.
All processes and changes take place here.
Partially works with the interface, displaying the success messages and download buttons.
:param file: user's file to generate tasks in
:param text: user's text input to generate tasks in
:param logs: widget to output logs to
:param progress: progress bar
:param progress_d: distractors progress bar
:param target_words: how target words are chosen: by user or automatically
:param tw_mode_automatic_mode:
:param level: user's specification of CEFR level of text
:param num_distractors: how many distractors does the user want the task to contain
:param save_name: user specifies name to save file in cloud
:param global_bad_target_words:global_bad_target_words
:param model_name
:return: Dictionary with output data: filename, amount_mode, text_with_gaps, tasks_as_list, correct_answers,
student_out, teacher_out, total_out, original_text
"""
# Clear bad target_words each time
if global_bad_target_words:
global_bad_target_words = []
# Define main global variables
GLOBAL_DISTRACTORS = set()
MAX_FREQUENCY = 0
# Get input text
if file is not None:
stringio = StringIO(file.getvalue().decode("utf-8"))
current_text = stringio.read()
elif text != '':
current_text = text
else:
esp_constants.st.warning('Вы и текст не вставили, и файл не выбрали 😢')
current_text = ''
esp_constants.st.stop()
# Process target words
if tw_mode_automatic_mode == 'Самостоятельно':
if target_words == '':
esp_constants.st.warning('Вы не ввели целевые слова')
esp_constants.st.stop()
# Cannot make up paradigm, so only USER_TARGET_WORDS is used
USER_TARGET_WORDS = prepare_target_words(target_words)
tw_mode_automatic_mode = False
else:
USER_TARGET_WORDS = None
tw_mode_automatic_mode = True
# Text preprocessing
original_text = current_text
current_text = (current_text.replace('.', '. ').replace('. . .', '...')
.replace(' ', ' ').replace('…', '...').replace('…', '...')
.replace('—', '-').replace('\u2014', '-').replace('—', '-')
.replace('-\n', '').replace('\n', '%^&*'))
current_text_sentences = [sent.text.strip() for sent in esp_constants.nlp(current_text).sents]
logs.update(label='Получили Ваш текст!', state='running')
progress.progress(10)
# Compute frequency dict
FREQ_DICT = compute_frequency_dict(current_text)
# Get maximum frequency (top 5% barrier)
_frequency_barrier_percent = 0.05
for j, tp in enumerate(FREQ_DICT.items()):
if j < len(FREQ_DICT) * _frequency_barrier_percent:
MAX_FREQUENCY = tp[1]
MAX_FREQUENCY = 3 if MAX_FREQUENCY < 3 else MAX_FREQUENCY
logs.update(label="Посчитали немного статистики!", state='running')
progress.progress(15)
# Choose necessary language minimum according to user's input
if level == 'A1':
target_minimum = esp_constants.a1_target_set
distractor_minimum = esp_constants.a1_distractor_set
elif level == 'A2':
target_minimum = esp_constants.a2_target_set
distractor_minimum = esp_constants.a2_distractor_set
elif level == 'B1':
target_minimum = esp_constants.b1_target_set
distractor_minimum = esp_constants.b1_distractor_set
elif level == 'B2':
target_minimum = esp_constants.b2_target_set
distractor_minimum = esp_constants.b2_distractor_set
elif level == 'C1':
target_minimum = esp_constants.c1_target_set
distractor_minimum = esp_constants.c1_distractor_set
elif level == 'C2':
target_minimum = esp_constants.c2_target_set
distractor_minimum = esp_constants.c2_distractor_set
elif level == 'Без уровня':
target_minimum = None
distractor_minimum = None
else:
target_minimum = None
distractor_minimum = None
logs.error('Вы не выбрали языковой уровень!')
st.stop()
# Define which model is used for distractor generation
logs.update(label='Загружаем языковые модели и другие данные', state='running')
if model_name == 'Модель-1':
mask_filler = load_w2v(w2v_model_1_path)
else:
mask_filler = load_w2v(w2v_model_2_path)
# Start generation process
workflow = [SENTENCE(original=sent.strip(), n_sentence=num, max_num_distractors=num_distractors)
for num, sent in enumerate(current_text_sentences)]
logs.update(label="Запускаем процесс генерации заданий!", state='running')
progress.progress(20)
for sentence in workflow:
sentence.lemmatize_sentence()
for sentence in workflow:
sentence.bind_phrases()
logs.update(label="Подготовили предложения для дальнейшей работы!", state='running')
progress.progress(30)
for j, sentence in enumerate(workflow):
sentence.search_target_words(model=mask_filler,
target_words_automatic_mode=tw_mode_automatic_mode,
target_minimum=target_minimum,
user_target_words=USER_TARGET_WORDS,
frequency_dict=FREQ_DICT)
progress.progress(int(30 + (j * (30 / len(workflow)))))
progress.progress(60)
DUPLICATE_TARGET_WORDS = defaultdict(list)
for sentence in workflow:
for target_word in sentence.target_words:
DUPLICATE_TARGET_WORDS[target_word['lemma']].append(target_word)
RESULT_TW = []
for tw_lemma, tw_data in DUPLICATE_TARGET_WORDS.items():
RESULT_TW.append(sample(tw_data, 1)[0])
for sentence in workflow:
for target_word in sentence.target_words:
if target_word not in RESULT_TW:
global_bad_target_words.append(target_word['original_text'])
sentence.target_words.remove(target_word)
progress.progress(65)
logs.update(label='Выбрали слова-пропуски!', state='running')
for sentence in workflow:
sentence.attach_distractors_to_target_word(model=mask_filler,
global_distractors=GLOBAL_DISTRACTORS,
distractor_minimum=distractor_minimum,
level_name=level,
max_frequency=MAX_FREQUENCY,
logs=logs, progress=progress_d)
progress.progress(70)
logs.update(label='Подобрали неправильные варианты!', state='running')
for sentence in workflow:
sentence.inflect_distractors()
progress.progress(80)
logs.update(label='Просклоняли и проспрягали неправильные варианты!', state='running')
for sentence in workflow:
sentence.filter_target_words(target_words_automatic_mode=tw_mode_automatic_mode)
for sentence in workflow:
sentence.sample_distractors(num_distractors=num_distractors)
progress.progress(90)
logs.update(label='Отобрали лучшие задания!', state='running')
RESULT_TASKS = []
for sentence in workflow:
for target_word in sentence.target_words:
task = TASK(task_data=target_word)
RESULT_TASKS.append(task)
del workflow
# Compute number of final tasks
if len(RESULT_TASKS) >= 20:
NUMBER_TASKS = 20
else:
if len(RESULT_TASKS) >= 15:
NUMBER_TASKS = 15
else:
if len(RESULT_TASKS) >= 10:
NUMBER_TASKS = 10
else:
NUMBER_TASKS = len(RESULT_TASKS)
RESULT_TASKS = sample(RESULT_TASKS, NUMBER_TASKS)
RESULT_TASKS = sorted(RESULT_TASKS, key=lambda t: (t.sentence_number, t.position_in_sentence))
for task in RESULT_TASKS:
task.compile_task(max_num_distractors=num_distractors)
TEXT_WITH_GAPS = []
VARIANTS = []
tasks_counter = 1
for i, sentence in enumerate(current_text_sentences):
for task in filter(lambda t: t.sentence_number == i, RESULT_TASKS):
sentence = sentence.replace(task.original_text, f'__________({tasks_counter})')
VARIANTS.append(task.variants)
tasks_counter += 1
TEXT_WITH_GAPS.append(sentence)
del RESULT_TASKS
TEXT_WITH_GAPS = ' '.join([sentence for sentence in TEXT_WITH_GAPS]).replace('%^&*', '\n')
PREPARED_TASKS = prepare_tasks(VARIANTS)
STUDENT_OUT = f'{TEXT_WITH_GAPS}\n\n{"=" * 70}\n\n{PREPARED_TASKS["TASKS_STUDENT"]}'
TEACHER_OUT = f'{TEXT_WITH_GAPS}\n\n{"=" * 70}\n\n{PREPARED_TASKS["TASKS_TEACHER"]}\n\n{"=" * 70}\n\n' \
f'{PREPARED_TASKS["KEYS_ONLY"]}'
TOTAL_OUT = f'{original_text}\n\n{"$" * 70}\n\n{STUDENT_OUT}\n\n{"=" * 70}\n\n{PREPARED_TASKS["TASKS_TEACHER"]}' \
f'\n\n{"$" * 70}\n\n{PREPARED_TASKS["KEYS_ONLY"]}'
logs.update(label='Сейчас все будет готово!', state='running')
progress.progress(90)
save_name = save_name if save_name != '' else f'{str(datetime.datetime.now())[:-7]}_{original_text[:20]}'
out = {
'name': save_name,
'STUDENT_OUT': STUDENT_OUT,
'TEACHER_OUT': TEACHER_OUT,
'TEXT_WITH_GAPS': TEXT_WITH_GAPS,
'TASKS_ONLY': PREPARED_TASKS["RAW_TASKS"],
'KEYS_ONLY': PREPARED_TASKS["KEYS_ONLY"],
'KEYS_ONLY_RAW': PREPARED_TASKS["RAW_KEYS_ONLY"],
'TOTAL_OUT': TOTAL_OUT,
'ORIGINAL': original_text,
'BAD_USER_TARGET_WORDS': sorted(set(global_bad_target_words))
}
return out
|