File size: 1,999 Bytes
5f1c16f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 |
import re
def clean_characters(sample, lower: bool = False, only_mesropatar: bool = False):
if 'sentence' not in sample:
if 'transcription' not in sample:
raise NotImplementedError()
else:
sample['sentence'] = sample['transcription']
allowed_chars = (
"-"
"a-z"
"A-Z"
"0-9"
"ԱԲԳԴԵԶԷԸԹԺԻԼԽԾԿՀՁՂՃՄՅՆՇՈՉՊՋՌՍՎՏՐՑՒՓՔՕՖ"
"աբգդեզէըթժիլխծկհձղճմյնշոչպջռսվտրցւփքօֆև"
" \"'։֊.:?;,ՙ՚՛՜՝՞՟\(\)"
)
if lower:
sample["sentence"] = sample["sentence"].lower()
if only_mesropatar:
allowed_chars = (
"ԱԲԳԴԵԶԷԸԹԺԻԼԽԾԿՀՁՂՃՄՅՆՇՈՉՊՋՌՍՎՏՐՑՒՓՔՕՖ"
"աբգդեզէըթժիլխծկհձղճմյնշոչպջռսվտրցւփքօֆև"
" -"
)
sample["sentence"] = re.sub(f"[^{allowed_chars}]", "", sample["sentence"])
# print(sample["sentence"])
return sample
def extract_all_chars(batch):
all_text = " ".join(batch["sentence"])
vocab = list(set(all_text))
return {"vocab": [vocab], "all_text": [all_text]}
def prepare_dataset(smaple, processor):
audio = smaple["audio"]
smaple["input_values"] = processor(
audio["array"], sampling_rate=audio["sampling_rate"]
).input_values[0]
smaple["input_length"] = len(smaple["input_values"])
with processor.as_target_processor():
smaple["labels"] = processor(smaple["sentence"]).input_ids
return smaple
def batched_prepare_dataset(batch, processor):
batch = batch.copy()
audio = batch["audio"]
batch["input_values"] = processor(
[i["array"] for i in audio], sampling_rate=16_000
).input_values
batch["input_length"] = [len(i) for i in batch["input_values"] ]
with processor.as_target_processor():
batch["labels"] = processor(batch["sentence"]).input_ids
return batch |