from datasets import load_dataset, Audio
import ctcalign


model_word_separator = '|'
model_blank_token = '[PAD]'

#ds_i = load_dataset("language-and-voice-lab/samromur_asr",split='train+validation+test')
# loading the whole corpus can be difficult for browser to handle live filtering
ds_i = load_dataset("language-and-voice-lab/samromur_asr",split='test')
ds_i = ds_i.data.to_pandas()
i_model_path="carlosdanielhernandezmena/wav2vec2-large-xlsr-53-icelandic-ep10-1000h"
a_i = ctcalign.aligner(i_model_path,model_word_separator,model_blank_token)

#ds_f = load_dataset("carlosdanielhernandezmena/ravnursson_asr",split='train+validation+test')
ds_f = load_dataset("carlosdanielhernandezmena/ravnursson_asr",split='test')
ds_f = ds_f.data.to_pandas()
f_model_path="carlosdanielhernandezmena/wav2vec2-large-xlsr-53-faroese-100h"
a_f = ctcalign.aligner(f_model_path,model_word_separator,model_blank_token)