from email.policy import default import os import csv import random import pandas as pd import gradio as gr from utils import * import scipy.io.wavfile as wavf from huggingface_hub import Repository, upload_file HF_TOKEN = os.environ.get("HF_TOKEN") NUMBER_DIR = './number' number_files = [f.name for f in os.scandir(NUMBER_DIR)] DATASET_REPO_URL = "https://huggingface.co/datasets/chrisjay/crowd-speech-africa" REPOSITORY_DIR = "data" LOCAL_DIR = 'data_local' os.makedirs(LOCAL_DIR,exist_ok=True) #DEFAULT_LANGS = {'Igbo':'ibo','Yoruba':'yor','Hausa':'hau'} GENDER = ['Choose Gender','Male','Female','Other','Prefer not to say'] #------------------Work on Languages-------------------- DEFAULT_LANGS = {} languages = read_json_lines('clean_languages.json') languages_lower=[l for l in languages] _ = [DEFAULT_LANGS.update({l['full'].lower():l['id'].lower()}) for l in languages_lower] #_ = [DEFAULT_LANGS.update({l_other.lower():[l['id'].lower()]}) for l in languages_lower for l_other in l['others'] if l_other.lower()!=l['full'].lower()] #------------------Work on Languages-------------------- repo = Repository( local_dir="data", clone_from=DATASET_REPO_URL, use_auth_token=HF_TOKEN ) #repo.git_pull() with open('app.css','r') as f: BLOCK_CSS = f.read() def save_record(language,text,record,number,age,gender,accent,number_history,current_number,done_recording): number_history = number_history or [0] # Save text and its corresponding record to flag speaker_metadata={} speaker_metadata['gender'] = gender if gender!=GENDER[0] else '' speaker_metadata['age'] = age if age !='' else '' speaker_metadata['accent'] = accent if accent!='' else '' if not done_recording: if language!=None and language!='Choose language' and record is not None and number is not None: language = language.lower() lang_id = DEFAULT_LANGS[language] text =text.strip() # Write audio to file audio_name = get_unique_name() SAVE_FILE_DIR = os.path.join(LOCAL_DIR,audio_name) os.makedirs(SAVE_FILE_DIR,exist_ok=True) audio_output_filename = os.path.join(SAVE_FILE_DIR,'audio.wav') wavf.write(audio_output_filename,record[0],record[1]) # Write metadata.json to file json_file_path = os.path.join(SAVE_FILE_DIR,'metadata.jsonl') metadata= {'id':audio_name,'file_name':'audio.wav', 'language_name':language,'language_id':lang_id, 'number':current_number, 'text':text,'frequency':record[0], 'age': speaker_metadata['age'],'gender': speaker_metadata['gender'], 'accent': speaker_metadata['accent'] } dump_json(metadata,json_file_path) # Simply upload the audio file and metadata using the hub's upload_file # Upload the audio repo_audio_path = os.path.join(REPOSITORY_DIR,os.path.join(audio_name,'audio.wav')) _ = upload_file(path_or_fileobj = audio_output_filename, path_in_repo =repo_audio_path, repo_id='chrisjay/crowd-speech-africa', repo_type='dataset', token=HF_TOKEN ) # Upload the metadata repo_json_path = os.path.join(REPOSITORY_DIR,os.path.join(audio_name,'metadata.jsonl')) _ = upload_file(path_or_fileobj = json_file_path, path_in_repo =repo_json_path, repo_id='chrisjay/crowd-speech-africa', repo_type='dataset', token=HF_TOKEN ) output = f'Recording successfully saved!' # Choose the next number number_history.append(current_number) number_choices = [num for num in [i for i in range(10)] if num not in number_history] if number_choices!=[]: next_number = random.choice(number_choices) next_number_image = f'number/{next_number}.jpg' else: done_recording=True next_number = 0 # the default number next_number_image = f'number/best.gif' output_string = "
language | audio | number | text |
---|---|---|---|
{lang} | {num_} | {text} |
Africa Crowdsource Speech
⌛ Please wait. Loading dataset...