Updated dashboard and util scripts

app.py

@@ -24,7 +24,7 @@ metrics_size = metrics_size_audio + metrics_size_text
 metrics_features_text_uniq = ["utts_unique", "words_unique", "chars_unique"]
 metrics_features_speech_rate = ["words_per_sec", "chars_per_sec"]
 metrics_features_duration = ["average_audio_duration[s]", "average_utterance_length[words]", "average_utterance_length[chars]"]
-metrics_features_meta = ["meta_cov_sex", "meta_cov_age"]
+metrics_features_meta = ["meta_cov_gender", "meta_cov_age"]
 metrics_features = metrics_features_text_uniq + metrics_features_speech_rate + metrics_features_duration + metrics_features_meta
 
 

run-analysis.py

@@ -48,7 +48,7 @@ dataset_contents = {}
 output_fn_contents = os.path.join(output_dir_reports_dataset, "dataset_contents.json")
 
 # specify features to load. Skip loading of audio data
-features_to_load = Features({'audioname': Value(dtype='string', id=None), 'split': Value(dtype='string', id=None), 'dataset': Value(dtype='string', id=None), 'speaker_id': Value(dtype='string', id=None), 'ref_orig': Value(dtype='string', id=None), 'audio_duration_samples': Value(dtype='int32', id=None), 'audio_duration_seconds': Value(dtype='float32', id=None), 'samplingrate_orig': Value(dtype='int32', id=None), 'sampling_rate': Value(dtype='int32', id=None), 'audiopath_bigos': Value(dtype='string', id=None), 'audiopath_local': Value(dtype='string', id=None), 'speaker_age': Value(dtype='string', id=None), 'speaker_sex': Value(dtype='string', id=None)})
+features_to_load = Features({'audioname': Value(dtype='string', id=None), 'split': Value(dtype='string', id=None), 'dataset': Value(dtype='string', id=None), 'speaker_id': Value(dtype='string', id=None), 'ref_orig': Value(dtype='string', id=None), 'audio_duration_samples': Value(dtype='int32', id=None), 'audio_duration_seconds': Value(dtype='float32', id=None), 'samplingrate_orig': Value(dtype='int32', id=None), 'sampling_rate': Value(dtype='int32', id=None), 'audiopath_bigos': Value(dtype='string', id=None), 'audiopath_local': Value(dtype='string', id=None), 'speaker_age': Value(dtype='string', id=None), 'speaker_gender': Value(dtype='string', id=None)})
 
 for config_name in dataset_configs:
     print("Generating stats for {}".format(config_name))
@@ -88,11 +88,11 @@ for config_name in dataset_configs:
     dataset_statistics[config_name]["average_audio_duration[s]"] = average_audio_duration_per_split(dataset_hf_subset)
     
     # metadata coverage per subset in percent - speaker accent
-    dataset_statistics[config_name]["meta_cov_sex"] = meta_cov_per_split(dataset_hf_subset, 'speaker_sex')
+    dataset_statistics[config_name]["meta_cov_gender"] = meta_cov_per_split(dataset_hf_subset, 'speaker_gender')
     dataset_statistics[config_name]["meta_cov_age"] = meta_cov_per_split(dataset_hf_subset, 'speaker_age')
 
     # speech rate per subset
-    dataset_statistics[config_name]["meta_dist_sex"] = meta_distribution_text(dataset_hf_subset, 'speaker_sex')
+    dataset_statistics[config_name]["meta_dist_gender"] = meta_distribution_text(dataset_hf_subset, 'speaker_gender')
     dataset_statistics[config_name]["meta_dist_age"] = meta_distribution_text(dataset_hf_subset, 'speaker_age')
 
     # dataset_statistics[config_name] = uniq_utts_per_speaker(dataset_hf_subset)
@@ -100,7 +100,7 @@ for config_name in dataset_configs:
 
     # distribution of audio duration per subset
     output_dir_plots_subset = os.path.join(output_dir_plots, config_name)
-    meta_distribution_violin_plot(dataset_hf_subset, output_dir_plots_subset, 'audio_duration_seconds', 'speaker_sex')
+    meta_distribution_violin_plot(dataset_hf_subset, output_dir_plots_subset, 'audio_duration_seconds', 'speaker_gender')
     
     # distribution of audio duration per age
     meta_distribution_violin_plot(dataset_hf_subset, output_dir_plots_subset, 'audio_duration_seconds', 'speaker_age')

utils.py

@@ -8,7 +8,7 @@ import numpy as np
 
 # move to consts
 buckets_age=['teens','twenties', 'thirties', 'fourties', 'fifties', 'sixties', 'seventies', 'eighties', 'nineties']
-buckets_sex=["male", "female"]
+buckets_gender=["male", "female"]
 
 def load_bigos_analyzer_report(fp:str)->dict:
     with open(fp, 'r') as f:
@@ -288,8 +288,8 @@ def meta_cov_per_split(dataset_hf, meta_field):
     # TODO move to config
     if meta_field == 'speaker_age':
         buckets = buckets_age
-    if meta_field == 'speaker_sex':
-        buckets = buckets_sex
+    if meta_field == 'speaker_gender':
+        buckets = buckets_gender
     out_dict = {}
     metric = "meta_cov_" + meta_field
     print("Calculating {}".format(metric))
@@ -378,8 +378,8 @@ def meta_distribution_text(dataset_hf, meta_field):
     no_meta=False
     if meta_field == 'speaker_age':
         buckets = buckets_age
-    if meta_field == 'speaker_sex':
-        buckets = buckets_sex
+    if meta_field == 'speaker_gender':
+        buckets = buckets_gender
     
     # input - huggingface dataset object
     # output - dictionary with statistics about audio duration per split
@@ -494,12 +494,12 @@ def recordings_per_speaker(dataset_hf):
     return out_dict_stats, out_dict_contents
 
 
-def meta_distribution_bar_plot(dataset_hf, output_dir, dimension = "speaker_sex"):
+def meta_distribution_bar_plot(dataset_hf, output_dir, dimension = "speaker_gender"):
     pass
 
-def meta_distribution_violin_plot(dataset_hf, output_dir, metric = "audio_duration_seconds",  dimension = "speaker_sex"):
+def meta_distribution_violin_plot(dataset_hf, output_dir, metric = "audio_duration_seconds",  dimension = "speaker_gender"):
     # input - huggingface dataset object
-    # output - figure with distribution of audio duration per sex
+    # output - figure with distribution of audio duration per gender
     out_dict = {}
 
     print("Generating violin plat for metric {} for dimension {}".format(metric, dimension))