Added support for datasets without secret test split

app.py

@@ -18,8 +18,17 @@ from datasets import get_dataset_config_names
 # TODO - compare the datasets
 
 st.set_page_config(layout="wide")
+metrics_size_audio = ["samples", "audio[h]", "speakers"]
+metrics_size_text = ["samples", "words", "chars"]
+metrics_size = metrics_size_audio + metrics_size_text
+metrics_features_text_uniq = ["utts_unique", "words_unique", "chars_unique"]
+metrics_features_speech_rate = ["words_per_sec", "chars_per_sec"]
+metrics_features_duration = ["average_audio_duration[s]", "average_utterance_length[words]", "average_utterance_length[chars]"]
+metrics_features_meta = ["meta_cov_sex", "meta_cov_age"]
+metrics_features = metrics_features_text_uniq + metrics_features_speech_rate + metrics_features_duration + metrics_features_meta
 
-about, analysis_bigos, analysis_bigos_pelcra = st.tabs(["About BIGOS datasets", "BIGOS V2 analysis", "PELCRA for BIGOS analysis"])
+
+about, analysis_bigos, analysis_bigos_diagnostic, analysis_bigos_pelcra = st.tabs(["About BIGOS datasets", "BIGOS V2 analysis", "BIGOS V2 diagnostic", "PELCRA for BIGOS analysis"])
 #analysis_bigos_diagnostic
 #########################################BIGOS################################################
 with about:
@@ -30,7 +39,6 @@ with about:
 
 with analysis_bigos:
     dataset_name = "amu-cai/pl-asr-bigos-v2"
-    #dataset_secret = "amu-cai/pl-asr-bigos-v2-secret"
     dataset_short_name = "BIGOS"
     dataset_version = "V2"
     
@@ -51,30 +59,93 @@ with analysis_bigos:
 
 
     st.header("Dataset level metrics")
-    metrics_size = ["samples", "audio[h]", "speakers", "words",  "chars"]
     df_sum_stats_agg = extract_stats_to_agg(df_multindex_for_agg, metrics_size)
 
     # split dataframe into separate dataframes for easier analysis and visualization
-    st.subheader("Dataset size (audio)")
-    df_sum_stats_audio = df_sum_stats_agg[["audio[h]", "samples", "speakers"]]
+    st.subheader("Audio content size")
+    df_sum_stats_audio = df_sum_stats_agg[metrics_size_audio]
     st.dataframe(df_sum_stats_audio)
     
-    st.subheader("Dataset size (text)")
-    df_sum_stats_text = df_sum_stats_agg[["samples", "words", "chars"]]
+    st.subheader("Text content size")
+    df_sum_stats_text = df_sum_stats_agg[metrics_size_text]
     st.dataframe(df_sum_stats_text)
 
+    df_sum_stats_all_splits = extract_stats_all_splits(df_multindex_all_splits, metrics_features)
+
+    st.subheader("Utterances, vocabulary and alphabet space")
+    df_sum_stats_feats_text = df_sum_stats_all_splits[metrics_features_text_uniq]
+    st.dataframe(df_sum_stats_feats_text)
+
+    st.subheader("Speech rates")
+    df_sum_stats_feats_speech_rate= df_sum_stats_all_splits[metrics_features_speech_rate]
+    st.dataframe(df_sum_stats_feats_speech_rate)
+
+    st.subheader("Average utterance lengths and audio duration")
+    df_sum_stats_feats_durations = df_sum_stats_all_splits[metrics_features_duration]
+    st.dataframe(df_sum_stats_feats_durations)
+
+    st.subheader("Metadata coverage")
+    df_sum_stats_feats_meta = df_sum_stats_all_splits[metrics_features_meta]
+    st.dataframe(df_sum_stats_feats_meta)
+
+    st.header("BIGOS subsets (source datasets) cards")
+    for subset in dataset_configs:
+        st.subheader("Dataset card for: {}".format(subset))
+        df_metrics_subset_size = extract_stats_for_dataset_card(df_multindex_for_agg, subset, metrics_size, add_total=True)
+        st.dataframe(df_metrics_subset_size)
+        df_metrics_subset_features = extract_stats_for_dataset_card(df_multindex_for_agg, subset, metrics_features, add_total=False)
+        st.dataframe(df_metrics_subset_features)
+
+with analysis_bigos_diagnostic:
+    dataset_name = "amu-cai/pl-asr-bigos-v2-diagnostic"
+    dataset_short_name = "BIGOS diagnostic"
+    dataset_version = "V2"
+    
+    dataset_configs = get_dataset_config_names(dataset_name,trust_remote_code=True)
+    # remove "all" subset, which is always the last config type
+    dataset_configs.pop()
+    print(dataset_configs)
+    # read the reports for public and secret datasets
+    [stats_dict_public, contents_dict_public] = read_reports(dataset_name)
+
+    # update the metrics for test split with the secret dataset statistics
+    #stats_dict_public = add_test_split_stats_from_secret_dataset(stats_dict_public, stats_dict_secret)
+    df_multindex_for_agg = dict_to_multindex_df(stats_dict_public, all_splits=False)
+    df_multindex_all_splits = dict_to_multindex_df(stats_dict_public, all_splits=True)
+
+    # extract metrics from dictionary and convert to various dataframes for easier analysis and visualization
+    #st.header("Summary statistics")
 
-    metrics_features = ["utts_unique", "words_unique", "chars_unique", "words_per_sec", "chars_per_sec", "average_audio_duration[s]"]
+
+    st.header("Dataset level metrics")
+    df_sum_stats_agg = extract_stats_to_agg(df_multindex_for_agg, metrics_size)
+
+    # split dataframe into separate dataframes for easier analysis and visualization
+    st.subheader("Audio content size")
+    df_sum_stats_audio = df_sum_stats_agg[metrics_size_audio]
+    st.dataframe(df_sum_stats_audio)
+    
+    st.subheader("Text content size")
+    df_sum_stats_text = df_sum_stats_agg[metrics_size_text]
+    st.dataframe(df_sum_stats_text)
 
     df_sum_stats_all_splits = extract_stats_all_splits(df_multindex_all_splits, metrics_features)
 
-    st.subheader("Dataset features (text)")
-    df_sum_stats_feats_text = df_sum_stats_all_splits[metrics_features[0:3]]
+    st.subheader("Utterances, vocabulary and alphabet space")
+    df_sum_stats_feats_text = df_sum_stats_all_splits[metrics_features_text_uniq]
     st.dataframe(df_sum_stats_feats_text)
 
-    st.subheader("Dataset features (audio)")
-    df_sum_stats_feats_audio = df_sum_stats_all_splits[metrics_features[3:]]
-    st.dataframe(df_sum_stats_feats_audio)
+    st.subheader("Speech rates")
+    df_sum_stats_feats_speech_rate= df_sum_stats_all_splits[metrics_features_speech_rate]
+    st.dataframe(df_sum_stats_feats_speech_rate)
+
+    st.subheader("Average utterance lengths and audio duration")
+    df_sum_stats_feats_durations = df_sum_stats_all_splits[metrics_features_duration]
+    st.dataframe(df_sum_stats_feats_durations)
+
+    st.subheader("Metadata coverage")
+    df_sum_stats_feats_meta = df_sum_stats_all_splits[metrics_features_meta]
+    st.dataframe(df_sum_stats_feats_meta)
 
     st.header("BIGOS subsets (source datasets) cards")
     for subset in dataset_configs:
@@ -84,14 +155,11 @@ with analysis_bigos:
         df_metrics_subset_features = extract_stats_for_dataset_card(df_multindex_for_agg, subset, metrics_features, add_total=False)
         st.dataframe(df_metrics_subset_features)
 
-
     
 #########################################PELCRA################################################
 with analysis_bigos_pelcra:
 
     dataset_name = "pelcra/pl-asr-pelcra-for-bigos"
-    #dataset_secret = "pelcra/pl-asr-pelcra-for-bigos-secret"
-
     dataset_short_name = "PELCRA"
 
     # local version with granted gated access
@@ -113,35 +181,35 @@ with analysis_bigos_pelcra:
     # extract metrics from dictionary and convert to various dataframes for easier analysis and visualization
     #st.header("Summary statistics")
 
-
     st.header("Dataset level metrics")
-    metrics_size = ["samples", "audio[h]", "speakers", "words",  "chars"]
     df_sum_stats_agg = extract_stats_to_agg(df_multindex_for_agg, metrics_size)
 
-    #st.dataframe(df_sum_stats_agg)
-    #print(df_sum_stats.columns)
-    
     # split dataframe into separate dataframes for easier analysis and visualization
-    st.subheader("Dataset size (audio)")
-    df_sum_stats_audio = df_sum_stats_agg[["audio[h]", "samples", "speakers"]]
+    st.subheader("Audio content size")
+    df_sum_stats_audio = df_sum_stats_agg[metrics_size_audio]
     st.dataframe(df_sum_stats_audio)
     
-    st.subheader("Dataset size (text)")
-    df_sum_stats_text = df_sum_stats_agg[["samples", "words", "chars"]]
+    st.subheader("Text content size")
+    df_sum_stats_text = df_sum_stats_agg[metrics_size_text]
     st.dataframe(df_sum_stats_text)
 
-
-    metrics_features = ["utts_unique", "words_unique", "chars_unique", "words_per_sec", "chars_per_sec", "average_audio_duration[s]"]
-
     df_sum_stats_all_splits = extract_stats_all_splits(df_multindex_all_splits, metrics_features)
 
-    st.subheader("Dataset features (text)")
-    df_sum_stats_feats_text = df_sum_stats_all_splits[metrics_features[0:3]]
+    st.subheader("Utterances, vocabulary and alphabet space")
+    df_sum_stats_feats_text = df_sum_stats_all_splits[metrics_features_text_uniq]
     st.dataframe(df_sum_stats_feats_text)
 
-    st.subheader("Dataset features (audio)")
-    df_sum_stats_feats_audio = df_sum_stats_all_splits[metrics_features[3:]]
-    st.dataframe(df_sum_stats_feats_audio)
+    st.subheader("Speech rates")
+    df_sum_stats_feats_speech_rate= df_sum_stats_all_splits[metrics_features_speech_rate]
+    st.dataframe(df_sum_stats_feats_speech_rate)
+
+    st.subheader("Average utterance lengths and audio duration")
+    df_sum_stats_feats_durations = df_sum_stats_all_splits[metrics_features_duration]
+    st.dataframe(df_sum_stats_feats_durations)
+
+    st.subheader("Metadata coverage")
+    df_sum_stats_feats_meta = df_sum_stats_all_splits[metrics_features_meta]
+    st.dataframe(df_sum_stats_feats_meta)
 
     st.header("BIGOS subsets (source datasets) cards")
     for subset in dataset_configs:

reports/amu-cai/pl-asr-bigos-v2-diagnostic/dataset_statistics.json

@@ -0,0 +1 @@
+{"pjatk-clarin_mobile-15": {"samples": {"test": 45, "train": 47, "validation": 45, "all_splits": 137}, "audio[h]": {"test": 0.16, "train": 0.16, "validation": 0.16, "all_splits": 0.48}, "speakers": {"test": 11, "train": 34, "validation": 10, "all_splits": 55}, "words": {"test": 1194, "train": 1268, "validation": 1203, "all_splits": 3665}, "chars": {"test": 8027, "train": 8258, "validation": 8109, "all_splits": 24394}, "utts_unique": {"test": 45, "train": 47, "validation": 45, "all_splits": 137}, "words_unique": {"test": 809, "train": 856, "validation": 809, "all_splits": 2161}, "chars_unique": {"test": 33, "train": 33, "validation": 33, "all_splits": 32}, "average_utterance_length[words]": {"test": 26.53, "train": 26.98, "validation": 26.73, "all_splits": 26.75}, "average_utterance_length[chars]": {"test": 178.38, "train": 175.7, "validation": 180.2, "all_splits": 178.06}, "samples_per_spk_stats": {"test": {"average": 4.09, "std": 2.07, "median": 4.0, "min": 1, "max": 7}, "train": {"average": 1.38, "std": 0.59, "median": 1.0, "min": 1, "max": 3}, "validation": {"average": 4.5, "std": 2.29, "median": 4.5, "min": 1, "max": 9}, "all_splits": {"average": 2.49, "std": 2.01, "median": 2.0, "min": 1, "max": 9}}, "words_per_sec": {"test": 2.06, "train": 2.17, "validation": 2.05, "all_splits": 2.09}, "chars_per_sec": {"test": 11.79, "train": 11.95, "validation": 11.78, "all_splits": 11.84}, "average_audio_duration[s]": {"test": 12.88, "train": 12.45, "validation": 13.03, "all_splits": 12.78}, "meta_cov_sex": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}, "meta_cov_age": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}, "meta_dist_sex": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}, "meta_dist_age": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}}, "pjatk-clarin_studio-15": {"samples": {"test": 42, "train": 44, "validation": 40, "all_splits": 126}, "audio[h]": {"test": 0.16, "train": 0.17, "validation": 0.17, "all_splits": 0.5}, "speakers": {"test": 32, "train": 43, "validation": 31, "all_splits": 106}, "words": {"test": 1660, "train": 1798, "validation": 1756, "all_splits": 5214}, "chars": {"test": 6549, "train": 6937, "validation": 6896, "all_splits": 20382}, "utts_unique": {"test": 42, "train": 44, "validation": 40, "all_splits": 126}, "words_unique": {"test": 593, "train": 636, "validation": 618, "all_splits": 1638}, "chars_unique": {"test": 34, "train": 34, "validation": 35, "all_splits": 35}, "average_utterance_length[words]": {"test": 39.52, "train": 40.86, "validation": 43.9, "all_splits": 41.38}, "average_utterance_length[chars]": {"test": 155.93, "train": 157.66, "validation": 172.4, "all_splits": 161.76}, "samples_per_spk_stats": {"test": {"average": 1.31, "std": 0.58, "median": 1.0, "min": 1, "max": 3}, "train": {"average": 1.02, "std": 0.15, "median": 1.0, "min": 1, "max": 2}, "validation": {"average": 1.29, "std": 0.52, "median": 1.0, "min": 1, "max": 3}, "all_splits": {"average": 1.19, "std": 0.46, "median": 1.0, "min": 1, "max": 3}}, "words_per_sec": {"test": 2.81, "train": 3.02, "validation": 2.96, "all_splits": 2.93}, "chars_per_sec": {"test": 8.29, "train": 8.63, "validation": 8.65, "all_splits": 8.52}, "average_audio_duration[s]": {"test": 14.04, "train": 13.54, "validation": 14.86, "all_splits": 14.12}, "meta_cov_sex": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}, "meta_cov_age": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}, "meta_dist_sex": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}, "meta_dist_age": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}}, "mailabs-corpus_librivox-19": {"samples": {"test": 42, "train": 48, "validation": 47, "all_splits": 137}, "audio[h]": {"test": 0.09, "train": 0.1, "validation": 0.09, "all_splits": 0.28}, "speakers": {"test": 32, "train": 34, "validation": 35, "all_splits": 101}, "words": {"test": 687, "train": 798, "validation": 721, "all_splits": 2206}, "chars": {"test": 4416, "train": 5230, "validation": 4691, "all_splits": 14337}, "utts_unique": {"test": 42, "train": 48, "validation": 47, "all_splits": 137}, "words_unique": {"test": 524, "train": 560, "validation": 539, "all_splits": 1412}, "chars_unique": {"test": 66, "train": 61, "validation": 69, "all_splits": 71}, "average_utterance_length[words]": {"test": 16.36, "train": 16.62, "validation": 15.34, "all_splits": 16.1}, "average_utterance_length[chars]": {"test": 105.14, "train": 108.96, "validation": 99.81, "all_splits": 104.65}, "samples_per_spk_stats": {"test": {"average": 1.31, "std": 0.53, "median": 1.0, "min": 1, "max": 3}, "train": {"average": 1.41, "std": 0.65, "median": 1.0, "min": 1, "max": 3}, "validation": {"average": 1.34, "std": 0.67, "median": 1.0, "min": 1, "max": 4}, "all_splits": {"average": 1.36, "std": 0.62, "median": 1.0, "min": 1, "max": 4}}, "words_per_sec": {"test": 2.22, "train": 2.18, "validation": 2.16, "all_splits": 2.18}, "chars_per_sec": {"test": 12.04, "train": 12.12, "validation": 11.87, "all_splits": 12.01}, "average_audio_duration[s]": {"test": 7.37, "train": 7.62, "validation": 7.12, "all_splits": 7.37}, "meta_cov_sex": {"test": 100.0, "train": 100.0, "validation": 100.0, "all_splits": 100.0}, "meta_cov_age": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}, "meta_dist_sex": {"test": {"male": 0.02, "female": 0.0}, "train": {"male": 0.0, "female": 0.0}, "validation": {"male": 0.0, "female": 0.09}, "all_splits": {"male": 0.2, "female": 0.8}}, "meta_dist_age": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}}, "pwr-azon_read-20": {"samples": {"test": 77, "train": 83, "validation": 95, "all_splits": 255}, "audio[h]": {"test": 0.17, "train": 0.17, "validation": 0.16, "all_splits": 0.5}, "speakers": {"test": 6, "train": 17, "validation": 4, "all_splits": 27}, "words": {"test": 780, "train": 805, "validation": 875, "all_splits": 2460}, "chars": {"test": 6744, "train": 6778, "validation": 7516, "all_splits": 21038}, "utts_unique": {"test": 73, "train": 82, "validation": 93, "all_splits": 239}, "words_unique": {"test": 589, "train": 618, "validation": 657, "all_splits": 1612}, "chars_unique": {"test": 33, "train": 33, "validation": 33, "all_splits": 32}, "average_utterance_length[words]": {"test": 10.13, "train": 9.7, "validation": 9.21, "all_splits": 9.65}, "average_utterance_length[chars]": {"test": 87.58, "train": 81.66, "validation": 79.12, "all_splits": 82.5}, "samples_per_spk_stats": {"test": {"average": 12.83, "std": 1.57, "median": 13.0, "min": 11, "max": 15}, "train": {"average": 4.88, "std": 2.0, "median": 5.0, "min": 1, "max": 8}, "validation": {"average": 23.75, "std": 6.61, "median": 23.5, "min": 15, "max": 33}, "all_splits": {"average": 9.44, "std": 7.45, "median": 7.0, "min": 1, "max": 33}}, "words_per_sec": {"test": 1.31, "train": 1.35, "validation": 1.48, "all_splits": 1.38}, "chars_per_sec": {"test": 10.04, "train": 10.0, "validation": 11.24, "all_splits": 10.42}, "average_audio_duration[s]": {"test": 7.72, "train": 7.2, "validation": 6.22, "all_splits": 6.99}, "meta_cov_sex": {"test": 100.0, "train": 100.0, "validation": 100.0, "all_splits": 100.0}, "meta_cov_age": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}, "meta_dist_sex": {"test": {"male": 0.14, "female": 0.86}, "train": {"male": 0.28, "female": 0.72}, "validation": {"male": 0.51, "female": 0.49}, "all_splits": {"male": 0.32, "female": 0.68}}, "meta_dist_age": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}}, "pwr-azon_spont-20": {"samples": {"test": 39, "train": 30, "validation": 31, "all_splits": 100}, "audio[h]": {"test": 0.17, "train": 0.16, "validation": 0.16, "all_splits": 0.49}, "speakers": {"test": 2, "train": 12, "validation": 2, "all_splits": 16}, "words": {"test": 1305, "train": 1190, "validation": 1711, "all_splits": 4206}, "chars": {"test": 8491, "train": 7791, "validation": 10497, "all_splits": 26779}, "utts_unique": {"test": 39, "train": 30, "validation": 31, "all_splits": 100}, "words_unique": {"test": 598, "train": 651, "validation": 724, "all_splits": 1646}, "chars_unique": {"test": 33, "train": 33, "validation": 33, "all_splits": 32}, "average_utterance_length[words]": {"test": 33.46, "train": 39.67, "validation": 55.19, "all_splits": 42.06}, "average_utterance_length[chars]": {"test": 217.72, "train": 259.7, "validation": 338.61, "all_splits": 267.79}, "samples_per_spk_stats": {"test": {"average": 19.5, "std": 7.5, "median": 19.5, "min": 12, "max": 27}, "train": {"average": 2.5, "std": 1.5, "median": 2.0, "min": 1, "max": 5}, "validation": {"average": 15.5, "std": 6.5, "median": 15.5, "min": 9, "max": 22}, "all_splits": {"average": 6.25, "std": 7.56, "median": 4.0, "min": 1, "max": 27}}, "words_per_sec": {"test": 2.18, "train": 2.03, "validation": 2.88, "all_splits": 2.36}, "chars_per_sec": {"test": 11.98, "train": 11.24, "validation": 14.81, "all_splits": 12.68}, "average_audio_duration[s]": {"test": 15.38, "train": 19.57, "validation": 19.13, "all_splits": 17.8}, "meta_cov_sex": {"test": 100.0, "train": 100.0, "validation": 100.0, "all_splits": 100.0}, "meta_cov_age": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}, "meta_dist_sex": {"test": {"male": 0.31, "female": 0.69}, "train": {"male": 0.53, "female": 0.47}, "validation": {"male": 1.0, "female": 0.0}, "all_splits": {"male": 0.59, "female": 0.41}}, "meta_dist_age": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}}, "pwr-maleset-unk": {"samples": {"test": 116, "train": 111, "validation": 118, "all_splits": 345}, "audio[h]": {"test": 0.16, "train": 0.16, "validation": 0.16, "all_splits": 0.48}, "speakers": {"test": 1, "train": 1, "validation": 1, "all_splits": 3}, "words": {"test": 1030, "train": 957, "validation": 972, "all_splits": 2959}, "chars": {"test": 7223, "train": 6779, "validation": 6467, "all_splits": 20469}, "utts_unique": {"test": 116, "train": 111, "validation": 117, "all_splits": 340}, "words_unique": {"test": 750, "train": 710, "validation": 692, "all_splits": 1866}, "chars_unique": {"test": 41, "train": 46, "validation": 49, "all_splits": 52}, "average_utterance_length[words]": {"test": 8.88, "train": 8.62, "validation": 8.24, "all_splits": 8.58}, "average_utterance_length[chars]": {"test": 62.27, "train": 61.07, "validation": 54.81, "all_splits": 59.33}, "samples_per_spk_stats": {"test": {"average": 116.0, "std": 0.0, "median": 116.0, "min": 116, "max": 116}, "train": {"average": 111.0, "std": 0.0, "median": 111.0, "min": 111, "max": 111}, "validation": {"average": 118.0, "std": 0.0, "median": 118.0, "min": 118, "max": 118}, "all_splits": {"average": 115.0, "std": 2.94, "median": 116.0, "min": 111, "max": 118}}, "words_per_sec": {"test": 1.73, "train": 1.68, "validation": 1.73, "all_splits": 1.71}, "chars_per_sec": {"test": 10.43, "train": 10.21, "validation": 9.79, "all_splits": 10.15}, "average_audio_duration[s]": {"test": 5.12, "train": 5.14, "validation": 4.76, "all_splits": 5.0}, "meta_cov_sex": {"test": 100.0, "train": 100.0, "validation": 100.0, "all_splits": 100.0}, "meta_cov_age": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}, "meta_dist_sex": {"test": {"male": 1.0, "female": 0.0}, "train": {"male": 1.0, "female": 0.0}, "validation": {"male": 1.0, "female": 0.0}, "all_splits": {"male": 1.0, "female": 0.0}}, "meta_dist_age": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}}, "pwr-shortwords-unk": {"samples": {"test": 86, "train": 102, "validation": 82, "all_splits": 270}, "audio[h]": {"test": 0.13, "train": 0.16, "validation": 0.12, "all_splits": 0.41000000000000003}, "speakers": {"test": 1, "train": 1, "validation": 1, "all_splits": 3}, "words": {"test": 822, "train": 984, "validation": 755, "all_splits": 2561}, "chars": {"test": 5649, "train": 6727, "validation": 5224, "all_splits": 17600}, "utts_unique": {"test": 84, "train": 92, "validation": 78, "all_splits": 232}, "words_unique": {"test": 585, "train": 644, "validation": 543, "all_splits": 1458}, "chars_unique": {"test": 37, "train": 47, "validation": 41, "all_splits": 48}, "average_utterance_length[words]": {"test": 9.56, "train": 9.65, "validation": 9.21, "all_splits": 9.49}, "average_utterance_length[chars]": {"test": 65.69, "train": 65.95, "validation": 63.71, "all_splits": 65.19}, "samples_per_spk_stats": {"test": {"average": 86.0, "std": 0.0, "median": 86.0, "min": 86, "max": 86}, "train": {"average": 102.0, "std": 0.0, "median": 102.0, "min": 102, "max": 102}, "validation": {"average": 82.0, "std": 0.0, "median": 82.0, "min": 82, "max": 82}, "all_splits": {"average": 90.0, "std": 8.64, "median": 86.0, "min": 82, "max": 102}}, "words_per_sec": {"test": 1.76, "train": 1.74, "validation": 1.75, "all_splits": 1.75}, "chars_per_sec": {"test": 10.33, "train": 10.13, "validation": 10.34, "all_splits": 10.26}, "average_audio_duration[s]": {"test": 5.43, "train": 5.56, "validation": 5.27, "all_splits": 5.43}, "meta_cov_sex": {"test": 100.0, "train": 100.0, "validation": 100.0, "all_splits": 100.0}, "meta_cov_age": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}, "meta_dist_sex": {"test": {"male": 1.0, "female": 0.0}, "train": {"male": 1.0, "female": 0.0}, "validation": {"male": 1.0, "female": 0.0}, "all_splits": {"male": 1.0, "female": 0.0}}, "meta_dist_age": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}}, "pwr-viu-unk": {"samples": {"test": 247, "train": 382, "validation": 266, "all_splits": 895}, "audio[h]": {"test": 0.09, "train": 0.15, "validation": 0.1, "all_splits": 0.33999999999999997}, "speakers": {"test": 1, "train": 1, "validation": 1, "all_splits": 3}, "words": {"test": 438, "train": 691, "validation": 457, "all_splits": 1586}, "chars": {"test": 2785, "train": 4470, "validation": 3012, "all_splits": 10267}, "utts_unique": {"test": 13, "train": 13, "validation": 13, "all_splits": 13}, "words_unique": {"test": 18, "train": 18, "validation": 18, "all_splits": 18}, "chars_unique": {"test": 28, "train": 28, "validation": 28, "all_splits": 27}, "average_utterance_length[words]": {"test": 1.77, "train": 1.81, "validation": 1.72, "all_splits": 1.77}, "average_utterance_length[chars]": {"test": 11.28, "train": 11.7, "validation": 11.32, "all_splits": 11.47}, "samples_per_spk_stats": {"test": {"average": 247.0, "std": 0.0, "median": 247.0, "min": 247, "max": 247}, "train": {"average": 382.0, "std": 0.0, "median": 382.0, "min": 382, "max": 382}, "validation": {"average": 266.0, "std": 0.0, "median": 266.0, "min": 266, "max": 266}, "all_splits": {"average": 298.33, "std": 59.67, "median": 266.0, "min": 247, "max": 382}}, "words_per_sec": {"test": 1.29, "train": 1.27, "validation": 1.25, "all_splits": 1.27}, "chars_per_sec": {"test": 6.92, "train": 6.95, "validation": 7.01, "all_splits": 6.96}, "average_audio_duration[s]": {"test": 1.37, "train": 1.42, "validation": 1.37, "all_splits": 1.39}, "meta_cov_sex": {"test": 100.0, "train": 100.0, "validation": 100.0, "all_splits": 100.0}, "meta_cov_age": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}, "meta_dist_sex": {"test": {"male": 1.0, "female": 0.0}, "train": {"male": 1.0, "female": 0.0}, "validation": {"male": 1.0, "female": 0.0}, "all_splits": {"male": 1.0, "female": 0.0}}, "meta_dist_age": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}}, "google-fleurs-22": {"samples": {"test": 56, "train": 47, "validation": 62, "all_splits": 165}, "audio[h]": {"test": 0.17, "train": 0.16, "validation": 0.16, "all_splits": 0.49}, "speakers": {"test": 1, "train": 1, "validation": 1, "all_splits": 3}, "words": {"test": 1149, "train": 930, "validation": 1100, "all_splits": 3179}, "chars": {"test": 8120, "train": 6479, "validation": 7817, "all_splits": 22416}, "utts_unique": {"test": 53, "train": 47, "validation": 55, "all_splits": 155}, "words_unique": {"test": 769, "train": 685, "validation": 705, "all_splits": 1908}, "chars_unique": {"test": 49, "train": 47, "validation": 45, "all_splits": 51}, "average_utterance_length[words]": {"test": 20.52, "train": 19.79, "validation": 17.74, "all_splits": 19.27}, "average_utterance_length[chars]": {"test": 145.0, "train": 137.85, "validation": 126.08, "all_splits": 135.85}, "samples_per_spk_stats": {"test": {"average": 56.0, "std": 0.0, "median": 56.0, "min": 56, "max": 56}, "train": {"average": 47.0, "std": 0.0, "median": 47.0, "min": 47, "max": 47}, "validation": {"average": 62.0, "std": 0.0, "median": 62.0, "min": 62, "max": 62}, "all_splits": {"average": 55.0, "std": 6.16, "median": 56.0, "min": 47, "max": 62}}, "words_per_sec": {"test": 1.93, "train": 1.6, "validation": 1.9, "all_splits": 1.81}, "chars_per_sec": {"test": 11.74, "train": 9.54, "validation": 11.63, "all_splits": 10.97}, "average_audio_duration[s]": {"test": 10.61, "train": 12.38, "validation": 9.32, "all_splits": 10.63}, "meta_cov_sex": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}, "meta_cov_age": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}, "meta_dist_sex": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}, "meta_dist_age": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}}, "polyai-minds14-21": {"samples": {"test": 24, "train": 26, "validation": 34, "all_splits": 84}, "audio[h]": {"test": 0.15, "train": 0.15, "validation": 0.16, "all_splits": 0.45999999999999996}, "speakers": {"test": 3, "train": 3, "validation": 3, "all_splits": 9}, "words": {"test": 456, "train": 524, "validation": 605, "all_splits": 1585}, "chars": {"test": 2874, "train": 3315, "validation": 3849, "all_splits": 10038}, "utts_unique": {"test": 24, "train": 26, "validation": 34, "all_splits": 84}, "words_unique": {"test": 253, "train": 274, "validation": 294, "all_splits": 551}, "chars_unique": {"test": 44, "train": 47, "validation": 49, "all_splits": 52}, "average_utterance_length[words]": {"test": 19.0, "train": 20.15, "validation": 17.79, "all_splits": 18.87}, "average_utterance_length[chars]": {"test": 119.75, "train": 127.5, "validation": 113.21, "all_splits": 119.5}, "samples_per_spk_stats": {"test": {"average": 8.0, "std": 9.2, "median": 2.0, "min": 1, "max": 21}, "train": {"average": 8.67, "std": 7.32, "median": 4.0, "min": 3, "max": 19}, "validation": {"average": 11.33, "std": 10.37, "median": 4.0, "min": 4, "max": 26}, "all_splits": {"average": 9.33, "std": 10.37, "median": 4.0, "min": 4, "max": 26}}, "words_per_sec": {"test": 0.82, "train": 0.97, "validation": 1.02, "all_splits": 0.94}, "chars_per_sec": {"test": 4.35, "train": 5.19, "validation": 5.47, "all_splits": 5.01}, "average_audio_duration[s]": {"test": 23.19, "train": 20.7, "validation": 17.44, "all_splits": 20.1}, "meta_cov_sex": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}, "meta_cov_age": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}, "meta_dist_sex": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}, "meta_dist_age": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}}, "all": {"samples": {"test": 774, "train": 920, "validation": 820, "all_splits": 2514}, "audio[h]": {"test": 1.45, "train": 1.54, "validation": 1.45, "all_splits": 4.44}, "speakers": {"test": 83, "train": 130, "validation": 71, "all_splits": 284}, "words": {"test": 9521, "train": 9945, "validation": 10155, "all_splits": 29621}, "chars": {"test": 60887, "train": 62773, "validation": 64087, "all_splits": 187747}, "utts_unique": {"test": 531, "train": 540, "validation": 552, "all_splits": 1558}, "words_unique": {"test": 4319, "train": 4502, "validation": 4384, "all_splits": 10565}, "chars_unique": {"test": 80, "train": 75, "validation": 81, "all_splits": 88}, "average_utterance_length[words]": {"test": 12.3, "train": 10.81, "validation": 12.38, "all_splits": 11.78}, "average_utterance_length[chars]": {"test": 78.67, "train": 68.23, "validation": 78.15, "all_splits": 74.68}, "samples_per_spk_stats": {"test": {"average": 8.6, "std": 30.0, "median": 1.0, "min": 1, "max": 247}, "train": {"average": 6.26, "std": 33.63, "median": 1.0, "min": 1, "max": 382}, "validation": {"average": 9.21, "std": 32.11, "median": 1.0, "min": 1, "max": 266}, "all_splits": {"average": 7.71, "std": 32.56, "median": 1.0, "min": 1, "max": 382}}, "words_per_sec": {"test": 1.82, "train": 1.8, "validation": 1.94, "all_splits": 1.85}, "chars_per_sec": {"test": 9.83, "train": 9.55, "validation": 10.32, "all_splits": 9.89}, "average_audio_duration[s]": {"test": 6.75, "train": 6.01, "validation": 6.38, "all_splits": 6.36}, "meta_cov_sex": {"test": 78.42, "train": 82.17, "validation": 77.93, "all_splits": 79.63}, "meta_cov_age": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}, "meta_dist_sex": {"test": {"male": 0.78, "female": 0.15}, "train": {"male": 0.84, "female": 0.1}, "validation": {"male": 0.85, "female": 0.08}, "all_splits": {"male": 0.88, "female": 0.12}}, "meta_dist_age": {"test": "N/A", "train": "N/A", "validation": "N/A", "all_splits": "N/A"}}}

reports/amu-cai/pl-asr-bigos-v2/dataset_contents.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:43e808b081d9b692c2469396565fb967105fd815894a7eaded34e89969dbc890
+oid sha256:51bc49c43a89556c627ad2b57143bceedc0a5510de81f2ceffc471200cdc7ff2
 size 46668863

reports/amu-cai/pl-asr-bigos-v2/dataset_statistics.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0acb30a9a215f9c96b567b8753f565f400eac2366df6dba6248ccba859e190e3
-size 23940
+oid sha256:3914471cf4e90fa7cef52beae12fe5a1162ae90a73b1e2e97cc38f9222ea8831
+size 26953

reports/pelcra/pl-asr-pelcra-for-bigos/dataset_contents.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9cea38447dc7485c0f628eba6e52f45e24d1d467fbe23c065162d6b36455ab1d
+oid sha256:5ccb4a2a854270fcd53a55ce09443c05392c3f15413013724a7975f7db9019ca
 size 95274266

reports/pelcra/pl-asr-pelcra-for-bigos/dataset_statistics.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0ab97523e5f4776bb566ed57c38126004bfac43f64bb3177e9ae39f1ee6e51d5
-size 30399
+oid sha256:18ea2da5c59735f8dcd2d78bffcbbda26e414abcdbf607ab4a66e15aff65acac
+size 33533

run-analysis.py

@@ -2,7 +2,7 @@ import os
 import json
 from datasets import load_dataset, get_dataset_config_names, Features, Value
 from utils import num_of_samples_per_split, uniq_utts_per_split, words_per_split, uniq_words_per_split, chars_per_split, uniq_chars_per_split
-from utils import total_audio_duration_per_split, average_audio_duration_per_split, speakers_per_split, meta_cov_per_split
+from utils import total_audio_duration_per_split, average_audio_duration_per_split, average_utterance_length_chars_per_split, average_utterance_length_words_per_split, speakers_per_split, meta_cov_per_split
 #, uniq_utts_per_speaker
 from utils import meta_distribution_text, meta_distribution_violin_plot, recordings_per_speaker, speech_rate_words_per_split, speech_rate_chars_per_split
 import argparse
@@ -16,14 +16,17 @@ os.makedirs(output_dir_plots, exist_ok=True)
 # read from command line argument
 parser = argparse.ArgumentParser()
 parser.add_argument("--dataset", type=str, required=True, help="Name of the dataset to generate reports for")
-parser.add_argument("--secret_test_split", default=True, type=bool, help="Should references for test split be retrieved from the secret distribution?")
+parser.add_argument('--no_secret_test_split', action='store_false', help="Should references for test split be retrieved from the secret distribution?")
 
 args = parser.parse_args()
 
+
 dataset_name = args.dataset
 print("Generating reports for dataset: {}".format(dataset_name))
-if (args.secret_test_split):
+if not (args.no_secret_test_split):
+
     dataset_name_secret = str.join("-", [dataset_name, "secret"])
+
     # check if secret repo exists
     print(dataset_name_secret)
     try:
@@ -31,7 +34,6 @@ if (args.secret_test_split):
     except:
         print("Config for secret dataset {} cannot be retrieved!".format(dataset_name_secret))
 
-#dataset_name = "amu-cai/pl-asr-bigos-v2"
 output_dir_reports_dataset = os.path.join(output_dir_reports, dataset_name)
 os.makedirs(output_dir_reports_dataset, exist_ok=True)
 
@@ -55,8 +57,11 @@ for config_name in dataset_configs:
     dataset_contents[config_name] = {}
 
     dataset_hf_subset = load_dataset(dataset_name, config_name, features=features_to_load, trust_remote_code=True)
-    if(args.secret_test_split):
+    
+    if not (args.no_secret_test_split):
         dataset_hf_subset_secret = load_dataset(dataset_name_secret, config_name, features=features_to_load, trust_remote_code=True)
+    else:
+        dataset_hf_subset_secret = None
 
     #audio content size
     dataset_statistics[config_name]["samples"] = num_of_samples_per_split(dataset_hf_subset)
@@ -73,6 +78,10 @@ for config_name in dataset_configs:
     dataset_statistics[config_name]["words_unique"], dataset_contents[config_name]["unique_words"] = uniq_words_per_split(dataset_hf_subset, dataset_hf_subset_secret)
     dataset_statistics[config_name]["chars_unique"], dataset_contents[config_name]["unique_chars"] = uniq_chars_per_split(dataset_hf_subset, dataset_hf_subset_secret)
     
+    dataset_statistics[config_name]["average_utterance_length[words]"] = average_utterance_length_words_per_split(dataset_hf_subset, dataset_hf_subset_secret)
+    dataset_statistics[config_name]["average_utterance_length[chars]"] = average_utterance_length_chars_per_split(dataset_hf_subset, dataset_hf_subset_secret)
+    dataset_statistics[config_name]["samples_per_spk_stats"], dataset_contents[config_name]["samples_per_spk"]  = recordings_per_speaker(dataset_hf_subset)
+
     # audio content derived features
     dataset_statistics[config_name]["words_per_sec"] = speech_rate_words_per_split(dataset_hf_subset, dataset_hf_subset_secret)
     dataset_statistics[config_name]["chars_per_sec"] = speech_rate_chars_per_split(dataset_hf_subset, dataset_hf_subset_secret)
@@ -86,7 +95,6 @@ for config_name in dataset_configs:
     dataset_statistics[config_name]["meta_dist_sex"] = meta_distribution_text(dataset_hf_subset, 'speaker_sex')
     dataset_statistics[config_name]["meta_dist_age"] = meta_distribution_text(dataset_hf_subset, 'speaker_age')
 
-    dataset_statistics[config_name]["samples_per_spk"], dataset_contents[config_name]["samples_per_spk"]  = recordings_per_speaker(dataset_hf_subset)
     # dataset_statistics[config_name] = uniq_utts_per_speaker(dataset_hf_subset)
     # number of words per speaker (min, max, med, avg, std)
 

utils.py

@@ -77,6 +77,58 @@ def average_audio_duration_per_split(dataset_hf):
     out_dict["all_splits"] = round(audio_length_total_seconds / samples_all,2)
     return out_dict
 
+def average_utterance_length_chars_per_split(dataset_hf, dataset_hf_secret):
+    # input - huggingface dataset object
+    # output - dictionary with statistics about average utterance length per split
+    out_dict = {}
+    metric = "average_utterance_length[chars]"
+    print("Calculating {}".format(metric))
+    chars_all=0
+    samples_all=0
+    for split in dataset_hf.keys():
+        # extract speakers from file_id 
+        if (split=="test" and dataset_hf_secret is not None):
+            utts_split = dataset_hf_secret[split]["ref_orig"]
+        else:
+            utts_split = dataset_hf[split]["ref_orig"]
+        words_split = " ".join(utts_split).split(" ")
+        chars_split = " ".join(words_split)
+        chars_split_count = len(chars_split)
+        chars_all += chars_split_count
+        samples_split = len(utts_split)
+        samples_all += samples_split
+        #print(split, chars_all_count)
+        out_dict[split] = round(chars_split_count/samples_split, 2)
+
+    # add number of samples for all splits
+    out_dict["all_splits"] = round(chars_all/samples_all, 2)
+    return out_dict
+
+def average_utterance_length_words_per_split(dataset_hf, dataset_hf_secret):
+    # input - huggingface dataset object
+    # output - dictionary with statistics about average utterance length per split
+    out_dict = {}
+    metric = "average_utterance_length[words]"
+    print("Calculating {}".format(metric))
+    words_all=0
+    samples_all=0
+    for split in dataset_hf.keys():
+        # extract speakers from file_id 
+        if (split=="test" and dataset_hf_secret is not None):
+            utts_split = dataset_hf_secret[split]["ref_orig"]
+        else:
+            utts_split = dataset_hf[split]["ref_orig"]
+        words_split_count = len(" ".join(utts_split).split(" "))
+        words_all += words_split_count
+        samples_split = len(utts_split)
+        samples_all += samples_split
+        #print(split, chars_all_count)
+        out_dict[split] = round(words_split_count/samples_split , 2)
+
+    # add number of samples for all splits
+    out_dict["all_splits"] = round(words_all/samples_all, 2)
+    return out_dict
+
 def speakers_per_split(dataset_hf):
     # input - huggingface dataset object
     # output - dictionary with statistics about audio duration per split
@@ -106,7 +158,7 @@ def uniq_utts_per_split(dataset_hf, dataset_hf_secret):
     utts_all = []
     for split in dataset_hf.keys():
         # extract speakers from file_id
-        if (split == "test"):
+        if (split == "test" and dataset_hf_secret is not None):
             utts_split = dataset_hf_secret[split]["ref_orig"]
         else:
             utts_split = dataset_hf[split]["ref_orig"]
@@ -129,7 +181,7 @@ def words_per_split(dataset_hf, dataset_hf_secret):
 
     for split in dataset_hf.keys():
         # extract speakers from file_id 
-        if (split == "test"):
+        if (split == "test" and dataset_hf_secret is not None):
             utts_all = dataset_hf_secret[split]["ref_orig"]
         else:
             utts_all = dataset_hf[split]["ref_orig"]
@@ -153,7 +205,7 @@ def uniq_words_per_split(dataset_hf, dataset_hf_secret):
     
     for split in dataset_hf.keys():
         # extract speakers from file_id 
-        if (split == "test"):
+        if (split == "test" and dataset_hf_secret is not None):
             utts_all = dataset_hf_secret[split]["ref_orig"]
         else:
             utts_all = dataset_hf[split]["ref_orig"]
@@ -185,7 +237,7 @@ def chars_per_split(dataset_hf, dataset_hf_secret):
     
     for split in dataset_hf.keys():
         # extract speakers from file_id 
-        if (split=="test"):
+        if (split=="test" and dataset_hf_secret is not None):
             utts_all = dataset_hf_secret[split]["ref_orig"]
         else:
             utts_all = dataset_hf[split]["ref_orig"]
@@ -210,7 +262,7 @@ def uniq_chars_per_split(dataset_hf, dataset_hf_secret):
     
     for split in dataset_hf.keys():
         # extract speakers from file_id 
-        if(split == "test"):
+        if(split == "test" and dataset_hf_secret is not None):
             utts_all = dataset_hf_secret[split]["ref_orig"]        
         else:
             utts_all = dataset_hf[split]["ref_orig"]
@@ -256,17 +308,13 @@ def meta_cov_per_split(dataset_hf, meta_field):
             out_dict[split] = "N/A"
             continue
         meta_info_not_null_all += meta_info_not_null_count
-        meta_info_coverage = round(meta_info_not_null_count / meta_info_count, 2)
-        #print(split, meta_info_coverage)
-
-        # add number of samples for all splits
+        meta_info_coverage = round(meta_info_not_null_count / meta_info_count * 100, 2)
         out_dict[split] = meta_info_coverage
 
-    # add number of samples for all splits
     if (meta_info_not_null_all == 0):
         out_dict["all_splits"] = "N/A"
     else:
-        out_dict["all_splits"] = round(meta_info_not_null_all/meta_info_all,2 )
+        out_dict["all_splits"] = round(meta_info_not_null_all/meta_info_all * 100,2 )
     return out_dict
 
 
@@ -282,7 +330,7 @@ def speech_rate_words_per_split(dataset_hf, dataset_hf_secret):
 
     for split in dataset_hf.keys():
         # extract speakers from file_id 
-        if (split == "test"):
+        if (split == "test" and dataset_hf_secret is not None):
             utts_split = dataset_hf_secret[split]["ref_orig"]
         else:
             utts_split = dataset_hf[split]["ref_orig"]
@@ -292,9 +340,8 @@ def speech_rate_words_per_split(dataset_hf, dataset_hf_secret):
         audio_split_length_seconds = sum(dataset_hf[split]["audio_duration_seconds"])
         audio_total_length_seconds += audio_split_length_seconds
         speech_rate = round(words_split_count / audio_split_length_seconds, 2)
-        #print(split, speech_rate)
         out_dict[split] = speech_rate
-    # add number of samples for all splits
+
     out_dict["all_splits"] = round(words_all_count / audio_total_length_seconds, 2)
     return out_dict
 
@@ -310,7 +357,7 @@ def speech_rate_chars_per_split(dataset_hf, dataset_hf_secret):
 
     for split in dataset_hf.keys():
         # extract speakers from file_id 
-        if (split == "test"):
+        if (split == "test" and dataset_hf_secret is not None):
             utts_split = dataset_hf_secret[split]["ref_orig"]
         else:
             utts_split = dataset_hf[split]["ref_orig"]
@@ -320,9 +367,8 @@ def speech_rate_chars_per_split(dataset_hf, dataset_hf_secret):
         audio_split_length_seconds = sum(dataset_hf[split]["audio_duration_seconds"])
         audio_total_length_seconds += audio_split_length_seconds
         speech_rate = round(chars_split_count / audio_split_length_seconds, 2)
-        #print(split, speech_rate)
         out_dict[split] = speech_rate
-    # add number of samples for all splits
+
     out_dict["all_splits"] = round(chars_all_count / audio_total_length_seconds, 2)
     return out_dict
 
@@ -362,7 +408,6 @@ def meta_distribution_text(dataset_hf, meta_field):
             out_dict[split][bucket] = round(values_count/len(meta_info_not_null),2)
         #print(split, out_dict[split])
     
-    # add number of samples for all splits
     if (no_meta):
         out_dict["all_splits"] = "N/A"
         return out_dict
@@ -428,6 +473,7 @@ def recordings_per_speaker(dataset_hf):
         recordings_total += recordings_split
 
         average_recordings_per_speaker = round( recordings_split / speakers_split,2)
+
         out_dict_stats[split]["average"] = average_recordings_per_speaker
         out_dict_stats[split]["std"] = round(np.std(list(recordings_per_speaker_stats_dict_split.values())),2)
         out_dict_stats[split]["median"] = np.median(list(recordings_per_speaker_stats_dict_split.values()))