Merge branch 'main' of https://huggingface.co/spaces/huggingface/data-measurements-tool-2 into main

app.py

@@ -117,7 +117,10 @@ def load_or_prepare(ds_args, show_embeddings, use_cache=False):
         logs.warning("Loading Embeddings")
         dstats.load_or_prepare_embeddings()
     logs.warning("Loading nPMI")
-    dstats.load_or_prepare_npmi()
+    try:
+        dstats.load_or_prepare_npmi()
+    except:
+        logs.warning("Missing a cache for npmi")
     logs.warning("Loading Zipf")
     dstats.load_or_prepare_zipf()
     return dstats

cache_dir/c4_realnewslike_train_text/text_dset/dataset.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9813f70c9be641905ca737aa8f16e29d6aa17155a76cd830e7a627aed91431f4
+size 529606944

cache_dir/c4_realnewslike_train_text/text_dset/dataset_info.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff9f59542efc98b40f23b64408e3fbaed544ad8f0d1fb1e7126ead5af52844ac
+size 945

cache_dir/c4_realnewslike_train_text/text_dset/state.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2f6884f5ee381e5df2d267dae699aaf4792ba06c8f16830c9c19c144b4b3003
+size 256

cache_dir/squad_v2_squad_v2_train_title/text_dset/dataset.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1900ec14c31551d40328c314dc2bf9f9a868b201a50a0a811ff81cc5013f03f7
+size 2414544

cache_dir/squad_v2_squad_v2_train_title/text_dset/dataset_info.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:622d414fc0ad5999a683527b4fe5f08608085c66fcd590a66a461b89858349a5
+size 2085

cache_dir/squad_v2_squad_v2_train_title/text_dset/state.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee31a935ee51d6d450089aa2d477d6db8e39d20076ad0ce8a204676a8e2c43c6
+size 256

data_measurements/dataset_statistics.py

@@ -498,7 +498,7 @@ class DatasetStatisticsCacheClass:
         if not self.live:
             if self.tokenized_df is None:
                 logs.warning("Tokenized dataset not yet loaded; doing so.")
-                self.load_or_prepare_dataset()
+                self.load_or_prepare_tokenized_df()
             if self.vocab_counts_df is None:
                 logs.warning("Vocab not yet loaded; doing so.")
                 self.load_or_prepare_vocab()
@@ -544,8 +544,8 @@ class DatasetStatisticsCacheClass:
         """
         logs.info("Doing text dset.")
         self.load_or_prepare_text_dset(save)
-        logs.info("Doing tokenized dataframe")
-        self.load_or_prepare_tokenized_df(save)
+        #logs.info("Doing tokenized dataframe")
+        #self.load_or_prepare_tokenized_df(save)
         logs.info("Doing dataset peek")
         self.load_or_prepare_dset_peek(save)
 

data_measurements/streamlit_utils.py

@@ -20,7 +20,7 @@ import streamlit as st
 from st_aggrid import AgGrid, GridOptionsBuilder
 
 from .dataset_utils import HF_DESC_FIELD, HF_FEATURE_FIELD, HF_LABEL_FIELD
-
+st.set_option('deprecation.showPyplotGlobalUse', False)
 
 def sidebar_header():
     st.sidebar.markdown(