Remove single datapoint datasets

app.py

@@ -4,14 +4,16 @@ import os
 import json
 from transformers import AutoTokenizer
 import ast
+import re
+
 
 CACHE_DIR = "cache_ds/" #Use this to build the dataset
 contribution_json = "contributors.json"
 
 contribution_dict = json.load(open(contribution_json,"r"))
+IGNORE_LIST = ["Bible","Tanzil",""]
 
-
-splits = os.listdir(CACHE_DIR)#['EuroParliamentProceedings', 'TED2020', 'PileOfLaw', 'StackExchange_ver2', 'GithubIssues', 'Opensubtitles', 'USPTO', 'S2ORC', 'DevDocs', 'CodePileReddit2022', 'DMMath', 'Gutenberg', 'USENET', 'GithubDiff', 'Enwiki', 'GNOME', 'ASFPublicMail', 'PileV2Reddit2020', 'CodePilePosts', 'Discourse', 'Tanzil', 'arXiv', 'UbuntuIRC', 'PubMed', 'CodePileReddit2020', 'CodePileReddit2021', 'GlobalVoices', 'FreeLaw_Options', 'PileV2Posts','Bible']
+splits = [split for split in os.listdir(CACHE_DIR) if split not in IGNORE_LIST]
 
 cached_ds = os.listdir(CACHE_DIR)
 tokenizer = AutoTokenizer.from_pretrained('EleutherAI/gpt-neox-20b')
@@ -43,9 +45,13 @@ def load_page(split):
             st.text(content)
             st.write("### Meta:")
             st.write(ast.literal_eval(meta))
+            #Tokenizer related count
             tokenized =  tokenizer(content, return_length=True)['length'][0]
-            token_count_metric = st.metric("Token Count",value=tokenized,delta=2048-tokenized)
-
+            token_count_metric = st.metric("Token Count(compared to 2048)",value=tokenized,delta=4096-tokenized)
+            #Word related count
+            split_words = re.findall(r'\w+', content)
+            word_count_metric = st.metric("Word Count",value=len(split_words))
+            
 
 
 demo_name = st.sidebar.selectbox("Choose a demo", splits)

load_dataset.py

@@ -1,17 +1,22 @@
-import datasets
-import logging
+# import datasets
+# import logging
 import os
-from tqdm import tqdm
-PATH = "/Users/reshinthadithyan/master/research/code-research/carperai/pile-v2-small-filtered/data"
-dataset_subs = os.listdir(PATH)
+import json
+# from tqdm import tqdm
+# dataset_subs = os.listdir(PATH)
 
-print(dataset_subs)
+# print(dataset_subs)
 
 
-for ds in tqdm(dataset_subs):
-    try:
-        print(ds)
-        dataset = datasets.load_dataset("CarperAI/pile-v2-small-filtered",data_files=f"data/{ds}/data.json", split="train")
-        dataset.save_to_disk(f"cache_ds/{ds}")
-    except:
-        print(f"Error at {ds}")
+# for ds in tqdm(dataset_subs):
+#     try:
+#         print(ds)
+#         dataset = datasets.load_dataset("CarperAI/pile-v2-small-filtered",data_files=f"data/{ds}/data.json", split="train")
+#         dataset.save_to_disk(f"cache_ds/{ds}")
+#     except:
+#         print(f"Error at {ds}")
+
+ds_subsets = os.listdir("cache_ds")
+
+with open("documentation.json","w") as f:
+    json.dump(ds_subsets,f)