Spaces:
Runtime error
Runtime error
File size: 4,101 Bytes
50e6fbc e0be252 82935d8 4c20fbb cf5eed6 3e4a220 50e6fbc 4b039b3 82935d8 4b039b3 4671e61 4b039b3 e0be252 3e4a220 e0be252 4b039b3 e0be252 cf5eed6 4b039b3 4c20fbb cf5eed6 4c20fbb cf5eed6 4c20fbb cf5eed6 4c20fbb e0be252 50e6fbc 4c20fbb 4b039b3 4c20fbb e0be252 4c20fbb 4b039b3 4c20fbb 50e6fbc e0be252 cf5eed6 e0be252 4c20fbb e0be252 4c20fbb e0be252 4b039b3 cf5eed6 e0be252 4c20fbb e0be252 4c20fbb e0be252 cf5eed6 e0be252 82935d8 4b039b3 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 |
import os
import random
import gradio as gr
import matplotlib.pyplot as plt
import numpy as np
from functools import partial
from datasets import load_dataset
dataset_names = [
"AI4Code",
"AMPS",
"ASFPublicMail",
"CPDataset",
"DMMath",
"Discourse",
"Enwiki",
"EuroParliamentProceedings",
"FreeLaw_Options",
"GithubDiff",
"GithubIssues",
"Gutenberg",
"LeetCode",
"PileOfLaw",
"PubMed",
"S2ORC",
"StackExchange",
"USENET",
"USPTO",
"UbuntuIRC",
"arXiv",
]
dataset_data = {}
for name in dataset_names:
path = f"data/{name}/data.json"
ds = load_dataset(
"CarperAI/pilev2_smol_metadata",
data_files=path,
use_auth_token=os.environ["HF_TOKEN"],
split="train",
# download_mode="force_redownload",
)
dataset_data[name] = {
"ds": ds,
"word_rep_ratios": np.random.randn(len(ds)),
"check_char_repetition_criteria": np.array(ds["check_char_repetition_criteria"]),
"check_flagged_words_criteria": np.array(ds["check_flagged_words_criteria"]),
}
def plt_plot(criteria, dataset, threshold):
plt.close("all")
x = dataset_data[dataset][criteria]
# calculate percentage of data that will be removed given threshold
perc = np.sum(x > threshold) / len(x)
# create a figure
fig = plt.figure()
# add a subplot
ax = fig.add_subplot(111)
# plot some data using black
ax.hist(x, bins=50, color="black")
# plot red dashed line at threshold
ax.axvline(threshold, color='r', linestyle='dashed', linewidth=2)
# set title
# add percentage of data removed
ax.set_title(f"{dataset} (removed {perc:.2%})")
plt.xlabel("Value")
plt.ylabel("Frequency")
# make it look nice
plt.tight_layout()
return fig
def check_filtered(criteria, dataset, threshold):
ds = dataset_data[dataset]["ds"]
filtered_ds = ds.filter(lambda x: x[criteria] > threshold)
if len(filtered_ds) == 0:
return "No examples found"
# get random sample of 1
sample = filtered_ds.select([random.randint(0, len(filtered_ds) - 1)])["text"][0]
return sample
with gr.Blocks() as demo:
dataset = gr.Radio(dataset_names, label="Dataset", value="arXiv")
with gr.Tab("Character Repetition Criteria"):
# plot some random data
plot = gr.Plot()
threshold = gr.Slider(minimum=0, maximum=1, label="Threshold")
calculate = gr.Button("Calculate")
check = gr.Button("Check Filtered Data")
filtered_data = gr.Textbox(lines=5, label="Filtered Data")
plot_fn = partial(plt_plot, "check_char_repetition_criteria")
calculate.click(plot_fn, [dataset, threshold], plot)
check_fn = partial(check_filtered, "check_char_repetition_criteria")
check.click(check_fn, [dataset, threshold], filtered_data)
with gr.Tab("Word Repetition Criteria"):# plot some random data
plot = gr.Plot()
threshold = gr.Slider(minimum=0, maximum=1, label="Threshold")
calculate = gr.Button("Calculate")
check = gr.Button("Check Filtered Data")
filtered_data = gr.Textbox(lines=5, label="Filtered Data")
plot_fn = partial(plt_plot, "word_rep_ratios")
calculate.click(plot_fn, [dataset, threshold], plot)
check_fn = partial(check_filtered, "word_rep_ratios")
check.click(check_fn, [dataset, threshold], filtered_data)
with gr.Tab("Flagged Word Criteria"):# plot some random data
plot = gr.Plot()
threshold = gr.Slider(minimum=0, maximum=1, label="Threshold")
calculate = gr.Button("Calculate")
check = gr.Button("Check Filtered Data")
filtered_data = gr.Textbox(lines=5, label="Filtered Data")
plot_fn = partial(plt_plot, "check_flagged_words_criteria")
calculate.click(plot_fn, [dataset, threshold], plot)
check_fn = partial(check_filtered, "check_flagged_words_criteria")
check.click(check_fn, [dataset, threshold], filtered_data)
if __name__ == "__main__":
demo.launch() |