pilev2_pipeline / app.py
ncoop57
Add additional checks
a2dd03e
raw
history blame
6.09 kB
import os
import random
import gradio as gr
import matplotlib.pyplot as plt
import numpy as np
from functools import partial
from datasets import load_dataset
dataset_names = [
"AI4Code",
"AMPS",
"ASFPublicMail",
"CPDataset",
"DMMath",
"Discourse",
"Enwiki",
"EuroParliamentProceedings",
"FreeLaw_Options",
"GithubDiff",
"GithubIssues",
"Gutenberg",
"LeetCode",
"PileOfLaw",
"PubMed",
"S2ORC",
"StackExchange",
"USENET",
"USPTO",
"UbuntuIRC",
"arXiv",
]
dataset_data = {}
for name in dataset_names:
path = f"data/{name}/data.json"
ds = load_dataset(
"CarperAI/pilev2_smol_metadata",
data_files=path,
use_auth_token=os.environ["HF_TOKEN"],
split="train",
# download_mode="force_redownload",
)
dataset_data[name] = {
"ds": ds,
"check_word_number_criteria": np.array(ds["check_word_number_criteria"]),
"check_char_repetition_criteria": np.array(ds["check_char_repetition_criteria"]),
"check_flagged_words_criteria": np.array(ds["check_flagged_words_criteria"]),
"check_stop_word_ratio_criteria": np.array(ds["check_stop_word_ratio_criteria"]),
"check_perplexity_criteria": np.array(ds["check_perplexity_criteria"]),
"check_language_criteria": np.array(ds["check_language_criteria"]),
}
def plt_plot(criteria, dataset, threshold):
plt.close("all")
x = dataset_data[dataset][criteria]
# calculate percentage of data that will be removed given threshold
perc = np.sum(x > threshold) / len(x)
# create a figure
fig = plt.figure()
# add a subplot
ax = fig.add_subplot(111)
# plot some data using black
ax.hist(x, bins=50, color="black")
# plot red dashed line at threshold
ax.axvline(threshold, color='r', linestyle='dashed', linewidth=2)
# set title
# add percentage of data removed
ax.set_title(f"{dataset} (removed {perc:.2%})")
plt.xlabel("Value")
plt.ylabel("Frequency")
# make it look nice
plt.tight_layout()
return fig
def check_filtered(criteria, dataset, threshold):
ds = dataset_data[dataset]["ds"]
filtered_ds = ds.filter(lambda x: x[criteria] > threshold)
if len(filtered_ds) == 0:
return "No examples found"
# get random sample of 1
sample = filtered_ds.select([random.randint(0, len(filtered_ds) - 1)])["text"][0]
return sample
with gr.Blocks() as demo:
dataset = gr.Radio(dataset_names, label="Dataset", value="arXiv")
with gr.Tab("Number of Words Criteria"):
# plot some random data
plot = gr.Plot()
threshold = gr.Slider(minimum=0, maximum=1, label="Threshold")
calculate = gr.Button("Calculate")
check = gr.Button("Check Filtered Data")
filtered_data = gr.Textbox(lines=5, label="Filtered Data")
plot_fn = partial(plt_plot, "check_word_number_criteria")
calculate.click(plot_fn, [dataset, threshold], plot)
check_fn = partial(check_filtered, "check_word_number_criteria")
check.click(check_fn, [dataset, threshold], filtered_data)
with gr.Tab("Character Repetition Criteria"):
# plot some random data
plot = gr.Plot()
threshold = gr.Slider(minimum=0, maximum=1, label="Threshold")
calculate = gr.Button("Calculate")
check = gr.Button("Check Filtered Data")
filtered_data = gr.Textbox(lines=5, label="Filtered Data")
plot_fn = partial(plt_plot, "check_char_repetition_criteria")
calculate.click(plot_fn, [dataset, threshold], plot)
check_fn = partial(check_filtered, "check_char_repetition_criteria")
check.click(check_fn, [dataset, threshold], filtered_data)
with gr.Tab("Stop Word Ratio Criteria"):
plot = gr.Plot()
threshold = gr.Slider(minimum=0, maximum=1, label="Threshold")
calculate = gr.Button("Calculate")
check = gr.Button("Check Filtered Data")
filtered_data = gr.Textbox(lines=5, label="Filtered Data")
plot_fn = partial(plt_plot, "check_stop_word_ratio_criteria")
calculate.click(plot_fn, [dataset, threshold], plot)
check_fn = partial(check_filtered, "check_stop_word_ratio_criteria")
check.click(check_fn, [dataset, threshold], filtered_data)
with gr.Tab("Flagged Word Criteria"):
plot = gr.Plot()
threshold = gr.Slider(minimum=0, maximum=1, label="Threshold")
calculate = gr.Button("Calculate")
check = gr.Button("Check Filtered Data")
filtered_data = gr.Textbox(lines=5, label="Filtered Data")
plot_fn = partial(plt_plot, "check_flagged_words_criteria")
calculate.click(plot_fn, [dataset, threshold], plot)
check_fn = partial(check_filtered, "check_flagged_words_criteria")
check.click(check_fn, [dataset, threshold], filtered_data)
with gr.Tab("Perplexity Criteria"):
plot = gr.Plot()
threshold = gr.Slider(minimum=0, maximum=1, label="Threshold")
calculate = gr.Button("Calculate")
check = gr.Button("Check Filtered Data")
filtered_data = gr.Textbox(lines=5, label="Filtered Data")
plot_fn = partial(plt_plot, "check_perplexity_criteria")
calculate.click(plot_fn, [dataset, threshold], plot)
check_fn = partial(check_filtered, "check_perplexity_criteria")
check.click(check_fn, [dataset, threshold], filtered_data)
with gr.Tab("Language Detection Criteria"):
plot = gr.Plot()
threshold = gr.Slider(minimum=0, maximum=1, label="Threshold")
calculate = gr.Button("Calculate")
check = gr.Button("Check Filtered Data")
filtered_data = gr.Textbox(lines=5, label="Filtered Data")
plot_fn = partial(plt_plot, "check_language_criteria")
calculate.click(plot_fn, [dataset, threshold], plot)
check_fn = partial(check_filtered, "check_language_criteria")
check.click(check_fn, [dataset, threshold], filtered_data)
if __name__ == "__main__":
demo.launch()