Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
File size: 5,489 Bytes
3acffd5 a1bb07d 3acffd5 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 |
from datasets import load_dataset
from collections import Counter, defaultdict
import pandas as pd
from huggingface_hub import list_datasets
import os
import gradio as gr
parti_prompt_results = []
ORG = "diffusers-parti-prompts"
SUBMISSIONS = {
"sd-v1-5": None,
"sd-v2-1": None,
"if-v1-0": None,
"karlo": None,
}
LINKS = {
"sd-v1-5": "https://huggingface.co/runwayml/stable-diffusion-v1-5",
"sd-v2-1": "https://huggingface.co/stabilityai/stable-diffusion-2-1",
"if-v1-0": "https://huggingface.co/DeepFloyd/IF-I-XL-v1.0",
"karlo": "https://huggingface.co/kakaobrain/karlo-v1-alpha",
}
MODEL_KEYS = "-".join(SUBMISSIONS.keys())
SUBMISSION_ORG = f"results-{MODEL_KEYS}"
submission_names = list(SUBMISSIONS.keys())
parti_prompt_categories = load_dataset(os.path.join(ORG, "sd-v1-5"))["train"]["Category"]
parti_prompt_challenge = load_dataset(os.path.join(ORG, "sd-v1-5"))["train"]["Challenge"]
def load_submissions():
all_datasets = list_datasets(author=SUBMISSION_ORG)
relevant_ids = [d.id for d in all_datasets]
ids = defaultdict(list)
challenges = defaultdict(list)
categories = defaultdict(list)
for _id in relevant_ids:
ds = load_dataset(_id)["train"]
for result, image_id in zip(ds["result"], ds["id"]):
ids[result].append(image_id)
challenges[parti_prompt_challenge[image_id]].append(result)
categories[parti_prompt_categories[image_id]].append(result)
all_values = sum(len(v) for v in ids.values())
main_dict = {k: '{:.2%}'.format(len(v)/all_values) for k, v in ids.items()}
challenges = {k: Counter(v) for k, v in challenges.items()}
categories = {k: Counter(v) for k, v in categories.items()}
return main_dict, challenges, categories
def get_dataframe_all():
main, challenges, categories = load_submissions()
main_frame = pd.DataFrame([main])
challenges_frame = pd.DataFrame.from_dict(challenges).fillna(0).T
challenges_frame = challenges_frame.div(challenges_frame.sum(axis=1), axis=0)
challenges_frame = challenges_frame.applymap(lambda x: '{:.2%}'.format(x))
categories_frame = pd.DataFrame.from_dict(categories).fillna(0).T
categories_frame = categories_frame.div(categories_frame.sum(axis=1), axis=0)
categories_frame = categories_frame.applymap(lambda x: '{:.2%}'.format(x))
categories_frame = categories_frame.reset_index().rename(columns={'index': 'Category'})
challenges_frame = challenges_frame.reset_index().rename(columns={'index': 'Challenge'})
return main_frame, challenges_frame, categories_frame
TITLE = "# Open Parti Prompts Leaderboard"
DESCRIPTION = """
*This leaderboard is retrieved from answers of [Community Evaluations on Parti Prompts](https://huggingface.co/spaces/OpenGenAI/open-parti-prompts)*
"""
EXPLANATION = """\n\n
## How the is data collected 📊 \n\n
In the [Community Parti Prompts](https://huggingface.co/spaces/OpenGenAI/open-parti-prompts), community members select for every prompt
of [Parti Prompts](https://huggingface.co/datasets/nateraw/parti-prompts) which open-source image generation model has generated the best image.
The community's answers are then stored and used in this space to give a human evaluation of the different models.
Currently the leaderboard includes the following models:
- [sd-v1-5](https://huggingface.co/runwayml/stable-diffusion-v1-5)
- [sd-v2-1](https://huggingface.co/stabilityai/stable-diffusion-2-1)
- [if-v1-0](https://huggingface.co/DeepFloyd/IF-I-XL-v1.0)
- [karlo](https://huggingface.co/kakaobrain/karlo-v1-alpha)
In the following you can see three result tables. The first shows you the overall preferences across all prompts. The second and third tables
show you a breakdown analysis per category and level of difficulty ("challenge") as defined by [Parti Prompts](https://huggingface.co/datasets/nateraw/parti-prompts).
"""
GALLERY_COLUMN_NUM = len(SUBMISSIONS)
def refresh():
return get_dataframe_all()
with gr.Blocks() as demo:
with gr.Column(visible=True) as intro_view:
gr.Markdown(TITLE)
gr.Markdown(DESCRIPTION)
gr.Markdown(EXPLANATION)
headers = list(SUBMISSIONS.keys())
datatype = "str"
main_df, category_df, challenge_df = get_dataframe_all()
with gr.Column():
gr.Markdown("# Open Parti Prompts")
main_dataframe = gr.Dataframe(
value=main_df,
headers=main_df.columns.to_list(),
datatype="str",
row_count=main_df.shape[0],
col_count=main_df.shape[1],
interactive=False,
)
with gr.Column():
gr.Markdown("## per category")
cat_dataframe = gr.Dataframe(
value=category_df,
headers=category_df.columns.to_list(),
datatype="str",
row_count=category_df.shape[0],
col_count=category_df.shape[1],
interactive=False,
)
with gr.Column():
gr.Markdown("## per challenge")
chal_dataframe = gr.Dataframe(
value=challenge_df,
headers=challenge_df.columns.to_list(),
datatype="str",
row_count=challenge_df.shape[0],
col_count=challenge_df.shape[1],
interactive=False,
)
with gr.Row():
refresh_button = gr.Button("Refresh")
refresh_button.click(refresh, inputs=[], outputs=[main_dataframe, cat_dataframe, chal_dataframe])
demo.launch()
|