Spaces:
Runtime error
Runtime error
File size: 6,780 Bytes
9b6f929 11bd448 82adb55 25bf2cc 11bd448 aae10fc 25bf2cc f5bf147 d15cd64 0984348 11bd448 9b6f929 82adb55 aae10fc f5bf147 d15cd64 0984348 aae10fc 11bd448 82adb55 349e976 82adb55 349e976 aae10fc 11bd448 aae10fc f62b8c4 aae10fc 25bf2cc aae10fc ac5d892 aae10fc 82adb55 490bc75 349e976 82adb55 490bc75 fe5361e 82adb55 eda68da f62b8c4 eda68da 25bf2cc 490bc75 cf5c8d2 25bf2cc cf5c8d2 fe4418c cf5c8d2 25bf2cc fe5361e 25bf2cc 490bc75 82adb55 aae10fc 25bf2cc 82adb55 490bc75 349e976 490bc75 82adb55 490bc75 82adb55 490bc75 aae10fc fe5361e 9b6f929 82adb55 25bf2cc 11bd448 82adb55 490bc75 25bf2cc 11bd448 9b6f929 25bf2cc |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 |
import os
import gradio as gr
from huggingface_hub import ModelCard, HfApi
from compliance_checks import (
ComplianceSuite,
ComplianceCheck,
IntendedPurposeCheck,
GeneralLimitationsCheck,
ComputationalRequirementsCheck,
EvaluationCheck,
)
hf_writer = gr.HuggingFaceDatasetSaver(
os.getenv('HUGGING_FACE_HUB_TOKEN'),
organization="society-ethics",
dataset_name="model-card-regulatory-check-flags",
private=True
)
hf_api = HfApi()
checks = [
IntendedPurposeCheck(),
GeneralLimitationsCheck(),
ComputationalRequirementsCheck(),
EvaluationCheck(),
]
suite = ComplianceSuite(checks=checks)
def status_emoji(status: bool):
return "✅" if status else "🛑"
def search_for_models(query: str):
if query.strip() == "":
return examples, ",".join([e[0] for e in examples])
models = [m.id for m in list(iter(hf_api.list_models(search=query, limit=10)))]
model_samples = [[m] for m in models]
models_text = ",".join(models)
return model_samples, models_text
def load_model_card(index, options_string: str):
options = options_string.split(",")
model_id = options[index]
card = ModelCard.load(repo_id_or_path=model_id).content
return card
def run_compliance_check(model_card: str):
results = suite.run(model_card)
return [
*[gr.Accordion.update(label=f"{r.name} - {status_emoji(r.status)}", open=not r.status) for r in results],
*[gr.Markdown.update(value=r.to_string()) for r in results],
]
def fetch_and_run_compliance_check(model_id: str):
model_card = ModelCard.load(repo_id_or_path=model_id).content
return run_compliance_check(model_card=model_card)
def compliance_result(compliance_check: ComplianceCheck):
accordion = gr.Accordion(label=f"{compliance_check.name}", open=False)
description = gr.Markdown("Run an evaluation to see results...")
return accordion, description
def read_file(file_obj):
with open(file_obj.name) as f:
model_card = f.read()
return model_card
model_card_box = gr.TextArea(label="Model Card")
# Have to destructure everything since I need to delay rendering.
col = gr.Column()
tab = gr.Tab(label="Results")
col2 = gr.Column()
compliance_results = [compliance_result(c) for c in suite.checks]
compliance_accordions = [c[0] for c in compliance_results]
compliance_descriptions = [c[1] for c in compliance_results]
examples = [
["bigscience/bloom"],
["roberta-base"],
["openai/clip-vit-base-patch32"],
["distilbert-base-cased-distilled-squad"],
]
with gr.Blocks(css="""\
#file-upload .boundedheight {
max-height: 100px;
}
code {
overflow: scroll;
}
""") as demo:
gr.Markdown("""\
# RegCheck AI
This Space matches information in [model cards](https://huggingface.co/docs/hub/model-cards) to proposed \
regulatory compliance descriptions in the \
[EU AI Act](https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=celex%3A52021PC0206).
This is a **prototype** to explore the feasibility of automatic checks for compliance, and is limited to specific \
provisions of Article 13 of the Act, “Transparency and provision of information to users”. \
**Please note: this is research work and NOT a commercial or legal product.**
""")
with gr.Accordion(label="Instructions", open=True):
gr.Markdown("""
To check a model card, first load it by doing any one of the following:
- If the model is on the Hugging Face Hub, search for a model and select it from the results.
- If you have the model card on your computer as a Markdown file, select the "Upload your own card" tab and \
click "Upload a Markdown file".
- Paste your model card's text directly into the "Model Card" text area.
""")
with gr.Accordion(label="Limitations", open=False):
gr.Markdown("""
This tool should be treated as a Proof Of Concept, and is not designed for production-level use.
- This is currently designed to only work on **English** model cards.
- This tool relies on a very strict model card schema, which may be different from your model card.
- Only material in the main card body is considered – any data in the YAML frontmatter is disregarded.
- If your model card contains any HTML fragments, this tool might not be able to read your model card.
""")
with gr.Row():
with gr.Column():
with gr.Tab(label="Load a card from the 🤗 Hugging Face Hub"):
with gr.Row():
model_id_search = gr.Text(label="Model ID")
search_results_text = gr.Text(visible=False, value=",".join([e[0] for e in examples]))
search_results_index = gr.Dataset(
label="Search Results",
components=[model_id_search],
samples=examples,
type="index",
)
model_id_search.change(
fn=search_for_models,
inputs=[model_id_search],
outputs=[search_results_index, search_results_text]
)
with gr.Tab(label="Upload your own card"):
file = gr.UploadButton(label="Upload a Markdown file", elem_id="file-upload")
# TODO: Bug – uploading more than once doesn't trigger the function? Gradio bug?
file.upload(
fn=read_file,
inputs=[file],
outputs=[model_card_box]
)
model_card_box.render()
with col.render():
with tab.render():
with col2.render():
for a, d in compliance_results:
with a.render():
d.render()
flag = gr.Button(value="Disagree with the result? Click here to flag it! 🚩")
flag_message = gr.Text(
show_label=False,
visible=False,
value="Thank you for flagging this! We'll use your report to improve the tool 🤗"
)
search_results_index.click(
fn=load_model_card,
inputs=[search_results_index, search_results_text],
outputs=[model_card_box]
)
model_card_box.change(
fn=run_compliance_check,
inputs=[model_card_box],
outputs=[*compliance_accordions, *compliance_descriptions]
)
flag.click(
fn=lambda x: hf_writer.flag(flag_data=[x]) and gr.Text.update(visible=True),
inputs=[model_card_box],
outputs=[flag_message]
)
hf_writer.setup(
components=[model_card_box],
flagging_dir="flagged"
)
demo.launch()
|