Spaces:
Runtime error
Runtime error
NimaBoscarino
commited on
Commit
•
f62b8c4
1
Parent(s):
d15cd64
Add initial instructional copy
Browse files- app.py +16 -4
- compliance_checks/base.py +6 -1
- compliance_checks/computational_requirements.py +40 -6
- compliance_checks/general_limitations.py +28 -1
- compliance_checks/intended_purpose.py +39 -1
- tests/test_computational_requirements_check.py +22 -0
- tests/test_general_limitations_check.py +19 -0
- tests/test_intended_purpose_check.py +28 -3
app.py
CHANGED
@@ -25,7 +25,7 @@ def run_compliance_check(model_card: str):
|
|
25 |
results = suite.run(model_card)
|
26 |
|
27 |
return [
|
28 |
-
*[gr.Accordion.update(label=f"{r.name} - {status_emoji(r.status)}") for r in results],
|
29 |
*[gr.Markdown.update(value=r.to_string()) for r in results],
|
30 |
]
|
31 |
|
@@ -57,14 +57,26 @@ with gr.Blocks(css="""\
|
|
57 |
#file-upload .boundedheight {
|
58 |
max-height: 100px;
|
59 |
}
|
|
|
|
|
|
|
|
|
60 |
""") as demo:
|
61 |
gr.Markdown("""\
|
62 |
# RegCheck AI
|
63 |
-
This Space uses model cards’ information as a source of regulatory
|
|
|
64 |
[EU AI Act](https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=celex%3A52021PC0206). For the moment being, the \
|
65 |
demo is a **prototype** limited to specific provisions of Article 13 of the AI Act, related to “Transparency and \
|
66 |
-
provision of information to users”.
|
67 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
68 |
""")
|
69 |
|
70 |
with gr.Row(elem_id="reverse-row"):
|
|
|
25 |
results = suite.run(model_card)
|
26 |
|
27 |
return [
|
28 |
+
*[gr.Accordion.update(label=f"{r.name} - {status_emoji(r.status)}", open=not r.status) for r in results],
|
29 |
*[gr.Markdown.update(value=r.to_string()) for r in results],
|
30 |
]
|
31 |
|
|
|
57 |
#file-upload .boundedheight {
|
58 |
max-height: 100px;
|
59 |
}
|
60 |
+
|
61 |
+
code {
|
62 |
+
overflow: scroll;
|
63 |
+
}
|
64 |
""") as demo:
|
65 |
gr.Markdown("""\
|
66 |
# RegCheck AI
|
67 |
+
This Space uses [model cards’](https://huggingface.co/docs/hub/model-cards) information as a source of regulatory \
|
68 |
+
compliance with some provisions of the proposed \
|
69 |
[EU AI Act](https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=celex%3A52021PC0206). For the moment being, the \
|
70 |
demo is a **prototype** limited to specific provisions of Article 13 of the AI Act, related to “Transparency and \
|
71 |
+
provision of information to users”. **(DISCLAIMER: this is NOT a commercial or legal advice-related product)**
|
72 |
+
|
73 |
+
To check a model card, first load it by doing any one of the following:
|
74 |
+
- If the model is on the Hugging Face Hub, enter its model ID and click "Load model card".
|
75 |
+
- If you have the model card on your computer as a Markdown file, select the "Upload your own card" tab and click \
|
76 |
+
"Upload a Markdown file".
|
77 |
+
- Paste your model card's text directly into the "Model Card" text area.
|
78 |
+
|
79 |
+
Once your card is loaded, click "Run validation checks" to receive your results.
|
80 |
""")
|
81 |
|
82 |
with gr.Row(elem_id="reverse-row"):
|
compliance_checks/base.py
CHANGED
@@ -21,7 +21,12 @@ def walk_to_next_heading(card, heading, heading_text) -> bool:
|
|
21 |
content.append(sibling.text.strip())
|
22 |
sibling = next(sibling_gen, None)
|
23 |
|
24 |
-
if all([c in [
|
|
|
|
|
|
|
|
|
|
|
25 |
return False # , None
|
26 |
|
27 |
return True # , content
|
|
|
21 |
content.append(sibling.text.strip())
|
22 |
sibling = next(sibling_gen, None)
|
23 |
|
24 |
+
if all([c in [
|
25 |
+
"[More Information Needed]",
|
26 |
+
"More information needed.",
|
27 |
+
"Users (both direct and downstream) should be made aware of the risks, biases and limitations of the "
|
28 |
+
"model. More information needed for further recommendations."
|
29 |
+
] for c in content]):
|
30 |
return False # , None
|
31 |
|
32 |
return True # , content
|
compliance_checks/computational_requirements.py
CHANGED
@@ -27,16 +27,50 @@ class ComputationalRequirementsResult(ComplianceResult):
|
|
27 |
return False
|
28 |
|
29 |
def to_string(self):
|
30 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
31 |
|
32 |
|
33 |
class ComputationalRequirementsCheck(ComplianceCheck):
|
34 |
name = "Computational Requirements"
|
35 |
|
36 |
def run_check(self, card: BeautifulSoup):
|
37 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
38 |
|
39 |
-
return ComputationalRequirementsResult(
|
40 |
-
status=check,
|
41 |
-
# requirements=content,
|
42 |
-
)
|
|
|
27 |
return False
|
28 |
|
29 |
def to_string(self):
|
30 |
+
if self.status:
|
31 |
+
return """\
|
32 |
+
In order for users to know what kind of hardware and software they need to run a model, a model card \
|
33 |
+
should have information about the model's computational requirements. We found some documentation \
|
34 |
+
for this in this model card. We look for this by searching for a heading called "Technical Specifications".
|
35 |
+
"""
|
36 |
+
else:
|
37 |
+
return """\
|
38 |
+
We weren't able to find a section in this model card for the model's computational requirements, but it's \
|
39 |
+
easy to add one! You can add the following section to the model card and, once you fill in the \
|
40 |
+
`[More Information Needed]` sections, the "Computational Requirements" check should pass 🤗
|
41 |
+
|
42 |
+
```md
|
43 |
+
## Technical Specifications [optional]
|
44 |
+
|
45 |
+
### Compute Infrastructure
|
46 |
+
|
47 |
+
[More Information Needed]
|
48 |
+
|
49 |
+
#### Hardware
|
50 |
+
|
51 |
+
[More Information Needed]
|
52 |
+
|
53 |
+
#### Software
|
54 |
+
|
55 |
+
[More Information Needed]
|
56 |
+
```
|
57 |
+
"""
|
58 |
|
59 |
|
60 |
class ComputationalRequirementsCheck(ComplianceCheck):
|
61 |
name = "Computational Requirements"
|
62 |
|
63 |
def run_check(self, card: BeautifulSoup):
|
64 |
+
combos = [
|
65 |
+
("h2", "Technical Specifications"),
|
66 |
+
("h2", "Technical Specifications [optional]"),
|
67 |
+
]
|
68 |
+
|
69 |
+
for hX, heading in combos:
|
70 |
+
purpose_check = walk_to_next_heading(card, hX, heading)
|
71 |
+
if purpose_check:
|
72 |
+
return ComputationalRequirementsResult(
|
73 |
+
status=True,
|
74 |
+
)
|
75 |
|
76 |
+
return ComputationalRequirementsResult()
|
|
|
|
|
|
compliance_checks/general_limitations.py
CHANGED
@@ -26,7 +26,34 @@ class GeneralLimitationsResult(ComplianceResult):
|
|
26 |
return False
|
27 |
|
28 |
def to_string(self):
|
29 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
30 |
|
31 |
|
32 |
class GeneralLimitationsCheck(ComplianceCheck):
|
|
|
26 |
return False
|
27 |
|
28 |
def to_string(self):
|
29 |
+
if self.status:
|
30 |
+
return """\
|
31 |
+
It's important for model cards to document the model's general limitations! We found some documentation \
|
32 |
+
for this in this model card. We look for this by searching for headings that say things like:
|
33 |
+
- Bias, Risks, and Limitations
|
34 |
+
- Intended uses & limitations
|
35 |
+
- Limitations
|
36 |
+
"""
|
37 |
+
else:
|
38 |
+
return """\
|
39 |
+
We weren't able to find a section in this model card for the model's limitations, but it's easy to \
|
40 |
+
add one! You can add the following section to the model card and, once you fill in the \
|
41 |
+
`[More Information Needed]` sections, the "General Limitations" check should pass 🤗
|
42 |
+
|
43 |
+
```md
|
44 |
+
## Bias, Risks, and Limitations
|
45 |
+
|
46 |
+
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
47 |
+
|
48 |
+
[More Information Needed]
|
49 |
+
|
50 |
+
### Recommendations
|
51 |
+
|
52 |
+
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
53 |
+
|
54 |
+
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
55 |
+
```
|
56 |
+
"""
|
57 |
|
58 |
|
59 |
class GeneralLimitationsCheck(ComplianceCheck):
|
compliance_checks/intended_purpose.py
CHANGED
@@ -33,7 +33,45 @@ class IntendedPurposeResult(ComplianceResult):
|
|
33 |
return False
|
34 |
|
35 |
def to_string(self):
|
36 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
37 |
|
38 |
|
39 |
class IntendedPurposeCheck(ComplianceCheck):
|
|
|
33 |
return False
|
34 |
|
35 |
def to_string(self):
|
36 |
+
if self.status:
|
37 |
+
return """\
|
38 |
+
It looks like this model card has some documentation for the model's intended purpose! We look for this by \
|
39 |
+
searching for headings that say things like:
|
40 |
+
- Intended uses & limitations
|
41 |
+
- Uses
|
42 |
+
- Model Use
|
43 |
+
"""
|
44 |
+
else:
|
45 |
+
return """\
|
46 |
+
We weren't able to find a section in this model card for the model's intended purpose, but it's easy to \
|
47 |
+
add one! You can add the following section to the model card and, once you fill in the \
|
48 |
+
`[More Information Needed]` sections, the "Intended Purpose" check should pass 🤗
|
49 |
+
|
50 |
+
```md
|
51 |
+
## Uses
|
52 |
+
|
53 |
+
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
54 |
+
|
55 |
+
[More Information Needed]
|
56 |
+
|
57 |
+
### Direct Use
|
58 |
+
|
59 |
+
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
60 |
+
|
61 |
+
[More Information Needed]
|
62 |
+
|
63 |
+
### Downstream Use [optional]
|
64 |
+
|
65 |
+
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
66 |
+
|
67 |
+
[More Information Needed]
|
68 |
+
|
69 |
+
### Out-of-Scope Use
|
70 |
+
|
71 |
+
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
72 |
+
[More Information Needed]
|
73 |
+
```
|
74 |
+
"""
|
75 |
|
76 |
|
77 |
class IntendedPurposeCheck(ComplianceCheck):
|
tests/test_computational_requirements_check.py
CHANGED
@@ -7,6 +7,21 @@ from compliance_checks import (
|
|
7 |
)
|
8 |
|
9 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
10 |
model_card_template = """\
|
11 |
# Model Card for Sample Model
|
12 |
|
@@ -41,3 +56,10 @@ def test_run_checks(card):
|
|
41 |
results = ComputationalRequirementsCheck().run_check(card_soup)
|
42 |
|
43 |
assert results == success_result
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
7 |
)
|
8 |
|
9 |
|
10 |
+
empty_template = """\
|
11 |
+
## Technical Specifications [optional]
|
12 |
+
|
13 |
+
### Compute Infrastructure
|
14 |
+
|
15 |
+
[More Information Needed]
|
16 |
+
|
17 |
+
#### Hardware
|
18 |
+
|
19 |
+
[More Information Needed]
|
20 |
+
|
21 |
+
#### Software
|
22 |
+
|
23 |
+
[More Information Needed]
|
24 |
+
"""
|
25 |
model_card_template = """\
|
26 |
# Model Card for Sample Model
|
27 |
|
|
|
56 |
results = ComputationalRequirementsCheck().run_check(card_soup)
|
57 |
|
58 |
assert results == success_result
|
59 |
+
|
60 |
+
|
61 |
+
def test_fail_on_empty_template():
|
62 |
+
model_card_html = markdown.markdown(empty_template)
|
63 |
+
card_soup = BeautifulSoup(model_card_html, features="html.parser")
|
64 |
+
results = ComputationalRequirementsCheck().run_check(card_soup)
|
65 |
+
assert results == ComputationalRequirementsResult()
|
tests/test_general_limitations_check.py
CHANGED
@@ -6,7 +6,19 @@ from compliance_checks import (
|
|
6 |
GeneralLimitationsCheck, GeneralLimitationsResult,
|
7 |
)
|
8 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
9 |
|
|
|
|
|
10 |
model_card_template = """\
|
11 |
# Model Card for Sample Model
|
12 |
|
@@ -99,3 +111,10 @@ def test_run_checks(card):
|
|
99 |
results = GeneralLimitationsCheck().run_check(card_soup)
|
100 |
|
101 |
assert results == success_result
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
6 |
GeneralLimitationsCheck, GeneralLimitationsResult,
|
7 |
)
|
8 |
|
9 |
+
empty_template = """\
|
10 |
+
## Bias, Risks, and Limitations
|
11 |
+
|
12 |
+
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
13 |
+
|
14 |
+
[More Information Needed]
|
15 |
+
|
16 |
+
### Recommendations
|
17 |
+
|
18 |
+
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
19 |
|
20 |
+
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
21 |
+
"""
|
22 |
model_card_template = """\
|
23 |
# Model Card for Sample Model
|
24 |
|
|
|
111 |
results = GeneralLimitationsCheck().run_check(card_soup)
|
112 |
|
113 |
assert results == success_result
|
114 |
+
|
115 |
+
|
116 |
+
def test_fail_on_empty_template():
|
117 |
+
model_card_html = markdown.markdown(empty_template)
|
118 |
+
card_soup = BeautifulSoup(model_card_html, features="html.parser")
|
119 |
+
results = GeneralLimitationsCheck().run_check(card_soup)
|
120 |
+
assert results == GeneralLimitationsResult()
|
tests/test_intended_purpose_check.py
CHANGED
@@ -6,15 +6,17 @@ from compliance_checks.intended_purpose import (
|
|
6 |
IntendedPurposeCheck, IntendedPurposeResult,
|
7 |
)
|
8 |
|
9 |
-
|
10 |
-
model_card_template = """\
|
11 |
## Uses
|
12 |
|
13 |
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
|
|
14 |
|
15 |
### Direct Use
|
16 |
|
17 |
-
|
|
|
|
|
18 |
|
19 |
### Downstream Use [optional]
|
20 |
|
@@ -25,6 +27,22 @@ Here is some info about direct uses...
|
|
25 |
### Out-of-Scope Use
|
26 |
|
27 |
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
28 |
|
29 |
Here is some info about out-of-scope uses...
|
30 |
"""
|
@@ -101,3 +119,10 @@ def test_run_checks(card):
|
|
101 |
results = IntendedPurposeCheck().run_check(card_soup)
|
102 |
|
103 |
assert results == success_result
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
6 |
IntendedPurposeCheck, IntendedPurposeResult,
|
7 |
)
|
8 |
|
9 |
+
empty_template = """\
|
|
|
10 |
## Uses
|
11 |
|
12 |
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
13 |
+
[More Information Needed]
|
14 |
|
15 |
### Direct Use
|
16 |
|
17 |
+
[More Information Needed]
|
18 |
+
|
19 |
+
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
20 |
|
21 |
### Downstream Use [optional]
|
22 |
|
|
|
27 |
### Out-of-Scope Use
|
28 |
|
29 |
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
30 |
+
[More Information Needed]
|
31 |
+
"""
|
32 |
+
model_card_template = """\
|
33 |
+
## Uses
|
34 |
+
|
35 |
+
Some info...
|
36 |
+
|
37 |
+
### Direct Use
|
38 |
+
|
39 |
+
Some more info.
|
40 |
+
|
41 |
+
### Downstream Use [optional]
|
42 |
+
|
43 |
+
[More Information Needed]
|
44 |
+
|
45 |
+
### Out-of-Scope Use
|
46 |
|
47 |
Here is some info about out-of-scope uses...
|
48 |
"""
|
|
|
119 |
results = IntendedPurposeCheck().run_check(card_soup)
|
120 |
|
121 |
assert results == success_result
|
122 |
+
|
123 |
+
|
124 |
+
def test_fail_on_empty_template():
|
125 |
+
model_card_html = markdown.markdown(empty_template)
|
126 |
+
card_soup = BeautifulSoup(model_card_html, features="html.parser")
|
127 |
+
results = IntendedPurposeCheck().run_check(card_soup)
|
128 |
+
assert results == IntendedPurposeResult()
|