NimaBoscarino commited on
Commit
f62b8c4
1 Parent(s): d15cd64

Add initial instructional copy

Browse files
app.py CHANGED
@@ -25,7 +25,7 @@ def run_compliance_check(model_card: str):
25
  results = suite.run(model_card)
26
 
27
  return [
28
- *[gr.Accordion.update(label=f"{r.name} - {status_emoji(r.status)}") for r in results],
29
  *[gr.Markdown.update(value=r.to_string()) for r in results],
30
  ]
31
 
@@ -57,14 +57,26 @@ with gr.Blocks(css="""\
57
  #file-upload .boundedheight {
58
  max-height: 100px;
59
  }
 
 
 
 
60
  """) as demo:
61
  gr.Markdown("""\
62
  # RegCheck AI
63
- This Space uses model cards’ information as a source of regulatory compliance with some provisions of the proposed \
 
64
  [EU AI Act](https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=celex%3A52021PC0206). For the moment being, the \
65
  demo is a **prototype** limited to specific provisions of Article 13 of the AI Act, related to “Transparency and \
66
- provision of information to users”. Choose a model card and check whether it has some useful info to comply with \
67
- the EU AI Act! **(DISCLAIMER: this is NOT a commercial or legal advice-related product)**\
 
 
 
 
 
 
 
68
  """)
69
 
70
  with gr.Row(elem_id="reverse-row"):
 
25
  results = suite.run(model_card)
26
 
27
  return [
28
+ *[gr.Accordion.update(label=f"{r.name} - {status_emoji(r.status)}", open=not r.status) for r in results],
29
  *[gr.Markdown.update(value=r.to_string()) for r in results],
30
  ]
31
 
 
57
  #file-upload .boundedheight {
58
  max-height: 100px;
59
  }
60
+
61
+ code {
62
+ overflow: scroll;
63
+ }
64
  """) as demo:
65
  gr.Markdown("""\
66
  # RegCheck AI
67
+ This Space uses [model cards’](https://huggingface.co/docs/hub/model-cards) information as a source of regulatory \
68
+ compliance with some provisions of the proposed \
69
  [EU AI Act](https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=celex%3A52021PC0206). For the moment being, the \
70
  demo is a **prototype** limited to specific provisions of Article 13 of the AI Act, related to “Transparency and \
71
+ provision of information to users”. **(DISCLAIMER: this is NOT a commercial or legal advice-related product)**
72
+
73
+ To check a model card, first load it by doing any one of the following:
74
+ - If the model is on the Hugging Face Hub, enter its model ID and click "Load model card".
75
+ - If you have the model card on your computer as a Markdown file, select the "Upload your own card" tab and click \
76
+ "Upload a Markdown file".
77
+ - Paste your model card's text directly into the "Model Card" text area.
78
+
79
+ Once your card is loaded, click "Run validation checks" to receive your results.
80
  """)
81
 
82
  with gr.Row(elem_id="reverse-row"):
compliance_checks/base.py CHANGED
@@ -21,7 +21,12 @@ def walk_to_next_heading(card, heading, heading_text) -> bool:
21
  content.append(sibling.text.strip())
22
  sibling = next(sibling_gen, None)
23
 
24
- if all([c in ["[More Information Needed]", "More information needed."] for c in content]):
 
 
 
 
 
25
  return False # , None
26
 
27
  return True # , content
 
21
  content.append(sibling.text.strip())
22
  sibling = next(sibling_gen, None)
23
 
24
+ if all([c in [
25
+ "[More Information Needed]",
26
+ "More information needed.",
27
+ "Users (both direct and downstream) should be made aware of the risks, biases and limitations of the "
28
+ "model. More information needed for further recommendations."
29
+ ] for c in content]):
30
  return False # , None
31
 
32
  return True # , content
compliance_checks/computational_requirements.py CHANGED
@@ -27,16 +27,50 @@ class ComputationalRequirementsResult(ComplianceResult):
27
  return False
28
 
29
  def to_string(self):
30
- return self.requirements
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
31
 
32
 
33
  class ComputationalRequirementsCheck(ComplianceCheck):
34
  name = "Computational Requirements"
35
 
36
  def run_check(self, card: BeautifulSoup):
37
- check = walk_to_next_heading(card, "h2", "Technical Specifications")
 
 
 
 
 
 
 
 
 
 
38
 
39
- return ComputationalRequirementsResult(
40
- status=check,
41
- # requirements=content,
42
- )
 
27
  return False
28
 
29
  def to_string(self):
30
+ if self.status:
31
+ return """\
32
+ In order for users to know what kind of hardware and software they need to run a model, a model card \
33
+ should have information about the model's computational requirements. We found some documentation \
34
+ for this in this model card. We look for this by searching for a heading called "Technical Specifications".
35
+ """
36
+ else:
37
+ return """\
38
+ We weren't able to find a section in this model card for the model's computational requirements, but it's \
39
+ easy to add one! You can add the following section to the model card and, once you fill in the \
40
+ `[More Information Needed]` sections, the "Computational Requirements" check should pass 🤗
41
+
42
+ ```md
43
+ ## Technical Specifications [optional]
44
+
45
+ ### Compute Infrastructure
46
+
47
+ [More Information Needed]
48
+
49
+ #### Hardware
50
+
51
+ [More Information Needed]
52
+
53
+ #### Software
54
+
55
+ [More Information Needed]
56
+ ```
57
+ """
58
 
59
 
60
  class ComputationalRequirementsCheck(ComplianceCheck):
61
  name = "Computational Requirements"
62
 
63
  def run_check(self, card: BeautifulSoup):
64
+ combos = [
65
+ ("h2", "Technical Specifications"),
66
+ ("h2", "Technical Specifications [optional]"),
67
+ ]
68
+
69
+ for hX, heading in combos:
70
+ purpose_check = walk_to_next_heading(card, hX, heading)
71
+ if purpose_check:
72
+ return ComputationalRequirementsResult(
73
+ status=True,
74
+ )
75
 
76
+ return ComputationalRequirementsResult()
 
 
 
compliance_checks/general_limitations.py CHANGED
@@ -26,7 +26,34 @@ class GeneralLimitationsResult(ComplianceResult):
26
  return False
27
 
28
  def to_string(self):
29
- return self.limitations
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
30
 
31
 
32
  class GeneralLimitationsCheck(ComplianceCheck):
 
26
  return False
27
 
28
  def to_string(self):
29
+ if self.status:
30
+ return """\
31
+ It's important for model cards to document the model's general limitations! We found some documentation \
32
+ for this in this model card. We look for this by searching for headings that say things like:
33
+ - Bias, Risks, and Limitations
34
+ - Intended uses & limitations
35
+ - Limitations
36
+ """
37
+ else:
38
+ return """\
39
+ We weren't able to find a section in this model card for the model's limitations, but it's easy to \
40
+ add one! You can add the following section to the model card and, once you fill in the \
41
+ `[More Information Needed]` sections, the "General Limitations" check should pass 🤗
42
+
43
+ ```md
44
+ ## Bias, Risks, and Limitations
45
+
46
+ <!-- This section is meant to convey both technical and sociotechnical limitations. -->
47
+
48
+ [More Information Needed]
49
+
50
+ ### Recommendations
51
+
52
+ <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
53
+
54
+ Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
55
+ ```
56
+ """
57
 
58
 
59
  class GeneralLimitationsCheck(ComplianceCheck):
compliance_checks/intended_purpose.py CHANGED
@@ -33,7 +33,45 @@ class IntendedPurposeResult(ComplianceResult):
33
  return False
34
 
35
  def to_string(self):
36
- return str((self.direct_use, self.direct_use, self.out_of_scope_use))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
37
 
38
 
39
  class IntendedPurposeCheck(ComplianceCheck):
 
33
  return False
34
 
35
  def to_string(self):
36
+ if self.status:
37
+ return """\
38
+ It looks like this model card has some documentation for the model's intended purpose! We look for this by \
39
+ searching for headings that say things like:
40
+ - Intended uses & limitations
41
+ - Uses
42
+ - Model Use
43
+ """
44
+ else:
45
+ return """\
46
+ We weren't able to find a section in this model card for the model's intended purpose, but it's easy to \
47
+ add one! You can add the following section to the model card and, once you fill in the \
48
+ `[More Information Needed]` sections, the "Intended Purpose" check should pass 🤗
49
+
50
+ ```md
51
+ ## Uses
52
+
53
+ <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
54
+
55
+ [More Information Needed]
56
+
57
+ ### Direct Use
58
+
59
+ <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
60
+
61
+ [More Information Needed]
62
+
63
+ ### Downstream Use [optional]
64
+
65
+ <!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
66
+
67
+ [More Information Needed]
68
+
69
+ ### Out-of-Scope Use
70
+
71
+ <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
72
+ [More Information Needed]
73
+ ```
74
+ """
75
 
76
 
77
  class IntendedPurposeCheck(ComplianceCheck):
tests/test_computational_requirements_check.py CHANGED
@@ -7,6 +7,21 @@ from compliance_checks import (
7
  )
8
 
9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10
  model_card_template = """\
11
  # Model Card for Sample Model
12
 
@@ -41,3 +56,10 @@ def test_run_checks(card):
41
  results = ComputationalRequirementsCheck().run_check(card_soup)
42
 
43
  assert results == success_result
 
 
 
 
 
 
 
 
7
  )
8
 
9
 
10
+ empty_template = """\
11
+ ## Technical Specifications [optional]
12
+
13
+ ### Compute Infrastructure
14
+
15
+ [More Information Needed]
16
+
17
+ #### Hardware
18
+
19
+ [More Information Needed]
20
+
21
+ #### Software
22
+
23
+ [More Information Needed]
24
+ """
25
  model_card_template = """\
26
  # Model Card for Sample Model
27
 
 
56
  results = ComputationalRequirementsCheck().run_check(card_soup)
57
 
58
  assert results == success_result
59
+
60
+
61
+ def test_fail_on_empty_template():
62
+ model_card_html = markdown.markdown(empty_template)
63
+ card_soup = BeautifulSoup(model_card_html, features="html.parser")
64
+ results = ComputationalRequirementsCheck().run_check(card_soup)
65
+ assert results == ComputationalRequirementsResult()
tests/test_general_limitations_check.py CHANGED
@@ -6,7 +6,19 @@ from compliance_checks import (
6
  GeneralLimitationsCheck, GeneralLimitationsResult,
7
  )
8
 
 
 
 
 
 
 
 
 
 
 
9
 
 
 
10
  model_card_template = """\
11
  # Model Card for Sample Model
12
 
@@ -99,3 +111,10 @@ def test_run_checks(card):
99
  results = GeneralLimitationsCheck().run_check(card_soup)
100
 
101
  assert results == success_result
 
 
 
 
 
 
 
 
6
  GeneralLimitationsCheck, GeneralLimitationsResult,
7
  )
8
 
9
+ empty_template = """\
10
+ ## Bias, Risks, and Limitations
11
+
12
+ <!-- This section is meant to convey both technical and sociotechnical limitations. -->
13
+
14
+ [More Information Needed]
15
+
16
+ ### Recommendations
17
+
18
+ <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
19
 
20
+ Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
21
+ """
22
  model_card_template = """\
23
  # Model Card for Sample Model
24
 
 
111
  results = GeneralLimitationsCheck().run_check(card_soup)
112
 
113
  assert results == success_result
114
+
115
+
116
+ def test_fail_on_empty_template():
117
+ model_card_html = markdown.markdown(empty_template)
118
+ card_soup = BeautifulSoup(model_card_html, features="html.parser")
119
+ results = GeneralLimitationsCheck().run_check(card_soup)
120
+ assert results == GeneralLimitationsResult()
tests/test_intended_purpose_check.py CHANGED
@@ -6,15 +6,17 @@ from compliance_checks.intended_purpose import (
6
  IntendedPurposeCheck, IntendedPurposeResult,
7
  )
8
 
9
-
10
- model_card_template = """\
11
  ## Uses
12
 
13
  <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
 
14
 
15
  ### Direct Use
16
 
17
- Here is some info about direct uses...
 
 
18
 
19
  ### Downstream Use [optional]
20
 
@@ -25,6 +27,22 @@ Here is some info about direct uses...
25
  ### Out-of-Scope Use
26
 
27
  <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
 
29
  Here is some info about out-of-scope uses...
30
  """
@@ -101,3 +119,10 @@ def test_run_checks(card):
101
  results = IntendedPurposeCheck().run_check(card_soup)
102
 
103
  assert results == success_result
 
 
 
 
 
 
 
 
6
  IntendedPurposeCheck, IntendedPurposeResult,
7
  )
8
 
9
+ empty_template = """\
 
10
  ## Uses
11
 
12
  <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
13
+ [More Information Needed]
14
 
15
  ### Direct Use
16
 
17
+ [More Information Needed]
18
+
19
+ <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
20
 
21
  ### Downstream Use [optional]
22
 
 
27
  ### Out-of-Scope Use
28
 
29
  <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
30
+ [More Information Needed]
31
+ """
32
+ model_card_template = """\
33
+ ## Uses
34
+
35
+ Some info...
36
+
37
+ ### Direct Use
38
+
39
+ Some more info.
40
+
41
+ ### Downstream Use [optional]
42
+
43
+ [More Information Needed]
44
+
45
+ ### Out-of-Scope Use
46
 
47
  Here is some info about out-of-scope uses...
48
  """
 
119
  results = IntendedPurposeCheck().run_check(card_soup)
120
 
121
  assert results == success_result
122
+
123
+
124
+ def test_fail_on_empty_template():
125
+ model_card_html = markdown.markdown(empty_template)
126
+ card_soup = BeautifulSoup(model_card_html, features="html.parser")
127
+ results = IntendedPurposeCheck().run_check(card_soup)
128
+ assert results == IntendedPurposeResult()