NimaBoscarino commited on
Commit
d15cd64
1 Parent(s): 7f9f5e2

Computational requirements check

Browse files
app.py CHANGED
@@ -6,11 +6,13 @@ from compliance_checks import (
6
  ComplianceCheck,
7
  IntendedPurposeCheck,
8
  GeneralLimitationsCheck,
 
9
  )
10
 
11
  checks = [
12
  IntendedPurposeCheck(),
13
  GeneralLimitationsCheck(),
 
14
  ]
15
  suite = ComplianceSuite(checks=checks)
16
 
 
6
  ComplianceCheck,
7
  IntendedPurposeCheck,
8
  GeneralLimitationsCheck,
9
+ ComputationalRequirementsCheck,
10
  )
11
 
12
  checks = [
13
  IntendedPurposeCheck(),
14
  GeneralLimitationsCheck(),
15
+ ComputationalRequirementsCheck(),
16
  ]
17
  suite = ComplianceSuite(checks=checks)
18
 
compliance_checks/__init__.py CHANGED
@@ -10,3 +10,7 @@ from compliance_checks.intended_purpose import (
10
  from compliance_checks.general_limitations import (
11
  GeneralLimitationsCheck, GeneralLimitationsResult,
12
  )
 
 
 
 
 
10
  from compliance_checks.general_limitations import (
11
  GeneralLimitationsCheck, GeneralLimitationsResult,
12
  )
13
+
14
+ from compliance_checks.computational_requirements import (
15
+ ComputationalRequirementsCheck, ComputationalRequirementsResult,
16
+ )
compliance_checks/base.py CHANGED
@@ -2,7 +2,7 @@ from abc import ABC, abstractmethod
2
  from typing import Optional, List
3
 
4
  import markdown
5
- from bs4 import BeautifulSoup, Comment
6
 
7
 
8
  def walk_to_next_heading(card, heading, heading_text) -> bool:
@@ -94,45 +94,6 @@ class ModelProviderIdentityCheck(ComplianceCheck):
94
  return ModelProviderIdentityResult()
95
 
96
 
97
- class ComputationalRequirementsResult(ComplianceResult):
98
- name = "Computational Requirements"
99
-
100
- def __init__(
101
- self,
102
- requirements: str = None,
103
- *args,
104
- **kwargs,
105
- ):
106
- super().__init__(*args, **kwargs)
107
- self.requirements = requirements
108
-
109
- def __eq__(self, other):
110
- if isinstance(other, ComputationalRequirementsResult):
111
- if super().__eq__(other):
112
- try:
113
- assert self.requirements == other.requirements
114
- return True
115
- except AssertionError:
116
- return False
117
- else:
118
- return False
119
-
120
- def to_string(self):
121
- return self.requirements
122
-
123
-
124
- class ComputationalRequirementsCheck(ComplianceCheck):
125
- name = "Computational Requirements"
126
-
127
- def run_check(self, card: BeautifulSoup):
128
- check, content = walk_to_next_heading(card, "h3", "Compute infrastructure")
129
-
130
- return ComputationalRequirementsResult(
131
- status=check,
132
- requirements=content,
133
- )
134
-
135
-
136
  class ComplianceSuite:
137
  def __init__(self, checks):
138
  self.checks = checks
 
2
  from typing import Optional, List
3
 
4
  import markdown
5
+ from bs4 import BeautifulSoup
6
 
7
 
8
  def walk_to_next_heading(card, heading, heading_text) -> bool:
 
94
  return ModelProviderIdentityResult()
95
 
96
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
97
  class ComplianceSuite:
98
  def __init__(self, checks):
99
  self.checks = checks
compliance_checks/computational_requirements.py ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from compliance_checks.base import ComplianceResult, ComplianceCheck, walk_to_next_heading
2
+ from bs4 import BeautifulSoup
3
+
4
+
5
+ class ComputationalRequirementsResult(ComplianceResult):
6
+ name = "Computational Requirements"
7
+
8
+ def __init__(
9
+ self,
10
+ requirements: str = None,
11
+ *args,
12
+ **kwargs,
13
+ ):
14
+ super().__init__(*args, **kwargs)
15
+ self.requirements = requirements
16
+
17
+ def __eq__(self, other):
18
+ if isinstance(other, ComputationalRequirementsResult):
19
+ if super().__eq__(other):
20
+ try:
21
+ # TODO: Do I want to do a deep equal?
22
+ # assert self.requirements == other.requirements
23
+ return True
24
+ except AssertionError:
25
+ return False
26
+ else:
27
+ return False
28
+
29
+ def to_string(self):
30
+ return self.requirements
31
+
32
+
33
+ class ComputationalRequirementsCheck(ComplianceCheck):
34
+ name = "Computational Requirements"
35
+
36
+ def run_check(self, card: BeautifulSoup):
37
+ check = walk_to_next_heading(card, "h2", "Technical Specifications")
38
+
39
+ return ComputationalRequirementsResult(
40
+ status=check,
41
+ # requirements=content,
42
+ )
tests/conftest.py CHANGED
@@ -31,9 +31,9 @@ expected_check_results = {
31
  "openai___clip-vit-large-patch14": [True, True, False],
32
  "philschmid___bart-large-cnn-samsum": [False, False, False],
33
  "prajjwal1___bert-tiny": [False, False, False],
34
- "roberta-base": [True, True, True], # For the computational requirements, sort of?
35
- "roberta-large": [True, True, True],
36
- "runwayml___stable-diffusion-v1-5": [True, True, True],
37
  "sentence-transformers___all-MiniLM-L6-v2": [True, False, False],
38
  "StanfordAIMI___stanford-deidentifier-base": [False, False, False],
39
  "t5-base": [True, False, False],
 
31
  "openai___clip-vit-large-patch14": [True, True, False],
32
  "philschmid___bart-large-cnn-samsum": [False, False, False],
33
  "prajjwal1___bert-tiny": [False, False, False],
34
+ "roberta-base": [True, True, False],
35
+ "roberta-large": [True, True, False],
36
+ "runwayml___stable-diffusion-v1-5": [True, True, False],
37
  "sentence-transformers___all-MiniLM-L6-v2": [True, False, False],
38
  "StanfordAIMI___stanford-deidentifier-base": [False, False, False],
39
  "t5-base": [True, False, False],
tests/test_compliance_checks.py CHANGED
@@ -5,6 +5,7 @@ from compliance_checks import (
5
  ComplianceSuite,
6
  IntendedPurposeCheck,
7
  GeneralLimitationsCheck,
 
8
  )
9
 
10
 
@@ -56,13 +57,10 @@ class TestComplianceSuite:
56
 
57
 
58
  def test_end_to_end_compliance_suite(real_model_card, expected_check_results):
59
- # TODO: TEMP
60
- expected_check_results = expected_check_results[:2]
61
-
62
  suite = ComplianceSuite(checks=[
63
  IntendedPurposeCheck(),
64
  GeneralLimitationsCheck(),
65
- # ComputationalRequirementsCheck()
66
  ])
67
 
68
  results = suite.run(real_model_card)
 
5
  ComplianceSuite,
6
  IntendedPurposeCheck,
7
  GeneralLimitationsCheck,
8
+ ComputationalRequirementsCheck,
9
  )
10
 
11
 
 
57
 
58
 
59
  def test_end_to_end_compliance_suite(real_model_card, expected_check_results):
 
 
 
60
  suite = ComplianceSuite(checks=[
61
  IntendedPurposeCheck(),
62
  GeneralLimitationsCheck(),
63
+ ComputationalRequirementsCheck()
64
  ])
65
 
66
  results = suite.run(real_model_card)
tests/test_computational_requirements_check.py CHANGED
@@ -7,23 +7,9 @@ from compliance_checks import (
7
  )
8
 
9
 
10
- expected_infrastructure = """\
11
- Jean Zay Public Supercomputer, provided by the French government.\
12
- Hardware\
13
- 384 A100 80GB GPUs (48 nodes)\
14
- Software\
15
- Megatron-DeepSpeed (Github link)\
16
- """
17
-
18
-
19
- @pytest.fixture
20
- def computational_requirements_model_card():
21
- # Adapted from: https://huggingface.co/bigscience/bloom/blob/main/README.md
22
- return """
23
  # Model Card for Sample Model
24
 
25
- ## Some Random Header
26
-
27
  ## Technical Specifications
28
 
29
  ### Compute infrastructure
@@ -37,45 +23,21 @@ Jean Zay Public Supercomputer, provided by the French government.
37
 
38
  * Megatron-DeepSpeed ([Github link](https://github.com/bigscience-workshop/Megatron-DeepSpeed))
39
  </details>
40
-
41
- ## Intended Use
42
-
43
- Etc..
44
  """
45
 
46
 
47
- @pytest.fixture
48
- def bad_computational_requirements_model_card():
49
- # Adapted from: https://huggingface.co/bigscience/bloom/blob/main/README.md
50
- return """
51
- # Model Card for Sample Model
52
-
53
- ## Some Random Header
54
-
55
- ## Technical Specifications
56
-
57
- ### Compute infrastructure
58
- [More Information Needed]
59
-
60
- ## Intended Use
61
-
62
- Etc..
63
- """
64
 
65
 
66
- @pytest.mark.parametrize("check,card,expected", [
67
- (ComputationalRequirementsCheck(), "computational_requirements_model_card", ComputationalRequirementsResult(
68
- status=True,
69
- requirements=expected_infrastructure,
70
- )),
71
- (ComputationalRequirementsCheck(), "bad_computational_requirements_model_card", ComputationalRequirementsResult()),
72
  ])
73
- def test_run_checks(check, card, expected, request):
74
- card = request.getfixturevalue(card)
75
-
76
  model_card_html = markdown.markdown(card)
77
  card_soup = BeautifulSoup(model_card_html, features="html.parser")
78
 
79
- results = check.run_check(card_soup)
80
 
81
- assert results == expected
 
7
  )
8
 
9
 
10
+ model_card_template = """\
 
 
 
 
 
 
 
 
 
 
 
 
11
  # Model Card for Sample Model
12
 
 
 
13
  ## Technical Specifications
14
 
15
  ### Compute infrastructure
 
23
 
24
  * Megatron-DeepSpeed ([Github link](https://github.com/bigscience-workshop/Megatron-DeepSpeed))
25
  </details>
 
 
 
 
26
  """
27
 
28
 
29
+ success_result = ComputationalRequirementsResult(
30
+ status=True
31
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
32
 
33
 
34
+ @pytest.mark.parametrize("card", [
35
+ model_card_template,
 
 
 
 
36
  ])
37
+ def test_run_checks(card):
 
 
38
  model_card_html = markdown.markdown(card)
39
  card_soup = BeautifulSoup(model_card_html, features="html.parser")
40
 
41
+ results = ComputationalRequirementsCheck().run_check(card_soup)
42
 
43
+ assert results == success_result