File size: 3,042 Bytes
f5bf147 f62b8c4 f5bf147 013f801 7c50704 f5bf147 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 |
from compliance_checks.base import ComplianceResult, ComplianceCheck, walk_to_next_heading
from bs4 import BeautifulSoup
class GeneralLimitationsResult(ComplianceResult):
name = "General Limitations"
def __init__(
self,
limitations: str = None,
*args,
**kwargs,
):
super().__init__(*args, **kwargs)
self.limitations = limitations
def __eq__(self, other):
if isinstance(other, GeneralLimitationsResult):
if super().__eq__(other):
try:
assert self.limitations == other.limitations
return True
except AssertionError:
return False
else:
return False
def to_string(self):
if self.status:
return """\
It's important for model cards to document the model's general limitations! We found some documentation \
for this in this model card. We look for this by searching for headings that say things like:
- Bias, Risks, and Limitations
- Intended uses & limitations
- Limitations
"""
else:
return """\
We weren't able to find a section in this model card for the model's limitations, but it's easy to \
add one! You can add the following section to the model card and, once you fill in the \
`[More Information Needed]` sections, the "General Limitations" check should pass 🤗
```md
## Bias, Risks, and Limitations
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
[More Information Needed]
### Recommendations
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
```
"""
class GeneralLimitationsCheck(ComplianceCheck):
name = "General Limitations"
def run_check(self, card: BeautifulSoup):
combos = [
("h1", "Bias, Risks, and Limitations"), ("h2", "Bias, Risks, and Limitations"),
("h2", "Intended uses & limitations"),
("h1", "Risks and Limitations"),
("h2", "Risks, Limitations and Biases"),
("h2", "Limitations and Bias"),
("h3", "Limitations and bias"),
("h1", "Limitations"), ("h2", "Limitations"),
("h2", "Performance and Limitations"),
]
for hX, heading in combos:
purpose_check = walk_to_next_heading(card, hX, heading)
if purpose_check:
return GeneralLimitationsResult(
status=True,
)
return GeneralLimitationsResult()
|