File size: 2,629 Bytes
e814211 f5bf147 e814211 f5bf147 e814211 f5bf147 e814211 f5bf147 e814211 f5bf147 e814211 f5bf147 e814211 f5bf147 e814211 f5bf147 e814211 f5bf147 e814211 f5bf147 e814211 f5bf147 e814211 f5bf147 e814211 f5bf147 e814211 f5bf147 e814211 f5bf147 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 |
import pytest
import markdown
from bs4 import BeautifulSoup
from compliance_checks import (
GeneralLimitationsCheck, GeneralLimitationsResult,
)
model_card_template = """\
# Model Card for Sample Model
## Bias, Risks, and Limitations
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
Hello world! These are some risks...
"""
albert_base_v2 = """\
# ALBERT Base v2
## Intended uses & limitations
You can use the raw model for either masked language modeling or next sentence prediction, but it's mostly intended to
be fine-tuned on a downstream task.
"""
distilbert_base_cased_distilled_squad = """\
# DistilBERT base cased distilled SQuAD
## Risks, Limitations and Biases
**CONTENT WARNING: Readers should be aware that language generated by this model can be disturbing or offensive to some and can propagate historical and current stereotypes.**
Significant research has explored bias and fairness issues with language models.
"""
gpt2 = """\
# GPT-2
### Limitations and bias
The training data used for this model has not been released as a dataset one can browse.
"""
clip = """\
# Model Card: CLIP
## Limitations
CLIP and our analysis of it have a number of limitations. CLIP currently struggles with respect to certain tasks such as fine grained classification and counting objects.
### Bias and Fairness
We find that the performance of CLIP - and the specific biases it exhibits - can depend significantly on class design and the choices one makes for categories to include and exclude.
"""
runway = """\
# Stable Diffusion v1-5 Model Card
## Limitations and Bias
### Limitations
- The model does not achieve perfect photorealism
### Bias
While the capabilities of image generation models are impressive, they can also reinforce or exacerbate social biases.
"""
distilroberta_base = """\
# Model Card for DistilRoBERTa base
# Bias, Risks, and Limitations
Significant research has explored bias and fairness issues with language models.
"""
bloom = """\
# BLOOM
# Risks and Limitations
*This section identifies foreseeable harms and misunderstandings.*
"""
success_result = GeneralLimitationsResult(
status=True
)
@pytest.mark.parametrize("card", [
model_card_template,
albert_base_v2,
distilbert_base_cased_distilled_squad,
gpt2,
clip,
runway,
distilroberta_base,
bloom,
])
def test_run_checks(card):
model_card_html = markdown.markdown(card)
card_soup = BeautifulSoup(model_card_html, features="html.parser")
results = GeneralLimitationsCheck().run_check(card_soup)
assert results == success_result
|