File size: 3,014 Bytes
e814211 f8c21da e814211 f62b8c4 e814211 f62b8c4 e814211 f62b8c4 e814211 f62b8c4 e814211 f8c21da e814211 f8c21da e814211 f8c21da e814211 f8c21da e814211 f8c21da e814211 f5bf147 f8c21da e814211 f8c21da e814211 f8c21da e814211 f8c21da e814211 f8c21da e814211 f8c21da e814211 f8c21da e814211 f8c21da e814211 f5bf147 5a34b97 f8c21da e814211 f8c21da f5bf147 f8c21da f5bf147 e814211 f8c21da e814211 f8c21da e814211 f8c21da f62b8c4 5a34b97 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 |
import pytest
import markdown
from bs4 import BeautifulSoup
from compliance_checks.intended_purpose import (
IntendedPurposeCheck, IntendedPurposeResult,
)
empty_template = """\
## Uses
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
[More Information Needed]
### Direct Use
[More Information Needed]
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
### Downstream Use [optional]
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
[More Information Needed]
### Out-of-Scope Use
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
[More Information Needed]
"""
model_card_template = """\
## Uses
Some info...
### Direct Use
Some more info.
### Downstream Use [optional]
[More Information Needed]
### Out-of-Scope Use
Here is some info about out-of-scope uses...
"""
albert_base_v2 = """\
# ALBERT Base v2
## Intended uses & limitations
Here is some info about direct uses...
"""
distilbert_base_cased_distilled_squad = """\
# DistilBERT base cased distilled SQuAD
## Uses
This model can be used for question answering.
"""
distilroberta_base = """\
# Model Card for DistilRoBERTa base
# Uses
You can use the raw model for masked language modeling, but it's mostly intended to be fine-tuned on a downstream task.
"""
clip = """\
# Model Card: CLIP
## Model Use
Stuff.
### Intended Use
Stuff.
#### Primary intended uses
Stuff.
### Out-of-Scope Use Cases
Stuff.
"""
sentence_transformers = """\
# all-MiniLM-L6-v2
## Intended uses
Our model is intented to be used as a sentence and short paragraph encoder.
"""
bloom = """\
# BLOOM
## Intended Use
This model is being created in order to enable public research on large language models (LLMs).
"""
bleed_over = """\
## Uses
"""
success_result = IntendedPurposeResult(
status=True
)
@pytest.mark.parametrize("card", [
model_card_template,
albert_base_v2,
distilbert_base_cased_distilled_squad,
distilroberta_base,
clip,
sentence_transformers,
bloom,
])
def test_run_checks(card):
model_card_html = markdown.markdown(card)
card_soup = BeautifulSoup(model_card_html, features="html.parser")
results = IntendedPurposeCheck().run_check(card_soup)
assert results == success_result
def test_fail_on_empty_template():
model_card_html = markdown.markdown(empty_template)
card_soup = BeautifulSoup(model_card_html, features="html.parser")
results = IntendedPurposeCheck().run_check(card_soup)
assert results == IntendedPurposeResult()
def test_fail_on_bleed_over():
model_card_html = markdown.markdown(bleed_over)
card_soup = BeautifulSoup(model_card_html, features="html.parser")
results = IntendedPurposeCheck().run_check(card_soup)
assert results == IntendedPurposeResult()
|