Sebastian Gehrmann commited on
Commit
ef1f338
1 Parent(s): 4081c39

added structured annotation section

Browse files
Files changed (1) hide show
  1. datacards/curation.py +71 -1
datacards/curation.py CHANGED
@@ -12,7 +12,7 @@ from .streamlit_utils import (
12
 
13
  N_FIELDS_ORIGINAL = 4
14
  N_FIELDS_LANGUAGE = 12
15
- N_FIELDS_ANNOTATIONS = 0
16
  N_FIELDS_CONSENT = 0
17
  N_FIELDS_PII = 0
18
  N_FIELDS_MAINTENANCE = 0
@@ -148,11 +148,81 @@ def curation_page():
148
  key_list=key_pref + ["filtered-criteria"],
149
  help="Describe the process for selecting instances to include in the dataset, including any tools used. If no selection was done, enter N/A.",
150
  )
 
151
  with st.expander("Structured Annotations", expanded=False):
152
  key_pref = ["curation", "annotations"]
153
  st.session_state.card_dict["curation"][
154
  "annotations"
155
  ] = st.session_state.card_dict["curation"].get("annotations", {})
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
156
  with st.expander("Consent", expanded=False):
157
  key_pref = ["curation", "consent"]
158
  st.session_state.card_dict["curation"]["consent"] = st.session_state.card_dict[
 
12
 
13
  N_FIELDS_ORIGINAL = 4
14
  N_FIELDS_LANGUAGE = 12
15
+ N_FIELDS_ANNOTATIONS = 10
16
  N_FIELDS_CONSENT = 0
17
  N_FIELDS_PII = 0
18
  N_FIELDS_MAINTENANCE = 0
 
148
  key_list=key_pref + ["filtered-criteria"],
149
  help="Describe the process for selecting instances to include in the dataset, including any tools used. If no selection was done, enter N/A.",
150
  )
151
+
152
  with st.expander("Structured Annotations", expanded=False):
153
  key_pref = ["curation", "annotations"]
154
  st.session_state.card_dict["curation"][
155
  "annotations"
156
  ] = st.session_state.card_dict["curation"].get("annotations", {})
157
+
158
+ make_radio(
159
+ label="Does the dataset have additional annotations for each instance?",
160
+ options=["none", "found", "automatically created", "expert created", "crowd-sourced"],
161
+ key_list=key_pref + ["origin"],
162
+ help="Was any additional data collected?",
163
+ )
164
+
165
+ # TODO: If yes....
166
+ # If expert or crowdsourced, this branch
167
+ make_radio(
168
+ label="What is the number of raters ",
169
+ options=["unknown", "1", "2<n<10", "11<n<50", "51<n<100", "n>100"],
170
+ key_list=key_pref + ["rater-number"],
171
+ help="How many raters were used to create the additional annotations?",
172
+ )
173
+ make_text_area(
174
+ label="Describe the qualifications required of an annotator.",
175
+ key_list=key_pref + ["rater-qualifications"],
176
+ help="e.g., languages or dialects they speak, education requirements, number of HITs (if MTurk).",
177
+ )
178
+ make_radio(
179
+ label="How many annotators saw each training example?",
180
+ options=["0", "1", "2", "3", "4", "5", ">5"],
181
+ key_list=key_pref + ["rater-training-num"],
182
+ help="",
183
+ )
184
+ make_radio(
185
+ label="How many annotators saw each test example?",
186
+ options=["0", "1", "2", "3", "4", "5", ">5"],
187
+ key_list=key_pref + ["rater-test-num"],
188
+ help="",
189
+ )
190
+ make_radio(
191
+ label="Was an annotation service used?",
192
+ options=["yes", "no", "unknown"],
193
+ key_list=key_pref + ["rater-annotation-service-bool"],
194
+ help="",
195
+ )
196
+ # TODO if yes
197
+ make_multiselect(
198
+ label="Which annotation services were used?",
199
+ options=[
200
+ "Amazon Mechanical Turk", "Prolific Academic",
201
+ "Upwork", "Appen", "Crowdflower", "other"
202
+ ],
203
+ key_list=key_pref + ["rater-annotation-service"],
204
+ )
205
+
206
+
207
+ make_text_area(
208
+ label="Purpose and values for each annoation",
209
+ key_list=key_pref + ["values"],
210
+ help="Describe the purpose and possible values for each kind of annotation.",
211
+ )
212
+ make_multiselect(
213
+ label="Quality control measures?",
214
+ options=["none", "unknown", "validated by another rater", "validated by data curators", "validated through automated script", "other"],
215
+ key_list=key_pref + ["quality-control"],
216
+ help="How was annotation quality controlled for / what control measures were put in place to ensure annotation quality?",
217
+ )
218
+ # TODO: If not none / unknown
219
+ make_text_area(
220
+ label="Describe the quality control measures that were taken.",
221
+ key_list=key_pref + ["quality-control-details"],
222
+ help="Describe how quality was ensured in the data curation process.",
223
+ )
224
+
225
+
226
  with st.expander("Consent", expanded=False):
227
  key_pref = ["curation", "consent"]
228
  st.session_state.card_dict["curation"]["consent"] = st.session_state.card_dict[