datacards/curation.py

import streamlit as st

from .streamlit_utils import make_text_input

from .streamlit_utils import (
    make_multiselect,
    make_selectbox,
    make_text_area,
    make_text_input,
    make_radio,
)

N_FIELDS_ORIGINAL = 4
N_FIELDS_LANGUAGE = 11
N_FIELDS_ANNOTATIONS = 10
N_FIELDS_CONSENT = 4
N_FIELDS_PII = 7
N_FIELDS_MAINTENANCE = 6

N_FIELDS = (
    N_FIELDS_ORIGINAL
    + N_FIELDS_LANGUAGE
    + N_FIELDS_ANNOTATIONS
    + N_FIELDS_CONSENT
    + N_FIELDS_PII
    + N_FIELDS_MAINTENANCE
)


def curation_page():
    st.session_state.card_dict["curation"] = st.session_state.card_dict.get(
        "curation", {}
    )
    with st.expander("Original Curation", expanded=False):
        key_pref = ["curation", "original"]
        st.session_state.card_dict["curation"]["original"] = st.session_state.card_dict[
            "curation"
        ].get("original", {})
        make_text_area(
            label="Original curation rationale",
            key_list=key_pref + ["rationale"],
            help="Describe the curation rationale behind the original dataset(s).",
        )
        make_text_area(
            label="What was the communicative goal?",
            key_list=key_pref + ["communicative"],
            help="Describe the communicative goal that the original dataset(s) was trying to represent.",
        )
        make_radio(
            label="Is the dataset aggregated from different data sources?",
            options=["no", "yes"],
            key_list=key_pref + ["is-aggregated"],
            help="e.g. Wikipedia, movi dialogues, etc.",
        )
        if st.session_state.card_dict["curation"]["original"]["is-aggregated"] == "yes":
            make_text_area(
                label="List the sources (one per line)",
                key_list=key_pref + ["aggregated-sources"],
                help="One source per line",
            )
        else:
            st.session_state.card_dict["curation"]["original"]["aggregated-sources"] = "N/A"

    with st.expander("Language Data", expanded=False):
        key_pref = ["curation", "language"]
        st.session_state.card_dict["curation"]["language"] = st.session_state.card_dict[
            "curation"
        ].get("language", {})
        make_multiselect(
            label="How was the language data obtained?",
            options=[
                "Found",
                "Created for the dataset",
                "Crowdsourced",
                "Machine-generated",
                "Other",
            ],
            key_list=key_pref + ["obtained"],
        )
        if "Found" in st.session_state.card_dict["curation"]["language"].get("obtained", []):
            make_multiselect(
                label="If found, where from?",
                options=["Multiple websites", "Single website", "Offline media collection", "Other"],
                key_list=key_pref + ["found"],
                help="select N/A if none of the language data was found",
            )
        else:
            st.session_state.card_dict["curation"]["language"]["found"] = []
        if "Crowdsourced" in st.session_state.card_dict["curation"]["language"].get("obtained", []):
            make_multiselect(
                label="If crowdsourced, where from?",
                options=[
                    "Amazon Mechanical Turk",
                    "Other crowdworker platform",
                    "Participatory experiment",
                    "Other",
                ],
                key_list=key_pref + ["crowdsourced"],
                help="select N/A if none of the language data was crowdsourced",
            )
        else:
            st.session_state.card_dict["curation"]["language"]["crowdsourced"] = []
        if "Created for the dataset" in st.session_state.card_dict["curation"]["language"].get("obtained", []):
            make_text_area(
                label="If created for the dataset, describe the creation process.",
                key_list=key_pref + ["created"],
            )
        else:
            st.session_state.card_dict["curation"]["language"]["created"] = "N/A"
        if "Machine-generated" in st.session_state.card_dict["curation"]["language"].get("obtained", []):
            make_text_input(
                label="If text was machine-generated for the dataset, provide a link to the generation method if available (N/A otherwise).",
                key_list=key_pref + ["machine-generated"],
                help="if the generation code is unavailable, enter N/A",
            )
        else:
            st.session_state.card_dict["curation"]["language"]["machine-generated"] = "N/A"
        make_text_area(
            label="What further information do we have on the language producers?",
            key_list=key_pref + ["producers-description"],
            help="Provide a description of the context in which the language was produced and who produced it.",
        )
        make_text_area(
            label="Does the language in the dataset focus on specific topics? How would you describe them?",
            key_list=key_pref + ["topics"],
            help="for example, tourism, entertainment, etc.",
        )
        make_selectbox(
            label="Was the text validated by a different worker or a data curator?",
            options=[
                "not validated",
                "validated by crowdworker",
                "validated by data curator",
                "other",
            ],
            key_list=key_pref + ["validated"],
            help="this question is about human or human-in-the-loop validation only",
        )
        make_text_area(
            label="How was the text data pre-processed? (Enter N/A if the text was not pre-processed)",
            key_list=key_pref + ["pre-processed"],
            help="List the steps in preprocessing the data for the dataset. Enter N/A if no steps were taken.",
        )
        make_selectbox(
            label="Were text instances selected or filtered?",
            options=["not filtered", "manually", "algorithmically", "hybrid"],
            key_list=key_pref + ["is-filtered"],
        )
        if st.session_state.card_dict["curation"]["language"]["is-filtered"] == "not filtered":
            st.session_state.card_dict["curation"]["language"]["filtered-criteria"] = "N/A"
        else:
            make_text_area(
                label="What were the selection criteria?",
                key_list=key_pref + ["filtered-criteria"],
                help="Describe the process for selecting instances to include in the dataset, including any tools used.",
            )

    with st.expander("Structured Annotations", expanded=False):
        key_pref = ["curation", "annotations"]
        st.session_state.card_dict["curation"][
            "annotations"
        ] = st.session_state.card_dict["curation"].get("annotations", {})

        make_selectbox(
            label="Does the dataset have additional annotations for each instance?",
            options=["none", "found", "automatically created", "expert created", "crowd-sourced"],
            key_list=key_pref + ["origin"],
            help="Was any additional data collected?",
        )

        # If expert or crowdsourced, this branch
        if st.session_state.card_dict["curation"]["annotations"]["origin"] in ["expert created", "crowd-sourced"]:
            make_selectbox(
                label="What is the number of raters?",
                options=["unknown", "1", "2<n<10", "11<n<50", "51<n<100", "n>100"],
                key_list=key_pref + ["rater-number"],
                help="How many raters were used to create the additional annotations?",
            )
            make_text_area(
                label="Describe the qualifications required of an annotator.",
                key_list=key_pref + ["rater-qualifications"],
                help="e.g., languages or dialects they speak, education requirements, number of HITs (if MTurk).",
            )
            make_selectbox(
                label="How many annotators saw each training example?",
                options=["0", "1", "2", "3", "4", "5", ">5"],
                key_list=key_pref + ["rater-training-num"],
                help="",
            )
            make_selectbox(
                label="How many annotators saw each test example?",
                options=["0", "1", "2", "3", "4", "5", ">5"],
                key_list=key_pref + ["rater-test-num"],
                help="",
            )
            make_radio(
                label="Was an annotation service used?",
                options=["no", "yes", "unknown"],
                key_list=key_pref + ["rater-annotation-service-bool"],
                help="",
            )
            if st.session_state.card_dict["curation"]["annotations"]["rater-annotation-service-bool"] == "yes":
                make_multiselect(
                    label="Which annotation services were used?",
                    options=[
                        "Amazon Mechanical Turk", "Prolific Academic",
                        "Upwork", "Appen", "Crowdflower", "other"
                    ],
                    key_list=key_pref + ["rater-annotation-service"],
                )
            else:
                st.session_state.card_dict["curation"]["annotations"]["rater-annotation-service"] = []
        else:
            st.session_state.card_dict["curation"]["annotations"]["rater-number"] = "N/A"
            st.session_state.card_dict["curation"]["annotations"]["rater-qualifications"] = "N/A"
            st.session_state.card_dict["curation"]["annotations"]["rater-training-num"] = "N/A"
            st.session_state.card_dict["curation"]["annotations"]["rater-test-num"] = "N/A"
            st.session_state.card_dict["curation"]["annotations"]["rater-annotation-service-bool"] = "no"
            st.session_state.card_dict["curation"]["annotations"]["rater-annotation-service"] = []

        if st.session_state.card_dict["curation"]["annotations"]["origin"] != "none":
            make_text_area(
                label="Purpose and values for each annoation",
                key_list=key_pref + ["values"],
                help="Describe the purpose and possible values for each kind of annotation.",
            )
            make_selectbox(
                label="Quality control measures?",
                options=["none", "unknown",  "validated by another rater", "validated by data curators", "validated through automated script", "other"],
                key_list=key_pref + ["quality-control"],
                help="How was annotation quality controlled for / what control measures were put in place to ensure annotation quality?",
            )
            if st.session_state.card_dict["curation"]["annotations"]["quality-control"] in ["none", "unknown"]:
                st.session_state.card_dict["curation"]["annotations"]["quality-control-details"] = "N/A"
            else:
                make_text_area(
                    label="Describe the quality control measures that were taken.",
                    key_list=key_pref + ["quality-control-details"],
                    help="Describe how quality was ensured in the data curation process.",
                )
        else:
            st.session_state.card_dict["curation"]["annotations"]["values"] = "N/A"
            st.session_state.card_dict["curation"]["annotations"]["quality-control"] = []
            st.session_state.card_dict["curation"]["annotations"]["quality-control-details"] = "N/A"


    with st.expander("Consent", expanded=False):
        key_pref = ["curation", "consent"]
        st.session_state.card_dict["curation"]["consent"] = st.session_state.card_dict[
            "curation"
        ].get("consent", {})
        make_radio(
            label="Was there a consent policy involved when gathering the data?",
            options=["no", "yes"],
            key_list=key_pref+["has-consent"],
        )
        if st.session_state.card_dict["curation"]["consent"]["has-consent"] == "yes":
            make_text_area(
                label="What was the consent policy?",
                key_list=key_pref+["consent-policy"],
                help="If available, provide the text that data creators were shown, else, describe the process.",
            )
            make_text_area(
                label="What other downstream uses of the data did the original data creators and the data curators consent to?",
                key_list=key_pref+["consent-other"],
            )
            st.session_state.card_dict["curation"]["consent"]["no-consent-justification"] = "N/A"
        else:
            st.session_state.card_dict["curation"]["consent"]["consent-policy"] = "N/A"
            st.session_state.card_dict["curation"]["consent"]["consent-other"] = "N/A"
            make_text_area(
                label="If not, what is the justification for reusing the data?",
                key_list=key_pref+["no-consent-justification"],
                help="Why would be a justification the data without consent of the data creators in this case?",
            )

    with st.expander("Private Identifying Information (PII)", expanded=False):
        key_pref = ["curation", "pii"]
        st.session_state.card_dict["curation"]["pii"] = st.session_state.card_dict[
            "curation"
        ].get("pii", {})
        make_radio(
            label="Does the source language data likely contain Personal Identifying Information about the data creators or subjects?",
            options=["yes/very likely", "likely", "unlikely", "no PII"],
            key_list=key_pref+["has-pii"],
            help="most datasets have some form of PII: names, addresses, emails, account names, personal beliefs, gender, etc. - select `no PII` only if sure",
        )
        if st.session_state.card_dict["curation"]["pii"]["has-pii"] == "no PII":
            make_text_area(
                label="Provide a justification for selecting `no PII` above.",
                key_list=key_pref+["no-pii-justification"],
                help="for example, if the text is about general knowledge without references to the author or to any persons.",
            )
            st.session_state.card_dict["curation"]["pii"]["pii-categories"] = []
            st.session_state.card_dict["curation"]["pii"]["is-pii-identified"] = "N/A"
            st.session_state.card_dict["curation"]["pii"]["pii-identified-method"] = "N/A"
            st.session_state.card_dict["curation"]["pii"]["is-pii-replaced"] = "N/A"
            st.session_state.card_dict["curation"]["pii"]["pii-replaced-method"] = "N/A"
        else:
            st.session_state.card_dict["curation"]["pii"]["no-pii-justification"] = "N/A"
            pii_help_text = """
                - Personally identifying general information includes names, physical and email addresses, website accounts with names or handles, dates (birth, death, etc.), full-face photographs and comparable images, URLS, and biometric identifiers (fingerprints, voice, etc.).
                - Personally identifying numbers include information such as telephone numbers, fax numbers, vehicle and device identifiers and serial numbers, social security numbers and equivalent, IP addresses, medical record numbers, health plan beneficiary numbers, account numbers, certificate/license numbers, and any other uniquely identifying numbers.
                - Sensitive information includes descriptions of racial or ethnic origin, political opinions, religious or philosophical beliefs, trade-union membership, genetic data, health-related data, and data concerning a person's sex life or sexual orientation.
            """
            make_multiselect(
                label="What categories of PII are present or suspected in the data?",
                options=["generic PII", "numeric PII", "sensitive information"],
                key_list=key_pref+["pii-categories"],
                help=pii_help_text,
            )
            make_radio(
                label="Did the curators use any automatic/manual method to identify PII in the dataset?",
                options=["no identification", "manual identification", "automatic identification", "mixed method"],
                key_list=key_pref+["is-pii-identified"],
            )
            if st.session_state.card_dict["curation"]["pii"]["is-pii-identified"] == "no identification":
                st.session_state.card_dict["curation"]["pii"]["pii-identified-method"] = "N/A"
                st.session_state.card_dict["curation"]["pii"]["is-pii-replaced"] = "N/A"
                st.session_state.card_dict["curation"]["pii"]["pii-replaced-method"] = "N/A"
            else:
                make_text_area(
                    label="Describe the method used to identify PII in the dataset",
                    key_list=key_pref+["pii-identified-method"],
                )
                make_radio(
                    label="Was the PII pseudonymized/handled somehow?",
                    options=["no", "yes"],
                    key_list=key_pref+["is-pii-replaced"],
                )
                if st.session_state.card_dict["curation"]["pii"]["is-pii-replaced"] == "yes":
                    make_text_area(
                        label="Describe the methods that were used to process the PII.",
                        key_list=key_pref+["pii-replaced-method"],
                    )
                else:
                    st.session_state.card_dict["curation"]["pii"]["pii-replaced-method"] = "N/A"

    with st.expander("Maintenance", expanded=False):
        key_pref = ["curation", "maintenance"]
        st.session_state.card_dict["curation"][
            "maintenance"
        ] = st.session_state.card_dict["curation"].get("maintenance", {})
        make_radio(
            label="Does the original dataset have a maintenance plan?",
            options=["no", "yes"],
            key_list=key_pref+["has-maintenance"],
            help="this can include planned update or a commitment to removing content on request",
        )
        if st.session_state.card_dict["curation"]["maintenance"]["has-maintenance"] == "yes":
            make_text_area(
                label="Describe the original dataset's maintenance plan.",
                key_list=key_pref+["description"],
            )
            make_text_area(
                label="Provide contact information of a person responsible for the dataset maintenance",
                key_list=key_pref+["contact"],
            )
            make_radio(
                label="Does the maintenance plan include a contestation mechanism allowing individuals to request removal fo content?",
                options=["no mechanism", "form submission", "contact maintainer", "other"],
                key_list=key_pref+["contestation-mechanism"],
            )
            if st.session_state.card_dict["curation"]["maintenance"]["contestation-mechanism"] == "no mechanism":
                st.session_state.card_dict["curation"]["maintenance"]["contestation-link"] = "N/A"
                st.session_state.card_dict["curation"]["maintenance"]["contestation-description"] = "N/A"
            elif st.session_state.card_dict["curation"]["maintenance"]["contestation-mechanism"] == "other":
                st.session_state.card_dict["curation"]["maintenance"]["contestation-link"] = "N/A"
                make_text_area(
                    label="Describe the contestation mechanism",
                    key_list=key_pref+["contestation-description"],
                )
            else:
                make_text_input(
                    label="Provide the form link or contact information",
                    key_list=key_pref+["contestation-link"],
                )
                st.session_state.card_dict["curation"]["maintenance"]["contestation-description"] = "N/A"
        else:
            st.session_state.card_dict["curation"]["maintenance"]["description"] = "N/A"
            st.session_state.card_dict["curation"]["maintenance"]["contact"] = "N/A"
            st.session_state.card_dict["curation"]["maintenance"]["contestation-mechanism"] = "N/A"
            st.session_state.card_dict["curation"]["maintenance"]["contestation-link"] = "N/A"
            st.session_state.card_dict["curation"]["maintenance"]["contestation-description"] = "N/A"


def curation_summary():
    total_filled = sum(
        [len(dct) for dct in st.session_state.card_dict.get("curation", {}).values()]
    )
    with st.expander(
        f"Dataset Curation Completion - {total_filled} of {N_FIELDS}", expanded=False
    ):
        completion_markdown = ""
        completion_markdown += (
            f"- **Overall completion:**\n  - {total_filled} of {N_FIELDS} fields\n"
        )
        completion_markdown += f"- **Sub-section - Original Curation:**\n  - {len(st.session_state.card_dict.get('curation', {}).get('original', {}))} of {N_FIELDS_ORIGINAL} fields\n"
        completion_markdown += f"- **Sub-section - Language Data:**\n  - {len(st.session_state.card_dict.get('curation', {}).get('language', {}))} of {N_FIELDS_LANGUAGE} fields\n"
        completion_markdown += f"- **Sub-section - Structured Annotations:**\n  - {len(st.session_state.card_dict.get('curation', {}).get('annotations', {}))} of {N_FIELDS_ANNOTATIONS} fields\n"
        completion_markdown += f"- **Sub-section - Consent:**\n  - {len(st.session_state.card_dict.get('curation', {}).get('consent', {}))} of {N_FIELDS_CONSENT} fields\n"
        completion_markdown += f"- **Sub-section - PII:**\n  - {len(st.session_state.card_dict.get('curation', {}).get('pii', {}))} of {N_FIELDS_PII} fields\n"
        completion_markdown += f"- **Sub-section - Maintenance:**\n  - {len(st.session_state.card_dict.get('curation', {}).get('maintenance', {}))} of {N_FIELDS_MAINTENANCE} fields\n"
        st.markdown(completion_markdown)