Spaces:
Sleeping
Sleeping
| import json | |
| import os | |
| from dataclasses import dataclass, field | |
| from typing import List, Optional, Dict | |
| import pandas as pd | |
| import streamlit as st | |
| from huggingface_hub import HfFileSystem | |
| import streamlit.components.v1 as components | |
| class Field: | |
| type: str | |
| title: str | |
| name: str = None | |
| mandatory: bool = True | |
| # if value of field is in the list of those values, makes following siblings mandatory | |
| following_mandatory_values: list = False | |
| help: Optional[str] = None | |
| children: Optional[List['Field']] = None | |
| other_params: Optional[Dict[str, object]] = field(default_factory=lambda: {}) | |
| # Function to get user ID from URL | |
| def get_user_id_from_url(): | |
| user_id = st.query_params.get("user_id", "") | |
| return user_id | |
| HF_TOKEN = os.environ.get("HF_TOKEN_WRITE") | |
| print("is none?", HF_TOKEN is None) | |
| hf_fs = HfFileSystem(token=HF_TOKEN) | |
| ######################################################################################## | |
| # CHANGE THE FOLLOWING VARIABLES ACCORDING TO YOUR NEEDS | |
| input_repo_path = 'datasets/ijundi/mod-gen-eval-test' | |
| output_repo_path = 'datasets/ijundi/mod-gen-eval-test' | |
| to_annotate_file_name = 'pilot_annotation_data.csv' # CSV file to annotate | |
| COLS_TO_SAVE = ['id', 'comment id', 'type'] | |
| agreement_labels = ['strongly disagree', 'disagree', 'neither agree no disagree', 'agree', 'strongly agree'] | |
| quality_labels = ['very poor', 'poor', 'acceptable', 'good', 'very good'] | |
| default_labels = agreement_labels | |
| # consent_text = 'By taking part in this study you agree that you read all the details in [the consent form](https://someurl.com) and that you give your consent to participate in the study.' | |
| consent_text = consent_text = ''' | |
| ## Consent Form | |
| You will be asked to take part in a research study. Before you decide to take part in this study, it is important that you understand why the study is being done and what it involves. Please read the following information carefully. | |
| ________________________________________________________________________________________ | |
| Project title: Moderation Properties\\ | |
| Researchers: I. Jundi, E.M. Vecchi, N. Falk, G. Lapesa\\ | |
| Institute: Institute for Machine Speech Processing (IMS)\\ | |
| University: University of Stuttgart\\ | |
| Contact: iman.jundi@ims.uni-stuttgart.de | |
| _________________________________________________________________________________________ | |
| ### Description of the research study | |
| In this study, we investigate the various properties of online moderation that aims to facilitate a productive discussion. The annotators' task is to evaluate and score various properties of a reply comment. | |
| The intended use of the results of this study includes an analysis as well as processed versions of the collected data in the context of a publicly available scientific publication. | |
| **Time required:** Your participation will take up to an estimated 1 hour. The time required may vary on an individual basis. | |
| **Risks and benefits:** The risks to your participation in this online survey are those associated with basic computer tasks, including boredom, fatigue, mild stress, or breach of confidentiality. Some of the topics discussed in the online posts to be annotated may include violence, suicide or rape. The only benefit to you is the learning experience from participating in a research study. The benefit to society is the contribution to scientific knowledge | |
| **Compensation:** You will be compensated for participating in this study. If you are interested, we will also be more than happy to share more information about our research with you. | |
| **Voluntary participation:** Your participation in this study is voluntary. It is your decision whether or not to participate in this study. If you decide to participate in this study, you will be asked to confirm this consent form ("I agree."). Even after signing the consent form, you can withdraw from participation at any time and without giving any reason. Partial data will not be analysed. | |
| **Confidentiality:** Your responses to this experiment will be anonymous. Please do not share any | |
| Information that can be used to identify you. The researcher(s) will make every effort to maintain your confidentiality. | |
| **Contact:** If at any time you have questions about this study or would like to report any adverse effects due to this study, please contact the researcher(s). | |
| **Trigger Warning:** The texts included in this study are produced in an online debate forum and some topics that are discussed, how they are discussed, and user perspectives may be uncomfortable or sensitive. First, all texts included here do not represent the views of the researchers conducting the study. Secondly, we provide the option [described in detail in the guidelines provided in the next step] to avoid having to annotate any instance that is problematic or uncomfortable for the annotator without penalty of compensation. | |
| ### Consent: | |
| Please indicate, in the box below, that you are at least 18 years old, have read and understood this consent form, are comfortable using the English language to complete the survey, and you agree to participate in this online research survey. | |
| - *I am age 18 or older.* | |
| - *I have read this consent form or had it read to me.* | |
| - *I am comfortable using the English language to participate in this survey.* | |
| - *I agree to participate in this research and I want to continue with the survey.* | |
| ''' | |
| guidelines_text = 'Please read <a href="https://someurl.com">the guidelines</a>' | |
| study_code = 'CE552C7F' | |
| redirect_url = f'https://app.prolific.com/submissions/complete?cc={study_code}' | |
| intro_fields: List[Field] = [ | |
| Field(name="intro_user_goals", type="text", title="As a user, what are your goals/objectives when commenting about a topic?"), | |
| Field(name="intro_experience", type="text", title="What do you feel contributes to a good experience for the users/discussion?"), | |
| Field(name="intro_valuable_comment", type="text", title="What makes a comment or contribution valuable?"), | |
| Field(name="intro_bad_comment", type="text", title="What makes a comment or contribution of poor quality, unconstructive or detrimental to the discussion?"), | |
| Field(name="intro_moderation_goals", type="text", title="As a user, what properties does a helpful moderator comment has?"), | |
| ] | |
| fields: List[Field] = [ | |
| Field(name="id", type="input_col", title="**Sample ID:**"), | |
| Field(name="type", type="input_col", title="**Topic:**"), | |
| Field(name="comment parent content", type="input_col", title="**Comment:**"), | |
| Field(name="comment content", type="input_col", title="**Reply:**"), | |
| # FieldDict(name="generated content", type="input_col", title="**Generated:**"), | |
| Field(type="container", title="", children=[ | |
| Field(name="reply_fitting", type="likert_radio", | |
| title="Is the reply comment **fitting and does it make sense** as a reply to the preceding comment?"), | |
| Field(name="actions_clear", type="likert_radio", | |
| title="**Actions clarity**: Does the reply make it clear what actions need to be done?"), | |
| ]), | |
| # Field(type="expander", | |
| # title="Expand and fill-out this section if you see **issues in the original comment**", | |
| # children=[]), | |
| Field(type="container", title="", children=[ | |
| Field(name="issues_in_comment", type="likert_radio", | |
| title="Do **you see some issues** in the original comment?", mandatory=True, | |
| following_mandatory_values=[1, 2, 3, 4]), | |
| Field(name="moderator_spotted", type="likert_radio", | |
| title="Based on the reply, has the **moderator spotted the issues** in the original comment?", | |
| mandatory=False), | |
| Field(name="reply_addresses_issues", type="likert_radio", | |
| title="How well does the reply **address those issues**?", mandatory=False) | |
| ]), | |
| Field(type="container", title="**Score the following properties of the moderator comment?**", children=[ | |
| Field(name="neutrality", type="likert_radio", title="Neutrality", | |
| help='Remain Neutral on the topic and on the Comment Substance and Commenter’s Viewpoint. The reply shouldn’t give away the opinion of the moderator on the topic or comment. '), | |
| # FieldDict(name="attitude", type="likert_radio", title="Attitude", help=''), | |
| Field(name="clarity", type="likert_radio", title="Clarity", | |
| help="Plain language, simple, clear, avoid overwhelming the user e.g. too many questions"), | |
| Field(name="curiosity", type="likert_radio", title="Curiosity", | |
| help="Moderators should model a spirit of inquiry and a desire to learn from and understand commenter’s experience and views. Try to be interested in the bases upon which each commenter stakes his or her claims and the lines of reasoning that has led each commenter to those particular conclusions."), | |
| # TODO | |
| Field(name="bias", type="likert_radio", title="Bias", | |
| help="Does the reply show some biases towards the commenter? Are there stereotypes or prejudices?"), | |
| Field(name="encouraging", type="likert_radio", title="Encouraging", | |
| help="Welcoming, encouraging and acknowledging. Avoid Evaluative and/or Condescending Responses"), | |
| ]), | |
| Field(type="markdown", title="---"), | |
| Field(name="score", type="likert_radio", | |
| title="**Overall score:** Based on the above, how good is the reply as a moderator comment? ", | |
| other_params={'labels': quality_labels}), | |
| Field(type="container", title="Which **moderator function** does the comment address and how well?", children=[ | |
| Field(name="moderator_function_quality", type="likert_radio", title="Improving quality", | |
| help="""Providing substantive information about the proposed rule\n | |
| Correcting misstatements or clarifying what the agency is looking for\n | |
| Pointing to relevant information in primary documents or other data\n | |
| Pointing out characteristics of effective commenting\n | |
| Asking users to provide more information, factual details, or data to support their statements\n | |
| Asking users to make or consider possible solutions/alternative approaches"""), | |
| Field(name="moderator_function_broadening", type="likert_radio", title="Broadening discussion", | |
| help="""Encouraging users to consider and engage comments of other users\n | |
| Posing a question to the community at large that encourages other users to respond"""), | |
| Field(name="moderator_function_other", type="text", title="Other moderator function(s)", mandatory=False), | |
| ]), | |
| Field(name="other_comments", type="text", title="Further comments: free text", mandatory=False), | |
| ] | |
| INPUT_FIELD_DEFAULT_VALUES = {'slider': 0, | |
| 'select_slider': 0, | |
| 'likert_radio': None, | |
| 'text': '', | |
| 'textarea': '', | |
| 'checkbox': False} | |
| SHOW_HELP_ICON = False | |
| SHOW_VALIDATION_ERROR_MESSAGE = True | |
| ######################################################################################## | |
| def read_data(_path): | |
| with hf_fs.open(input_repo_path + '/' + _path) as f: | |
| return pd.read_csv(f) | |
| def read_saved_data(): | |
| _path = get_path() | |
| if hf_fs.exists(output_repo_path + '/' + _path): | |
| with hf_fs.open(output_repo_path + '/' + _path) as f: | |
| try: | |
| return json.load(f) | |
| except json.JSONDecodeError as e: | |
| print(e) | |
| return None | |
| # Write a remote file | |
| def save_data(data): | |
| hf_fs.mkdir(f"{output_repo_path}/{data['user_id']}") | |
| with hf_fs.open(f"{output_repo_path}/{get_path()}", "w") as f: | |
| f.write(json.dumps(data)) | |
| def get_path(): | |
| return f"{st.session_state.user_id}/{st.session_state.current_index}.json" | |
| #################################### Streamlit App #################################### | |
| # Function to navigate rows | |
| def navigate(index_change): | |
| st.session_state.current_index += index_change | |
| # only works consistently if done before rerun | |
| js = ''' | |
| <script> | |
| var body = window.parent.document.querySelector(".main"); | |
| body.scrollTop = 0; | |
| window.scrollY = 0; | |
| </script> | |
| ''' | |
| st.components.v1.html(js, height=0) | |
| # https://discuss.streamlit.io/t/click-twice-on-button-for-changing-state/45633/2 | |
| # disable text input enter to submit | |
| # https://discuss.streamlit.io/t/text-input-how-to-disable-press-enter-to-apply/14457/6 | |
| components.html( | |
| """ | |
| <script> | |
| const inputs = window.parent.document.querySelectorAll('input'); | |
| inputs.forEach(input => { | |
| input.addEventListener('keydown', function(event) { | |
| if (event.key === 'Enter') { | |
| event.preventDefault(); | |
| } | |
| }); | |
| }); | |
| </script> | |
| """, | |
| height=0 | |
| ) | |
| st.rerun() | |
| def show_field(f: Field, index: int, data_collected): | |
| if f.type not in INPUT_FIELD_DEFAULT_VALUES.keys(): | |
| st.session_state.following_mandatory = False | |
| match f.type: | |
| case 'input_col': | |
| st.write(f.title) | |
| st.write(st.session_state.data.iloc[index][f.name]) | |
| case 'markdown': | |
| st.markdown(f.title) | |
| case 'expander' | 'container': | |
| with (st.expander(f.title) if f.type == 'expander' else st.container(border=True)): | |
| if f.type == 'container': | |
| st.markdown(f.title) | |
| for child in f.children: | |
| show_field(child, index, data_collected) | |
| else: | |
| key = f.name + str(index) | |
| st.session_state.data_inputs_keys.append(f.name) | |
| value = st.session_state[key] if key in st.session_state else \ | |
| (data_collected[f.name] if data_collected else INPUT_FIELD_DEFAULT_VALUES[f.type]) | |
| if not SHOW_HELP_ICON: | |
| f.title = f'**{f.title}**\n\n{f.help}' if f.help else f.title | |
| validation_error = False | |
| if f.mandatory or st.session_state.following_mandatory: | |
| # form is not displayed for first time | |
| if st.session_state.form_displayed == st.session_state.current_index: | |
| if st.session_state[key] == INPUT_FIELD_DEFAULT_VALUES[f.type]: | |
| st.session_state.valid = False | |
| validation_error = True | |
| elif f.following_mandatory_values and st.session_state[key] in f.following_mandatory_values: | |
| st.session_state.following_mandatory = True | |
| f.title += " :red[* required!]" if (validation_error and not SHOW_VALIDATION_ERROR_MESSAGE) else' :red[*]' | |
| f.help = None | |
| match f.type: | |
| case 'checkbox': | |
| st.checkbox(f.title, | |
| key=key, | |
| value=value, help=f.help) | |
| case 'slider': | |
| st.slider(f.title, | |
| min_value=0, max_value=100, step=25, | |
| key=key, | |
| value=value, help=f.help) | |
| case 'select_slider': | |
| labels = default_labels if not f.other_params.get('labels') else f.other_params.get('labels') | |
| st.select_slider(f.title, | |
| options=[0, 25, 50, 75, 100], | |
| format_func=lambda x: labels[x // 25], | |
| key=key, | |
| value=value, help=f.help) | |
| case 'likert_radio': | |
| labels = default_labels if not f.other_params.get('labels') else f.other_params.get('labels') | |
| st.radio(f.title, | |
| options=[0, 1, 2, 3, 4], | |
| format_func=lambda x: labels[x], | |
| key=key, | |
| index=value, help=f.help, horizontal=True) | |
| case 'text': | |
| st.text_input(f.title, key=key, value=value) | |
| case 'textarea': | |
| st.text_area(f.title, key=key, value=value) | |
| if validation_error: | |
| st.error(f"Mandatory field") | |
| def show_fields(fields: List[Field]): | |
| st.session_state.valid = True | |
| index = st.session_state.current_index | |
| data_collected = read_saved_data() | |
| st.session_state.data_inputs_keys = [] | |
| st.session_state.following_mandatory = False | |
| for field in fields: | |
| show_field(field, index, data_collected) | |
| submitted = st.form_submit_button("Submit") | |
| if submitted: | |
| if not st.session_state.valid: | |
| st.error("Please fill in all mandatory fields") | |
| # st.rerun() # filed-out values are not shown otherwise | |
| else: | |
| with st.spinner(text="saving"): | |
| save_data({ | |
| 'user_id': st.session_state.user_id, | |
| 'index': st.session_state.current_index, | |
| **(st.session_state.data.iloc[index][COLS_TO_SAVE].to_dict() if index >= 0 else {}), | |
| **{k: st.session_state[k+str(index)] for k in st.session_state.data_inputs_keys} | |
| }) | |
| st.success("Feedback submitted successfully!") | |
| navigate(1) | |
| st.session_state.form_displayed = st.session_state.current_index | |
| # st.set_page_config(layout='wide') | |
| # Title of the app | |
| st.title("Moderation Properties") | |
| st.markdown( | |
| """<style> | |
| div[data-testid="stMarkdownContainer"] > p { | |
| font-size: 1rem; | |
| } | |
| section.main > div {max-width:60rem} | |
| </style> | |
| """, unsafe_allow_html=True) | |
| # Load the data to annotate | |
| if 'data' not in st.session_state: | |
| st.session_state.data = read_data(to_annotate_file_name) | |
| # Initialize the current index | |
| if 'current_index' not in st.session_state: | |
| st.session_state.current_index = -3 | |
| st.session_state.form_displayed = -3 | |
| def add_validated_submit(fields, message): | |
| st.session_state.form_displayed = st.session_state.current_index | |
| if st.form_submit_button("Submit"): | |
| if all(not x for x in fields): | |
| st.error(message) | |
| else: | |
| navigate(1) | |
| def add_checked_submit(): | |
| check = st.checkbox('I agree', key='consent') | |
| add_validated_submit([check], "Please agree to give your consent to proceed") | |
| def add_annotation_guidelines(): | |
| st.write(f"username is {st.session_state.user_id}") | |
| st.markdown( | |
| "<details open><summary>Annotation Guidelines</summary>" + guidelines_text + "</details>" | |
| , unsafe_allow_html=True) | |
| if st.session_state.current_index == -3: | |
| with st.form("data_form"): | |
| st.markdown(consent_text) | |
| add_checked_submit() | |
| elif st.session_state.current_index == -2: | |
| user_id_from_url = get_user_id_from_url() | |
| if user_id_from_url: | |
| st.session_state.user_id = user_id_from_url | |
| navigate(1) | |
| else: | |
| with st.form("data_form"): | |
| st.session_state.user_id = st.text_input('User ID', value=user_id_from_url) | |
| add_validated_submit([st.session_state.user_id], "Please enter a valid user ID") | |
| elif st.session_state.current_index == -1: | |
| add_annotation_guidelines() | |
| with st.form("intro_form"): | |
| show_fields(intro_fields) | |
| elif st.session_state.current_index < len(st.session_state.data): | |
| add_annotation_guidelines() | |
| with st.form("data_form"+str(st.session_state.current_index)): | |
| show_fields(fields) | |
| else: | |
| st.write(f"Thank you for taking part in this study! [Click here]({redirect_url}) to complete the study.") | |
| # Navigation buttons | |
| if st.session_state.current_index > 0: | |
| if st.button("Previous"): | |
| navigate(-1) | |
| if 0 <= st.session_state.current_index < len(st.session_state.data): | |
| st.write(f"Page {st.session_state.current_index + 1} out of {len(st.session_state.data)}") | |
| st.markdown( | |
| """<style> | |
| div[data-testid="InputInstructions"] { | |
| visibility: hidden; | |
| } | |
| </style>""", unsafe_allow_html=True | |
| ) | |