import json
import os
from dataclasses import dataclass, field
from typing import List, Optional, Dict
from PIL import Image
import pandas as pd
import streamlit as st
from huggingface_hub import HfFileSystem
@dataclass
class Field:
type: str
title: str
name: str = None
help: Optional[str] = None
children: Optional[List['Field']] = None
other_params: Optional[Dict[str, object]] = field(default_factory=lambda: {})
# Function to get user ID from URL
def get_user_id_from_url():
user_id = st.query_params.get("user_id", "")
return user_id
HF_TOKEN = os.environ.get("HF_TOKEN_WRITE")
print("is none?", HF_TOKEN is None)
hf_fs = HfFileSystem(token=HF_TOKEN)
input_repo_path = 'datasets/emvecchi/annotate-pilot'
output_repo_path = 'datasets/emvecchi/annotate-pilot'
to_annotate_file_name = 'to_annotate.csv' # CSV file to annotate
COLS_TO_SAVE = ['comment_id']
agreement_labels = ['strongly disagree', 'disagree', 'neither agree no disagree', 'agree', 'strongly agree']
quality_labels = ['very poor', 'poor', 'acceptable', 'good', 'very good']
priority_labels = ['not selected', 'not a priority', 'low priority', 'neutral', 'moderate priority', 'high priority']
default_labels = agreement_labels
function_choices = ['Broadening Discussion',
'Improving Comment Quality',
'Content Correction',
'Keeping Discussion on Topic',
'Organizing Discussion',
'Policing',
'Resolving Site Use Issues',
'Social Functions',
'Other (please specify)']
property_choices = ['appropriateness',
'clarity',
'constructiveness',
'common good',
'effectiveness',
'emotion',
'impact',
'overall quality',
'proposal',
'Q for justification',
'storytelling',
'rationality',
'reasonableness',
'reciprocity',
'reference',
'respect',
'Other (please specify)']
assistance_choices = ['Expand the breadth of moderator role',
'Reduce my own bias',
#'Assist with recall',
'Avoids missing relevant instances',
'Improve speed of moderation tasks',
'Manage prioritization of comments to consider',
'Visualization of properties narrows down moderator contribution',
'Other (please specify)']
default_choices = function_choices
consent_text = '''
**Consent Form**
You will be asked to take part in a research study. Before you decide to take part in this study, it is important that you understand why the study is being done and what it involves. Please read the following information carefully.
________________________________________________________________________________________
Project title: Forms of User Moderation in Online Discussion
Researchers: N. Falk, E.M. Vecchi, I. Jundi, G. Lapesa
Institute: Institute for Machine Speech Processing (IMS)
University: University of Stuttgart
Contact: neele.falk@ims.uni-stuttgart.de
_________________________________________________________________________________________
Description of the research study
In this study, we investigate how users in online discussions take on certain functions of moderators in deliberation. The annotators' task is to evaluate whether a comment takes on any of the possible moderation functions described. In addition, we want to investigate to what extent such comments differ in style from expert moderators and whether the comment contributes in a constructive way to the discussion.
The intended use of the results of this study includes an analysis as well as processed versions of the collected data in the context of a publicly available scientific publication.
Time required: Your participation will take up to an estimated 1 hour. The time required may vary on an individual basis.
Risks and benefits: The risks to your participation in this online survey are those associated with basic computer tasks, including boredom, fatigue, mild stress, or breach of confidentiality. Some of the topics discussed in the online posts to be annotated may include violence, suicide or rape. The only benefit to you is the learning experience from participating in a research study. The benefit to society is the contribution to scientific knowledge
Compensation: You will be compensated for participating in this study. If you are interested, we will also be more than happy to share more information about our research with you.
Voluntary participation: Your participation in this study is voluntary. It is your decision whether or not to participate in this study. If you decide to participate in this study, you will be asked to confirm this consent form ("I agree."). Even after signing the consent form, you can withdraw from participation at any time and without giving any reason. Partial data will not be analysed.
Confidentiality: Your responses to this experiment will be anonymous. Please do not share any
Information that can be used to identify you. The researcher(s) will make every effort to maintain your confidentiality.
Contact: If at any time you have questions about this study or would like to report any adverse effects due to this study, please contact the researcher(s).
Trigger Warning: The texts included in this study are produced in an online debate forum and some topics that are discussed, how they are discussed, and user perspectives may be uncomfortable or sensitive. First, all texts included here do not represent the views of the researchers conducting the study. Secondly, we provide the option [described in detail in the guidelines provided in the next step] to avoid having to annotate any instance that is problematic or uncomfortable for the annotator without penalty of compensation.
Consent:
Please indicate, in the box below, that you are at least 18 years old, have read and understood this consent form, are comfortable using the English language to complete the survey, and you agree to participate in this online research survey.
I am age 18 or older.
I have read this consent form or had it read to me.
I am comfortable using the English language to participate in this survey.
I agree to participate in this research and I want to continue with the survey.
'''
#By taking part in this study you agree that you read all the details in [the consent form](https://someurl.com) and that you give your consent to participate in the study.'
guidelines_text = 'Please read the guidelines'
study_code = 'some code here'
intro_fields: List[Field] = [
Field(name="intro_moderation_goals", type="text", title="As a moderator, what are your goals/objectives for the comment section? (free text)"),
Field(name="intro_experience", type="text", title="What do you feel contributes to a good experience for the users/discussion?"),
Field(name="intro_valuable_comment", type="text", title="What makes a comment or contribution valuable?"),
Field(name="intro_bad_comment", type="text", title="What makes a comment or contribution of poor quality, unconstructive or detrimental to the discussion?"),
]
fields: List[Field] = [
Field(name="topic", type="input_col", title="**Topic:**"),
Field(type="expander", title="**Preceeding Comment:** *(expand)*", children=[
Field(name="parent_comment", type="input_col", title=""),
]),
#Field(name="parent_comment", type="input_col", title="**Preceeding Comment:**"),
Field(name="comment", type="input_col", title="**Comment:**"),
Field(name="image_name", type="input_col", title=""),# "**Visualization of high contributing properties:**"),
Field(type="container", title="**Need for Moderation**", children=[
Field(name="to_moderate", type="radio",
title="Do feel this comment/discussion would benefit from moderator intervention?"),
Field(name="actions_clear", type="select_slider",
title="With what level of **priority** would you need to interact with this comment?", other_params={'labels': priority_labels}),
]),
Field(type="container", title="**Moderation Function**", children=[
Field(name="mod_function", type="multiselect",
title="What type of moderation function is needed here? *(Multiple selection possible)*"),
Field(name="mod_function_other", type="text", title="*If Other, please specify:*"),
]),
Field(type="container", title="**Contributing properties**", children=[
Field(name="relevant_properties", type="multiselect",
title="Which property(s) is most impactful in your assessment? *(Multiple selection possible)*",
other_params={'choices': property_choices}),
Field(name="relevant_properties_other", type="text", title="*If Other, please specify:*"),
]),
Field(type="container", title="**Moderator Assistance**", children=[
Field(name="helpful", type="radio",
title="If this comment/discussion was flagged to you, would it be helpful in your task of moderation?"),
Field(name="mod_assistance", type="multiselect",
title="If yes, please motivate the benefit it would contribute to the task. *(Multiple selection possible)*",
other_params={'choices': assistance_choices}),
Field(name="mod_assistance_other", type="text", title="*If Other, please specify:*"),
]),
Field(type="container", title="**Other**", children=[
Field(name="other_comments", type="text", title="Please provide any additional comments or information: *(optional)*"),
]),
]
INPUT_FIELD_DEFAULT_VALUES = {'slider': 0,
'text': None,
'textarea': None,
'checkbox': False,
'radio': False,
'select_slider': 0,
'multiselect': None}
SHOW_HELP_ICON = False
def read_data(_path):
with hf_fs.open(input_repo_path + '/' + _path) as f:
return pd.read_csv(f)
def read_saved_data():
_path = get_path()
if hf_fs.exists(output_repo_path + '/' + _path):
with hf_fs.open(output_repo_path + '/' + _path) as f:
try:
return json.load(f)
except json.JSONDecodeError as e:
print(e)
return None
# Write a remote file
def save_data(data):
hf_fs.mkdir(f"{output_repo_path}/{data['user_id']}")
with hf_fs.open(f"{output_repo_path}/{get_path()}", "w") as f:
f.write(json.dumps(data))
def get_path():
return f"{st.session_state.user_id}/{st.session_state.current_index}.json"
def display_image(image_path):
with hf_fs.open(image_path) as f:
img = Image.open(f)
st.image(img, caption='8 most contributing properties', use_column_width=True)
#################################### Streamlit App ####################################
# Function to navigate rows
def navigate(index_change):
st.session_state.current_index += index_change
print(st.session_state.current_index)
# https://discuss.streamlit.io/t/click-twice-on-button-for-changing-state/45633/2
st.rerun()
def show_field(f: Field, index: int, data_collected):
if f.type not in INPUT_FIELD_DEFAULT_VALUES.keys():
match f.type:
case 'input_col':
st.write(f.title)
if f.name == 'image_name':
st.write(f.title)
image_name = st.session_state.data.iloc[index][f.name]
if image_name: # Ensure the image name is not empty
image_path = os.path.join(input_repo_path, 'images', image_name)
display_image(image_path)
else:
st.write(st.session_state.data.iloc[index][f.name])
case 'markdown':
st.markdown(f.title)
case 'expander' | 'container':
with (st.expander(f.title) if f.type == 'expander' else st.container(border=True)):
if f.type == 'container':
st.markdown(f.title)
for child in f.children:
show_field(child, index, data_collected)
else:
key = f.name + str(index)
value = st.session_state.default_values[f.name] = data_collected[f.name] if data_collected else \
INPUT_FIELD_DEFAULT_VALUES[f.type]
if not SHOW_HELP_ICON:
f.title = f'**{f.title}**\n\n{f.help}' if f.help else f.title
f.help = None
match f.type:
case 'checkbox':
st.session_state.data_inputs[f.name] = st.checkbox(f.title,
key=key,
value=value, help=f.help)
case 'radio':
st.session_state.data_inputs[f.name] = st.radio(f.title,
["not selected","yes","no","other"],
key=key,
help=f.help)
case 'slider':
st.session_state.data_inputs[f.name] = st.slider(f.title,
min_value=0, max_value=6, step=1,
key=key,
value=value, help=f.help)
case 'select_slider':
labels = default_labels if not f.other_params.get('labels') else f.other_params.get('labels')
st.session_state.data_inputs[f.name] = st.select_slider(f.title,
options=[0, 20, 40, 60, 80, 100],
format_func=lambda x: labels[x // 20],
key=key,
value=value, help=f.help)
case 'multiselect':
choices = default_choices if not f.other_params.get('choices') else f.other_params.get('choices')
st.session_state.data_inputs[f.name] = st.multiselect(f.title,
options = choices,
key=key, max_selections=3,
help=f.help)
case 'text':
st.session_state.data_inputs[f.name] = st.text_input(f.title, key=key, value=value)
case 'textarea':
st.session_state.data_inputs[f.name] = st.text_area(f.title, key=key, value=value)
def show_fields(fields: List[Field]):
index = st.session_state.current_index
data_collected = read_saved_data()
st.session_state.default_values = {}
st.session_state.data_inputs = {}
for field in fields:
show_field(field, index, data_collected)
submitted = st.form_submit_button("Submit")
if submitted:
with st.spinner(text="saving"):
save_data({
'user_id': st.session_state.user_id,
'index': st.session_state.current_index,
**(st.session_state.data.iloc[index][COLS_TO_SAVE].to_dict() if index >= 0 else {}),
**st.session_state.data_inputs
})
st.success("Feedback submitted successfully!")
navigate(1)
st.set_page_config(layout='wide')
# Title of the app
st.title("Moderator Intervention Prediction")
st.markdown(
"""
""", unsafe_allow_html=True)
#with st.expander(label="Annotation Guidelines", expanded=False):
# st.write('some guidelines here')
# Load the data to annotate
if 'data' not in st.session_state:
st.session_state.data = read_data(to_annotate_file_name)
# Initialize the current index
if 'current_index' not in st.session_state:
st.session_state.current_index = -3
def add_validated_submit(fields, message):
if st.form_submit_button("Submit"):
if all(not x for x in fields):
st.error(message)
else:
navigate(1)
def add_checked_submit():
check = st.checkbox('I agree', key='consent')
add_validated_submit([check], "Please agree to give your consent to proceed")
def add_annotation_guidelines():
st.markdown(
"Annotation Guidelines
"+guidelines_text+"