|
import time |
|
import random |
|
import logging |
|
import streamlit as st |
|
import pandas as pd |
|
|
|
from dotenv import load_dotenv |
|
|
|
import utils |
|
import db |
|
import modeling |
|
import plots |
|
|
|
def set_if_not_in_session_state(key, value): |
|
"""Helper function to initialize a session state variable if it doesn't exist.""" |
|
if key not in st.session_state: |
|
st.session_state[key] = value |
|
|
|
def initialize(): |
|
"""Initialization function to set up logging, load environment variables, and initialize session state variables.""" |
|
load_dotenv() |
|
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO) |
|
|
|
keys = ['selected_rating', 'collect_data', 'gender_value', 'expert_value', 'show_launch', 'user_id', 'statements', 'current_statement', 'db'] |
|
values = [0, None, None, None, True, random.randint(1, 999_999_999), None, None, None] |
|
|
|
for key, value in zip(keys, values): |
|
set_if_not_in_session_state(key, value) |
|
|
|
connect_to_database() |
|
|
|
def connect_to_database(): |
|
"""Establishes a connection to the database.""" |
|
if st.session_state.db is None: |
|
credentials_dict = db.load_credentials() |
|
connection_attempts = 0 |
|
|
|
while st.session_state.db is None and connection_attempts < 3: |
|
st.session_state.db = db.connect_to_db(credentials_dict) |
|
if st.session_state.db is None: |
|
logging.info('Retrying to connect to db...') |
|
connection_attempts += 1 |
|
time.sleep(1) |
|
else: |
|
retrieve_statements() |
|
|
|
def retrieve_statements(): |
|
"""Retrieves statements from the database.""" |
|
retrieval_attempts = 0 |
|
|
|
while st.session_state.statements is None and retrieval_attempts < 3: |
|
st.session_state.statements = db.get_statements_from_db(st.session_state.db) |
|
st.session_state.current_statement = db.pick_random(st.session_state.statements) |
|
if st.session_state.statements is None: |
|
logging.info('Retrying to retrieve statements from db...') |
|
retrieval_attempts += 1 |
|
time.sleep(1) |
|
|
|
def get_user_consent(): |
|
st.markdown(""" |
|
### Support Future Research |
|
Additionally, we kindly ask for your agreement to collect anonymous data from your app usage in order to improve future research. |
|
You may choose to agree or decline this data collection. |
|
""") |
|
|
|
collect_data_options = ['Yes, I agree and want to support and help improve this research', 'No'] |
|
collect_data_input = st.radio( |
|
label='You may choose to agree or decline this data collection.', |
|
options=collect_data_options, |
|
horizontal=True, |
|
label_visibility='collapsed' |
|
) |
|
return collect_data_options.index(collect_data_input) == 0 |
|
|
|
|
|
def get_user_info(): |
|
gender_options = ['[Please select]', 'Female', 'Male', 'Other'] |
|
gender_input = st.selectbox( |
|
label='Please select your gender', |
|
options=gender_options, |
|
) |
|
gender_value = gender_options.index(gender_input) |
|
|
|
expert_options = [ |
|
'[Please select]', |
|
'No, I do not have a background in social or behavioral sciences', |
|
'Yes, I have either studied social or behavioral sciences or I am currently a student in this field', |
|
'Yes, I have either worked as a researcher in the field of social or behavioral sciences or I have had past experience as a researcher in this area' |
|
] |
|
expert_input = st.selectbox( |
|
label='Please indicate whether you have any experience or educational background in social or behavioral sciences (e.g., psychology)', |
|
options=expert_options, |
|
) |
|
expert_value = expert_options.index(expert_input) |
|
|
|
return expert_value, gender_value |
|
|
|
def get_user_rating(placeholder): |
|
|
|
with placeholder: |
|
with st.container(): |
|
st.markdown(f""" |
|
### How desirable is the following statement? |
|
To support future research, rate the following statement according to whether it is socially desirable or undesirable. |
|
Is it socially desirable or undesirable to endorse the following statement? |
|
#### <center>\"{st.session_state.current_statement.capitalize()}\"</center> |
|
""", unsafe_allow_html=True) |
|
|
|
rating_options = ['[Please select]', 'Very undesirable', 'Undesirable', 'Neutral', 'Desirable', 'Very desirable'] |
|
|
|
selected_rating = st.selectbox( |
|
label='Rate the statement above according to whether it is socially desirable or undesirable.', |
|
options=rating_options, |
|
key='selection' |
|
) |
|
|
|
suitability_options = ['No, I\'m just playing around', 'Yes, my input can help improve this research'] |
|
|
|
research_suitability = st.radio( |
|
label='Is your input suitable for research purposes?', |
|
options=suitability_options, |
|
horizontal=True |
|
) |
|
|
|
st.session_state.collect_data_optout = st.checkbox( |
|
label='Don\'t ask me to rate further statements.', |
|
value=False |
|
) |
|
|
|
st.session_state.item_rating = rating_options.index(selected_rating) |
|
st.session_state.suitability_rating = suitability_options.index(research_suitability) |
|
|
|
def handle_acceptance(collect_data_value, expert_value, gender_value, message): |
|
if st.button(label='Accept Disclaimer', type='primary', use_container_width=True): |
|
if collect_data_value and not (expert_value > 0 and gender_value > 0): |
|
message.error('Please answer the questions above!') |
|
else: |
|
st.session_state.expert_value = expert_value |
|
st.session_state.gender_value = gender_value |
|
st.session_state.show_launch = False |
|
st.session_state.collect_data = collect_data_value |
|
st.experimental_rerun() |
|
|
|
def show_launch(placeholder): |
|
with placeholder: |
|
with st.container(): |
|
st.divider() |
|
st.markdown(""" |
|
## Before Using the App |
|
### Disclaimer |
|
This application is provided as-is, without any warranty or guarantee of any kind, expressed or implied. It is intended for educational, non-commercial use only. |
|
The developers of this app shall not be held liable for any damages or losses incurred from its use. By using this application, you agree to the terms and conditions |
|
outlined herein and acknowledge that any commercial use or reliance on its functionality is strictly prohibited. |
|
""", unsafe_allow_html=True) |
|
|
|
collect_data_value = False |
|
if st.session_state.db: |
|
collect_data_value = get_user_consent() |
|
|
|
expert_value, gender_value = (0, 0) |
|
if collect_data_value: |
|
expert_value, gender_value = get_user_info() |
|
|
|
message = st.empty() |
|
|
|
handle_acceptance(collect_data_value, expert_value, gender_value, message) |
|
|
|
def show_summary(placeholder): |
|
with placeholder: |
|
with st.container(): |
|
st.markdown(""" |
|
## What is the focus of this research? |
|
Certain biases can affect how people respond to surveys and psychological questionnaires. |
|
For example, survey respondents may attempt to conceal socially undesirable traits (e.g., |
|
being ill-tempered) and endorse statements that cast them in a favorable manner (e.g., |
|
being cooperative). |
|
|
|
Developers of psychological questionnaires hence sometimes aim to ensure that questions |
|
are neutral, or that a subset of questions is equally (un)desirable. In the past, human |
|
judges have been tasked with quantifying item desirability. In contrast, the research |
|
underlying this web application demonstrates that large language models (LLMs) can |
|
achieve this too! |
|
""") |
|
|
|
def handle_demo_input(): |
|
|
|
if st.session_state.collect_data: |
|
if st.session_state.item_rating > 0: |
|
|
|
st.session_state.sentiment, st.session_state.desirability = modeling.score_text(st.session_state.input_text) |
|
|
|
payload = { |
|
'user_id': st.session_state.user_id, |
|
'gender_value': st.session_state.gender_value, |
|
'expert_value': st.session_state.expert_value, |
|
'statement': st.session_state.current_statement, |
|
'rating': st.session_state.item_rating, |
|
'suitability': st.session_state.suitability_rating, |
|
'input_text': st.session_state.input_text, |
|
'sentiment': st.session_state.sentiment, |
|
'desirability': st.session_state.desirability, |
|
} |
|
write_to_db_success = db.write_to_db(st.session_state.db, payload) |
|
|
|
if st.session_state.collect_data_optout: |
|
st.session_state.collect_data = False |
|
|
|
if write_to_db_success: |
|
st.session_state.current_statement = db.pick_random(st.session_state.statements) |
|
st.session_state.selection = '[Please select]' |
|
else: |
|
return None |
|
else: |
|
st.session_state.sentiment, st.session_state.desirability = modeling.score_text(st.session_state.input_text) |
|
|
|
if st.session_state.db: |
|
payload = { |
|
'user_id': st.session_state.user_id, |
|
'gender_value': None, |
|
'expert_value': None, |
|
'statement': None, |
|
'rating': None, |
|
'suitability': None, |
|
'input_text': st.session_state.input_text, |
|
'sentiment': st.session_state.sentiment, |
|
'desirability': st.session_state.desirability, |
|
} |
|
|
|
write_to_db_success = db.write_to_db(st.session_state.db, payload) |
|
|
|
|
|
def show_demo(placeholder): |
|
with placeholder: |
|
with st.container(): |
|
st.divider() |
|
st.markdown(""" |
|
## Try it yourself! |
|
Use the text field below to enter a statement that might be part of a psychological |
|
questionnaire (e.g., "I love a good fight."). Your input will be processed by |
|
language models, returning a machine-based estimate of item sentiment (i.e., valence) |
|
and desirability. |
|
|
|
""") |
|
modeling.load_model() |
|
|
|
if 'sentiment' in st.session_state and 'desirability' in st.session_state: |
|
plots.show_scores( |
|
sentiment=st.session_state.sentiment, |
|
desirability=st.session_state.desirability, |
|
input_text=st.session_state.input_text |
|
) |
|
|
|
st.session_state.input_text = st.text_input( |
|
label='Item text/statement:', |
|
value='I love a good fight.', |
|
placeholder='Enter item text' |
|
) |
|
|
|
user_rating_placeholder = st.empty() |
|
|
|
if st.session_state.collect_data: |
|
get_user_rating(user_rating_placeholder) |
|
|
|
if st.button( |
|
label='Evaluate Item Text', |
|
on_click=handle_demo_input, |
|
type='primary', |
|
use_container_width=True |
|
): |
|
if st.session_state.collect_data and st.session_state.item_rating == 0: |
|
st.error('Please rate the statement presented above!') |
|
|
|
def show_data(placeholder): |
|
with placeholder: |
|
with st.container(): |
|
st.divider() |
|
st.markdown(""" |
|
## Explore the data |
|
Figures show the accuarcy in precitions of human-rated item desirability by |
|
the sentiment model (left) and the desirability model (right), using |
|
`test`-partition data only. |
|
""") |
|
|
|
show_covariates = st.checkbox('Show covariates', value=True) |
|
if show_covariates: |
|
option = st.selectbox('Group by', options=list(utils.covariate_columns.values())) |
|
else: |
|
option = None |
|
|
|
if 'df' not in st.session_state: |
|
utils.load_data() |
|
|
|
plot = plots.scatter_plot(st.session_state.df, option) |
|
st.plotly_chart(plot, theme=None, use_container_width=True) |
|
|
|
def main(): |
|
st.set_page_config(page_title='Machine-Based Item Desirability Ratings') |
|
|
|
|
|
|
|
logo_url = "logo-130x130.svg" |
|
logo_link = "https://example.com/" |
|
|
|
col1, col2 = st.columns([2, 5]) |
|
with col1: |
|
st.image('logo-130x130.svg') |
|
|
|
with col2: |
|
st.markdown("# Machine-Based Item Desirability Ratings") |
|
|
|
st.markdown(""" |
|
This web application showcases the process of obtaining item desirability ratings using natural language processing (AI), accompanying the paper "Expanding the Methodological Toolbox: Machine-Based Item Desirability Ratings as an Alternative to Human-Based Ratings". |
|
|
|
📖 Paper: https://doi.org/10.1016/j.paid.2023.112307 |
|
|
|
📋 Preprint: https://psyarxiv.com/7bpeq |
|
|
|
💾 Data: https://osf.io/67mkz/ |
|
|
|
🖊️ Cite:<br> Hommel, B. E. (2023). Expanding the methodological toolbox: Machine-based item desirability ratings as an alternative to human-based ratings. Personality and Individual Differences, 213, 112307. https://doi.org/10.1016/j.paid.2023.112307 |
|
|
|
#️⃣ Twitter: https://twitter.com/BjoernHommel |
|
|
|
The web application is maintained by [magnolia psychometrics](https://www.magnolia-psychometrics.com/). |
|
""", unsafe_allow_html=True) |
|
|
|
placeholder_launch = st.empty() |
|
placeholder_summary = st.empty() |
|
placeholder_demo = st.empty() |
|
placeholder_data = st.empty() |
|
|
|
if st.session_state.show_launch is True: |
|
show_launch(placeholder_launch) |
|
else: |
|
placeholder_launch = st.empty() |
|
show_summary(placeholder_summary) |
|
show_demo(placeholder_demo) |
|
show_data(placeholder_data) |
|
|
|
|
|
if __name__ == '__main__': |
|
|
|
initialize() |
|
main() |
