Spaces:
Sleeping
Sleeping
| import streamlit as st | |
| from huggingface import Model | |
| from db import DB | |
| from helpers import validate_key, label_speaker, expand_for, check_none_in_dict, check_conversation_input_validity | |
| def main(): | |
| cred_path = "scamgpt-j-evaluation-firebase-adminsdk-xt1xg-71792b5117.json" | |
| db = DB(cred_path) | |
| key = st.text_input("Enter Survey Key", "") | |
| # conversations = {'q1_input': None, 'q1_reply': None, 'q2_input': None, 'q2_reply': None, 'q3_input': None,'q3_reply': None} | |
| # Initialize 'conversations' in session state if not present | |
| if 'conversations' not in st.session_state: | |
| st.session_state.conversations = {'q1_input': None, 'q1_reply': None, 'q2_input': None, 'q2_reply': None, 'q3_input': None, 'q3_reply': None} | |
| if validate_key(key, db): | |
| st.write("Survey Key Accepted. Please proceed.") | |
| model_name = 'ksee/testgptj' #'txw123/gptj-prompttuned' | |
| API_URL = "https://w82gkg78xjt469wa.us-east-1.aws.endpoints.huggingface.cloud" | |
| model = Model(model_name, API_URL) # replace with the name of the model deployed to Hugging Face | |
| st.write('') | |
| st.write('') | |
| with st.container(): | |
| st.header('Conversation 1') | |
| entry = [None] * 10 | |
| for entry_num in range(10): | |
| with st.expander('Conversation entry ' + str(entry_num + 1), expanded=expand_for(entry_num)): | |
| col1, col2 = st.columns([1, 3]) | |
| with col1: | |
| speaker = st.selectbox("Speaker", ["You", "Other Party"], index=1-entry_num%2, key='speaker1 ' + str(entry_num)) | |
| with col2: | |
| text = st.text_area("", key='text1 ' + str(entry_num)) | |
| if text != '': | |
| entry_addition = label_speaker(speaker) + ': ' + text | |
| entry[entry_num] = entry_addition | |
| if st.button('Generate Response', key='response1'): | |
| with st.spinner('Generating a response...'): | |
| if check_conversation_input_validity(entry): | |
| input_text = ' '.join([text for text in entry if text is not None]) | |
| output_text = model.generate_text(input_text) | |
| st.text('Response: ' + output_text) | |
| st.session_state.conversations['q1_input'] = input_text | |
| st.session_state.conversations['q1_reply'] = output_text | |
| else: | |
| st.error('Invalid conversation. Please ensure that conversation has at least two interactions and ends with a text from you.') | |
| checkbox1 = st.checkbox('Is this response believable based on your known context of the conversation?', key='checkbox1') | |
| scamcheck1 = st.checkbox('Was this conversation an actual scam convesation?', key='scamcheck1') | |
| with st.container(): | |
| st.header('Conversation 2') | |
| entry = [None] * 10 | |
| for entry_num in range(10): | |
| with st.expander('Conversation entry ' + str(entry_num + 1)): | |
| col1, col2 = st.columns([1, 3]) | |
| with col1: | |
| speaker = st.selectbox("Speaker", ["You", "Other Party"], index=1-entry_num%2, key='speaker2 ' + str(entry_num)) | |
| with col2: | |
| text = st.text_area("", key='text2 ' + str(entry_num)) | |
| if text != '': | |
| entry_addition = label_speaker(speaker) + ': ' + text | |
| entry[entry_num] = entry_addition | |
| if st.button('Generate Response', key='response2'): | |
| with st.spinner('Generating a response...'): | |
| if check_conversation_input_validity(entry): | |
| input_text = ' '.join([text for text in entry if text is not None]) | |
| output_text = model.generate_text(input_text) | |
| st.text('Response: ' + output_text) | |
| st.session_state.conversations['q2_input'] = input_text | |
| st.session_state.conversations['q2_reply'] = output_text | |
| else: | |
| st.error('Invalid conversation. Please ensure that conversation has at least two interactions and ends with a text from you.') | |
| checkbox2 = st.checkbox('Is this response believable based on your known context of the conversation?', key='checkbox2') | |
| scamcheck2 = st.checkbox('Was this conversation an actual scam convesation?', key='scamcheck2') | |
| with st.container(): | |
| st.header('Conversation 3') | |
| entry = [None] * 10 | |
| for entry_num in range(10): | |
| with st.expander('Conversation entry ' + str(entry_num + 1)): | |
| col1, col2 = st.columns([1, 3]) | |
| with col1: | |
| speaker = st.selectbox("Speaker", ["You", "Other Party"], index=1-entry_num%2, key='speaker3 ' + str(entry_num)) | |
| with col2: | |
| text = st.text_area("", key='text3 ' + str(entry_num)) | |
| if text != '': | |
| entry_addition = label_speaker(speaker) + ': ' + text | |
| entry[entry_num] = entry_addition | |
| if st.button('Generate Response', key='response3'): | |
| with st.spinner('Generating a response...'): | |
| if check_conversation_input_validity(entry): | |
| input_text = ' '.join([text for text in entry if text is not None]) | |
| output_text = model.generate_text(input_text) | |
| st.text('Response: ' + output_text) | |
| st.session_state.conversations['q3_input'] = input_text | |
| st.session_state.conversations['q3_reply'] = output_text | |
| else: | |
| st.error('Invalid conversation. Please ensure that conversation has at least two interactions and ends with a text from you.') | |
| checkbox3 = st.checkbox('Is this response believable based on your known context of the conversation?', key='checkbox3') | |
| scamcheck3 = st.checkbox('Was this conversation an actual scam convesation?', key='scamcheck3') | |
| st.header('Usefulness Evaluation') | |
| satisfaction = st.slider("Imagine you have a conversation where you're uncertain if it's a scam. How useful do you think this bot would be in understanding and assessing the situation?", 1, 5, 5) | |
| if st.button("Submit"): | |
| with st.spinner('Submitting your responses...'): | |
| if check_none_in_dict(st.session_state.conversations): | |
| st.error('Please ensure all questions have been attempted.') | |
| else: | |
| db.add_survey_responses(key, model_name, checkbox1, scamcheck1, st.session_state.conversations['q1_input'], st.session_state.conversations['q1_reply'], checkbox2, scamcheck2, st.session_state.conversations['q2_input'], st.session_state.conversations['q2_reply'], checkbox3, scamcheck3, st.session_state.conversations['q3_input'], st.session_state.conversations['q3_reply'], satisfaction) | |
| st.success('Responses have been submitted. Thank you for your participation!') | |
| else: | |
| if key != '': | |
| st.error('Survey key provided is either invalid or has already been used.') | |
| def display_survey_introduction(): | |
| st.title("Instant Messaging Scam Interaction Survey") | |
| st.markdown(""" | |
| Thank you for participating in this survey. Your input is invaluable in helping us evaluate the efficacy of our conversational bot, designed to mimic the responses of a scammer in various conversation scenarios within instant messaging platforms. Please read the following instructions carefully before you begin: | |
| **Objective of the Survey**: | |
| - The primary goal of this survey is to assess how effectively our bot can replicate scammer-like responses when provided with a conversation log. Your feedback will directly contribute to improving the bot's accuracy and realism in such interactions. | |
| **Inputting Conversations**: | |
| - You will be presented with three separate conversation scenarios. For each scenario, you are required to input the dialogue. | |
| **Number of Interactions**: | |
| - Each conversation should consist of a minimum of two interactions (one from the counterparty and one from you) and can include up to a maximum of ten interactions. This structure is crucial for the bot to understand the flow and context of the conversation. | |
| **Final Interaction Requirement**: | |
| - Please ensure that the last interaction in each conversation log comes from you. This setup allows the bot to generate the subsequent response as if it were the counterparty, thereby simulating a scammer's potential reply. | |
| **Contextual Believability**: | |
| - Given that you are familiar with the actual context of each conversation, we ask you to use the provided checkbox to indicate whether the bot's generated response seems believable within the given context. This assessment is key to understanding the bot's effectiveness in real-world scenarios. | |
| **Evaluating the Bot's Efficacy**: | |
| - After interacting with the bot across these scenarios, you will be asked to evaluate its overall efficacy as a potential tool for scam intervention. Your evaluation will help us gauge the bot's practical utility in identifying and responding to scam-related conversations. | |
| Your honest and thoughtful responses are crucial for the success of this project. We appreciate your time and effort in participating in this survey. | |
| """) | |
| if __name__ == "__main__": | |
| display_survey_introduction() | |
| main() | |