Spaces:
Sleeping
Sleeping
Zekun Wu
commited on
Commit
β’
823c0be
1
Parent(s):
c9ac1ea
update
Browse files- app.py +26 -102
- pages/1_Generation_Demo.py +104 -0
- analysis.py β pages/util/analysis.py +0 -0
- generation.py β pages/util/generation.py +0 -0
- model.py β pages/util/model.py +0 -0
app.py
CHANGED
@@ -1,104 +1,28 @@
|
|
1 |
import streamlit as st
|
2 |
-
import pandas as pd
|
3 |
-
from io import StringIO
|
4 |
-
from generation import process_scores
|
5 |
-
from model import AzureAgent, GPTAgent
|
6 |
-
from analysis import statistical_tests, result_evaluation
|
7 |
|
8 |
-
|
9 |
-
|
10 |
-
|
11 |
-
|
12 |
-
|
13 |
-
|
14 |
-
|
15 |
-
|
16 |
-
|
17 |
-
|
18 |
-
|
19 |
-
|
20 |
-
|
21 |
-
|
22 |
-
|
23 |
-
|
24 |
-
|
25 |
-
|
26 |
-
|
27 |
-
|
28 |
-
|
29 |
-
|
30 |
-
|
31 |
-
|
32 |
-
|
33 |
-
|
34 |
-
st.experimental_rerun()
|
35 |
-
|
36 |
-
if st.sidebar.button("Submit Model Info"):
|
37 |
-
st.session_state.model_submitted = True
|
38 |
-
|
39 |
-
|
40 |
-
|
41 |
-
|
42 |
-
# Ensure experiment settings are only shown if model info is submitted
|
43 |
-
if st.session_state.model_submitted:
|
44 |
-
df = None
|
45 |
-
file_options = st.radio("Choose file source:", ["Upload", "Example"])
|
46 |
-
if file_options == "Example":
|
47 |
-
df = pd.read_csv("prompt_test.csv")
|
48 |
-
else:
|
49 |
-
st.session_state.uploaded_file = st.file_uploader("Choose a file")
|
50 |
-
if st.session_state.uploaded_file is not None:
|
51 |
-
data = StringIO(st.session_state.uploaded_file.getvalue().decode("utf-8"))
|
52 |
-
df = pd.read_csv(data)
|
53 |
-
if df is not None:
|
54 |
-
|
55 |
-
st.write('Data:', df)
|
56 |
-
|
57 |
-
st.session_state.occupation = st.text_input("Occupation", value=st.session_state.occupation)
|
58 |
-
st.session_state.group_name = st.text_input("Group Name", value=st.session_state.group_name)
|
59 |
-
st.session_state.privilege_label = st.text_input("Privilege Label", value=st.session_state.privilege_label)
|
60 |
-
st.session_state.protect_label = st.text_input("Protect Label", value=st.session_state.protect_label)
|
61 |
-
st.session_state.num_run = st.number_input("Number of Runs", 1, 10, st.session_state.num_run)
|
62 |
-
|
63 |
-
if st.button('Process Data') and not st.session_state.data_processed:
|
64 |
-
# Initialize the correct agent based on model type
|
65 |
-
if model_type == 'AzureAgent':
|
66 |
-
agent = AzureAgent(st.session_state.api_key, st.session_state.endpoint_url, st.session_state.deployment_name)
|
67 |
-
else:
|
68 |
-
agent = GPTAgent(st.session_state.api_key, st.session_state.endpoint_url, st.session_state.deployment_name, api_version)
|
69 |
-
|
70 |
-
# Process data and display results
|
71 |
-
with st.spinner('Processing data...'):
|
72 |
-
parameters = {"temperature": st.session_state.temperature, "max_tokens": st.session_state.max_tokens}
|
73 |
-
df = process_scores(df, st.session_state.num_run, parameters, st.session_state.privilege_label, st.session_state.protect_label, agent, st.session_state.group_name, st.session_state.occupation)
|
74 |
-
st.session_state.data_processed = True # Mark as processed
|
75 |
-
|
76 |
-
# Add ranks for each score within each row
|
77 |
-
ranks = df[['Privilege_Avg_Score', 'Protect_Avg_Score', 'Neutral_Avg_Score']].rank(axis=1,ascending=False)
|
78 |
-
|
79 |
-
df['Privilege_Rank'] = ranks['Privilege_Avg_Score']
|
80 |
-
df['Protect_Rank'] = ranks['Protect_Avg_Score']
|
81 |
-
df['Neutral_Rank'] = ranks['Neutral_Avg_Score']
|
82 |
-
|
83 |
-
st.write('Processed Data:', df)
|
84 |
-
|
85 |
-
# use the data to generate a plot
|
86 |
-
st.write("Plotting the data")
|
87 |
-
|
88 |
-
test_results = statistical_tests(df)
|
89 |
-
print(test_results)
|
90 |
-
evaluation_results = result_evaluation(test_results)
|
91 |
-
print(evaluation_results)
|
92 |
-
|
93 |
-
for key, value in evaluation_results.items():
|
94 |
-
st.write(f"{key}: {value}")
|
95 |
-
|
96 |
-
|
97 |
-
if st.button("Reset Experiment Settings"):
|
98 |
-
st.session_state.occupation = "Programmer"
|
99 |
-
st.session_state.group_name = "Gender"
|
100 |
-
st.session_state.privilege_label = "Male"
|
101 |
-
st.session_state.protect_label = "Female"
|
102 |
-
st.session_state.num_run = 1
|
103 |
-
st.session_state.data_processed = False
|
104 |
-
st.session_state.uploaded_file = None
|
|
|
1 |
import streamlit as st
|
|
|
|
|
|
|
|
|
|
|
2 |
|
3 |
+
st.set_page_config(
|
4 |
+
page_title="app",
|
5 |
+
page_icon="π",
|
6 |
+
)
|
7 |
+
|
8 |
+
st.write("# Welcome to Streamlit! π")
|
9 |
+
|
10 |
+
st.sidebar.success("Select a demo above.")
|
11 |
+
|
12 |
+
st.markdown(
|
13 |
+
"""
|
14 |
+
Streamlit is an open-source app framework built specifically for
|
15 |
+
Machine Learning and Data Science projects.
|
16 |
+
**π Select a demo from the sidebar** to see some examples
|
17 |
+
of what Streamlit can do!
|
18 |
+
### Want to learn more?
|
19 |
+
- Check out [streamlit.io](https://streamlit.io)
|
20 |
+
- Jump into our [documentation](https://docs.streamlit.io)
|
21 |
+
- Ask a question in our [community
|
22 |
+
forums](https://discuss.streamlit.io)
|
23 |
+
### See more complex demos
|
24 |
+
- Use a neural net to [analyze the Udacity Self-driving Car Image
|
25 |
+
Dataset](https://github.com/streamlit/demo-self-driving)
|
26 |
+
- Explore a [New York City rideshare dataset](https://github.com/streamlit/demo-uber-nyc-pickups)
|
27 |
+
"""
|
28 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
pages/1_Generation_Demo.py
ADDED
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import streamlit as st
|
2 |
+
import pandas as pd
|
3 |
+
from io import StringIO
|
4 |
+
from util.generation import process_scores
|
5 |
+
from util.model import AzureAgent, GPTAgent
|
6 |
+
from util.analysis import statistical_tests, result_evaluation
|
7 |
+
|
8 |
+
# Set up the Streamlit interface
|
9 |
+
st.title('JobFair: A Benchmark for Fairness in LLM Employment Decision')
|
10 |
+
st.sidebar.title('Model Settings')
|
11 |
+
|
12 |
+
# Define a function to manage state initialization
|
13 |
+
def initialize_state():
|
14 |
+
keys = ["model_submitted", "api_key", "endpoint_url", "deployment_name", "temperature", "max_tokens",
|
15 |
+
"data_processed", "group_name","occupation", "privilege_label", "protect_label", "num_run", "uploaded_file"]
|
16 |
+
defaults = [False, "", "https://safeguard-monitor.openai.azure.com/", "gpt35-1106", 0.5, 150, False,"Gender", "Programmer", "Male", "Female", 1, None]
|
17 |
+
for key, default in zip(keys, defaults):
|
18 |
+
if key not in st.session_state:
|
19 |
+
st.session_state[key] = default
|
20 |
+
|
21 |
+
initialize_state()
|
22 |
+
|
23 |
+
# Model selection and configuration
|
24 |
+
model_type = st.sidebar.radio("Select the type of agent", ('GPTAgent', 'AzureAgent'))
|
25 |
+
st.session_state.api_key = st.sidebar.text_input("API Key", type="password", value=st.session_state.api_key)
|
26 |
+
st.session_state.endpoint_url = st.sidebar.text_input("Endpoint URL", value=st.session_state.endpoint_url)
|
27 |
+
st.session_state.deployment_name = st.sidebar.text_input("Model Name", value=st.session_state.deployment_name)
|
28 |
+
api_version = '2024-02-15-preview' if model_type == 'GPTAgent' else ''
|
29 |
+
st.session_state.temperature = st.sidebar.slider("Temperature", 0.0, 1.0, st.session_state.temperature, 0.01)
|
30 |
+
st.session_state.max_tokens = st.sidebar.number_input("Max Tokens", 1, 1000, st.session_state.max_tokens)
|
31 |
+
|
32 |
+
if st.sidebar.button("Reset Model Info"):
|
33 |
+
initialize_state() # Reset all state to defaults
|
34 |
+
st.experimental_rerun()
|
35 |
+
|
36 |
+
if st.sidebar.button("Submit Model Info"):
|
37 |
+
st.session_state.model_submitted = True
|
38 |
+
|
39 |
+
|
40 |
+
|
41 |
+
|
42 |
+
# Ensure experiment settings are only shown if model info is submitted
|
43 |
+
if st.session_state.model_submitted:
|
44 |
+
df = None
|
45 |
+
file_options = st.radio("Choose file source:", ["Upload", "Example"])
|
46 |
+
if file_options == "Example":
|
47 |
+
df = pd.read_csv("prompt_test.csv")
|
48 |
+
else:
|
49 |
+
st.session_state.uploaded_file = st.file_uploader("Choose a file")
|
50 |
+
if st.session_state.uploaded_file is not None:
|
51 |
+
data = StringIO(st.session_state.uploaded_file.getvalue().decode("utf-8"))
|
52 |
+
df = pd.read_csv(data)
|
53 |
+
if df is not None:
|
54 |
+
|
55 |
+
st.write('Data:', df)
|
56 |
+
|
57 |
+
st.session_state.occupation = st.text_input("Occupation", value=st.session_state.occupation)
|
58 |
+
st.session_state.group_name = st.text_input("Group Name", value=st.session_state.group_name)
|
59 |
+
st.session_state.privilege_label = st.text_input("Privilege Label", value=st.session_state.privilege_label)
|
60 |
+
st.session_state.protect_label = st.text_input("Protect Label", value=st.session_state.protect_label)
|
61 |
+
st.session_state.num_run = st.number_input("Number of Runs", 1, 10, st.session_state.num_run)
|
62 |
+
|
63 |
+
if st.button('Process Data') and not st.session_state.data_processed:
|
64 |
+
# Initialize the correct agent based on model type
|
65 |
+
if model_type == 'AzureAgent':
|
66 |
+
agent = AzureAgent(st.session_state.api_key, st.session_state.endpoint_url, st.session_state.deployment_name)
|
67 |
+
else:
|
68 |
+
agent = GPTAgent(st.session_state.api_key, st.session_state.endpoint_url, st.session_state.deployment_name, api_version)
|
69 |
+
|
70 |
+
# Process data and display results
|
71 |
+
with st.spinner('Processing data...'):
|
72 |
+
parameters = {"temperature": st.session_state.temperature, "max_tokens": st.session_state.max_tokens}
|
73 |
+
df = process_scores(df, st.session_state.num_run, parameters, st.session_state.privilege_label, st.session_state.protect_label, agent, st.session_state.group_name, st.session_state.occupation)
|
74 |
+
st.session_state.data_processed = True # Mark as processed
|
75 |
+
|
76 |
+
# Add ranks for each score within each row
|
77 |
+
ranks = df[['Privilege_Avg_Score', 'Protect_Avg_Score', 'Neutral_Avg_Score']].rank(axis=1,ascending=False)
|
78 |
+
|
79 |
+
df['Privilege_Rank'] = ranks['Privilege_Avg_Score']
|
80 |
+
df['Protect_Rank'] = ranks['Protect_Avg_Score']
|
81 |
+
df['Neutral_Rank'] = ranks['Neutral_Avg_Score']
|
82 |
+
|
83 |
+
st.write('Processed Data:', df)
|
84 |
+
|
85 |
+
# use the data to generate a plot
|
86 |
+
st.write("Plotting the data")
|
87 |
+
|
88 |
+
test_results = statistical_tests(df)
|
89 |
+
print(test_results)
|
90 |
+
evaluation_results = result_evaluation(test_results)
|
91 |
+
print(evaluation_results)
|
92 |
+
|
93 |
+
for key, value in evaluation_results.items():
|
94 |
+
st.write(f"{key}: {value}")
|
95 |
+
|
96 |
+
|
97 |
+
if st.button("Reset Experiment Settings"):
|
98 |
+
st.session_state.occupation = "Programmer"
|
99 |
+
st.session_state.group_name = "Gender"
|
100 |
+
st.session_state.privilege_label = "Male"
|
101 |
+
st.session_state.protect_label = "Female"
|
102 |
+
st.session_state.num_run = 1
|
103 |
+
st.session_state.data_processed = False
|
104 |
+
st.session_state.uploaded_file = None
|
analysis.py β pages/util/analysis.py
RENAMED
File without changes
|
generation.py β pages/util/generation.py
RENAMED
File without changes
|
model.py β pages/util/model.py
RENAMED
File without changes
|