import altair as alt from experiment_terminator import ExperimentTerminator import numpy as np import pandas as pd import streamlit as st def analyze_experiment(): """ Upon clicking the button, run the experiment analyzer and display all of the output. This even displays a histogram and some Arnold images. """ exp_output = et.analyze_experiment(planned_trials_a, planned_trials_b, completed_trials_a, completed_trials_b, successes_a, successes_b) # Only display the parameter summary information after the analysis has concluded. input_summary.markdown(input_display) # Display the summary statistics about the posterior distribution and posterior predictive # distribution output_col1.metric("Observed Conversion Rate in Control Group", value=np.round(exp_output[0], 4)) output_col2.metric("Observed Conversion Rate in Test Group", value=np.round(exp_output[1], 4)) output_col3.metric("Estimated Lift (Posterior Mean)", value=str(np.round(100 * exp_output[2], 2)) + "%") output_col4.metric("Probability Test Rate > Control Rate", value=np.round(exp_output[3], 4)) output_col5.metric("Probability of Significance at Experiment End", value=np.round(exp_output[4], 4)) # Apply logic determining what conclusion is reached. if exp_output[4] < 0.01 or exp_output[4] > .99: # We know the future of the experiment - conclude. outstring = "## Experiment can be terminated. " if exp_output[4] < .01: # The future is that there is almost no way we'll see significance outstring += "No difference between experiment and control will be found." else: # The future is that we almost certainly will see significance if exp_output[3] > .5: outstring += "Test is superior to control." else: outstring += "Control is superior to test." conclusion_container.markdown(outstring) # Add an Arnold image to drive home the point. graph_col1.image('images/terminator_okay.gif', use_column_width=True) else: # We don't know the future of the experiment. Continue conclusion_container.markdown("## Experiment should not be terminated.") # Add an Arnold image to drive home the point. graph_col1.image('images/thou_shalt_not_terminate.jpeg', use_column_width=True) # Create the histogram showing the posterior distribution of lifts. chart_data = pd.DataFrame({'lift': exp_output[5]}) chart = alt.Chart(chart_data).mark_bar().encode( alt.X("lift", bin=alt.Bin(maxbins=30)), y="count()" ).properties(title="Histogram of Posterior Distribution of Lift") graph_col2.altair_chart(chart, use_container_width=True) et = ExperimentTerminator() # Set overall page configuration variables st.set_page_config(layout="wide") st.title('Experiment Terminator') # Create the container where users can input information about their experiment. main_container = st.container() # Create input boxes in two columns, one for Control and one for Test. For each one, # make input boxes for planned number of trials, completed trials, and successes. col1, col2 = main_container.columns(2) col1.header("Control") planned_trials_a = col1.number_input("Planned Number of Trials", min_value=100, key='planned_trials_a', value=500) completed_trials_a = col1.number_input("Completed Trials", min_value=100, max_value=planned_trials_a, key='completed_trials_a', value=300) successes_a = col1.number_input("Successes", min_value=0, key='successes_a', value=25) col2.header("Test") planned_trials_b = col2.number_input("Planned Number of Trials", min_value=100, key='planned_trials_b', value=500) completed_trials_b = col2.number_input("Completed Trials", min_value=100, max_value=planned_trials_b, key='completed_trials_b', value=300) successes_b = col2.number_input("Successes", min_value=0, key='successes_b', value=35) # Add a button to click to make the analysis run. analyze = st.button("Analyze Experiment for Termination", on_click=analyze_experiment) # Add a container to holds summary information on the parameters used to run the analysis. This gets # displayed back to the user so they know what information the code is using. input_summary = st.container() input_display = "Analysis run with the following parameters: Control Group (" input_display += str(planned_trials_a) + " planned trials, " input_display += str(completed_trials_a) + " completed trials, " input_display += str(successes_a) + " successes), Test Group (" input_display += str(planned_trials_b) + " planned trials, " input_display += str(completed_trials_b) + " completed trials, " input_display += str(successes_b) + " successes)" input_summary.markdown("") # Add a container to hold summary statistics about the posterior distribution and posterior # predictive distribution. result_container = st.container() output_col1, output_col2, output_col3, output_col4, output_col5 = result_container.columns(5) # Add a container with a text statement about the conclusions reached. conclusion_container = st.container() conclusion_container.markdown("") # Add a container to hold graphiics. This will hold an Arnold graphic on the left and a histogram on the right. graph_container = st.container() graph_col1, graph_col2 = graph_container.columns(2) graph_col1.markdown("") graph_col2.markdown("")