Spaces:
Runtime error
Runtime error
| import numpy as np | |
| class ExperimentTerminator(): | |
| def __init__(self): | |
| # The number of samples to be used in Monte Carlo elements of the code. Increasing this number | |
| # improves accuracy of estimates at the expense of run time. | |
| self.mc_samples = 5_000 | |
| # The Type I Error rate for the experiment. This determines the credible interval size used | |
| # for all calculations (e.g., alpha = 0.05 produces 95% credible intervals) | |
| self.alpha = 0.05 | |
| def get_posterior_samples(self, | |
| completed_trials_a, | |
| completed_trials_b, | |
| successes_a, | |
| successes_b): | |
| """ | |
| Get samples from the posterior distribution of the probability of success in the | |
| control group and the test group. | |
| :param completed_trials_a: Integer giving the number of trials completed in the control group | |
| :param completed_trials_b: Integer giving the number of trials completed in the test group | |
| :param successes_a: Integer giving the number of successes observed so far in the control group | |
| :param successes_b: Integer giving the number of successes observed so far in the test group | |
| :return: A list with two arrays of samples. The first array is a set of posterior samples from | |
| the distribution of the probability of success in the control group. The second array is the | |
| same for the test group. | |
| """ | |
| posterior_samples_a = np.random.beta(successes_a, | |
| completed_trials_a - successes_a, | |
| self.mc_samples) | |
| posterior_samples_b = np.random.beta(successes_b, | |
| completed_trials_b - successes_b, | |
| self.mc_samples) | |
| return [posterior_samples_a, posterior_samples_b] | |
| def get_prob_reject_null(self, | |
| planned_trials_a, | |
| planned_trials_b, | |
| completed_trials_a, | |
| completed_trials_b, | |
| successes_a, | |
| successes_b, | |
| posterior_samples_a, | |
| posterior_samples_b): | |
| """ | |
| Calculate the probability that the null hypothesis will be rejected by the planned end | |
| of the experiment | |
| :param planned_trials_a: Integer giving the number of trials planned to be completed in | |
| the control group in the experiment | |
| :param planned_trials_b: Integer giving the number of trials planned to be completed in | |
| the test group in the experiment | |
| :param completed_trials_a: Integer giving the number of trials completed in the control group | |
| :param completed_trials_b: Integer giving the number of trials completed in the teest group | |
| :param successes_a: Integer giving the number of successes observed so far in the control group | |
| :param successes_b: Integer giving the number of successes observed so far in the test group | |
| :param posterior_samples_a: Posterior samples for the control group returned by get_posterior_samples | |
| :param posterior_samples_a: Posterior samples for the test group returned by get_posterior_samples | |
| :return: Float with the posterior predictive probability of rejecting the null hypothesis. | |
| """ | |
| potential_successes_a = np.random.binomial(planned_trials_a - completed_trials_a, | |
| posterior_samples_a, | |
| self.mc_samples) | |
| potential_successes_a += successes_a | |
| potential_successes_b = np.random.binomial(planned_trials_b - completed_trials_b, | |
| posterior_samples_b, | |
| self.mc_samples) | |
| potential_successes_b += successes_b | |
| rejection = np.zeros(self.mc_samples) | |
| for i in range(self.mc_samples): | |
| post_samples_a = np.random.beta(potential_successes_a[i] + 1, | |
| planned_trials_a - potential_successes_a[i] + 1, | |
| self.mc_samples) | |
| post_samples_b = np.random.beta(potential_successes_b[i] + 1, | |
| planned_trials_b - potential_successes_b[i] + 1, | |
| self.mc_samples) | |
| post_samples_b_minus_a = post_samples_b - post_samples_a | |
| interval = np.quantile(post_samples_b_minus_a, [self.alpha / 2, 1 - (self.alpha / 2)]) | |
| if (interval[0] > 0 or interval[1] < 0): | |
| rejection[i] = 1 | |
| return np.mean(rejection) | |
| def analyze_experiment(self, | |
| planned_trials_a, | |
| planned_trials_b, | |
| completed_trials_a, | |
| completed_trials_b, | |
| successes_a, | |
| successes_b): | |
| """ | |
| Based on the number of planned trials, completed trials, and successes observed so far in the | |
| control and test groups, calculate a bunch of summary measures of the posterior distribution | |
| and predicted posterior. | |
| :param planned_trials_a: Integer giving the number of trials planned to be completed in | |
| the control group in the experiment | |
| :param planned_trials_b: Integer giving the number of trials planned to be completed in | |
| the test group in the experiment | |
| :param completed_trials_a: Integer giving the number of trials completed in the control group | |
| :param completed_trials_b: Integer giving the number of trials completed in the teest group | |
| :param successes_a: Integer giving the number of successes observed so far in the control group | |
| :param successes_b: Integer giving the number of successes observed so far in the test group | |
| """ | |
| posterior_samples = self.get_posterior_samples(completed_trials_a, | |
| completed_trials_b, | |
| successes_a, | |
| successes_b) | |
| posterior_lift = (posterior_samples[1] - posterior_samples[0]) / posterior_samples[0] | |
| conversion_rate_a = successes_a / completed_trials_a | |
| conversion_rate_b = successes_b / completed_trials_b | |
| expected_lift = np.mean(posterior_lift) | |
| pr_b_gt_a = np.mean(posterior_lift >= 0) | |
| pr_reject_null = self.get_prob_reject_null(planned_trials_a, | |
| planned_trials_b, | |
| completed_trials_a, | |
| completed_trials_b, | |
| successes_a, | |
| successes_b, | |
| posterior_samples[0], | |
| posterior_samples[1]) | |
| return [conversion_rate_a, | |
| conversion_rate_b, | |
| expected_lift, | |
| pr_b_gt_a, | |
| pr_reject_null, | |
| posterior_lift] | |
| if __name__ == "__main__": | |
| et = ExperimentTerminator() | |
| exp_outcomes = et.analyze_experiment(2000, 2000, 1000, 1000, 250, 270) | |
| print(exp_outcomes) |