Repository: lmc2179/bayesian_bootstrap

Issue:
Implementation of weight bootstrapping
Hello,

first of all: This is a great package!

Now to my question: Why is the only method of Bayesian bootstrapping resampling? Many sklearn regressor implement the weight keyword in the `fit` function and therefore could use Bayesian bootstrapping with weights instead of resampling. This approach should be much faster and less resource hungry, since only the size of the original training data matters.

Also I'm interested in implementing such a feature :)

Greetings!

Comments:

Patch:
diff --git a/README.md b/README.md
index 3d45649..05e754b 100644
--- a/README.md
+++ b/README.md
@@ -124,6 +124,7 @@ Interested in contributing? We'd love to have your help! Please keep the followi
 Credit for past contributions:
 
 * [roya0045](https://github.com/roya0045) implemented the original version of the low-memory optimizations.
+* [JulianWgs](https://github.com/JulianWgs) implemented the Bayesian machine learning model using weight distributions instead of resampling.
 
 # Further reading
 
diff --git a/bayesian_bootstrap/bootstrap.py b/bayesian_bootstrap/bootstrap.py
index 70c7fe5..1b5af80 100644
--- a/bayesian_bootstrap/bootstrap.py
+++ b/bayesian_bootstrap/bootstrap.py
@@ -124,17 +124,21 @@ def bayesian_bootstrap_regression(X, y, statistic, n_replications, resample_size
     else:
         weights = np.random.dirichlet([1] * len(X), n_replications)
     for w in weights:
-        resample_i = np.random.choice(range(len(X_arr)), p=w, size=resample_size)
-        resample_X = X_arr[resample_i]
-        resample_y = y_arr[resample_i]
-        s = statistic(resample_X, resample_y)
+        if resample_size is None:
+            s = statistic(X, y, w)
+        else:
+            resample_i = np.random.choice(range(len(X_arr)), p=w, size=resample_size)
+            resample_X = X_arr[resample_i]
+            resample_y = y_arr[resample_i]
+            s = statistic(resample_X, resample_y)
         samples.append(s)
 
     return samples
 
+
 class BayesianBootstrapBagging(object):
     """A bootstrap aggregating model using the bayesian bootstrap. Similar to scikit-learn's BaggingRegressor."""
-    def __init__(self, base_learner, n_replications, resample_size, low_mem=False):
+    def __init__(self, base_learner, n_replications, resample_size=None, low_mem=False):
         """Initialize the base learners of the ensemble.
 
         Parameter base_learner: A scikit-learn like estimator. This object should implement a fit() and predict()
@@ -161,12 +165,18 @@ class BayesianBootstrapBagging(object):
 
         Returns: Fitted model
         """
-        self.base_models_ = bayesian_bootstrap_regression(X,
-                                                          y,
-                                                          lambda X, y: deepcopy(self.base_learner).fit(X, y),
-                                                          self.n_replications,
-                                                          self.resample_size,
-                                                          low_mem=self.memo)
+        if self.resample_size is None:
+            statistic = lambda X, y, w: deepcopy(self.base_learner).fit(X, y, w)
+        else:
+            statistic = lambda X, y: deepcopy(self.base_learner).fit(X, y)
+        self.base_models_ = bayesian_bootstrap_regression(
+            X,
+            y,
+            statistic,
+            self.n_replications,
+            self.resample_size,
+            low_mem=self.memo
+        )
         return self
 
     def predict(self, X):


Test patch:
diff --git a/bayesian_bootstrap/tests/test_bootstrap.py b/bayesian_bootstrap/tests/test_bootstrap.py
index e817c81..91bf78d 100644
--- a/bayesian_bootstrap/tests/test_bootstrap.py
+++ b/bayesian_bootstrap/tests/test_bootstrap.py
@@ -36,7 +36,7 @@ class TestMoments(unittest.TestCase):
         posterior_samples = bayesian_bootstrap(X, np.mean, 10000, 100,low_mem=False)
         self.assertAlmostEqual(np.mean(posterior_samples), 0, delta=0.01)
         self.assertAlmostEqual(len([s for s in posterior_samples if s < 0]), 5000, delta=1000)
-        
+
     def test_var_resample(self):
         X = np.random.uniform(-1, 1, 500)
         posterior_samples = bayesian_bootstrap(X, np.var, 10000, 5000, low_mem=True)
@@ -72,7 +72,7 @@ class TestIntervals(unittest.TestCase):
         return x
 
 class TestRegression(unittest.TestCase):
-    def test_parameter_estimation_low_memory(self):
+    def test_parameter_estimation_resampling_low_memory(self):
         X = np.random.uniform(0, 4, 1000)
         y = X + np.random.normal(0, 1, 1000)
         m = BayesianBootstrapBagging(LinearRegression(), 10000, 1000, low_mem=True)
@@ -96,7 +96,7 @@ class TestRegression(unittest.TestCase):
         self.assertGreater(r, 0)
 
 
-    def test_parameter_estimation(self):
+    def test_parameter_estimation_resampling(self):
         X = np.random.uniform(0, 4, 1000)
         y = X + np.random.normal(0, 1, 1000)
         m = BayesianBootstrapBagging(LinearRegression(), 10000, 1000, low_mem=False)
@@ -117,7 +117,54 @@ class TestRegression(unittest.TestCase):
         self.assertAlmostEqual(np.mean(intercept_samples), 0, delta=0.3)
         l, r = highest_density_interval(intercept_samples, alpha=0.05)
         self.assertLess(l, 0)
-        self.assertGreater(r, 0)       
+        self.assertGreater(r, 0)
+
+    def test_parameter_estimation_bayes(self):
+        X = np.random.uniform(0, 4, 1000)
+        y = X + np.random.normal(0, 1, 1000)
+        m = BayesianBootstrapBagging(LinearRegression(), 10000, low_mem=False)
+        m.fit(X.reshape(-1, 1), y)
+        coef_samples = [b.coef_ for b in m.base_models_]
+        intercept_samples = [b.intercept_ for b in m.base_models_]
+        self.assertAlmostEqual(np.mean(coef_samples), 1, delta=0.3)
+        l, r = central_credible_interval(coef_samples, alpha=0.05)
+        self.assertLess(l, 1)
+        self.assertGreater(r, 1)
+        l, r = highest_density_interval(coef_samples, alpha=0.05)
+        self.assertLess(l, 1)
+        self.assertGreater(r, 1)
+        self.assertAlmostEqual(np.mean(intercept_samples), 0, delta=0.3)
+        l, r = central_credible_interval(intercept_samples, alpha=0.05)
+        self.assertLess(l, 0)
+        self.assertGreater(r, 0)
+        self.assertAlmostEqual(np.mean(intercept_samples), 0, delta=0.3)
+        l, r = highest_density_interval(intercept_samples, alpha=0.05)
+        self.assertLess(l, 0)
+        self.assertGreater(r, 0)
+
+    def test_parameter_estimation_bayes_low_memory(self):
+        X = np.random.uniform(0, 4, 1000)
+        y = X + np.random.normal(0, 1, 1000)
+        m = BayesianBootstrapBagging(LinearRegression(), 10000, low_mem=True)
+        m.fit(X.reshape(-1, 1), y)
+        coef_samples = [b.coef_ for b in m.base_models_]
+        intercept_samples = [b.intercept_ for b in m.base_models_]
+        self.assertAlmostEqual(np.mean(coef_samples), 1, delta=0.3)
+        l, r = central_credible_interval(coef_samples, alpha=0.05)
+        self.assertLess(l, 1)
+        self.assertGreater(r, 1)
+        l, r = highest_density_interval(coef_samples, alpha=0.05)
+        self.assertLess(l, 1)
+        self.assertGreater(r, 1)
+        self.assertAlmostEqual(np.mean(intercept_samples), 0, delta=0.3)
+        l, r = central_credible_interval(intercept_samples, alpha=0.05)
+        self.assertLess(l, 0)
+        self.assertGreater(r, 0)
+        self.assertAlmostEqual(np.mean(intercept_samples), 0, delta=0.3)
+        l, r = highest_density_interval(intercept_samples, alpha=0.05)
+        self.assertLess(l, 0)
+        self.assertGreater(r, 0)
+
 
 if __name__ == '__main__':
     unittest.main()