Repository: scikit-learn/scikit-learn

Issue:
# BinMapper within HGBT does not handle sample weights

### Describe the bug

BinMapper under _hist_gradient_boosting does not accept sample weights as input leading to mismatch of bin thresholds outputted when calculating weighted versus repeated samples. Linked to Issue #27117

### Steps/Code to Reproduce

```python
from sklearn.ensemble._hist_gradient_boosting import binning

from sklearn.datasets import make_regression
import numpy as np

n_samples = 50  
n_features = 2
rng = np.random.RandomState(42)
    
X, y = make_regression(
    n_samples=n_samples,
    n_features=n_features,
    n_informative=n_features,
    random_state=0,
)

# Create dataset with repetitions and corresponding sample weights
sample_weight = rng.randint(0, 10, size=X.shape[0])
X_resampled_by_weights = np.repeat(X, sample_weight, axis=0)

bins_fit_weighted = binning._BinMapper(255).fit(X)
bins_fit_resampled = binning._BinMapper(255).fit(X_resampled_by_weights)

np.testing.assert_allclose(bins_fit_resampled.bin_thresholds_, bins_fit_weighted.bin_thresholds_)
```

### Expected Results

No error thrown

### Actual Results

```
AssertionError: 
Not equal to tolerance rtol=1e-07, atol=0

(shapes (2, 47), (2, 49) mismatch)
 ACTUAL: array([[-2.12963 , -1.668234, -1.622048, -1.433347, -1.208973, -1.117951,
        -1.059653, -0.977926, -0.901382, -0.891626, -0.879291, -0.841972,
        -0.742803, -0.653391, -0.572564, -0.510229, -0.456415, -0.395252,...
 DESIRED: array([[-2.12963 , -1.668234, -1.622048, -1.433347, -1.208973, -1.117951,
        -1.059653, -0.977926, -0.901382, -0.891626, -0.879291, -0.841972,
        -0.742803, -0.653391, -0.572564, -0.510229, -0.456415, -0.395252,...
```

### Versions

```shell
System:
    python: 3.12.4 | packaged by conda-forge | (main, Jun 17 2024, 10:13:44) [Clang 16.0.6 ]
executable: /Users/shrutinath/micromamba/envs/scikit-learn/bin/python
   machine: macOS-14.3-arm64-arm-64bit

Python dependencies:
      sklearn: 1.6.dev0
          pip: 24.0
   setuptools: 70.1.1
        numpy: 2.0.0
        scipy: 1.14.0
       Cython: 3.0.10
       pandas: None
   matplotlib: 3.9.0
       joblib: 1.4.2
threadpoolctl: 3.5.0

Built with OpenMP: True

threadpoolctl info:
       user_api: blas
   internal_api: openblas
    num_threads: 8
         prefix: libopenblas
...
    num_threads: 8
         prefix: libomp
       filepath: /Users/shrutinath/micromamba/envs/scikit-learn/lib/libomp.dylib
        version: None
```


Implementation patch:
diff --git a/sklearn/ensemble/_hist_gradient_boosting/binning.py b/sklearn/ensemble/_hist_gradient_boosting/binning.py
index b0745b58ae8dd..3b3a4108359ee 100644
--- a/sklearn/ensemble/_hist_gradient_boosting/binning.py
+++ b/sklearn/ensemble/_hist_gradient_boosting/binning.py
@@ -10,6 +10,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 import numpy as np
+from numpy.lib.stride_tricks import sliding_window_view
 
 from sklearn.base import BaseEstimator, TransformerMixin
 from sklearn.ensemble._hist_gradient_boosting._binning import _map_to_bins
@@ -23,10 +24,11 @@
 from sklearn.utils import check_array, check_random_state
 from sklearn.utils._openmp_helpers import _openmp_effective_n_threads
 from sklearn.utils.parallel import Parallel, delayed
+from sklearn.utils.stats import _weighted_percentile
 from sklearn.utils.validation import check_is_fitted
 
 
-def _find_binning_thresholds(col_data, max_bins):
+def _find_binning_thresholds(col_data, max_bins, sample_weight=None):
     """Extract quantiles from a continuous feature.
 
     Missing values are ignored for finding the thresholds.
@@ -50,31 +52,63 @@ def _find_binning_thresholds(col_data, max_bins):
     """
     # ignore missing values when computing bin thresholds
     missing_mask = np.isnan(col_data)
-    if missing_mask.any():
+    any_missing = missing_mask.any()
+    if any_missing:
         col_data = col_data[~missing_mask]
+
+    # If sample_weight is not None and 0-weighted values exist, we need to
+    # remove those before calculating the distinct points.
+    if sample_weight is not None:
+        if any_missing:
+            sample_weight = sample_weight[~missing_mask]
+        nnz_sw = sample_weight != 0
+        col_data = col_data[nnz_sw]
+        sample_weight = sample_weight[nnz_sw]
+
     # The data will be sorted anyway in np.unique and again in percentile, so we do it
     # here. Sorting also returns a contiguous array.
-    col_data = np.sort(col_data)
+    sort_idx = np.argsort(col_data)
+    col_data = col_data[sort_idx]
+    if sample_weight is not None:
+        sample_weight = sample_weight[sort_idx]
+
     distinct_values = np.unique(col_data).astype(X_DTYPE)
+
+    if len(distinct_values) == 1:
+        return np.asarray([])
+
     if len(distinct_values) <= max_bins:
-        midpoints = distinct_values[:-1] + distinct_values[1:]
-        midpoints *= 0.5
+        # Calculate midpoints if distinct values <= max_bins
+        bin_thresholds = sliding_window_view(distinct_values, 2).mean(axis=1)
+    elif sample_weight is None:
+        # We compute bin edges using the output of np.percentile with
+        # the "averaged_inverted_cdf" interpolation method that is consistent
+        # with the code for the sample_weight != None case.
+        percentiles = np.linspace(0, 100, num=max_bins + 1)
+        percentiles = percentiles[1:-1]
+        bin_thresholds = np.percentile(
+            col_data, percentiles, method="averaged_inverted_cdf"
+        )
+        assert bin_thresholds.shape[0] == max_bins - 1
     else:
-        # We could compute approximate midpoint percentiles using the output of
-        # np.unique(col_data, return_counts) instead but this is more
-        # work and the performance benefit will be limited because we
-        # work on a fixed-size subsample of the full data.
         percentiles = np.linspace(0, 100, num=max_bins + 1)
         percentiles = percentiles[1:-1]
-        midpoints = np.percentile(col_data, percentiles, method="midpoint").astype(
-            X_DTYPE
+        bin_thresholds = np.array(
+            [
+                _weighted_percentile(col_data, sample_weight, percentile, average=True)
+                for percentile in percentiles
+            ]
         )
-        assert midpoints.shape[0] == max_bins - 1
+        assert bin_thresholds.shape[0] == max_bins - 1
+    # Remove duplicated thresholds if they exist.
+    unique_bin_values = np.unique(bin_thresholds)
+    if unique_bin_values.shape[0] != bin_thresholds.shape[0]:
+        bin_thresholds = unique_bin_values
 
     # We avoid having +inf thresholds: +inf thresholds are only allowed in
     # a "split on nan" situation.
-    np.clip(midpoints, a_min=None, a_max=ALMOST_INF, out=midpoints)
-    return midpoints
+    np.clip(bin_thresholds, a_min=None, a_max=ALMOST_INF, out=bin_thresholds)
+    return bin_thresholds
 
 
 class _BinMapper(TransformerMixin, BaseEstimator):
@@ -175,7 +209,7 @@ def __init__(
         self.random_state = random_state
         self.n_threads = n_threads
 
-    def fit(self, X, y=None):
+    def fit(self, X, y=None, sample_weight=None):
         """Fit data X by computing the binning thresholds.
 
         The last bin is reserved for missing values, whether missing values
@@ -202,12 +236,25 @@ def fit(self, X, y=None):
 
         X = check_array(X, dtype=[X_DTYPE], ensure_all_finite=False)
         max_bins = self.n_bins - 1
-
         rng = check_random_state(self.random_state)
         if self.subsample is not None and X.shape[0] > self.subsample:
-            subset = rng.choice(X.shape[0], self.subsample, replace=False)
+            subsampling_probabilities = None
+            if sample_weight is not None:
+                subsampling_probabilities = sample_weight / np.sum(sample_weight)
+            # Sampling with replacement to implement frequency semantics
+            # for sample weights. Note that we need `replace=True` even when
+            # `sample_weight is None` to make sure that passing no weights is
+            # statistically equivalent to passing unit weights.
+            subset = rng.choice(
+                X.shape[0], self.subsample, p=subsampling_probabilities, replace=True
+            )
             X = X.take(subset, axis=0)
 
+            # Add a switch to replace sample weights with None
+            # since sample weights were already used in subsampling
+            # and should not then be propagated to _find_binning_thresholds
+            sample_weight = None
+
         if self.is_categorical is None:
             self.is_categorical_ = np.zeros(X.shape[1], dtype=np.uint8)
         else:
@@ -238,11 +285,12 @@ def fit(self, X, y=None):
         n_bins_non_missing = [None] * n_features
 
         non_cat_thresholds = Parallel(n_jobs=self.n_threads, backend="threading")(
-            delayed(_find_binning_thresholds)(X[:, f_idx], max_bins)
+            delayed(_find_binning_thresholds)(
+                X[:, f_idx], max_bins, sample_weight=sample_weight
+            )
             for f_idx in range(n_features)
             if not self.is_categorical_[f_idx]
         )
-
         non_cat_idx = 0
         for f_idx in range(n_features):
             if self.is_categorical_[f_idx]:
diff --git a/sklearn/ensemble/_hist_gradient_boosting/gradient_boosting.py b/sklearn/ensemble/_hist_gradient_boosting/gradient_boosting.py
index 4a4fa319f4ab7..a2977cdc6b64c 100644
--- a/sklearn/ensemble/_hist_gradient_boosting/gradient_boosting.py
+++ b/sklearn/ensemble/_hist_gradient_boosting/gradient_boosting.py
@@ -719,9 +719,13 @@ def fit(
             random_state=self._random_seed,
             n_threads=n_threads,
         )
-        X_binned_train = self._bin_data(X_train, is_training_data=True)
+        X_binned_train = self._bin_data(
+            X_train, sample_weight_train, is_training_data=True
+        )
         if X_val is not None:
-            X_binned_val = self._bin_data(X_val, is_training_data=False)
+            X_binned_val = self._bin_data(
+                X_val, sample_weight_val, is_training_data=False
+            )
         else:
             X_binned_val = None
 
@@ -1218,7 +1222,7 @@ def _should_stop(self, scores):
         recent_improvements = [score > reference_score for score in recent_scores]
         return not any(recent_improvements)
 
-    def _bin_data(self, X, is_training_data):
+    def _bin_data(self, X, sample_weight, is_training_data):
         """Bin data X.
 
         If is_training_data, then fit the _bin_mapper attribute.
@@ -1234,7 +1238,9 @@ def _bin_data(self, X, is_training_data):
             )
         tic = time()
         if is_training_data:
-            X_binned = self._bin_mapper.fit_transform(X)  # F-aligned array
+            X_binned = self._bin_mapper.fit_transform(
+                X, sample_weight=sample_weight
+            )  # F-aligned array
         else:
             X_binned = self._bin_mapper.transform(X)  # F-aligned array
             # We convert the array to C-contiguous since predicting is faster
@@ -1736,7 +1742,7 @@ class HistGradientBoostingRegressor(RegressorMixin, BaseHistGradientBoosting):
     >>> X, y = load_diabetes(return_X_y=True)
     >>> est = HistGradientBoostingRegressor().fit(X, y)
     >>> est.score(X, y)
-    0.92...
+    0.93...
     """
 
     _parameter_constraints: dict = {


Test patch:
diff --git a/sklearn/ensemble/_hist_gradient_boosting/tests/test_binning.py b/sklearn/ensemble/_hist_gradient_boosting/tests/test_binning.py
index 6f9fcd0057141..6193594fa74b7 100644
--- a/sklearn/ensemble/_hist_gradient_boosting/tests/test_binning.py
+++ b/sklearn/ensemble/_hist_gradient_boosting/tests/test_binning.py
@@ -1,6 +1,7 @@
 import numpy as np
 import pytest
 from numpy.testing import assert_allclose, assert_array_equal
+from scipy.stats import kstest
 
 from sklearn.ensemble._hist_gradient_boosting.binning import (
     _BinMapper,
@@ -113,6 +114,22 @@ def test_map_to_bins(max_bins):
         assert binned[max_idx, feature_idx] == max_bins - 1
 
 
+def test_unique_bins_repeated_weighted():
+    # Test sample weight equivalence for the degenerate case
+    # when only one unique bin value exists and should be trimmed
+    # due to repeated values. Since the first discrete value of 1
+    # is repeated/weighted 1000 times we expect the quantile bin
+    # threshold values found to be repeated values of 1, which are
+    # trimmed to return a single unique bin threshold of [1.]
+    col_data = np.asarray([1, 2, 3, 4, 5, 6]).reshape(-1, 1)
+    sample_weight = np.asarray([1000, 1, 1, 1, 1, 1])
+    col_data_repeated = np.asarray(([1] * 1000) + [2, 3, 4, 5, 6]).reshape(-1, 1)
+
+    binmapper = _BinMapper(n_bins=4).fit(col_data, sample_weight=sample_weight)
+    binmapper_repeated = _BinMapper(n_bins=4).fit(col_data_repeated)
+    assert_array_equal(binmapper.bin_thresholds_, binmapper_repeated.bin_thresholds_)
+
+
 @pytest.mark.parametrize("max_bins", [5, 10, 42])
 def test_bin_mapper_random_data(max_bins):
     n_samples, n_features = DATA.shape
@@ -198,6 +215,69 @@ def test_bin_mapper_repeated_values_invariance(n_distinct):
     assert_array_equal(binned_1, binned_2)
 
 
+@pytest.mark.parametrize("n_bins", [50, 250])
+def test_binmapper_weighted_vs_repeated_equivalence(global_random_seed, n_bins):
+    rng = np.random.RandomState(global_random_seed)
+
+    n_samples = 200
+    X = rng.randn(n_samples, 3)
+    sw = rng.randint(0, 5, size=n_samples)
+    X_repeated = np.repeat(X, sw, axis=0)
+
+    est_weighted = _BinMapper(n_bins=n_bins).fit(X, sample_weight=sw)
+    est_repeated = _BinMapper(n_bins=n_bins).fit(X_repeated, sample_weight=None)
+    assert_allclose(est_weighted.bin_thresholds_, est_repeated.bin_thresholds_)
+
+    X_trans_weighted = est_weighted.transform(X)
+    X_trans_repeated = est_repeated.transform(X)
+    assert_array_equal(X_trans_weighted, X_trans_repeated)
+
+
+# Note: we use a small number of RNG seeds to check that the tests is not seed
+# dependent while keeping the statistical test valid. If we had used the
+# global_random_seed fixture, it would have been expected to get some wrong
+# rejections of the null hypothesis because of the large number of
+# tests run by the fixture.
+@pytest.mark.parametrize("seed", [0, 1, 42])
+@pytest.mark.parametrize("n_bins", [3, 5])
+def test_subsampled_weighted_vs_repeated_equivalence(seed, n_bins):
+    rng = np.random.RandomState(seed)
+
+    n_samples = 500
+    X = rng.randn(n_samples, 3)
+
+    sw = rng.randint(0, 5, size=n_samples)
+    X_repeated = np.repeat(X, sw, axis=0)
+
+    # Collect estimated bins thresholds on the weighted/repeated datasets for
+    # `n_resampling_iterations` subsampling. `n_resampling_iterations` is large
+    # enough to ensure a well-powered statistical test.
+    n_resampling_iterations = 500
+    bins_weighted = []
+    bins_repeated = []
+    for _ in range(n_resampling_iterations):
+        params = dict(n_bins=n_bins, subsample=300, random_state=rng)
+        est_weighted = _BinMapper(**params).fit(X, sample_weight=sw)
+        est_repeated = _BinMapper(**params).fit(X_repeated, sample_weight=None)
+        bins_weighted.append(np.hstack(est_weighted.bin_thresholds_))
+        bins_repeated.append(np.hstack(est_repeated.bin_thresholds_))
+
+    bins_weighted = np.stack(bins_weighted).T
+    bins_repeated = np.stack(bins_repeated).T
+    # bins_weighted and bins_weighted of shape (n_thresholds, n_subsample)
+    # kstest_pval of shape (n_thresholds,)
+    kstest_pval = np.asarray(
+        [
+            kstest(bin_weighted, bin_repeated).pvalue
+            for bin_weighted, bin_repeated in zip(bins_weighted, bins_repeated)
+        ]
+    )
+    # We should not be able to reject the null hypothesis that the two samples
+    # come from the same distribution for all bins at level 5% with Bonferroni
+    # correction.
+    assert np.min(kstest_pval) > 0.05 / len(kstest_pval)
+
+
 @pytest.mark.parametrize(
     "max_bins, scale, offset",
     [
diff --git a/sklearn/ensemble/_hist_gradient_boosting/tests/test_gradient_boosting.py b/sklearn/ensemble/_hist_gradient_boosting/tests/test_gradient_boosting.py
index e5992e27840b8..cb8cc1971a0d2 100644
--- a/sklearn/ensemble/_hist_gradient_boosting/tests/test_gradient_boosting.py
+++ b/sklearn/ensemble/_hist_gradient_boosting/tests/test_gradient_boosting.py
@@ -708,6 +708,7 @@ def test_zero_sample_weights_classification():
 
     X = [[1, 0], [1, 0], [1, 0], [0, 1], [1, 1]]
     y = [0, 0, 1, 0, 2]
+
     # ignore the first 2 training samples by setting their weight to 0
     sample_weight = [0, 0, 1, 1, 1]
     gb = HistGradientBoostingClassifier(loss="log_loss", min_samples_leaf=1)
@@ -718,16 +719,19 @@ def test_zero_sample_weights_classification():
 @pytest.mark.parametrize(
     "problem", ("regression", "binary_classification", "multiclass_classification")
 )
-@pytest.mark.parametrize("duplication", ("half", "all"))
-def test_sample_weight_effect(problem, duplication):
+def test_sample_weight_effect(problem, global_random_seed):
     # High level test to make sure that duplicating a sample is equivalent to
     # giving it weight of 2.
 
-    # fails for n_samples > 255 because binning does not take sample weights
-    # into account. Keeping n_samples <= 255 makes
-    # sure only unique values are used so SW have no effect on binning.
-    n_samples = 255
+    # This test assumes that subsampling in `_BinMapper` is disabled
+    # (when `n_samples < 2e5`) and therefore the binning results should be
+    # deterministic.
+    # Otherwise, this test would require being rewritten as a statistical test.
+    # We also set `n_samples` large enough to ensure that columns have more than
+    # 255 distinct values and that we test the impact of weight-aware binning.
+    n_samples = 300
     n_features = 2
+    rng = np.random.RandomState(global_random_seed)
     if problem == "regression":
         X, y = make_regression(
             n_samples=n_samples,
@@ -755,21 +759,17 @@ def test_sample_weight_effect(problem, duplication):
     # duplicated samples.
     est = Klass(min_samples_leaf=1)
 
-    # Create dataset with duplicate and corresponding sample weights
-    if duplication == "half":
-        lim = n_samples // 2
-    else:
-        lim = n_samples
-    X_dup = np.r_[X, X[:lim]]
-    y_dup = np.r_[y, y[:lim]]
-    sample_weight = np.ones(shape=(n_samples))
-    sample_weight[:lim] = 2
+    # Create dataset with repetitions and corresponding sample weights
+    sample_weight = rng.randint(0, 3, size=X.shape[0])
+    X_repeated = np.repeat(X, sample_weight, axis=0)
+    assert X_repeated.shape[0] < 2e5
+    y_repeated = np.repeat(y, sample_weight, axis=0)
 
-    est_sw = clone(est).fit(X, y, sample_weight=sample_weight)
-    est_dup = clone(est).fit(X_dup, y_dup)
+    est_weighted = clone(est).fit(X, y, sample_weight=sample_weight)
+    est_repeated = clone(est).fit(X_repeated, y_repeated)
 
     # checking raw_predict is stricter than just predict for classification
-    assert np.allclose(est_sw._raw_predict(X_dup), est_dup._raw_predict(X_dup))
+    assert_allclose(est_weighted._raw_predict(X), est_repeated._raw_predict(X))
 
 
 @pytest.mark.parametrize("Loss", (HalfSquaredError, AbsoluteError))
@@ -1303,7 +1303,7 @@ def test_check_interaction_cst(interaction_cst, n_features, result):
 def test_interaction_cst_numerically():
     """Check that interaction constraints have no forbidden interactions."""
     rng = np.random.RandomState(42)
-    n_samples = 1000
+    n_samples = 2000
     X = rng.uniform(size=(n_samples, 2))
     # Construct y with a strong interaction term
     # y = x0 + x1 + 5 * x0 * x1
diff --git a/sklearn/ensemble/_hist_gradient_boosting/tests/test_predictor.py b/sklearn/ensemble/_hist_gradient_boosting/tests/test_predictor.py
index 3c3c9ae81bac2..ae9770ca14a83 100644
--- a/sklearn/ensemble/_hist_gradient_boosting/tests/test_predictor.py
+++ b/sklearn/ensemble/_hist_gradient_boosting/tests/test_predictor.py
@@ -28,7 +28,7 @@
 @pytest.mark.parametrize("n_bins", [200, 256])
 def test_regression_dataset(n_bins):
     X, y = make_regression(
-        n_samples=500, n_features=10, n_informative=5, random_state=42
+        n_samples=1000, n_features=10, n_informative=5, random_state=42
     )
     X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)