⚠️ CI failed on Linux.ubuntu_atlas ⚠️

**CI is still failing on [Linux.ubuntu_atlas](https://dev.azure.com/scikit-learn/scikit-learn/_build/results?buildId=46059&view=logs&j=0a287ed6-22f4-5cb4-88b1-d5fcdc4d8b7e)** (Aug 30, 2022)
- Test Collection Failure

COMMENT:
**Edit**: should be fixed by #24168 

Hmmm the latest failure is a separate one:
```
test_pairwise_distances_argkmin[80-float32-parallel_on_X-cityblock-1000000.0-500]
test_pairwise_distances_argkmin[80-float32-parallel_on_Y-cityblock-1000000.0-500]
```

Maybe related to https://github.com/scikit-learn/scikit-learn/pull/23865, cc @jjerphan for more visibility.

I can reproduce both failures on a Ubuntu 20.04 box with atlas with GLOBAL_RANDOM_SEED=72 (for some reason with 80 only the parallel_on_X one fails ...):
```
 SKLEARN_RUN_FLOAT32_TESTS=1 SKLEARN_TESTS_GLOBAL_RANDOM_SEED=72 pytest sklearn/metrics/tests/test_pairwise_distances_reduction.py -k 'parallel_on_ and float32 and cityblock and 1000000.0'
```

<details>
<summary>Full pytest failure</summary>

```
========================================================================= FAILURES ==========================================================================
_____________________________________ test_pairwise_distances_argkmin[72-float32-parallel_on_X-cityblock-1000000.0-500] _____________________________________

global_random_seed = 72, n_features = 500, translation = 1000000.0, metric = 'cityblock', strategy = 'parallel_on_X', dtype = <class 'numpy.float32'>
n_samples = 100, k = 10

    @pytest.mark.filterwarnings("ignore:WMinkowskiDistance:FutureWarning:sklearn")
    @pytest.mark.parametrize("n_features", [50, 500])
    @pytest.mark.parametrize("translation", [0, 1e6])
    @pytest.mark.parametrize("metric", CDIST_PAIRWISE_DISTANCES_REDUCTION_COMMON_METRICS)
    @pytest.mark.parametrize("strategy", ("parallel_on_X", "parallel_on_Y"))
    @pytest.mark.parametrize("dtype", [np.float64, np.float32])
    def test_pairwise_distances_argkmin(
        global_random_seed,
        n_features,
        translation,
        metric,
        strategy,
        dtype,
        n_samples=100,
        k=10,
    ):
        # TODO: can we easily fix this discrepancy?
        edge_cases = [
            (np.float32, "chebyshev", 1000000.0),
            (np.float32, "chebyshev", 1000000.0),
        ]
        if (dtype, metric, translation) in edge_cases:
            pytest.xfail("Numerical differences lead to small differences in results.")
    
        rng = np.random.RandomState(global_random_seed)
        spread = 1000
        X = translation + rng.rand(n_samples, n_features).astype(dtype) * spread
        Y = translation + rng.rand(n_samples, n_features).astype(dtype) * spread
    
        # Haversine distance only accepts 2D data
        if metric == "haversine":
            X = np.ascontiguousarray(X[:, :2])
            Y = np.ascontiguousarray(Y[:, :2])
    
        metric_kwargs = _get_metric_params_list(metric, n_features)[0]
    
        # Reference for argkmin results
        if metric == "euclidean":
            # Compare to scikit-learn GEMM optimized implementation
            dist_matrix = euclidean_distances(X, Y)
        else:
            dist_matrix = cdist(X, Y, metric=metric, **metric_kwargs)
        # Taking argkmin (indices of the k smallest values)
        argkmin_indices_ref = np.argsort(dist_matrix, axis=1)[:, :k]
        # Getting the associated distances
        argkmin_distances_ref = np.zeros(argkmin_indices_ref.shape, dtype=np.float64)
        for row_idx in range(argkmin_indices_ref.shape[0]):
            argkmin_distances_ref[row_idx] = dist_matrix[
                row_idx, argkmin_indices_ref[row_idx]
            ]
    
        argkmin_distances, argkmin_indices = PairwiseDistancesArgKmin.compute(
            X,
            Y,
            k,
            metric=metric,
            metric_kwargs=metric_kwargs,
            return_distance=True,
            # So as to have more than a chunk, forcing parallelism.
            chunk_size=n_samples // 4,
            strategy=strategy,
        )
    
>       ASSERT_RESULT[(PairwiseDistancesArgKmin, dtype)](
            argkmin_distances,
            argkmin_distances_ref,
            argkmin_indices,
            argkmin_indices_ref,
        )

sklearn/metrics/tests/test_pairwise_distances_reduction.py:919: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

ref_dist = array([[155344.875 , 155930.1875, 156232.6875, 157066.75  , 157164.8125,
        157680.3125, 157733.4375, 158197.25  ...4375, 154084.75  , 154663.4375, 155331.    ,
        156545.8125, 156738.125 , 157577.4375, 157760.375 , 158019.8125]])
dist = array([[155344.875 , 155930.1875, 156232.6875, 157066.75  , 157164.8125,
        157680.3125, 157733.4375, 158197.25  ...4375, 154084.75  , 154663.4375, 155331.    ,
        156545.8125, 156738.125 , 157577.4375, 157760.375 , 158019.8125]])
ref_indices = array([[66, 61, 96, 60, 76, 22, 38, 89, 97, 37],
       [59, 26, 47, 50,  4, 14, 69, 46, 60, 76],
       [27, 38, 22, ...38, 37, 34, 70, 87],
       [29, 78, 22, 27, 73, 56, 45, 15, 97,  0],
       [60, 98, 16, 61, 66, 76,  6, 68, 26, 59]])
indices = array([[66, 61, 96, 60, 76, 22, 38, 89, 97, 37],
       [59, 26, 47, 50,  4, 14, 69, 46, 60, 76],
       [27, 38, 22, ...38, 37, 34, 70, 87],
       [29, 78, 22, 27, 73, 56, 45, 15, 97,  0],
       [60, 98, 16, 61, 66, 76,  6, 68, 26, 59]])
rtol = 0.0001

    def assert_argkmin_results_quasi_equality(
        ref_dist,
        dist,
        ref_indices,
        indices,
        rtol=1e-4,
    ):
        """Assert that argkmin results are valid up to:
          - relative tolerance on computed distance values
          - permutations of indices for distances values that differ up to
            a precision level
    
        To be used for testing neighbors queries on float32 datasets: we
        accept neighbors rank swaps only if they are caused by small
        rounding errors on the distance computations.
        """
        is_sorted = lambda a: np.all(a[:-1] <= a[1:])
    
        n_significant_digits = -(int(floor(log10(abs(rtol)))) + 1)
    
        assert (
            ref_dist.shape == dist.shape == ref_indices.shape == indices.shape
        ), "Arrays of results have various shapes."
    
        n_queries, n_neighbors = ref_dist.shape
    
        # Asserting equality results one row at a time
        for query_idx in range(n_queries):
            ref_dist_row = ref_dist[query_idx]
            dist_row = dist[query_idx]
    
            assert is_sorted(
                ref_dist_row
            ), f"Reference distances aren't sorted on row {query_idx}"
            assert is_sorted(dist_row), f"Distances aren't sorted on row {query_idx}"
    
            assert_allclose(ref_dist_row, dist_row, rtol=rtol)
    
            ref_indices_row = ref_indices[query_idx]
            indices_row = indices[query_idx]
    
            # Grouping indices by distances using sets on a rounded distances up
            # to a given number of decimals of significant digits derived from rtol.
            reference_neighbors_groups = defaultdict(set)
            effective_neighbors_groups = defaultdict(set)
    
            for neighbor_rank in range(n_neighbors):
                rounded_dist = relative_rounding(
                    ref_dist_row[neighbor_rank],
                    n_significant_digits=n_significant_digits,
                )
                reference_neighbors_groups[rounded_dist].add(ref_indices_row[neighbor_rank])
                effective_neighbors_groups[rounded_dist].add(indices_row[neighbor_rank])
    
            # Asserting equality of groups (sets) for each distance
            msg = (
                f"Neighbors indices for query {query_idx} are not matching "
                f"when rounding distances at {n_significant_digits} significant digits "
                f"derived from rtol={rtol:.1e}"
            )
            for rounded_distance in reference_neighbors_groups.keys():
>               assert (
                    reference_neighbors_groups[rounded_distance]
                    == effective_neighbors_groups[rounded_distance]
                ), msg
E               AssertionError: Neighbors indices for query 39 are not matching when rounding distances at 3 significant digits derived from rtol=1.0e-04
E               assert {33, 60, 76, 99} == {41, 60, 76, 99}
E                 Extra items in the left set:
E                 33
E                 Extra items in the right set:
E                 41
E                 Use -v to get more diff

sklearn/metrics/tests/test_pairwise_distances_reduction.py:181: AssertionError
_____________________________________ test_pairwise_distances_argkmin[72-float32-parallel_on_Y-cityblock-1000000.0-500] _____________________________________

global_random_seed = 72, n_features = 500, translation = 1000000.0, metric = 'cityblock', strategy = 'parallel_on_Y', dtype = <class 'numpy.float32'>
n_samples = 100, k = 10

    @pytest.mark.filterwarnings("ignore:WMinkowskiDistance:FutureWarning:sklearn")
    @pytest.mark.parametrize("n_features", [50, 500])
    @pytest.mark.parametrize("translation", [0, 1e6])
    @pytest.mark.parametrize("metric", CDIST_PAIRWISE_DISTANCES_REDUCTION_COMMON_METRICS)
    @pytest.mark.parametrize("strategy", ("parallel_on_X", "parallel_on_Y"))
    @pytest.mark.parametrize("dtype", [np.float64, np.float32])
    def test_pairwise_distances_argkmin(
        global_random_seed,
        n_features,
        translation,
        metric,
        strategy,
        dtype,
        n_samples=100,
        k=10,
    ):
        # TODO: can we easily fix this discrepancy?
        edge_cases = [
            (np.float32, "chebyshev", 1000000.0),
            (np.float32, "chebyshev", 1000000.0),
        ]
        if (dtype, metric, translation) in edge_cases:
            pytest.xfail("Numerical differences lead to small differences in results.")
    
        rng = np.random.RandomState(global_random_seed)
        spread = 1000
        X = translation + rng.rand(n_samples, n_features).astype(dtype) * spread
        Y = translation + rng.rand(n_samples, n_features).astype(dtype) * spread
    
        # Haversine distance only accepts 2D data
        if metric == "haversine":
            X = np.ascontiguousarray(X[:, :2])
            Y = np.ascontiguousarray(Y[:, :2])
    
        metric_kwargs = _get_metric_params_list(metric, n_features)[0]
    
        # Reference for argkmin results
        if metric == "euclidean":
            # Compare to scikit-learn GEMM optimized implementation
            dist_matrix = euclidean_distances(X, Y)
        else:
            dist_matrix = cdist(X, Y, metric=metric, **metric_kwargs)
        # Taking argkmin (indices of the k smallest values)
        argkmin_indices_ref = np.argsort(dist_matrix, axis=1)[:, :k]
        # Getting the associated distances
        argkmin_distances_ref = np.zeros(argkmin_indices_ref.shape, dtype=np.float64)
        for row_idx in range(argkmin_indices_ref.shape[0]):
            argkmin_distances_ref[row_idx] = dist_matrix[
                row_idx, argkmin_indices_ref[row_idx]
            ]
    
        argkmin_distances, argkmin_indices = PairwiseDistancesArgKmin.compute(
            X,
            Y,
            k,
            metric=metric,
            metric_kwargs=metric_kwargs,
            return_distance=True,
            # So as to have more than a chunk, forcing parallelism.
            chunk_size=n_samples // 4,
            strategy=strategy,
        )
    
>       ASSERT_RESULT[(PairwiseDistancesArgKmin, dtype)](
            argkmin_distances,
            argkmin_distances_ref,
            argkmin_indices,
            argkmin_indices_ref,
        )

sklearn/metrics/tests/test_pairwise_distances_reduction.py:919: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

ref_dist = array([[155344.875 , 155930.1875, 156232.6875, 157066.75  , 157164.8125,
        157680.3125, 157733.4375, 158197.25  ...4375, 154084.75  , 154663.4375, 155331.    ,
        156545.8125, 156738.125 , 157577.4375, 157760.375 , 158019.8125]])
dist = array([[155344.875 , 155930.1875, 156232.6875, 157066.75  , 157164.8125,
        157680.3125, 157733.4375, 158197.25  ...4375, 154084.75  , 154663.4375, 155331.    ,
        156545.8125, 156738.125 , 157577.4375, 157760.375 , 158019.8125]])
ref_indices = array([[66, 61, 96, 60, 76, 22, 38, 89, 97, 37],
       [59, 26, 47, 50,  4, 14, 69, 46, 60, 76],
       [27, 38, 22, ...38, 37, 34, 70, 87],
       [29, 78, 22, 27, 73, 56, 45, 15, 97,  0],
       [60, 98, 16, 61, 66, 76,  6, 68, 26, 59]])
indices = array([[66, 61, 96, 60, 76, 22, 38, 89, 97, 37],
       [59, 26, 47, 50,  4, 14, 69, 46, 60, 76],
       [27, 38, 22, ...38, 37, 34, 70, 87],
       [29, 78, 22, 27, 73, 56, 45, 15, 97,  0],
       [60, 98, 16, 61, 66, 76,  6, 68, 26, 59]])
rtol = 0.0001

    def assert_argkmin_results_quasi_equality(
        ref_dist,
        dist,
        ref_indices,
        indices,
        rtol=1e-4,
    ):
        """Assert that argkmin results are valid up to:
          - relative tolerance on computed distance values
          - permutations of indices for distances values that differ up to
            a precision level
    
        To be used for testing neighbors queries on float32 datasets: we
        accept neighbors rank swaps only if they are caused by small
        rounding errors on the distance computations.
        """
        is_sorted = lambda a: np.all(a[:-1] <= a[1:])
    
        n_significant_digits = -(int(floor(log10(abs(rtol)))) + 1)
    
        assert (
            ref_dist.shape == dist.shape == ref_indices.shape == indices.shape
        ), "Arrays of results have various shapes."
    
        n_queries, n_neighbors = ref_dist.shape
    
        # Asserting equality results one row at a time
        for query_idx in range(n_queries):
            ref_dist_row = ref_dist[query_idx]
            dist_row = dist[query_idx]
    
            assert is_sorted(
                ref_dist_row
            ), f"Reference distances aren't sorted on row {query_idx}"
            assert is_sorted(dist_row), f"Distances aren't sorted on row {query_idx}"
    
            assert_allclose(ref_dist_row, dist_row, rtol=rtol)
    
            ref_indices_row = ref_indices[query_idx]
            indices_row = indices[query_idx]
    
            # Grouping indices by distances using sets on a rounded distances up
            # to a given number of decimals of significant digits derived from rtol.
            reference_neighbors_groups = defaultdict(set)
            effective_neighbors_groups = defaultdict(set)
    
            for neighbor_rank in range(n_neighbors):
                rounded_dist = relative_rounding(
                    ref_dist_row[neighbor_rank],
                    n_significant_digits=n_significant_digits,
                )
                reference_neighbors_groups[rounded_dist].add(ref_indices_row[neighbor_rank])
                effective_neighbors_groups[rounded_dist].add(indices_row[neighbor_rank])
    
            # Asserting equality of groups (sets) for each distance
            msg = (
                f"Neighbors indices for query {query_idx} are not matching "
                f"when rounding distances at {n_significant_digits} significant digits "
                f"derived from rtol={rtol:.1e}"
            )
            for rounded_distance in reference_neighbors_groups.keys():
>               assert (
                    reference_neighbors_groups[rounded_distance]
                    == effective_neighbors_groups[rounded_distance]
                ), msg
E               AssertionError: Neighbors indices for query 39 are not matching when rounding distances at 3 significant digits derived from rtol=1.0e-04
E               assert {33, 60, 76, 99} == {41, 60, 76, 99}
E                 Extra items in the left set:
E                 33
E                 Extra items in the right set:
E                 41
E                 Use -v to get more diff

sklearn/metrics/tests/test_pairwise_distances_reduction.py:181: AssertionError
======================================================== 2 failed, 6 passed, 426 deselected in 0.53s ========================================================

```

</details>

Labels: Build / CI