Repository: adap/flower

Issue:
Implement Krum as a new strategy
### Describe the type of feature and its functionality.

Krum is described [here](https://proceedings.neurips.cc/paper/2017/file/f4b9ec30ad9f68f89b29639786cb62ef-Paper.pdf).

### Describe step by step what files and adjustments are you planning to include.

I am planning to create two new files in the strategy directory: krum.py and krum_test.py

### Is there something else you want to add?

_No response_
Implement MultiKrum as a new strategy
### Describe the type of feature and its functionality.

MultiKrum is described [here](https://proceedings.neurips.cc/paper/2017/file/f4b9ec30ad9f68f89b29639786cb62ef-Paper.pdf).

### Describe step by step what files and adjustments are you planning to include.

I am planning to create two new files in the strategy directory: multikrum.py and multikrum_test.py

### Is there something else you want to add?

_No response_


Discussion:

Resolving patch:
diff --git a/src/py/flwr/server/strategy/aggregate.py b/src/py/flwr/server/strategy/aggregate.py
--- a/src/py/flwr/server/strategy/aggregate.py
+++ b/src/py/flwr/server/strategy/aggregate.py
@@ -20,7 +20,7 @@
 
 import numpy as np
 
-from flwr.common import NDArrays
+from flwr.common import NDArray, NDArrays
 
 
 def aggregate(results: List[Tuple[NDArrays, int]]) -> NDArrays:
@@ -53,6 +53,44 @@ def aggregate_median(results: List[Tuple[NDArrays, int]]) -> NDArrays:
     return median_w
 
 
+def aggregate_krum(
+    results: List[Tuple[NDArrays, int]], num_malicious: int, to_keep: int
+) -> NDArrays:
+    """Choose one parameter vector according to the Krum fucntion.
+
+    If to_keep is not None, then MultiKrum is applied.
+    """
+    # Create a list of weights and ignore the number of examples
+    weights = [weights for weights, _ in results]
+
+    # Compute distances between vectors
+    distance_matrix = _compute_distances(weights)
+
+    # For each client, take the n-f-2 closest parameters vectors
+    num_closest = max(1, len(weights) - num_malicious - 2)
+    closest_indices = []
+    for i, _ in enumerate(distance_matrix):
+        closest_indices.append(
+            np.argsort(distance_matrix[i])[1 : num_closest + 1].tolist()  # noqa: E203
+        )
+
+    # Compute the score for each client, that is the sum of the distances
+    # of the n-f-2 closest parameters vectors
+    scores = [
+        np.sum(distance_matrix[i, closest_indices[i]])
+        for i in range(len(distance_matrix))
+    ]
+
+    if to_keep > 0:
+        # Choose to_keep clients and return their average (MultiKrum)
+        best_indices = np.argsort(scores)[::-1][len(scores) - to_keep :]  # noqa: E203
+        best_results = [results[i] for i in best_indices]
+        return aggregate(best_results)
+
+    # Return the index of the client which minimizes the score (Krum)
+    return weights[np.argmin(scores)]
+
+
 def weighted_loss_avg(results: List[Tuple[int, float]]) -> float:
     """Aggregate evaluation results obtained from multiple clients."""
     num_total_evaluation_examples = sum([num_examples for num_examples, _ in results])
@@ -76,3 +114,21 @@ def aggregate_qffl(
         updates.append(tmp)
     new_parameters = [(u - v) * 1.0 for u, v in zip(parameters, updates)]
     return new_parameters
+
+
+def _compute_distances(weights: List[NDArrays]) -> NDArray:
+    """Compute distances between vectors.
+
+    Input: weights - list of weights vectors
+    Output: distances - matrix distance_matrix of squared distances between the vectors
+    """
+    flat_w = np.array(
+        [np.concatenate(p, axis=None).ravel() for p in weights]  # type: ignore
+    )
+    distance_matrix = np.zeros((len(weights), len(weights)))
+    for i, _ in enumerate(flat_w):
+        for j, _ in enumerate(flat_w):
+            delta = flat_w[i] - flat_w[j]
+            norm = np.linalg.norm(delta)  # type: ignore
+            distance_matrix[i, j] = norm**2
+    return distance_matrix
diff --git a/src/py/flwr/server/strategy/krum.py b/src/py/flwr/server/strategy/krum.py
new file mode 100644
--- /dev/null
+++ b/src/py/flwr/server/strategy/krum.py
@@ -0,0 +1,167 @@
+# Copyright 2020 Adap GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Machine Learning with Adversaries: Byzantine Tolerant Gradient Descent.
+
+[Blanchard et al., 2017].
+
+Paper: https://proceedings.neurips.cc/paper/2017/file/f4b9ec30ad9f68f89b29639786cb62ef-Paper.pdf
+"""
+
+
+from logging import WARNING
+from typing import Callable, Dict, List, Optional, Tuple, Union
+
+from flwr.common import (
+    FitRes,
+    MetricsAggregationFn,
+    NDArrays,
+    Parameters,
+    Scalar,
+    ndarrays_to_parameters,
+    parameters_to_ndarrays,
+)
+from flwr.common.logger import log
+from flwr.server.client_proxy import ClientProxy
+
+from .aggregate import aggregate_krum
+from .fedavg import FedAvg
+
+WARNING_MIN_AVAILABLE_CLIENTS_TOO_LOW = """
+Setting `min_available_clients` lower than `min_fit_clients` or
+`min_evaluate_clients` can cause the server to fail when there are too few clients
+connected to the server. `min_available_clients` must be set to a value larger
+than or equal to the values of `min_fit_clients` and `min_evaluate_clients`.
+"""
+
+# flake8: noqa: E501
+class Krum(FedAvg):
+    """Configurable Krum strategy implementation."""
+
+    # pylint: disable=too-many-arguments,too-many-instance-attributes,line-too-long
+    def __init__(
+        self,
+        *,
+        fraction_fit: float = 1.0,
+        fraction_evaluate: float = 1.0,
+        min_fit_clients: int = 2,
+        min_evaluate_clients: int = 2,
+        min_available_clients: int = 2,
+        num_malicious_clients: int = 0,
+        num_clients_to_keep: int = 0,
+        evaluate_fn: Optional[
+            Callable[
+                [int, NDArrays, Dict[str, Scalar]],
+                Optional[Tuple[float, Dict[str, Scalar]]],
+            ]
+        ] = None,
+        on_fit_config_fn: Optional[Callable[[int], Dict[str, Scalar]]] = None,
+        on_evaluate_config_fn: Optional[Callable[[int], Dict[str, Scalar]]] = None,
+        accept_failures: bool = True,
+        initial_parameters: Optional[Parameters] = None,
+        fit_metrics_aggregation_fn: Optional[MetricsAggregationFn] = None,
+        evaluate_metrics_aggregation_fn: Optional[MetricsAggregationFn] = None,
+    ) -> None:
+        """Configurable Krum strategy.
+
+        Parameters
+        ----------
+        fraction_fit : float, optional
+            Fraction of clients used during training. Defaults to 0.1.
+        fraction_evaluate : float, optional
+            Fraction of clients used during validation. Defaults to 0.1.
+        min_fit_clients : int, optional
+            Minimum number of clients used during training. Defaults to 2.
+        min_evaluate_clients : int, optional
+            Minimum number of clients used during validation. Defaults to 2.
+        min_available_clients : int, optional
+            Minimum number of total clients in the system. Defaults to 2.
+        num_malicious_clients : int, optional
+            Number of malicious clients in the system. Defaults to 0.
+        num_clients_to_keep : int, optional
+            Number of clients to keep before averaging (MultiKrum). Defaults to 0, in that case classical Krum is applied.
+        evaluate_fn : Optional[Callable[[int, NDArrays, Dict[str, Scalar]], Optional[Tuple[float, Dict[str, Scalar]]]]]
+            Optional function used for validation. Defaults to None.
+        on_fit_config_fn : Callable[[int], Dict[str, Scalar]], optional
+            Function used to configure training. Defaults to None.
+        on_evaluate_config_fn : Callable[[int], Dict[str, Scalar]], optional
+            Function used to configure validation. Defaults to None.
+        accept_failures : bool, optional
+            Whether or not accept rounds containing failures. Defaults to True.
+        initial_parameters : Parameters, optional
+            Initial global model parameters.
+        """
+
+        if (
+            min_fit_clients > min_available_clients
+            or min_evaluate_clients > min_available_clients
+        ):
+            log(WARNING, WARNING_MIN_AVAILABLE_CLIENTS_TOO_LOW)
+
+        super().__init__(
+            fraction_fit=fraction_fit,
+            fraction_evaluate=fraction_evaluate,
+            min_fit_clients=min_fit_clients,
+            min_evaluate_clients=min_evaluate_clients,
+            min_available_clients=min_available_clients,
+            evaluate_fn=evaluate_fn,
+            on_fit_config_fn=on_fit_config_fn,
+            on_evaluate_config_fn=on_evaluate_config_fn,
+            accept_failures=accept_failures,
+            initial_parameters=initial_parameters,
+            fit_metrics_aggregation_fn=fit_metrics_aggregation_fn,
+            evaluate_metrics_aggregation_fn=evaluate_metrics_aggregation_fn,
+        )
+        self.fit_metrics_aggregation_fn = fit_metrics_aggregation_fn
+        self.evaluate_metrics_aggregation_fn = evaluate_metrics_aggregation_fn
+        self.num_malicious_clients = num_malicious_clients
+        self.num_clients_to_keep = num_clients_to_keep
+
+    def __repr__(self) -> str:
+        rep = f"Krum(accept_failures={self.accept_failures})"
+        return rep
+
+    def aggregate_fit(
+        self,
+        server_round: int,
+        results: List[Tuple[ClientProxy, FitRes]],
+        failures: List[Union[Tuple[ClientProxy, FitRes], BaseException]],
+    ) -> Tuple[Optional[Parameters], Dict[str, Scalar]]:
+        """Aggregate fit results using Krum."""
+        if not results:
+            return None, {}
+        # Do not aggregate if there are failures and failures are not accepted
+        if not self.accept_failures and failures:
+            return None, {}
+
+        # Convert results
+        weights_results = [
+            (parameters_to_ndarrays(fit_res.parameters), fit_res.num_examples)
+            for _, fit_res in results
+        ]
+        parameters_aggregated = ndarrays_to_parameters(
+            aggregate_krum(
+                weights_results, self.num_malicious_clients, self.num_clients_to_keep
+            )
+        )
+
+        # Aggregate custom metrics if aggregation fn was provided
+        metrics_aggregated = {}
+        if self.fit_metrics_aggregation_fn:
+            fit_metrics = [(res.num_examples, res.metrics) for _, res in results]
+            metrics_aggregated = self.fit_metrics_aggregation_fn(fit_metrics)
+        elif server_round == 1:  # Only log this warning once
+            log(WARNING, "No fit_metrics_aggregation_fn provided")
+
+        return parameters_aggregated, metrics_aggregated


Test patch:
diff --git a/src/py/flwr/server/strategy/krum_test.py b/src/py/flwr/server/strategy/krum_test.py
new file mode 100644
--- /dev/null
+++ b/src/py/flwr/server/strategy/krum_test.py
@@ -0,0 +1,200 @@
+# Copyright 2020 Adap GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Krum tests."""
+
+from typing import List, Tuple
+from unittest.mock import MagicMock
+
+from numpy import array, float32
+
+from flwr.common import (
+    Code,
+    FitRes,
+    NDArrays,
+    Parameters,
+    Status,
+    ndarrays_to_parameters,
+    parameters_to_ndarrays,
+)
+from flwr.server.client_proxy import ClientProxy
+from flwr.server.grpc_server.grpc_client_proxy import GrpcClientProxy
+
+from .krum import Krum
+
+
+def test_krum_num_fit_clients_20_available() -> None:
+    """Test num_fit_clients function."""
+    # Prepare
+    strategy = Krum()
+    expected = 20
+
+    # Execute
+    actual, _ = strategy.num_fit_clients(num_available_clients=20)
+
+    # Assert
+    assert expected == actual
+
+
+def test_krum_num_fit_clients_19_available() -> None:
+    """Test num_fit_clients function."""
+    # Prepare
+    strategy = Krum()
+    expected = 19
+
+    # Execute
+    actual, _ = strategy.num_fit_clients(num_available_clients=19)
+
+    # Assert
+    assert expected == actual
+
+
+def test_krum_num_fit_clients_10_available() -> None:
+    """Test num_fit_clients function."""
+    # Prepare
+    strategy = Krum()
+    expected = 10
+
+    # Execute
+    actual, _ = strategy.num_fit_clients(num_available_clients=10)
+
+    # Assert
+    assert expected == actual
+
+
+def test_krum_num_fit_clients_minimum() -> None:
+    """Test num_fit_clients function."""
+    # Prepare
+    strategy = Krum()
+    expected = 9
+
+    # Execute
+    actual, _ = strategy.num_fit_clients(num_available_clients=9)
+
+    # Assert
+    assert expected == actual
+
+
+def test_krum_num_evaluation_clients_40_available() -> None:
+    """Test num_evaluation_clients function."""
+    # Prepare
+    strategy = Krum(fraction_evaluate=0.05)
+    expected = 2
+
+    # Execute
+    actual, _ = strategy.num_evaluation_clients(num_available_clients=40)
+
+    # Assert
+    assert expected == actual
+
+
+def test_krum_num_evaluation_clients_39_available() -> None:
+    """Test num_evaluation_clients function."""
+    # Prepare
+    strategy = Krum(fraction_evaluate=0.05)
+    expected = 2
+
+    # Execute
+    actual, _ = strategy.num_evaluation_clients(num_available_clients=39)
+
+    # Assert
+    assert expected == actual
+
+
+def test_krum_num_evaluation_clients_20_available() -> None:
+    """Test num_evaluation_clients function."""
+    # Prepare
+    strategy = Krum(fraction_evaluate=0.05)
+    expected = 2
+
+    # Execute
+    actual, _ = strategy.num_evaluation_clients(num_available_clients=20)
+
+    # Assert
+    assert expected == actual
+
+
+def test_krum_num_evaluation_clients_minimum() -> None:
+    """Test num_evaluation_clients function."""
+    # Prepare
+    strategy = Krum(fraction_evaluate=0.05)
+    expected = 2
+
+    # Execute
+    actual, _ = strategy.num_evaluation_clients(num_available_clients=19)
+
+    # Assert
+    assert expected == actual
+
+
+def test_aggregate_fit() -> None:
+    """Tests if Krum is aggregating correctly."""
+    # Prepare
+    previous_weights: NDArrays = [array([0.1, 0.1, 0.1, 0.1], dtype=float32)]
+    strategy = Krum(
+        initial_parameters=ndarrays_to_parameters(previous_weights),
+        num_malicious_clients=1,
+    )
+    param_0: Parameters = ndarrays_to_parameters(
+        [array([0.2, 0.2, 0.2, 0.2], dtype=float32)]
+    )
+    param_1: Parameters = ndarrays_to_parameters(
+        [array([0.7, 0.7, 0.7, 0.7], dtype=float32)]
+    )
+    param_2: Parameters = ndarrays_to_parameters(
+        [array([1.0, 1.0, 1.0, 1.0], dtype=float32)]
+    )
+    bridge = MagicMock()
+    client_0 = GrpcClientProxy(cid="0", bridge=bridge)
+    client_1 = GrpcClientProxy(cid="1", bridge=bridge)
+    client_2 = GrpcClientProxy(cid="2", bridge=bridge)
+    results: List[Tuple[ClientProxy, FitRes]] = [
+        (
+            client_0,
+            FitRes(
+                status=Status(code=Code.OK, message="Success"),
+                parameters=param_0,
+                num_examples=5,
+                metrics={},
+            ),
+        ),
+        (
+            client_1,
+            FitRes(
+                status=Status(code=Code.OK, message="Success"),
+                parameters=param_1,
+                num_examples=5,
+                metrics={},
+            ),
+        ),
+        (
+            client_2,
+            FitRes(
+                status=Status(code=Code.OK, message="Success"),
+                parameters=param_2,
+                num_examples=5,
+                metrics={},
+            ),
+        ),
+    ]
+    expected: NDArrays = [array([0.7, 0.7, 0.7, 0.7], dtype=float32)]
+
+    # Execute
+    actual_aggregated, _ = strategy.aggregate_fit(
+        server_round=1, results=results, failures=[]
+    )
+    if actual_aggregated:
+        actual_list = parameters_to_ndarrays(actual_aggregated)
+        actual = actual_list[0]
+    assert (actual == expected[0]).all()
diff --git a/src/py/flwr/server/strategy/multikrum_test.py b/src/py/flwr/server/strategy/multikrum_test.py
new file mode 100644
--- /dev/null
+++ b/src/py/flwr/server/strategy/multikrum_test.py
@@ -0,0 +1,99 @@
+# Copyright 2020 Adap GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Krum tests."""
+
+from typing import List, Tuple
+from unittest.mock import MagicMock
+
+from numpy import array, float32
+
+from flwr.common import (
+    Code,
+    FitRes,
+    NDArrays,
+    Parameters,
+    Status,
+    ndarrays_to_parameters,
+    parameters_to_ndarrays,
+)
+from flwr.server.client_proxy import ClientProxy
+from flwr.server.grpc_server.grpc_client_proxy import GrpcClientProxy
+
+from .krum import Krum
+
+
+def test_aggregate_fit() -> None:
+    """Tests if MultiKrum is aggregating correctly."""
+    # Prepare
+    previous_weights: NDArrays = [array([0.1, 0.1, 0.1, 0.1], dtype=float32)]
+    strategy = Krum(
+        initial_parameters=ndarrays_to_parameters(previous_weights),
+        num_malicious_clients=1,
+        num_clients_to_keep=2,
+    )
+    param_0: Parameters = ndarrays_to_parameters(
+        [array([0.2, 0.2, 0.2, 0.2], dtype=float32)]
+    )
+    param_1: Parameters = ndarrays_to_parameters(
+        [array([0.5, 0.5, 0.5, 0.5], dtype=float32)]
+    )
+    param_2: Parameters = ndarrays_to_parameters(
+        [array([1.0, 1.0, 1.0, 1.0], dtype=float32)]
+    )
+
+    bridge = MagicMock()
+    client_0 = GrpcClientProxy(cid="0", bridge=bridge)
+    client_1 = GrpcClientProxy(cid="1", bridge=bridge)
+    client_2 = GrpcClientProxy(cid="2", bridge=bridge)
+
+    results: List[Tuple[ClientProxy, FitRes]] = [
+        (
+            client_0,
+            FitRes(
+                status=Status(code=Code.OK, message="Success"),
+                parameters=param_0,
+                num_examples=5,
+                metrics={},
+            ),
+        ),
+        (
+            client_1,
+            FitRes(
+                status=Status(code=Code.OK, message="Success"),
+                parameters=param_1,
+                num_examples=5,
+                metrics={},
+            ),
+        ),
+        (
+            client_2,
+            FitRes(
+                status=Status(code=Code.OK, message="Success"),
+                parameters=param_2,
+                num_examples=5,
+                metrics={},
+            ),
+        ),
+    ]
+    expected: NDArrays = [array([0.35, 0.35, 0.35, 0.35], dtype=float32)]
+
+    # Execute
+    actual_aggregated, _ = strategy.aggregate_fit(
+        server_round=1, results=results, failures=[]
+    )
+    if actual_aggregated:
+        actual_list = parameters_to_ndarrays(actual_aggregated)
+        actual = actual_list[0]
+    assert (actual == expected[0]).all()