internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: adap/flower

Issue:
Upgrade to PyTorch 1.6
Upgrade all torch dependency to 1.6 (torchvision 0.7) which hopefully improves PyTorch-related mypy type checks.


Discussion:

Resolving patch:
diff --git a/src/py/flwr_example/pytorch/cifar.py b/src/py/flwr_example/pytorch/cifar.py
--- a/src/py/flwr_example/pytorch/cifar.py
+++ b/src/py/flwr_example/pytorch/cifar.py
@@ -22,6 +22,7 @@
 
 
 # mypy: ignore-errors
+# pylint: disable=W0223
 
 
 from collections import OrderedDict
diff --git a/src/py/flwr_example/quickstart/client.py b/src/py/flwr_example/quickstart/client.py
--- a/src/py/flwr_example/quickstart/client.py
+++ b/src/py/flwr_example/quickstart/client.py
@@ -43,7 +43,7 @@ def evaluate(self, weights, config):
     x_train, x_test = x_train / 255.0, x_test / 255.0
 
     # Instanstiate client
-    client = MnistClient("0", model, x_train, y_train, x_test, y_test)
+    client = MnistClient("0", model, x_train, y_train, x_test, y_test)  # type: ignore
 
     # Start client
     fl.app.client.start_keras_client(server_address="[::]:8080", client=client)
diff --git a/src/py/flwr_experimental/logserver/server.py b/src/py/flwr_experimental/logserver/server.py
--- a/src/py/flwr_experimental/logserver/server.py
+++ b/src/py/flwr_experimental/logserver/server.py
@@ -25,7 +25,7 @@
 from http.server import BaseHTTPRequestHandler, HTTPServer
 from pathlib import Path
 from threading import Event, Thread
-from typing import Dict, List, Optional, Tuple, Union, cast
+from typing import Dict, List, Optional, Tuple, Union
 
 import boto3
 import numpy as np
@@ -136,7 +136,7 @@ def plot_accuracies(values: Accuracies) -> str:
         filename=filename,
     )
     upload_file(local_path, filename + ".pdf")
-    return cast(str, local_path)
+    return local_path
 
 
 class RequestHandler(BaseHTTPRequestHandler):


Test patch:
diff --git a/dev/test.sh b/dev/test.sh
--- a/dev/test.sh
+++ b/dev/test.sh
@@ -6,7 +6,7 @@ echo "=== test.sh ==="
 
 isort --skip src/py/flwr/proto --check-only -rc src/py/flwr    && echo "- isort:  done" &&
 black -q --exclude "src\/py\/flwr\/proto" --check src/py/flwr  && echo "- black:  done" &&
-# mypy src/py                                                  && echo "- mypy:   done" &&
-pylint --ignore=src/py/flwr/proto src/py/flwr                     && echo "- pylint: done" &&
+mypy src/py                                                    && echo "- mypy:   done" &&
+pylint --ignore=src/py/flwr/proto src/py/flwr                  && echo "- pylint: done" &&
 pytest -q src/py/flwr                                          && echo "- pytest: done" &&
 echo "- All Python checks passed"
Repository: adap/flower

Issue:
Improve docstring for `start_server`



Discussion:

Resolving patch:
diff --git a/src/py/flwr/server/app.py b/src/py/flwr/server/app.py
--- a/src/py/flwr/server/app.py
+++ b/src/py/flwr/server/app.py
@@ -35,7 +35,33 @@ def start_server(
     strategy: Optional[Strategy] = None,
     grpc_max_message_length: int = GRPC_MAX_MESSAGE_LENGTH,
 ) -> None:
-    """Start a Flower server using the gRPC transport layer."""
+    """Start a Flower server using the gRPC transport layer.
+
+    Arguments:
+        server_address: Optional[str] (default: `"[::]:8080"`). The IPv6
+            address of the server.
+        server: Optional[flwr.server.Server] (default: None). An implementation
+            of the abstract base class `flwr.server.Server`. If no instance is
+            provided, then `start_server` will create one.
+        config: Optional[Dict[str, int]] (default: None). The only currently
+            supported values is `num_rounds`, so a full configuration object
+            instructing the server to perform three rounds of federated
+            learning looks like the following: `{"num_rounds": 3}`.
+        strategy: Optional[flwr.server.Strategy] (default: None). An
+            implementation of the abstract base class `flwr.server.Strategy`.
+            If no strategy is provided, then `start_server` will use
+            `flwr.server.strategy.FedAvg`.
+        grpc_max_message_length: int (default: 536_870_912, this equals 512MB).
+            The maximum length of gRPC messages that can be exchanged with the
+            Flower clients. The default should be sufficient for most models.
+            Users who train very large models might need to increase this
+            value. Note that the Flower clients needs to started with the same
+            value (see `flwr.client.start_client`), otherwise clients will not
+            know about the increased limit and block larger messages.
+
+    Returns:
+        None.
+    """
 
     # Create server instance if none was given
     if server is None:


Test patch:
Repository: adap/flower

Issue:
Evaluation of the final model on empty client set
Hello,
I'm trying to run the basic [Tensorflow Example](https://github.com/adap/flower/tree/main/examples/quickstart_tensorflow) but when the [final model evaluation is executed](https://github.com/adap/flower/blob/42ef63b6a8097c3b571b65e11b5471477253a48d/src/py/flwr/server/app.py#L97) the client returns an empty set, thus the evaluation is not executed.
This should be due to the fact that the server [closes the connections](https://github.com/adap/flower/blob/42ef63b6a8097c3b571b65e11b5471477253a48d/src/py/flwr/server/server.py#L117) with the clients after the fit.

Thank you


Discussion:
Thanks for the report @NicholasRasi ! What you're describing is right, good catch. We'll fix this asap.

Resolving patch:
diff --git a/src/py/flwr/server/app.py b/src/py/flwr/server/app.py
--- a/src/py/flwr/server/app.py
+++ b/src/py/flwr/server/app.py
@@ -108,5 +108,8 @@ def start_server(
     else:
         log(INFO, "app_evaluate: no evaluation result")
 
+    # Graceful shutdown
+    server.disconnect_all_clients()
+
     # Stop the gRPC server
     grpc_server.stop(1)
diff --git a/src/py/flwr/server/server.py b/src/py/flwr/server/server.py
--- a/src/py/flwr/server/server.py
+++ b/src/py/flwr/server/server.py
@@ -114,10 +114,6 @@ def fit(self, num_rounds: int) -> History:
                     rnd=current_round, loss=cast(float, loss_fed)
                 )
 
-        # Send shutdown signal to all clients
-        all_clients = self._client_manager.all()
-        _ = shutdown(clients=[all_clients[k] for k in all_clients.keys()])
-
         # Bookkeeping
         end_time = timeit.default_timer()
         elapsed = end_time - start_time
@@ -182,6 +178,11 @@ def fit_round(self, rnd: int) -> Optional[Weights]:
         # Aggregate training results
         return self.strategy.aggregate_fit(rnd, results, failures)
 
+    def disconnect_all_clients(self) -> None:
+        """Send shutdown signal to all clients."""
+        all_clients = self._client_manager.all()
+        _ = shutdown(clients=[all_clients[k] for k in all_clients.keys()])
+
     def _get_initial_weights(self) -> Weights:
         """Get initial weights from one of the available clients."""
         random_client = self._client_manager.sample(1)[0]


Test patch:
Repository: adap/flower

Issue:
Create basic repository structure
Flower Datasets related code is going to live in a new `flwr.datasets` package.

Code to generate partitioned datasets is going in `flwr.dataset.generator`.


Discussion:

Resolving patch:
diff --git a/src/py/flwr/dataset/__init__.py b/src/py/flwr/dataset/__init__.py
new file mode 100644
--- /dev/null
+++ b/src/py/flwr/dataset/__init__.py
@@ -0,0 +1,15 @@
+# Copyright 2020 Adap GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower Datasets."""
diff --git a/src/py/flwr/dataset/generator/__init__.py b/src/py/flwr/dataset/generator/__init__.py
new file mode 100644
--- /dev/null
+++ b/src/py/flwr/dataset/generator/__init__.py
@@ -0,0 +1,15 @@
+# Copyright 2020 Adap GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower Datasets generator."""


Test patch:
Repository: adap/flower

Issue:
Drop Python 3.6 Support
Several major libraries have started to drop Python 3.6 support


Discussion:

Resolving patch:
diff --git a/src/py/flwr/server/strategy/fedavg_android.py b/src/py/flwr/server/strategy/fedavg_android.py
--- a/src/py/flwr/server/strategy/fedavg_android.py
+++ b/src/py/flwr/server/strategy/fedavg_android.py
@@ -272,7 +272,7 @@ def parameters_to_weights(self, parameters: Parameters) -> Weights:
     # pylint: disable=R0201
     def ndarray_to_bytes(self, ndarray: np.ndarray) -> bytes:
         """Serialize NumPy array to bytes."""
-        return cast(bytes, ndarray.tobytes())
+        return ndarray.tobytes()
 
     # pylint: disable=R0201
     def bytes_to_ndarray(self, tensor: bytes) -> np.ndarray:
diff --git a/src/py/flwr/server/strategy/qfedavg.py b/src/py/flwr/server/strategy/qfedavg.py
--- a/src/py/flwr/server/strategy/qfedavg.py
+++ b/src/py/flwr/server/strategy/qfedavg.py
@@ -175,7 +175,7 @@ def norm_grad(grad_list: List[Weights]) -> float:
                 client_grads = np.append(
                     client_grads, grad_list[i]
                 )  # output a flattened array
-            squared = np.square(client_grads)
+            squared = np.square(client_grads)  # type: ignore
             summed = np.sum(squared)
             return float(summed)
 


Test patch:
Repository: adap/flower

Issue:
Secure gRPC: Logging still reporing insecure connection
### Discussed in https://github.com/adap/flower/discussions/1020

<div type='discussions-op-text'>

<sup>Originally posted by **NathanSchot** January 21, 2022</sup>
Hey,

My team and I have enjoyed working with flower for a few months now. We are very excited with the recent implementation of secure gRPC connections, as security is one of our main focus points.

I realise I may be a bit early (and potentially nitpicky) with this question as secure gRPC is still under construction, but we noticed that the logging of the FL process does not take the existence of root certificates in account, therefore printing incorrect information. The two lines where this happens are:

https://github.com/adap/flower/blob/cb66854dcbf2e0c5bf14ee48149053a84be8a3be/src/py/flwr/client/app.py#L89
and 
https://github.com/adap/flower/blob/cb66854dcbf2e0c5bf14ee48149053a84be8a3be/src/py/flwr/client/grpc_client/connection.py#L114

Thanks in advance.</div>


Discussion:
Thanks for the report @NathanSchot, appreciated! CC @tanertopal 
@NathanSchot much appreciated! We will update that asap!

Resolving patch:
diff --git a/src/py/flwr/client/app.py b/src/py/flwr/client/app.py
--- a/src/py/flwr/client/app.py
+++ b/src/py/flwr/client/app.py
@@ -86,7 +86,6 @@ class `flwr.client.Client`.
             root_certificates=root_certificates,
         ) as conn:
             receive, send = conn
-            log(INFO, "Opened (insecure) gRPC connection")
 
             while True:
                 server_message = receive()
diff --git a/src/py/flwr/client/grpc_client/connection.py b/src/py/flwr/client/grpc_client/connection.py
--- a/src/py/flwr/client/grpc_client/connection.py
+++ b/src/py/flwr/client/grpc_client/connection.py
@@ -15,7 +15,7 @@
 """Provides contextmanager which manages a gRPC channel to connect to the
 server."""
 from contextlib import contextmanager
-from logging import DEBUG
+from logging import DEBUG, INFO
 from queue import Queue
 from typing import Callable, Iterator, Optional, Tuple
 
@@ -91,8 +91,10 @@ def grpc_connection(
         channel = grpc.secure_channel(
             server_address, ssl_channel_credentials, options=channel_options
         )
+        log(INFO, "Opened secure gRPC connection using certificates")
     else:
         channel = grpc.insecure_channel(server_address, options=channel_options)
+        log(INFO, "Opened insecure gRPC connection (no certificates were passed)")
 
     channel.subscribe(on_channel_state_change)
 
@@ -111,4 +113,4 @@ def grpc_connection(
     finally:
         # Make sure to have a final
         channel.close()
-        log(DEBUG, "Insecure gRPC channel closed")
+        log(DEBUG, "gRPC channel closed")


Test patch:
Repository: adap/flower

Issue:
Unify documentation to use `-` instead of `_`
### Describe what the documentation is missing.

Some docs use `_` as the separator and some use `-`, for example:
- `https://flower.dev/docs/example_walkthrough_pytorch_mnist.html`
- `https://flower.dev/docs/example-mxnet-walk-through.html`

This is inconsistent.

### Suggest your improvement.

We want to unify all docs to use `-` ("minus") as the only separator. This involves the following TODO's:

- [ ] Rename all docs that use `_` (and replace it with `-`)
- [ ] For each renamed doc, configure a redirect from the old naming (e.g., `quickstart_pytorch`) to the new page (e.g., `quickstart-pytorch`) using, for example, https://documatt.gitlab.io/sphinx-reredirects/index.html
- [ ] Update all references in the docs with the new naming (e.g., in `index.rst`)
- [ ] Update all other references with the new naming (e.g., README.md, code examples, website, ...)

Last, but not least, this should all be done in a single PR. We should not split, for example, the renaming of a page and the configuration of a redirect into two PRs because it would lead to a deployed version of the docs where users would see a 404 page when they use a link that points to the old naming.


Discussion:
Hello @danieljanes , I would like to take up this issue
@RISHIKESHAVAN that would be awesome, please go ahead!

Resolving patch:
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -51,6 +51,7 @@
     "sphinx_copybutton",
     "sphinx_panels",
     "sphinxcontrib.mermaid",
+    "sphinx_reredirects"
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -61,6 +62,19 @@
 # This pattern also affects html_static_path and html_extra_path.
 exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
+# Sphinx redirects, implemented after the doc filename changes.
+# To prevent 404 errors and redirect to the new pages.
+redirects = {
+    "quickstart_mxnet": "quickstart-mxnet.html",
+    "quickstart_pytorch_lightning": "quickstart-pytorch-lightning.html",
+    "example_walkthrough_pytorch_mnist": "example-walkthrough-pytorch-mnist.html",
+    "quickstart_huggingface": "quickstart-huggingface.html",
+    "quickstart_pytorch": "quickstart-pytorch.html",
+    "quickstart_tensorflow": "quickstart-tensorflow.html",
+    "release_process": "release-process.html",
+    "quickstart_scikitlearn": "quickstart-scikitlearn.html"
+}
+
 
 # -- Options for HTML output -------------------------------------------------
 


Test patch:
Repository: adap/flower

Issue:
sklearn-logreg-mnist example is outdated
### Describe the bug

The sklearn-logreg-mnist example is outdated and does not work with flower 1.0.0. I will make a pull request to fix this.

### Steps/Code to Reproduce

sh ./run.sh

### Expected Results

The example should run with no errors, both on the client and on the server.

### Actual Results

Number of errors arise when the script runs.


Discussion:

Resolving patch:
diff --git a/examples/sklearn-logreg-mnist/client.py b/examples/sklearn-logreg-mnist/client.py
--- a/examples/sklearn-logreg-mnist/client.py
+++ b/examples/sklearn-logreg-mnist/client.py
@@ -27,7 +27,7 @@
 
     # Define Flower client
     class MnistClient(fl.client.NumPyClient):
-        def get_parameters(self):  # type: ignore
+        def get_parameters(self, config):  # type: ignore
             return utils.get_model_parameters(model)
 
         def fit(self, parameters, config):  # type: ignore
@@ -46,4 +46,4 @@ def evaluate(self, parameters, config):  # type: ignore
             return loss, len(X_test), {"accuracy": accuracy}
 
     # Start Flower client
-    fl.client.start_numpy_client("0.0.0.0:8080", client=MnistClient())
+    fl.client.start_numpy_client(server_address="0.0.0.0:8080", client=MnistClient())
diff --git a/examples/sklearn-logreg-mnist/server.py b/examples/sklearn-logreg-mnist/server.py
--- a/examples/sklearn-logreg-mnist/server.py
+++ b/examples/sklearn-logreg-mnist/server.py
@@ -17,7 +17,7 @@ def get_evaluate_fn(model: LogisticRegression):
     _, (X_test, y_test) = utils.load_mnist()
 
     # The `evaluate` function will be called after every round
-    def evaluate(parameters: fl.common.Weights):
+    def evaluate(server_round, parameters: fl.common.NDArrays, config):
         # Update model with the latest parameters
         utils.set_model_params(model, parameters)
         loss = log_loss(y_test, model.predict_proba(X_test))
@@ -39,5 +39,5 @@ def evaluate(parameters: fl.common.Weights):
     fl.server.start_server(
         server_address="0.0.0.0:8080",
         strategy=strategy,
-        config={"num_rounds": 5},
+        config=fl.server.ServerConfig(num_rounds=5),
     )


Test patch:
Repository: adap/flower

Issue:
deprecated eval_fn still used in examples
### Describe the bug

While running the `embedded_devices` example, an issue is faced due to the use of `eval_fn` keyword which was deprecated after Flower 1.0.0 and has now been changed to `evaluate_fn`

### Steps/Code to Reproduce

Setup the server as mentioned in the `examples/embedded_devices` readme

### Expected Results

The server should start without any error

### Actual Results

The following error is encountered:
```
File "/embedded_devices/server.py", line 109, in main
     strategy = fl.server.strategy.FedAvg(
TypeError: FedAvg.__init__() got an unexpected keyword argument 'eval_fn'
```


Discussion:

Resolving patch:
diff --git a/examples/embedded_devices/server.py b/examples/embedded_devices/server.py
--- a/examples/embedded_devices/server.py
+++ b/examples/embedded_devices/server.py
@@ -110,7 +110,7 @@ def main() -> None:
         fraction_fit=args.sample_fraction,
         min_fit_clients=args.min_sample_size,
         min_available_clients=args.min_num_clients,
-        eval_fn=get_eval_fn(testset),
+        evaluate_fn=get_eval_fn(testset),
         on_fit_config_fn=fit_config,
     )
     server = fl.server.Server(client_manager=client_manager, strategy=strategy)


Test patch:
Repository: getmoto/moto

Issue:
dynamo query with hash key returns wrong results
When executing a query in dynamodb wrong result is returned

Step to reproduce:
add an insert to the default query_test in the unit tests:

```
@mock_dynamodb
def test_query():
    conn = boto.connect_dynamodb()
    table = create_table(conn)

    item_data = {
        'Body': 'http://url_to_lolcat.gif',
        'SentBy': 'User A',
        'ReceivedTime': '12/9/2011 11:36:03 PM',
    }
    # the extra item
    item = table.new_item(
        hash_key='the-key1',
        range_key='4561',
        attrs=item_data,
    )
    item.put()

    item = table.new_item(
        hash_key='the-key',
        range_key='456',
        attrs=item_data,
    )
    item.put()

    item = table.new_item(
        hash_key='the-key',
        range_key='123',
        attrs=item_data,
    )
    item.put()

    item = table.new_item(
        hash_key='the-key',
        range_key='789',
        attrs=item_data,
    )
    item.put()

    results = table.query(hash_key='the-key', range_key_condition=condition.GT('1'))
    results.response['Items'].should.have.length_of(3)
```

This new test will fail as the library will return 4 results in stead of 3



Discussion:

Resolving patch:
diff --git a/moto/dynamodb/models.py b/moto/dynamodb/models.py
--- a/moto/dynamodb/models.py
+++ b/moto/dynamodb/models.py
@@ -182,7 +182,11 @@ def query(self, hash_key, range_comparison, range_objs):
         results = []
         last_page = True  # Once pagination is implemented, change this
 
-        possible_results = list(self.all_items())
+        if self.range_key_attr:
+            possible_results = self.items[hash_key].values()
+        else:
+            possible_results = list(self.all_items())
+
         if range_comparison:
             for result in possible_results:
                 if result.range_key.compare(range_comparison, range_objs):


Test patch:
Repository: getmoto/moto

Issue:
Cleanup: Duplicate import
In "ec2/models.py", the util "random_internet_gateway_id" is imported twice. Cleanup candidate.



Discussion:

Resolving patch:
diff --git a/moto/ec2/models.py b/moto/ec2/models.py
--- a/moto/ec2/models.py
+++ b/moto/ec2/models.py
@@ -53,7 +53,6 @@
     random_eip_association_id,
     random_eni_attach_id,
     random_eni_id,
-    random_internet_gateway_id,
     random_instance_id,
     random_internet_gateway_id,
     random_ip,


Test patch: