Repository: OpenMined/PySyft

Issue:
GSoC Project: Build a federated learning worker for PySyft
Based on the [Federated Learning roadmap](https://github.com/OpenMined/Roadmap/blob/master/web_and_mobile_team/projects/federated_learning.md) we need to build a worker library for static federated learning in PySyft. The worker will be based on the other 3 worker libraries that already exist: [syft.js](https://github.com/OpenMined/syft.js) (web), [SwiftSyft](https://github.com/OpenMined/SwiftSyft) (iOS), and [KotlinSyft](https://github.com/OpenMined/KotlinSyft) (Android). The API will be a mirror image of the other federated learning worker libraries, allowing for environments other than a web browser or mobile device to be supported: IoT, Raspberry Pi, desktop application, Windows phone, etc.

**Requirements:**
- Willing to work with the Web & Mobile team and be mentored by @vkkhare, the author of KotlinSyft
- Familiarity with Websockets
- Familiarity with WebRTC
- Good knowledge of PySyft
- Intimate knowledge of Python
- History of developing libraries (in any language)
- Good knowledge and understanding of the problems and challenges described in the [Federated Learning roadmap](https://github.com/OpenMined/Roadmap/blob/master/web_and_mobile_team/projects/federated_learning.md) (questions can be directed to @cereallarceny)

**Difficulty:** High

This is quite a large project and would more or less allow for static federated learning to be performed in any environment. _A federated learning system of this breadth has not previously been developed, anywhere._ Make no mistake, this is a big project, but it also has very far-reaching ramifications for federated learning as a whole. Fortunately, the web and mobile team has already built 3 federated learning worker libraries in the past, so you'll be inheriting the wisdom, patterns, and roadmap that has basically been finalized by your new colleagues. If you're not afraid of hard projects with a pretty large payoff, this is the one for you.


Discussion:

Resolving patch:
diff --git a/syft/federated/fl_client.py b/syft/federated/fl_client.py
new file mode 100644
--- /dev/null
+++ b/syft/federated/fl_client.py
@@ -0,0 +1,26 @@
+from urllib.parse import urlparse
+from syft.grid.grid_client import GridClient
+from syft.federated.fl_job import FLJob
+
+
+class FLClient:
+    def __init__(self, url, auth_token, verbose=False):
+        self.url = url
+        self.auth_token = auth_token
+        self.worker_id = None
+
+        url_fragments = urlparse(url)
+        self.grid_client = GridClient(id="", address=url_fragments.netloc, secure=not verbose,)
+
+    def new_job(self, model_name, model_version) -> FLJob:
+        if self.worker_id is None:
+            auth_response = self.grid_client.authenticate(self.auth_token)
+            self.worker_id = auth_response["data"]["worker_id"]
+
+        job = FLJob(
+            fl_client=self,
+            grid_client=self.grid_client,
+            model_name=model_name,
+            model_version=model_version,
+        )
+        return job
diff --git a/syft/federated/fl_job.py b/syft/federated/fl_job.py
new file mode 100644
--- /dev/null
+++ b/syft/federated/fl_job.py
@@ -0,0 +1,96 @@
+from syft.grid.grid_client import GridClient
+from syft.grid.grid_client import GridError
+from syft.execution.state import State
+from syft.execution.placeholder import PlaceHolder
+
+
+class EventEmitter:
+    def __init__(self):
+        self.listeners = {}
+        pass
+
+    def add_listener(self, event_name, fn):
+        if event_name not in self.listeners:
+            self.listeners[event_name] = []
+        self.listeners[event_name].append(fn)
+
+    def trigger(self, event_name, *args, **kwargs):
+        if event_name in self.listeners:
+            for fn in self.listeners[event_name]:
+                fn(*args, **kwargs)
+
+
+class FLJob(EventEmitter):
+    EVENT_ACCEPTED = "accepted"
+    EVENT_REJECTED = "rejected"
+    EVENT_ERROR = "error"
+
+    def __init__(self, fl_client, grid_client: GridClient, model_name: str, model_version: str):
+        super().__init__()
+        self.fl_client = fl_client
+        self.grid_client = grid_client
+        self.model_name = model_name
+        self.model_version = model_version
+
+        self.model = None
+        self.plans = {}
+        self.protocols = {}
+        self.cycle_params = {}
+        self.client_config = {}
+
+    def _init_cycle(self, cycle_params: dict):
+        self.cycle_params = cycle_params
+        self.client_config = cycle_params["client_config"]
+
+        worker_id = self.fl_client.worker_id
+        request_key = cycle_params["request_key"]
+
+        # Load model
+        self.model = self.grid_client.get_model(worker_id, request_key, cycle_params["model_id"])
+
+        # Load plans
+        for plan_name, plan_id in cycle_params["plans"].items():
+            self.plans[plan_name] = self.grid_client.get_plan(
+                worker_id, request_key, plan_id, GridClient.PLAN_TYPE_TORCHSCRIPT
+            )
+
+        # Load protocols
+        for protocol_name, protocol_id in cycle_params["protocols"].items():
+            self.protocols[protocol_name] = self.grid_client.get_protocol(
+                worker_id, request_key, protocol_id
+            )
+
+    def start(self):
+        try:
+            speed_info = self.grid_client.get_connection_speed(self.fl_client.worker_id)
+            cycle_request_response = self.grid_client.cycle_request(
+                worker_id=self.fl_client.worker_id,
+                model_name=self.model_name,
+                model_version=self.model_version,
+                speed_info=speed_info,
+            )
+            cycle_params = cycle_request_response["data"]
+
+            if cycle_params["status"] == GridClient.CYCLE_STATUS_ACCEPTED:
+                self._init_cycle(cycle_params)
+                self.trigger(self.EVENT_ACCEPTED, self)
+            elif cycle_params["status"] == GridClient.CYCLE_STATUS_REJECTED:
+                self.trigger(self.EVENT_REJECTED, self, cycle_params.get("timeout", None))
+        except GridError as e:
+            self.trigger(self.EVENT_ERROR, self, f"Grid communication error: {e.error}")
+
+    def report(self, updated_model_params: list):
+        # Calc params diff
+        orig_params = self.model.tensors()
+        diff_params = [orig_params[i] - updated_model_params[i] for i in range(len(orig_params))]
+
+        # Wrap diff in State
+        diff_ph = [PlaceHolder().instantiate(t) for t in diff_params]
+        diff = State(state_placeholders=diff_ph)
+
+        response = self.grid_client.report(
+            worker_id=self.fl_client.worker_id,
+            request_key=self.cycle_params["request_key"],
+            diff=diff,
+        )
+        return response
diff --git a/syft/grid/grid_client.py b/syft/grid/grid_client.py
--- a/syft/grid/grid_client.py
+++ b/syft/grid/grid_client.py
@@ -1,16 +1,34 @@
 import json
 
 import binascii
+import base64
 import websocket
 import websockets
+import requests
 
 import syft as sy
 from syft.serde import protobuf
 
+from syft.execution.state import State
+from syft_proto.execution.v1.plan_pb2 import Plan as PlanPB
+from syft_proto.execution.v1.state_pb2 import State as StatePB
+from syft_proto.execution.v1.protocol_pb2 import Protocol as ProtocolPB
+
 TIMEOUT_INTERVAL = 60
 
 
+class GridError(BaseException):
+    def __init__(self, error, status):
+        self.status = status
+        self.error = error
+
+
 class GridClient:
+    CYCLE_STATUS_ACCEPTED = "accepted"
+    CYCLE_STATUS_REJECTED = "rejected"
+    PLAN_TYPE_LIST = "list"
+    PLAN_TYPE_TORCHSCRIPT = "torchscript"
+
     def __init__(self, id: str, address: str, secure: bool = False):
         self.id = id
         self.address = address
@@ -19,11 +37,15 @@ def __init__(self, id: str, address: str, secure: bool = False):
         self.serialize_worker = sy.VirtualWorker(hook=None)
 
     @property
-    def url(self):
+    def ws_url(self):
         return f"wss://{self.address}" if self.secure else f"ws://{self.address}"
 
+    @property
+    def http_url(self):
+        return f"https://{self.address}" if self.secure else f"http://{self.address}"
+
     def connect(self):
-        args_ = {"max_size": None, "timeout": TIMEOUT_INTERVAL, "url": self.url}
+        args_ = {"max_size": None, "timeout": TIMEOUT_INTERVAL, "url": self.ws_url}
 
         self.ws = websocket.create_connection(**args_)
 
@@ -34,8 +56,32 @@ def _send_msg(self, message: dict) -> dict:
             Returns:
                 response (dict) : response payload.
         """
+        if self.ws is None or not self.ws.connected:
+            self.connect()
+
         self.ws.send(json.dumps(message))
-        return json.loads(self.ws.recv())
+        json_response = json.loads(self.ws.recv())
+
+        # print("REQ", message)
+        # print("RES", json_response)
+
+        error = json_response["data"].get("error", None)
+        if error is not None:
+            raise GridError(error, None)
+
+        return json_response
+
+    def _send_http_req(self, method, path: str, params: dict = None, body: bytes = None):
+        if method == "GET":
+            res = requests.get(self.http_url + path, params)
+        elif method == "POST":
+            res = requests.post(self.http_url + path, body)
+
+        if not res.ok:
+            raise GridError("HTTP response is not OK", res.status_code)
+
+        response = res.content
+        return response
 
     def _serialize(self, obj):
         """Serializes object to protobuf"""
@@ -48,6 +94,12 @@ def _serialize_object(self, obj):
             serialized_object[k] = binascii.hexlify(self._serialize(v)).decode()
         return serialized_object
 
+    def _unserialize(self, serialized_obj, obj_protobuf_type):
+        pb = obj_protobuf_type()
+        pb.ParseFromString(serialized_obj)
+        serialization_worker = sy.VirtualWorker(hook=None, auto_add=False)
+        return protobuf.serde._unbufferize(serialization_worker, pb)
+
     def close(self):
         self.ws.shutdown()
 
@@ -79,3 +131,64 @@ def host_federated_training(
         }
 
         return self._send_msg(message)
+
+    def authenticate(self, auth_token):
+        message = {
+            "type": "federated/authenticate",
+            "data": {"auth_token": auth_token},
+        }
+
+        return self._send_msg(message)
+
+    def cycle_request(self, worker_id, model_name, model_version, speed_info):
+        message = {
+            "type": "federated/cycle-request",
+            "data": {
+                "worker_id": worker_id,
+                "model": model_name,
+                "version": model_version,
+                **speed_info,
+            },
+        }
+        return self._send_msg(message)
+
+    def get_model(self, worker_id, request_key, model_id):
+        params = {
+            "worker_id": worker_id,
+            "request_key": request_key,
+            "model_id": model_id,
+        }
+        serialized_model = self._send_http_req("GET", "/federated/get-model", params)
+        return self._unserialize(serialized_model, StatePB)
+
+    def get_plan(self, worker_id, request_key, plan_id, receive_operations_as):
+        params = {
+            "worker_id": worker_id,
+            "request_key": request_key,
+            "plan_id": plan_id,
+            "receive_operations_as": receive_operations_as,
+        }
+        serialized_plan = self._send_http_req("GET", "/federated/get-plan", params)
+        return self._unserialize(serialized_plan, PlanPB)
+
+    def get_protocol(self, worker_id, request_key, protocol_id):
+        params = {
+            "worker_id": worker_id,
+            "request_key": request_key,
+            "plan_id": protocol_id,
+        }
+        serialized_protocol = self._send_http_req("GET", "/federated/get-protocol", params)
+        return self._unserialize(serialized_protocol, ProtocolPB)
+
+    def report(self, worker_id: str, request_key: str, diff: State):
+        diff_serialized = self._serialize(diff)
+        diff_base64 = base64.b64encode(diff_serialized).decode("ascii")
+        params = {
+            "type": "federated/report",
+            "data": {"worker_id": worker_id, "request_key": request_key, "diff": diff_base64,},
+        }
+        return self._send_msg(params)
+
+    def get_connection_speed(self, worker_id):
+        # TODO
+        return {"ping": 5, "download": 100, "upload": 100}


Test patch: