Repository: scaleoutsystems/fedn
Issue: Side case when client joins during an active round

**Severity**
- [ ] Critical/Blocker (select if the issue makes the application unusable or causes serious data loss)
- [x] High (select if the issue affects a major feature and there is no workaround or the available workaround is very complex)
- [ ] Medium (select if the issue affects a minor feature or affects a major feature but has an easy enough workaround to not cause any major inconvenience)
- [ ] Low (select if the issue doesn't significantly affect the user experience, like minor visual bugs)

**Describe the bug**
If a client joins during an active round (reducer/controller in monitoring state), for some reason the client is not added to the MongoDB database. 

But it gets correctly attached and assigned to a combiner and immediately joins the training. 

**Environment:**
 - OS: [e.g. mac OS]
 - Version: [e.g. macOS Catalina]
 - Browser [e.g. chrome, safari]
 - Version [e.g. 22]

**Reproduction Steps**
Start say 10 rounds. 
Attach a client, look in the MongoExpress interface. 

**Expected behavior**
One of: 

1. New Clients should not be able to join when reducer is in monitoring state. 
2. The Clients should be properly added in the network/database.  

**Screenshots**
If applicable, add screenshots to help explain your problem.

**Contact Details**
An e-mail address in case we need to contact you for further details regarding this issue.


Changed files:
fedn/fedn/common/tracer/mongotracer.py, fedn/fedn/clients/reducer/restservice.py, fedn/fedn/common/net/connect.py, fedn/fedn/clients/reducer/network.py, fedn/fedn/client.py

diff --git a/fedn/fedn/client.py b/fedn/fedn/client.py
index 08c4d5a..eaf4815 100644
--- a/fedn/fedn/client.py
+++ b/fedn/fedn/client.py
@@ -146,11 +146,10 @@ class Client:
         """Contacts the controller and asks for combiner assignment. """
 
         print("Asking for assignment!", flush=True)
-        import time
         while True:
             status, response = self.connector.assign()
             if status == Status.TryAgain:
-                print("Make sure to upload the compute package!", flush=True)
+                print(response, flush=True)
                 time.sleep(5)
                 continue
             if status == Status.Assigned:
diff --git a/fedn/fedn/clients/reducer/network.py b/fedn/fedn/clients/reducer/network.py
index 746b7db..2c64762 100644
--- a/fedn/fedn/clients/reducer/network.py
+++ b/fedn/fedn/clients/reducer/network.py
@@ -60,17 +60,14 @@ class Network:
         self.statestore.set_combiner(combiner.to_dict())
 
     def add_client(self, client):
-        """
+        """ Add a new client to the network. 
 
         :param client:
         :return:
         """
-        if not self.control.idle():
-            print("Reducer is not idle, cannot add additional client.")
-            return
 
         if self.find_client(client['name']):
-            return
+            return 
 
         print("adding client {}".format(client['name']), flush=True)
         self.statestore.set_client(client)
diff --git a/fedn/fedn/clients/reducer/restservice.py b/fedn/fedn/clients/reducer/restservice.py
index bd3cbca..0c51206 100644
--- a/fedn/fedn/clients/reducer/restservice.py
+++ b/fedn/fedn/clients/reducer/restservice.py
@@ -451,7 +451,12 @@ class ReducerRestService:
             """Handle client assignment requests. """
 
             if self.control.state() == ReducerState.setup:
-                return jsonify({'status': 'retry'})
+                return jsonify({'status': 'retry', 
+                                'msg': "Controller is not configured. Make sure to upload the compute package."})
+
+            if not self.control.idle():
+                return jsonify({'status': 'retry',
+                                'msg': "Conroller is not in idle state, try again later. "})
 
             name = request.args.get('name', None)
             combiner_preferred = request.args.get('combiner', None)
@@ -462,11 +467,8 @@ class ReducerRestService:
                 combiner = self.control.find_available_combiner()
 
             if combiner is None:
-                return jsonify({'status': 'retry'})
-            ## Check that a framework has been selected prior to assigning clients.
-            framework = self.control.statestore.get_framework()
-            if not framework:
-                return jsonify({'status': 'retry'})
+                return jsonify({'status': 'retry',
+                                'msg': "Failed to assign to a combiner, try again later."})
 
             client = {
                 'name': name,
@@ -475,25 +477,23 @@ class ReducerRestService:
                 'ip': request.remote_addr,
                 'status': 'available'
             }
+
+            # Add client to database 
             self.control.network.add_client(client)
 
-            if combiner:
-                import base64
-                cert_b64 = base64.b64encode(combiner.certificate)
-                response = {
-                    'status': 'assigned',
-                    'host': combiner.address,
-                    'ip': combiner.ip,
-                    'port': combiner.port,
-                    'certificate': str(cert_b64).split('\\'')[1],
-                    'model_type': self.control.statestore.get_framework()
-                }
-
-                return jsonify(response)
-            elif combiner is None:
-                return jsonify({'status': 'retry'})
+            # Return connection information to client
+            import base64
+            cert_b64 = base64.b64encode(combiner.certificate)
+            response = {
+                'status': 'assigned',
+                'host': combiner.address,
+                'ip': combiner.ip,
+                'port': combiner.port,
+                'certificate': str(cert_b64).split('\\'')[1],
+                'model_type': self.control.statestore.get_framework()
+            }
 
-            return jsonify({'status': 'retry'})
+            return jsonify(response)
 
         @app.route('/infer')
         def infer():
diff --git a/fedn/fedn/common/net/connect.py b/fedn/fedn/common/net/connect.py
index 4a6c586..e3c844d 100644
--- a/fedn/fedn/common/net/connect.py
+++ b/fedn/fedn/common/net/connect.py
@@ -86,8 +86,12 @@ class ConnectorClient:
 
         if retval.status_code >= 200 and retval.status_code < 204:
             if retval.json()['status'] == 'retry':
-                print("Reducer was not ready. Try again later.")
-                return Status.TryAgain, None
+                if 'msg' in retval.json():
+                    reason = retval.json()['msg']
+                else:
+                    reason = "Reducer was not ready. Try again later."
+
+                return Status.TryAgain, reason
 
             return Status.Assigned, retval.json()
 
diff --git a/fedn/fedn/common/tracer/mongotracer.py b/fedn/fedn/common/tracer/mongotracer.py
index 21e088b..be401d9 100644
--- a/fedn/fedn/common/tracer/mongotracer.py
+++ b/fedn/fedn/common/tracer/mongotracer.py
@@ -37,7 +37,7 @@ class MongoTracer(Tracer):
 
         print("LOG: \\n {} \\n".format(data), flush=True)
 
-        if self.status:
+        if self.status!=None:
             self.status.insert_one(data)
 
     def drop_round_time(self):