Repository: opentensor/bittensor

Issue:
Bittensor throws exception if one peer dies or stops contributing
We need to surround things with try/catch blocks to get Bittensor to continue without interruption if a specific peer dies or stops contributing to it. 


Discussion:

Resolving patch:
diff --git a/bittensor/dendrite.py b/bittensor/dendrite.py
--- a/bittensor/dendrite.py
+++ b/bittensor/dendrite.py
@@ -56,6 +56,7 @@ def forward(self, synapses: List[bittensor_pb2.Synapse], x: List[ object ], mode
                 
             # Call remote synapse.
             results.append(remote_synapse(forward_inputs, mode))
+
         return results
     
 # NOTE: (const) This code has been ported from hivemind thanks to Yozh and Max.
@@ -130,10 +131,14 @@ def forward(ctx, caller: RemoteSynapse, dummy: torch.Tensor, inputs: object, mod
                                             )
         
         # Make rpc call.
-        response = ctx.caller.stub.Forward(request)
-                
-        # Deserialize outputs and return.
-        outputs = PyTorchSerializer.deserialize_tensor(response.tensors[0])
+        try:
+            response = ctx.caller.stub.Forward(request)                
+            # Deserialize outputs and return.
+            outputs = PyTorchSerializer.deserialize_tensor(response.tensors[0])
+        except grpc._channel._InactiveRpcError as ire:
+            #logger.error("Could not forward() to peer: {}".format(ire))
+            outputs = torch.zeros((inputs.size(0), bittensor.__network_dim__))
+        
         return outputs
 
     @staticmethod
@@ -154,14 +159,19 @@ def backward(ctx, grads: torch.Tensor) -> Optional[torch.Tensor]:
                                                 tensors = [serialized_inputs, serialized_grads]
                                             )
         
-        # Attain backward response
-        response = ctx.caller.stub.Backward(request)
+        deserialized_grad_inputs = torch.zeros(1,1)
+        
+        try:
+            # Attain backward response
+            response = ctx.caller.stub.Backward(request)
 
-        # Deserialize grad responses.
-        # TODO (const) maybe remove this?
-        if ctx.mode == bittensor_pb2.Modality.TEXT:
-            return (None, None, None, None)       
-        else:
-            deserialized_grad_inputs = PyTorchSerializer.deserialize (response.tensors[0])
-            return (None, None, deserialized_grad_inputs, None)        
-        
\ No newline at end of file
+            # Deserialize grad responses.
+            # TODO (const) maybe remove this?
+            if ctx.mode == bittensor_pb2.Modality.TEXT:
+                return (None, None, None, None)       
+            else:
+                deserialized_grad_inputs = PyTorchSerializer.deserialize (response.tensors[0])
+                return (None, None, deserialized_grad_inputs, None)        
+        except grpc._channel._InactiveRpcError as ire:
+            #logger.error("Could not backward() to peer: {}".format(ire))
+            return (None, None, deserialized_grad_inputs, None)
\ No newline at end of file
diff --git a/bittensor/metagraph.py b/bittensor/metagraph.py
--- a/bittensor/metagraph.py
+++ b/bittensor/metagraph.py
@@ -82,8 +82,8 @@ def _sink(self, request: bittensor_pb2.GossipBatch):
             self._heartbeat[synapse.synapse_key] = time.time()      
 
     def Gossip(self, request: bittensor_pb2.GossipBatch, context):
-        synapses = self.get_synapses(1000)
-        peers = self.get_peers(10)
+        synapses = self.synapses(1000)
+        peers = self.peers(10)
         self._sink(request)
         response = bittensor_pb2.GossipBatch(peers=peers, synapses=synapses)
         return response
@@ -93,15 +93,15 @@ def do_gossip(self):
         if len(self._peers) == 0:
             return
         
-        synapses = self.get_synapses(1000)
-        peers = self.get_peers(10)
+        synapses = self.synapses(1000)
+        peers = self.peers(10)
         metagraph_address = random.choice(list(self._peers))        
         realized_address = metagraph_address
         if metagraph_address.split(':')[0] == self._config.remote_ip:
             realized_address = 'localhost:' + str(metagraph_address.split(":")[1])
             
         try:
-            version = bittensor.__version__
+
             channel = grpc.insecure_channel(realized_address)
             stub = bittensor_grpc.MetagraphStub(channel)
             request = bittensor_pb2.GossipBatch(peers=peers, synapses=synapses)
@@ -131,18 +131,20 @@ def _update(self):
             while self._running:
                 self.do_gossip()
                 if len(self._peers) > 0:
-                    self.do_clean(60*60)
+                    self.do_clean(15)
                 time.sleep(10)
-        except (KeyboardInterrupt, SystemExit):
+        except (KeyboardInterrupt, SystemExit) as e:
             logger.info('stop metagraph')
             self._running = False
             self.stop()
+            raise e
 
     def _serve(self):
         try:
             self._server.start()
-        except (KeyboardInterrupt, SystemExit):
+        except (KeyboardInterrupt, SystemExit) as ex:
             self.stop()
+            raise ex
         except Exception as e:
             logger.error(e)
         
diff --git a/examples/mnist/main.py b/examples/mnist/main.py
--- a/examples/mnist/main.py
+++ b/examples/mnist/main.py
@@ -96,7 +96,7 @@ def train(model, epoch, global_step):
                             
             # Logs:
             if batch_idx % log_interval == 0:
-                n_peers = len(bittensor.metagraph.peers)
+                n_peers = len(bittensor.metagraph.peers())
                 n_synapses = len(bittensor.metagraph.synapses())
                 writer.add_scalar('n_peers', n_peers, global_step)
                 writer.add_scalar('n_synapses', n_synapses, global_step)
@@ -104,7 +104,7 @@ def train(model, epoch, global_step):
             
                 n = len(train_data)
                 logger.info('Train Epoch: {} [{}/{} ({:.0f}%)]\tLocal Loss: {:.6f}\nNetwork Loss: {:.6f}\tDistillation Loss: {:.6f}\tnP|nS: {}|{}'.format(
-                    epoch, (batch_idx * batch_size_train), n, (100. * batch_idx * batch_size_train)/n, output['local_target_loss'].item(), output['network_target_loss'].item(), output['distillation_loss'].item(), len(bittensor.metagraph.peers), 
+                    epoch, (batch_idx * batch_size_train), n, (100. * batch_idx * batch_size_train)/n, output['local_target_loss'].item(), output['network_target_loss'].item(), output['distillation_loss'].item(), len(bittensor.metagraph.peers()), 
                             len(bittensor.metagraph.synapses())))
 
     # Test loop.


Test patch: