Repository: sonic-net/sonic-mgmt

Issue:
[test-gap][dualtor] Add grpc error config mux downstream testcase
![image](https://user-images.githubusercontent.com/35479537/223737856-ff1315c8-c525-4191-9fdc-cd38df7d4150.png)


Discussion:

Resolving patch:
diff --git a/ansible/dualtor/nic_simulator/nic_simulator.py b/ansible/dualtor/nic_simulator/nic_simulator.py
--- a/ansible/dualtor/nic_simulator/nic_simulator.py
+++ b/ansible/dualtor/nic_simulator/nic_simulator.py
@@ -12,7 +12,6 @@
 import argparse
 import contextlib
 import fcntl
-import functools
 import grpc
 import json
 import logging
@@ -638,17 +637,19 @@ def join(self, timeout=None, suppress_exception=False):
             if suppress_exception:
                 return self._e
             else:
-                raise(self._e) from None
+                raise (self._e) from None
 
 
 class NiCServer(nic_simulator_grpc_service_pb2_grpc.DualToRActiveServicer):
     """gRPC for a NiC."""
 
-    def __init__(self, nic_addr, ovs_bridge):
+    def __init__(self, nic_addr, ovs_bridge, binding_port):
         self.nic_addr = nic_addr
         self.ovs_bridge = ovs_bridge
+        self.binding_port = binding_port
         self.server = None
         self.thread = None
+        self.started = False
 
     def QueryAdminForwardingPortState(self, request, context):
         logging.debug("QueryAdminForwardingPortState: request to server %s from client %s\n",
@@ -710,14 +711,16 @@ def _run_server(self, binding_port):
         self.server.start()
         self.server.wait_for_termination()
 
-    def start(self, binding_port):
+    def start(self):
         """Start the gRPC server thread."""
-        self.thread = InterruptableThread(target=self._run_server, args=(binding_port,))
+        self.thread = InterruptableThread(target=self._run_server, args=(self.binding_port,))
         self.thread.start()
+        self.started = True
 
     def stop(self):
         """Stop the gRPC server thread."""
-        self.server._state.termination_event.set()
+        self.server.stop(grace=None)
+        self.started = False
 
     def join(self, timeout=None, suppress_exception=False):
         """Wait the gRPC server thread termination."""
@@ -727,9 +730,10 @@ def join(self, timeout=None, suppress_exception=False):
 class MgmtServer(nic_simulator_grpc_mgmt_service_pb2_grpc.DualTorMgmtServiceServicer):
     """Management gRPC server to interact with sonic-mgmt."""
 
-    def __init__(self, binding_address, binding_port):
+    def __init__(self, binding_address, binding_port, nic_servers):
         self.binding_address = binding_address
         self.binding_port = binding_port
+        self.nic_servers = nic_servers
         self.client_stubs = {}
         self.server = None
 
@@ -823,6 +827,43 @@ def SetDrop(self, request, context):
         logging.debug("SetDrop[mgmt]: response of set drop: %s\n", response)
         return response
 
+    def SetNicServerAdminState(self, request, context):
+        nic_addresses = request.nic_addresses
+        admin_states = request.admin_states
+        logging.debug("SetNicServerAdminState[mgmt]: request set nic server admin state:%s\n", request)
+
+        successes = []
+        for nic_address, admin_state in zip(nic_addresses, admin_states):
+            nic_server = self.nic_servers[nic_address]
+            success = True
+            if admin_state:
+                if not nic_server.started:
+                    try:
+                        nic_server.start()
+                    except Exception:
+                        logging.error("Failed to start nic server %s", nic_address, exc_info=True)
+                        success = False
+                logging.debug("Started nic server %s", nic_address)
+            else:
+                if nic_server.started:
+                    try:
+                        nic_server.stop()
+                        nic_server.join()
+                    except Exception:
+                        logging.error("Failed to stop nic server %s", nic_address, exc_info=True)
+                        success = False
+                logging.debug("Stopped nic server %s", nic_address)
+
+            successes.append(success)
+
+        response = nic_simulator_grpc_mgmt_service_pb2.ListOfNiCServerAdminStateReply(
+            nic_addresses=nic_addresses,
+            admin_states=admin_states,
+            successes=successes
+        )
+        logging.debug("SetNicServerAdminState[mgmt]: response of set nic server admin state:%s\n", response)
+        return response
+
     def start(self):
         self.server = grpc.server(
             futures.ThreadPoolExecutor(max_workers=THREAD_CONCURRENCY_PER_SERVER),
@@ -857,8 +898,9 @@ def __init__(self, vm_set, mgmt_port, binding_port):
                      json.dumps(list(self.ovs_bridges.keys()), indent=4))
 
         self.servers = {}
-        self.servers = {nic_addr: NiCServer(nic_addr, ovs_bridge) for nic_addr, ovs_bridge in self.ovs_bridges.items()}
-        self.mgmt_server = MgmtServer(self.mgmt_port_address, binding_port)
+        self.servers = {nic_addr: NiCServer(nic_addr, ovs_bridge, binding_port)
+                        for nic_addr, ovs_bridge in self.ovs_bridges.items()}
+        self.mgmt_server = MgmtServer(self.mgmt_port_address, binding_port, self.servers)
 
     def _find_all_server_nics(self):
         return [_ for _ in os.listdir('/sys/class/net') if re.search(NETNS_IFACE_PATTERN, _)]
@@ -872,7 +914,7 @@ def _find_all_bridges(self):
     def start_nic_servers(self):
         for nic_addr, server in self.servers.items():
             logging.debug("Starting gRPC server on NiC %s", nic_addr)
-            server.start(self.binding_port)
+            server.start()
 
     def stop_nic_servers(self):
         for nic_addr, server in self.servers.items():
diff --git a/ansible/dualtor/nic_simulator/nic_simulator_grpc_mgmt_service_pb2.py b/ansible/dualtor/nic_simulator/nic_simulator_grpc_mgmt_service_pb2.py
--- a/ansible/dualtor/nic_simulator/nic_simulator_grpc_mgmt_service_pb2.py
+++ b/ansible/dualtor/nic_simulator/nic_simulator_grpc_mgmt_service_pb2.py
@@ -3,368 +3,106 @@
 # source: nic_simulator_grpc_mgmt_service.proto
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
 from google.protobuf import message as _message
 from google.protobuf import reflection as _reflection
 from google.protobuf import symbol_database as _symbol_database
-import nic_simulator_grpc_service_pb2 as nic__simulator__grpc__service__pb2
 # @@protoc_insertion_point(imports)
 
 _sym_db = _symbol_database.Default()
 
-DESCRIPTOR = _descriptor.FileDescriptor(
-  name='nic_simulator_grpc_mgmt_service.proto',
-  package='',
-  syntax='proto3',
-  serialized_options=None,
-  create_key=_descriptor._internal_create_key,
-  serialized_pb=b'\n%nic_simulator_grpc_mgmt_service.proto\x1a nic_simulator_grpc_service.proto\"R\n\x12ListOfAdminRequest\x12\x15\n\rnic_addresses\x18\x01 \x03(\t\x12%\n\x0e\x61\x64min_requests\x18\x02 \x03(\x0b\x32\r.AdminRequest\"M\n\x10ListOfAdminReply\x12\x15\n\rnic_addresses\x18\x01 \x03(\t\x12\"\n\radmin_replies\x18\x02 \x03(\x0b\x32\x0b.AdminReply\"^\n\x16ListOfOperationRequest\x12\x15\n\rnic_addresses\x18\x01 \x03(\t\x12-\n\x12operation_requests\x18\x02 \x03(\x0b\x32\x11.OperationRequest\"Y\n\x14ListOfOperationReply\x12\x15\n\rnic_addresses\x18\x01 \x03(\t\x12*\n\x11operation_replies\x18\x02 \x03(\x0b\x32\x0f.OperationReply\"O\n\x11ListOfDropRequest\x12\x15\n\rnic_addresses\x18\x01 \x03(\t\x12#\n\rdrop_requests\x18\x02 \x03(\x0b\x32\x0c.DropRequest\"J\n\x0fListOfDropReply\x12\x15\n\rnic_addresses\x18\x01 \x03(\t\x12 \n\x0c\x64rop_replies\x18\x02 \x03(\x0b\x32\n.DropReply2\xa8\x02\n\x12\x44ualTorMgmtService\x12I\n\x1dQueryAdminForwardingPortState\x12\x13.ListOfAdminRequest\x1a\x11.ListOfAdminReply\"\x00\x12G\n\x1bSetAdminForwardingPortState\x12\x13.ListOfAdminRequest\x1a\x11.ListOfAdminReply\"\x00\x12K\n\x17QueryOperationPortState\x12\x17.ListOfOperationRequest\x1a\x15.ListOfOperationReply\"\x00\x12\x31\n\x07SetDrop\x12\x12.ListOfDropRequest\x1a\x10.ListOfDropReply\"\x00\x62\x06proto3'  # noqa E501
-  ,
-  dependencies=[nic__simulator__grpc__service__pb2.DESCRIPTOR, ])
-
-_LISTOFADMINREQUEST = _descriptor.Descriptor(
-  name='ListOfAdminRequest',
-  full_name='ListOfAdminRequest',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  create_key=_descriptor._internal_create_key,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='nic_addresses', full_name='ListOfAdminRequest.nic_addresses', index=0,
-      number=1, type=9, cpp_type=9, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-    _descriptor.FieldDescriptor(
-      name='admin_requests', full_name='ListOfAdminRequest.admin_requests', index=1,
-      number=2, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  serialized_options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=75,
-  serialized_end=157,
-)
-
-
-_LISTOFADMINREPLY = _descriptor.Descriptor(
-  name='ListOfAdminReply',
-  full_name='ListOfAdminReply',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  create_key=_descriptor._internal_create_key,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='nic_addresses', full_name='ListOfAdminReply.nic_addresses', index=0,
-      number=1, type=9, cpp_type=9, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-    _descriptor.FieldDescriptor(
-      name='admin_replies', full_name='ListOfAdminReply.admin_replies', index=1,
-      number=2, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  serialized_options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=159,
-  serialized_end=236,
-)
-
-
-_LISTOFOPERATIONREQUEST = _descriptor.Descriptor(
-  name='ListOfOperationRequest',
-  full_name='ListOfOperationRequest',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  create_key=_descriptor._internal_create_key,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='nic_addresses', full_name='ListOfOperationRequest.nic_addresses', index=0,
-      number=1, type=9, cpp_type=9, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-    _descriptor.FieldDescriptor(
-      name='operation_requests', full_name='ListOfOperationRequest.operation_requests', index=1,
-      number=2, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  serialized_options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=238,
-  serialized_end=332,
-)
-
-
-_LISTOFOPERATIONREPLY = _descriptor.Descriptor(
-  name='ListOfOperationReply',
-  full_name='ListOfOperationReply',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  create_key=_descriptor._internal_create_key,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='nic_addresses', full_name='ListOfOperationReply.nic_addresses', index=0,
-      number=1, type=9, cpp_type=9, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-    _descriptor.FieldDescriptor(
-      name='operation_replies', full_name='ListOfOperationReply.operation_replies', index=1,
-      number=2, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  serialized_options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=334,
-  serialized_end=423,
-)
 
+import nic_simulator_grpc_service_pb2 as nic__simulator__grpc__service__pb2
 
-_LISTOFDROPREQUEST = _descriptor.Descriptor(
-  name='ListOfDropRequest',
-  full_name='ListOfDropRequest',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  create_key=_descriptor._internal_create_key,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='nic_addresses', full_name='ListOfDropRequest.nic_addresses', index=0,
-      number=1, type=9, cpp_type=9, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-    _descriptor.FieldDescriptor(
-      name='drop_requests', full_name='ListOfDropRequest.drop_requests', index=1,
-      number=2, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  serialized_options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=425,
-  serialized_end=504,
-)
 
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n%nic_simulator_grpc_mgmt_service.proto\x1a nic_simulator_grpc_service.proto\"R\n\x12ListOfAdminRequest\x12\x15\n\rnic_addresses\x18\x01 \x03(\t\x12%\n\x0e\x61\x64min_requests\x18\x02 \x03(\x0b\x32\r.AdminRequest\"M\n\x10ListOfAdminReply\x12\x15\n\rnic_addresses\x18\x01 \x03(\t\x12\"\n\radmin_replies\x18\x02 \x03(\x0b\x32\x0b.AdminReply\"^\n\x16ListOfOperationRequest\x12\x15\n\rnic_addresses\x18\x01 \x03(\t\x12-\n\x12operation_requests\x18\x02 \x03(\x0b\x32\x11.OperationRequest\"Y\n\x14ListOfOperationReply\x12\x15\n\rnic_addresses\x18\x01 \x03(\t\x12*\n\x11operation_replies\x18\x02 \x03(\x0b\x32\x0f.OperationReply\"O\n\x11ListOfDropRequest\x12\x15\n\rnic_addresses\x18\x01 \x03(\t\x12#\n\rdrop_requests\x18\x02 \x03(\x0b\x32\x0c.DropRequest\"J\n\x0fListOfDropReply\x12\x15\n\rnic_addresses\x18\x01 \x03(\t\x12 \n\x0c\x64rop_replies\x18\x02 \x03(\x0b\x32\n.DropReply\"O\n ListOfNiCServerAdminStateRequest\x12\x15\n\rnic_addresses\x18\x01 \x03(\t\x12\x14\n\x0c\x61\x64min_states\x18\x02 \x03(\x08\"`\n\x1eListOfNiCServerAdminStateReply\x12\x15\n\rnic_addresses\x18\x01 \x03(\t\x12\x14\n\x0c\x61\x64min_states\x18\x02 \x03(\x08\x12\x11\n\tsuccesses\x18\x03 \x03(\x08\x32\x88\x03\n\x12\x44ualTorMgmtService\x12I\n\x1dQueryAdminForwardingPortState\x12\x13.ListOfAdminRequest\x1a\x11.ListOfAdminReply\"\x00\x12G\n\x1bSetAdminForwardingPortState\x12\x13.ListOfAdminRequest\x1a\x11.ListOfAdminReply\"\x00\x12K\n\x17QueryOperationPortState\x12\x17.ListOfOperationRequest\x1a\x15.ListOfOperationReply\"\x00\x12\x31\n\x07SetDrop\x12\x12.ListOfDropRequest\x1a\x10.ListOfDropReply\"\x00\x12^\n\x16SetNicServerAdminState\x12!.ListOfNiCServerAdminStateRequest\x1a\x1f.ListOfNiCServerAdminStateReply\"\x00\x62\x06proto3')
 
-_LISTOFDROPREPLY = _descriptor.Descriptor(
-  name='ListOfDropReply',
-  full_name='ListOfDropReply',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  create_key=_descriptor._internal_create_key,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='nic_addresses', full_name='ListOfDropReply.nic_addresses', index=0,
-      number=1, type=9, cpp_type=9, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-    _descriptor.FieldDescriptor(
-      name='drop_replies', full_name='ListOfDropReply.drop_replies', index=1,
-      number=2, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  serialized_options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=506,
-  serialized_end=580,
-)
 
-_LISTOFADMINREQUEST.fields_by_name['admin_requests'].message_type = nic__simulator__grpc__service__pb2._ADMINREQUEST
-_LISTOFADMINREPLY.fields_by_name['admin_replies'].message_type = nic__simulator__grpc__service__pb2._ADMINREPLY
-_LISTOFOPERATIONREQUEST.fields_by_name['operation_requests'].message_type = \
-    nic__simulator__grpc__service__pb2._OPERATIONREQUEST
-_LISTOFOPERATIONREPLY.fields_by_name['operation_replies'].message_type = \
-    nic__simulator__grpc__service__pb2._OPERATIONREPLY
-_LISTOFDROPREQUEST.fields_by_name['drop_requests'].message_type = nic__simulator__grpc__service__pb2._DROPREQUEST
-_LISTOFDROPREPLY.fields_by_name['drop_replies'].message_type = nic__simulator__grpc__service__pb2._DROPREPLY
-DESCRIPTOR.message_types_by_name['ListOfAdminRequest'] = _LISTOFADMINREQUEST
-DESCRIPTOR.message_types_by_name['ListOfAdminReply'] = _LISTOFADMINREPLY
-DESCRIPTOR.message_types_by_name['ListOfOperationRequest'] = _LISTOFOPERATIONREQUEST
-DESCRIPTOR.message_types_by_name['ListOfOperationReply'] = _LISTOFOPERATIONREPLY
-DESCRIPTOR.message_types_by_name['ListOfDropRequest'] = _LISTOFDROPREQUEST
-DESCRIPTOR.message_types_by_name['ListOfDropReply'] = _LISTOFDROPREPLY
-_sym_db.RegisterFileDescriptor(DESCRIPTOR)
 
+_LISTOFADMINREQUEST = DESCRIPTOR.message_types_by_name['ListOfAdminRequest']
+_LISTOFADMINREPLY = DESCRIPTOR.message_types_by_name['ListOfAdminReply']
+_LISTOFOPERATIONREQUEST = DESCRIPTOR.message_types_by_name['ListOfOperationRequest']
+_LISTOFOPERATIONREPLY = DESCRIPTOR.message_types_by_name['ListOfOperationReply']
+_LISTOFDROPREQUEST = DESCRIPTOR.message_types_by_name['ListOfDropRequest']
+_LISTOFDROPREPLY = DESCRIPTOR.message_types_by_name['ListOfDropReply']
+_LISTOFNICSERVERADMINSTATEREQUEST = DESCRIPTOR.message_types_by_name['ListOfNiCServerAdminStateRequest']
+_LISTOFNICSERVERADMINSTATEREPLY = DESCRIPTOR.message_types_by_name['ListOfNiCServerAdminStateReply']
 ListOfAdminRequest = _reflection.GeneratedProtocolMessageType('ListOfAdminRequest', (_message.Message,), {
-  'DESCRIPTOR': _LISTOFADMINREQUEST,
-  '__module__': 'nic_simulator_grpc_mgmt_service_pb2'
+  'DESCRIPTOR' : _LISTOFADMINREQUEST,
+  '__module__' : 'nic_simulator_grpc_mgmt_service_pb2'
   # @@protoc_insertion_point(class_scope:ListOfAdminRequest)
   })
 _sym_db.RegisterMessage(ListOfAdminRequest)
 
 ListOfAdminReply = _reflection.GeneratedProtocolMessageType('ListOfAdminReply', (_message.Message,), {
-  'DESCRIPTOR': _LISTOFADMINREPLY,
-  '__module__': 'nic_simulator_grpc_mgmt_service_pb2'
+  'DESCRIPTOR' : _LISTOFADMINREPLY,
+  '__module__' : 'nic_simulator_grpc_mgmt_service_pb2'
   # @@protoc_insertion_point(class_scope:ListOfAdminReply)
   })
 _sym_db.RegisterMessage(ListOfAdminReply)
 
 ListOfOperationRequest = _reflection.GeneratedProtocolMessageType('ListOfOperationRequest', (_message.Message,), {
-  'DESCRIPTOR': _LISTOFOPERATIONREQUEST,
-  '__module__': 'nic_simulator_grpc_mgmt_service_pb2'
+  'DESCRIPTOR' : _LISTOFOPERATIONREQUEST,
+  '__module__' : 'nic_simulator_grpc_mgmt_service_pb2'
   # @@protoc_insertion_point(class_scope:ListOfOperationRequest)
   })
 _sym_db.RegisterMessage(ListOfOperationRequest)
 
 ListOfOperationReply = _reflection.GeneratedProtocolMessageType('ListOfOperationReply', (_message.Message,), {
-  'DESCRIPTOR': _LISTOFOPERATIONREPLY,
-  '__module__': 'nic_simulator_grpc_mgmt_service_pb2'
+  'DESCRIPTOR' : _LISTOFOPERATIONREPLY,
+  '__module__' : 'nic_simulator_grpc_mgmt_service_pb2'
   # @@protoc_insertion_point(class_scope:ListOfOperationReply)
   })
 _sym_db.RegisterMessage(ListOfOperationReply)
 
 ListOfDropRequest = _reflection.GeneratedProtocolMessageType('ListOfDropRequest', (_message.Message,), {
-  'DESCRIPTOR': _LISTOFDROPREQUEST,
-  '__module__': 'nic_simulator_grpc_mgmt_service_pb2'
+  'DESCRIPTOR' : _LISTOFDROPREQUEST,
+  '__module__' : 'nic_simulator_grpc_mgmt_service_pb2'
   # @@protoc_insertion_point(class_scope:ListOfDropRequest)
   })
 _sym_db.RegisterMessage(ListOfDropRequest)
 
 ListOfDropReply = _reflection.GeneratedProtocolMessageType('ListOfDropReply', (_message.Message,), {
-  'DESCRIPTOR': _LISTOFDROPREPLY,
-  '__module__': 'nic_simulator_grpc_mgmt_service_pb2'
+  'DESCRIPTOR' : _LISTOFDROPREPLY,
+  '__module__' : 'nic_simulator_grpc_mgmt_service_pb2'
   # @@protoc_insertion_point(class_scope:ListOfDropReply)
   })
 _sym_db.RegisterMessage(ListOfDropReply)
 
-_DUALTORMGMTSERVICE = _descriptor.ServiceDescriptor(
-  name='DualTorMgmtService',
-  full_name='DualTorMgmtService',
-  file=DESCRIPTOR,
-  index=0,
-  serialized_options=None,
-  create_key=_descriptor._internal_create_key,
-  serialized_start=583,
-  serialized_end=879,
-  methods=[
-           _descriptor.MethodDescriptor(
-             name='QueryAdminForwardingPortState',
-             full_name='DualTorMgmtService.QueryAdminForwardingPortState',
-             index=0,
-             containing_service=None,
-             input_type=_LISTOFADMINREQUEST,
-             output_type=_LISTOFADMINREPLY,
-             serialized_options=None,
-             create_key=_descriptor._internal_create_key,
-           ),
-           _descriptor.MethodDescriptor(
-             name='SetAdminForwardingPortState',
-             full_name='DualTorMgmtService.SetAdminForwardingPortState',
-             index=1,
-             containing_service=None,
-             input_type=_LISTOFADMINREQUEST,
-             output_type=_LISTOFADMINREPLY,
-             serialized_options=None,
-             create_key=_descriptor._internal_create_key,
-            ),
-           _descriptor.MethodDescriptor(
-             name='QueryOperationPortState',
-             full_name='DualTorMgmtService.QueryOperationPortState',
-             index=2,
-             containing_service=None,
-             input_type=_LISTOFOPERATIONREQUEST,
-             output_type=_LISTOFOPERATIONREPLY,
-             serialized_options=None,
-             create_key=_descriptor._internal_create_key,
-            ),
-           _descriptor.MethodDescriptor(
-             name='SetDrop',
-             full_name='DualTorMgmtService.SetDrop',
-             index=3,
-             containing_service=None,
-             input_type=_LISTOFDROPREQUEST,
-             output_type=_LISTOFDROPREPLY,
-             serialized_options=None,
-             create_key=_descriptor._internal_create_key,
-            ),
-           ])
-_sym_db.RegisterServiceDescriptor(_DUALTORMGMTSERVICE)
-
-DESCRIPTOR.services_by_name['DualTorMgmtService'] = _DUALTORMGMTSERVICE
+ListOfNiCServerAdminStateRequest = _reflection.GeneratedProtocolMessageType('ListOfNiCServerAdminStateRequest', (_message.Message,), {
+  'DESCRIPTOR' : _LISTOFNICSERVERADMINSTATEREQUEST,
+  '__module__' : 'nic_simulator_grpc_mgmt_service_pb2'
+  # @@protoc_insertion_point(class_scope:ListOfNiCServerAdminStateRequest)
+  })
+_sym_db.RegisterMessage(ListOfNiCServerAdminStateRequest)
 
+ListOfNiCServerAdminStateReply = _reflection.GeneratedProtocolMessageType('ListOfNiCServerAdminStateReply', (_message.Message,), {
+  'DESCRIPTOR' : _LISTOFNICSERVERADMINSTATEREPLY,
+  '__module__' : 'nic_simulator_grpc_mgmt_service_pb2'
+  # @@protoc_insertion_point(class_scope:ListOfNiCServerAdminStateReply)
+  })
+_sym_db.RegisterMessage(ListOfNiCServerAdminStateReply)
+
+_DUALTORMGMTSERVICE = DESCRIPTOR.services_by_name['DualTorMgmtService']
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+  DESCRIPTOR._options = None
+  _LISTOFADMINREQUEST._serialized_start=75
+  _LISTOFADMINREQUEST._serialized_end=157
+  _LISTOFADMINREPLY._serialized_start=159
+  _LISTOFADMINREPLY._serialized_end=236
+  _LISTOFOPERATIONREQUEST._serialized_start=238
+  _LISTOFOPERATIONREQUEST._serialized_end=332
+  _LISTOFOPERATIONREPLY._serialized_start=334
+  _LISTOFOPERATIONREPLY._serialized_end=423
+  _LISTOFDROPREQUEST._serialized_start=425
+  _LISTOFDROPREQUEST._serialized_end=504
+  _LISTOFDROPREPLY._serialized_start=506
+  _LISTOFDROPREPLY._serialized_end=580
+  _LISTOFNICSERVERADMINSTATEREQUEST._serialized_start=582
+  _LISTOFNICSERVERADMINSTATEREQUEST._serialized_end=661
+  _LISTOFNICSERVERADMINSTATEREPLY._serialized_start=663
+  _LISTOFNICSERVERADMINSTATEREPLY._serialized_end=759
+  _DUALTORMGMTSERVICE._serialized_start=762
+  _DUALTORMGMTSERVICE._serialized_end=1154
 # @@protoc_insertion_point(module_scope)
diff --git a/ansible/dualtor/nic_simulator/nic_simulator_grpc_mgmt_service_pb2_grpc.py b/ansible/dualtor/nic_simulator/nic_simulator_grpc_mgmt_service_pb2_grpc.py
--- a/ansible/dualtor/nic_simulator/nic_simulator_grpc_mgmt_service_pb2_grpc.py
+++ b/ansible/dualtor/nic_simulator/nic_simulator_grpc_mgmt_service_pb2_grpc.py
@@ -34,6 +34,11 @@ def __init__(self, channel):
                 request_serializer=nic__simulator__grpc__mgmt__service__pb2.ListOfDropRequest.SerializeToString,
                 response_deserializer=nic__simulator__grpc__mgmt__service__pb2.ListOfDropReply.FromString,
                 )
+        self.SetNicServerAdminState = channel.unary_unary(
+                '/DualTorMgmtService/SetNicServerAdminState',
+                request_serializer=nic__simulator__grpc__mgmt__service__pb2.ListOfNiCServerAdminStateRequest.SerializeToString,
+                response_deserializer=nic__simulator__grpc__mgmt__service__pb2.ListOfNiCServerAdminStateReply.FromString,
+                )
 
 
 class DualTorMgmtServiceServicer(object):
@@ -63,6 +68,12 @@ def SetDrop(self, request, context):
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
+    def SetNicServerAdminState(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
 
 def add_DualTorMgmtServiceServicer_to_server(servicer, server):
     rpc_method_handlers = {
@@ -86,88 +97,102 @@ def add_DualTorMgmtServiceServicer_to_server(servicer, server):
                     request_deserializer=nic__simulator__grpc__mgmt__service__pb2.ListOfDropRequest.FromString,
                     response_serializer=nic__simulator__grpc__mgmt__service__pb2.ListOfDropReply.SerializeToString,
             ),
+            'SetNicServerAdminState': grpc.unary_unary_rpc_method_handler(
+                    servicer.SetNicServerAdminState,
+                    request_deserializer=nic__simulator__grpc__mgmt__service__pb2.ListOfNiCServerAdminStateRequest.FromString,
+                    response_serializer=nic__simulator__grpc__mgmt__service__pb2.ListOfNiCServerAdminStateReply.SerializeToString,
+            ),
     }
     generic_handler = grpc.method_handlers_generic_handler(
             'DualTorMgmtService', rpc_method_handlers)
     server.add_generic_rpc_handlers((generic_handler,))
 
 
-# This class is part of an EXPERIMENTAL API.
+ # This class is part of an EXPERIMENTAL API.
 class DualTorMgmtService(object):
     """Missing associated documentation comment in .proto file."""
 
     @staticmethod
     def QueryAdminForwardingPortState(request,
-                                      target,
-                                      options=(),
-                                      channel_credentials=None,
-                                      call_credentials=None,
-                                      insecure=False,
-                                      compression=None,
-                                      wait_for_ready=None,
-                                      timeout=None,
-                                      metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/DualTorMgmtService/QueryAdminForwardingPortState',
-                                             nic__simulator__grpc__mgmt__service__pb2.ListOfAdminRequest
-                                             .SerializeToString,
-                                             nic__simulator__grpc__mgmt__service__pb2.ListOfAdminReply.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression,
-                                             wait_for_ready, timeout, metadata)
+            nic__simulator__grpc__mgmt__service__pb2.ListOfAdminRequest.SerializeToString,
+            nic__simulator__grpc__mgmt__service__pb2.ListOfAdminReply.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def SetAdminForwardingPortState(request,
-                                    target,
-                                    options=(),
-                                    channel_credentials=None,
-                                    call_credentials=None,
-                                    insecure=False,
-                                    compression=None,
-                                    wait_for_ready=None,
-                                    timeout=None,
-                                    metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/DualTorMgmtService/SetAdminForwardingPortState',
-                                             nic__simulator__grpc__mgmt__service__pb2.ListOfAdminRequest
-                                             .SerializeToString,
-                                             nic__simulator__grpc__mgmt__service__pb2.ListOfAdminReply.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression,
-                                             wait_for_ready, timeout, metadata)
+            nic__simulator__grpc__mgmt__service__pb2.ListOfAdminRequest.SerializeToString,
+            nic__simulator__grpc__mgmt__service__pb2.ListOfAdminReply.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def QueryOperationPortState(request,
-                                target,
-                                options=(),
-                                channel_credentials=None,
-                                call_credentials=None,
-                                insecure=False,
-                                compression=None,
-                                wait_for_ready=None,
-                                timeout=None,
-                                metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/DualTorMgmtService/QueryOperationPortState',
-                                             nic__simulator__grpc__mgmt__service__pb2.ListOfOperationRequest
-                                             .SerializeToString,
-                                             nic__simulator__grpc__mgmt__service__pb2.ListOfOperationReply.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression,
-                                             wait_for_ready, timeout, metadata)
+            nic__simulator__grpc__mgmt__service__pb2.ListOfOperationRequest.SerializeToString,
+            nic__simulator__grpc__mgmt__service__pb2.ListOfOperationReply.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def SetDrop(request,
-                target,
-                options=(),
-                channel_credentials=None,
-                call_credentials=None,
-                insecure=False,
-                compression=None,
-                wait_for_ready=None,
-                timeout=None,
-                metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/DualTorMgmtService/SetDrop',
-                                             nic__simulator__grpc__mgmt__service__pb2.ListOfDropRequest
-                                             .SerializeToString,
-                                             nic__simulator__grpc__mgmt__service__pb2.ListOfDropReply.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression,
-                                             wait_for_ready, timeout, metadata)
+            nic__simulator__grpc__mgmt__service__pb2.ListOfDropRequest.SerializeToString,
+            nic__simulator__grpc__mgmt__service__pb2.ListOfDropReply.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def SetNicServerAdminState(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/DualTorMgmtService/SetNicServerAdminState',
+            nic__simulator__grpc__mgmt__service__pb2.ListOfNiCServerAdminStateRequest.SerializeToString,
+            nic__simulator__grpc__mgmt__service__pb2.ListOfNiCServerAdminStateReply.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)


Test patch:
diff --git a/tests/common/dualtor/nic_simulator_control.py b/tests/common/dualtor/nic_simulator_control.py
--- a/tests/common/dualtor/nic_simulator_control.py
+++ b/tests/common/dualtor/nic_simulator_control.py
@@ -29,7 +29,8 @@
     "set_drop_active_active",
     "TrafficDirection",
     "ForwardingState",
-    "toggle_active_active_simulator_ports"
+    "toggle_active_active_simulator_ports",
+    "stop_nic_grpc_server"
 ]
 
 logger = logging.getLogger(__name__)
@@ -353,3 +354,34 @@ def _toggle_active_active_simulator_ports(mux_ports, portid, state):
                 raise ValueError("failed to toggle port %s, portid %s, state %s" % (mux_port, portid, state))
 
     return _toggle_active_active_simulator_ports
+
+
+@pytest.fixture(scope="function")
+def stop_nic_grpc_server(mux_config, nic_simulator_client, restart_nic_simulator):      # noqa F811
+    """Return a helper function to simulate grpc failure for active-active ports."""
+
+    def _call_set_nic_server_admin_state_nic_simulator(nic_addresses, admin_state):
+        request = nic_simulator_grpc_mgmt_service_pb2.ListOfNiCServerAdminStateRequest(
+            nic_addresses=list(nic_addresses),
+            admin_states=[admin_state for _ in nic_addresses]
+        )
+        client_stub = nic_simulator_client()
+        response = call_grpc(client_stub.SetNicServerAdminState, args=(request,))
+        logging.debug("stop nic grpc server response:\n%s", response)
+        for nic_address, success in zip(response.nic_addresses, response.successes):
+            if not success:
+                raise ValueError("failed to stop grpc server %s" % nic_address)
+
+    def _stop_nic_grpc_server(interface_names):
+        """Stop the grpc servers."""
+        nic_addresses = []
+        for interface_name in interface_names:
+            nic_address = mux_config[interface_name]["SERVER"]["soc_ipv4"].split("/")[0]
+            logger.debug("Stop grpc server on port %s, address %s", interface_name, nic_address)
+            nic_addresses.append(nic_address)
+
+        _call_set_nic_server_admin_state_nic_simulator(nic_addresses, False)
+
+    yield _stop_nic_grpc_server
+
+    restart_nic_simulator()
diff --git a/tests/dualtor_io/test_grpc_server_failure.py b/tests/dualtor_io/test_grpc_server_failure.py
new file mode 100644
--- /dev/null
+++ b/tests/dualtor_io/test_grpc_server_failure.py
@@ -0,0 +1,87 @@
+import pytest
+
+from tests.common.dualtor.control_plane_utils import verify_tor_states
+from tests.common.dualtor.data_plane_utils import send_t1_to_server_with_action     # noqa F401
+from tests.common.dualtor.data_plane_utils import send_server_to_t1_with_action     # noqa F401
+from tests.common.dualtor.dual_tor_utils import upper_tor_host                      # noqa F401
+from tests.common.dualtor.dual_tor_utils import lower_tor_host                      # noqa F401
+from tests.common.fixtures.ptfhost_utils import run_icmp_responder                  # noqa F401
+from tests.common.fixtures.ptfhost_utils import run_garp_service                    # noqa F401
+from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory             # noqa F401
+from tests.common.fixtures.ptfhost_utils import change_mac_addresses                # noqa F401
+from tests.common.dualtor.dual_tor_common import active_active_ports                # noqa F401
+from tests.common.dualtor.dual_tor_common import cable_type                         # noqa F401
+from tests.common.dualtor.dual_tor_common import CableType
+from tests.common.dualtor.nic_simulator_control import stop_nic_grpc_server         # noqa F401
+from tests.common.dualtor.nic_simulator_control import restart_nic_simulator        # noqa F401
+
+
+pytestmark = [
+    pytest.mark.topology("dualtor")
+]
+
+
+@pytest.mark.enable_active_active
+@pytest.mark.skip_active_standby
+def test_grpc_server_failure_config_standby_config_auto_upstream_active(
+    upper_tor_host, lower_tor_host, send_server_to_t1_with_action,      # noqa F811
+    cable_type, active_active_ports, stop_nic_grpc_server               # noqa F811
+):
+    if cable_type == CableType.active_active:
+        stop_nic_grpc_server(active_active_ports)
+        upper_tor_host.shell_cmds(cmds=["config mux mode standby %s" % _ for _ in active_active_ports])
+
+        verify_tor_states(
+            expected_active_host=lower_tor_host,
+            expected_standby_host=upper_tor_host,
+            expected_standby_health='unhealthy',
+            cable_type=cable_type,
+            skip_state_db=True,
+        )
+
+        send_server_to_t1_with_action(
+            upper_tor_host,
+            verify=True,
+            allowed_disruption=0,
+            action=lambda: upper_tor_host.shell("config mux mode auto all")
+        )
+
+        verify_tor_states(
+            expected_active_host=[upper_tor_host, lower_tor_host],
+            expected_standby_host=None,
+            cable_type=cable_type,
+            skip_state_db=True
+        )
+
+
+@pytest.mark.enable_active_active
+@pytest.mark.skip_active_standby
+def test_grpc_server_failure_config_standby_config_auto_downstream_standby(
+    upper_tor_host, lower_tor_host, send_t1_to_server_with_action,      # noqa F811
+    cable_type, active_active_ports, stop_nic_grpc_server               # noqa F811
+):
+    if cable_type == CableType.active_active:
+        stop_nic_grpc_server(active_active_ports)
+        upper_tor_host.shell_cmds(cmds=["config mux mode standby %s" % _ for _ in active_active_ports])
+
+        verify_tor_states(
+            expected_active_host=lower_tor_host,
+            expected_standby_host=upper_tor_host,
+            expected_standby_health='unhealthy',
+            cable_type=cable_type,
+            skip_state_db=True,
+        )
+
+        send_t1_to_server_with_action(
+            upper_tor_host,
+            verify=True,
+            allowed_disruption=0,
+            action=lambda: upper_tor_host.shell("config mux mode auto all")
+        )
+
+        verify_tor_states(
+            expected_active_host=[upper_tor_host, lower_tor_host],
+            expected_standby_host=None,
+            cable_type=cable_type,
+            skip_state_db=True
+        )