Repository: sonic-net/sonic-mgmt

Issue:
Investigate restart-ptf failures on backend T0/T1 nightly run
Failures seen on 09/15 run
TASK [vm_set : Setup vlan port for vlan tunnel] ********************************
Thursday 16 September 2021  12:08:09 +0000 (0:00:00.067)       0:00:28.030 **** 
fatal: [STR2-ACS-SERV-18]: FAILED! => {"changed": false, "module_stderr": "Shared connection to 10.64.246.97 closed.\r\n", "module_stdout": "Traceback (most recent call last):\r\n  File \"/home/azure/.ansible/tmp/ansible-tmp-1631794089.6-51123-143018373515312/AnsiballZ_vlan_port.py\", line 114, in <module>\r\n    _ansiballz_main()\r\n  File \"/home/azure/.ansible/tmp/ansible-tmp-1631794089.6-51123-143018373515312/AnsiballZ_vlan_port.py\", line 106, in _ansiballz_main\r\n    invoke_module(zipped_mod, temp_path, ANSIBALLZ_PARAMS)\r\n  File \"/home/azure/.ansible/tmp/ansible-tmp-1631794089.6-51123-143018373515312/AnsiballZ_vlan_port.py\", line 49, in invoke_module\r\n    imp.load_module('__main__', mod, module, MOD_DESC)\r\n  File \"/tmp/ansible_vlan_port_payload_cWs6KG/__main__.py\", line 151, in <module>\r\n  File \"/tmp/ansible_vlan_port_payload_cWs6KG/__main__.py\", line 139, in main\r\n  File \"/tmp/ansible_vlan_port_payload_cWs6KG/__main__.py\", line 65, in create_vlan_ports\r\n  File \"/tmp/ansible_vlan_port_payload_cWs6KG/__main__.py\", line 50, in create_vlan_port\r\n  File \"/tmp/ansible_vlan_port_payload_cWs6KG/__main__.py\", line 113, in cmd\r\nException: ret_code=3, error message=ERROR: trying to add VLAN #2020 to IF -:enp59s0f1:-  error: File exists\r\n. cmd=vconfig add enp59s0f1 2020\r\n", "msg": "MODULE FAILURE\nSee stdout/stderr for the exact error", "rc": 1}



Discussion:
The below failures is seen occassionally on T0 and T1
TASK [vm_set : Renumber topology t1-backend to VMs. base vm = VM1884] **********
Wednesday 06 October 2021  12:17:35 +0000 (0:00:00.038)       0:00:34.448 ***** 
fatal: [STR2-ACS-SERV-18]: FAILED! => {"changed": false, "msg": "ret_code=2, error message=\"RTNETLINK answers: File exists\n\". cmd=\"ip link add ptf-vms18-7-b type veth peer name backplane\""}


Resolving patch:
diff --git a/ansible/roles/vm_set/library/vm_topology.py b/ansible/roles/vm_set/library/vm_topology.py
--- a/ansible/roles/vm_set/library/vm_topology.py
+++ b/ansible/roles/vm_set/library/vm_topology.py
@@ -9,6 +9,7 @@
 import shlex
 import time
 import traceback
+import random
 
 import docker
 
@@ -390,16 +391,9 @@ def add_mgmt_port_to_docker(self, mgmt_bridge, mgmt_ip, mgmt_gw, mgmt_ipv6_addr=
             self.pid = api_server_pid
         if VMTopology.intf_not_exists(MGMT_PORT_NAME, self.pid):
             if api_server_pid is None:
-                tmp_mgmt_if = hashlib.md5((PTF_NAME_TEMPLATE % self.vm_set_name).encode("utf-8")).hexdigest()[0:6] + MGMT_PORT_NAME
-                self.add_br_if_to_docker(mgmt_bridge, PTF_MGMT_IF_TEMPLATE % self.vm_set_name, tmp_mgmt_if)
+                self.add_br_if_to_docker(mgmt_bridge, PTF_MGMT_IF_TEMPLATE % self.vm_set_name, MGMT_PORT_NAME)
             else:
-                tmp_mgmt_if = hashlib.md5(('apiserver').encode("utf-8")).hexdigest()[0:6] + MGMT_PORT_NAME
-                self.add_br_if_to_docker(mgmt_bridge, 'apiserver', tmp_mgmt_if)
-
-            VMTopology.iface_down(tmp_mgmt_if, self.pid)
-            VMTopology.cmd("nsenter -t %s -n ip link set dev %s name %s" % (self.pid, tmp_mgmt_if, MGMT_PORT_NAME))
-
-        VMTopology.iface_up(MGMT_PORT_NAME, self.pid)
+                self.add_br_if_to_docker(mgmt_bridge, 'apiserver', MGMT_PORT_NAME)
         self.add_ip_to_docker_if(MGMT_PORT_NAME, mgmt_ip, mgmt_ipv6_addr=mgmt_ipv6_addr, mgmt_gw=mgmt_gw, mgmt_gw_v6=mgmt_gw_v6, api_server_pid=api_server_pid)
 
     def add_bp_port_to_docker(self, mgmt_ip, mgmt_ipv6):
@@ -408,9 +402,12 @@ def add_bp_port_to_docker(self, mgmt_ip, mgmt_ipv6):
         VMTopology.iface_disable_txoff(BP_PORT_NAME, self.pid)
 
     def add_br_if_to_docker(self, bridge, ext_if, int_if):
-        logging.info('=== For veth pair, add %s to bridge %s, set %s to PTF docker' % (ext_if, bridge, int_if))
+        # add random suffix to int_if to support multiple tasks run concurrently
+        random_suffix = generate_random_code(6)
+        tmp_int_if = int_if + random_suffix
+        logging.info('=== For veth pair, add %s to bridge %s, set %s to PTF docker, tmp intf %s' % (ext_if, bridge, int_if, tmp_int_if))
         if VMTopology.intf_not_exists(ext_if):
-            VMTopology.cmd("ip link add %s type veth peer name %s" % (ext_if, int_if))
+            VMTopology.cmd("ip link add %s type veth peer name %s" % (ext_if, tmp_int_if))
 
         _, if_to_br = VMTopology.brctl_show(bridge)
         if ext_if not in if_to_br:
@@ -418,8 +415,9 @@ def add_br_if_to_docker(self, bridge, ext_if, int_if):
 
         VMTopology.iface_up(ext_if)
 
-        if VMTopology.intf_exists(int_if) and VMTopology.intf_not_exists(int_if, self.pid):
-            VMTopology.cmd("ip link set netns %s dev %s" % (self.pid, int_if))
+        if VMTopology.intf_exists(tmp_int_if) and VMTopology.intf_not_exists(tmp_int_if, self.pid):
+            VMTopology.cmd("ip link set netns %s dev %s" % (self.pid, tmp_int_if))
+            VMTopology.cmd("nsenter -t %s -n ip link set dev %s name %s" % (self.pid, tmp_int_if, int_if))
 
         VMTopology.iface_up(int_if, self.pid)
 
@@ -1171,6 +1169,18 @@ def _assert(condition, exctype, msg):
     return hostif_exists, vms_exists
 
 
+def generate_random_code(digit=6):
+    """
+    Generate random code
+    Args:
+        digit (int): digit of code, e.g. 6
+
+    Returns:
+        str: random code, e.g. k9A27L
+    """
+    return ''.join(random.sample("abcdefghijklmnABCDEFGHIJKLMN0123456789", digit))
+
+
 def check_params(module, params, mode):
     for param in params:
         if param not in module.params:


Test patch: