Repository: sonic-net/sonic-mgmt

Issue:
Investigate restart-ptf failures on backend T0/T1 nightly run
Failures seen on 09/15 run
TASK [vm_set : Setup vlan port for vlan tunnel] ********************************
Thursday 16 September 2021  12:08:09 +0000 (0:00:00.067)       0:00:28.030 **** 
fatal: [STR2-ACS-SERV-18]: FAILED! => {"changed": false, "module_stderr": "Shared connection to 10.64.246.97 closed.\r\n", "module_stdout": "Traceback (most recent call last):\r\n  File \"/home/azure/.ansible/tmp/ansible-tmp-1631794089.6-51123-143018373515312/AnsiballZ_vlan_port.py\", line 114, in <module>\r\n    _ansiballz_main()\r\n  File \"/home/azure/.ansible/tmp/ansible-tmp-1631794089.6-51123-143018373515312/AnsiballZ_vlan_port.py\", line 106, in _ansiballz_main\r\n    invoke_module(zipped_mod, temp_path, ANSIBALLZ_PARAMS)\r\n  File \"/home/azure/.ansible/tmp/ansible-tmp-1631794089.6-51123-143018373515312/AnsiballZ_vlan_port.py\", line 49, in invoke_module\r\n    imp.load_module('__main__', mod, module, MOD_DESC)\r\n  File \"/tmp/ansible_vlan_port_payload_cWs6KG/__main__.py\", line 151, in <module>\r\n  File \"/tmp/ansible_vlan_port_payload_cWs6KG/__main__.py\", line 139, in main\r\n  File \"/tmp/ansible_vlan_port_payload_cWs6KG/__main__.py\", line 65, in create_vlan_ports\r\n  File \"/tmp/ansible_vlan_port_payload_cWs6KG/__main__.py\", line 50, in create_vlan_port\r\n  File \"/tmp/ansible_vlan_port_payload_cWs6KG/__main__.py\", line 113, in cmd\r\nException: ret_code=3, error message=ERROR: trying to add VLAN #2020 to IF -:enp59s0f1:-  error: File exists\r\n. cmd=vconfig add enp59s0f1 2020\r\n", "msg": "MODULE FAILURE\nSee stdout/stderr for the exact error", "rc": 1}



Discussion:
The below failures is seen occassionally on T0 and T1
TASK [vm_set : Renumber topology t1-backend to VMs. base vm = VM1884] **********
Wednesday 06 October 2021  12:17:35 +0000 (0:00:00.038)       0:00:34.448 ***** 
fatal: [STR2-ACS-SERV-18]: FAILED! => {"changed": false, "msg": "ret_code=2, error message=\"RTNETLINK answers: File exists\n\". cmd=\"ip link add ptf-vms18-7-b type veth peer name backplane\""}

reopened since the error shows again

Resolving patch:
diff --git a/ansible/roles/vm_set/library/vlan_port.py b/ansible/roles/vm_set/library/vlan_port.py
--- a/ansible/roles/vm_set/library/vlan_port.py
+++ b/ansible/roles/vm_set/library/vlan_port.py
@@ -1,10 +1,9 @@
 #!/usr/bin/python
 
-import re
-import sys
-import time
 import subprocess
-from pprint import pprint
+import logging
+import traceback
+
 from ansible.module_utils.basic import *
 
 DOCUMENTATION = '''
@@ -28,34 +27,43 @@
 '''
 
 CMD_DEBUG_FNAME = '/tmp/vlan_port.cmds.txt'
-EXCEPTION_DEBUG_FNAME = '/tmp/vlan_port.exception.txt'
+
+
+logging.basicConfig(
+    filename=CMD_DEBUG_FNAME,
+    level=logging.DEBUG,
+    format="%(asctime)s %(levelname)-8s %(message)s"
+)
+
 
 class VlanPort(object):
     def __init__(self, external_port, vlan_ids):
         self.external_port = external_port
         self.vlan_ids = vlan_ids
-        self.host_ifaces = VlanPort.ifconfig('ifconfig -a')
 
         return
 
     def up_external_port(self):
-        if self.external_port in self.host_ifaces:
+        if VlanPort.iface_exists(self.external_port):
             VlanPort.iface_up(self.external_port)
 
         return
 
     def create_vlan_port(self, port, vlan_id):
         vlan_port = "%s.%d" % (port, vlan_id)
-        if vlan_port not in self.host_ifaces:
-            VlanPort.cmd('vconfig rem %s' % vlan_port, True)
-            VlanPort.cmd('vconfig add %s %d' % (port, vlan_id))
+        VlanPort.log_show_vlan_intf(port, vlan_id)
+        try:
+            self.destroy_vlan_port(vlan_port)
+        except Exception:
+            pass
 
+        VlanPort.cmd('vconfig add %s %d' % (port, vlan_id))
         VlanPort.iface_up(vlan_port)
 
         return
 
     def destroy_vlan_port(self, vlan_port):
-        if vlan_port in self.host_ifaces:
+        if VlanPort.iface_exists(vlan_port):
             VlanPort.iface_down(vlan_port)
             VlanPort.cmd('vconfig rem %s' % vlan_port)
 
@@ -94,6 +102,21 @@ def iface_up(iface_name, pid=None):
     def iface_down(iface_name, pid=None):
         return VlanPort.iface_updown(iface_name, 'down', pid)
 
+    @staticmethod
+    def iface_exists(iface_name):
+        iface = VlanPort.ifconfig("ifconfig -a %s" % iface_name)
+        return bool(iface)
+
+    @staticmethod
+    def log_show_vlan_intf(port, vlan_id):
+        cmdline = "cat /proc/net/vlan/config | grep %s" % vlan_id
+        out = VlanPort.cmd(cmdline, ignore_error=True)
+        if out:
+            vlan_intf, vlan_id, port = out.strip().split("|")
+            logging.debug("Port %s has vlan interface %s with vlan id %s" % (port, vlan_intf, vlan_id))
+        else:
+            logging.debug("Port %s doesn't has vlan interface with vlan id %s" % (port, vlan_id))
+
     @staticmethod
     def iface_updown(iface_name, state, pid):
         if pid is None:
@@ -103,21 +126,18 @@ def iface_updown(iface_name, state, pid):
 
     @staticmethod
     def cmd(cmdline, ignore_error=False):
-        with open(CMD_DEBUG_FNAME, 'a') as fp:
-            pprint("CMD: %s" % cmdline, fp)
-        cmd = cmdline.split(' ')
-        process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
+        logging.debug("CMD: %s", cmdline)
+        process = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
         stdout, stderr = process.communicate()
         ret_code = process.returncode
 
         if ret_code != 0 and not ignore_error:
             raise Exception("ret_code=%d, error message=%s. cmd=%s" % (ret_code, stderr, cmdline))
 
-        with open(CMD_DEBUG_FNAME, 'a') as fp:
-            if ret_code == 0:
-                pprint("OUTPUT: %s" % stdout, fp)
-            else:
-                pprint("ERR: %s" % stderr, fp)
+        if ret_code == 0:
+            logging.info("OUTPUT: %s", stdout)
+        else:
+            logging.error("ERR: %s", stderr)
 
         return stdout.decode('utf-8')
 
@@ -130,6 +150,9 @@ def main():
         vlan_ids=dict(required=True, type='dict'),
     ))
 
+    # log separator
+    logging.info("--------------------------------------------------------------------")
+
     cmd = module.params['cmd']
     external_port = module.params['external_port']
     vlan_ids = module.params['vlan_ids']
@@ -137,19 +160,20 @@ def main():
     fp_ports = {}
 
     vp = VlanPort(external_port, vlan_ids)
-
-    vp.up_external_port()
-    if cmd == "create":
-        vp.create_vlan_ports()
-    elif cmd == "remove":
-        vp.remove_vlan_ports()
-
-    fp_port_templ = external_port + ".%s"
-    for a_port_index, vid in vlan_ids.items():
-        fp_ports[a_port_index] = fp_port_templ % vid
-
-    module.exit_json(changed=False, ansible_facts={'dut_fp_ports': fp_ports})
-
+    try:
+        vp.up_external_port()
+        if cmd == "create":
+            vp.create_vlan_ports()
+        elif cmd == "remove":
+            vp.remove_vlan_ports()
+
+        fp_port_templ = external_port + ".%s"
+        for a_port_index, vid in vlan_ids.items():
+            fp_ports[a_port_index] = fp_port_templ % vid
+
+        module.exit_json(changed=False, ansible_facts={'dut_fp_ports': fp_ports})
+    except Exception as detail:
+        module.fail_json(msg="ERROR: %s, TRACEBACK: %s" % (repr(detail), traceback.format_exc()))
 
 if __name__ == "__main__":
     main()


Test patch: