Repository: faucetsdn/faucet
Issue: New MC-LAG implementation doesn't work after cold start

Steps to reproduce:

    1. Bring standby LAG port down
    2. Cold-start switch (either by bringing switch down and back up again, or reloading faucet with enough of a config change that it will cold start switches on the network)
    3. Bring standby LAG port up

It never recognizes that the port is back up for whatever reason.  The dreaded cold-start cached state!  After this point, something about the LACP state machine seems to be borked, since I can't get it to recover with any sequence of bringing ports up/down, but restarting Faucet does fix the situation. 

```
<cold start with sw2-28 down>
Feb 10 19:19:14 faucet.valve INFO     DPID 178 (0xb2) nz-kiwi-t1sw2 Cold start configuring DP
Feb 10 19:19:14 faucet.valve INFO     DPID 178 (0xb2) nz-kiwi-t1sw2 Port 9 (to t2sw1 port 52) configured
Feb 10 19:19:14 faucet.valve INFO     DPID 178 (0xb2) nz-kiwi-t1sw2 Port 10 (to t2sw2 port 52) configured
Feb 10 19:19:14 faucet.valve INFO     DPID 178 (0xb2) nz-kiwi-t1sw2 Port 11 (to t2sw3 port 52) configured
Feb 10 19:19:14 faucet.valve INFO     DPID 178 (0xb2) nz-kiwi-t1sw2 Port 6 (to t1sw1 port 6) configured
Feb 10 19:19:14 faucet.valve INFO     DPID 178 (0xb2) nz-kiwi-t1sw2 Configuring VLAN 171 vid:171 tagged: Port 28
<bring sw2-28 up>
Feb 10 19:20:25 faucet.valve INFO     DPID 178 (0xb2) nz-kiwi-t1sw2 status change: Port 28 up status True reason MODIFY state 4
Feb 10 19:20:25 faucet.valve INFO     DPID 178 (0xb2) nz-kiwi-t1sw2 Port 28 (egress) up
Feb 10 19:20:25 faucet.valve INFO     DPID 178 (0xb2) nz-kiwi-t1sw2 Configuring VLAN 171 vid:171 tagged: Port 28
Feb 10 19:20:26 faucet.valve INFO     DPID 178 (0xb2) nz-kiwi-t1sw2 status did not change: Port 28 up status True reason MODIFY state 4
<bring sw1-28 down>
Feb 10 19:21:04 faucet.valve INFO     DPID 177 (0xb1) nz-kiwi-t1sw1 status did not change: Port 28 up status True reason MODIFY state 0
Feb 10 19:21:04 faucet.valve INFO     DPID 177 (0xb1) nz-kiwi-t1sw1 Port 28 state OFPPS_LIVE reset, ignoring in expectation of port down
Feb 10 19:21:04 faucet.valve INFO     DPID 177 (0xb1) nz-kiwi-t1sw1 status change: Port 28 up status False reason MODIFY state 1
Feb 10 19:21:04 faucet.valve INFO     DPID 177 (0xb1) nz-kiwi-t1sw1 Port 28 (egress) down
Feb 10 19:21:04 faucet.valve INFO     DPID 177 (0xb1) nz-kiwi-t1sw1 LAG 3 Port 28 actor state INITIALIZING (previous state UP)
Feb 10 19:21:04 faucet.valve INFO     DPID 177 (0xb1) nz-kiwi-t1sw1 LAG 3 Port 28 UNSELECTED (previous state SELECTED)
Feb 10 19:21:04 faucet.valve INFO     DPID 177 (0xb1) nz-kiwi-t1sw1 Configuring VLAN 171 vid:171 tagged: Port 28
```

Changed files:
tests/integration/mininet_multidp_tests.py, faucet/valve.py

diff --git a/faucet/valve.py b/faucet/valve.py
index 8b979097..429f9e04 100644
--- a/faucet/valve.py
+++ b/faucet/valve.py
@@ -215,8 +215,9 @@ class Valve:
             for vlan in self.dp.vlans.values():
                 if vlan.faucet_vips_by_ipv(route_manager.IPV):
                     route_manager.active = True
+                    vips_str = list(str(vip) for vip in vlan.faucet_vips_by_ipv(route_manager.IPV))
                     self.logger.info('IPv%u routing is active on %s with VIPs %s' % (
-                        route_manager.IPV, vlan, vlan.faucet_vips_by_ipv(route_manager.IPV)))
+                                     route_manager.IPV, vlan, vips_str))
             for eth_type in route_manager.CONTROL_ETH_TYPES:
                 self._route_manager_by_eth_type[eth_type] = route_manager
         eth_dst_hairpin_table = self.dp.tables.get('eth_dst_hairpin', None)
@@ -1069,7 +1070,7 @@ class Valve:
         ofmsgs = []
         updated = self.lacp_update_actor_state(port, lacp_up, now, lacp_pkt)
         select_updated = self.lacp_update_port_selection_state(port, other_valves)
-        if updated or select_updated:
+        if updated or select_updated or cold_start:
             if updated:
                 self._reset_lacp_status(port)
             if port.is_port_selected() and port.is_actor_up():
diff --git a/tests/integration/mininet_multidp_tests.py b/tests/integration/mininet_multidp_tests.py
index 7cd7da2d..48293345 100644
--- a/tests/integration/mininet_multidp_tests.py
+++ b/tests/integration/mininet_multidp_tests.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import os
+import time
 import networkx
 
 from mininet.log import error
@@ -1109,6 +1110,62 @@ class FaucetSingleLAGTest(FaucetTopoTestBase):
         self.verify_stack_up()
         self.verify_lag_connectivity(self.LACP_HOST)
 
+    def restart_on_down_lag_port(self, port_dp_index, cold_start_dp_index):
+        """Down a port on port_dpid_index, cold-start on cold_start_dp, UP previous port"""
+        # Bring a LACP port DOWN
+        chosen_dpid = self.dpids[port_dp_index]
+        port_no = self.host_information[self.LACP_HOST]['ports'][chosen_dpid][0]
+        self.set_port_down(port_no, chosen_dpid)
+        self.verify_num_lag_up_ports(1, chosen_dpid)
+        # Cold start switch, cold-start twice to get back to initial condition
+        cold_start_dpid = self.dpids[cold_start_dp_index]
+        conf = self._get_faucet_conf()
+        interfaces_conf = conf['dps'][self.dp_name(cold_start_dp_index)]['interfaces']
+        for port, port_conf in interfaces_conf.items():
+            if 'lacp' not in port_conf and 'stack' not in port_conf:
+                # Change host VLAN to enable cold-starting on faucet-2
+                curr_vlan = port_conf['native_vlan']
+                port_conf['native_vlan'] = (
+                    self.vlan_name(1) if curr_vlan == self.vlan_name(0) else self.vlan_name(0))
+                # VLAN changed so just delete the host information instead of recomputing
+                #   routes etc..
+                for _id in self.host_information:
+                    if cold_start_dpid in self.host_information[_id]['ports']:
+                        ports = self.host_information[_id]['ports'][cold_start_dpid]
+                        if port in ports:
+                            del self.host_information[_id]
+                            break
+                break
+        self.reload_conf(conf, self.faucet_config_path, restart=True,
+                         cold_start=True, change_expected=False)
+        time.sleep(5)
+        # Bring LACP port UP
+        self.set_port_up(port_no, chosen_dpid)
+        self.verify_num_lag_up_ports(2, chosen_dpid)
+        # Take down all of the other ports
+        for dpid, ports in self.host_information[self.LACP_HOST]['ports'].items():
+            for port in ports:
+                if dpid != chosen_dpid and port != port_no:
+                    self.set_port_down(port, dpid)
+
+    def test_mclag_coldstart(self):
+        """Test LACP MCLAG after a cold start"""
+        lacp_host_links = [0, 0, 1, 1]
+        self.set_up(lacp_host_links)
+        self.verify_stack_up()
+        self.verify_lag_host_connectivity()
+        self.restart_on_down_lag_port(1, 1)
+        self.verify_lag_host_connectivity()
+
+    def test_mclag_warmstart(self):
+        """Test LACP MCLAG after a warm start"""
+        lacp_host_links = [0, 0, 1, 1]
+        self.set_up(lacp_host_links)
+        self.verify_stack_up()
+        self.verify_lag_host_connectivity()
+        self.restart_on_down_lag_port(0, 1)
+        self.verify_lag_host_connectivity()
+
 
 class FaucetSingleLAGOnUniqueVLANTest(FaucetSingleLAGTest):
     """Test LACP LAG on Faucet stack topologies with a distributed LAG bundle on a unique VLAN"""