Code change:
It("each node by ordering clean reboot and ensure they function upon restart", func() { // clean shutdown and restart <del> testReboot(c, "sudo reboot") </del> <ins> // We sleep 10 seconds to give some time for ssh command to cleanly finish before the node is rebooted. testReboot(c, "nohup sh -c 'sleep 10 && sudo reboot' >/dev/null 2>&1 &") </ins> }) It("each node by ordering unclean reboot and ensure they function upon restart", func() { // unclean shutdown and restart <del> testReboot(c, "echo b | sudo tee /proc/sysrq-trigger") </del> <ins> // We sleep 10 seconds to give some time for ssh command to cleanly finish before the node is shutdown. testReboot(c, "nohup sh -c 'sleep 10 && echo b | sudo tee /proc/sysrq-trigger' >/dev/null 2>&1 &") </ins> }) It("each node by triggering kernel panic and ensure they function upon restart", func() { // kernel panic <del> testReboot(c, "echo c | sudo tee /proc/sysrq-trigger") </del> <ins> // We sleep 10 seconds to give some time for ssh command to cleanly finish before kernel panic is triggered. testReboot(c, "nohup sh -c 'sleep 10 && echo c | sudo tee /proc/sysrq-trigger' >/dev/null 2>&1 &") </ins> }) It("each node by switching off the network interface and ensure they function upon switch on", func() { // switch the network interface off for a while to simulate a network outage <del> testReboot(c, "sudo ifdown eth0 && sleep 120 && sudo ifup eth0") </del> <ins> // We sleep 10 seconds to give some time for ssh command to cleanly finish before network is down. testReboot(c, "nohup sh -c 'sleep 10 && sudo ifdown eth0 && sleep 120 && sudo ifup eth0' >/dev/null 2>&1 &") </ins> }) <del> It("each node by dropping all inbound packages for a while and ensure they function afterwards", func() { </del> <ins> It("each node by dropping all inbound packets for a while and ensure they function afterwards", func() { </ins> // tell the firewall to drop all inbound packets for a while <del> testReboot(c, "sudo iptables -A INPUT -j DROP && sleep 120 && sudo iptables -D INPUT -j DROP") </del> <ins> // We sleep 10 seconds to give some time for ssh command to cleanly finish before starting dropping inbound packets. testReboot(c, "nohup sh -c 'sleep 10 && sudo iptables -A INPUT -j DROP && sleep 120 && sudo iptables -D INPUT -j DROP' >/dev/null 2>&1 &") </ins> }) <del> It("each node by dropping all outbound packages for a while and ensure they function afterwards", func() { </del> <ins> It("each node by dropping all outbound packets for a while and ensure they function afterwards", func() { </ins> // tell the firewall to drop all outbound packets for a while <del> testReboot(c, "sudo iptables -A OUTPUT -j DROP && sleep 120 && sudo iptables -D OUTPUT -j DROP") </del> <ins> // We sleep 10 seconds to give some time for ssh command to cleanly finish before starting dropping outbound packets. testReboot(c, "nohup sh -c 'sleep 10 && sudo iptables -A OUTPUT -j DROP && sleep 120 && sudo iptables -D OUTPUT -j DROP' >/dev/null 2>&1 &") </ins> }) })

Related discussion:
text: All reboot tests are pretty stable (1 failure in 180 runs, in total): each node by dropping all inbound packets for a while and ensure they function afterwards (No known failures. Stability 100 %) 4 min 5 sec Passed each node by dropping all outbound packets for a while and ensure they function afterwards (No known failures. Stability 100 %) 4 min 6 sec Passed each node by ordering clean reboot and ensure they function upon restart (Failed 1 times in the last 30 runs. Stability: 96 %) 1 min 41 sec Passed each node by ordering unclean reboot and ensure they function upon restart (No known 1. failures. Stability 100 %) 1 min 25 sec Passed each node by switching off the network interface and ensure they function upon switch on (No known failures. Stability 100 %) 4 min 0 sec Passed each node by triggering kernel panic and ensure they function upon restart (No known failures. Stability 100 %)