for m in machines: m.wait_for_unit("consul.service") <del> for m in machines: m.wait_until_succeeds("[ $(consul members | grep -o alive | wc -l) == 5 ]") </del> <ins> def wait_for_healthy_servers(): # See https://github.com/hashicorp/consul/issues/8118#issuecomment-645330040 # for why the `Voter` column of `list-peers` has that info. # TODO: The `grep true` relies on the fact that currently in #       the output like #           # consul operator raft list-peers #           Node     ID   Address           State     Voter  RaftProtocol #           server3  ...  192.168.1.3:8300  leader    true   3 #           server2  ...  192.168.1.2:8300  follower  true   3 #           server1  ...  192.168.1.1:8300  follower  false  3 #       `Voter`is the only boolean column. #       Change this to the more reliable way to be defined by #       https://github.com/hashicorp/consul/issues/8118 #       once that ticket is closed. for m in machines: m.wait_until_succeeds( "[ $(consul operator raft list-peers | grep true | wc -l) == 3 ]" ) def wait_for_all_machines_alive(): """ Note that Serf-"alive" does not mean "Raft"-healthy; see `wait_for_healthy_servers()` for that instead. """ for m in machines: m.wait_until_succeeds("[ $(consul members | grep -o alive | wc -l) == 5 ]") wait_for_healthy_servers() # Also wait for clients to be alive. wait_for_all_machines_alive() </ins> client1.succeed("consul kv put testkey 42") client2.succeed("[ $(consul kv get testkey) == 42 ]") <del> # Test that the cluster can tolearate failures of any single server: for server in servers: server.crash() # For each client, wait until they have connection again # using `kv get -recurse` before issuing commands. client1.wait_until_succeeds("consul kv get -recurse") client2.wait_until_succeeds("consul kv get -recurse") </del> <ins> def rolling_reboot_test(proper_rolling_procedure=True): """ Tests that the cluster can tolearate failures of any single server, following the recommended rolling upgrade procedure from https://www.consul.io/docs/upgrading#standard-upgrades. </ins> <del> # Do some consul actions while one server is down. client1.succeed("consul kv put testkey 43") client2.succeed("[ $(consul kv get testkey) == 43 ]") client2.succeed("consul kv delete testkey") </del> <ins> Optionally, `proper_rolling_procedure=False` can be given to wait only for each server to be back `Healthy`, not `Stable` in the Raft consensus, see Consul setting `ServerStabilizationTime` and https://github.com/hashicorp/consul/issues/8118#issuecomment-645330040. """ for server in servers: server.crash() # For each client, wait until they have connection again # using `kv get -recurse` before issuing commands. client1.wait_until_succeeds("consul kv get -recurse") client2.wait_until_succeeds("consul kv get -recurse") </ins> <del> # Restart crashed machine. server.start() </del> <ins> # Do some consul actions while one server is down. client1.succeed("consul kv put testkey 43") client2.succeed("[ $(consul kv get testkey) == 43 ]") client2.succeed("consul kv delete testkey") # Restart crashed machine. server.start() if proper_rolling_procedure: # Wait for recovery. wait_for_healthy_servers() else: # NOT proper rolling upgrade procedure, see above. wait_for_all_machines_alive() # Wait for client connections. client1.wait_until_succeeds("consul kv get -recurse") client2.wait_until_succeeds("consul kv get -recurse") # Do some consul actions with server back up. client1.succeed("consul kv put testkey 44") client2.succeed("[ $(consul kv get testkey) == 44 ]") client2.succeed("consul kv delete testkey") def all_servers_crash_simultaneously_test(): """ Tests that the cluster will eventually come back after all servers crash simultaneously. """ for server in servers: server.crash() for server in servers: server.start() </ins> # Wait for recovery. <del> for m in machines: m.wait_until_succeeds("[ $(consul members | grep -o alive | wc -l) == 5 ]") </del> <ins> wait_for_healthy_servers() </ins> # Wait for client connections. client1.wait_until_succeeds("consul kv get -recurse") client2.wait_until_succeeds("consul kv get -recurse") <del> # Do some consul actions with server back up. </del> <ins> # Do some consul actions with servers back up. </ins> client1.succeed("consul kv put testkey 44") client2.succeed("[ $(consul kv get testkey) == 44 ]") client2.succeed("consul kv delete testkey") <ins> # Run the tests. print("rolling_reboot_test()") rolling_reboot_test() print("all_servers_crash_simultaneously_test()") all_servers_crash_simultaneously_test() print("rolling_reboot_test(proper_rolling_procedure=False)") rolling_reboot_test(proper_rolling_procedure=False) </ins> ''; })