Bug: Test Failure: test_assassinate_valid_node
Status: Resolved
Priority: Normal
Resolution: Fixed
Created: 08/Apr/24 16:35
Resolved: 18/Apr/24 12:30

Failing consistently on trunk:

{code:java}
ccmlib.node.TimeoutError: 03 Apr 2024 19:39:32 [node1] after 120.11/120 seconds Missing: ['127.0.0.4:7000.* is now UP'] not found in system.log:
 Head: INFO  [Messaging-EventLoop-3-1] 2024-04-03 19:37:3
 Tail: ... some nodes were not ready
INFO  [OptionalTasks:1] 2024-04-03 19:39:30,454 CassandraRoleManager.java:484 - Setup task failed with error, rescheduling
self = <gossip_test.TestGossip object at 0x7f1e63fed400>

    def test_assassinate_valid_node(self):
        """
        @jira_ticket CASSANDRA-16588
        Test that after taking two non-seed nodes down and assassinating
        one of them, the other can come back up.
        """
        cluster = self.cluster
    
        cluster.populate(5).start()
        node1 = cluster.nodelist()[0]
        node3 = cluster.nodelist()[2]
    
        self.cluster.set_configuration_options({
            'seed_provider': [{'class_name': 'org.apache.cassandra.locator.SimpleSeedProvider',
                               'parameters': [{'seeds': node1.address()}]
                              }]
            })
    
        non_seed_nodes = cluster.nodelist()[-2:]
        for node in non_seed_nodes:
            node.stop()
    
        assassination_target = non_seed_nodes[0]
        logger.debug("Assassinating non-seed node {}".format(assassination_target.address()))
        out, err, _ = node1.nodetool("assassinate {}".format(assassination_target.address()))
        assert_stderr_clean(err)
    
        logger.debug("Starting non-seed nodes")
        for node in non_seed_nodes:
>           node.start()

gossip_test.py:78: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../env3.8/lib/python3.8/site-packages/ccmlib/node.py:915: in start
    node.watch_log_for_alive(self, from_mark=mark)
../env3.8/lib/python3.8/site-packages/ccmlib/node.py:684: in watch_log_for_alive
    self.watch_log_for(tofind, from_mark=from_mark, timeout=timeout, filename=filename)
../env3.8/lib/python3.8/site-packages/ccmlib/node.py:608: in watch_log_for
    TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

start = 1712173052.8186479, timeout = 120
msg = "Missing: ['127.0.0.4:7000.* is now UP'] not found in system.log:\n Head: INFO  [Messaging-EventLoop-3-1] 2024-04-03 1...[OptionalTasks:1] 2024-04-03 19:39:30,454 CassandraRoleManager.java:484 - Setup task failed with error, rescheduling\n"
node = 'node1'

    @staticmethod
    def raise_if_passed(start, timeout, msg, node=None):
        if start + timeout < time.time():
>           raise TimeoutError.create(start, timeout, msg, node)
E           ccmlib.node.TimeoutError: 03 Apr 2024 19:39:32 [node1] after 120.11/120 seconds Missing: ['127.0.0.4:7000.* is now UP'] not found in system.log:
E            Head: INFO  [Messaging-EventLoop-3-1] 2024-04-03 19:37:3
E            Tail: ... some nodes were not ready
E           INFO  [OptionalTasks:1] 2024-04-03 19:39:30,454 CassandraRoleManager.java:484 - Setup task failed with error, rescheduling

../env3.8/lib/python3.8/site-packages/ccmlib/node.py:56: TimeoutError
{code}

https://app.circleci.com/pipelines/github/ekaterinadimitrova2/cassandra/2680/workflows/8b1c0d0a-7458-4b43-9bba-ac96b9bfe64f/jobs/58929/tests#failed-test-0

https://ci-cassandra.apache.org/job/Cassandra-trunk/1859/#showFailuresLink