Bug: Test Failure: test_resumable_decommission
Status: Open
Priority: Normal
Created: 08/Apr/24 16:45


{code:java}
self = <topology_test.TestTopology object at 0x7f985e7c5070>

    @since('3.10')
    def test_resumable_decommission(self):
        """
        @jira_ticket CASSANDRA-12008
    
        Test decommission operation is resumable
        """
        self.fixture_dtest_setup.ignore_log_patterns = [r'Streaming error occurred',
                                                        r'Error while decommissioning node',
                                                        r'Remote peer 127.0.0.2 failed stream session',
                                                        r'Remote peer \/?127.0.0.2:7000 failed stream session',
                                                        r'peer 127.0.0.2:7000 is probably down',]
        cluster = self.cluster
        cluster.set_configuration_options(values={'stream_throughput_outbound_megabits_per_sec': 1})
        cluster.populate(3, install_byteman=True).start()
        node1, node2, node3 = cluster.nodelist()
    
        session = self.patient_cql_connection(node2)
        # reduce system_distributed RF to 2 so we don't require forceful decommission
        session.execute("ALTER KEYSPACE system_distributed WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor':'2'};")
        create_ks(session, 'ks', 2)
        create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'})
        insert_c1c2(session, n=10000, consistency=ConsistencyLevel.ALL)
    
        # Execute first rebuild, should fail
        with pytest.raises(ToolError):
            if cluster.version() >= '4.0':
                script = [mk_bman_path('4.0/decommission_failure_inject.btm')]
            else:
                script = [mk_bman_path('pre4.0/decommission_failure_inject.btm')]
            node2.byteman_submit(script)
            node2.nodetool('decommission')
    
        # Make sure previous ToolError is due to decommission
        node2.watch_log_for('Error while decommissioning node')
    
        # Decommission again
        mark = node2.mark_log()
        node2.nodetool('decommission')
    
        # Check decommision is done and we skipped transfereed ranges
        node2.watch_log_for('DECOMMISSIONED', from_mark=mark)
        node2.grep_log("Skipping transferred range .* of keyspace ks, endpoint {}".format(node2.address_for_current_version_slashy()), filename='debug.log')
    
        # Check data is correctly forwarded to node1 and node3
        cluster.remove(node2)
        node3.stop(gently=False)
        session = self.patient_exclusive_cql_connection(node1)
        session.execute('USE ks')
        for i in range(0, 10000):
            query_c1c2(session, i, ConsistencyLevel.ONE)
        node1.stop(gently=False)
        node3.start()
        session.shutdown()
        mark = node3.mark_log()
        node3.watch_log_for('Starting listening for CQL clients', from_mark=mark)
        session = self.patient_exclusive_cql_connection(node3)
        session.execute('USE ks')
        for i in range(0, 10000):
>           query_c1c2(session, i, ConsistencyLevel.ONE)

topology_test.py:275: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tools/data.py:43: in query_c1c2
    assertions.assert_length_equal(rows, 1)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

object_with_length = [], expected_length = 1

    def assert_length_equal(object_with_length, expected_length):
        """
        Assert an object has a specific length.
        @param object_with_length The object whose length will be checked
        @param expected_length The expected length of the object
    
        Examples:
        assert_length_equal(res, nb_counter)
        """
>       assert len(object_with_length) == expected_length, \
            "Expected {} to have length {}, but instead is of length {}"\
            .format(object_with_length, expected_length, len(object_with_length))
E       AssertionError: Expected [] to have length 1, but instead is of length 0

tools/assertions.py:267: AssertionError
{code}
https://app.circleci.com/pipelines/github/ekaterinadimitrova2/cassandra/2680/workflows/8b1c0d0a-7458-4b43-9bba-ac96b9bfe64f/jobs/58929/tests#failed-test-0