Bug: Fix TestGossipingPropertyFileSnitch.test_prefer_local_reconnect_on_listen_address
Status: Resolved
Priority: Normal
Resolution: Fixed
Created: 20/Jun/22 16:44
Resolved: 01/Jul/22 03:34

It is consistently failing in trunk after

CASSANDRA-17661. Maybe a debug message has changed or something?

CC [~Jyothsnakonisa], [~jmeredithco] , [~ycai] 
{code:java}
Error Message
ccmlib.node.TimeoutError: 14 Jun 2022 02:38:50 [node1] after 60.07/60 seconds Missing: ['tiated reconnect to an Internal IP (\\/)?127.0.0.2(:7000)? for the (\\/)?127.0.0.4(:7000)?'] not found in debug.log:  Head: INFO  [main] 2022-06-14 02:36:25,360 YamlConfigura  Tail: ...7.0.0.3:7000(/127.0.0.1:43890)->/127.0.0.4:7000-LARGE_MESSAGES-8f378c72 successfully connected, version = 12, framing = CRC, encryption = unencrypted
Stacktrace
self = <snitch_test.TestGossipingPropertyFileSnitch object at 0x7f2e6ec50dc0>

    def test_prefer_local_reconnect_on_listen_address(self):
        """
            @jira_ticket CASSANDRA-9748
            @jira_ticket CASSANDRA-8084
    
            Test that it's possible to connect over the broadcast_address when
            listen_on_broadcast_address=true and that GossipingPropertyFileSnitch
            reconnect via listen_address when prefer_local=true
            """
    
        NODE1_LISTEN_ADDRESS = '127.0.0.1'
        NODE1_BROADCAST_ADDRESS = '127.0.0.3'
    
        NODE2_LISTEN_ADDRESS = '127.0.0.2'
        NODE2_BROADCAST_ADDRESS = '127.0.0.4'
    
        STORAGE_PORT = 7000
    
        cluster = self.cluster
        cluster.populate(2)
        node1, node2 = cluster.nodelist()
    
        running40 = node1.get_base_cassandra_version() >= 4.0
    
        cluster.seeds = [NODE1_BROADCAST_ADDRESS]
        cluster.set_configuration_options(values={'endpoint_snitch': 'org.apache.cassandra.locator.GossipingPropertyFileSnitch',
                                                  'listen_on_broadcast_address': 'true'})
        node1.set_configuration_options(values={'broadcast_address': NODE1_BROADCAST_ADDRESS})
        node2.auto_bootstrap = True
        node2.set_configuration_options(values={'broadcast_address': NODE2_BROADCAST_ADDRESS})
    
        for node in cluster.nodelist():
            with open(os.path.join(node.get_conf_dir(), 'cassandra-rackdc.properties'), 'w') as snitch_file:
                snitch_file.write("dc=dc1" + os.linesep)
                snitch_file.write("rack=rack1" + os.linesep)
                snitch_file.write("prefer_local=true" + os.linesep)
    
        node1.start(wait_for_binary_proto=True)
    
        self._test_connect(NODE1_LISTEN_ADDRESS, STORAGE_PORT)
        self._test_connect(NODE1_BROADCAST_ADDRESS, STORAGE_PORT)
    
        # write some data to node1
        node1.stress(['write', 'n=10K', 'no-warmup', '-rate', 'threads=8'])
    
        session = self.patient_cql_connection(node1)
        stress_table = 'keyspace1.standard1'
        original_rows = list(session.execute("SELECT * FROM {}".format(stress_table)))
    
        node2.start(wait_for_binary_proto=True, wait_other_notice=False)
    
        self._test_connect(NODE2_LISTEN_ADDRESS, STORAGE_PORT)
        self._test_connect(NODE2_BROADCAST_ADDRESS, STORAGE_PORT)
    
        # substring for Intiated -> Initiated typo was fixed in 3.10
        matchn1 = 'tiated reconnect to an Internal IP (\/)?{}(:7000)? for the (\/)?{}(:7000)?'.format(NODE1_LISTEN_ADDRESS, NODE1_BROADCAST_ADDRESS)
        matchn2 = 'tiated reconnect to an Internal IP (\/)?{}(:7000)? for the (\/)?{}(:7000)?'.format(NODE2_LISTEN_ADDRESS, NODE2_BROADCAST_ADDRESS)
>       node1.watch_log_for(matchn2, filename='debug.log', timeout=60)

snitch_test.py:85: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../venv/lib/python3.8/site-packages/ccmlib/node.py:589: in watch_log_for
    TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

start = 1655174270.069921, timeout = 60
msg = "Missing: ['tiated reconnect to an Internal IP (\\\\/)?127.0.0.2(:7000)? for the (\\\\/)?127.0.0.4(:7000)?'] not found...127.0.0.4:7000-LARGE_MESSAGES-8f378c72 successfully connected, version = 12, framing = CRC, encryption = unencrypted\n"
node = 'node1'

    @staticmethod
    def raise_if_passed(start, timeout, msg, node=None):
        if start + timeout < time.time():
>           raise TimeoutError.create(start, timeout, msg, node)
E           ccmlib.node.TimeoutError: 14 Jun 2022 02:38:50 [node1] after 60.07/60 seconds Missing: ['tiated reconnect to an Internal IP (\\/)?127.0.0.2(:7000)? for the (\\/)?127.0.0.4(:7000)?'] not found in debug.log:
E            Head: INFO  [main] 2022-06-14 02:36:25,360 YamlConfigura
E            Tail: ...7.0.0.3:7000(/127.0.0.1:43890)->/127.0.0.4:7000-LARGE_MESSAGES-8f378c72 successfully connected, version = 12, framing = CRC, encryption = unencrypted

../venv/lib/python3.8/site-packages/ccmlib/node.py:56: TimeoutError
{code}