Bug: Test Failure: org.apache.cassandra.simulator.test.ShortPaxosSimulationTest
Status: Open
Priority: Normal
Created: 18/Jan/24 22:23

Flaky on trunk as seen in this repeated run:
[https://app.circleci.com/pipelines/github/ekaterinadimitrova2/cassandra/2620/workflows/7c4dc2de-730c-4d7b-aa10-4462c7d36ec7/jobs/53268/tests]

Some of the failures (there are 37 and I did not check all of them, but this can be reproduced with the below command, in case the logs expired before someone takes a look into this ticket:
{code:java}
.circleci/generate.sh -p  \    
 -e REPEATED_SIMULATOR_DTESTS=org.apache.cassandra.simulator.test.ShortPaxosSimulationTest
{code}
):
{code:java}
org.apache.cassandra.simulator.SimulationException: Failed on seed 0x62061d2b91315cfc
	Suppressed: java.lang.RuntimeException: java.util.concurrent.TimeoutException
		at org.apache.cassandra.utils.Throwables.maybeFail(Throwables.java:79)
		at org.apache.cassandra.utils.FBUtilities.waitOnFutures(FBUtilities.java:547)
		at org.apache.cassandra.distributed.impl.AbstractCluster.close(AbstractCluster.java:1098)
		at org.apache.cassandra.simulator.ClusterSimulation.close(ClusterSimulation.java:854)
		at org.apache.cassandra.simulator.SimulationRunner$Run.run(SimulationRunner.java:361)
		at org.apache.cassandra.simulator.SimulationRunner$BasicCommand.run(SimulationRunner.java:346)
		at org.apache.cassandra.simulator.paxos.PaxosSimulationRunner$Run.run(PaxosSimulationRunner.java:34)
		at org.apache.cassandra.simulator.paxos.PaxosSimulationRunner.main(PaxosSimulationRunner.java:148)
		at org.apache.cassandra.simulator.test.ShortPaxosSimulationTest.simulationTest(ShortPaxosSimulationTest.java:101)
		at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
		at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
		at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	Caused by: java.util.concurrent.TimeoutException
		at org.apache.cassandra.utils.concurrent.AbstractFuture.get(AbstractFuture.java:253)
		at org.apache.cassandra.utils.FBUtilities.waitOnFutures(FBUtilities.java:539)
		Suppressed: java.util.concurrent.TimeoutException
		Suppressed: java.util.concurrent.TimeoutException
		Suppressed: java.util.concurrent.TimeoutException
		Suppressed: java.util.concurrent.TimeoutException
Caused by: java.lang.IllegalStateException: Can not commit transformation: "INVALID"(Rejecting this plan as the node NodeId{id=1} is still a part of CMS.).
	at org.apache.cassandra.tcm.ClusterMetadataService.lambda$commit$6(ClusterMetadataService.java:470)
	at org.apache.cassandra.tcm.ClusterMetadataService.commit(ClusterMetadataService.java:514)
	at org.apache.cassandra.tcm.ClusterMetadataService.commit(ClusterMetadataService.java:467)
	at org.apache.cassandra.simulator.cluster.OnClusterLeave$SubmitPrepareLeave.lambda$new$3ac88fac$1(OnClusterLeave.java:86)
	at org.apache.cassandra.distributed.api.IInvokableInstance.unsafeRunOnThisThread(IInvokableInstance.java:85)
	at org.apache.cassandra.simulator.systems.SimulatedActionTask.lambda$asSafeRunnable$0(SimulatedActionTask.java:83)
	at org.apache.cassandra.simulator.systems.SimulatedActionTask$1.run(SimulatedActionTask.java:93)
	at org.apache.cassandra.simulator.systems.InterceptingExecutor$InterceptingPooledExecutor$WaitingThread.lambda$new$1(InterceptingExecutor.java:318)
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	at java.base/java.lang.Thread.run(Thread.java:829)
{code}
{code:java}
org.apache.cassandra.simulator.SimulationException: Failed on seed 0xabf4ff09282d1aaa
Caused by: java.lang.AssertionError: (from replicasForPrimaryKey) [4, 5, 6] != [3, 4, 5] (predicted)
	at org.apache.cassandra.simulator.cluster.ClusterActions.validateReplicasForKeys(ClusterActions.java:274)
	at org.apache.cassandra.simulator.cluster.OnClusterChangeTopology.lambda$before$0(OnClusterChangeTopology.java:62)
	at org.apache.cassandra.simulator.systems.NonInterceptible.execute(NonInterceptible.java:58)
	at org.apache.cassandra.simulator.cluster.OnClusterChangeTopology.before(OnClusterChangeTopology.java:61)
	at org.apache.cassandra.simulator.cluster.OnClusterJoin.performSimple(OnClusterJoin.java:53)
	at org.apache.cassandra.simulator.Action.performAndRegister(Action.java:468)
	at org.apache.cassandra.simulator.Action.perform(Action.java:486)
	at org.apache.cassandra.simulator.ActionSchedule.next(ActionSchedule.java:379)
	at org.apache.cassandra.simulator.paxos.PaxosSimulation$2.next(PaxosSimulation.java:217)
	at org.apache.cassandra.simulator.paxos.PaxosSimulation.run(PaxosSimulation.java:189)
	at org.apache.cassandra.simulator.paxos.PairOfSequencesPaxosSimulation.run(PairOfSequencesPaxosSimulation.java:351)
	at org.apache.cassandra.simulator.SimulationRunner$Run.run(SimulationRunner.java:365)
	at org.apache.cassandra.simulator.SimulationRunner$BasicCommand.run(SimulationRunner.java:346)
	at org.apache.cassandra.simulator.paxos.PaxosSimulationRunner$Run.run(PaxosSimulationRunner.java:34)
	at org.apache.cassandra.simulator.paxos.PaxosSimulationRunner.main(PaxosSimulationRunner.java:148)
	at org.apache.cassandra.simulator.test.ShortPaxosSimulationTest.simulationTest(ShortPaxosSimulationTest.java:101)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
{code}
{code:java}
org.apache.cassandra.simulator.SimulationException: Failed on seed 0x777b0b65b7b3cf0e
	Suppressed: java.lang.RuntimeException: java.util.concurrent.TimeoutException
		at org.apache.cassandra.utils.Throwables.maybeFail(Throwables.java:79)
		at org.apache.cassandra.utils.FBUtilities.waitOnFutures(FBUtilities.java:547)
		at org.apache.cassandra.distributed.impl.AbstractCluster.close(AbstractCluster.java:1098)
		at org.apache.cassandra.simulator.ClusterSimulation.close(ClusterSimulation.java:854)
		at org.apache.cassandra.simulator.SimulationRunner$Run.run(SimulationRunner.java:361)
		at org.apache.cassandra.simulator.SimulationRunner$BasicCommand.run(SimulationRunner.java:346)
		at org.apache.cassandra.simulator.paxos.PaxosSimulationRunner$Run.run(PaxosSimulationRunner.java:34)
		at org.apache.cassandra.simulator.paxos.PaxosSimulationRunner.main(PaxosSimulationRunner.java:148)
		at org.apache.cassandra.simulator.test.ShortPaxosSimulationTest.simulationTest(ShortPaxosSimulationTest.java:101)
		at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
		at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
		at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	Caused by: java.util.concurrent.TimeoutException
		at org.apache.cassandra.utils.concurrent.AbstractFuture.get(AbstractFuture.java:253)
		at org.apache.cassandra.utils.FBUtilities.waitOnFutures(FBUtilities.java:539)
Caused by: org.apache.cassandra.simulator.paxos.HistoryViolation: history violation on -1209807090: [6, 14, 29, 42, 56, 63, 73, 75, 85, 94, 118, 165, 171, 190, 196, 200, 210, 212, 236, 241, 251, 279, 280, 301, 328, 341, 379, 367, 401, 426, 437, 441, 442, 447, 457, 466] != [171, 190, 196, 200, 210, 212, 236, 241, 251, 279, 280, 301, 328, 341, 379, 367, 401, 426, 437, 441, 442, 447, 457, 466]
	at org.apache.cassandra.simulator.paxos.HistoryChecker.fail(HistoryChecker.java:329)
	at org.apache.cassandra.simulator.paxos.PairOfSequencesPaxosSimulation$VerifyingOperation.verify(PairOfSequencesPaxosSimulation.java:113)
	at org.apache.cassandra.simulator.paxos.PaxosSimulation$Operation.accept(PaxosSimulation.java:100)
	at org.apache.cassandra.simulator.paxos.PaxosSimulation$Operation.accept(PaxosSimulation.java:65)
	at org.apache.cassandra.simulator.systems.SimulatedActionCallable$1.run(SimulatedActionCallable.java:47)
	at org.apache.cassandra.simulator.systems.InterceptingExecutor$InterceptingPooledExecutor$WaitingThread.lambda$new$1(InterceptingExecutor.java:318)
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	at java.base/java.lang.Thread.run(Thread.java:829)
{code}