Bug: org.apache.cassandra.distributed.test.ReadRepairTest#movingTokenReadRepairTest passes because the message drop filters are not dropping the messages
Status: Resolved
Priority: Normal
Resolution: Fixed
Created: 14/Aug/20 00:23
Resolved: 02/Oct/20 08:36

The test tries to block messages from node 4 to 3 but used verb ordinal rather than id; this causes the verbs to sent to node 3 and allow all replies to happen.

If you fix the test to use id (and actually filter), then the test fails

Patch to show the issue

{code}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java
index f0c82b85e1..55d2e8f303 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java
@@ -30,14 +30,14 @@ import org.apache.cassandra.dht.Token;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.ICluster;
-import org.apache.cassandra.distributed.shared.NetworkTopology;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.service.PendingRangeCalculatorService;
import org.apache.cassandra.service.StorageService;

+import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
+import static org.apache.cassandra.distributed.shared.AssertUtils.row;
import static org.apache.cassandra.net.Verb.READ_REPAIR_REQ;
import static org.apache.cassandra.net.Verb.READ_REQ;
-import static org.apache.cassandra.distributed.shared.AssertUtils.*;

public class ReadRepairTest extends TestBaseImpl
{
@@ -115,8 +115,8 @@ public class ReadRepairTest extends TestBaseImpl

            // prevent #4 from reading or writing to #3, so our QUORUM must contain #2 and #4
            // since #1 is taking over the range, this means any read-repair must make it to #1 as well
-            cluster.filters().verbs(READ_REQ.ordinal()).from(4).to(3).drop();
-            cluster.filters().verbs(READ_REPAIR_REQ.ordinal()).from(4).to(3).drop();
+            cluster.filters().verbs(READ_REQ.id).from(4).to(3).drop();
+            cluster.filters().verbs(READ_REPAIR_REQ.id).from(4).to(3).drop();
            assertRows(cluster.coordinator(4).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = ?",
                                                      ConsistencyLevel.ALL, i),
                       row(i, 1, 1));
{code}

Exception

{code}
org.apache.cassandra.exceptions.ReadTimeoutException: Operation timed out - received only 2 responses.

	at org.apache.cassandra.service.reads.ReadCallback.awaitResults(ReadCallback.java:120)
	at org.apache.cassandra.service.reads.AbstractReadExecutor.awaitResponses(AbstractReadExecutor.java:373)
	at org.apache.cassandra.service.StorageProxy.fetchRows(StorageProxy.java:1823)
	at org.apache.cassandra.service.StorageProxy.readRegular(StorageProxy.java:1713)
	at org.apache.cassandra.service.StorageProxy.read(StorageProxy.java:1630)
	at org.apache.cassandra.db.SinglePartitionReadCommand$Group.execute(SinglePartitionReadCommand.java:1123)
	at org.apache.cassandra.cql3.statements.SelectStatement.execute(SelectStatement.java:294)
	at org.apache.cassandra.cql3.statements.SelectStatement.execute(SelectStatement.java:246)
	at org.apache.cassandra.cql3.statements.SelectStatement.execute(SelectStatement.java:88)
	at org.apache.cassandra.distributed.impl.Coordinator.executeInternal(Coordinator.java:100)
	at org.apache.cassandra.distributed.impl.Coordinator.lambda$executeWithResult$0(Coordinator.java:62)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	at java.lang.Thread.run(Thread.java:748)
{code}