Bug: CorruptSSTableException after a 2.1 SSTable is upgraded to 3.0, failing reads
Status: Resolved
Priority: Normal
Resolution: Fixed
Created: 01/May/20 15:39
Resolved: 04/Jun/20 18:21

Below is the exception with stack trace. This issue is consistently reproduce-able.
{code:java}
ERROR [SharedPool-Worker-1] 2020-05-01 14:57:57,661 AbstractLocalAwareExecutorService.java:169 - Uncaught exception on thread Thread[SharedPool-Worker-1,5,main]ERROR [SharedPool-Worker-1] 2020-05-01 14:57:57,661 AbstractLocalAwareExecutorService.java:169 - Uncaught exception on thread Thread[SharedPool-Worker-1,5,main]org.apache.cassandra.io.sstable.CorruptSSTableException: Corrupted: /mnt/data/cassandra/data/<ks>/<cf-fda511301fb311e7bd79fd24f2fcfb0d/md-10151-big-Data.db at org.apache.cassandra.db.columniterator.AbstractSSTableIterator$Reader.hasNext(AbstractSSTableIterator.java:349) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.columniterator.AbstractSSTableIterator.hasNext(AbstractSSTableIterator.java:220) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.columniterator.SSTableIterator.hasNext(SSTableIterator.java:33) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.rows.LazilyInitializedUnfilteredRowIterator.computeNext(LazilyInitializedUnfilteredRowIterator.java:95) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.rows.LazilyInitializedUnfilteredRowIterator.computeNext(LazilyInitializedUnfilteredRowIterator.java:32) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.transform.BaseRows.hasNext(BaseRows.java:129) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.rows.LazilyInitializedUnfilteredRowIterator.computeNext(LazilyInitializedUnfilteredRowIterator.java:95) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.rows.LazilyInitializedUnfilteredRowIterator.computeNext(LazilyInitializedUnfilteredRowIterator.java:32) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.transform.BaseRows.hasNext(BaseRows.java:129) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.transform.BaseRows.hasNext(BaseRows.java:129) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.rows.UnfilteredRowIteratorSerializer.serialize(UnfilteredRowIteratorSerializer.java:131) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.rows.UnfilteredRowIteratorSerializer.serialize(UnfilteredRowIteratorSerializer.java:87) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.rows.UnfilteredRowIteratorSerializer.serialize(UnfilteredRowIteratorSerializer.java:77) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.partitions.UnfilteredPartitionIterators$Serializer.serialize(UnfilteredPartitionIterators.java:294) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.ReadResponse$LocalDataResponse.build(ReadResponse.java:187) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.ReadResponse$LocalDataResponse.<init>(ReadResponse.java:180) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.ReadResponse$LocalDataResponse.<init>(ReadResponse.java:176) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.ReadResponse.createDataResponse(ReadResponse.java:76) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.ReadCommand.createResponse(ReadCommand.java:341) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.ReadCommandVerbHandler.doVerb(ReadCommandVerbHandler.java:47) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:67) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[na:1.8.0_231] at org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:165) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$LocalSessionFutureTask.run(AbstractLocalAwareExecutorService.java:137) [nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:109) [nf-cassandra-3.0.19.8.jar:3.0.19.8] at java.lang.Thread.run(Thread.java:748) [na:1.8.0_231]Caused by: java.lang.ArrayIndexOutOfBoundsException: 121 at org.apache.cassandra.db.ClusteringPrefix$Deserializer.prepare(ClusteringPrefix.java:425) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.UnfilteredDeserializer$CurrentDeserializer.prepareNext(UnfilteredDeserializer.java:170) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.UnfilteredDeserializer$CurrentDeserializer.hasNext(UnfilteredDeserializer.java:151) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.columniterator.SSTableIterator$ForwardReader.computeNext(SSTableIterator.java:140) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.columniterator.SSTableIterator$ForwardReader.hasNextInternal(SSTableIterator.java:172) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] at org.apache.cassandra.db.columniterator.AbstractSSTableIterator$Reader.hasNext(AbstractSSTableIterator.java:336) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] ... 27 common frames omitted

Caused by: java.lang.ArrayIndexOutOfBoundsException: 121
    at org.apache.cassandra.db.ClusteringPrefix$Deserializer.prepare(ClusteringPrefix.java:425) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
    at org.apache.cassandra.db.UnfilteredDeserializer$CurrentDeserializer.prepareNext(UnfilteredDeserializer.java:170) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
    at org.apache.cassandra.db.UnfilteredDeserializer$CurrentDeserializer.hasNext(UnfilteredDeserializer.java:151) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
    at org.apache.cassandra.db.columniterator.SSTableIterator$ForwardReader.computeNext(SSTableIterator.java:140) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
    at org.apache.cassandra.db.columniterator.SSTableIterator$ForwardReader.hasNextInternal(SSTableIterator.java:172) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8]
    at org.apache.cassandra.db.columniterator.AbstractSSTableIterator$Reader.hasNext(AbstractSSTableIterator.java:336) ~[nf-cassandra-3.0.19.8.jar:3.0.19.8] ... 27 common frames omitted
{code}

Column family definition
{code:java}
CREATE TABLE <keyspace>."<cf>" (
 key text,
 value text,
 PRIMARY KEY (key, value)
 ) WITH COMPACT STORAGE
 AND CLUSTERING ORDER BY (value ASC)
 AND bloom_filter_fp_chance = 0.01
 AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
 AND comment = ''
 AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
 AND compression = {'enabled': 'false'}
 AND crc_check_chance = 1.0
 AND dclocal_read_repair_chance = 0.1
 AND default_time_to_live = 0
 AND gc_grace_seconds = 864000
 AND max_index_interval = 2048
 AND memtable_flush_period_in_ms = 0
 AND min_index_interval = 128
 AND read_repair_chance = 0.0
 AND speculative_retry = '99PERCENTILE';{code}