Buggy code:
@ Slow @ ThreadLeakLingering ( linger = 60000 ) public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase { public static Logger log = LoggerFactory . getLogger ( ChaosMonkeyNothingIsSafeTest . class ) ; private static final Integer RUN_LENGTH = Integer . parseInt ( System . getProperty ( "solr.tests.cloud.cm.runlength" , "-1" ) ) ; @ BeforeClass public static void beforeSuperClass ( ) { } @ AfterClass public static void afterSuperClass ( ) { } @ Before @ Override public void setUp ( ) throws Exception { } @ Override @ After public void tearDown ( ) throws Exception { } public ChaosMonkeyNothingIsSafeTest ( ) { } @ Override public void doTest ( ) throws Exception { boolean testsSuccesful = false ; try { handle . clear ( ) ; handle . put ( "QTime" , SKIPVAL ) ; handle . put ( "timestamp" , SKIPVAL ) ; ZkStateReader zkStateReader = cloudClient . getZkStateReader ( ) ; for ( int j = 1 ; j < ( sliceCount ) ; j ++ ) { zkStateReader . getLeaderRetry ( DEFAULT_COLLECTION , ( "shard" + j ) , 10000 ) ; } waitForRecoveriesToFinish ( false ) ; del ( "*:*" ) ; List < StopableThread > threads = new ArrayList < StopableThread > ( ) ; int threadCount = 1 ; int i = 0 ; for ( i = 0 ; i < threadCount ; i ++ ) { StopableIndexingThread indexThread = new StopableIndexingThread ( ( ( i + 1 ) * 50000 ) , true ) ; threads . add ( indexThread ) ; indexThread . start ( ) ; } threadCount = 1 ; i = 0 ; for ( i = 0 ; i < threadCount ; i ++ ) { StopableSearchThread searchThread = new StopableSearchThread ( ) ; threads . add ( searchThread ) ; searchThread . start ( ) ; } boolean runFullThrottle = random ( ) . nextBoolean ( ) ; if ( runFullThrottle ) { ChaosMonkeyNothingIsSafeTest . FullThrottleStopableIndexingThread ftIndexThread = new ChaosMonkeyNothingIsSafeTest . FullThrottleStopableIndexingThread ( clients , ( ( i + 1 ) * 50000 ) , true ) ; threads . add ( ftIndexThread ) ; ftIndexThread . start ( ) ; } chaosMonkey . startTheMonkey ( true , 10000 ) ; long runLength ; if ( ( ChaosMonkeyNothingIsSafeTest . RUN_LENGTH ) != ( - 1 ) ) { runLength = ChaosMonkeyNothingIsSafeTest . RUN_LENGTH ; } else { <START_BUG> int [ ] runTimes = new int [ ] { 5000 , 6000 , 10000 , 15000 , 15000 , 30000 , 30000 , 45000 , 90000 , 120000 } ; <END_BUG> runLength = runTimes [ random ( ) . nextInt ( ( ( runTimes . length ) - 1 ) ) ] ; } try { Thread . sleep ( runLength ) ; } finally { chaosMonkey . stopTheMonkey ( ) ; } for ( StopableThread indexThread : threads ) { indexThread . safeStop ( ) ; } for ( StopableThread indexThread : threads ) { indexThread . join ( ) ; } for ( StopableThread indexThread : threads ) { if ( ( indexThread instanceof StopableIndexingThread ) && ( ! ( indexThread instanceof ChaosMonkeyNothingIsSafeTest . FullThrottleStopableIndexingThread ) ) ) { assertEquals ( 0 , ( ( StopableIndexingThread ) ( indexThread ) ) . getFails ( ) ) ; } } Thread . sleep ( 2000 ) ; waitForThingsToLevelOut ( Integer . MAX_VALUE ) ; for ( int j = 1 ; j < ( sliceCount ) ; j ++ ) { zkStateReader . getLeaderRetry ( DEFAULT_COLLECTION , ( "shard" + j ) , 30000 ) ; } commit ( ) ; zkStateReader . updateClusterState ( true ) ; assertTrue ( ( ( zkStateReader . getClusterState ( ) . getLiveNodes ( ) . size ( ) ) > 0 ) ) ; checkShardConsistency ( ( ! runFullThrottle ) , true ) ; long ctrlDocs = controlClient . query ( new SolrQuery ( "*:*" ) ) . getResults ( ) . getNumFound ( ) ; long cloudClientDocs = cloudClient . query ( new SolrQuery ( "*:*" ) ) . getResults ( ) . getNumFound ( ) ; assertTrue ( ( ( "Found<seq2seq4repair_space>" + ctrlDocs ) + "<seq2seq4repair_space>control<seq2seq4repair_space>docs" ) , ( cloudClientDocs > 0 ) ) ; if ( VERBOSE ) System . out . println ( ( ( "control<seq2seq4repair_space>docs:" + ( controlClient . query ( new SolrQuery ( "*:*" ) ) . getResults ( ) . getNumFound ( ) ) ) + "\n\n" ) ) ; if ( random ( ) . nextBoolean ( ) ) { zkServer . shutdown ( ) ; zkServer = new ZkTestServer ( zkServer . getZkDir ( ) , zkServer . getPort ( ) ) ; zkServer . run ( ) ; } CloudSolrServer client = createCloudClient ( "collection1" ) ; try { createCollection ( null , "testcollection" , 1 , 1 , 1 , client , null , "conf1" ) ; } finally { client . shutdown ( ) ; } List < Integer > numShardsNumReplicas = new ArrayList < Integer > ( 2 ) ; numShardsNumReplicas . add ( 1 ) ; numShardsNumReplicas . add ( 1 ) ; checkForCollection ( "testcollection" , numShardsNumReplicas , null ) ; testsSuccesful = true ; } finally { if ( ! testsSuccesful ) { printLayout ( ) ; } } } class FullThrottleStopableIndexingThread extends StopableIndexingThread { private HttpClient httpClient = HttpClientUtil . createClient ( null ) ; private volatile boolean stop = false ; int clientIndex = 0 ; private ConcurrentUpdateSolrServer suss ; private List < SolrServer > clients ; public FullThrottleStopableIndexingThread ( List < SolrServer > clients , int startI , boolean doDeletes ) { } @ Override public void run ( ) { } private void changeUrlOnError ( Exception e ) { } @ Override public void safeStop ( ) { } @ Override public int getFails ( ) { } } @ Override protected void indexr ( Object ... fields ) throws Exception { } }

Fix:
int [ ] runTimes = new int [ ] { 5000 , 6000 , 10000 , 15000 , 25000 , 30000 , 30000 , 45000 , 90000 , 120000 } ;