Buggy code:
private static final Integer RUN_LENGTH = Integer . parseInt ( System . getProperty ( "solr.tests.cloud.cm.runlength" , "-1" ) ) ; @ BeforeClass public static void beforeSuperClass ( ) { } @ AfterClass public static void afterSuperClass ( ) { } public static String [ ] fieldNames = new String [ ] { "f_i" , "f_f" , "f_d" , "f_l" , "f_dt" } ; public static RandVal [ ] randVals = new RandVal [ ] { rint , rfloat , rdouble , rlong , rdate } ; protected String [ ] getFieldNames ( ) { } protected RandVal [ ] getRandValues ( ) { } @ Before @ Override public void setUp ( ) throws Exception { } @ Override @ After public void tearDown ( ) throws Exception { } public ChaosMonkeyNothingIsSafeTest ( ) { } @ Override public void doTest ( ) throws Exception { boolean testsSuccesful = false ; try { handle . clear ( ) ; handle . put ( "QTime" , SKIPVAL ) ; handle . put ( "timestamp" , SKIPVAL ) ; ZkStateReader zkStateReader = cloudClient . getZkStateReader ( ) ; for ( int j = 1 ; j < ( sliceCount ) ; j ++ ) { zkStateReader . getLeaderRetry ( DEFAULT_COLLECTION , ( "shard" + j ) , 10000 ) ; } waitForRecoveriesToFinish ( false ) ; del ( "*:*" ) ; List < StopableThread > threads = new ArrayList < StopableThread > ( ) ; int threadCount = 1 ; int i = 0 ; for ( i = 0 ; i < threadCount ; i ++ ) { StopableIndexingThread indexThread = new StopableIndexingThread ( Integer . toString ( i ) , true ) ; threads . add ( indexThread ) ; indexThread . start ( ) ; } threadCount = 1 ; i = 0 ; for ( i = 0 ; i < threadCount ; i ++ ) { StopableSearchThread searchThread = new StopableSearchThread ( ) ; threads . add ( searchThread ) ; searchThread . start ( ) ; } boolean runFullThrottle = random ( ) . nextBoolean ( ) ; if ( runFullThrottle ) { ChaosMonkeyNothingIsSafeTest . FullThrottleStopableIndexingThread ftIndexThread = new ChaosMonkeyNothingIsSafeTest . FullThrottleStopableIndexingThread ( clients , "ft1" , true ) ; threads . add ( ftIndexThread ) ; ftIndexThread . start ( ) ; } chaosMonkey . startTheMonkey ( true , 10000 ) ; try { long runLength ; if ( ( ChaosMonkeyNothingIsSafeTest . RUN_LENGTH ) != ( - 1 ) ) { runLength = ChaosMonkeyNothingIsSafeTest . RUN_LENGTH ; } else { int [ ] runTimes = new int [ ] { 5000 , 6000 , 10000 , 15000 , 25000 , 30000 , 30000 , 45000 , 90000 , 120000 } ; runLength = runTimes [ random ( ) . nextInt ( ( ( runTimes . length ) - 1 ) ) ] ; } Thread . sleep ( runLength ) ; } finally { chaosMonkey . stopTheMonkey ( ) ; } for ( StopableThread indexThread : threads ) { indexThread . safeStop ( ) ; } for ( StopableThread indexThread : threads ) { indexThread . join ( ) ; } Thread . sleep ( 2000 ) ; waitForThingsToLevelOut ( Integer . MAX_VALUE ) ; for ( int j = 1 ; j < ( sliceCount ) ; j ++ ) { zkStateReader . getLeaderRetry ( DEFAULT_COLLECTION , ( "shard" + j ) , 30000 ) ; } commit ( ) ; zkStateReader . updateClusterState ( true ) ; assertTrue ( ( ( zkStateReader . getClusterState ( ) . getLiveNodes ( ) . size ( ) ) > 0 ) ) ; for ( StopableThread indexThread : threads ) { if ( ( indexThread instanceof StopableIndexingThread ) && ( ! ( indexThread instanceof ChaosMonkeyNothingIsSafeTest . FullThrottleStopableIndexingThread ) ) ) { <START_BUG> assertFalse ( "There<seq2seq4repair_space>were<seq2seq4repair_space>too<seq2seq4repair_space>many<seq2seq4repair_space>update<seq2seq4repair_space>fails<seq2seq4repair_space>-<seq2seq4repair_space>we<seq2seq4repair_space>expect<seq2seq4repair_space>it<seq2seq4repair_space>can<seq2seq4repair_space>happen,<seq2seq4repair_space>but<seq2seq4repair_space>shouldn't<seq2seq4repair_space>easily" , ( ( ( ( StopableIndexingThread ) ( indexThread ) ) . getFails ( ) ) > 1 ) ) ; <END_BUG> } } checkShardConsistency ( ( ! runFullThrottle ) , true ) ; long ctrlDocs = controlClient . query ( new SolrQuery ( "*:*" ) ) . getResults ( ) . getNumFound ( ) ; long cloudClientDocs = cloudClient . query ( new SolrQuery ( "*:*" ) ) . getResults ( ) . getNumFound ( ) ; assertTrue ( ( ( "Found<seq2seq4repair_space>" + ctrlDocs ) + "<seq2seq4repair_space>control<seq2seq4repair_space>docs" ) , ( cloudClientDocs > 0 ) ) ; if ( VERBOSE ) System . out . println ( ( ( "control<seq2seq4repair_space>docs:" + ( controlClient . query ( new SolrQuery ( "*:*" ) ) . getResults ( ) . getNumFound ( ) ) ) + "\n\n" ) ) ; if ( random ( ) . nextBoolean ( ) ) { zkServer . shutdown ( ) ; zkServer = new ZkTestServer ( zkServer . getZkDir ( ) , zkServer . getPort ( ) ) ; zkServer . run ( ) ; } CloudSolrServer client = createCloudClient ( "collection1" ) ; try { createCollection ( null , "testcollection" , 1 , 1 , 1 , client , null , "conf1" ) ; } finally { client . shutdown ( ) ; } List < Integer > numShardsNumReplicas = new ArrayList < Integer > ( 2 ) ; numShardsNumReplicas . add ( 1 ) ; numShardsNumReplicas . add ( 1 ) ; checkForCollection ( "testcollection" , numShardsNumReplicas , null ) ; testsSuccesful = true ; } finally { if ( ! testsSuccesful ) { printLayout ( ) ; } } } class FullThrottleStopableIndexingThread extends StopableIndexingThread { private HttpClient httpClient = HttpClientUtil . createClient ( null ) ; private volatile boolean stop = false ; int clientIndex = 0 ; private ConcurrentUpdateSolrServer suss ; private List < SolrServer > clients ; public FullThrottleStopableIndexingThread ( List < SolrServer > clients , String id , boolean doDeletes ) { } @ Override public void run ( ) { } private void changeUrlOnError ( Exception e ) { } @ Override public void safeStop ( ) { }

Fix:
assertFalse ( "There<seq2seq4repair_space>were<seq2seq4repair_space>too<seq2seq4repair_space>many<seq2seq4repair_space>update<seq2seq4repair_space>fails<seq2seq4repair_space>-<seq2seq4repair_space>we<seq2seq4repair_space>expect<seq2seq4repair_space>it<seq2seq4repair_space>can<seq2seq4repair_space>happen,<seq2seq4repair_space>but<seq2seq4repair_space>shouldn't<seq2seq4repair_space>easily" , ( ( ( ( StopableIndexingThread ) ( indexThread ) ) . getFails ( ) ) > 10 ) ) ;