static void dedupWithDownsample() throws IOException{

		dup=0;
		int memoryPolicy = 5;
		resultRecord.println("Total " + storageLoad() + " entries added into the index");
		File[] incomingFile = null;
		incomingFile = new File(outputpath).listFiles();
		int cacheSizeControl = 0;
//		int sampleNum = (int) ((Math.log(2)+Math.log(10000))/2/0.0001/Math.pow(expectRaio,2));
		
		resultRecord.println("Pick out total " + sampleNumPerSeg + " samples per segment for estimation");
		
		System.out.println("Pick out total " + sampleNumPerSeg + " samples per segment for estimation");
//		int sampleNumPerSeg = sampleNum/incomingFile.length; // samples per segment
		
//		for(int i = 1; i <=incomingFile.length; i++){
		
		for(int segmentNo=1;segmentNo<=incomingFile.length;segmentNo++){
//			while((total-dup)>= downSampleTrig*storageLimit){
			while(index.size()>=downSampleTrig*indexEntryNum){
			   sampleRate = migratePara*sampleRate; //this is downsampleing
//				   storageLimit = storageLimit*2;
				
				resultRecord.println("The current index size is: " + index.size()+
						  "\nIt's approaching " + downSampleTrig + "of supported index capacity: "+ indexEntryNum);
				resultRecord.println("Index is gonna be full, downsampling...");
				resultRecord.println("Index size before downsampling: "+ index.size());
				indexShrink();
				resultRecord.println("Index size after downsampling: "+ index.size());
				resultRecord.println("\nDownsampling ... The new sampling rate is: " + (double)1/sampleRate);
			}
//			}
//			consecutiveDownSample(segmentNo,sampleRate,cacheSizeControl,memoryPolicy,sampleNumPerSeg);
			sampleTotal+=sampleNumPerSeg;
			File file = null;
			file = new File(outputpath+"/Segment_"+  segmentNo);
			hashRecord = new String[segsize*1024/chunksize];
			uniqueRecord = new int[segsize*1024/chunksize];
			sampledRecord = new int[segsize*1024/chunksize];
			containerTrack = new long[segsize*1024/chunksize];
		cacheSizeControl++;
		if(cacheSizeControl==memoryPolicy ){  //cache size control
		containerRecord.clear();
		cache.clear();
		cacheSizeControl=0;
		}
		
		dedupProcess_v2 dP = new dedupProcess_v2(index,indexSize, M,cache,chunksize, segsize, hashRecord,
				 uniqueRecord,sampledRecord,containerTrack,storageFileFolder,bf, bloomfilterRecord, 
				 file,containerRecord,estimateBase,sampleNumPerSeg,segAmount,consecutiveDedup,sampleRate);		
		dP.dedup();
		total += dP.getTotal();
		dup += dP.getDup();
		if(consecutiveDedup=true && segmentNo%segAmount == 0){	
			nodeQue.add(new Node(total,dup,(double)1/sampleRate,index.size(),(double)dup/total,(double)dup/(index.size()),sampleTotal));
			resultRecord.println();
			resultRecord.println("The culmulative segments are: " + segmentNo +
								"\nCurrent data amount is: "+total+
								"\nThe duplicates are: "+dup+
								"\nThe current index size is: " + index.size()+
								"\nThe deduplication rate is : " + (double)dup/total*100 +"%"+
								"\nThe estimated dedup rate is: " + getEstimateRatio()*100 +" %");
			resultRecord.println();
//			resultRecord.println("####The dedup efficiency is: " + (double)dup/(indexEntryNum-index.size())+" dup/wasted entry");	
			resultRecord.println("####The dedup efficiency is: " + (double)dup/(index.size())+" dup/entry slot");	
		}
		
		totalChunkNum += dP.getTotalChunkNum();
		dP.storageWrite(); 	

//		System.out.println("The current cache size is: " + dP.getCacheSize());
		}
			
		resultRecord.println("The current index size is: " + index.size());
		
		System.out.println("The current index size is: " + index.size());
//		System.out.println("The current BloomFilter size is: " + bf.size());		
		

		resultRecord.println("The total data is: "+total+"\nThe duplicates are: "+dup+
					 "\nThe deduplication rate is : " + (double)dup/total*100 +"%");
			
		System.out.println("The total data is: "+total+"\nThe duplicates are: "+dup+
				"\nThe sampling rate is: " + sampleRate + "\nThe deduplication rate is : " + (double)dup/total*100 +"%");
		measuredRatio = (double)dup/total;

			estiRatio = getEstimateRatio();
			
			resultRecord.println("The estimated dedup rate is: " + getEstimateRatio()*100 +" %");
			
			System.out.println("The estimated dedup rate is: " + getEstimateRatio()*100 +" %");
//			resultRecord.println("####The dedup efficiency is: " + (double)dup/(indexEntryNum-index.size())+" dup/wasted entry");	
			resultRecord.println("####The dedup efficiency is: " + (double)dup/(index.size())+" dup/entry slot");	
			nodeQue.add(new Node(total,dup,(double)1/sampleRate,index.size(),(double)dup/total,(double)dup/(index.size()),sampleTotal));
			/*record # of entries with different hitting number */
			matlabStatistic();
			indexStatistic();
			
		
		
	}