log大文件中高效率检索固定字符串之1

1、使用方式之一、先把文件读到内存中，使用java中的正则表达式去匹配，使用的是Matcher Pattern等类 
2、使用方式之二、先把文件读到内存中，在内存中检索文件所有换行符，并根据换行符和字substring（）截取每一行文字进行判断。不适用正则表达式，使用的是indexOf()、substring()两个函数。

【总结】 
    方式1 逻辑简单，只需要准备好对应的正则表达式就可以.但执行效率非常慢，非常耗时。1380行数据，查找其中3行数据，需要6s左右
    方式2 实现逻辑需要自己写实现代码（indexOf()\substring()），具体代码如下 getMatchrfLines（）方法


```java 
package com.log;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.sql.main.CommonUtil;

public class LoggerFileDiv {
	private static final String LOG_FILE_PATH ="C:\\Users\\cloud-user\\work\\テスト\\ITテスト2\\xml\\";
	private static String applogcnts="";
	private static String errlogcnts="";
	private static int[] nCountArry;
	private static int[] nCountArryErr;
	
	static{
		initFileCnts("MRO030");
		
		int r =0;
		nCountArry=new int[12000];//\n index array
		int count=0;
		while((r=applogcnts.indexOf("\n",r+50)) > 0){
			nCountArry[count]=r;
			count++;
		}
		//err log
		r =0;
		count=0;
		nCountArryErr=new int[12000];//\n index array
		while((r=errlogcnts.indexOf("\n",r+20)) > 0){
			nCountArryErr[count]=r;
			count++;
		}
	}
	
	
	public static void main(String[] args) {
//		HashMap data =new HashMap();
//		data.put("null", null);
//		data.put("null2", null);
//		data.put(null, "null2222");
//		System.out.println(data.get(null));;
//		
//		System.out.println(data);
		
//		String pattern =",20180220113652,A004002990501,029905010001167,";
				//,20180220132637,A004002990501,029905010001340,
//		divFileByTranatmseq("MRO020");
		divFileByTranatmseq("MRO030");
//		copyRequestXml("MRO020");
//		copyRequestXml("MRO030");
	}
	public static void copyRequestXml(String progrmid){
		File dirP =new File(LOG_FILE_PATH +progrmid +"\\output\\day_test\\");
		String inputXmlPath =LOG_FILE_PATH +"\\input\\";
		
		List<String> lst =new ArrayList<String>();
		lst.add("123458");
		lst.add("acbsdsd");
		lst.add("212gffff");
//		lst.forEach((n) -> {System.out.println(n);});
		//lambda 公式
		dirP.listFiles( 
				( dir , name) ->{  
					System.out.println(name);
					return true;
				}
				
				);
		
		
//		dirP.listFiles( (File dir, String name) -> {
//			if(!CommonUtil.isEmpty(dir) && name.endsWith("-request.xml")){
//				System.out.println("logggg:"+name);
//				return true;
//			}else {
//				return false;
//			}
//		}) ;
		
		
//		for(File caseDir : dir.listFiles(new MROPrefixFilter())){
//			for(File xml : caseDir.listFiles(new RequestXmlFilenameFilter())){
//				xml.renameTo(new File(inputXmlPath +xml.getName()));
//			}
//		}
		
	}
	
	
	public static String getTranatmseqFromXml(String progrmid,String xmlName){
		String[] tags =new String[]{"<transaction_date>(.*)</transaction_date>",
					"<terminal_no>(.*)</terminal_no>",
				"<terminal_serial_no>(.*)</terminal_serial_no>"};
		
		String xmlPath =LOG_FILE_PATH+progrmid+"\\output\\"+xmlName;
		File dir =new File(xmlPath);
	    BufferedReader bfr = null;	
	    StringBuffer xmlStr=new StringBuffer();
	    StringBuffer sb =new StringBuffer();
		try {
			bfr=new BufferedReader(new InputStreamReader(new FileInputStream(dir), "Shift_JIS"));
			while(bfr.ready()){
				xmlStr.append(bfr.readLine()+"\r\n");
			}
			for(int i=0;i<3;i++){
				Pattern p =Pattern.compile(tags[i]);
				Matcher m =p.matcher(xmlStr);
				if(m.find()){
					sb.append(","+m.group(1));
				}
			}
		} catch (Exception e) {
			e.printStackTrace();
		}finally {
			try {
				bfr.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		return sb.toString();
		
	}
	
	public static void divFileByTranatmseq(String progrmid){
		File dir =new File(LOG_FILE_PATH+progrmid+"\\output\\");
		int i=0;
		if(dir.isDirectory()){
			File[] caseDirs =dir.listFiles(new MROPrefixFilter());
			for (File caseDir : caseDirs){
				i++;
				String caseNo =caseDir.getName();
				String responseXml =caseNo+"\\"+caseNo+"-request.xml";
				String patternStr =getTranatmseqFromXml(progrmid,responseXml);
				//getProperLogcnts(progrmid,caseNo,patternStr);
				getMatchrfLines(progrmid,caseNo,patternStr);
//				if(i>=20) break;
			}
		}
		
	}
	
	public static void getProperLogcnts(String progrmid,String caseno,String tranatmseq){
		//,20180220132637,A004002990501,029905010001340,
		String pattern="(.*"+tranatmseq+".*)";
		File dir =new File(LOG_FILE_PATH+progrmid+"\\");
		
		if(dir.isDirectory()){
			File[] loggFs=dir.listFiles((dir2,name) ->{
				if(name.endsWith(".log")) return true;
				else return false;});
			
			//read file line by line
			BufferedReader bfr =null;
			
			for(File f : loggFs){
				String fname =f.getName();
				StringBuffer findedLineCnts =new StringBuffer();
				String preHeadLine ="";
				try {
					 bfr=new BufferedReader(new InputStreamReader(new FileInputStream(f), "UTF-8"));
					 
					 boolean headLineAddedFlg =false;
					 boolean flushFileFlg =false;
					 while(bfr.ready()){

						 String line =bfr.readLine();
						 
						 if(matchLine(line,pattern)){
							 if(!headLineAddedFlg && fname.contains("mr-on-ap")){
								 findedLineCnts.append(preHeadLine+"\r\n");
								 headLineAddedFlg=true;
							 }
							 findedLineCnts.append(line+"\r\n");
							 flushFileFlg=true;
						 }else {
							 if(flushFileFlg){
								 //write file to path
								 String logfilepath =LOG_FILE_PATH+progrmid+"\\output\\"+caseno+"\\"+fname;
								 CommonUtil.writeString(logfilepath, findedLineCnts.toString());
								 System.out.println(caseno+" log writen.");
								 
								 break;
							 }
							 preHeadLine=line;//pre line 
							 
							 headLineAddedFlg =false;
							 flushFileFlg=false;
							 
						}
//						 fileCnts.append();
					 }
					 bfr.close();
					 
				} catch (Exception e) {
					e.printStackTrace();
					break;
				}
			}
		}
	}
	
	public static void initFileCnts(String progrmid){
		File aplogFile =new File(LOG_FILE_PATH+progrmid+"\\mr-on-ap.log");
		File errlogFile =new File(LOG_FILE_PATH+progrmid+"\\mr-on-err.log");
		BufferedReader bfr =null;
		BufferedReader bfr2 =null;
		StringBuffer applogsb =new StringBuffer();
		StringBuffer errlogsb =new StringBuffer();
		 try {
			bfr=new BufferedReader(new InputStreamReader(new FileInputStream(aplogFile), "UTF-8"));
			bfr2=new BufferedReader(new InputStreamReader(new FileInputStream(errlogFile), "UTF-8"));
			while(bfr.ready()){
				applogsb.append(bfr.readLine()+"\r\n"); 
			 }
			while(bfr2.ready()){
				errlogsb.append(bfr2.readLine()+"\r\n"); 
			 }
			applogcnts = applogsb.toString();
			errlogcnts = errlogsb.toString();
			
			System.out.println("applogcnts size:"+applogcnts.length());
			System.out.println("errlogcnts size:"+errlogcnts.length());
			
		} catch (Exception e) {
			e.printStackTrace();
		}finally {
			try {
				bfr.close();
				bfr2.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
			
		}
		 
		
	}
	private static boolean matchLine(String cnts,String pattern ){
		Pattern p =Pattern.compile(pattern);
		Matcher m =p.matcher(cnts);
		return m.find();
	}
	private static void  getMatchrfLines(String progrmid,String caseno,String tranatmseq){
		String test =",20180220113652,A004002990501,029905010001166";
		String pattern="(.*"+test+".*)";
		Pattern p =Pattern.compile(pattern);
		Matcher m =p.matcher(applogcnts);
		
		long starttime =System.currentTimeMillis();
		 
//		while(m.find()){
//			System.out.println(m.group(1));
//		}
//		int r =0;
//		int[] nCountArry=new int[1500];//\n index array
//		int count=0;
//		while((r=applogcnts.indexOf("\n",r+50)) > 0){
////			System.out.println("index:"+r);
//			nCountArry[count]=r;
//			count++;
//		}
		int startpos =0;
		StringBuffer sbCnts = new StringBuffer();
		boolean headAddedFlg =false;
		String prelineStr ="";
		
		//applog file 
		for(int i=0;i<nCountArry.length;i++){
			if(nCountArry[i] <=0) break;
			String lineStr = applogcnts.substring(startpos, nCountArry[i]);
			
			if(lineStr.indexOf(tranatmseq)>0){
				if(!headAddedFlg){
					sbCnts.append(prelineStr+"\n");
					headAddedFlg=true;
				}
				sbCnts.append(lineStr+"\n");
			}
			prelineStr=lineStr;
			startpos = nCountArry[i] +1;
		}
		
		String applogfilepath =LOG_FILE_PATH+progrmid+"\\output\\"+caseno+"\\"+"mr-on-ap.log";
		CommonUtil.writeString(applogfilepath, sbCnts.toString());
		
		startpos =0;
		sbCnts = new StringBuffer();
		prelineStr ="";
		
		//errlog file 
		for(int i=0;i<nCountArryErr.length;i++){
			if(nCountArryErr[i] <=0) break;
			String lineStr = errlogcnts.substring(startpos, nCountArryErr[i]);
			
			if(lineStr.indexOf(tranatmseq)>0){
				sbCnts.append(lineStr+"\n");
			}
			prelineStr=lineStr;
			startpos = nCountArryErr[i] +1;
		}
		String errlogfilepath =LOG_FILE_PATH+progrmid+"\\output\\"+caseno+"\\"+"mr-on-err.log";
		CommonUtil.writeString(errlogfilepath, sbCnts.toString());

		long endtime =System.currentTimeMillis();
		System.out.println(caseno+"execute time :"+(endtime-starttime)+"ms");
	}
	
}
class LogFileFilenameFilter implements FilenameFilter{

	@Override
	public boolean accept(File dir, String name) {
		
		if(!CommonUtil.isEmpty(name) && name.endsWith(".log")){
			System.out.println("logggg:"+name);
			return true;
		}else {
			return false;
		}
	}
	
}
class RequestXmlFilenameFilter implements FilenameFilter{

	@Override
	public boolean accept(File dir, String name) {
		
		if(!CommonUtil.isEmpty(name) && name.endsWith("-request.xml")){
			System.out.println("logggg:"+name);
			return true;
		}else {
			return false;
		}
	}
	
}
class MROPrefixFilter implements FilenameFilter{

	@Override
	public boolean accept(File dir, String name) {
		
		if(!CommonUtil.isEmpty(name) && name.startsWith("MRO")){
//			System.out.println("MRO:"+name);
			return true;
		}else {
			return false;
		}
	}
	
}
```