public void filterDocument(Dictionary dictionary, int docId, Map<Integer, String> docTable){
       String line = "";
       String []splitString;
       String totalString = "";
       StringBuilder sb = new StringBuilder();
       while( true){
           line = fileProcessor.readLineFromFile();
           if( line == null)
               break;
//           sb.append(line);
           splitString = line.split(" ");
           for( String temp : splitString){
               tokens.add(temp);
           }
       }

       //System.out.println("Title is : " + getTitle(tokens));
       docTable.put( docId, getTitle(tokens).replace(',',' '));
       removeTags( tokens);
       totalString = tokens.toString();
//       System.out.println( sb.toString().replaceAll(" ,","").replaceAll("[a-zA-Z]*[-]+[^a-z]*",""));

       totalString = totalString.replaceAll(" ,","");
       totalString = totalString.replaceAll("\\<.*?>","");
       totalString = totalString.replaceAll(",","");
       totalString = totalString.replaceAll("\\[","");
       totalString = totalString.replaceAll("\\]","");
       totalString = totalString.replaceAll("\"","");
       totalString = totalString.replaceAll("\\*","");
       totalString = totalString.replaceAll(":"," ");
       totalString = totalString.replaceAll("\\("," ");
       totalString = totalString.replaceAll("\\)"," ");
       totalString = totalString.replaceAll("!"," ");
       totalString = totalString.replaceAll("/"," ");
       totalString = totalString.replaceAll("\\?","");
       totalString = totalString.replaceAll("\\$[^0-9]"," ");
       //totalString = totalString.replaceAll("\\.[^0-9^a-z]*","");
       totalString = totalString.replace("\'","");
       totalString = totalString.replace("\"","");
       totalString = totalString.replace("\\{","");
       totalString = totalString.replace("\\}","");
       totalString = totalString.replace("...","");
       totalString = totalString.replace("\\`","");
       totalString = totalString.replace("&","");
       totalString = totalString.replace(";","");
       //totalString = totalString.replace("[^0-9][*a-z]@[^0-9][*a-z]","");



       StringBuilder stringBuilder = new StringBuilder( totalString);
       //stringBuilder = checkFirstDots(stringBuilder);
       for( int i =0; i< stringBuilder.length(); i++){
           char tempChar = stringBuilder.charAt(i);
           if( i+1 == stringBuilder.length()) {
               if( tempChar == '-' && tempChar == '-')
                   stringBuilder.setCharAt(i,' ');
               break;
           }
           char nextChar = stringBuilder.charAt(i + 1);
           if( tempChar =='-' && !Character.isDigit(nextChar) ){
               stringBuilder.setCharAt(i,' ');
           }
       }

       removeDots( stringBuilder, dictionary, docId);
   }