public static void main(String args[]) throws IOException {
		int training_size = DataSet.TRAININGSIZE;
		int testing_size = DataSet.TESTINGSIZE;
		DataSet training = new DataSet(
				"data/train.txt",
				true, training_size);
		DataSet testing = new DataSet(
				"data/test.txt",
				false, testing_size);
		String solpath = "data/test_label.txt";
		LogisticRegressionWithHashing lr = new LogisticRegressionWithHashing();
		
		boolean personal = true; // switch this for personalization

		// filter the testing data that has common users in the training set. 
 		Set<Integer> userInTraining = (new BasicAnalysis()).uniqUsers(training);
  	ArrayList<Boolean> includeList = new ArrayList<Boolean>();			
  	while(testing.hasNext()) {
  		int userid = testing.nextInstance().userid;
  		if (userInTraining.contains(userid))
  			includeList.add(true);
  		else
  			includeList.add(false);
  	}
  	testing.reset();
		
		if (!personal) {
  		double step = 0.01;
  		double lambda = 0.001;
  		int[] dims = {97, 12289, 1572869};
  		//int[] dims = {12289};
  		for (int dim : dims) {
  			System.err.println("Running dim = " + dim);
  			ArrayList<Double> avgLoss  = new ArrayList<Double>();
  			Weights weights = lr.train(training, dim, lambda, step, avgLoss, personal);
  			ArrayList<Double> ctr_prediction = lr.predict(weights, testing,
  				personal);
  			double rmse = EvalUtil.eval(solpath, ctr_prediction);
  			double rmseKnownUser = EvalUtil.evalWithIncludingList(solpath, ctr_prediction, includeList);
  		  System.out.println("rmse: " + rmse + "\n");
  		  System.out.println("rmseKnownUser: " + rmseKnownUser + "\n");
  		  
  		  
  		  // save the weights and the prediction
  			String outpathbase = "experiments/lrhashing/";
  			String suffix = "_"+dim;
  			BufferedWriter writer = new BufferedWriter(new FileWriter(outpathbase + "weights" + suffix));
  			writer.write(weights.toString());
  			writer.close();
  			
  			writer = new BufferedWriter(new FileWriter(outpathbase + "ctr" + suffix));
  			writer.write("rmse: " + rmse + "\n");
  			for (double ctr : ctr_prediction)
  				writer.write(ctr + "\n");
  			writer.close();
  			
  			writer = new BufferedWriter(new FileWriter(outpathbase + "loss" + suffix));
  			for (double loss : avgLoss)
  				writer.write(loss + "\n");
  			writer.close();
    	}
  	} else {  		
  	  		double step = 0.01;
    		double lambda = 0.001;
    		int[] dims = {12289};
    		for (int dim : dims) {
    			System.err.println("Running dim = " + dim);
    			ArrayList<Double> avgLoss  = new ArrayList<Double>();
    			Weights weights = lr.train(training, dim, lambda, step, avgLoss, personal);
    			ArrayList<Double> ctr_prediction = lr.predict(weights, testing,
    				personal);
    			double rmse = EvalUtil.evalWithIncludingList(solpath, ctr_prediction, includeList);
    		  System.out.println("rmse: " + rmse + "\n");
    		  
    		  // save the weights and the prediction
    			String outpathbase = "experiments/lrpersonal/";
    			String suffix = "_"+dim;
    			BufferedWriter writer = new BufferedWriter(new FileWriter(outpathbase + "weights" + suffix));
    			writer.write(weights.toString());
    			writer.close();
    			
    			writer = new BufferedWriter(new FileWriter(outpathbase + "ctr" + suffix));
    			writer.write("rmse: " + rmse + "\n");
    			for (double ctr : ctr_prediction)
    				writer.write(ctr + "\n");
    			writer.close();
    			
    			writer = new BufferedWriter(new FileWriter(outpathbase + "loss" + suffix));
    			for (double loss : avgLoss)
    				writer.write(loss + "\n");
    			writer.close();  		
  	}
  	}
	}