public static void main(String[] args) throws Exception
	{
		JobConf conf;
		
		if (args.length < 4)
		{
			System.err.println("Invalid number of arguments.");
			System.err.println("Usage: BuildAndComparePages <old pages dir> <new pages dir> <number of chunks> <chunk size>");
			System.out.println("Output is placed on hdfs at /out/newpages/part-00000");
			throw new IllegalArgumentException("Invalid number of arguments");
		}
		

		// Build the bloom filter and serialize it to hdfs for the next step
		if (args.length == 4 || args[4].equals("1") || args[4].equals("3"))
		{
			conf = new JobConf(BuildBloomFilter.class);
			conf.setJobName("BuildBloomFilter");
	
			conf.setOutputKeyClass(Text.class);
			conf.setOutputValueClass(BloomFilterPair.class);
	
			conf.setMapperClass(BuildBloomFilter.Map.class); 
			conf.setReducerClass(BuildBloomFilter.Reduce.class);
	
			conf.setInputFormat(TextInputFormat.class);
			conf.setOutputFormat(TextOutputFormat.class);
			//conf.setNumReduceTasks(1);
			
			
			Path pageDir		= new Path(args[0]);
			//convert to int to get a runtime exception immediately if the value is bad
			Integer numChunks 	= Integer.valueOf(args[2]);
			Integer chunkSize	= Integer.valueOf(args[3]);
			
			//Provide configuration options for determining bloom filter size
			conf.set("old_path", args[0]);
			conf.set("new_path", args[1]);
			conf.set("num_chunks", numChunks.toString());
			conf.set("chunk_size", chunkSize.toString());
			
	
			FileInputFormat.setInputPaths(conf, pageDir);
			FileOutputFormat.setOutputPath(conf, new Path("/dev/null"));
			
			JobClient.runJob(conf);
		}
		
	
		// Now use the bloom filters to determine changes between the shards
		if (args.length == 4 || args[4].equals("2") || args[4].equals("3"))
		{
			conf = new JobConf(CompareInternetPages.class);
			conf.setJobName("CompareInternetPages");
			
			conf.setOutputKeyClass(Text.class);
			conf.setOutputValueClass(Text.class);
	
			conf.setMapperClass(CompareInternetPages.Map.class); 
			conf.setReducerClass(CompareInternetPages.Reduce.class);
			
			conf.setInputFormat(TextInputFormat.class);
			conf.setOutputFormat(TextOutputFormat.class);
			
			Path pageDir		= new Path(args[1]);
			//convert to int to get a runtime exception immediately if the value is bad
			Integer numChunks 	= Integer.valueOf(args[2]);
			Integer chunkSize	= Integer.valueOf(args[3]);
			
			conf.set("old_path", args[0]);
			conf.set("new_path", args[1]);
			conf.set("num_chunks", numChunks.toString());
			conf.set("chunk_size", chunkSize.toString());
			
			FileInputFormat.setInputPaths(conf, pageDir);
			
	
			//FileOutputFormat.setOutputPath(conf, new Path("s3://blakebucket/out/newpages"));
			FileOutputFormat.setOutputPath(conf, new Path("/out/newpages"));
	
			JobClient.runJob(conf);
		}
		
	}