public void crawlRobotFiles() {
		try {
			GutenbergCrawlerHelper.init();
		} catch (IOException e) {
			e.printStackTrace();
		}
		File robotFileDir = new File(GUTENBERG_ROBOT_FILE_DIR);
		File[] fileNames = robotFileDir.listFiles();
		List<File> files = Collections.synchronizedList(new ArrayList<File>());

		for (int i = 0; i < fileNames.length; i++) {
			if ((i + 1) % GUTENBERG_ROBOT_DISPATCH_THRESHOLD == 0) {
				// dispatch file tasks
				synchronized (files) {
					dispatchDownloadTask(Collections
							.synchronizedList(new ArrayList<File>(files)));
				}
				files.clear();
			} else {
				files.add(fileNames[i]);
			}
		}
		if (files.size() != 0) {
			synchronized (files) {
				dispatchDownloadTask(files);
			}
		}
	}