public void createIndex() throws IOException {

        validate();

        String tweet = null;

        TweetProcessor tweetProcessor = new TweetProcessor(indexFolder, dumpFile);

        BufferedReader dumpReader = new BufferedReader(new FileReader(dumpFile));

        while((tweet = dumpReader.readLine()) != null) {

            String processedTweet = tweet.trim();
            if( processedTweet.length() > 0 )
                tweetProcessor.process(processedTweet);

        }

        dumpReader.close();
        tweetProcessor.finish();

    }