public static void generateInvertedIndex(boolean includeApproximateMatches, String tokenizationType, File directory) { //NB Change nGramSize to tokenizationType

        dictionaryMap.clear();
        inverseDocFreqMap.clear();

        //read all files in a directory
        //http://stackoverflow.com/questions/4917326/how-to-iterate-over-the-files-of-a-certain-directory-in-java 
        File[] files = directory.listFiles();
        int collectionSize = files.length; //total number of docs in collection

        ReadFile rf = new ReadFile();
        TokenGenerator generator = new TokenGenerator();

        for (int i = 0; i < files.length; i++) {
            if (files[i].isFile()) {
                String docName = files[i].getName();
                try {
                    String docContents = rf.readFile(files[i].toString());
                    StringTokenizer st = new StringTokenizer(docContents);
                    double docLength = st.countTokens();

                    tokens = generator.generateTokens(docContents, tokenizationType);
                    calculateTFandIDF(docContents, docName, collectionSize, docLength);

                } catch (IOException ioe) {
                    System.out.println(ioe);
                }
            }
        }
        //re-calculate with approximate values
        if (includeApproximateMatches) {
            int count = 0;
            for (int i = 0; i < files.length; i++) {
                count++;
                if (files[i].isFile()) {
                    String docName = files[i].getName();

                    try {

                        String docContents = rf.readFile(files[i].toString());
                        StringTokenizer tokenizer = new StringTokenizer(docContents);
                        double docLength = tokenizer.countTokens();
                        System.out.println("new doc: " + docName + " count: " + count);
                        long startTime = System.nanoTime();

                        tokens = generator.generateTokens(docContents, tokenizationType);
                        reCalculateTFandIDFwithApproxValues(docContents, docName, collectionSize, docLength);

                        long endTime = System.nanoTime();
                        System.out.println("that took :" + (endTime - startTime));
                    } catch (IOException ioe) {
                        System.out.println("error" + ioe);
                    }
                }
            }
        }

        calculateTFIDF();

    }