src/main/java/edu/columbia/main/screen_logging/ViewManager.java
public class ViewManager {

    HashMap<String,TaskLogger> taskLoggers = new HashMap<>();

    public ViewManager(String [] languages) {
        for(String lang : languages){
            taskLoggers.put(lang, new TaskLogger(lang));
        }
    }

    public TaskLogger getLogger(String lang){
        return this.taskLoggers.get(lang);
    }


    public void printToConsole(){
        clearScreen();
        for(TaskLogger logger : this.taskLoggers.values()){
            System.out.println(logger);
        }
    }

    public static void clearScreen() {
        System.out.print("\033[H\033[2J");
        System.out.flush();
    }
}
src/main/java/edu/columbia/main/configuration/BabelConfig.java
public class BabelConfig {
    private static Logger log = Logger.getLogger(BabelConfig.class);
    private static volatile BabelConfig instance = null;
    private ConfigFromFile cfg;
    private String[] collectionLanguage;
    private String pathToWordsList;
    private String pathToDataFolder;

    /**
     * Create config object from file
     *
     * @param f file descriptor for config file
     */
    private BabelConfig(File f) {
        try {
            Properties props = new Properties();
            props.load(new FileInputStream(f));
            cfg = ConfigFactory.create(ConfigFromFile.class, props);
        } catch (IOException e) {
            cfg = ConfigFactory.create(ConfigFromFile.class);
        }
    }

    /**
     * Creates a config object from file in path or from default if fails to load file
     *
     * @param pathToConfig
     */
    private BabelConfig(String pathToConfig) {
        //if string null or empty, use or own configFromFile.properties.
        if (pathToConfig == null) {
            cfg = ConfigFactory.create(ConfigFromFile.class);
        } else {
            //get config.properties file from path

            try {
                Properties props = new Properties();
                props.load(new FileInputStream(new File(pathToConfig)));
                cfg = ConfigFactory.create(ConfigFromFile.class, props);
            } catch (IOException e) {
                cfg = ConfigFactory.create(ConfigFromFile.class);
            }

        }

    }

    /**
     * Wrapper for constructor since class is singelton
     *
     * @param pathToFile path to config file
     */
    public static void InsteantiateWithConfigFile(String pathToFile) {
        instance = new BabelConfig(pathToFile);
    }

    /**
     * Wrapper for constructor since class is singelton
     *
     * @param f file descriptor for config file
     */
    public static void InsteantiateWithConfigFile(File f) {
        instance = new BabelConfig(f);
    }

    /**
     * Fetch singlton object
     *
     * @return Instance of BabelConfig (this class)
     */
    public static BabelConfig getInstance() {
        String p = null;
        if (instance == null) {
            synchronized (BabelConfig.class) {
                if (instance == null)
                    instance = new BabelConfig(p);
            }
        }
        return instance;
    }

    /**
     * @return string with path to where scraping files are saved
     */
    public static String getPathToScrapingFolder() {
        ConfigFromFile cfg = BabelConfig.getInstance().getConfigFromFile();
        String remote = cfg.productionScarpingFolderPath();
        return remote;
    }

    /**
     * Loads a resource as stream from resources folder or temp folder
     */

    public ConfigFromFile getConfigFromFile() {

        return cfg;
    }

    /**
     * Loads a resource as stream from resources folder
     *
     * @param str path to resource
     * @return
     */
    public static InputStream getResourceAsStream(String str) {

        File f = new File(str);
        if (f.exists()) {
            try {
                FileInputStream fs = new FileInputStream(f);
                return fs;
            } catch (FileNotFoundException e) {

            }
        }
        ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
        return classLoader.getResourceAsStream(str);
    }

    /**
     * Returns path to local resource
     *
     * @param str local path
     * @return absolute path
     */
    public static String getResource(String str) {

        File f = new File(str);
        if (f.exists())
            return f.getAbsolutePath();


        try {
            return BabelConfig.class.getResource("/" + str).toURI().getPath();
        } catch (Exception e) {
            log.error(e);
            return null;
        }
    }

    /**
     * @return absolute path to resources folder
     */
    public static String getResourcesPath() {
        try {
            return BabelConfig.class.getResource("/completeModel3.gm").toURI().getPath().replace("completeModel3.gm", "");
            //using completeModel3.gm as a hack to get the path to its directory
        } catch (URISyntaxException e) {
            log.error(e);
            return null;
        }
    }

    public String[] getListOfLanguages() {
        return collectionLanguage;
    }

    public void setCollectionLanguage(String langCode) {
        if (langCode != null && !langCode.isEmpty()) {
            collectionLanguage = new String[1];
            collectionLanguage[0] = langCode;
        }
    }

    public String getPathToWordsList() {
        if (pathToWordsList != null)
            return pathToWordsList;
        else
            return BabelConfig.getInstance().getConfigFromFile().pathToWordsFile();
    }

    public void setPathToWordsList(String path) {
        this.pathToWordsList = path;
    }

    public void setPathToDataFolder(String pathToDataFolder) {
        this.pathToDataFolder = pathToDataFolder;
    }

    public String getPathToDataFolder() {
        return this.pathToDataFolder;
    }

    public void setLanguages(String[] languages) {
        this.collectionLanguage = languages;
    }
}
src/main/java/edu/columbia/main/normalization/TwitterNormalizer.java
public class TwitterNormalizer {

    private Extractor extractor;
    public TwitterNormalizer(){
        extractor = new Extractor();
    }
    public String cleanTweet(String tweet){

        if (tweet == null)
             return null;
        if (tweet.equals(""))
            return "";

        ArrayList<String>toRemove = new ArrayList<String>();

        ArrayList<String> hashtags = (ArrayList<String>) extractor.extractHashtags(tweet);
        ArrayList<String> cashtags = (ArrayList<String>) extractor.extractCashtags(tweet);
        ArrayList<String> urls = (ArrayList<String>) extractor.extractURLs(tweet);
        ArrayList<String> mentions = (ArrayList<String>) extractor.extractMentionedScreennames(tweet);


        toRemove.addAll(hashtags);
        toRemove.addAll(cashtags);
        toRemove.addAll(urls);
        toRemove.addAll(mentions);


        for(String remove: toRemove){
            tweet = tweet.replaceAll(remove,"");
        }

        tweet = tweet.replaceAll("@","").replaceAll("RT","").replaceAll(":","");
        tweet = removeEmojis(tweet);

        return tweet;
    }


    /**
     * Removes all emojis and variations from a String
     */
    public String removeEmojis(String str) {
        //remove all fitzpatrick modifiers
        for (Fitzpatrick fitzpatrick : Fitzpatrick.values()) {
            str = str.replaceAll(fitzpatrick.unicode, "");
        }
        //remove all emojis
        for (Emoji emoji : EmojiManager.getAll()) {
            str = str.replaceAll(emoji.getUnicode(), "");
        }

        return str;
    }

}
src/main/java/edu/columbia/main/language_id/Result.java
public class Result {

    public Result(String languageCode, boolean isReliable, double confidence) {

        if(languageCode == null){
            this.languageCode = "UNKNOWN";
            isReliable = false;
            confidence = 0;
        }
        else {
            this.languageCode = languageCode;
            this.isReliable = isReliable;
            this.confidence = confidence;
        }
    }

    public String languageCode;
    public boolean isReliable;
    public double confidence;

    public String getLanguageName(){
        return com.neovisionaries.i18n.LanguageAlpha3Code.getByCode(this.languageCode).getName();
    }

    @Override
    public String toString() {
        return "Result{" +
                "languageCode='" + languageCode + '\'' +
                ", isReliable=" + isReliable +
                ", confidence=" + confidence +
                '}';
    }


    @Override
    public boolean equals(Object o) {
        if (this == o) return true;
        if (!(o instanceof Result)) return false;

        Result result = (Result) o;

        return languageCode.equals(result.languageCode);

    }

    @Override
    public int hashCode() {
        return languageCode.hashCode();
    }
}
src/main/java/edu/columbia/main/LogDB.java
public class LogDB implements Serializable{

    public static final String SCRAPING_FOLDER = BabelConfig.getPathToScrapingFolder()+"/";
    public static final String LOG_FILE = "/log2.txt";
    private HashMap<String,LogDBEntry> db;
    File dbFile;
    static Logger log = Logger.getLogger(LogDB.class);

    public LogDB(String lang){

       this.dbFile = new File(SCRAPING_FOLDER+lang+LOG_FILE);

        if(!dbFile.exists()){
            File dir = new File(SCRAPING_FOLDER +lang);
            if(!dir.exists() || !dir.isDirectory())
                dir.mkdir();
            try {
                dbFile.createNewFile();
            } catch (IOException e) {
                log.error(e);
            }
        }
       readDBFromFile();
   }

    public LogDB(String lang, String type){

        this.dbFile = new File(SCRAPING_FOLDER+lang+"/"+type+".txt");
        if(!dbFile.exists())
            try {
                dbFile.createNewFile();
            } catch (IOException e) {
                log.error(e);
            }

        readDBFromFile();
    }

    private void readDBFromFile() {

        File f = dbFile;
        db = new HashMap<String, LogDBEntry>();
        Scanner in = null;
        try {
            in = new Scanner(new FileReader(f));
            while(in.hasNextLine()) {
                String nextLine = in.nextLine();
                LogDBEntry entry = new LogDBEntry(nextLine);
                db.put(entry.getId(),entry);
            }
        }
        catch(IOException e) {
            log.error(e);
        }
        finally {
            try { in.close() ; } catch(Exception e) { /* ignore */ }
        }
    }

    @Deprecated
    public static void log(String id, String lang){
        try {
            File f = new File(SCRAPING_FOLDER+lang+LOG_FILE);
            FileUtils.writeStringToFile(f, "id: "+id+"\n",true);

        } catch (IOException e) {
            log.error(e);
        }

    }

    /**
     *
     * @param id the id of the post
     * @param url the location from which we got that post
     * @param lang the languageCode of the post
     */
    public static void logWithUrl(String id, String url, String lang){
        try {
            File f = new File(SCRAPING_FOLDER+lang+LOG_FILE);
            FileUtils.writeStringToFile(f, "id: " +id+ ", url:"+url+"\n",true);

        } catch (IOException e) {
            log.error(e);
        }

    }



    public void logWithUrlNonStatic(String id, String url, String lang){
        try {
            File f = dbFile;
            FileUtils.writeStringToFile(f, "id: " +id+ ", url:"+url+"\n",true);
            LogDBEntry entry = new LogDBEntry(id,url);
            db.put(entry.getId(),entry);

        } catch (IOException e) {
            log.error(e);
        }

    }


    /**
     * Checks if a post was already save
     * @param id the ID of the post
     * @param lang the languageCode of the post
     * @return true for new posts
     */

    public static boolean isNew(String id, String lang){

        File f = new File(SCRAPING_FOLDER+lang+LOG_FILE);
        if(f.exists())
            return !find(f,id);
        else {
            try {
                FileUtils.writeStringToFile(f, "id: 1,url:www.test.com\n");
                return true;
            } catch (IOException e) {
                log.error(e);
            }
        }
        return true;

    }


    /**
     * Generates a new ID by checking the last ID in the log, if the log contains non int ID's
     * the method will generate a new random ID.
     * @param lang The languageCode of the log
     * @return Last id in the log +1 OR a random int
     * @throws Exception if failed to get id
     */
    public static String getNewId(String lang) throws Exception {
        File f = new File(SCRAPING_FOLDER+lang+LOG_FILE);
        String id = "";

        if(!f.exists()){
            FileUtils.writeStringToFile(f,"id: 1,url:www.test.com\n");
        }

        try {
            ReversedLinesFileReader fr = new ReversedLinesFileReader(f);

            String lastLine = fr.readLine();
            int commaLocation;
            try {
                 commaLocation = lastLine.indexOf(",");
            }
            catch (Exception ex){
                lastLine = fr.readLine();
                commaLocation = lastLine.indexOf(",");
            }
            id = lastLine.substring(4, commaLocation);

            if (id == "0") {
                throw new Exception("Couldn't get ID");
            }

        } catch (IOException e) {
            log.error(e);
        }

        try{
            int numId = Integer.parseInt(id);
            id = String.valueOf(++numId);
            return id;
        }
        catch(Exception exx){
            Random rand = new Random();
            id = String.valueOf(rand.nextInt()%10000);
            return id;
        }

    }

    /**
     * Search a file for a searchString
     * @param f the file to search in
     * @param searchString
     * @return true if found
     */
    public static boolean find(File f, String searchString) {
        boolean result = false;
        Scanner in = null;
        try {
            in = new Scanner(new FileReader(f));
            while(in.hasNextLine() && !result) {
                String nextLine = in.nextLine();
                result = nextLine.indexOf(searchString) >= 0;
            }
        }
        catch(IOException e) {
            log.error(e);
        }
        finally {
            try { in.close() ; } catch(Exception e) { /* ignore */ }
        }
        return result;
    }

    public boolean isNew(String id){
        return !db.containsKey(id);
    }

}
src/main/java/edu/columbia/main/db/Models/Tweet.java
public class Tweet extends DBEntry {


    public Tweet(String data, String unNormalizedData, String language, String user, DateTime dateTime, String collectedFrom, String url, String twitter_id, String filename) {
        super(filename);
        setData(data);
        setNonNormalizedData(unNormalizedData);
        setLanguage(language);
        setUser(user);
        setDateTime(dateTime);
        setSource(collectedFrom);
        setUrl(url);
        setTwitterId(twitter_id);
    }

    public Tweet(BasicDBObject obj) {
        super(obj);
    }


    public String getLanguage() {
        return (String) get("languageCode");
    }

    public void setLanguage(String language) {
        put("languageCode",language);
    }

    public String getUser() {
        return (String) get("user");
    }

    public void setUser(String user) {
        put("user",user);
    }

    public DateTime getDateTime() {
        return new DateTime((String)get("dateTime"));
    }

    public void setDateTime(DateTime dateTime) {
        if(dateTime == null)
            dateTime = new DateTime();
        put("dateTime",dateTime.toString());
    }

    public String getSource() {
        return (String) get("collectedFrom");
    }

    public void setSource(String source) {
        put("collectedFrom",source);
    }

    public String getTwitterId() {
        return (String) get("twitter_id");
    }
    public void setTwitterId(String id) {
        put("twitter_id",id);
    }



    public void setNonNormalizedData(String nonNormalizedData) {
        put("non_normalized",nonNormalizedData);
    }

    public void getNonNormalizedData() {
        get("non_normalized");
    }


    public void setUrl(String url) {
        put("url",url);
    }

    public String getUrl() {
        return (String)get("url");
    }


}
src/main/java/edu/columbia/main/db/DAO.java
public class DAO {


    /**
     * Saves an entry to file
     * @param entry
     * @param dbName usually scrapig
     * @return true if success
     */
    public static boolean saveEntry(DBEntry entry, String dbName){

        if(entry == null || !entry.isValid())
            return false;

        Logger log = Logger.getLogger(DAO.class);

        MongoDatabase db = MongoDB.INSTANCE.getDatabase(dbName);

        String collectionName = getCollectionName(entry);


        MongoCollection collection = db.getCollection(collectionName,BasicDBObject.class);

        try {
            collection.insertOne(entry);
            return true;
        }
        catch (MongoWriteException ex){
            if (ex.getCode() != 11000) // Ignore errors about duplicates
                log.error(ex.getError().getMessage());
            return false;
        }

    }

    /**
     * Gets collection name based on class of entry
     * @param entry one of the optional model in Models.
     * @return name of collection
     */
    private static String getCollectionName(DBEntry entry) {
        String collectionName = "";
        if (entry instanceof Tweet)
            collectionName = "tweets";
        else if(entry instanceof BlogPost)
            collectionName = "blogPosts";
        else if(entry instanceof ForumPost)
            collectionName = "forumPosts";
        return collectionName;
    }


    /**
     * Saves an entry to scraping db
     * @param entry to save
     * @return true if success
     */
    public static boolean saveEntry(DBEntry entry){
        return saveEntry(entry,"scraping");
    }


    /**
     * checks if an entry exists in the dbName DB
     * @param entry to check
     * @param dbName to check in
     * @return true if new
     */
    public static boolean isNew(DBEntry entry, String dbName){

        if(entry == null)
            return false;

        MongoDatabase db = MongoDB.INSTANCE.getDatabase(dbName);
        String collectionName = getCollectionName(entry);

        MongoCollection collection = db.getCollection(collectionName,BasicDBObject.class);

        BasicDBObject obj = (BasicDBObject) collection.find(eq("_id", entry.getId())).first();
        if(obj != null)
            return false;
        else
            return true;
    }

    /**
     * Check if an entry exists in scraping db
     * @param entry to check
     * @return true if new
     */
    public static boolean isNew(DBEntry entry){
        return isNew(entry,"scraping");
    }


    /**
     * Returns a MongoCollection based on Model Class
     * @param type Model Class
     * @return MongoCollection for that class
     */
    public static MongoCollection getCollectionForClass(Class type) {
        MongoDatabase db = MongoDB.INSTANCE.getDatabase("scraping");
        String collectionName = "";
        if (Tweet.class == type)
            collectionName = "tweets";
        else if (BlogPost.class == type)
            collectionName = "blogPosts";
        else if (ForumPost.class == type)
            collectionName = "forumPosts";

        return db.getCollection(collectionName, BasicDBObject.class);
    }


    /**
     * Search the DB  for tweets in a specific language
      * @param language to search for
     * @return list of users with tweets in that language
     */
    public static Set<String> getUsersWithTweetInLanguage(String language) {
        MongoCollection collection = getCollectionForClass(Tweet.class);
        Set<String> s = new HashSet<String>();
        //find users with tweets having language
        MongoCursor cursor = collection.find(and(eq("languageCode", language))).projection(include("url")).iterator();
        while(cursor.hasNext())
        {
            JSONObject obj = new JSONObject(cursor.next().toString());
            String url = obj.get("url").toString();
            String [] urlParts = url.split("/");
            s.add(urlParts[3]);
        }
        return s;
    }

    /**
     * returns a list of all twitter users in db
     * @return
     */
    public static Set<String> getAllTwitterUserIds(){
        MongoCollection collection = getCollectionForClass(Tweet.class);
        Set<String> s = new HashSet<String>();
        MongoCursor cursor = collection.find().projection(include("url")).iterator();
        while(cursor.hasNext())
        {
            JSONObject obj = new JSONObject(cursor.next().toString());
            String url = obj.get("url").toString();
            String [] urlParts = url.split("/");
            s.add(urlParts[3]);
        }
        return s;

    }

    /**
     *
     * @param c class to count
     * @param source where data was collected from
     * @param language
     * @return count of all documents in db
     */
    public static long countBySourceAndLanguage(Class c, String source, String language) {
        MongoCollection collection = getCollectionForClass(c);
        return collection.count(and(eq("collectedFrom", source), eq("languageCode", language)));
    }


    /**
     * Counts blog posts in language that hasn't been optimizd by PostExtractor
     * @param lang of posts
     * @return count
     */
    public static long countNonOpimizedBlogPostsInLanguage(String lang) {
        MongoCollection collection = getCollectionForClass(BlogPost.class);
        return collection.count(and(exists("optimized", false), eq("languageCode", lang)));

    }
    /**
     * fetches blog posts in language that hasn't been optimizd by PostExtractor
     * supports pagination
     *
     * @param lang of posts
     * @param n number of posts to fetch
     * @param skip post number to start for
     * @return list of BlogPosts
     */
    public static ArrayList<BlogPost> getNNonOpimizedBlogPostsInLanguage(String lang, int n, int skip) {
        MongoCollection collection = getCollectionForClass(BlogPost.class);
        ArrayList<BlogPost> posts = new ArrayList<>();
        collection.find(and(exists("optimized", false), eq("languageCode", lang))).skip(skip).batchSize(n).into(posts);
        return posts;
    }

    /**
     * returns count of number of docs belonging to collection
     *
     * @param collectionName
     * @param dbName
     * @return count of docs in collection
     */
    public static long getCollectionCountInLanguage(String collectionName, String dbName) {
        MongoDatabase db = MongoDB.INSTANCE.getDatabase(dbName);
        MongoCollection collection = db.getCollection(collectionName, BasicDBObject.class);
        return collection.count();
    }

    /**
     * Runs query to get 'k' docs starting from position 'start'
     *
     * @param start index
     * @param k number of docs to fetch
     * @param langcode docs belonging to langcode
     * @return list of BasicDBObjects
     */
    public static ArrayList<BasicDBObject> getBasicDBObjects(int start, int k, String langcode, String collectionName, String dbName) {
        ArrayList<BasicDBObject> data = new ArrayList<BasicDBObject>();
        MongoDatabase db = MongoDB.INSTANCE.getDatabase(dbName);
        MongoCollection collection = db.getCollection(collectionName, BasicDBObject.class);
        collection.find(and(exists("optimized", false), eq("languageCode", langcode))).skip(start).batchSize(k).into(data);
        return data;
    }

    /**
     * Replace a current db entry with another
     * @param type
     * @param old entry
     * @param newPost entry
     */
    public static void replaceEntry(Class type, DBEntry old, DBEntry newPost) {
        MongoCollection collection = getCollectionForClass(type);
        collection.replaceOne(eq("_id", old.getId()), newPost);
    }

    /**
     * deletes an entry from the DB
     * @param entry to delete
     */
    public static void deleteEntry(DBEntry entry) {
        if(entry == null)
            return;

        MongoDatabase db = MongoDB.INSTANCE.getDatabase("scraping");
        String collectionName = getCollectionName(entry);

        MongoCollection collection = db.getCollection(collectionName, BasicDBObject.class);
        collection.deleteOne(eq("_id",entry.getId()));
    }

    /**
     * Checks if a document exists in the DB by url
     * @param c type of document
     * @param url to check for
     * @return true if not exists
     */
    public static boolean isNewByURL(Class c, String url) {

        MongoCollection collection = getCollectionForClass(c);
        BasicDBObject obj = (BasicDBObject) collection.find(eq("url",url)).first();
        if(obj != null)
            return false;
        else
            return true;
    }
}
src/main/java/edu/columbia/main/language_id/LanguageDetector.java
public class LanguageDetector {

    /* LingPipe classifier */
    LingPipe lp = new LingPipe("completeModel3.gm");
    /* CLD2 Classifier */
    Cld2 cld = null;

    /* TextCat Classifier */
    edu.columbia.main.language_id.textcat.TextCategorizer tc = null;
    private ArrayList<String> lpLangs = new ArrayList<String>();
    Logger log = Logger.getLogger(LanguageDetector.class);


    /* Constructor */
    public LanguageDetector(){
        tc = new edu.columbia.main.language_id.textcat.TextCategorizer();
        try {
            cld = new Cld2();
        }
        catch (IOException e)
        {
            log.error(e);
        }
        catch (ClassNotFoundException e)
        {
            log.error(e);
        }
        catch (Error e){
            log.info("failed to load CLD. Continuing using LP language classifier. ");
        }

    }

    /**
     * Preforms language detection using majority vote approach over all classifiers
     * If CLD2 is not installed than fallback to standard language detection detectLanguage()
     * @param text to preform language detection on
     * @param lang what language is this text should be in? Used to pick classifiers
     * @return Classification results
     * @throws IOException
     * @throws ClassNotFoundException
     */
    public Result detectMajorityVote(String text, String lang) throws IOException, ClassNotFoundException {

        if(cld == null){ //if we can't load CLD no point in doing majority vote
            return detectLanguage(text,lang);
        }

        ArrayList<Result> results = new ArrayList<>();

        LanguageCode code = new LanguageCode(lang, LanguageCode.CodeTypes.ISO_639_2);
        if(lp.getSupportedLanguages().contains(code))
            results.add(lp.detectLanguage(text));
        if(tc.getSupportedLanguages().contains(code))
            results.add(tc.detectLanguage(text));
        if(cld.getSupportedLanguages().contains(code))
            results.add(cld.detectLanguage(text));

        Result res = mostCommon(results);

        if(res == null)
            return new Result(null,false,0);
        else
            return res;

    }

    /**
     * Preforms language detection with the best available classifier (based on measured accuracy)
     * @param text to preform language detection on
     * @param lang what language is this text should be in? Used to pick classifiers
     * @return Classification results
     * @throws IOException
     * @throws ClassNotFoundException
     */
    public Result detectHierarchy(String text, String lang) throws IOException, ClassNotFoundException {
        if(text == null)
            return null;

        LanguageCode code = new LanguageCode(lang, LanguageCode.CodeTypes.ISO_639_2);


        if(cld != null && cld.getSupportedLanguages().contains(code)) {
            return cld.detectLanguage(text);
        }
        else if (lp.getSupportedLanguages().contains(code)){
            return lp.detectLanguage(text);
        }
        else if(tc.getSupportedLanguages().contains(code)){
            return tc.detectLanguage(text);
        }
        else{
            log.info("Language: " + lang + " not supported!");
        }

        return null;

    }

    /**
     * Main entry point for identification. Will choose approach based on configuration provided in config file
     * @param text to preform language detection on
     * @param lang what language is this text should be in? Used to pick classifiers
     * @return Classification results
     */
    public Result detectLanguage(String text, String lang) {
        try {
            if (BabelConfig.getInstance().getConfigFromFile().useMajorityVote() && cld!=null)
                return detectMajorityVote(text, lang);
            else
                return detectHierarchy(text, lang);

            }
        catch(NullPointerException np){
            log.error(np);
            return null;
        }
        catch (Exception e) {
            log.error("Can't run language id - > Shutting down!");
            log.error(e);
            System.exit(0);
        }
        return null;
    }

    @Deprecated
    public Result detectOld(String text, String lang){
        LanguageCode code = new LanguageCode(lang, LanguageCode.CodeTypes.ISO_639_2);

        if(lang == null || lp.getSupportedLanguages().contains(code)) {
            try {
                return lp.detectLanguage(text);
            } catch (IOException e) {
                log.error(e);
            } catch (ClassNotFoundException e) {
                log.error(e);
            }
        }

        else{
            return new Result(tc.categorize(Utils.removePuntuation(text)),true,2);
        }

        return null;
    }


    private static <T> T mostCommon(List<T> list) {

        if(list == null || list.size() == 0)
            return null;
        if(list.size() == 1)
            return list.get(0);

        Map<T, Integer> map = new HashMap<>();

        for (T t : list) {
            Integer val = map.get(t);
            map.put(t, val == null ? 1 : val + 1);
        }

        Map.Entry<T, Integer> max = null;

        for (Map.Entry<T, Integer> e : map.entrySet()) {
            if (max == null || e.getValue() > max.getValue())
                max = e;
        }
        if(map.size() == list.size()){  //if sizes are the same it means that all the values in list are unique
            return null;
        }
        return max.getKey();
    }
}
src/main/java/edu/columbia/main/FileSaver.java
public class FileSaver {

    private  String hash;
    /** post it */
    String id;
    /** data of the post */
    String content;
    /** languageCode of the post */
    String language;
    /** the source of the post  - blog name*/
    String source;
    /** the url of the post */
    String url;
    Logger log = Logger.getLogger(FileSaver.class);

    /**
     * Constructur
     *
     * @param content
     * @param language
     * @param source
     * @param url
     * @param id
     */
    public FileSaver(String content, String language, String source, String url, String id) {

        this.id = id;
        this.content = content;
        this.language = language;
        this.source = source;
        this.url = url;

    }
    public FileSaver(String content, String language, String source, String url, String id, String hash) {
        this(content, language, source, url, id);
        this.hash = hash;

    }

    /**
     * Constructor that gets a new id from the DB
     * @param content
     * @param language
     * @param source
     * @param url
     */
    public FileSaver(String content, String language, String source, String url) {

            try {
                this.id = LogDB.getNewId(language);
            } catch (Exception e) {
                log.error(e);
                System.exit(0);
            }
        this.content = content;
        this.language = language;
        this.source = source;
        this.url = url;

    }



    public String getFileName(){
        String filename;
        if(hash != null && !hash.isEmpty())
            filename = BabelConfig.getPathToScrapingFolder()+"/"+language+"/"+source+"/"+ hash +".txt".replace("/","");
        else {
            filename = BabelConfig.getPathToScrapingFolder()+"/" + language + "/" + source + "/" + id + ".txt".replace("/", "");
            hash = "";
        }

        return filename;
    }

    /**
     * saves the file according to a specific directory structure
     * scraping/languageCode/source/id.txt
     * logs the saved file to the DB
     */
    public String save(LogDB logDb){
        try {
            String filename  = getFileName();
            File f = new File(filename);
            if(f.exists()) {
                log.warn(filename + " Already exists");
                return f.getName();
            }

            FileUtils.writeStringToFile(f, content, "UTF-8");


            LogDB.logWithUrl(id + "#" + hash, url, language);


            if(logDb != null){
                logDb.logWithUrlNonStatic(id+"#"+hash,url,language);
            }
            return f.getName();


        } catch (FileNotFoundException e) {
            log.error(e);
        } catch (IOException e) {
            log.error(e);
        }

        return null;
    }

    public String getContent() {
        return content;
    }

    public void setContent(String content) {
        this.content = content;
    }

    public String getLanguage() {
        return language;
    }

    public void setLanguage(String language) {
        this.language = language;
    }

    public String getSource() {
        return source;
    }

    public void setSource(String source) {
        this.source = source;
    }
}
src/main/java/edu/columbia/main/LanguageDataManager.java
public enum LanguageDataManager {
    INSTANCE;

    static Logger log = Logger.getLogger(LanguageDataManager.class);

    /* path to resources folder with seed word files */
    public static final String LANGUAGE_DATA_FOLDER = "languageData/";



    public static ArrayList<String> readFile(String path){
        if(path != null){
            File fw = new File(path);
            //read the words from the path
            ArrayList<String> lst = null;
            try {
                lst = (ArrayList) FileUtils.readLines(fw,"UTF-8");
            } catch (IOException e) {
                log.debug(e);
            }
            //return as list
            return lst;
        }
        return null;
    }

    public static ArrayList<String> getMostCommonWords(String lang, int n, int ngram) {

        String path = BabelConfig.getInstance().getPathToWordsList();
        ArrayList<String> seedQueries = null;

        if(path != null){
            seedQueries = readFile(path);
            if(seedQueries != null)
                return seedQueries;
        }


        //if MostCommon file exists in user's data folder, load from there
        String userPath = BabelConfig.getInstance().getPathToDataFolder();
        seedQueries = readFile(userPath + lang);
        if(seedQueries != null)
            return seedQueries;


        //If mostcommon file exists in resources, load it from file
        InputStream f = BabelConfig.getResourceAsStream(LANGUAGE_DATA_FOLDER + lang);
        if(f != null){
            seedQueries =  getMostCommonWordsFromFile(f,n);
        }

        //No data found
        if(seedQueries == null || seedQueries.isEmpty()) {
            log.error("Could not locate any word list for seeding. Shutting down");
            System.exit(0);
        }
        return null;
    }


    /**
     * loads most common words from file
     * @param f stream to load from
     * @param n number of words to load
     * @return
     */
    public static ArrayList<String> getMostCommonWordsFromFile(InputStream f, int n) {

        ArrayList<String> words = new ArrayList<String>();
        BufferedReader br = new BufferedReader(new InputStreamReader(f, StandardCharsets.UTF_8));
        int counter = 0;
        String line;
        try {
            while ((line = br.readLine()) != null && counter <= n) {
                words.add(line);
            }
        } catch (IOException e) {
            log.error(e);
        }

        return words;
    }
    static class ValueComparator implements Comparator<String> {

        Map<String, Integer> base;
        public ValueComparator(Map<String, Integer> base) {
            this.base = base;
        }

        // Note: this comparator imposes orderings that are inconsistent with equals.
        public int compare(String a, String b) {
            if (base.get(a) >= base.get(b)) {
                return -1;
            } else {
                return 1;
            } // returning 0 would merge keys
        }
    }


    public static String[] getLanguages() {
        return BabelConfig.getInstance().getListOfLanguages();
    }


}


Question: package edu.columbia.main.twitter;
import edu.columbia.main.FileSaver;
import edu.columbia.main.LanguageDataManager;
import edu.columbia.main.LogDB;
import edu.columbia.main.configuration.BabelConfig;
import edu.columbia.main.db.DAO;
import edu.columbia.main.db.Models.Tweet;
import edu.columbia.main.language_id.LanguageDetector;
import edu.columbia.main.language_id.Result;
import edu.columbia.main.normalization.TwitterNormalizer;
import edu.columbia.main.screen_logging.ViewManager;
import org.apache.log4j.Logger;
import twitter4j.*;
import java.io.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;



/**
 * Created by Gideon on 2/20/15.
 */

/**
 * Consumes the Twitter API for a given language
 * Saves all found tweets
 */
public class TwitterScraper implements Serializable {

    private ViewManager viewManager;
    private Twitter twitter;
    private int counter = 0;
    private int numOfRequests = 0;
    private HashMap<String, Boolean> map = new HashMap<String, Boolean>();
    private ArrayList<String> words;
    private LogDB logDb;
    private String lang;
    private LanguageDetector languageDetector;
    static Logger log = Logger.getLogger(TwitterScraper.class);
    static int ngram = BabelConfig.getInstance().getConfigFromFile().ngram();


    public TwitterScraper(String language, LanguageDetector languageDetector, ViewManager viewManager) {
        this.lang = language; //1 lang per scraper
        this.words = LanguageDataManager.getMostCommonWords(this.lang, 5000, ngram);
        this.logDb = new LogDB(this.lang); //saving text files
        this.languageDetector = languageDetector;
        this.viewManager = viewManager;
    }


    /**
     * Iterates over all seed words
     *
     * @throws TwitterException
     */
    public void scrapeByLanguage() throws TwitterException {

        while (true) {
            Iterator it = words.iterator();
            while (it.hasNext()) {
                String word = (String) it.next();
                word = word.trim();
                if (word.equals("")) {
                    continue;
                }
                searchAndSave(word, languageDetector, this.lang);
                it.remove();
            }

            //after finishing all the words refill the list
            this.words = LanguageDataManager.getMostCommonWords(this.lang, 5000, ngram);
        }

    }


    /**
     * Queries the twitter api for a given words and saves all matched results
     *
     * @param word to query
     * @param lp   language identification instance
     * @param lang that tweets should be in
     * @throws TwitterException
     */
    private void searchAndSave(String word, LanguageDetector lp, String lang) throws TwitterException {
        log.info("Searching for posts that contain the word: " + word);
        Query query = new Query(word);
        if (lang.equals("lit")) {
            query.setLang("lt");
        } else if (lang.equals("tel"))
            query.setLang("te");

        query.setCount(100);
        query.setSince("2010-01-01");

        QueryResult result;
        do {
            result = twitter.search(query);
            if (numOfRequests++ == 178) {
                throw new TwitterException("API Limit");
            }
            List<Status> tweets = result.getTweets();
            for (Status tweet : tweets) {
                try {
                    if (map.get(tweet.getText()) == null) {
                        String content = tweet.getText();
                        String url = "http://twitter.com/" + tweet.getUser().getId() + "/status/" + tweet.getId();
                        String origContent = content;
                        content = new TwitterNormalizer().cleanTweet(content);
Answer:                         Result res = lp.detectLanguage(content, this.lang);