Pinpoint/FeatureExtraction.py

import ast
import base64
import codecs
import csv
import gc
import json
import os
import pickle
import re
import shutil
import time

import numpy
import pandas as pd
import uuid
from scipy.spatial import distance

from Pinpoint.Aggregator_NGram import n_gram_aggregator
from Pinpoint.Aggregator_TfIdf import tf_idf_aggregator
from Pinpoint.Aggregator_Word2Vec import word_2_vec_aggregator
from Pinpoint.Aggregator_WordingChoice import wording_choice_aggregator
from Pinpoint.Grapher import grapher
from Pinpoint.Logger import logger
from Pinpoint.Sanitizer import sanitization, sys


class feature_extraction():
    """
    This class is used to wrap the functionality of aggregating tweets from CSV files and extracting features pertinent
    to building a random forest extremist classifier.
    """

    # A graph used to store connections between aggregated users
    graph = grapher()
    archived_graphs = []  # an archive of the previous graphs
    # A list storing dictionaries of user ids and their features.
    tweet_user_features = []
    completed_tweet_user_features = []  # has centrality added
    # the global TF IDF model used for the Word 2 Vec model
    saved_tf_idf_model = None
    # A dictionary used for the translation of actual Twitter username to UUID
    dict_of_users = {}

    # The max size for all data entries  (i.e. baseline tweets)
    MAX_RECORD_SIZE = sys.maxsize  # 3050

    # Datasets for training
    violent_words_dataset_location = None
    tf_idf_training_dataset_location = None
    outputs_location = None

    # Used for knowing which columns to access data from. For Twitter data.
    # Summary variables
    DEFAULT_USERNAME_COLUMN_ID = 0
    DEFAULT_DATE_COLUMN_ID = 1
    DEFAULT_MESSAGE_COLUMN_ID = 2
    DEFAULT_ANALYTIC_COLUMN_ID = 4
    DEFAULT_CLOUT_COLUMN_ID = 5
    DEFAULT_AUTHENTIC_COLUMN_ID = 6
    DEFAULT_TONE_COLUMN_ID = 7
    # Emotional Analysis
    DEFAULT_ANGER_COLUMN_ID = 36
    DEFAULT_SADNESS_COLUMN_ID = 37
    DEFAULT_ANXIETY_COLUMN_ID = 35
    # Personal Drives:
    DEFAULT_POWER_COLUMN_ID = 62
    DEFAULT_REWARD_COLUMN_ID = 63
    DEFAULT_RISK_COLUMN_ID = 64
    DEFAULT_ACHIEVEMENT_COLUMN_ID = 61
    DEFAULT_AFFILIATION_COLUMN_ID = 60
    # Personal pronouns
    DEFAULT_P_PRONOUN_COLUMN_ID = 13
    DEFAULT_I_PRONOUN_COLUMN_ID = 19

    # Constants for the fields in the baseline data set (i.e. ISIS magazine/ Stormfront, etc)
    DEFAULT_BASELINE_MESSAGE_COLUMN_ID = 5
    # Summary variables
    DEFAULT_BASELINE_CLOUT_COLUMN_ID = 10
    DEFAULT_BASELINE_ANALYTIC_COLUMN_ID = 9
    DEFAULT_BASELINE_TONE_COLUMN_ID = 12
    DEFAULT_BASELINE_AUTHENTIC_COLUMN_ID = 11
    # Emotional Analysis
    DEFAULT_BASELINE_ANGER_COLUMN_ID = 41
    DEFAULT_BASELINE_SADNESS_COLUMN_ID = 42
    DEFAULT_BASELINE_ANXIETY_COLUMN_ID = 40
    # Personal Drives
    DEFAULT_BASELINE_POWER_COLUMN_ID = 67
    DEFAULT_BASELINE_REWARD_COLUMN_ID = 68
    DEFAULT_BASELINE_RISK_COLUMN_ID = 69
    DEFAULT_BASELINE_ACHIEVEMENT_COLUMN_ID = 66
    DEFAULT_BASELINE_AFFILIATION_COLUMN_ID = 65
    # Personal pronouns
    DEFAULT_BASELINE_P_PRONOUN_COLUMN_ID = 18
    DEFAULT_BASELINE_I_PRONOUN_COLUMN_ID = 24

    # Used for Minkowski distance
    _average_clout = 0
    _average_analytic = 0
    _average_tone = 0
    _average_authentic = 0
    _average_anger = 0
    _average_sadness = 0
    average_anxiety = 0
    average_power = 0
    average_reward = 0
    average_risk = 0
    average_achievement = 0
    average_affiliation = 0
    average_p_pronoun = 0
    average_i_pronoun = 0

    # Used to chache messages to free memory
    MESSAGE_TMP_CACHE_LOCATION = "message_cache"

    def __init__(self, violent_words_dataset_location=None
                 , baseline_training_dataset_location=None,
                 outputs_location=r"outputs"):
        """
        Constructor

        The feature_extraction() class can be initialised with violent_words_dataset_location,
        tf_idf_training_dataset_location, and outputs_location locations. All files in the violent_words_dataset_location
        will be read (one line at a time) and added to the corpus of violent and swear words. The csv file at
        baseline_training_dataset_location is used to train the TFIDF model and a Minkowski distance score is calculated based on the LIWC scores present.

        If the constant variable need to be changed, do this by setting the member variables.
        """

        # Error if datasets not provided
        if violent_words_dataset_location is None:
            raise Exception("No Violent Words dir provided. Provide a directory that contains new line seperated "
                            "files where each line is a violent, extremist, etc word")

        if baseline_training_dataset_location is None:
            raise Exception("No baseline (TF-IDF/ Minkowski) dataset provided. Thus should be a csv file containing "
                            "extremist content and LIWC scores.")

        # Set datasets to member variables
        self.violent_words_dataset_location = violent_words_dataset_location
        self.tf_idf_training_dataset_location = baseline_training_dataset_location
        self.outputs_location = outputs_location

        # Attempt to make the outputs folder if it doesn't exist
        try:
            os.makedirs(outputs_location)
        except:
            pass

    def _reset_stored_feature_data(self):
        """
        Resets memeber variables from a previous run. Importantly does not reset to TF IDF model.
        :return:
        """

        # A graph used to store connections between aggregated users
        self.graph = grapher()
        archived_graphs = []  # an archive of the previous graphs
        # A list storing dictionaries of user ids and their features.
        self.tweet_user_features = []
        self.completed_tweet_user_features = []  # has centrality added
        # the global TF IDF model used for the Word 2 Vec model
        self.dict_of_users = {}

        # Used for Minkowski distance
        self._average_clout = 0
        self._average_analytic = 0
        self._average_tone = 0
        self._average_authentic = 0
        self._average_anger = 0
        self._average_sadness = 0
        self.average_anxiety = 0
        self.average_power = 0
        self.average_reward = 0
        self.average_risk = 0
        self.average_achievement = 0
        self.average_affiliation = 0
        self.average_p_pronoun = 0
        self.average_i_pronoun = 0

    def _get_unique_id_from_username(self, username):
        """
        A function used to retrieve a UUID based on a twitter username. If a username has been used before the same UUID
        will be returned as it is stored in a dictionary.
        :param username:
        :return: a string representation of a UUID relating to a Twitter username
        """

        if username in self.dict_of_users:
            # username already in dictionary
            unique_id = self.dict_of_users[username]
        else:
            # make new UUID
            unique_id = uuid.uuid4().hex
            # stops uuid collisions
            while unique_id in self.dict_of_users.values():
                unique_id = uuid.uuid4().hex

            # Add new user id to dictionary
            self.dict_of_users[username] = unique_id

        # todo it's less efficient writing the whole file every run
        path = os.path.join(self.outputs_location, "users.json")

        with open(path, 'w') as outfile:
            json.dump(self.dict_of_users, outfile)

        return unique_id

    def _add_to_graph(self, originating_user_name, message):
        """
        A wrapper function used for adding a node/ connection to the graph.
        :param originating_user_name: the Twitter username
        :param message: The Tweet
        """

        # Adds node to graph so that if they don't interact with anyone they still have a centrality
        self.graph.add_node(originating_user_name)

        # Process mentions
        mentions = re.findall("\@([a-zA-Z\-\_]+)", message)

        # For all mentions in the tweet add them to the graph as a node
        for mention in mentions:
            self.graph.add_edge_wrapper(originating_user_name, mention, 1, "mention")

        # process hashtags
        hashtags = re.findall("\#([a-zA-Z\-\_]+)", message)

        # For all hashtags in the tweet add them to the graph as a node
        for hashtag in hashtags:
            self.graph.add_edge_wrapper(originating_user_name, hashtag, 1, "hashtag")

    def _get_capitalised_word_frequency(self, message):
        """
        A wrapper function for returning the frequency of capitalised words in a message.
        :param message:
        :return: the frequency of capitalised words in a message.
        """
        return wording_choice_aggregator().get_frequency_of_capatalised_words(
            message)  # NEEDS TO BE DONE before lower case

    def _get_violent_word_frequency(self, message):
        """
        A wrapper function used to retrieve the frequency of violent words in a message.
        :param message: a string representation of a social media message
        :return: The frequency of violent words in the message
        """
        return wording_choice_aggregator().get_frequency_of_violent_or_curse_words(message,
                                                                                   self.violent_words_dataset_location)

    def _get_tweet_vector(self, message):
        """
        A wrapper function used retrieve the 200 size vector representation (Average and Max vector concatenated)
        of that message.
        :param message: a string representation of a message
        :param tf_idf_model:
        :return: a 200 size vector of the tweet
        """
        vectors = []
        tf_idf_model = self._get_tf_idf_model()

        for word in message.split(" "):
            # todo add  back word = sanitization().sanitize(word, self.outputs_location, force_new_data_and_dont_persisit=True)
            try:
                vectors.append(tf_idf_model.wv[word])
                logger().print_message("Word '{}' in vocabulary...".format(word))
            except KeyError as e:
                pass
                logger().print_message(e)
                logger().print_message("Word '{}' not in vocabulary...".format(word))

        # Lists of the values used to store the max and average vector values
        max_value_list = []
        average_value_list = []

        # Check for if at least one word in the message is in the vocabulary of the model
        final_array_of_vectors = pd.np.zeros(100)
        if len(vectors) > 0:

            # Loop through the elements in the vectors
            for iterator in range(vectors[0].size):

                list_of_all_values = []

                # Loop through each vector
                for vector in vectors:
                    value = vector[iterator]
                    list_of_all_values.append(value)

                average_value = sum(list_of_all_values) / len(list_of_all_values)
                max_value = max(list_of_all_values)
                max_value_list.append(max_value)
                average_value_list.append(average_value)

            final_array_of_vectors = pd.np.append(pd.np.array([max_value_list]), pd.np.array([average_value_list]))

        # Convert array to list
        list_of_vectors = []
        for vector in final_array_of_vectors:
            list_of_vectors.append(vector)

        return list_of_vectors

    def _process_tweet(self, user_name, message, row):
        """
        Wrapper function for taking a username and tweet and extracting the features.
        :param user_name:
        :param message:
        :return: a dictionary of all features from the message
        """
        self._add_to_graph(user_name, message)

        features_dict = {"cap_freq": self._get_capitalised_word_frequency(message),
                         "violent_freq": self._get_violent_word_frequency(message),
                         "message_vector": self._get_tweet_vector(message)}


        return features_dict

    def _get_average_liwc_scores_for_baseline_data(self):
        """
        Calculate the LIWC scores for the baseline dataset and the minkowski dataset.
        """

        # Checks if the values have already been set this run, if so don't calculate again
        # TODO what of the edge case where average clout is 0?
        if self._average_clout == 0:
            logger.print_message("Opening dataset {} for LIWC feature extraction and Minkowski distance".format(
                self.tf_idf_training_dataset_location))
            baseline_data_set_name = self.tf_idf_training_dataset_location

            clout_list = []
            analytic_list = []
            tone_list = []
            authentic_list = []
            anger_list = []
            sadness_list = []
            anxiety_list = []
            power_list = []
            reward_list = []
            risk_list = []
            achievement_list = []
            affiliation_list = []
            p_pronoun_list = []
            i_pronoun_list = []

            with open(baseline_data_set_name, 'r', encoding='cp1252') as file:
                reader = csv.reader(file)

                is_header = True
                for row in reader:

                    if is_header:
                        is_header = False
                        continue

                    # Try and access columns, if can't then LIWC fields haven't been set and should be set to 0
                    try:
                        clout = row[self.DEFAULT_BASELINE_CLOUT_COLUMN_ID]
                        analytic = row[self.DEFAULT_BASELINE_ANALYTIC_COLUMN_ID]
                        tone = row[self.DEFAULT_BASELINE_TONE_COLUMN_ID]
                        authentic = row[self.DEFAULT_BASELINE_AUTHENTIC_COLUMN_ID]
                        anger = row[self.DEFAULT_BASELINE_ANGER_COLUMN_ID]
                        sadness = row[self.DEFAULT_BASELINE_SADNESS_COLUMN_ID]
                        anxiety = row[self.DEFAULT_BASELINE_ANXIETY_COLUMN_ID]
                        power = row[self.DEFAULT_BASELINE_POWER_COLUMN_ID]
                        reward = row[self.DEFAULT_BASELINE_REWARD_COLUMN_ID]
                        risk = row[self.DEFAULT_BASELINE_RISK_COLUMN_ID]
                        achievement = row[self.DEFAULT_BASELINE_ACHIEVEMENT_COLUMN_ID]
                        affiliation = row[self.DEFAULT_BASELINE_AFFILIATION_COLUMN_ID]
                        p_pronoun = row[self.DEFAULT_BASELINE_P_PRONOUN_COLUMN_ID]
                        i_pronoun = row[self.DEFAULT_BASELINE_I_PRONOUN_COLUMN_ID]
                    except:
                        clout = 0
                        analytic = 0
                        tone = 0
                        authentic = 0
                        anger = 0
                        sadness = 0
                        anxiety = 0
                        power = 0
                        reward = 0
                        risk = 0
                        achievement = 0
                        affiliation = 0
                        p_pronoun = 0
                        i_pronoun = 0

                    clout_list.append(float(clout))
                    analytic_list.append(float(analytic))
                    tone_list.append(float(tone))
                    authentic_list.append(float(authentic))
                    anger_list.append(float(anger))
                    sadness_list.append(float(sadness))
                    anxiety_list.append(float(anxiety))
                    power_list.append(float(power))
                    reward_list.append(float(reward))
                    risk_list.append(float(risk))
                    achievement_list.append(float(achievement))
                    affiliation_list.append(float(affiliation))
                    p_pronoun_list.append(float(p_pronoun))
                    i_pronoun_list.append(float(i_pronoun))

            #  Get average for variables, used for distance score. These are member variables so that they don't
            #  have to be re-calculated on later runs
            self._average_clout = sum(clout_list) / len(clout_list)
            self._average_analytic = sum(analytic_list) / len(analytic_list)
            self._average_tone = sum(tone_list) / len(tone_list)
            self._average_authentic = sum(authentic_list) / len(authentic_list)
            self._average_anger = sum(anger_list) / len(anger_list)
            self._average_sadness = sum(sadness_list) / len(sadness_list)
            self.average_anxiety = sum(anxiety_list) / len(anxiety_list)
            self.average_power = sum(power_list) / len(power_list)
            self.average_reward = sum(reward_list) / len(reward_list)
            self.average_risk = sum(risk_list) / len(risk_list)
            self.average_achievement = sum(achievement_list) / len(achievement_list)
            self.average_affiliation = sum(affiliation_list) / len(affiliation_list)
            self.average_p_pronoun = sum(p_pronoun_list) / len(p_pronoun_list)
            self.average_i_pronoun = sum(i_pronoun_list) / len(i_pronoun_list)

        return [self._average_clout, self._average_analytic, self._average_tone, self._average_authentic,
                self._average_anger, self._average_sadness, self.average_anxiety,
                self.average_power, self.average_reward, self.average_risk, self.average_achievement,
                self.average_affiliation,
                self.average_p_pronoun, self.average_i_pronoun]

    def _get_tf_idf_model(self):
        """
        A function used to retrieve the TFIDF model trained on the extremist dataset. If the model has already been
        created then the previously created model will be used.
        :return: a TF-IDF model
        """

        # if already made model, reuse
        if self.saved_tf_idf_model is None:
            logger.print_message("Opening dataset {} for TF-IDF".format(self.tf_idf_training_dataset_location))
            baseline_data_set_name = self.tf_idf_training_dataset_location

            data_set = ""

            with open(baseline_data_set_name, 'r', encoding='cp1252') as file:
                reader = csv.reader(file)

                is_header = True
                for row in reader:

                    if is_header:
                        is_header = False
                        continue

                    # take quote from dataset and add it to dataset
                    message = row[self.DEFAULT_BASELINE_MESSAGE_COLUMN_ID]  # data column
                    data_set = data_set + message + "/n"

            # clean data set
            # todo should we be doing sanitization clean_data = sanitization().sanitize(data_set, self.outputs_location) # if so remove line below
            clean_data = data_set

            # get ngrams
            uni_grams, bi_grams, tri_grams = n_gram_aggregator().get_ngrams(clean_data)
            ngrams = uni_grams + bi_grams + tri_grams

            # todo The TF_IDF most important ngrams arn't being used. Should these be used instead of the other ngrams
            tf_idf_scores = tf_idf_aggregator().get_tf_idf_scores(ngrams, data_set)
            number_of_most_important_ngrams = int(len(ngrams) / 2)  # number is half all ngrams
            list_of_most_important_ngrams = sorted(tf_idf_scores, key=tf_idf_scores.get, reverse=True)[
                                            :number_of_most_important_ngrams]

            # create a word 2 vec model
            model = word_2_vec_aggregator().get_model(list_of_sentences=list_of_most_important_ngrams)
            self.saved_tf_idf_model = model
        else:
            model = self.saved_tf_idf_model

        return model

    def open_wrapper(self, location, access_type, list_of_encodings=["utf-8", 'latin-1', 'cp1252']):
        """
        A wrapper around the open built in function that has fallbacks for different encodings.
        :return:
        """

        for encoding in list_of_encodings:
            try:
                file = open(location, access_type, encoding=encoding)
                # Attempt to read file, if fails try other encoding
                file.readlines()
                file.seek(0)
                file.close()
                file = open(location, access_type, encoding=encoding)
                return file
            except LookupError as e:
                continue
            except UnicodeDecodeError as e:
                continue

        raise Exception(
            "No valid encoding provided for file: '{}'. Encodings provided: '{}'".format(location, list_of_encodings))

    def _add_user_post_db_cache(self, user_id, dict_to_add):
        """
        Used to add data to the post message db cache used to free up memory.
        """

        if not os.path.isdir(self.MESSAGE_TMP_CACHE_LOCATION):
            os.mkdir(self.MESSAGE_TMP_CACHE_LOCATION)

        # Save file as pickle
        file_name = "{}-{}.pickle".format(user_id,int(time.time()))
        file_name = os.path.join(self.MESSAGE_TMP_CACHE_LOCATION, file_name)
        with open(file_name, 'wb') as pickle_handle:
            pickle.dump({"description":"a temporery file used for saving memory",
                         "data":dict_to_add}, pickle_handle, protocol=pickle.HIGHEST_PROTOCOL)

    def _get_user_post_db_cache(self, file_name):
        """
        Retrieves data from the cache database used to free up memory.
        """
        if not os.path.isdir(self.MESSAGE_TMP_CACHE_LOCATION):
            raise Exception("Attempted to access temporery cache files before files are created")

        if not os.path.isfile(file_name):
            raise Exception("Attempted to access cache file {}, however, it does not exist".format(file_name))

        with (open(file_name, "rb")) as openfile:
            cache_data = pickle.load(openfile)

        return cache_data["data"]

    def _delete_user_post_db_cache(self):
        try:
            if os.path.isdir(self.MESSAGE_TMP_CACHE_LOCATION):
                shutil.rmtree(self.MESSAGE_TMP_CACHE_LOCATION)
        except:
            pass

    def _get_type_of_message_data(self, data_set_location, has_header=True, is_extremist=None):
        # Ensure all temp files are deleted
        self._delete_user_post_db_cache()

        # Counts the total rows in the CSV. Used for progress reporting.
        print("Starting entity count. Will count '{}'".format(self.MAX_RECORD_SIZE))

        # Read one entry at a time
        max_chunksize = 1
        row_count = 0

        for row in pd.read_csv(data_set_location, iterator=True,encoding='latin-1'):

            row_count = row_count + 1

            if row_count >= self.MAX_RECORD_SIZE:
                break


        print("Finished entity count. Count is: '{}'".format(row_count))
        print("")
        # Loops through all rows in the dataset CSV file.
        current_processed_rows = 0
        is_header = False

        for row in pd.read_csv(data_set_location, iterator=True,encoding='latin-1'):
            row = row.columns
            # Makes sure same number for each dataset
            if current_processed_rows > row_count:
                break

            # Skips the first entry, as it's the CSV header
            if has_header and is_header:
                is_header = False
                continue

            # Retrieve username
            try:
                username = row[self.DEFAULT_USERNAME_COLUMN_ID]
                date = row[self.DEFAULT_DATE_COLUMN_ID]
                user_unique_id = self._get_unique_id_from_username(username)
            except:
                # if empty entry
                continue
            # Attempt to get LIWC scores from csv, if not present return 0's
            try:
                # Summary variables
                clout = float(row[self.DEFAULT_CLOUT_COLUMN_ID])
                analytic = float(row[self.DEFAULT_ANALYTIC_COLUMN_ID])
                tone = float(row[self.DEFAULT_TONE_COLUMN_ID])
                authentic = float(row[self.DEFAULT_AUTHENTIC_COLUMN_ID])
                # Emotional Analysis
                anger = float(row[self.DEFAULT_ANGER_COLUMN_ID])
                sadness = float(row[self.DEFAULT_SADNESS_COLUMN_ID])
                anxiety = float(row[self.DEFAULT_ANXIETY_COLUMN_ID])
                # Personal Drives:
                power = float(row[self.DEFAULT_POWER_COLUMN_ID])
                reward = float(row[self.DEFAULT_REWARD_COLUMN_ID])
                risk = float(row[self.DEFAULT_RISK_COLUMN_ID])
                achievement = float(row[self.DEFAULT_ACHIEVEMENT_COLUMN_ID])
                affiliation = float(row[self.DEFAULT_AFFILIATION_COLUMN_ID])
                # Personal pronouns
                i_pronoun = float(row[self.DEFAULT_I_PRONOUN_COLUMN_ID])
                p_pronoun = float(row[self.DEFAULT_P_PRONOUN_COLUMN_ID])

            except:
                # Summary variables
                clout = 0
                analytic = 0
                tone = 0
                authentic = 0
                # Emotional Analysis
                anger = 0
                sadness = 0
                anxiety = 0
                # Personal Drives:
                power = 0
                reward = 0
                risk = 0
                achievement = 0
                affiliation = 0
                # Personal pronouns
                i_pronoun = 0
                p_pronoun = 0

            liwc_dict = {
                "clout": clout,
                "analytic": analytic,
                "tone": tone,
                "authentic": authentic,
                "anger": anger,
                "sadness": sadness,
                "anxiety": anxiety,
                "power": power,
                "reward": reward,
                "risk": risk,
                "achievement": achievement,
                "affiliation": affiliation,
                "i_pronoun": i_pronoun,
                "p_pronoun": p_pronoun,
            }

            # Calculate minkowski distance
            average_row = self._get_average_liwc_scores_for_baseline_data()

            actual_row = [clout, analytic, tone, authentic,
                          anger, sadness, anxiety,
                          power, reward, risk, achievement, affiliation,
                          p_pronoun, i_pronoun
                          ]

            try:
                liwc_dict["minkowski"] = distance.minkowski(actual_row, average_row, 1)
            except ValueError:
                continue

            # Retrieve Tweet for message
            tweet = str(row[self.DEFAULT_MESSAGE_COLUMN_ID])

            # clean/ remove markup in dataset
            sanitised_message = sanitization().sanitize(tweet, self.outputs_location,
                                                        force_new_data_and_dont_persisit=True)

            # If no message skip entry
            if not len(tweet) > 0 or not len(sanitised_message) > 0 or sanitised_message == '' or not len(
                    sanitised_message.split(" ")) > 0:
                continue

            # Process Tweet and save as dict
            tweet_dict = self._process_tweet(user_unique_id, tweet, row)

            # If the message vector is not 200 skip (meaning that a blank message was processed)
            if not len(tweet_dict["message_vector"]) == 200:
                continue

            if is_extremist is not None:
                tweet_dict["is_extremist"] = is_extremist

            tweet_dict["date"] = date

            # Merge liwc dict with tweet dict
            tweet_dict = {**tweet_dict, **liwc_dict}

            #tweet_dict["user_unique_id"]= user_unique_id

            self._add_user_post_db_cache(user_unique_id, {user_unique_id: tweet_dict})
            #self.tweet_user_features.append()
            # TODO here save to cache json instead of list and graph

            logger().print_message("Added message from user: '{}', from dataset: '{}'. {} rows of {} completed."
                                   .format(user_unique_id, data_set_location, current_processed_rows, row_count), 1)
            current_processed_rows = current_processed_rows + 1
            print("Finished reading row")

        # Add the centrality (has to be done after all users are added to graph)
        completed_tweet_user_features = []
        # Loops through each item in the list which represents each message/ tweet

        # Loop through all data in cache file
        for cached_message_file in os.listdir(self.MESSAGE_TMP_CACHE_LOCATION):
            cached_message_file = os.fsdecode(cached_message_file)
            cached_message_file = os.path.join(self.MESSAGE_TMP_CACHE_LOCATION,cached_message_file)

            # Only process pickle files
            if not cached_message_file.endswith(".pickle"):
                continue

            print("Reading cache file: '{}'".format(cached_message_file))
            cached_message_data = self._get_user_post_db_cache(cached_message_file)
            # Loops through the data in that tweet (Should only be one entry per tweet).
            for user_id in cached_message_data.keys():
                updated_entry = {}
                updated_entry[user_id] = cached_message_data[user_id]
                # Adds centrality
                updated_entry[user_id]["centrality"] = self.graph.get_degree_centrality_for_user(user_id)
                logger().print_message(
                    "Added '{}' Centrality for user '{}'".format(updated_entry[user_id]["centrality"], user_id), 1)
                completed_tweet_user_features.append(updated_entry)
                gc.collect()
                break  # Only one entry per list


        self._delete_user_post_db_cache()
        self.completed_tweet_user_features = self.completed_tweet_user_features + completed_tweet_user_features
        self.tweet_user_features = []
        #self.archived_graphs.append(self.graph)
        self.graph = grapher()
        print("Finished messages")

    def _get_extremist_data(self, dataset_location):
        """
        This function is responsible for aggregating tweets from the extremist dataset, extracting the features, and
        saving them to a file for a model to be created.
        """

        self._get_type_of_message_data(data_set_location=dataset_location, is_extremist=True)

    def _get_counterpoise_data(self, dataset_location):
        """
        This function is responsible for aggregating tweets from the counterpoise (related to the topic but from
        legitimate sources, e.g. news outlets) dataset, extracting the features, and saving them to a file for a
        model to be created.
        """

        self._get_type_of_message_data(data_set_location=dataset_location, is_extremist=False)

    def _get_standard_tweets(self, dataset_location):
        """
        This function is responsible for aggregating tweets from the baseline (random sample of twitter posts)
        dataset, extracting the features, and saving them to a file for a model to be created.
        """

        self._get_type_of_message_data(data_set_location=dataset_location, is_extremist=False)

    def dump_features_for_list_of_datasets(self, feature_file_path_to_save_to, list_of_dataset_locations,
                                           force_new_dataset=True):
        """
        Saves features representing a provided dataset to a json file. Designed to be used for testing after a
        model has been created.
        :param feature_file_path_to_save_to:
        :param dataset_location:
        :return:
        """

        self._reset_stored_feature_data()

        if force_new_dataset or not os.path.isfile(feature_file_path_to_save_to):
            for dataset in list_of_dataset_locations:
                self._get_type_of_message_data(data_set_location=dataset, is_extremist=None)

            with open(feature_file_path_to_save_to, 'w') as outfile:
                json.dump(self.completed_tweet_user_features, outfile, indent=4)

        else:
            with open(feature_file_path_to_save_to, 'r') as file:
                data = file.read()

            # parse file
            self.completed_tweet_user_features = json.loads(data)

    def dump_training_data_features(self, feature_file_path_to_save_to, extremist_data_location,
                                    baseline_data_location, force_new_dataset=True):
        """
        The entrypoint function, used to dump all features, for all users in the extreamist, counterpoise, and baseline
        datsets to a json file.
        :param feature_file_path_to_save_to: The filepath to save the datasets to
        """

        self._reset_stored_feature_data()

        if force_new_dataset or not os.path.isfile(feature_file_path_to_save_to):
            print("Starting baseline messages")
            self._get_standard_tweets(baseline_data_location)
            print("Starting extremist messages")
            self._get_extremist_data(extremist_data_location)


            with open(feature_file_path_to_save_to, 'w') as outfile:
                json.dump(self.completed_tweet_user_features, outfile, indent=4)