Update radical_tweet_aggregator.py

radical_tweet_aggregator.py

@@ -3,21 +3,20 @@ import json
 import os
 from datetime import date
 from pathlib import Path
-
+import time
+import tweepy
+from googletrans import Translator
+from predictor import predictor
 import unicodedata
 
+# Twitter API keys
 consumer_token = os.getenv('CONSUMER_TOKEN')
 consumer_secret = os.getenv('CONSUMER_SECRET')
 my_access_token = os.getenv('ACCESS_TOKEN')
 my_access_secret = os.getenv('ACCESS_SECRET')
 bearer = os.getenv('BEARER')
 
-import time
-import tweepy
-from googletrans import Translator
-
-from predictor import predictor
-
+# TODO: is this needed for mapping the object type after reading the pickle files? If not remove.
 class grapher():
     """
     A wrapper class used for generating a graph for interactions between users
@@ -67,12 +66,7 @@ class grapher():
             self.graph.vs[node_count - 1]["label"] = node_2_name.capitalize()
             node_2 = self.graph.vs[node_count - 1]
 
-
-
-        #print("User one {} - {}, user two {} - {}".format(node_1["label"], str(node_1["id"]),
-        #                                                  node_2["label"], str(node_2["id"])))
         self.graph.add_edges([(node_1["id"], node_2["id"])])
-        #self.graph.add_edge(node_1_name, node_2_name, weight=weight, relation=relationship)  # , attr={""}
 
     def add_node(self, node_name):
         """
@@ -92,8 +86,7 @@ class grapher():
             self.graph.vs[node_count-1]["label"] = node_name.capitalize()
             node_1 = self.graph.vs[node_count-1]
 
-global_oauth1_user_handler = None
-
+# Setup Tweepy API and client objects
 auth = tweepy.OAuth1UserHandler(
    consumer_token, consumer_secret,
    my_access_token, my_access_secret
@@ -108,26 +101,30 @@ client = tweepy.Client(
     access_token_secret=my_access_secret
 )
 
+# This class is used for streaming Tweets via Tweepy
 class IDPrinter(tweepy.StreamingClient):
-
     def on_tweet(self, tweet):
         self.translator = Translator()
         gc.collect()
         if len(tweet.data["text"]) > 100:
-            #tweet = client.get_tweet(id=tweet.id)
             if tweet and tweet.data:
 
                 if tweet.data["author_id"]:
                     tweet_data = tweet.data["text"].strip().replace("@", "").replace("\n","")
                     if tweet_data is not None or tweet != "":
                         username = client.get_user(id=tweet.author_id).data
+                        # Ensure that Tweet is in English
                         lang = self.translator.detect(tweet_data).lang
 
                         if lang == "en":
                             tweet_data = unicodedata.normalize('NFKD', tweet_data).encode('ascii', 'ignore').decode()
                             if tweet_data != None:
+                                # Use Pinpoint to identify if a Tweet is extremist or not
                                 is_extremist = predictor().predict(tweet_data)
                                 print("user {} post extremist {} - message: {}".format(username, is_extremist, str(tweet_data)))
+                                
+                                # If a tweet is extremist go through 10 of that users posts and identify the percentage 
+                                # of posts that are extremist
                                 if is_extremist != None and is_extremist == 1:
                                     tweets = client.get_users_tweets(id=tweet.author_id, max_results=10)
 
@@ -140,9 +137,9 @@ class IDPrinter(tweepy.StreamingClient):
                                                 if is_extremist == True:
                                                     number_extreme = number_extreme + 1
 
-                                    print(number_extreme)
+                                    #print(number_extreme)
                                     threshold = number_extreme/len(tweets[0]) * 100
-                                    print("Threshold {}".format(threshold))
+                                    #print("Threshold {}".format(threshold))
                                     if threshold > 1: #
 
                                         file_name = os.path.join("users","{}-{}-radical_users.txt".format(username,date.today().strftime("%b-%d-%Y")))
@@ -150,7 +147,7 @@ class IDPrinter(tweepy.StreamingClient):
                                         file_path = Path(file_name)
                                         file_path.touch(exist_ok=True)
 
-
+                                        # Write user to a file in the user folder with the percentage of extremist posts
                                         with open(file_name, 'w') as outfile:
                                             json_to_dump = [{"username": username.id, "threshold": threshold,
                                                              "date": date.today().strftime("%b-%d-%Y")}]
@@ -158,9 +155,8 @@ class IDPrinter(tweepy.StreamingClient):
                                         print("Got user {}".format(username))
 
         gc.collect()
-    # calling the api
-
 
+# Continue indefinitely and collects Twitter posts
 while True:
     try:
         printer = IDPrinter(bearer_token=bearer,wait_on_rate_limit =True,chunk_size=10000)