Update radical_tweet_aggregator.py

radical_tweet_aggregator.py

@@ -121,35 +121,44 @@ class IDPrinter(tweepy.StreamingClient):
                             if tweet_data != None:
                                 # Use Pinpoint to identify if a Tweet is extremist or not
                                 is_extremist = predictor().predict(tweet_data)
-                                print("user {} post extremist {} - message: {}".format(username, is_extremist, str(tweet_data)))
-                                
-                                # If a tweet is extremist go through 10 of that users posts and identify the percentage 
+                                toxicity_score = Detoxify('original').predict(tweet_data)["toxicity"]
+
+                                print("user {} post extremist {}, toxicity {} - message: {}".format(username, is_extremist, toxicity_score, str(tweet_data)))
+
+                                # If a tweet is extremist or toxicity above 0.5, go through 200 of that users posts and identify the percentage
                                 # of posts that are extremist
-                                if is_extremist != None and is_extremist == 1:
-                                    tweets = client.get_users_tweets(id=tweet.author_id, max_results=10)
+                                if (is_extremist != None and is_extremist == 1) or (toxicity_score != None and toxicity_score >= 0.5):
+                                    tweets = client.get_users_tweets(id=tweet.author_id, max_results=100)
 
                                     number_extreme = 0
                                     tweets = tweets[0]
+
+                                    list_of_toxicity_scores = []
                                     for users_tweet in tweets:
                                         if users_tweet.text != None:
-                                            is_extremist = predictor().predict(users_tweet.text)
+                                            tweet_msg = users_tweet.text
+                                            is_extremist = predictor().predict(tweet_msg)
+                                            toxicity_score = Detoxify('original').predict(tweet_msg)["toxicity"]
+                                            list_of_toxicity_scores.append(toxicity_score)
                                             if is_extremist != None:
                                                 if is_extremist == True:
                                                     number_extreme = number_extreme + 1
 
                                     #print(number_extreme)
-                                    threshold = number_extreme/len(tweets[0]) * 100
+                                    threshold = number_extreme/len(tweets)
+                                    threshold = threshold * 100
+                                    toxicity_avg = sum(list_of_toxicity_scores) / len(tweets)
+                                    toxicity_avg = toxicity_avg * 100
                                     #print("Threshold {}".format(threshold))
-                                    if threshold > 1: #
+                                    if threshold > 1 or toxicity_avg > 1: #
 
                                         file_name = os.path.join("users","{}-{}-radical_users.txt".format(username,date.today().strftime("%b-%d-%Y")))
-                                        print("User {} was found to be extremist".format(username))
                                         file_path = Path(file_name)
                                         file_path.touch(exist_ok=True)
 
                                         # Write user to a file in the user folder with the percentage of extremist posts
                                         with open(file_name, 'w') as outfile:
-                                            json_to_dump = [{"username": username.id, "threshold": threshold,
+                                            json_to_dump = [{"username": username.id, "violence-threshold": threshold, "toxicity-threshold":toxicity_avg,
                                                              "date": date.today().strftime("%b-%d-%Y")}]
                                             json.dump(json_to_dump, outfile, indent=4)
                                         print("Got user {}".format(username))