Upload 5 files

app.py

@@ -0,0 +1,42 @@
+import streamlit as st
+from inference import Keyword_oracle
+from datetime import date
+from extract_tweets import extract_tweets
+import torch
+import gc
+from pathlib import Path
+
+
+header = st.container()
+get_tweet= st.container()
+features= st.container()
+modelTraining = st.container()
+
+
+
+with get_tweet:
+    Path('sheets/').mkdir(exist_ok=True)
+    st.header("Place the topic you want to research on Twitter :bird:")
+    input_keyword =  st.text_input('Write the keyword:')
+    if input_keyword:
+        current_date = date.today()
+        data_since = st.date_input('from which date:',current_date)
+        data_until = st.date_input('until which date:',current_date)
+        max_kw = st.slider('maximum words per keyword', 1, 3, 1)
+        st.text('This process may take a few seconds')
+        st.text(f'plot of the keywords asociated with the topic {input_keyword}:')
+        extract_tweets(input_keyword,data_since,data_until)
+        oracle = Keyword_oracle(input_keyword,
+                                        keyphrase_ngram_range = (1,max_kw),
+                                        diversity=0.3,top_n=3)
+        st.pyplot(oracle.plot())
+        st.text("Table of the most popular keywords")
+        table = oracle.return_table()
+        st.dataframe(table)
+        st.download_button(
+            label="Download data as CSV",
+            data= table.to_csv().encode('utf-8'),
+            file_name= f'{input_keyword}.csv',
+            mime='text/csv',
+        )
+        del oracle

extract_tweets.py

@@ -0,0 +1,50 @@
+import tweepy
+import math
+import pandas as pd
+from tweepy import Client
+from openpyxl import load_workbook
+import re
+import streamlit as st
+
+api_key = st.secrets['api_key']
+api_key_secret = st.secrets['api_key_secret']
+access_token = st.secrets['access_token']
+access_token_secret = st.secrets['access_token_secret']
+
+auth = tweepy.OAuthHandler(api_key,api_key_secret)
+auth.set_access_token(access_token,access_token_secret)
+
+api = tweepy.API(auth)
+
+def preprocess(tweets):
+    proccesed_tweets = []
+    for tweet in tweets.split():
+        tweet = '@user' if tweet.startswith('@') and len(tweet) > 1 else tweet
+        tweet = 'http' if tweet.startswith('http') else tweet
+        proccesed_tweets.append(tweet)
+    return " ".join(proccesed_tweets)
+
+
+def extract_tweets(words,date_since,date_until,num_tweets=300):
+    tweets = tweepy.Cursor(
+                            api.search_tweets,
+                            words, lang="en",
+                            since_id=date_since,
+                            until=date_until,
+                            tweet_mode='extended').items(num_tweets)
+    tweet_cont,tweet_rt,tweet_heart=[],[],[]
+    for tweet in tweets:
+        try:
+            tweet_cont.append(preprocess(tweet.full_text))
+            tweet_rt.append(tweet.retweet_count)
+            tweet_heart.append(tweet.retweeted_status.favorite_count)
+        except AttributeError:
+            tweet_heart.append(0)
+    data = {
+        'Tweet': tweet_cont,
+        'Retweet': tweet_rt,
+        'Favs':tweet_heart
+            }
+    df = pd.DataFrame(data)
+    with pd.ExcelWriter(f'sheets/{words}.xlsx') as writer:
+        df.to_excel(writer)

inference.py

@@ -0,0 +1,113 @@
+from keybert import KeyBERT
+from sen_model import Sentiment
+from sampling import sampling_inference
+import torch
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import mplcyberpunk
+from adjustText import adjust_text
+class Keyword_oracle():
+    def __init__(self,file_name,
+                    weight_rt_fav = [1,4],
+                    noise_threshold = 75,
+                    words_exp = ["user","http","rt","fav",'https'],
+                    **kwargs
+                ):
+            self.key_bert = KeyBERT()
+            self.file_name = file_name
+            self.keybert_args = kwargs
+            self.weight_rt_fav =  weight_rt_fav
+            self.raw_tweets= sampling_inference(file_name).sampled_df()
+            self.noise_threshold = noise_threshold if kwargs['top_n'] == 1 else 90 if kwargs['top_n'] == 2 else 95 
+            self.tweets = self.raw_tweets['Tweet']
+            self.retweet = self.raw_tweets['Retweet']
+            self.favs = self.raw_tweets['Favs']
+            self.sentiment_eval = self.__sentimient_eval__()
+            self.words_exp = words_exp
+            self.mined_tweets = self.__tweets_mined__()
+            self.denoised_df = self.__denoised_df__()
+            self.percentiles = self.__find_threshold__()
+            self.categorical = self.__categorical__()
+
+    def __sentimient_eval__(self):
+        return Sentiment(self.tweets)
+
+    def __tweets_mined__(self):
+        raw_keywords = self.key_bert.extract_keywords(self.tweets,
+               keyphrase_ngram_range = self.keybert_args['keyphrase_ngram_range'],
+                diversity = self.keybert_args['diversity'],
+                top_n = self.keybert_args['top_n']
+                )
+        key_words,engagement,acum_sents = [],[],[]
+        for keys,retweet,fav,sent in zip(raw_keywords,self.retweet,self.favs,self.sentiment_eval):
+            for key in keys:
+                if not set(key[0].split()).intersection(set(self.words_exp)):
+                    key_words.append(key[0])
+                    engagement.append(1+retweet/self.weight_rt_fav[0]+fav/self.weight_rt_fav[1])
+                    acum_sents.append(sent+retweet/self.weight_rt_fav[0]*(sent)+fav/self.weight_rt_fav[1]*sent)
+        key_word_data = {
+            "Key": key_words,
+            'engagement': engagement,
+            'emotions overall':acum_sents
+            }
+        return pd.DataFrame(key_word_data).groupby(['Key'], as_index=False).sum()
+
+
+    def __denoised_df__(self):
+        df = self.mined_tweets
+        tweets = df['engagement']
+        percentile =  np.percentile(tweets, self.noise_threshold)
+        return df[tweets > percentile].reset_index(drop=True)
+
+    def __find_threshold__(self):
+        df = self.mined_tweets
+        tweets = df['emotions overall']
+        top_threshold = self.noise_threshold
+        bottom_threshold = 100-top_threshold
+        while np.percentile(tweets,top_threshold) <= 0 and np.percentile(tweets,100-top_threshold):
+            try:
+                top_threshold +=5
+                bottom_threshold -= 5
+            except top_threshold == 95: 
+                top_threshold,bottom_threshold = 0,0
+        bottom_threshold,top_threshold = np.percentile(tweets,bottom_threshold),np.percentile(tweets,top_threshold)
+        return bottom_threshold,top_threshold
+    def __categorical__(self):
+        df = self.denoised_df
+        tweets = df['emotions overall'].to_numpy()
+        categorical = ['neutral','positive','negative']
+        bottom_threshold,top_threshold = self.percentiles
+        pos = (tweets >= top_threshold) if top_threshold > 0 else np.zeros(tweets.shape[0])
+        neg = (tweets <= bottom_threshold)*-1 if bottom_threshold < 0 else np.zeros(tweets.shape[0])
+        numerical = pos+neg
+        return [categorical[index] for index in numerical.astype(int)]
+    
+    def return_table(self):
+        self.denoised_df['Categorical'] = self.__categorical__()
+        return self.denoised_df.sort_values(by=['emotions overall'],ascending = False).reset_index(drop=True)
+
+    def plot(self):
+        df = self.denoised_df
+        plt.style.use("cyberpunk")
+        keys = df['Key']
+        x,y = df['engagement'],df['emotions overall']
+        fig, ax = plt.subplots()
+        ax.scatter(x, y)
+        text = [plt.text(x_value,y_value,key_value) for x_value,y_value,key_value in zip(x,y,keys)]
+        adjust_text(text)
+        bottom_threshold,top_threshold =  self.percentiles
+        plt.axhline(bottom_threshold ,c= "red", marker='.', linestyle=':') if bottom_threshold < 0 else None 
+        plt.axhline(top_threshold,c= "magenta", marker='.', linestyle=':') if top_threshold > 0 else None
+        plt.title(f"Denoised sentiment analysis of {self.file_name}")
+        plt.xlabel("Engagement")
+        plt.ylabel("Emotions Overall")
+        return fig
+
+if __name__ == "__main__":
+    file_name ='Graham Potter'
+    Keyword_oracle = Keyword_oracle(file_name,
+                                    keyphrase_ngram_range = (1,2),
+                                    diversity=0.3,top_n=3)
+    Keyword_oracle.plot()
+    print(Keyword_oracle.return_table())

sampling.py

@@ -0,0 +1,34 @@
+import random
+import pandas as pd
+import numpy as np
+import scipy.stats
+np.set_printoptions(suppress=True)
+
+class sampling_inference():
+    def __init__(self,file_name,weight = [1,4]):
+        self.raw_tweets= pd.read_excel(f"sheets/{file_name}.xlsx")
+        self.weight = weight
+        self.engagement = self.__engagement__()
+        self.perc = self.__eval_perc__()
+        self.perc_thres = np.percentile(self.engagement,self.perc)
+    def __engagement__(self):
+        tweets = self.raw_tweets['Tweet']
+        raw_retweets = self.raw_tweets['Retweet'].to_numpy()
+        raw_favs = self.raw_tweets['Favs'].to_numpy()
+        engagement = raw_retweets/self.weight[0]+raw_favs/self.weight[1]
+        return engagement 
+    def __eval_perc__(self,perc=75):
+        engagement = self.engagement
+        while np.percentile(engagement,perc) == 0 and perc < 95:
+            perc += 5
+        return perc
+    def sampled_df(self):
+        engagement = self.engagement
+        above_perc = np.where(self.engagement >= self.perc_thres)[0]
+        bellow_perc = np.where(self.engagement < self.perc_thres)[0].tolist()
+        bellow_perc = np.array(random.sample(bellow_perc,above_perc.shape[0]))
+        sampled_rows = np.concatenate((above_perc,bellow_perc))
+        sampled_df= self.raw_tweets.loc[sampled_rows].reset_index(drop=True)
+        del sampled_df['Unnamed: 0']
+        return sampled_df
+

sen_model.py

@@ -0,0 +1,13 @@
+from transformers import pipeline
+import numpy as np
+
+specific_model = pipeline(model="finiteautomata/bertweet-base-sentiment-analysis")
+
+def Sentiment(tweets):
+    output_model = specific_model(tweets.tolist())
+    labels = ["NEG","NEU","POS"]
+    idx = []
+    for output in output_model:
+        idx.append(labels.index(output["label"])-1)
+    return np.array(idx)
+