fixed minor bugs

.idea/misc.xml

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (politweet)" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (politweet)" project-jdk-type="Python SDK" />
 </project>

.idea/politweet.iml

@@ -6,7 +6,7 @@
       <excludeFolder url="file://$MODULE_DIR$/env" />
       <excludeFolder url="file://$MODULE_DIR$/venv" />
     </content>
-    <orderEntry type="jdk" jdkName="Python 3.10 (politweet)" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Python 3.9 (politweet)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="PyNamespacePackagesService">

app.py

@@ -15,11 +15,11 @@ import matplotlib.pyplot as plt
 from functions import functions as f
 import time
 
-
-SELECTED_COLUMN_DICT = { 'merged_topic': ['tweet', 'main_topic' , 'sub_topic' ,'synonym_topic' , 'cos_sim_topic', 'merged_topic' ],
-                         'sentiment':['tweet', 'sentiment'],
-                        'merged_target': ['tweet', 'target','synonym_target', 'cos_sim_target' , 'merged_target']
-                        }
+SELECTED_COLUMN_DICT = {
+    'merged_topic': ['tweet', 'main_topic', 'sub_topic', 'synonym_topic', 'cos_sim_topic', 'merged_topic'],
+    'sentiment': ['tweet', 'sentiment'],
+    'merged_target': ['tweet', 'target', 'synonym_target', 'cos_sim_target', 'merged_target']
+    }
 
 USER_LIST = ['jimmieakesson', 'BuschEbba', 'annieloof', 'JohanPehrson', 'bolund', 'martastenevi', 'SwedishPM',
              'dadgostarnooshi']
@@ -42,16 +42,16 @@ num_tweet = 1000
 LIMIT = 0.05
 
 
-def show_all_stats(SeeFullStats):
+def show_all_stats(see_full_stats):
     dataframe = pd.read_csv("{}/data/twitterdata.csv".format(tc.ROOT_PATH))
-    if SeeFullStats:
+    if see_full_stats:
         return dataframe
     else:
         return pd.DataFrame()
 
 
-def fixChoicesCorrectOrder(Choices):
-    ListChoices = [x for x in Columns if x in Choices]
+def fixChoicesCorrectOrder(choices):
+    ListChoices = [x for x in Columns if x in choices]
     return ListChoices
 
 
@@ -74,14 +74,12 @@ def main(From,
          To,
          UserNameChoices,
          plot_choice,
-         save_selected ,
+         save_selected,
          rb1, rb2, rb3, rb4, rb5, rb6, rb7, rb8,
-         v1, v2, v3, v4, v5, v6, v7, v8 ,
+         v1, v2, v3, v4, v5, v6, v7, v8,
          s1, s2, s3, s4, s5, s6, s7, s8
 
          ):
-
-
     save_file_bool = s1, s2, s3, s4, s5, s6, s7, s8
 
     def Add_Pychart(df, leaders, plot_choices):
@@ -102,7 +100,7 @@ def main(From,
 
         for db in df_list:
             for col in PLOT_CHOICES_REVERSE_DICT:  # plot_choices:
-                if col=='merged_target':
+                if col == 'merged_target':
                     pie_charts.append(bar(db[0], col + ": " + db[1]))
                 else:
                     pie_charts.append(pie_chart(db[0], col, col + ": " + db[1]))
@@ -116,43 +114,47 @@ def main(From,
         if db.empty:
             return None
         else:
-            db['merged_target']= db["merged_target"].apply(lambda x: "other" if x == "ERROR_9000" or x == "ERROR_496" else x)  # replacing Different Error type with string "other"
-            db['sentiment'] = db['sentiment'].apply(lambda x: re.sub('\s+', "", x))   # removing extra spaces in  at the end and beginning of the sentiments.
+            db['merged_target'] = db["merged_target"].apply(lambda
+                                                                x: "other" if x == "ERROR_9000" or x == "ERROR_496" else x)  # replacing Different Error type with string "other"
+            db['sentiment'] = db['sentiment'].apply(
+                lambda x: re.sub('\s+', "", x))  # removing extra spaces in  at the end and beginning of the sentiments.
             # This can be removed after we remove all unnessary spaces from twitter data
-            all_targets= ['v', 'mp', 's', 'c', 'l', 'kd', 'm', 'sd', 'Red-Greens', 'The opposition']
+            all_targets = ['v', 'mp', 's', 'c', 'l', 'kd', 'm', 'sd', 'Red-Greens', 'The opposition']
             db_new = db.loc[db["merged_target"] != "other"]  # dataframe with other category removed
-            percent_target = (len(db_new) / len(db))*100
-            targets= db_new["merged_target"].value_counts().keys().to_list()
-            positive=[0]*len(all_targets)
-            negative=[0]*len(all_targets)
-            neutral=[0]*len(all_targets)
-            other =[0]*len(all_targets)
-            for i,target in enumerate(all_targets):
-                temp_db= db_new.loc[db_new["merged_target"] == target]
+            percent_target = (len(db_new) / len(db)) * 100
+            targets = db_new["merged_target"].value_counts().keys().to_list()
+            positive = [0] * len(all_targets)
+            negative = [0] * len(all_targets)
+            neutral = [0] * len(all_targets)
+            other = [0] * len(all_targets)
+            for i, target in enumerate(all_targets):
+                temp_db = db_new.loc[db_new["merged_target"] == target]
                 if temp_db.empty:
                     pass
                 else:
                     sent = temp_db['sentiment'].to_list()
-                    positive[i] +=sent.count('positive')
+                    positive[i] += sent.count('positive')
                     negative[i] += sent.count('negative')
                     neutral[i] += sent.count('neutral')
-                    other[i] +=  sent.count('other')
+                    other[i] += sent.count('other')
             font1 = {'family': 'serif', 'color': 'blue', 'size': 10}
             fig = plt.figure()
-            y1 = np.array(positive)/len(db_new)
-            y2 = np.array(negative)/len(db_new)
-            y3 = np.array(neutral)/len(db_new)
-            y4 = np.array(other)/len(db_new)
-            plt.bar(all_targets, y1 , color='g')
-            plt.bar(all_targets, y2 , bottom=y1, color='r')
-            plt.bar(all_targets, y3 , bottom=(y1+y2), color ='yellow')
-            plt.bar(all_targets, y4 , bottom=(y1+y2+y3) , color= 'b')
+            y1 = np.array(positive) / len(db_new) if len(db_new) > 0 else np.array(positive)
+            y2 = np.array(negative) / len(db_new) if len(db_new) > 0 else np.array(negative)
+            y3 = np.array(neutral) / len(db_new) if len(db_new) > 0 else np.array(neutral)
+            y4 = np.array(other) / len(db_new) if len(db_new) > 0 else np.array(other)
+            plt.bar(all_targets, y1, color='g')
+            plt.bar(all_targets, y2, bottom=y1, color='r')
+            plt.bar(all_targets, y3, bottom=(y1 + y2), color='yellow')
+            plt.bar(all_targets, y4, bottom=(y1 + y2 + y3), color='b')
             plt.xticks(rotation=15)
             plt.ylim(0, 1)
-            plt.title( str(percent_target)[0:4] + "% "+ " of tweets have  target. "+ "Number of tweets with target:" +str(len(db_new)),loc='right',fontdict=font1)
-            #plt.xlabel("Targets")
+            plt.title(
+                str(percent_target)[0:4] + "% " + " of tweets have  target. " + "Number of tweets with target:" + str(
+                    len(db_new)), loc='right', fontdict=font1)
+            # plt.xlabel("Targets")
             plt.ylabel("Procent")
-            plt.legend(["positive","negative", "neutral","other"])
+            plt.legend(["positive", "negative", "neutral", "other"])
             return fig
 
     def pie_chart(db, col_name, title):
@@ -178,46 +180,50 @@ def main(From,
     #      os.path.dirname(os.path.abspath(__file__))) + "/politweet/data/twitterdata.csv")  #
     df = dataframe
 
-
     if save_selected:
         user_list = MatchNameToUser(UserNameChoices)
-        df_l=[]
+        df_l = []
         for user in user_list:
-            df_l.append( pd.DataFrame(df.loc[df['username']== user]) )
+            df_l.append(pd.DataFrame(df.loc[df['username'] == user]))
 
-        selected_df= pd.concat(df_l).reset_index(drop=True)
-        export_to_download(selected_df,"selected_leaders")
-        save_selected_checkbox= [gr.Checkbox.update(interactive=False)]
+        selected_df = pd.concat(df_l).reset_index(drop=True)
+        export_to_download(selected_df, "selected_leaders")
+        save_selected_checkbox = [gr.Checkbox.update(interactive=False)]
 
+    else:
+        save_selected_checkbox = [gr.Checkbox.update(interactive=True)]
 
     pycharts = Add_Pychart(df, UserNameChoices, convert_plot_choices(plot_choice))
 
-    rb_components = [rb1, rb2, rb3, rb4, rb5, rb6, rb7, rb8]  #radio_buttons
-    df_visibility_check = [v1,v2,v3,v4,v5,v6,v7,v8]
-    def get_selected_df_list(d_frame,save_or_no ,selected_users, radio, visiblity):
+    rb_components = [rb1, rb2, rb3, rb4, rb5, rb6, rb7, rb8]  # radio_buttons
+    df_visibility_check = [v1, v2, v3, v4, v5, v6, v7, v8]
+
+    def get_selected_df_list(d_frame, save_or_no, selected_users, radio, visibility):
 
         leader_bool_list = [True if leader in selected_users else False for leader in USER_NAMES]
-        df_list=[]
+        df_list = []
         number_tweets = []
-        save_file_components_list =[]
-        for i , u_bool in enumerate(leader_bool_list):
+        save_file_components_list = []
+        for i, u_bool in enumerate(leader_bool_list):
             user_df = d_frame.loc[d_frame['username'] == USER_LIST[i]]
-            number_tweets.append(gr.Number.update(value=len(user_df),visible=u_bool))
+            number_tweets.append(gr.Number.update(value=len(user_df), visible=u_bool))
 
             if save_or_no[i]:
-                export_to_download(pd.DataFrame(user_df) ,"one_leader" )
-                save_file_components_list.append( gr.Checkbox.update(visible=u_bool , interactive=False) )
+                export_to_download(pd.DataFrame(user_df), "one_leader")
+                save_file_components_list.append(gr.Checkbox.update(visible=u_bool, interactive=False))
             else:
-                save_file_components_list.append( gr.Checkbox.update(visible=u_bool) )
+                save_file_components_list.append(gr.Checkbox.update(visible=u_bool))
 
-            if u_bool and visiblity[i]:
-                df_list.append(   get_exemple_df(user_df,PLOT_CHOICES_DICT[radio[i]])  )
+            if u_bool and visibility[i]:
+                df_list.append(get_exemple_df(user_df, PLOT_CHOICES_DICT[radio[i]]))
             else:
                 df_list.append(None)
 
-        return  df_list +number_tweets+save_file_components_list
+        return df_list + number_tweets + save_file_components_list
+
+    return pycharts + save_selected_checkbox + get_selected_df_list(df, save_file_bool, list(UserNameChoices),
+                                                                    rb_components, df_visibility_check)
 
-    return  pycharts + save_selected_checkbox +get_selected_df_list(df,save_file_bool,list(UserNameChoices), rb_components, df_visibility_check)
 
 ''' END OF MAIN 
 ####
@@ -227,35 +233,31 @@ def main(From,
 '''
 
 
-
-
-
-def get_exemple_df(df:pd.DataFrame, column:str):
+def get_exemple_df(df: pd.DataFrame, column: str):
     print(column)
-    df=df[SELECTED_COLUMN_DICT[column] ]
+    df = df[SELECTED_COLUMN_DICT[column]]
     unique_labels = df[column].value_counts().keys()
-    stat=[]
+    stat = []
     for label in unique_labels:
         df_temp = df.loc[df[column] == label]
         if len(df_temp) > 5:
-            df_temp =df_temp[0:5]
+            df_temp = df_temp[0:5]
         stat.append(df_temp)
 
-    exemple_df= pd.concat(stat)
+    exemple_df = pd.concat(stat)
 
-    #stat =stat.reset_index(drop=True)     just in case u want to reset indexing
+    # stat =stat.reset_index(drop=True)     just in case u want to reset indexing
 
     return exemple_df
 
 
-def export_to_download(_data_frame,_type:str ):
-
+def export_to_download(_data_frame, _type: str):
     downloads_path = str(Path.home()) + "/Downloads/"
     if _type == "one_leader":
-        file_name = _data_frame['username'].to_list()[0] #df['username'][0] + "_data"
+        file_name = _data_frame['username'].to_list()[0]  # df['username'][0] + "_data"
     else:
         file_name = "selected_leaders"
-    full_path = downloads_path + file_name+".csv"
+    full_path = downloads_path + file_name + ".csv"
 
     while full_path in glob.glob(downloads_path + "*"):
         search_list = re.findall('\p{N}+', full_path)
@@ -268,24 +270,22 @@ def export_to_download(_data_frame,_type:str ):
 
     _data_frame.to_csv(full_path, index=False)
 
+    # , pie_chart(df, "main_topic"), pie_chart("target")
 
 
-
-
-
-    # , pie_chart(df, "main_topic"), pie_chart("target")
 def piechart_input(df, column, limit):
     df_len = len(df)
     df_v = df[column].value_counts()
     df_len = len(df)
     if column == "sentiment":
         ds_sentiment = df[column].apply(lambda x: re.sub("\s+", "", str(x)))
-        df_v = ds_sentiment.apply(lambda x: x if str(x).lower() == "positive" or str(x).lower() == "negative" or str(x).lower() == "neutral" else "other").value_counts()
+        df_v = ds_sentiment.apply(lambda x: x if str(x).lower() == "positive" or str(x).lower() == "negative" or str(
+            x).lower() == "neutral" else "other").value_counts()
     elif column == "merged_target":
         ds_target = df[column].apply(lambda x: "other" if x == "ERROR_9000" or x == "ERROR_496" else x)
         df_v = ds_target.value_counts()
         freq = df_v.to_list()
-        labels= df_v.keys().to_list
+        labels = df_v.keys().to_list
         freq_dict = {}
         freq_dict[column] = labels
         freq_dict["frequency"] = freq
@@ -321,50 +321,34 @@ def convert_to_boolean(leaders, plot_choices):
     return leaders_converted, plot_converted
 
 
-
-
-
-
-
-
-
-
-
-
-
-
 def update_window(leaders: list, plot_choices: list,
                   v1, v2, v3, v4, v5, v6, v7, v8
                   ):
-
-
     leader_bool_list, plot_bool_list = convert_to_boolean(leaders, plot_choices)
 
     bool_list = []
     df_visiblity_bool = [v1, v2, v3, v4, v5, v6, v7, v8]
 
-
-#this loop sets boolean for plots
+    # this loop sets boolean for plots
     for leader in leader_bool_list:
         if leader:
             for choice in plot_bool_list:
                 bool_list.append(choice)
-           #bool_list.append(True)  ## this is for radio component
+        # bool_list.append(True)  ## this is for radio component
         else:
             for i in range(len(plot_bool_list)):
                 bool_list.append(False)
-            #bool_list.append(False)
+            # bool_list.append(False)
 
     update_blocks = []
     update_plots = []
     update_radio = []
-    update_nr_tweet =[]
+    update_nr_tweet = []
     update_checkbox = []
-    update_save_file_checkboxes =[]
+    update_save_file_checkboxes = []
     update_df = []
 
-    #all_visual = block_list + plots + radio_list + nr_tweet_list + checkbox_list + saving_file_checkboxes + df_list
-
+    # all_visual = block_list + plots + radio_list + nr_tweet_list + checkbox_list + saving_file_checkboxes + df_list
 
     for i, vis_or_not in enumerate(leader_bool_list):
         update_blocks.append(gr.Row.update(visible=vis_or_not))
@@ -374,10 +358,10 @@ def update_window(leaders: list, plot_choices: list,
             update_df.append(gr.DataFrame.update(visible=df_visiblity_bool[i]))
         else:
 
-            update_blocks.append(gr.Row.update(visible=False   ))
-            update_df.append(gr.DataFrame.update(visible= False ))
+            update_blocks.append(gr.Row.update(visible=False))
+            update_df.append(gr.DataFrame.update(visible=False))
 
-        update_nr_tweet.append( gr.Number.update(visible = vis_or_not) )
+        update_nr_tweet.append(gr.Number.update(visible=vis_or_not))
         update_radio.append(gr.Radio.update(visible=vis_or_not))
         update_checkbox.append(gr.Checkbox.update(visible=vis_or_not))
         update_save_file_checkboxes.append(gr.Checkbox.update(visible=vis_or_not))
@@ -387,34 +371,20 @@ def update_window(leaders: list, plot_choices: list,
     return update_blocks + update_plots + update_radio + update_nr_tweet + update_checkbox + update_save_file_checkboxes + update_df
 
 
-
-
-
-
-
-
-
-
-
-
-
-
 def add_plots(user):
     plot_list = []
     for plot_type in PLOT_CHOICES_DICT:
-        plot_list.append(gr.Plot(label=plot_type+ " for " + user, visible=False))
+        plot_list.append(gr.Plot(label=plot_type + " for " + user, visible=False))
     return plot_list
 
 
-
-
-
 def add_nbr_boxes():
     return [gr.Number(value=0, label='Tweets by ' + user, visible=False) for user in USER_NAMES]
 
 
 if __name__ == "__main__":
     import gradio as gr
+
     demo = gr.Blocks(title='Politweet')
     with demo:
         with gr.Column():
@@ -427,7 +397,7 @@ if __name__ == "__main__":
                                                label="")
                     plot_choices = gr.CheckboxGroup(choices=CHOICE_LIST, label='Choose what to show')
 
-                    save_selected_data_checkbox= gr.Checkbox(label="Export selected data")
+                    save_selected_data_checkbox = gr.Checkbox(label="Export selected data")
                     with gr.Row():
                         update = gr.Button('Apply')
                         btn = gr.Button("Run")
@@ -436,52 +406,49 @@ if __name__ == "__main__":
                         # show_plots = gr.components.Checkbox(label='Show topics', value=True)
                 with gr.Column():
                     selected = gr.DataFrame(label="Summary statistics for the selected choices",
-                                                       max_rows=None, visible=False)
-                    # all_data = gr.components.DataFrame(label="Summary statistics of the total database", max_rows=None)
+                                            max_rows=None, visible=False)
+                    # all_data = gr.components.DataFrame(label="Summary statistics of the total database",
+                    # max_rows=None)
 
             plots = []
             radio_list = []
             checkbox_list = []
             df_list = []
             block_list = []
-            saving_file_checkboxes =[]
+            saving_file_checkboxes = []
             nr_tweet_list = []
             with gr.Column():
                 for i in range(len(USER_NAMES)):
-                    block_list +=[gr.Row()] * 3
+                    block_list += [gr.Row()] * 3
                 for i, leader in enumerate(USER_NAMES):
                     with gr.Row():
                         plots += add_plots(leader)
                     with gr.Row():
-                        radio_list.append(gr.Radio(list(PLOT_CHOICES_DICT.keys()), visible=False ,interactive=True))
-                        nr_tweet_list.append( gr.Number(visible=False)  )
-                        checkbox_list.append(gr.Checkbox(label="Show stats ",value=False,visible=False))
-                        saving_file_checkboxes.append( gr.Checkbox(label= "Export file" , value=False , visible= False)  )
+                        radio_list.append(gr.Radio(list(PLOT_CHOICES_DICT.keys()), visible=False, interactive=True))
+                        nr_tweet_list.append(gr.Number(visible=False))
+                        checkbox_list.append(gr.Checkbox(label="Show stats ", value=False, visible=False))
+                        saving_file_checkboxes.append(gr.Checkbox(label="Export file", value=False, visible=False))
 
                     with gr.Row():
                         df_list.append(gr.DataFrame(visible=False))
 
-
         inp = [date1,
                date2,
                leaders,
-               plot_choices , save_selected_data_checkbox] + radio_list + checkbox_list + saving_file_checkboxes
+               plot_choices, save_selected_data_checkbox] + radio_list + checkbox_list + saving_file_checkboxes
 
-        output = plots + [save_selected_data_checkbox]+ df_list + nr_tweet_list + saving_file_checkboxes
+        output = plots + [save_selected_data_checkbox] + df_list + nr_tweet_list + saving_file_checkboxes
 
-
-        all_visual = block_list + plots + radio_list + nr_tweet_list +checkbox_list + saving_file_checkboxes +df_list  #+ df_list # df_comps
+        all_visual = block_list + plots + radio_list + nr_tweet_list + checkbox_list + saving_file_checkboxes + df_list  # + df_list # df_comps
 
         update_inp = [leaders, plot_choices] + checkbox_list
 
-
         update.click(fn=update_window, inputs=update_inp, outputs=all_visual)
 
         btn.click(fn=main, inputs=inp, outputs=output)
         # input.change(fn=main, inputs=input, outputs=output)
     demo.launch(share=False)
 
+# df= pd.read_csv(os.getcwd()+"/data/twitterdata.csv")
 
-#df= pd.read_csv(os.getcwd()+"/data/twitterdata.csv")
-
-#https://51285.gradio.app
+# https://51285.gradio.app

textclassifier/TextClassifier.py

@@ -44,8 +44,14 @@ class TextClassifier:
         # add timer in time-loop and stop after 10 seconds
         # self.df = self.ts.scrape_by_user(user_name)
         self.df = self.ts.scrape_by_several_users(user_list)
-        # Make id as type int64
-        self.df.loc[:, 'id'] = self.df.id.copy().apply(lambda x: int(x))
+
+        # Check if 'id' is in self.df
+        if 'id' in self.df.columns:
+            # Make id as type int64
+            self.df.loc[:, 'id'] = self.df.id.copy().apply(lambda x: int(x))
+        else:
+            # If not do nothing
+            pass
         openai.api_key = OPENAI_API_KEY
 
     def classify_all(self, tweet: str):

twitterscraper/TwitterScraper.py

@@ -27,7 +27,7 @@ class TwitterScraper(object):
         # Make sure to_date is later than from_date
         assert from_date < to_date, "from_date must be earlier than to_date"
         # Make sure num_tweets is a positive integer
-        assert 0 < num_tweets <= 60, "num_tweets must be a positive integer and at most 60"
+        # assert 0 < num_tweets <= 60, "num_tweets must be a positive integer and at most 60"
 
         self.from_date = from_date
         self.to_date = to_date