ändringar i run_main_pipeline

.idea/misc.xml

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (politweet)" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (politweet)" project-jdk-type="Python SDK" />
 </project>

.idea/politweet.iml

@@ -5,7 +5,7 @@
       <excludeFolder url="file://$MODULE_DIR$/politweet-environment" />
       <excludeFolder url="file://$MODULE_DIR$/env" />
     </content>
-    <orderEntry type="inheritedJdk" />
+    <orderEntry type="jdk" jdkName="Python 3.9 (politweet)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="PyNamespacePackagesService">

textclassifier/TextClassifier.py

@@ -275,21 +275,22 @@ class TextClassifier:
         #We presume that all tweets inside the twitterdata.csv file are already classified.
         :return: Pandas dataframe
         """
-        temp_df = pd.DataFrame(
-            columns=['id', 'tweet', 'date', 'user_id', 'username', 'urls', 'nlikes', 'nreplies', 'nretweets'])
-        already_classified_df = pd.read_csv(filename)
-
-        for index, row in self.df.iterrows():
-            if row['id'] in already_classified_df['id'].values:
-                temp_df = temp_df.append(self.get_tweet_by_id(row['id']))
-                self.df = self.df.drop(index)
+        # temp_df = pd.DataFrame(
+        #     columns=['id', 'tweet', 'date', 'user_id', 'username', 'urls', 'nlikes', 'nreplies', 'nretweets'])
+        if os.path.exists(filename):
+            already_classified_df = pd.read_csv(filename)
+            # Remove all entries in self.df that are already in already_classified_df
+            self.df = self.df.merge(already_classified_df, how='left', on='id')
+            # Create a new dataframe where entries in already_classified_df but not in self.df
+            temp_df = already_classified_df.merge(self.df, how='left', on='id')
+        else:
+            print("No csv file found. Continuing without removing already classified tweets.")
 
         self.df = self.classify_topics_of_tweets()
         self.df = self.classify_sentiment_of_tweets()
         # self.df = self.analyze_sentiment_of_tweets()
+
         self.df_to_csv(filename)
-        self.df = self.df.append(temp_df)
-        return self.df
 
     def __repr__(self):
         """
@@ -299,10 +300,6 @@ class TextClassifier:
         return "Classifier for user: " + self.user_name + " with model: " + self.model_name + "."
 
 
-
 if __name__ == "__main__":
-    tc = TextClassifier(from_date="2020-01-01", to_date="2020-12-31", user_name='jimmieakesson', num_tweets=20)
-    tc.test()
-
-
-
+    tc = TextClassifier(from_date="2019-12-01", to_date="2020-12-31", user_name='jimmieakesson', num_tweets=100)
+    tc.run_main_pipeline()