added remove duplicates method to csv data

data/twitterdata.csv

@@ -17,5 +17,3 @@ id,tweet,date,user_id,username,urls,nlikes,nreplies,nretweets
 1194553636946350080,Morgan Johansson måste avgå. #pldebatt #svpol  https://t.co/QsVAhqvaou,2019-11-13 10:52:35,95972673,jimmieakesson,[],1713,47,203
 1194528503284346881,Idag begär jag att riksdagen avsätter justitieminister Morgan Johansson. #svpol  https://t.co/tL703x5eYQ,2019-11-13 09:12:43,95972673,jimmieakesson,[],1844,80,140
 1194495733858222080,Åtta år senare och Morgan kämpar vidare... 🦸‍♂️  https://t.co/iCWrEwhgHP,2019-11-13 07:02:30,95972673,jimmieakesson,[],1769,87,271
-1193987880609341440,Stefan Löfven kommenterar oppositionens misstroendeförklaring mot justitieministern.  Kan vara något av det mest arroganta jag någonsin läst.  #avgå  https://t.co/MZHSF1eyMm,2019-11-11 21:24:28,95972673,jimmieakesson,[],1946,160,288
-1193852463390035968,"Det är uppenbart att justitieministern, som i det här fallet är ytterst ansvarig för att klara den här situationen, inte förstår allvaret och inte klarar av att hantera det.   https://t.co/YoSQnJiluZ  https://t.co/eBkBy1SQdU",2019-11-11 12:26:22,95972673,jimmieakesson,['https://sverigesradio.se/sida/artikel.aspx?programid=83&artikel=7341552'],1657,62,178

textclassifier/TextClassifier.py

@@ -45,6 +45,13 @@ class TextClassifier:
         # self.api_key = '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'
         openai.api_key = OPENAI_AUTHTOKEN
 
+    def classify_topic_and_sentiment(self):
+        self.classify_topic_of_tweets()
+        self.classify_sentiment_of_tweets()
+
+        # save the dataframe to a csv file
+
+
     @staticmethod
     def cleanup_sentiment_results(classification_unclean):
         """
@@ -219,7 +226,7 @@ class TextClassifier:
 
     def df_to_csv(self, filename="{}/data/twitterdata.csv".format(ROOT_PATH)):
         """
-        Writes pandas df to csv file. If it already exists, it appends.
+        Writes pandas df to csv file. If it already exists, it appends. If not, it creates. It also removes duplicates.
         :param filename:
         :return:
         """
@@ -228,18 +235,32 @@ class TextClassifier:
         else:
             self.df.to_csv(filename, mode='a', header=False, index=False)
 
-    def return_row_if_ID_exists(self, id: str, filename="{}/data/twitterdata.csv".format(ROOT_PATH)):
+        self.remove_duplicates_from_csv(filename)
+
+    @staticmethod
+    def remove_duplicates_from_csv(filename="{}/data/twitterdata.csv".format(ROOT_PATH)):
         """
-        Checks if a ID is already in the Data.csv file and if it is, it returns the row
-        :param id:
-        :return:
+        Removes duplicates from csv file.
+        :param filename: filename of csv file
+        :return: None
+        """
+        with open(filename, 'r') as f:
+            lines = f.readlines()
+        with open(filename, 'w') as f:
+            for line in lines:
+                if line not in lines[lines.index(line) + 1:]:
+                    f.write(line)
+
+    def remove_already_classified_tweets(self, filename="{}/data/twitterdata.csv".format(ROOT_PATH)):
+        """
+        Removes tweets that have already been classified.
+        :param filename: filename of csv file
+        :return: None
         """
-        with open(filename, 'r') as csvfile:
-            datareader = csv.reader(csvfile)
-            for row in datareader:
-                if row[0] == id:
-                    return row
-            return None
+        df = self.df
+        df = df[df['sentiment'].isnull()]
+        self.df = df
+        self.df_to_csv(filename)
 
     def __repr__(self):
         """
@@ -248,7 +269,9 @@ class TextClassifier:
         """
         return "Classifier for user: " + self.user_name + " with model: " + self.model_name + "."
 
-# if __name__ == "__main__":
+if __name__ == "__main__":
+    tc = TextClassifier(from_date="2022-01-01", to_date="2022-05-31", user_name='jimmieakesson', num_tweets=20)
+    tc.remove_duplicates_from_csv()
 #     import pandas as pd
 #     from datetime import datetime
 #     import os