Merge branch 'develop' into 36-create-process-to-store-and-retrieve-data-for-textclassifier

.idea/politweet.iml

@@ -3,7 +3,6 @@
   <component name="NewModuleRootManager">
     <content url="file://$MODULE_DIR$">
       <excludeFolder url="file://$MODULE_DIR$/politweet-environment" />
-      <excludeFolder url="file://$MODULE_DIR$/venv" />
     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />

README.md

@@ -20,9 +20,14 @@ För att få alla dependencies:
 
 1. skapa en virtual environment: https://docs.python.org/3/library/venv.html
 2. Aktivera din virtual environment
-2. gå till projektets root path och skriv i terminalen: 
-      $ env2/bin/python -m pip install -r requirements.txt
-3. I vissa fall funkar det inte att installera twint för Ubuntu. Efter att ha ställt in allt funkade det efter att ha kört "sudo apt-get install build-        essential" i terminalen.
+3. gå till projektets root path och skriv i terminalen: 
+      $ pip install -r requirements.txt
+4. I vissa fall funkar det inte att installera twint för Ubuntu. Efter att ha ställt in allt funkade det efter att ha kört "sudo apt-get install build-        essential" i terminalen.
+5. För att använda openai behövs en auktoriserings-token. Detta skapas genom att skapa en '.env' fil i projektets root path. 
+6. Skriv in följande i den filen:
+      OPENAI_AUTHTOKEN=din open-ai token
+7. Nu borde TextClassifier kunna använda openai, givet att du har timmar att lägga till din token.
+
 
 
 

requirements.txt

@@ -37,6 +37,7 @@ pycparser==2.21
 pyparsing==3.0.9
 PySocks==1.7.1
 python-dateutil==2.8.2
+python-dotenv==0.20.0
 python-socks==2.0.3
 pytz==2022.1
 regex==2022.6.2

textclassifier/TextClassifier.py

@@ -5,9 +5,17 @@ from twitterscraper import TwitterScraper
 from datetime import date
 import os
 
+
 # Set one directory up into ROOT_PATH
 ROOT_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
+from dotenv import find_dotenv, load_dotenv
+
+dotenv_path = find_dotenv()
+load_dotenv(dotenv_path)
+OPENAI_AUTHTOKEN = os.environ.get("OPENAI_AUTHTOKEN")
+
+
 
 class TextClassifier:
     def __init__(self, model_name="text-davinci-002", from_date='2022-01-01', to_date=str(date.today()),
@@ -21,6 +29,14 @@ class TextClassifier:
         :param to_date: string of the format 'YYYY-MM-DD'.
         :param num_tweets: integer value of the maximum number of tweets to be scraped.
         """
+        # Make sure to_date is later than from_date
+        assert from_date < to_date, "from_date must be earlier than to_date"
+        # Make sure the dates are in the correct format
+        assert re.match(r'^\d{4}-\d{2}-\d{2}$', from_date) is not None, "from_date must be in the format YYYY-MM-DD"
+        # Make sure user_name is not empty
+        assert user_name is not None, "user_name cannot be empty"
+        # Make sure num_tweets is a positive integer
+        assert num_tweets > 0, "num_tweets must be a positive integer"
 
         self.model_name = model_name
         self.from_date = from_date
@@ -30,7 +46,7 @@ class TextClassifier:
         self.ts = TwitterScraper.TwitterScraper(from_date, to_date, num_tweets)
         self.df = self.ts.scrape_by_user(user_name)
         # self.api_key = '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'
-        openai.api_key = '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'
+        openai.api_key = OPENAI_AUTHTOKEN
 
     def scrape_tweets(self):
         """
@@ -51,7 +67,6 @@ class TextClassifier:
             classification_clean = classification_clean.replace(" ", "")
 
         return classification_clean
-        return response.choices[0]['text']
 
     def classify_sentiment(self, text: str):
         """
@@ -205,9 +220,7 @@ class TextClassifier:
         df_topic['topic'] = df_topic['tweet'].apply(self.classify_topic)
         return df_topic
 
-    def __repr__(self):
-        return "TwitterScraper(from_date={}, to_date={}, num_tweets={})".format(self.from_date, self.to_date,
-                                                                                self.num_tweets)
+
 
     @staticmethod
     def cleanup_topic_results(prediction_dict, text):
@@ -240,9 +253,27 @@ class TextClassifier:
             return None
 
 
-if __name__ == "__main__":
-    import pandas as pd
+    def __repr__(self):
+        """
+        Gives a string that describes which user is classified
+        :return:
+        """
+        return "Classifier for user: " + self.user_name + " with model: " + self.model_name + "."
+
+
+# if __name__ == "__main__":
+#     import pandas as pd
+#     from datetime import datetime
+#     import os
+#     # show all columns
+#     pd.set_option('display.max_columns', None)
+#
+#     tc = TextClassifier(from_date="2019-01-01", to_date="2019-05-31", user_name='jimmieakesson', num_tweets=20)
+#     tc.classify_sentiment_of_tweets()
+#     # df = tc.analyze_sentiment_of_tweets()
+#     # print(df)
+#     df = tc.classify_topics_of_tweets()
+#     print(df)
+#     # save to csv in a folder under politweet with timestamp in name
+#     df.to_csv(f"{datetime.now().strftime('%Y-%m-%d %H-%M-%S')}_tweets.csv")
 
-    pd.set_option('display.max_columns', None)
-    TC = TextClassifier(from_date="2019-01-01", to_date="2019-12-31", user_name='jimmieakesson', num_tweets=100)
-    TC.df_to_csv()