changed topic so that topics are divided into columns

app.py

@@ -1,5 +1,7 @@
 import gradio as gr
 from textclassifier import TextClassifier as tc
+from functions import *
+import re
 
 
 def main(from_date, to_date, user_name):
@@ -12,18 +14,75 @@ def main(from_date, to_date, user_name):
     user_name = user_name
     text_classifier = tc.TextClassifier(from_date=from_date, to_date=to_date, user_name=user_name, num_tweets=20)
     text_classifier.run_main_pipeline()
-    return text_classifier.get_dataframe()
+    dataframe = text_classifier.dataframe
+    return (dataframe, get_summary_statistics(dataframe))
 
 
-def greet(name):
-    return "Hello " + name + "!"
+def get_summary_statistics(dataframe):
+    """
+    This function returns a summary statistics of the dataframe. Returns a string with the summary statistics.
+    :param dataframe: dataframe
+    :return: str
+    """
+    summary_statistics = dataframe.describe()
+    return summary_statistics.to_string()
+
+
+def separate_string(string):
+    list_string = string.split('.')
+    list_useable = []
+    for list_part in list_string :
+        list_useable.append(list_part.split(' ', 1))
+
+    final_list = []
+    for li in list_useable[1:]:
+        final_list.append(li[1])
+    # remove numeric characters and spaces
+    filter_numeric_regex = '[^a-z]'
+    final_final_list = []
+    for li in final_list:
+        final_final_list.append(re.sub(filter_numeric_regex,' ',li).strip())
+    return final_final_list
+
+
+def summary_categorical(dataframe):
+    """
+    This function returns a string of a summary of categorical variables of a dataframe
+    :param dataframe:
+    :return:
+    """
+    # First find all numeric columns in the dataframe
+    numeric_columns = dataframe._get_numeric_data().columns
+    # Then find all categorical columns in the dataframe
+    categorical_columns = dataframe.select_dtypes(include=['object']).columns
+    print("Numeric columns: " + str(numeric_columns) + "\n" + "Categorical columns: " + str(categorical_columns))
+    # Then find the summary statistics of each categorical column
+    summary = ""
+    for column in categorical_columns:
+        # skip 'tweet', 'urls' and 'date' columns
+        if column == 'tweet' or column == 'date' or column == 'urls':
+            continue
+        else:
+            summary += column + ": " + str(dataframe[column].value_counts()) + "\n"
+    # Return a nice decoration of summary
+    return "Summary of categorical variables:\n" + summary
 
 
 if __name__ == "__main__":
-    demo = gr.Interface(
-        fn=main,
-        inputs=['text', 'text', 'text'],
-        outputs="dataframe",
-    )
+    from datetime import date
 
-    demo.launch()
+    # demo = gr.Interface(
+    #     fn=main,
+    #     inputs=['text', 'text', 'text'],
+    #     outputs=["dataframe", "text"],
+    # )
+    # demo.launch()
+
+    text_classifier = tc.TextClassifier(from_date='2020-01-01', to_date='2020-01-31', user_name="jimmieakesson",
+                                        num_tweets=20)
+    text_classifier.run_main_pipeline()
+    print(get_summary_statistics(text_classifier.get_dataframe()))
+    print(type(get_summary_statistics(text_classifier.get_dataframe())))
+    print(summary_categorical(text_classifier.get_dataframe()))
+    string = '1. swedish 2. nuclear 3. hello world 4. uha yhd ikv hahd vva 5. '
+    print(separate_string(string))

flagged/log.csv

@@ -1,2 +1,3 @@
 'from_date','to_date','user_name','output','flag','username','timestamp'
 '2020-01-01','2020-01-01','jimmieakesson','{"data": [["", "", ""], ["", "", ""], ["", "", ""]], "headers": ["1", "2", "3"]}','','','2022-07-19 14:58:49.268002'
+'2021-01-01','2021-01-31','jimmieakesson','{"data": [["", "", ""], ["", "", ""], ["", "", ""]], "headers": ["1", "2", "3"]}','','','','2022-07-20 10:01:35.767463'

functions/functions.py

@@ -0,0 +1,25 @@
+from re import sub
+
+
+def separate_string(string):
+    """
+    This function returns a list of strings from a string.
+    Example: separate_string('1. swedish 2. nuclear 3. hello world 4. uha yhd ikv hahd vva 5. ')
+    returns ['swedish', 'nuclear', 'hello world', 'uha yhd ikv hahd vva', '']
+    :param string: string to be separated
+    :return: list of string items
+    """
+    list_string = string.split('.')
+    list_useable = []
+    for list_part in list_string:
+        list_useable.append(list_part.split(' ', 1))
+
+    final_list = []
+    for li in list_useable[1:]:
+        final_list.append(li[1])
+    # remove numeric characters and spaces
+    filter_numeric_regex = '[^a-z]'
+    final_final_list = []
+    for li in final_list:
+        final_final_list.append(sub(filter_numeric_regex, ' ', li).strip())
+    return final_final_list

textclassifier/TextClassifier.py

@@ -1,16 +1,16 @@
+import os
 import time
+import warnings
+from datetime import date
 
 import openai
-import csv
+import pandas as pd
 import regex as re
-from twitterscraper import TwitterScraper
-from datetime import date
-import os
 from dotenv import find_dotenv, load_dotenv
-import pandas as pd
-import warnings
 from pandas.core.common import SettingWithCopyWarning
-import matplotlib.pyplot as plt
+
+from twitterscraper import TwitterScraper
+from functions import functions as f
 
 warnings.simplefilter(action="ignore", category=SettingWithCopyWarning)
 
@@ -269,12 +269,17 @@ class TextClassifier:
         self.df = df
         self.df_to_csv(filename)
 
-    def get_tweet_by_id(self, id, filename="{}/data/twitterdata.csv".format(ROOT_PATH)):
+    def split_topics_into_columns(self):
         """
-        Returns tweet by id.
-        :param id: id of tweet
-        :return: tweet
+        Splits the topics into columns.
+        :return: None
         """
+        df_topic = self.df.copy()
+        df_topic['topic_temp'] = df_topic['topic'].apply(lambda x: f.separate_string(x))
+        df_topic_split = pd.DataFrame(df_topic['topic_temp'].tolist(),
+                                      columns=['main_topic', 'sub_topic_1', 'sub_topic_2'])
+        self.df = df_topic.merge(df_topic_split, how='left', left_index=True, right_index=True)
+        self.df.drop(['topic_temp'], axis=1, inplace=True)
 
     def run_main_pipeline(self, filename="{}/data/twitterdata.csv".format(ROOT_PATH)):
         """
@@ -340,8 +345,6 @@ class TextClassifier:
 
 
 if __name__ == "__main__":
-    text_classifier = TextClassifier(from_date="2020-01-01", to_date="2020-01-31", user_name='dadgostarnooshi', num_tweets=20)
+    text_classifier = TextClassifier(from_date="2020-01-01", to_date="2020-01-31", user_name='jimmieakesson', num_tweets=20)
     text_classifier.run_main_pipeline()
-    print(text_classifier.split_topics("1. topic1 2. topic2 3. topic3"))
-