PAN subreddit 200 threads added

app.py

@@ -267,7 +267,7 @@ def main_demo():
     demo = gr.Blocks()
 
     with demo:
-        data_sets = ["platform-engg_messages", "apps-ui_messages", "ux-reviews_messages"]
+        data_sets = ["platform-engg_messages", "apps-ui_messages", "ux-reviews_messages", "paloaltonetworks_messages"]
         load_data(data_sets)
         with gr.Tab("Thread"):
             data_sets_dd = gr.Dropdown(data_sets,

data_models.py

@@ -0,0 +1,6 @@
+
+class GenericMessage:
+    def __init__(self, user, timestamp, content):
+        self.user = user
+        self.timestamp = timestamp
+        self.content = content

reddit_collect.py

@@ -0,0 +1,52 @@
+import json
+import time
+
+from prawcore import RequestException
+
+import csv
+
+import praw
+
+from data_models import GenericMessage
+
+reddit = praw.Reddit(
+    client_id='LV2nS-xiWYIEn6YpwOhWpg',
+    client_secret='PhC4AYKkL0OUR8miVIuZF45Iz_saiA',
+    user_agent='PythonScript:com.example.passive_monitoring:v0.0.1 (by /u/vvsatya)',
+)
+
+subreddit ='paloaltonetworks'
+# Access subreddit
+subreddit = reddit.subreddit(subreddit)
+
+retry_count = 0
+max_retries = 5
+retry_delay = 5  # in seconds
+
+while retry_count < max_retries:
+    try:
+        thread_messages_file = f'csv/{subreddit}_messages.csv'
+        with open(thread_messages_file, 'w', newline='') as csvfile:
+            writer = csv.writer(csvfile)
+            writer.writerow(
+                ['thread_ts', 'messages_json'])
+            for index, submission in enumerate(subreddit.hot(limit=1000), 1):
+                messages = []
+                messages.append(GenericMessage(submission.author.id, submission.created, f"Title: {submission.title}\n Body: {submission.selftext}"))
+                # Collect and print comments
+                submission.comments.replace_more(limit=None)  # Get more comments if there are "load more" placeholders
+                for comment in submission.comments.list():
+                    author = comment.author.id if comment.author else 'unknown'
+                    messages.append(GenericMessage(author, comment.created, comment.body))
+                message_dicts = [msg.__dict__ for msg in messages]
+                writer.writerow([f'{subreddit}-{index}', json.dumps(message_dicts)])
+                if ( index % 10 == 0 ):
+                    print("Fetched threads : ", index)
+    except RequestException as e:
+        if hasattr(e, 'response') and e.response is not None:
+            if e.response.status_code == 429:
+                print(f"Rate limit exceeded. Retrying in {retry_delay} seconds.")
+                time.sleep(retry_delay)
+                retry_count += 1
+                continue
+        raise

slack_summary.py

@@ -14,6 +14,7 @@ from ratelimit import limits, sleep_and_retry
 
 import csv
 from channel_id_mapper import ChannelIdMapper
+from data_models import GenericMessage
 from metadata_extracter import MetadataExtractor, ThreadMetadata
 
 SKIP_AI = False
@@ -43,13 +44,8 @@ def load_mapping_from_json(filepath):
 
 userIdMapping = load_mapping_from_json('user_id_to_name_mapping.json')
 
-
-class SlackMessage:
-    def __init__(self, user, timestamp, content):
-        self.user = userIdMapping.get(user, user)
-        self.timestamp = timestamp
-        self.content = content
-
+def map_user(user_id):
+    return userIdMapping.get(user_id, user_id)
 
 class SlackThread:
     def __init__(self, min_ts: str, max_ts: str, user: str, unique_users: set[str], summary: str, message_count: int,
@@ -149,8 +145,8 @@ class SlackChannelReader:
                 for row in csv_reader:
                     thread_ts = str(row['thread_ts']).strip()
                     thread_messages_dict = json.loads(str(row['messages_json']))
-                    thread_messages: list[SlackMessage] = [SlackMessage(d['user'], d['timestamp'], d['content']) for d
-                                                           in thread_messages_dict]
+                    thread_messages: list[GenericMessage] = [GenericMessage(map_user(d['user']), d['timestamp'], d['content']) for d
+                                                             in thread_messages_dict]
                     messages_count += len(thread_messages)
                     print(f"Summarizing thread {thread_ts} with {len(thread_messages)} messages")
                     slack_thread = self._get_thread_summary(thread_messages)
@@ -197,8 +193,8 @@ class SlackChannelReader:
         )
         return response
 
-    def read_thread_messages(self, channel_id, thread_ts) -> List[SlackMessage]:
-        all_messages: list[SlackMessage] = []
+    def read_thread_messages(self, channel_id, thread_ts) -> List[GenericMessage]:
+        all_messages: list[GenericMessage] = []
         next_cursor = None
         has_more = True
         while has_more:
@@ -209,9 +205,9 @@ class SlackChannelReader:
             for message in messages:
                 try:
                     if 'user' in message:
-                        all_messages.append(SlackMessage(message['user'], message['ts'], message['text']))
+                        all_messages.append(GenericMessage(map_user(message['user']), message['ts'], message['text']))
                     elif 'subtype' in message:
-                        all_messages.append(SlackMessage(message['subtype'], message['ts'], message['text']))
+                        all_messages.append(GenericMessage(message['subtype'], message['ts'], message['text']))
                     else:
                         print(f"Unknown message type: {message}")
                 except KeyError:
@@ -232,7 +228,7 @@ class SlackChannelReader:
         return result
 
     @staticmethod
-    def _get_thread_summary(thread_messages: List[SlackMessage]) -> Optional[SlackThread]:
+    def _get_thread_summary(thread_messages: List[GenericMessage]) -> Optional[SlackThread]:
 
         if len(thread_messages) == 1:
             return None