Init commit

.gitignore

@@ -0,0 +1,2 @@
+.idea/
+notebooks/.ipynb_checkpoints

Dockerfile

@@ -0,0 +1,34 @@
+# Use the official Python base image
+FROM python:3.9
+
+# Install Git LFS
+RUN apt-get update && apt-get install -y git-lfs
+
+# Set the working directory
+WORKDIR /app
+
+# Copy requirements.txt into the container
+COPY requirements.txt .
+
+# Install the required packages
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Set the git credential helper to "store"
+RUN git config --global credential.helper store
+
+# Copy the rest of the application files into the container
+COPY . .
+
+# Set environment variables (Replace with your actual values)
+ENV HUGGINGFACE_AUTH_TOKEN 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
+ENV SUBREDDIT askreddit
+ENV START_DATE 2013-01-01
+
+# Copy supervisord.conf into the container
+COPY supervisord.conf .
+
+# Expose the desired port
+EXPOSE 7860
+
+# Run supervisord
+CMD ["supervisord", "-c", "supervisord.conf"]

app.py

@@ -0,0 +1,24 @@
+import gradio as gr
+from rich.console import Console
+from rich.syntax import Syntax
+
+
+def log_file_to_html_string():
+    log_file = "mylog.log"
+
+    console = Console(record=True, width=150)
+    with open(log_file, "rt") as f:
+        syntax = Syntax(f.read(), "python", theme="monokai", word_wrap=True)
+
+    console.print(syntax)
+    html_content = console.export_html(inline_styles=True)
+
+    return html_content
+
+
+with gr.Blocks() as demo:
+    name = gr.Markdown("# Reddit Scraper")
+    output = gr.HTML(log_file_to_html_string, every=1)
+
+if __name__ == '__main__':
+    demo.launch(server_name="0.0.0.0", show_error=True, server_port=7860, enable_queue=True)

archive/subreddit_downloader.py

@@ -0,0 +1,145 @@
+import csv
+import json
+import sys
+import time
+import traceback
+from datetime import datetime
+
+import requests
+
+username = ""  # put the username you want to download in the quotes
+subreddit = "BestofRedditorUpdates"  # put the subreddit you want to download in the quotes
+thread_id = ""  # put the id of the thread you want to download in the quotes, it's the first 5 to 7 character string of letters and numbers from the url, like 107xayi
+# leave either one blank to download an entire user's or subreddit's history
+# or fill in both to download a specific users history from a specific subreddit
+
+# change this to one of "human", "csv" or "json"
+# - human: the score, creation date, author, link and then the comment/submission body on a second line. Objects are separated by lines of dashes
+# - csv: a comma seperated value file with the fields score, date, title, author, link and then body or url
+# - json: the full json object
+output_format = "csv"
+
+# default start time is the current time and default end time is all history
+# you can change out the below lines to set a custom start and end date. The script works backwards, so the end date has to be before the start date
+# start_time = datetime.utcnow()  # datetime.strptime("10/05/2021", "%m/%d/%Y")
+start_time = datetime.strptime("04/02/2023", "%m/%d/%Y")
+end_time = None  # datetime.strptime("09/25/2021", "%m/%d/%Y")
+
+convert_to_ascii = False  # don't touch this unless you know what you're doing
+convert_thread_id_to_base_ten = True  # don't touch this unless you know what you're doing
+
+
+def write_csv_line(writer, obj, is_submission):
+    output_list = []
+    output_list.append(str(obj['score']))
+    output_list.append(datetime.fromtimestamp(obj['created_utc']).strftime("%Y-%m-%d"))
+    if is_submission:
+        output_list.append(obj['title'])
+    output_list.append(f"u/{obj['author']}")
+    output_list.append(f"https://www.reddit.com{obj['permalink']}")
+    if is_submission:
+        if obj['is_self']:
+            if 'selftext' in obj:
+                output_list.append(obj['selftext'])
+            else:
+                output_list.append("")
+        else:
+            output_list.append(obj['url'])
+    else:
+        output_list.append(obj['body'])
+    writer.writerow(output_list)
+
+
+def write_json_line(handle, obj):
+    handle.write(json.dumps(obj))
+    handle.write("\n")
+
+
+def download_from_url(filename, url_base, output_format, start_datetime, end_datetime, is_submission, convert_to_ascii):
+    print(f"Saving to {filename}")
+
+    count = 0
+    if output_format == "human" or output_format == "json":
+        if convert_to_ascii:
+            handle = open(filename, 'w', encoding='ascii')
+        else:
+            handle = open(filename, 'w', encoding='UTF-8')
+    else:
+        handle = open(filename, 'w', encoding='UTF-8', newline='')
+        writer = csv.writer(handle)
+
+    previous_epoch = int(start_datetime.timestamp())
+    break_out = False
+    while True:
+        new_url = url_base + str(previous_epoch)
+        json_text = requests.get(new_url, headers={'User-Agent': "Post downloader by /u/Watchful1"})
+        time.sleep(1)  # pushshift has a rate limit, if we send requests too fast it will start returning error messages
+        try:
+            json_data = json_text.json()
+        except json.decoder.JSONDecodeError:
+            time.sleep(1)
+            continue
+
+        if 'data' not in json_data:
+            break
+        objects = json_data['data']
+        if len(objects) == 0:
+            break
+
+        for obj in objects:
+            previous_epoch = obj['created_utc'] - 1
+            if end_datetime is not None and datetime.utcfromtimestamp(previous_epoch) < end_datetime:
+                break_out = True
+                break
+            count += 1
+            try:
+                if output_format == "csv":
+                    write_csv_line(writer, obj, is_submission)
+                elif output_format == "json":
+                    write_json_line(handle, obj)
+            except Exception as err:
+                if 'permalink' in obj:
+                    print(f"Couldn't print object: https://www.reddit.com{obj['permalink']}")
+                else:
+                    print(f"Couldn't print object, missing permalink: {obj['id']}")
+                print(err)
+                print(traceback.format_exc())
+
+        if break_out:
+            break
+
+        print(f"Saved {count} through {datetime.fromtimestamp(previous_epoch).strftime('%Y-%m-%d')}")
+
+    print(f"Saved {count}")
+    handle.close()
+
+
+if __name__ == "__main__":
+    filter_string = None
+    if username == "" and subreddit == "" and thread_id == "":
+        print("Fill in username, subreddit or thread id")
+        sys.exit(0)
+    if output_format not in ("human", "csv", "json"):
+        print("Output format must be one of human, csv, json")
+        sys.exit(0)
+
+    filters = []
+    if username:
+        filters.append(f"author={username}")
+    if subreddit:
+        filters.append(f"subreddit={subreddit}")
+    if thread_id:
+        if convert_thread_id_to_base_ten:
+            filters.append(f"link_id={int(thread_id, 36)}")
+        else:
+            filters.append(f"link_id=t3_{thread_id}")
+    filter_string = '&'.join(filters)
+
+    url_template = "https://api.pushshift.io/reddit/{}/search?limit=1000&order=desc&{}&before="
+
+    if not thread_id:
+        download_from_url("posts.txt", url_template.format("submission", filter_string), output_format, start_time,
+                          end_time, True, convert_to_ascii)
+    # download_from_url("comments.txt", url_template.format("comment", filter_string), output_format, start_time,
+    #                   end_time, False, convert_to_ascii)
+

main.py

@@ -0,0 +1,145 @@
+import os
+import time
+from datetime import datetime, timedelta
+
+import pandas as pd
+from datasets import Dataset, DatasetDict, load_dataset
+from huggingface_hub import login
+
+from my_logger import setup_logger
+from utilities.pushshift_data import scrape_submissions_by_day, submissions_to_dataframe
+
+# Set dataset name, path to README.md, and existing dataset details
+dataset_name = "derek-thomas/askreddit_test"
+dataset_readme_path = "README.md"
+subreddit = os.environ["SUBREDDIT"]
+
+# Authenticate with Hugging Face using an auth token
+auth_token = os.environ["HUGGINGFACE_AUTH_TOKEN"]
+login(auth_token, add_to_git_credential=True)
+
+logger = setup_logger(__name__)
+
+
+def update_readme(dataset_name, subreddit, date_to_fetch):
+    readme_text = f"""
+    # {dataset_name}
+
+    ## Dataset Overview
+    The goal is to have an open dataset of `{subreddit}` submissions. This has been taken from the Pushshift API.
+
+    ## Data Collection
+    This has been collected with sequential calls that follow the pagination of the pushshift request. 
+
+
+    ## Data Structure
+    - `all_days`: All the data after `{os.environ["START_DATE"]}`
+
+    ## Update Frequency
+    The dataset is updated daily and covers the period from `{os.environ["START_DATE"]}` to two days ago.
+
+    ## Attribution
+    Data sourced from the Pushshift API.
+
+    ## Change Log
+    <details>
+    <summary>Click to expand</summary>
+
+    - **{datetime.now().strftime('%Y-%m-%d')}:** Added data for {date_to_fetch} to the 'all_days' split and saved as CSV
+
+    </details>
+    """
+
+    return readme_text
+
+
+def main(date_to_fetch):
+    """
+    Runs the main data processing function to fetch and process subreddit data for the specified date.
+
+    Args:
+        date_to_fetch (str): The date to fetch subreddit data for, in the format "YYYY-MM-DD".
+
+    Returns:
+        most_recent_date (str): Most recent date in dataset
+    """
+
+    # Load the existing dataset from the Hugging Face hub or create a new one
+    try:
+        logger.info("Loading existing dataset")
+        dataset = load_dataset(dataset_name)
+        if "__index_level_0__" in dataset["all_days"].column_names:
+            dataset = dataset.remove_columns(["__index_level_0__"])
+    except FileNotFoundError:
+        logger.info("Creating new dataset")
+        dataset = DatasetDict()
+
+    # Call get_subreddit_day with the calculated date
+    logger.info(f"Fetching data for {date_to_fetch}")
+    submissions = scrape_submissions_by_day(subreddit, date_to_fetch)
+    df = submissions_to_dataframe(submissions)
+    logger.info(f"Data fetched for {date_to_fetch}")
+    most_recent_date = datetime.strptime(date_to_fetch, '%Y-%m-%d').date()
+
+    # Append DataFrame to split 'all_days' or create new split
+    if "all_days" in dataset:
+        logger.info("Appending data to split 'all_days'")
+        # Merge the new submissions
+        old_data = dataset['all_days'].to_pandas()
+        new_data = pd.concat([old_data, df], ignore_index=True)
+
+        # Drop duplicates just in case
+        new_data = new_data.drop_duplicates(subset=['id'], keep="first")
+        new_data_most_recent_date_raw = new_data['created_utc'].max()
+        new_data_most_recent_date_dt = datetime.strptime(new_data_most_recent_date_raw.split(' ')[0], '%Y-%m-%d').date()
+        # Adding timedelta in case there is rounding error
+        most_recent_date = max(new_data_most_recent_date_dt - timedelta(days=1), most_recent_date)
+
+        # Convert back to dataset
+        dataset["all_days"] = Dataset.from_pandas(new_data)
+    else:
+        logger.info("Creating new split 'all_days'")
+        dataset["all_days"] = Dataset.from_pandas(df)
+    # Log appending or creating split 'all'
+    logger.info("Appended or created split 'all_days'")
+
+    # Push the augmented dataset to the Hugging Face hub
+    logger.info(f"Pushing data for {date_to_fetch} to the Hugging Face hub")
+    readme_text = update_readme(dataset_name, subreddit, date_to_fetch)
+    dataset.description = readme_text
+    dataset.push_to_hub(dataset_name, token=auth_token)
+    logger.info(f"Processed and pushed data for {date_to_fetch} to the Hugging Face Hub")
+    return most_recent_date
+
+
+def run_main_continuously():
+    """
+    This function runs the given `main_function` continuously, starting from the date specified
+    in the environment variable "START_DATE" until two days ago. Once it reaches two days ago,
+    it will wait until tomorrow to start again at the same time as when it started today.
+    """
+    start_date_str = os.environ.get("START_DATE")
+    start_date = datetime.strptime(start_date_str, "%Y-%m-%d").date()
+
+    # Calculate the start time for running the main_function every day.
+    start_time = datetime.now().time()
+
+    while True:
+        today = datetime.now().date()
+        two_days_ago = today - timedelta(days=2)
+
+        if start_date <= two_days_ago:
+            logger.info(f"Running main function for date: {start_date}")
+            most_recent_date = main(str(start_date))
+            start_date = most_recent_date + timedelta(days=1)
+        else:
+            tomorrow = today + timedelta(days=1)
+            now = datetime.now()
+            start_of_tomorrow = datetime.combine(tomorrow, start_time)
+            wait_until_tomorrow = (start_of_tomorrow - now).total_seconds()
+            logger.info(f"Waiting until tomorrow: {wait_until_tomorrow} seconds")
+            time.sleep(wait_until_tomorrow)
+
+
+if __name__ == '__main__':
+    run_main_continuously()

my_logger.py

@@ -0,0 +1,22 @@
+import logging
+
+
+def setup_logger(name: str):
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.DEBUG)
+
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+    # Create a file handler to write logs to a file
+    file_handler = logging.FileHandler('mylog.log')
+    file_handler.setLevel(logging.DEBUG)
+    file_handler.setFormatter(formatter)
+    logger.addHandler(file_handler)
+
+    # Create a stream handler to write logs to the console
+    stream_handler = logging.StreamHandler()
+    stream_handler.setLevel(logging.DEBUG)
+    stream_handler.setFormatter(formatter)
+    logger.addHandler(stream_handler)
+
+    return logger

notebooks/explore.ipynb

@@ -0,0 +1,323 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "730ba509",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from IPython.core.interactiveshell import InteractiveShell\n",
+    "InteractiveShell.ast_node_interactivity = \"all\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d9acd4b6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pathlib import Path\n",
+    "import sys\n",
+    "proj_dir = Path.cwd().parent\n",
+    "\n",
+    "sys.path.append(str(proj_dir))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "62452860",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from utilities.pushshift_data import scrape_submissions_by_day, submissions_to_dataframe, get_post_count_for_day"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "a956a623",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "17df3f2812084d3591e914ffcfd948b0",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "0it [00:00, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2023-04-12 16:23:59,392 - INFO - Fetching data between timestamps 2013-02-28 20:00:00 and 2013-03-01 20:00:00\n",
+      "2023-04-12 16:24:03,524 - INFO - Fetching data between timestamps 2013-02-28 20:00:00 and 2013-03-01 14:37:16\n",
+      "2023-04-12 16:24:08,443 - INFO - Fetching data between timestamps 2013-02-28 20:00:00 and 2013-03-01 05:02:52\n",
+      "2023-04-12 16:24:13,409 - INFO - Fetching data between timestamps 2013-02-28 20:00:00 and 2013-03-01 00:43:35\n",
+      "2023-04-12 16:24:17,548 - INFO - Fetching data between timestamps 2013-02-28 20:00:00 and 2013-02-28 20:28:35\n",
+      "2023-04-12 16:24:21,490 - INFO - Fetching data between timestamps 2013-02-28 20:00:00 and 2013-02-28 20:00:48\n",
+      "2023-04-12 16:24:23,658 - INFO - Finished scraping 4106 submissions in 28.86 seconds\n"
+     ]
+    }
+   ],
+   "source": [
+    "subreddit_to_scrape = \"askreddit\"\n",
+    "day_to_scrape = \"2013-03-01\"\n",
+    "submissions = scrape_submissions_by_day(subreddit_to_scrape, day_to_scrape)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "b1cc845b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>permalink</th>\n",
+       "      <th>selftext</th>\n",
+       "      <th>url</th>\n",
+       "      <th>created_utc</th>\n",
+       "      <th>author</th>\n",
+       "      <th>num_comments</th>\n",
+       "      <th>score</th>\n",
+       "      <th>title</th>\n",
+       "      <th>id</th>\n",
+       "      <th>downs</th>\n",
+       "      <th>ups</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>/r/AskReddit/comments/19hbm0/in_the_way_that_p...</td>\n",
+       "      <td>Basically, do other parts of the world have th...</td>\n",
+       "      <td>http://www.reddit.com/r/AskReddit/comments/19h...</td>\n",
+       "      <td>2013-03-01 19:58:55</td>\n",
+       "      <td>sjr63</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>In the way that popular English and American m...</td>\n",
+       "      <td>19hbm0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>/r/AskReddit/comments/19hblp/could_i_buy_an_an...</td>\n",
+       "      <td></td>\n",
+       "      <td>http://www.reddit.com/r/AskReddit/comments/19h...</td>\n",
+       "      <td>2013-03-01 19:58:50</td>\n",
+       "      <td>WeirdPlane</td>\n",
+       "      <td>13</td>\n",
+       "      <td>1</td>\n",
+       "      <td>Could I buy an Android phone without a plan an...</td>\n",
+       "      <td>19hblp</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>/r/AskReddit/comments/19hblj/how_do_i_reddit/</td>\n",
+       "      <td>Yeah.\n",
+       "\n",
+       "How do I reddit? I don't use or read re...</td>\n",
+       "      <td>http://www.reddit.com/r/AskReddit/comments/19h...</td>\n",
+       "      <td>2013-03-01 19:58:47</td>\n",
+       "      <td>xxnovaroxgg</td>\n",
+       "      <td>14</td>\n",
+       "      <td>0</td>\n",
+       "      <td>How do I reddit</td>\n",
+       "      <td>19hblj</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>/r/AskReddit/comments/19hbjx/xpost_rsurvival_h...</td>\n",
+       "      <td>My brothers, dad and I have always been huge L...</td>\n",
+       "      <td>http://www.reddit.com/r/AskReddit/comments/19h...</td>\n",
+       "      <td>2013-03-01 19:58:07</td>\n",
+       "      <td>tuffstough</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>(x-post r/survival) Have any redditors seen Le...</td>\n",
+       "      <td>19hbjx</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>/r/AskReddit/comments/19hbjk/female_redditors_...</td>\n",
+       "      <td>I'm curious, guys tend to get asked the usual ...</td>\n",
+       "      <td>http://www.reddit.com/r/AskReddit/comments/19h...</td>\n",
+       "      <td>2013-03-01 19:57:58</td>\n",
+       "      <td>redditredditx3</td>\n",
+       "      <td>13</td>\n",
+       "      <td>2</td>\n",
+       "      <td>Female Redditors, which part of the male physi...</td>\n",
+       "      <td>19hbjk</td>\n",
+       "      <td>0</td>\n",
+       "      <td>2</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                           permalink  \\\n",
+       "0  /r/AskReddit/comments/19hbm0/in_the_way_that_p...   \n",
+       "1  /r/AskReddit/comments/19hblp/could_i_buy_an_an...   \n",
+       "2      /r/AskReddit/comments/19hblj/how_do_i_reddit/   \n",
+       "3  /r/AskReddit/comments/19hbjx/xpost_rsurvival_h...   \n",
+       "4  /r/AskReddit/comments/19hbjk/female_redditors_...   \n",
+       "\n",
+       "                                            selftext  \\\n",
+       "0  Basically, do other parts of the world have th...   \n",
+       "1                                                      \n",
+       "2  Yeah.\n",
+       "\n",
+       "How do I reddit? I don't use or read re...   \n",
+       "3  My brothers, dad and I have always been huge L...   \n",
+       "4  I'm curious, guys tend to get asked the usual ...   \n",
+       "\n",
+       "                                                 url          created_utc  \\\n",
+       "0  http://www.reddit.com/r/AskReddit/comments/19h...  2013-03-01 19:58:55   \n",
+       "1  http://www.reddit.com/r/AskReddit/comments/19h...  2013-03-01 19:58:50   \n",
+       "2  http://www.reddit.com/r/AskReddit/comments/19h...  2013-03-01 19:58:47   \n",
+       "3  http://www.reddit.com/r/AskReddit/comments/19h...  2013-03-01 19:58:07   \n",
+       "4  http://www.reddit.com/r/AskReddit/comments/19h...  2013-03-01 19:57:58   \n",
+       "\n",
+       "           author  num_comments  score  \\\n",
+       "0           sjr63             1      1   \n",
+       "1      WeirdPlane            13      1   \n",
+       "2     xxnovaroxgg            14      0   \n",
+       "3      tuffstough             0      1   \n",
+       "4  redditredditx3            13      2   \n",
+       "\n",
+       "                                               title      id  downs  ups  \n",
+       "0  In the way that popular English and American m...  19hbm0      0    1  \n",
+       "1  Could I buy an Android phone without a plan an...  19hblp      0    1  \n",
+       "2                                    How do I reddit  19hblj      0    0  \n",
+       "3  (x-post r/survival) Have any redditors seen Le...  19hbjx      0    1  \n",
+       "4  Female Redditors, which part of the male physi...  19hbjk      0    2  "
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df = submissions_to_dataframe(submissions)\n",
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "518addff",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6e5490dc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "start_date = datetime.strptime(\"2013-01-01\", \"%Y-%m-%d\")\n",
+    "start_date"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bf13555a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df[\"created_utc\"] = pd.to_datetime(df[\"created_utc\"], unit=\"s\").dt.tz_localize(\"UTC\").dt.strftime('%Y-%m-%d %H:%M:%S')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "48e413f3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9e83befa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df.dtypes"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ba84be68",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

requirements.txt

@@ -0,0 +1,9 @@
+praw==7.7.0
+gradio==3.23
+nbdev==2.3.12
+datasets==2.11.0
+requests==2.28.2
+loguru==0.7.0
+rich==13.3.4
+gradio==3.23.0
+supervisor==4.2.5

supervisord.conf

@@ -0,0 +1,20 @@
+[supervisord]
+nodaemon=true
+
+[program:main]
+command=python main.py
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autostart=true
+# autorestart=true
+
+[program:app]
+command=python app.py
+stdout_logfile=/dev/null
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autostart=true
+autorestart=true

utilities/pushshift_data.py

@@ -0,0 +1,162 @@
+import time
+from datetime import datetime, timedelta, timezone
+from typing import Any, Dict, List, Optional
+
+import pandas as pd
+import requests
+
+from my_logger import setup_logger
+
+logger = setup_logger(__name__)
+
+
+def get_pushshift_data(subreddit: str, before: Optional[int] = None,
+                       after: Optional[int] = None, aggs: Optional[str] = None) -> Optional[Dict[str, Any]]:
+    """
+    Fetch data from the Pushshift API for the specified subreddit.
+
+    :param subreddit: The name of the subreddit to scrape.
+    :param before: The upper limit for the created_utc attribute of the submissions.
+    :param after: The lower limit for the created_utc attribute of the submissions.
+    :param aggs: The aggregation summary option to use.
+    :return: A dictionary containing the fetched data and aggregations if available.
+    """
+    url = "https://api.pushshift.io/reddit/search/submission/"
+    params = {
+        "subreddit": subreddit,
+        "size": 1000,
+        "sort": "created_utc",
+        "sort_type": "desc",
+        }
+    if before is not None:
+        params["before"] = before
+    if after is not None:
+        params["after"] = after
+    if aggs is not None:
+        params["aggs"] = aggs
+
+    response = requests.get(url, params=params)
+    if response.status_code == 200:
+        return response.json()
+    else:
+        logger.error(f"Error fetching data: {response.status_code}")
+        return None
+
+
+def get_post_count_for_day(subreddit: str, day_to_scrape: str) -> int:
+    """
+    Get the total number of posts for a specific day in the specified subreddit using the Pushshift API.
+
+    :param subreddit: The name of the subreddit to get the post count for.
+    :param day_to_scrape: The date for which to get the post count (format: "YYYY-MM-DD").
+    :return: The total number of posts for the specified day.
+    """
+    date_obj = datetime.strptime(day_to_scrape, "%Y-%m-%d")
+    after = int(date_obj.timestamp())
+    before = int((date_obj + timedelta(days=1)).timestamp())
+
+    response = get_pushshift_data(subreddit, before=before, after=after, aggs="created_utc")
+    if response is not None:
+        aggs = response.get("aggs", {}).get("created_utc", [])
+        if aggs:
+            return aggs[0]["doc_count"]
+    return 0
+
+
+def fetch_data(subreddit: str, before: int, after: int) -> Optional[Dict[str, Any]]:
+    url = "https://api.pushshift.io/reddit/search/submission/"
+    params = {
+        "subreddit": subreddit,
+        "size": 1000,
+        "sort": "created_utc",
+        "sort_type": "desc",
+        "before": before,
+        "after": after,
+        }
+
+    response = requests.get(url, params=params)
+    if response.status_code == 200:
+        return response.json()
+    else:
+        logger.error(f"Error fetching data: {response.status_code}")
+        return None
+
+
+def convert_timestamp_to_datetime(timestamp: int) -> str:
+    # Convert the timestamp to a datetime object
+    datetime_obj = datetime.utcfromtimestamp(timestamp)
+
+    # Add timezone information
+    datetime_obj_utc = datetime_obj.replace(tzinfo=timezone.utc)
+
+    # Convert the datetime object to a formatted string
+    datetime_str = datetime_obj_utc.strftime('%Y-%m-%d %H:%M:%S')
+
+    return datetime_str
+
+
+def scrape_submissions_by_day(subreddit_to_scrape: str, day_to_scrape: str) -> List[Dict[str, Any]]:
+    start_time = time.time()
+    scraped_submissions = []
+    date_obj = datetime.strptime(day_to_scrape, "%Y-%m-%d")
+
+    if date_obj > datetime.now() - timedelta(days=7):
+        logger.error("The specified date might not be available in the Pushshift API yet. "
+                     "Please try an earlier date or wait for the API to be updated.")
+        return scraped_submissions
+
+    after = int(date_obj.timestamp())
+    before = int((date_obj + timedelta(days=1)).timestamp())
+
+    # todo get_post_count_for_day didnt seem to work
+    # post_count = get_post_count_for_day(subreddit_to_scrape, day_to_scrape)
+    # total_requests = (post_count + 99) // 100  # Estimate the total number of requests
+
+    actual_requests = 0
+    while after < before:
+        after_str, before_str = convert_timestamp_to_datetime(after), convert_timestamp_to_datetime(before)
+        logger.info(f"Fetching data between timestamps {after_str} and {before_str}")
+        data = get_pushshift_data(subreddit_to_scrape, before=before, after=after)
+        if data is None or len(data["data"]) == 0:
+            break
+
+        scraped_submissions.extend(data["data"])
+        before = data["data"][-1]["created_utc"]
+
+        actual_requests += 1
+        time.sleep(1)
+
+    elapsed_time = time.time() - start_time
+    if actual_requests:
+        logger.info(
+                f"{actual_requests}it [{elapsed_time // 60:02}:{elapsed_time % 60:.2f} {elapsed_time / actual_requests:.2f}s/it]")
+    logger.info(
+            f"Finished scraping {len(scraped_submissions)} submissions in {elapsed_time:.2f} seconds in {actual_requests} requests")
+    return scraped_submissions
+
+
+def submissions_to_dataframe(submissions: List[Dict[str, Any]]) -> pd.DataFrame:
+    """
+    Parse a list of submissions into a pandas DataFrame.
+
+    :param submissions: A list of dictionaries containing the scraped submission data.
+    :return: A pandas DataFrame containing the submission data.
+    """
+    cols = ['score', 'num_comments', 'title', 'permalink', 'selftext', 'url', 'created_utc', 'author', 'id',
+            'downs', 'ups']
+    df = pd.DataFrame(submissions)
+    df = df.convert_dtypes()
+    df = df[cols]
+    # Convert the "created_utc" column to a datetime column with timezone information
+    df['created_utc'] = pd.to_datetime(df['created_utc'], unit='s').dt.tz_localize('UTC').dt.strftime(
+            '%Y-%m-%d %H:%M:%S')
+    return df
+
+
+if __name__ == '__main__':
+    subreddit_to_scrape = "askreddit"
+    day_to_scrape = "2013-03-01"
+    submissions = scrape_submissions_by_day(subreddit_to_scrape, day_to_scrape)
+    df = submissions_to_dataframe(submissions)
+    print(df.head().to_string())
+    logger.info(f"Scraped {len(submissions)} submissions from r/{subreddit_to_scrape} on {day_to_scrape}")