archive/subreddit_downloader.py

import csv
import json
import sys
import time
import traceback
from datetime import datetime

import requests

username = ""  # put the username you want to download in the quotes
subreddit = "BestofRedditorUpdates"  # put the subreddit you want to download in the quotes
thread_id = ""  # put the id of the thread you want to download in the quotes, it's the first 5 to 7 character string of letters and numbers from the url, like 107xayi
# leave either one blank to download an entire user's or subreddit's history
# or fill in both to download a specific users history from a specific subreddit

# change this to one of "human", "csv" or "json"
# - human: the score, creation date, author, link and then the comment/submission body on a second line. Objects are separated by lines of dashes
# - csv: a comma seperated value file with the fields score, date, title, author, link and then body or url
# - json: the full json object
output_format = "csv"

# default start time is the current time and default end time is all history
# you can change out the below lines to set a custom start and end date. The script works backwards, so the end date has to be before the start date
# start_time = datetime.utcnow()  # datetime.strptime("10/05/2021", "%m/%d/%Y")
start_time = datetime.strptime("04/02/2023", "%m/%d/%Y")
end_time = None  # datetime.strptime("09/25/2021", "%m/%d/%Y")

convert_to_ascii = False  # don't touch this unless you know what you're doing
convert_thread_id_to_base_ten = True  # don't touch this unless you know what you're doing


def write_csv_line(writer, obj, is_submission):
    output_list = []
    output_list.append(str(obj['score']))
    output_list.append(datetime.fromtimestamp(obj['created_utc']).strftime("%Y-%m-%d"))
    if is_submission:
        output_list.append(obj['title'])
    output_list.append(f"u/{obj['author']}")
    output_list.append(f"https://www.reddit.com{obj['permalink']}")
    if is_submission:
        if obj['is_self']:
            if 'selftext' in obj:
                output_list.append(obj['selftext'])
            else:
                output_list.append("")
        else:
            output_list.append(obj['url'])
    else:
        output_list.append(obj['body'])
    writer.writerow(output_list)


def write_json_line(handle, obj):
    handle.write(json.dumps(obj))
    handle.write("\n")


def download_from_url(filename, url_base, output_format, start_datetime, end_datetime, is_submission, convert_to_ascii):
    print(f"Saving to {filename}")

    count = 0
    if output_format == "human" or output_format == "json":
        if convert_to_ascii:
            handle = open(filename, 'w', encoding='ascii')
        else:
            handle = open(filename, 'w', encoding='UTF-8')
    else:
        handle = open(filename, 'w', encoding='UTF-8', newline='')
        writer = csv.writer(handle)

    previous_epoch = int(start_datetime.timestamp())
    break_out = False
    while True:
        new_url = url_base + str(previous_epoch)
        json_text = requests.get(new_url, headers={'User-Agent': "Post downloader by /u/Watchful1"})
        time.sleep(1)  # pushshift has a rate limit, if we send requests too fast it will start returning error messages
        try:
            json_data = json_text.json()
        except json.decoder.JSONDecodeError:
            time.sleep(1)
            continue

        if 'data' not in json_data:
            break
        objects = json_data['data']
        if len(objects) == 0:
            break

        for obj in objects:
            previous_epoch = obj['created_utc'] - 1
            if end_datetime is not None and datetime.utcfromtimestamp(previous_epoch) < end_datetime:
                break_out = True
                break
            count += 1
            try:
                if output_format == "csv":
                    write_csv_line(writer, obj, is_submission)
                elif output_format == "json":
                    write_json_line(handle, obj)
            except Exception as err:
                if 'permalink' in obj:
                    print(f"Couldn't print object: https://www.reddit.com{obj['permalink']}")
                else:
                    print(f"Couldn't print object, missing permalink: {obj['id']}")
                print(err)
                print(traceback.format_exc())

        if break_out:
            break

        print(f"Saved {count} through {datetime.fromtimestamp(previous_epoch).strftime('%Y-%m-%d')}")

    print(f"Saved {count}")
    handle.close()


if __name__ == "__main__":
    filter_string = None
    if username == "" and subreddit == "" and thread_id == "":
        print("Fill in username, subreddit or thread id")
        sys.exit(0)
    if output_format not in ("human", "csv", "json"):
        print("Output format must be one of human, csv, json")
        sys.exit(0)

    filters = []
    if username:
        filters.append(f"author={username}")
    if subreddit:
        filters.append(f"subreddit={subreddit}")
    if thread_id:
        if convert_thread_id_to_base_ten:
            filters.append(f"link_id={int(thread_id, 36)}")
        else:
            filters.append(f"link_id=t3_{thread_id}")
    filter_string = '&'.join(filters)

    url_template = "https://api.pushshift.io/reddit/{}/search?limit=1000&order=desc&{}&before="

    if not thread_id:
        download_from_url("posts.txt", url_template.format("submission", filter_string), output_format, start_time,
                          end_time, True, convert_to_ascii)
    # download_from_url("comments.txt", url_template.format("comment", filter_string), output_format, start_time,
    #                   end_time, False, convert_to_ascii)