import os from datetime import datetime from typing import Any, Dict, List import praw from utilities.my_logger import setup_logger # Setup logging logger = setup_logger(__name__) # Get subreddit subreddit_var = os.getenv("SUBREDDIT") reddit_pull_limit = int(os.getenv("REDDIT_PULL_LIMIT")) # Dummy row for when we create a new repo dummy_data = { "id": ['id'], "content": ["This is a sample post content. Just for demonstration purposes!"], "poster": ["sampleUser123"], "date_utc": [datetime.strptime("2023-10-26 14:30:45", '%Y-%m-%d %H:%M:%S')], "flair": ["Discussion"], "title": ["Sample Post Title: How to Use Hugging Face?"], "score": [457], "permalink": ["/r/sampleSubreddit/comments/sampleID/sample_post_title_how_to_use_hugging_face/"], "updated": False, "new": False, } def get_reddit_instance() -> praw.Reddit: """Initialize and return a Reddit instance using PRAW.""" return praw.Reddit( client_id=os.getenv('REDDIT_CLIENT_ID'), client_secret=os.getenv('REDDIT_CLIENT_SECRET'), user_agent=os.getenv('REDDIT_USER_AGENT'), ratelimit_seconds=20, ) def extract_submission_data(submission: praw.models.Submission) -> Dict[str, Any]: """Extract and return relevant data from a given Reddit submission.""" return { "content": submission.selftext, "poster": str(submission.author), "date_utc": datetime.utcfromtimestamp(submission.created_utc).strftime('%Y-%m-%d %H:%M:%S'), "flair": submission.link_flair_text, "title": submission.title, "score": submission.ups, "permalink": submission.permalink, } def praw_downloader() -> List[Dict[str, str]]: """Main function to extract and save all submissions from the subreddit.""" reddit = get_reddit_instance() subreddit = reddit.subreddit(subreddit_var) logger.info(f'Starting to fetch submissions from {os.getenv("SUBREDDIT")}.') submissions = [] for submission in subreddit.new(limit=reddit_pull_limit): # Set limit=None to get all posts # logger.debug(f'Processing post {submission.id} - {submission.title}') data = extract_submission_data(submission) submissions.append(data) logger.info(f'Finished downloading {len(submissions)} submissions.') return submissions if __name__ == "__main__": praw_downloader()