import requests
from bs4 import BeautifulSoup
from flask import Flask, Response
from feedgen.feed import FeedGenerator
from datetime import datetime
import pytz
import re
import time

# Function to process comments and their replies
def process_comment(comment, work_title, fg):
    # Extract relevant information from the comment
    comment_id = comment.get("id")
    if comment_id is None:
        return  # Skip comments without an id

    comment_author_elem = comment.find("a", href=True)
    comment_author = comment_author_elem.text.strip() if comment_author_elem is not None else "Unknown"
    comment_text_elem = comment.find("blockquote", class_="userstuff")
    if comment_text_elem is not None:
        comment_text = comment_text_elem.text.strip()
    else:
        return  # Skip comments without any text

    # Extract chapter information if available
    chapter_info_elem = comment.find("span", class_="parent")
    chapter_info = chapter_info_elem.text.strip() if chapter_info_elem is not None else "Unknown Chapter"

    # Extract date and time information
    date_elem = comment.find("span", class_="posted").find("abbr", class_="day")
    day = date_elem.text.strip() if date_elem is not None else "Unknown"
    date_elem = comment.find("span", class_="posted").find("span", class_="date")
    date = date_elem.text.strip() if date_elem is not None else "Unknown"
    date_elem = comment.find("span", class_="posted").find("abbr", class_="month")
    month = date_elem.text.strip() if date_elem is not None else "Unknown"
    date_elem = comment.find("span", class_="posted").find("span", class_="year")
    year = date_elem.text.strip() if date_elem is not None else "Unknown"
    date_elem = comment.find("span", class_="posted").find("span", class_="time")
    time_str = date_elem.text.strip() if date_elem is not None else "Unknown"

    # Create an RSS entry for the comment with chapter information and date
    entry = fg.add_entry()
    entry.title(f"{work_title} - [{chapter_info}] Comment by {comment_author}")
    entry.description(comment_text)

    # Generate URL for the comment
    comment_url = f"https://archiveofourown.org/comments/{comment_id.split('_')[1]}" if '_' in comment_id else ""

    # Set the GUID value for the comment entry
    entry.guid(comment_url, permalink=True)

    # Combine date and time strings into a single string
    datetime_str = f"{day} {date} {month} {year} {time_str}"

    # Parse the datetime string into a timezone-aware datetime object
    dt = datetime.strptime(datetime_str, "%a %d %b %Y %I:%M%p")

    # Convert the datetime object to UTC if it's not already in UTC
    if dt.tzinfo is None or dt.tzinfo.utcoffset(dt) is None:
        dt = pytz.utc.localize(dt)
    else:
        dt = dt.astimezone(pytz.utc)

    formatted_dt = dt.strftime("%a, %d %b %Y %H:%M:%S %z")
    entry.pubDate(formatted_dt)


# Function to handle rate limiting (429 errors)
def handle_rate_limiting(response):
    if response.status_code == 429:
        retry_after = int(response.headers.get("Retry-After", 1))
        print(f"Rate limited. Retrying after {retry_after} seconds...")
        time.sleep(retry_after)
        raise Exception("Rate limited. Stopping the script.")


# Function to extract title and work ID with retry
def extract_title_with_retry(url):
    max_retries = 3
    retries = 0
    while retries < max_retries:
        response = requests.get(url)
        handle_rate_limiting(response)

        soup = BeautifulSoup(response.content, "html.parser")
        title_elem = soup.find("title")
        if title_elem:
            return title_elem.text.strip()

        retries += 1
        print(f"Title extraction failed. Retrying... Attempt #{retries}")

    return "Unknown"  # Return default value if extraction fails after maximum retries


# Flask app initialization
app = Flask(__name__)

@app.route("/<work_id>")
def generate_rss_feed(work_id):
    # Check if the work ID is in the correct format (8-digit number)
    if not re.match(r"^\d{8}$", work_id):
        return "Invalid work ID format. Please enter an 8-digit number."

    # Create the feed generator object
    fg = FeedGenerator()

    # Rest of the code remains the same
    base_url = f"https://archiveofourown.org/works/{work_id}"
    urls = [base_url]

    # Process each URL and create RSS feed
    for url in urls:
        # Extract the title and work ID from the work page URL
        work_url = url + "?view_adult=true&view_full_work=true"
        work_title = extract_title_with_retry(work_url)

        # Set the title of the RSS feed
        fg.title(f"{work_title} - AO3 Comments RSS Feed")

        # Rest of the code remains the same
        fg.link(href=base_url, rel='alternate')
        fg.description('RSS feed for AO3 comments')

        # Construct the base URL for comments
        base_url = url + "/comments?page={}&show_comments=true&view_full_work=true&view_adult=true#comments"

        # Extract the title and work ID from the work page URL
        work_url = url + "?view_adult=true&view_full_work=true"
        work_title = extract_title_with_retry(work_url)
        work_id = work_url.split("/")[-1].split("?")[0]  # Remove the query parameters from the work ID

        # Start with the first page
        page = 1
        has_comments = True
        last_page = 1  # Initialize last_page outside the loop

        while has_comments:
            # Construct the URL for the current page
            url = base_url.format(page)

            # Send a GET request to the URL
            response = requests.get(url)

            # Check if rate limited and handle it
            handle_rate_limiting(response)

            # Create a BeautifulSoup object from the response content
            soup = BeautifulSoup(response.content, "html.parser")

            # Find the comments section on the page
            comments_section = soup.find("div", id="comments_placeholder")

            # Check if comments section exists
            if comments_section is not None:
                # Find all individual comments
                comments = comments_section.find_all("li")

                # Loop through each comment and process it
                for comment in comments:
                    process_comment(comment, work_title, fg)  # Pass the respective fg instance

                # Find the pagination links
                pagination = comments_section.find("ol", class_="pagination actions")

                # Check if there are multiple pages of comments
                if pagination is not None:
                    # Find the last page number
                    last_page = int(pagination.find_all("li")[-2].text)

                # Update page after processing the current page
                page += 1

            else:
                print("Comments section not found on the current page.")
                break  # Stop the script if comments section not found

            # Update has_comments based on the presence of more pages
            has_comments = page <= last_page

    # Generate RSS feed XML
    response = Response(fg.rss_str(pretty=True), mimetype='application/xml')
    return response


if __name__ == '__main__':
    app.run(port=4329)

this is what i want the script to do:  want it to extract the number of comments the fic has and saves it as a variable to compare to, so that only when it increases, does it do a full scan of all the comment pages. the current fic i'm looking at has 621 comments currently. if that number goes up, i want the script to scrape all the comments. if it stays the same, no scraping should happen. every time the rss is refreshed (every 5 minutes is the current setting in my rss reader), the only requests being sent out are for the title extraction and comment count extraction to compare to the number to the previous run. if it increases, only THEN scrape all the comments

the comment count scraping should be done on the same function and the same url that's used to extract the title. please post the full modified code with nothing cut.


here's the structure for the the part that shows the number of comments: 

<dl class="stats"><div class="published"><dt class="published">Published:</dt><dd class="published">2023-01-16</dd></div><div class="status"><dt class="status">Updated:</dt><dd class="status">2023-06-22</dd></div><div class="words"><dt class="words">Words:</dt><dd class="words" data-ao3e-original="57,715">57 715</dd></div><div class="AO3E"><dt class="reading-time AO3E">Reading time:</dt><dd class="reading-time AO3E">4 hours, 48 mins</dd></div><div class="AO3E"><dt class="finish-at AO3E">Finish reading at:</dt><dd class="finish-at AO3E"><span title="Using current time of 13:25. Click to update." aria-label="18:13. Using current time of 13:25. Click to update." tabindex="0" role="button"><span class="time">18:13</span><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" class="AO3E--icon" role="img"><path fill="currentColor" d="M17.65,6.35C16.2,4.9 14.21,4 12,4A8,8 0 0,0 4,12A8,8 0 0,0 12,20C15.73,20 18.84,17.45 19.73,14H17.65C16.83,16.33 14.61,18 12,18A6,6 0 0,1 6,12A6,6 0 0,1 12,6C13.66,6 15.14,6.69 16.22,7.78L13,11H20V4L17.65,6.35Z"></path></svg></span></dd></div><div class="chapters"><dt class="chapters">Chapters:</dt><dd class="chapters" data-ao3e-original="18/?">18/?</dd></div><div class="comments"><dt class="comments">Comments:</dt><dd class="comments" data-ao3e-original="621">621</dd></div><div class="kudos"><dt class="kudos">Kudos:</dt><dd class="kudos" data-ao3e-original="1,055">1 055</dd></div><div class="bookmarks"><dt class="bookmarks">Bookmarks:</dt><dd class="bookmarks"><a href="/works/44300110/bookmarks" data-ao3e-original="103">103</a></dd></div><div class="hits"><dt class="hits">Hits:</dt><dd class="hits" data-ao3e-original="19,841">19 841</dd></div><div class="AO3E"><dt class="kudos-hits-ratio AO3E">Kudos/Hits:</dt><dd class="kudos-hits-ratio AO3E">5.32%</dd></div></dl>