import requests
import re
import sys
from collections import deque, defaultdict
from urllib.parse import unquote

# Install 'mwparserfromhell' for parsing the wiki text
# You can install it via pip: pip install mwparserfromhell
import mwparserfromhell

# Step 1: Gather all song titles from the specified categories
def get_song_titles():
    categories = [
        "Category:NoCopyrightSounds_songs",
        "Category:NCS_Arcade_songs",
        "Category:Removed_songs"
    ]
    base_url = "https://nocopyrightsounds.fandom.com"

    song_titles = set()

    # Helper function to fetch pages from a category
    def fetch_category_members(category):
        url = f"{base_url}/api.php"
        params = {
            "action": "query",
            "list": "categorymembers",
            "cmtitle": category,
            "cmlimit": "500",
            "format": "json",
            "continue": ""
        }
        while True:
            response = requests.get(url, params=params).json()
            members = response.get('query', {}).get('categorymembers', [])
            for member in members:
                song_titles.add(member['title'])
            if 'continue' in response:
                params.update(response['continue'])
            else:
                break

    # Fetch songs from categories
    for category in categories:
        fetch_category_members(category)

    # Handle SoundCloud promotions
    # Fetch the page content of "List of NCS SoundCloud promotions by date"
    promotions_url = f"{base_url}/api.php"
    params = {
        "action": "parse",
        "page": "List of NCS SoundCloud promotions by date",
        "prop": "wikitext",
        "format": "json"
    }
    response = requests.get(promotions_url, params=params).json()
    wikitext = response['parse']['wikitext']['*']
    wikicode = mwparserfromhell.parse(wikitext)
    # Now parse the table and extract songs with 🟡 emoji
    yellow_circle = "🟡"
    for table in wikicode.filter_tags(matches=lambda node: node.tag == 'table'):
        for tr in table.contents.filter_tags(matches=lambda node: node.tag == 'tr'):
            td_list = tr.contents.filter_tags(matches=lambda node: node.tag == 'td')
            if len(td_list) >= 2:
                artist_td = str(td_list[0])
                song_td = str(td_list[1])
                if yellow_circle in artist_td:
                    # Extract song title
                    song_title = mwparserfromhell.parse(song_td).strip_code().strip()
                    if song_title:
                        song_titles.add(song_title)

    return list(song_titles)

# Step 2: For each song, get the lyrics
def get_song_lyrics(song_titles):
    base_url = "https://nocopyrightsounds.fandom.com"
    song_lyrics = {}
    no_lyrics_songs = set()

    for title in song_titles:
        # Fetch the page wikitext
        page_url = f"{base_url}/api.php"
        params = {
            "action": "parse",
            "page": title,
            "prop": "wikitext",
            "format": "json"
        }
        response = requests.get(page_url, params=params).json()
        if 'error' in response:
            continue  # Page doesn't exist
        wikitext = response['parse']['wikitext']['*']
        wikicode = mwparserfromhell.parse(wikitext)
        # Extract lyrics from the 'Lyrics' template
        lyrics = None
        for template in wikicode.filter_templates():
            if template.name.strip().lower() == 'lyrics':
                lyrics = template.get(1).value.strip_code().strip()
                break
        if lyrics:
            song_lyrics[title] = lyrics
        else:
            no_lyrics_songs.add(title)

    return song_lyrics, no_lyrics_songs

# Step 3: Normalize song titles and lyrics
def normalize_text(text):
    # Convert to lowercase
    text = text.lower()
    # Replace '&' with 'and' and vice versa
    text = text.replace('&', 'and')
    # Replace numbers with words and vice versa
    numbers = {
        '0': 'zero',
        '1': 'one',
        '2': 'two',
        '3': 'three',
        '4': 'four',
        '5': 'five',
        '6': 'six',
        '7': 'seven',
        '8': 'eight',
        '9': 'nine'
    }
    for num, word in numbers.items():
        text = text.replace(num, word)
    # Remove punctuation
    text = re.sub(r'[^\w\s]', '', text)
    # Handle UK/US spelling differences (simplified example)
    uk_us_spelling = {
        'colour': 'color',
        'flavour': 'flavor',
        'honour': 'honor',
        'labour': 'labor',
        'centre': 'center',
        'theatre': 'theater',
        'metre': 'meter',
        'defence': 'defense',
        'licence': 'license',
        'offence': 'offense',
        'pretence': 'pretense',
        'analyse': 'analyze',
        'realise': 'realize',
        'recognise': 'recognize',
        'paralyse': 'paralyze',
        'dialogue': 'dialog',
        'catalogue': 'catalog',
        'programme': 'program',
        'aeroplane': 'airplane',
        'cheque': 'check',
        'jewellery': 'jewelry',
        'plough': 'plow',
        'savoury': 'savory',
        'tyre': 'tire',
        'aluminium': 'aluminum',
        'favour': 'favor',
        'humour': 'humor',
        'labourer': 'laborer',
        'mould': 'mold',
        'sceptical': 'skeptical',
        'specialise': 'specialize',
        'theorise': 'theorize',
        'brexit': 'brexit'
    }
    for uk, us in uk_us_spelling.items():
        text = text.replace(uk, us)
    # Normalize whitespace
    text = ' '.join(text.split())
    return text

def normalize_songs_and_lyrics(song_lyrics):
    normalized_song_titles = {}
    for title in song_lyrics.keys():
        normalized_title = normalize_text(title)
        normalized_song_titles[normalized_title] = title

    normalized_lyrics = {}
    for title, lyrics in song_lyrics.items():
        normalized_lyrics[title] = normalize_text(lyrics)
    return normalized_song_titles, normalized_lyrics

# Step 4: Build a mapping from song titles to referenced songs
def build_song_graph(normalized_song_titles, normalized_lyrics):
    song_graph = defaultdict(list)
    song_titles_set = set(normalized_song_titles.keys())

    for title, lyrics in normalized_lyrics.items():
        normalized_title = normalize_text(title)
        # Search for song titles in the lyrics
        for song_title in song_titles_set:
            if song_title != normalized_title and song_title in lyrics:
                original_song_title = normalized_song_titles[song_title]
                song_graph[title].append(original_song_title)
    return song_graph

# Step 5: Implement the chain-finding algorithms
def find_longest_chain_first_reference(song_graph, song_lyrics):
    longest_chain = []
    visited_global = set()

    for start_song in song_lyrics.keys():
        chain = []
        visited = set()
        current_song = start_song

        while True:
            if current_song in visited or current_song not in song_graph:
                break
            visited.add(current_song)
            chain.append(current_song)
            visited_global.add(current_song)

            references = song_graph[current_song]
            if not references:
                break
            # Consider only the first referenced song
            next_song = references[0]
            current_song = next_song
        if len(chain) > len(longest_chain):
            longest_chain = chain
    return longest_chain

def find_longest_chain_any_reference(song_graph, song_lyrics):
    longest_chain = []
    visited_global = set()

    for start_song in song_lyrics.keys():
        stack = [(start_song, [start_song], set([start_song]))]

        while stack:
            current_song, path, visited = stack.pop()
            references = song_graph.get(current_song, [])
            extended = False
            for next_song in references:
                if next_song not in visited:
                    extended = True
                    stack.append((next_song, path + [next_song], visited | set([next_song])))
            if not extended:
                if len(path) > len(longest_chain):
                    longest_chain = path
    return longest_chain

# Step 6: Main execution
def main():
    print("Fetching song titles...")
    song_titles = get_song_titles()
    print(f"Found {len(song_titles)} songs.")

    print("Fetching song lyrics...")
    song_lyrics, no_lyrics_songs = get_song_lyrics(song_titles)
    print(f"Retrieved lyrics for {len(song_lyrics)} songs.")
    print(f"{len(no_lyrics_songs)} songs have no lyrics.")

    print("Normalizing song titles and lyrics...")
    normalized_song_titles, normalized_lyrics = normalize_songs_and_lyrics(song_lyrics)

    print("Building song graph...")
    song_graph = build_song_graph(normalized_song_titles, normalized_lyrics)
    print("Song graph constructed.")

    print("Finding the longest chain (first reference)...")
    longest_chain_first = find_longest_chain_first_reference(song_graph, song_lyrics)
    print("Longest chain considering only the first referenced song:")
    print(" -> ".join(longest_chain_first))
    print(f"Length of chain: {len(longest_chain_first)}")

    print("\nFinding the longest chain (any reference)...")
    longest_chain_any = find_longest_chain_any_reference(song_graph, song_lyrics)
    print("Longest chain considering any referenced song:")
    print(" -> ".join(longest_chain_any))
    print(f"Length of chain: {len(longest_chain_any)}")

if __name__ == "__main__":
    main()



I have this program here. I need to add a function that saves all scraped information locally for future use, to save time and also to avoid frequently sending requests to the servers of the scraped site.