import pandas as pd
import requests
import xml.etree.ElementTree as ET
import streamlit as st
import streamlit.components.v1 as components
import logging
import scrapy
from scrapy.crawler import CrawlerProcess
import sys
import advertools as adv
import json

page_title='Sitemap Analyzer'

# streamlit
logo_file = ''
st.set_page_config(
    '{}'.format(page_title),
    '{}'.format(logo_file),
    layout='wide',
    initial_sidebar_state='auto',
    menu_items={
        "About": "{}".format(page_title),
    },
)

def faq():
    st.header("Frequently Asked Questions")
    st.markdown(
        """
    1. [What is a sitemap?](#what-is-a-sitemap)
    2. [How can I get started?](#how-can-i-get-started)
    3. [Who is behind this project?](#who-is-behind-this-project)
    4. [How do I report a bug or issue?](#how-do-i-report-a-bug-or-issue)
    
    ---
    
    ### What is a sitemap? 
    
    A sitemap is a file that lists the links of a collection of webpages or Youtube videos, which helps chatbot builders like CustomGPT discover and index the video content. Using a sitemap collection, you can build your own ChatGPT based on what is being said in the videos.
    
    [Back to top](#frequently-asked-questions)
    
    ### How can I get started?
    
    1. Just enter the link to the Youtube channel. 
    2. This app will then create a dynamic sitemap based on all the videos in that channel. 
    3. Copy-paste the sitemap into CustomGPT to build your customized ChatGPT chatbot based on whats being said in the videos. 
    
    [Back to top](#frequently-asked-questions)
    
    ### Who is behind this project?
    
    This free tool was created by [CustomGPT](https://customgpt.ai/).
    
    [Back to top](#frequently-asked-questions)
    
    ### How do I report a bug or issue?
    
    Please [contact us](https://customgpt.ai/contact-us/)
    
    [Back to top](#frequently-asked-questions)
    """
    )


def instructions():
    st.header("Instructions")
    st.markdown(
        """
    1. Enter the link to the Youtube channel.
    2. Click the 'Generate Sitemap' button.
    3. Copy the sitemap link and use it in [CustomGPT](https://customgpt.ai)
    """
    )

logging.basicConfig(filename='log.txt', level=logging.DEBUG)

# Function to estimate the size of a webpage using the HEAD method
def estimate_size(url):
    try:
        response = requests.head(url)
        size = int(response.headers.get('Content-Length'))
        return size
    except:
        logging.error(f"Failed to estimate size of {url}")
        return 28000

# Function to crawl a webpage and get its actual size using the HEAD method
class MySpider(scrapy.Spider):
    name = 'myspider'

    def start_requests(self):
        yield scrapy.Request(self.start_urls[0], method='HEAD')

    def parse(self, response):
        print('VISITED: %s' % response.url)
        if 'Content-Length' in response.headers:
            size = int(response.headers['Content-Length'].decode('utf-8'))
            print('Size %d bytes' % size)
        else:
            print('Content-Length not found in the response headers')

def crawl_size(url):
    try:
        process = CrawlerProcess(settings={
            'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
            'LOG_LEVEL': 'DEBUG'
        })
        process.crawl(MySpider, start_urls=[url])
        process.start()
        return size
    except:
        logging.error(f"Failed to crawl size of {url}")
        return None

# Function to parse a sitemap and return a list of URLs
def sitemap_parse(sitemap_url):
    try:
        adv.spider.user_agent = 'CustomGPT <PRESIDIO_ANONYMIZED_EMAIL_ADDRESS>'
        df = adv.sitemap_to_df(sitemap_url)
        urls = df['loc'].tolist()
        return urls
    except:
        logging.error(f"Failed to parse sitemap {sitemap_url}")
        return []

# Function to calculate the size of all pages in a sitemap
def calculate_sizes(sitemap_url, method):
    urls = sitemap_parse(sitemap_url)
    num_urls = len(urls)
    sizes = []
    if method == "Estimated Method (recommended)":
        st.write("Estimating sizes...")
        for i in range(min(num_urls, 10)):
            size = estimate_size(urls[i])
            sizes.append(size)
            st.progress((i+1)/min(num_urls, 10))
        avg_size = sum(sizes) / len(sizes)
        sizes = [avg_size] * num_urls
    else:
        st.write("Crawling sizes...")
        if num_urls <= 1000:
            for i in range(num_urls):
                size = crawl_size(urls[i])
                sizes.append(size)
                st.progress((i+1)/num_urls)
        else:
            for i in range(1000):
                size = crawl_size(urls[i])
                sizes.append(size)
                st.progress((i+1)/1000)
            avg_size = sum(sizes) / len(sizes)
            sizes += [avg_size] * (num_urls - 1000)
    return sizes

# Function to generate the output table
def generate_table(sitemap_url, method):
    sizes = calculate_sizes(sitemap_url, method)
    data = {"URL": [], "Size": [], "Size Calculation Method": [], "Metadata": []}
    for i, url in enumerate(sitemap_parse(sitemap_url)):
        data["URL"].append(url)
        data["Size"].append(sizes[i])
        data["Size Calculation Method"].append(method)
        data["Metadata"].append("") # Add metadata here if available
    df = pd.DataFrame(data)
    return df

# Main function to run the app
def main():
    st.sidebar.title('Navigation')
    page = st.sidebar.radio('Go to', ['Home', 'Instructions', 'FAQ', 'Reset'])
    
    if page == 'Home':
        st.header("{}".format(page_title))
        st.info("This is a Streamlit app to analyze a sitemap and give information about it.")
        sitemap_url = st.text_input("Sitemap URL")
        method = st.radio("Data Calculation Method", ["Estimated Method (recommended)", "Actual Crawl Method (slow)"])
        if st.button("Analyze"):
            st.write("Analyzing sitemap...")
            df = generate_table(sitemap_url, method)
            st.write(f"Is sitemap valid? {len(df) > 0}")
            st.write(f"Number of pages detected in the sitemap: {len(df)}")
            st.write(f"Estimated size of all pages: {sum(df['Size'].fillna(0))} bytes")
            st.write("Data from Sitemap and calculations:")
            st.write(df)
            
            components.html(f"""
            <button id="toggleButton">Show log</button>
            <div id="log" style="display: none;">
            </div>

           <script>
                // Parse the log data from JSON
                var logData = [];

                var logDiv = document.getElementById("log");
                var button = document.getElementById("toggleButton");

                for (var i = 0; i < logData.length; i++) {{
                    var para = document.createElement("p");
                    var node = document.createTextNode(logData[i]);
                    para.appendChild(node);

                    logDiv.appendChild(para);
                }}

                button.addEventListener('click', function() {{
                    if (logDiv.style.display === "none") {{
                        logDiv.style"display = "block";
                        button.textContent = "Hide log";
                    }} else {{
                        logDiv.style.display = "none";
                        button.textContent = "Show log";
                    }}
                }});
            </script>
            """, height=800, scrolling=True)
    elif page == 'Instructions':
        instructions()
    elif page == 'FAQ':
        st.title('FAQ')
        faq()
    elif page == 'Reset':
        st_refresh.st_refresh()
   



if __name__ == "__main__":
    main()

fix the logging and the actual crawl flow it doesnt seem to work always getting exception failed to crawl