make this extremely more advanced and complex
use reqwest::{Client, Url};
use scraper::{Html, Selector};
use std::collections::{HashMap, HashSet};
use std::error::Error;
use std::time::{Duration, Instant};
use tokio::time::sleep;
use regex::Regex;
use serde_json::json;
use futures::stream::{self, StreamExt};
use swc_common::sync::Lrc;
use swc_common::SourceMap;
use swc_ecma_ast::*;
use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax};
use swc_ecma_visit::{Visit, VisitWith};

const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36";
const RATE_LIMIT: Duration = Duration::from_millis(1000);
const MAX_CONCURRENT_REQUESTS: usize = 5;

#[derive(Debug, Clone)]
struct CrawlResult {
    url: String,
    title: Option<String>,
    links: Vec<String>,
    text_content: String,
    metadata: HashMap<String, String>,
    js_files: Vec<String>,
    images: Vec<String>,
    other_resources: Vec<String>,
}

#[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
    let start_url = "https://roblox.com";
    let max_depth = 3;
    let max_pages = 100;

    let client = Client::builder()
        .user_agent(USER_AGENT)
        .build()?;

    let mut visited = HashSet::new();
    let mut to_visit = vec![(start_url.to_string(), 0)];
    let mut results = Vec::new();

    let start_time = Instant::now();

    while let Some((url, depth)) = to_visit.pop() {
        if visited.contains(&url) || depth > max_depth || results.len() >= max_pages {
            continue;
        }

        println!("Crawling: {} (Depth: {})", url, depth);

        match crawl_page(&client, &url).await {
            Ok(mut result) => {
                visited.insert(url.clone());
                
                // Process JavaScript files
                let js_resources = process_js_files(&client, &result.js_files).await;
                result.images.extend(js_resources.images);
                result.other_resources.extend(js_resources.other_resources);

                results.push(result.clone());

                for link in &result.links {
                    if !visited.contains(link) {
                        to_visit.push((link.clone(), depth + 1));
                    }
                }

                // Store results in JSON format
                let json_result = json!({
                    "url": result.url,
                    "title": result.title,
                    "text_content": result.text_content,
                    "metadata": result.metadata,
                    "links": result.links,
                    "js_files": result.js_files,
                    "images": result.images,
                    "other_resources": result.other_resources,
                });
                std::fs::write(
                    format!("crawl_result_{}.json", results.len()),
                    serde_json::to_string_pretty(&json_result)?,
                )?;
            }
            Err(e) => eprintln!("Error crawling {}: {}", url, e),
        }

        sleep(RATE_LIMIT).await;
    }

    println!("Crawl completed in {:?}", start_time.elapsed());
    println!("Total pages crawled: {}", results.len());

    Ok(())
}

async fn crawl_page(client: &Client, url: &str) -> Result<CrawlResult, Box<dyn Error>> {
    let response = client.get(url).send().await?;
    let base_url = Url::parse(url)?;
    let body = response.text().await?;
    let document = Html::parse_document(&body);

    let title = document
        .select(&Selector::parse("title").unwrap())
        .next()
        .and_then(|title| Some(title.text().collect::<String>()));

    let text_content = document
        .select(&Selector::parse("body").unwrap())
        .next()
        .map(|body| body.text().collect::<String>())
        .unwrap_or_default();

    let metadata = extract_metadata(&document);
    let links = extract_links(&document, &base_url);
    let (js_files, images, other_resources) = extract_resources(&document, &base_url);

    Ok(CrawlResult {
        url: url.to_string(),
        title,
        links,
        text_content,
        metadata,
        js_files,
        images,
        other_resources,
    })
}

fn extract_metadata(document: &Html) -> HashMap<String, String> {
    let mut metadata = HashMap::new();
    let meta_selector = Selector::parse("meta").unwrap();

    for meta in document.select(&meta_selector) {
        if let (Some(name), Some(content)) = (meta.value().attr("name"), meta.value().attr("content")) {
            metadata.insert(name.to_string(), content.to_string());
        }
    }

    metadata
}

fn extract_links(document: &Html, base_url: &Url) -> Vec<String> {
    let link_selector = Selector::parse("a[href]").unwrap();
    let mut links = Vec::new();

    for link in document.select(&link_selector) {
        if let Some(href) = link.value().attr("href") {
            if let Ok(absolute_url) = base_url.join(href) {
                links.push(absolute_url.to_string());
            }
        }
    }

    links
}

fn extract_resources(document: &Html, base_url: &Url) -> (Vec<String>, Vec<String>, Vec<String>) {
    let js_selector = Selector::parse("script[src]").unwrap();
    let img_selector = Selector::parse("img[src]").unwrap();
    let link_selector = Selector::parse("link[href]").unwrap();

    let mut js_files = Vec::new();
    let mut images = Vec::new();
    let mut other_resources = Vec::new();

    for script in document.select(&js_selector) {
        if let Some(src) = script.value().attr("src") {
            if let Ok(absolute_url) = base_url.join(src) {
                js_files.push(absolute_url.to_string());
            }
        }
    }

    for img in document.select(&img_selector) {
        if let Some(src) = img.value().attr("src") {
            if let Ok(absolute_url) = base_url.join(src) {
                images.push(absolute_url.to_string());
            }
        }
    }

    for link in document.select(&link_selector) {
        if let Some(href) = link.value().attr("href") {
            if let Ok(absolute_url) = base_url.join(href) {
                other_resources.push(absolute_url.to_string());
            }
        }
    }

    (js_files, images, other_resources)
}

struct JsResources {
    images: Vec<String>,
    other_resources: Vec<String>,
}

async fn process_js_files(client: &Client, js_files: &[String]) -> JsResources {
    let mut images = Vec::new();
    let mut other_resources = Vec::new();

    let js_contents = stream::iter(js_files)
        .map(|url| {
            let client = client.clone();
            async move {
                match client.get(url).send().await {
                    Ok(response) => response.text().await.ok(),
                    Err(_) => None,
                }
            }
        })
        .buffer_unordered(MAX_CONCURRENT_REQUESTS)
        .collect::<Vec<_>>()
        .await;

    for js_content in js_contents.into_iter().flatten() {
        let (img, other) = parse_js_for_resources(&js_content);
        images.extend(img);
        other_resources.extend(other);
    }

    JsResources {
        images,
        other_resources,
    }
}

fn parse_js_for_resources(js_content: &str) -> (Vec<String>, Vec<String>) {
    let cm: Lrc<SourceMap> = Default::default();
    let fm = cm.new_source_file(swc_common::FileName::Anon.into(), js_content.into());

    let lexer = Lexer::new(
        Syntax::Es(Default::default()),
        Default::default(),
        StringInput::from(&*fm),
        None,
    );

    let mut parser = Parser::new_from(lexer);
    let module = parser.parse_module().ok();

    let mut url_visitor = UrlVisitor::default();
    if let Some(module) = module {
        module.visit_with(&mut url_visitor);
    }

    (url_visitor.images, url_visitor.other_resources)
}

#[derive(Default)]
struct UrlVisitor {
    images: Vec<String>,
    other_resources: Vec<String>,
}

impl Visit for UrlVisitor {
    fn visit_str(&mut self, s: &swc_ecma_ast::Str) {
        if s.value.starts_with("http") || s.value.starts_with("//") {
            if s.value.ends_with(".jpg") || s.value.ends_with(".png") || s.value.ends_with(".gif") || s.value.ends_with(".svg") {
                self.images.push(s.value.to_string());
            } else {
                self.other_resources.push(s.value.to_string());
            }
        }
    }
}

// The extract_data_with_regex function remains unchanged
fn extract_data_with_regex(text: &str, patterns: &[(&str, &str)]) -> HashMap<String, String> {
    let mut extracted_data = HashMap::new();

    for (key, pattern) in patterns {
        let re = Regex::new(pattern).unwrap();
        if let Some(captures) = re.captures(text) {
            if let Some(matched) = captures.get(1) {
                extracted_data.insert(key.to_string(), matched.as_str().to_string());
            }
        }
    }

    extracted_data
}