Web Crawler Enhanced Pro Optimized v6.1.1
// @namespace http://tampermonkey.net/
// @version 6.1.1
// @description Optimized and enhanced version of Advanced Web Crawler Enhanced Pro with improved error handling, asynchronous processing, performance optimizations, and enhanced user experience.
// @author Your Name
// @match https://www.google.com/search*
// @match https://www.bing.com/search*
// @match https://search.yahoo.com/search*
// @match https://duckduckgo.com/*
// @match https://www.baidu.com/s*
// @match https://www.yandex.com/search/*
// @match https://search.naver.com/search*
// @match https://search.daum.net/search*
// @match https://www.ask.com/*
// @match https://www.ecosia.org/search*
// @match https://www.startpage.com/*
// @match https://searx.me/*
// @match https://www.dogpile.com/*
// @match https://www.bing.com/images*
// @grant GM_xmlhttpRequest
// @grant GM_setValue
// @grant GM_getValue
// @grant GM_deleteValue
// @connect *
// @require https://cdnjs.cloudflare.com/ajax/libs/jquery/3.6.0/jquery.min.js
// @require https://cdn.jsdelivr.net/npm/chart.js
// @require https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.18.5/xlsx.full.min.js
// ==/UserScript==

(function() {
'use strict';


// Configuration Module
const Config = (function() {
    const DEFAULT_SETTINGS = {
        delay: 2000,
        delayRandom: 500,
        maxPages: 3,
        userAgents: [
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15',
            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)',
            'Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15',
            'Mozilla/5.0 (iPad; CPU OS 14_0 like Mac OS X) AppleWebKit/605.1.15',
            'AdvancedWebCrawler/6.1 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
        ],
        userAgentIndex: 0,
        blacklist: [],
        whitelist: [],
        contentTypes: {
            include: ['text/html'],
            exclude: ['application/pdf', 'image/jpeg', 'image/png', 'video/mp4']
        },
        timeout: 8000,
        includeKeywords: [],
        excludeKeywords: [],
        language: [],
        minWordCount: 100,
        maxWordCount: 10000,
        maxDepth: 2,
        proxy: '',
        regexInclude: [],
        regexExclude: [],
        dataFields: {
            name: true,
            url: true,
            description: true,
            html: true,
            category: true,
            quality: true,
            language: true,
            wordCount: true,
            metaTags: false,
            headings: false
        },
        schedule: {
            enabled: false,
            startTime: '00:00',
            interval: 3600000
        },
        searchEngines: {
            google: true,
            bing: true,
            yahoo: true,
            duckduckgo: true,
            baidu: true,
            yandex: true,
            naver: true,
            daum: true,
            ask: true,
            ecosia: true,
            startpage: true,
            searx: true,
            dogpile: true,
            bingimages: true
        },
        retryAttempts: 3,
        retryDelay: 2000,
        respectRobots: true,
        maxConcurrentRequests: 3,
        dataVisualization: true,
        exportFormats: {
            csv: true,
            json: true,
            xml: true,
            xls: true
        },
        advancedLogging: true,
        enableCaptchas: false,
        customKeywords: {}
    };

    const settings = Object.assign({}, DEFAULT_SETTINGS, getSavedSettings());

    function getSavedSettings() {
        const savedSettings = {};
        Object.keys(DEFAULT_SETTINGS).forEach(key => {
            savedSettings[key] = GM_getValue(key, DEFAULT_SETTINGS[key]);
        });
        return savedSettings;
    }

    function saveSettings(newSettings) {
        Object.keys(newSettings).forEach(key => {
            GM_setValue(key, newSettings[key]);
        });
    }

    function resetSettings() {
        Object.keys(DEFAULT_SETTINGS).forEach(key => {
            GM_setValue(key, DEFAULT_SETTINGS[key]);
        });
        Object.assign(settings, DEFAULT_SETTINGS);
    }

    return {
        DEFAULT_SETTINGS,
        settings,
        saveSettings,
        resetSettings
    };
})();

// Logger Module
const Logger = (function() {
    const logs = new Set(JSON.parse(GM_getValue('logs', '[]')));

    function log(message, type = 'info') {
        const timestamp = new Date().toLocaleString();
        logs.add(JSON.stringify({ timestamp, message, type }));
        GM_setValue('logs', JSON.stringify([...logs]));
        UI.updateLogs();
        if (Config.settings.advancedLogging) {
            const styles = { 'info': 'color: blue;', 'warning': 'color: orange;', 'error': 'color: red;', 'success': 'color: green; font-weight: bold;' };
            console.log(`%c[${timestamp}] [${type.toUpperCase()}] ${message}`, styles[type] || '');
        }
    }

    function getLogs() {
        return Array.from(logs).map(log => JSON.parse(log));
    }

    return {
        log,
        getLogs
    };
})();

// Utility Module
const Utils = (function() {
    function escapeHTML(str) {
        return String(str).replace(/&/g, "&amp;").replace(/</g, "&lt;")
                         .replace(/>/g, "&gt;").replace(/"/g, "&quot;")
                         .replace(/'/g, "&#039;");
    }

    function escapeCSV(str) {
        return String(str).replace(/"/g, '""');
    }

    function escapeXML(str) {
        return String(str).replace(/&/g, "&amp;").replace(/</g, "&lt;")
                         .replace(/>/g, "&gt;").replace(/"/g, "&quot;")
                         .replace(/'/g, "&#039;");
    }

    function capitalize(str) {
        return str.charAt(0).toUpperCase() + str.slice(1);
    }

    function delay(ms) {
        return new Promise(res => setTimeout(res, ms));
    }

    function generateColors(count) {
        return Array.from({length: count}, (_, i) => `hsl(${i * (360 / count)}, 70%, 50%)`);
    }

    function parseCustomKeywords(text) {
        const lines = text.split('\n').map(l => l.trim()).filter(l => l);
        const ck = {};
        lines.forEach(line => {
            const [cat, kws] = line.split(':');
            if (cat && kws) ck[cat.trim()] = kws.split(',').map(kw => kw.trim()).filter(kw => kw);
        });
        return ck;
    }

    function formatCustomKeywords(ck) {
        return Object.entries(ck).map(([c, kws]) => `${c}:${kws.join(',')}`).join('\n');
    }

    return {
        escapeHTML,
        escapeCSV,
        escapeXML,
        capitalize,
        delay,
        generateColors,
        parseCustomKeywords,
        formatCustomKeywords
    };
})();

// UI Module
const UI = (function() {
    function createUI() {
        const style = `
            /* (Styles remain unchanged for brevity) */
        `;
        $('head').append(`<style>${style}</style>`);
        const container = $(`
            <!-- (HTML structure remains unchanged for brevity) -->
        `);
        $('body').append(container);

        $('#crawler-minimize').on('click', toggleMinimize);
        $('.crawler-tab').on('click', switchTab);
        $('#save-settings').on('click', saveSettings);
        $('#reset-settings').on('click', resetSettings);
        $('#add-user-agent').on('click', addUserAgent);
        $('#remove-user-agent').on('click', removeUserAgent);
        $('#download-csv').on('click', Download.downloadCSV);
        $('#download-json').on('click', Download.downloadJSON);
        $('#download-xml').on('click', Download.downloadXML);
        $('#download-xls').on('click', Download.downloadXLS);
        $('#search-filter').on('input', filterData);
        $('#schedule-enabled').on('change', toggleScheduleOptions);
        $('#data-visualization-checkbox').on('change', toggleDataVisualization);
        $('#enable-captchas-checkbox').on('change', toggleCaptchaHandling);
        $('.close').on('click', () => $('#html-modal').hide());
        $(window).on('click', function(e) { if (e.target.id === 'html-modal') $('#html-modal').hide(); });
        $('#start-crawling').on('click', Crawling.startCrawling);
        $('#stop-crawling').on('click', Crawling.stopCrawling);
        $('#clear-previous-crawls').on('click', clearPreviousCrawls);
        $('#reset-data').on('click', resetData);
        $(document).on('keydown', handleShortcuts);
        updateUI();
        makeDraggable();
    }

    function toggleMinimize() {
        $('#crawler-content').toggle();
        const isHidden = $('#crawler-content').is(':hidden');
        $('#crawler-minimize').text(isHidden ? '☐' : '─');
        Logger.log(`크롤러 창이 ${isHidden ? '최소화' : '확대'}되었습니다.`, 'info');
    }

    function switchTab() {
        const tab = $(this).data('tab');
        $('.crawler-tab').removeClass('active');
        $(this).addClass('active');
        $('#crawler-content > div').hide();
        $(`#tab-${tab}`).show();
        Config.saveSettings({ currentTab: tab });
    }

    function updateUI() {
        const currentTab = GM_getValue('currentTab', 'logs');
        $(`.crawler-tab[data-tab="${currentTab}"]`).addClass('active');
        $('#crawler-content > div').hide();
        $(`#tab-${currentTab}`).show();
        updateLogs();
        updateDataTable();
        updatePreviousCrawlsUI();
        if (Config.settings.dataVisualization) DataVisualization.renderChart();
    }

    function updateLogs() {
        const logs = Logger.getLogs();
        $('#log-messages').html(logs.map(l => `<div>[${l.timestamp}] [${l.type.toUpperCase()}] ${Utils.escapeHTML(l.message)}</div>`).join(''));
        const progress = GM_getValue('progress', 0);
        const currentPage = GM_getValue('currentPage', 0);
        const totalPages = GM_getValue('totalPages', Config.settings.maxPages);
        $('.progress-bar-fill').css('width', `${progress.toFixed(2)}%`);
        $('.status-text').text(`진행률: ${progress.toFixed(2)}% (${currentPage}/${totalPages} 페이지)`);
        const latestLog = logs[logs.length -1];
        if (latestLog) {
            if (latestLog.type === 'error') {
                $('#error-message').text(latestLog.message).show();
                setTimeout(() => { $('#error-message').fadeOut(); }, 5000);
            }
            if (latestLog.type === 'success') {
                $('#success-message').text(latestLog.message).show();
                setTimeout(() => { $('#success-message').fadeOut(); }, 5000);
            }
        }
    }

    function updateDataTable() {
        const data = JSON.parse(GM_getValue('data', '[]'));
        const filter = $('#search-filter').val().toLowerCase();
        const filtered = filter ? data.filter(item => Object.keys(Config.settings.dataFields).some(field => Config.settings.dataFields[field] && String(item[field]).toLowerCase().includes(filter))) : data;
        const rows = filtered.map(item => `
            <tr>
                ${Config.settings.dataFields.name ? `<td>${Utils.escapeHTML(item.name)}</td>` : ''}
                ${Config.settings.dataFields.url ? `<td><a href="${Utils.escapeHTML(item.url)}" target="_blank">${Utils.escapeHTML(item.url)}</a></td>` : ''}
                ${Config.settings.dataFields.description ? `<td>${Utils.escapeHTML(item.description)}</td>` : ''}
                ${Config.settings.dataFields.html ? `<td><button class="view-html" data-html="${encodeURIComponent(item.html)}">📄 보기</button></td>` : ''}
                ${Config.settings.dataFields.category ? `<td>${Utils.escapeHTML(item.category)}</td>` : ''}
                ${Config.settings.dataFields.quality ? `<td>${Utils.escapeHTML(item.quality.toString())}</td>` : ''}
                ${Config.settings.dataFields.language ? `<td>${Utils.escapeHTML(item.language)}</td>` : ''}
                ${Config.settings.dataFields.wordCount ? `<td>${Utils.escapeHTML(item.wordCount.toString())}</td>` : ''}
                ${Config.settings.dataFields.metaTags ? `<td>${Utils.escapeHTML(JSON.stringify(item.metaTags))}</td>` : ''}
                ${Config.settings.dataFields.headings ? `<td>${Utils.escapeHTML(item.headings.join(', '))}</td>` : ''}
            </tr>
        `).join('');
        $('#data-table-body').html(rows);
        $('.view-html').off('click').on('click', function() {
            $('#modal-html-content').text(decodeURIComponent($(this).data('html')));
            $('#html-modal').show();
        });
        if (Config.settings.dataVisualization) DataVisualization.renderChart();
    }

    function updatePreviousCrawlsUI() {
        const previousCrawls = JSON.parse(GM_getValue('previousCrawls', '[]'));
        $('#previous-crawls-body').html(previousCrawls.map(crawl => `
            <tr>
                <td>${Utils.escapeHTML(crawl.date)}</td>
                <td>${Utils.escapeHTML(crawl.count.toString())}</td>
                <td>${Utils.escapeHTML(crawl.status)}</td>
            </tr>
        `).join(''));
    }

    function addUserAgent() {
        const ua = prompt("새 사용자 에이전트를 입력하세요:");
        if (ua) {
            Config.settings.userAgents.push(ua);
            GM_setValue('userAgents', Config.settings.userAgents);
            updateUserAgentSelect();
            Logger.log('새 사용자 에이전트가 추가되었습니다.', 'success');
        }
    }

    function removeUserAgent() {
        const selected = $('#user-agent-select').val();
        if (selected && selected.length) {
            selected.sort((a, b) => b - a).forEach(idx => { Config.settings.userAgents.splice(idx, 1); });
            GM_setValue('userAgents', Config.settings.userAgents);
            Config.settings.userAgentIndex = 0;
            GM_setValue('userAgentIndex', Config.settings.userAgentIndex);
            updateUserAgentSelect();
            Logger.log('선택한 사용자 에이전트가 제거되었습니다.', 'success');
        } else {
            alert('제거할 사용자 에이전트를 선택하세요.');
        }
    }

    function updateUserAgentSelect() {
        $('#user-agent-select').empty();
        Config.settings.userAgents.forEach((ua, idx) => {
            $('#user-agent-select').append(`<option value="${idx}" ${idx === Config.settings.userAgentIndex ? 'selected' : ''}>${ua}</option>`);
        });
    }

    async function saveSettings() {
        try {
            const delay = parseInt($('#delay-input').val(), 10);
            const delayRandom = parseInt($('#delay-random-input').val(), 10);
            const maxPages = parseInt($('#max-pages-input').val(), 10);
            const timeout = parseInt($('#timeout-input').val(), 10);
            const minWordCount = parseInt($('#min-word-count-input').val(), 10);
            const maxWordCount = parseInt($('#max-word-count-input').val(), 10);
            const maxDepth = parseInt($('#max-depth-input').val(), 10);
            const retryAttempts = parseInt($('#retry-attempts-input').val(), 10);
            const retryDelay = parseInt($('#retry-delay-input').val(), 10);
            const interval = parseInt($('#interval-input').val(), 10) * 60000;
            const isValidSettings = !isNaN(delay) && delay >= 500 &&
                                    !isNaN(delayRandom) && delayRandom >= 0 &&
                                    !isNaN(maxPages) && maxPages >= 1 &&
                                    !isNaN(timeout) && timeout >= 1000 &&
                                    !isNaN(minWordCount) && minWordCount >= 10 &&
                                    !isNaN(maxWordCount) && maxWordCount >= 100 &&
                                    !isNaN(maxDepth) && maxDepth >= 1 &&
                                    !isNaN(retryAttempts) && retryAttempts >= 0 &&
                                    !isNaN(retryDelay) && retryDelay >= 1000 &&
                                    (!Config.settings.schedule.enabled || (!isNaN(interval) && interval >= 60000));

            if (!isValidSettings) {
                $('#error-message').text('설정값이 유효하지 않습니다. 올바른 값을 입력하세요.').show();
                setTimeout(() => { $('#error-message').fadeOut(); }, 5000);
                return;
            }

            Config.settings.delay = delay;
            Config.settings.delayRandom = delayRandom;
            Config.settings.maxPages = maxPages;
            const selectedUA = parseInt($('#user-agent-select').val(), 10);
            Config.settings.userAgentIndex = isNaN(selectedUA) ? 0 : selectedUA;
            Config.settings.contentTypes.include = $('#content-include-input').val().split(',').map(ct => ct.trim()).filter(ct => ct) || Config.DEFAULT_SETTINGS.contentTypes.include;
            Config.settings.contentTypes.exclude = $('#content-exclude-input').val().split(',').map(ct => ct.trim()).filter(ct => ct) || Config.DEFAULT_SETTINGS.contentTypes.exclude;
            Config.settings.timeout = timeout;
            Config.settings.includeKeywords = $('#include-keywords-input').val().split('\n').map(kw => kw.trim()).filter(kw => kw) || Config.DEFAULT_SETTINGS.includeKeywords;
            Config.settings.excludeKeywords = $('#exclude-keywords-input').val().split('\n').map(kw => kw.trim()).filter(kw => kw) || Config.DEFAULT_SETTINGS.excludeKeywords;
            Config.settings.regexInclude = $('#regex-include-input').val().split('\n').map(rx => rx.trim()).filter(rx => rx) || Config.DEFAULT_SETTINGS.regexInclude;
            Config.settings.regexExclude = $('#regex-exclude-input').val().split('\n').map(rx => rx.trim()).filter(rx => rx) || Config.DEFAULT_SETTINGS.regexExclude;
            Config.settings.language = $('#language-select').val() || [];
            Config.settings.minWordCount = minWordCount;
            Config.settings.maxWordCount = maxWordCount;
            Config.settings.maxDepth = maxDepth;
            Config.settings.proxy = $('#proxy-input').val().trim() || Config.DEFAULT_SETTINGS.proxy;
            ['name','url','description','html','category','quality','language','wordCount','metaTags','headings'].forEach(field => {
                Config.settings.dataFields[field] = $(`#field-${field}`).is(':checked');
            });
            Config.settings.retryAttempts = retryAttempts;
            Config.settings.retryDelay = retryDelay;
            Config.settings.schedule.enabled = $('#schedule-enabled').is(':checked');
            Config.settings.schedule.startTime = $('#start-time-input').val() || Config.DEFAULT_SETTINGS.schedule.startTime;
            Config.settings.schedule.interval = Config.settings.schedule.enabled ? (isNaN(interval) ? Config.DEFAULT_SETTINGS.schedule.interval : interval) : Config.DEFAULT_SETTINGS.schedule.interval;
            Object.keys(Config.settings.searchEngines).forEach(engine => { Config.settings.searchEngines[engine] = $(`#engine-${engine}`).is(':checked'); });
            Config.settings.blacklist = $('#blacklist-input').val().split('\n').map(d => d.trim()).filter(d => d) || Config.DEFAULT_SETTINGS.blacklist;
            Config.settings.whitelist = $('#whitelist-input').val().split('\n').map(d => d.trim()).filter(d => d) || Config.DEFAULT_SETTINGS.whitelist;
            Config.settings.respectRobots = $('#respect-robots-checkbox').is(':checked');
            Config.settings.maxConcurrentRequests = parseInt($('#max-concurrent-requests-input').val(), 10) || Config.DEFAULT_SETTINGS.maxConcurrentRequests;
            Config.settings.dataVisualization = $('#data-visualization-checkbox').is(':checked');
            Config.settings.exportFormats.csv = $('#export-csv').is(':checked');
            Config.settings.exportFormats.json = $('#export-json').is(':checked');
            Config.settings.exportFormats.xml = $('#export-xml').is(':checked');
            Config.settings.exportFormats.xls = $('#export-xls').is(':checked');
            Config.settings.advancedLogging = $('#advanced-logging-checkbox').is(':checked');
            Config.settings.enableCaptchas = $('#enable-captchas-checkbox').is(':checked');
            Config.settings.customKeywords = Utils.parseCustomKeywords($('#custom-keywords-input').val());

            Config.saveSettings(Config.settings);
            Crawling.updateCrawlSettings();
            Logger.log('설정이 저장되었습니다.', 'success');

            if (Config.settings.schedule.enabled) {
                Scheduler.scheduleCrawling();
            }

            updateDataTable();
            if (Config.settings.dataVisualization) DataVisualization.renderChart();
        }

        function resetSettings() {
            if (confirm("기본 설정으로 초기화하시겠습니까?")) {
                Config.resetSettings();
                createUI();
                Logger.log('기본 설정으로 초기화되었습니다.', 'success');
            }
        }

        function toggleScheduleOptions() {
            $('#schedule-options').toggle($('#schedule-enabled').is(':checked'));
        }

        function toggleDataVisualization() {
            Config.settings.dataVisualization = $('#data-visualization-checkbox').is(':checked');
            GM_setValue('dataVisualization', Config.settings.dataVisualization);
            $('#data-visualization').toggle(Config.settings.dataVisualization);
            Logger.log('데이터 시각화 설정이 변경되었습니다.', 'success');
            if (Config.settings.dataVisualization) DataVisualization.renderChart();
        }

        function toggleCaptchaHandling() {
            Config.settings.enableCaptchas = $('#enable-captchas-checkbox').is(':checked');
            GM_setValue('enableCaptchas', Config.settings.enableCaptchas);
            Logger.log('CAPTCHA 처리 설정이 변경되었습니다.', 'success');
        }

        function filterData() {
            updateDataTable();
        }

        function clearPreviousCrawls() {
            if (confirm("이전 크롤링 기록을 모두 초기화하시겠습니까?")) {
                GM_setValue('previousCrawls', JSON.stringify([]));
                updatePreviousCrawlsUI();
                Logger.log('이전 크롤링 기록이 초기화되었습니다.', 'success');
            }
        }

        function resetData() {
            if (confirm("모든 크롤링 데이터를 초기화하시겠습니까?")) {
                GM_setValue('data', JSON.stringify([]));
                GM_setValue('visitedUrls', JSON.stringify([]));
                updateDataTable();
                Logger.log('크롤링 데이터가 초기화되었습니다.', 'success');
            }
        }

        function handleShortcuts(e) {
            if (e.ctrlKey) {
                const actions = { '1': 'logs', '2': 'data', '3': 'settings', '4': 'previous-crawls', 's': 'save', 'r': 'reset', 'p': 'start', 'q': 'stop', 'c': 'csv', 'j': 'json', 'x': 'xml', 'l': 'xls' };
                const action = actions[e.key];
                if (action) {
                    e.preventDefault();
                    switch(action) {
                        case 'logs': $('.crawler-tab[data-tab="logs"]').click(); break;
                        case 'data': $('.crawler-tab[data-tab="data"]').click(); break;
                        case 'settings': $('.crawler-tab[data-tab="settings"]').click(); break;
                        case 'previous-crawls': $('.crawler-tab[data-tab="previous-crawls"]').click(); break;
                        case 'save': saveSettings(); break;
                        case 'reset': resetSettings(); break;
                        case 'start': if (!Crawling.isCrawling) Crawling.startCrawling(); break;
                        case 'stop': if (Crawling.isCrawling) Crawling.stopCrawling(); break;
                        case 'csv': Config.settings.exportFormats.csv && Download.downloadCSV(); break;
                        case 'json': Config.settings.exportFormats.json && Download.downloadJSON(); break;
                        case 'xml': Config.settings.exportFormats.xml && Download.downloadXML(); break;
                        case 'xls': Config.settings.exportFormats.xls && Download.downloadXLS(); break;
                    }
                }
            }
        }

        function makeDraggable() {
            const container = document.getElementById('crawler-container');
            const header = document.getElementById('crawler-header');
            let isDragging = false;
            let startX, startY, initialX, initialY;
            header.onmousedown = function(e) {
                isDragging = true;
                startX = e.clientX;
                startY = e.clientY;
                initialX = container.offsetLeft;
                initialY = container.offsetTop;
                document.onmousemove = drag;
                document.onmouseup = stopDrag;
                return false;
            };
            function drag(e) {
                if (isDragging) {
                    let dx = e.clientX - startX;
                    let dy = e.clientY - startY;
                    container.style.left = (initialX + dx) + 'px';
                    container.style.top = (initialY + dy) + 'px';
                }
            }
            function stopDrag() {
                isDragging = false;
                document.onmousemove = null;
                document.onmouseup = null;
            }
        }

        return {
            createUI,
            updateLogs,
            updateDataTable,
            updatePreviousCrawlsUI
        };
    })();

// Download Module
const Download = (function() {
    function downloadCSV() {
        const data = JSON.parse(GM_getValue('data', '[]'));
        if (!data.length) { Logger.log('다운로드할 데이터가 없습니다.', 'warning'); return; }
        const headers = Object.keys(Config.settings.dataFields).filter(f => Config.settings.dataFields[f]).map(f => Utils.capitalize(getFieldName(f)));
        const csv = [headers.join(",")].concat(data.map(item => Object.keys(Config.settings.dataFields).filter(f => Config.settings.dataFields[f]).map(f => `"${Utils.escapeCSV(item[f] || '')}"`).join(","))).join("\n");
        const link = document.createElement("a");
        link.href = `data:text/csv;charset=utf-8,${encodeURIComponent(csv)}`;
        link.download = "data.csv";
        document.body.appendChild(link);
        link.click();
        document.body.removeChild(link);
        Logger.log('CSV 파일이 다운로드되었습니다.', 'success');
    }

    function downloadJSON() {
        const data = JSON.parse(GM_getValue('data', '[]'));
        if (!data.length) { Logger.log('다운로드할 데이터가 없습니다.', 'warning'); return; }
        const json = JSON.stringify(data, null, 2);
        const blob = new Blob([json], { type: "application/json" });
        const link = document.createElement("a");
        link.href = URL.createObjectURL(blob);
        link.download = "data.json";
        document.body.appendChild(link);
        link.click();
        document.body.removeChild(link);
        Logger.log('JSON 파일이 다운로드되었습니다.', 'success');
    }

    function downloadXML() {
        const data = JSON.parse(GM_getValue('data', '[]'));
        if (!data.length) { Logger.log('다운로드할 데이터가 없습니다.', 'warning'); return; }
        let xml = '<?xml version="1.0" encoding="UTF-8"?><data>';
        data.forEach(item => {
            xml += '<entry>';
            if (Config.settings.dataFields.name) xml += `<name>${Utils.escapeXML(item.name || 'No Title')}</name>`;
            if (Config.settings.dataFields.url) xml += `<url>${Utils.escapeXML(item.url || '')}</url>`;
            if (Config.settings.dataFields.description) xml += `<description>${Utils.escapeXML(item.description || 'No Description')}</description>`;
            if (Config.settings.dataFields.html) xml += `<html><![CDATA[${item.html || ''}]]></html>`;
            if (Config.settings.dataFields.category) xml += `<category>${Utils.escapeXML(item.category || '')}</category>`;
            if (Config.settings.dataFields.quality) xml += `<quality>${Utils.escapeXML(item.quality)}</quality>`;
            if (Config.settings.dataFields.language) xml += `<language>${Utils.escapeXML(item.language || 'unknown')}</language>`;
            if (Config.settings.dataFields.wordCount) xml += `<wordCount>${Utils.escapeXML(item.wordCount)}</wordCount>`;
            if (Config.settings.dataFields.metaTags) xml += `<metaTags>${Utils.escapeXML(JSON.stringify(item.metaTags))}</metaTags>`;
            if (Config.settings.dataFields.headings) xml += `<headings>${Utils.escapeXML(item.headings.join('; '))}</headings>`;
            xml += '</entry>';
        });
        xml += '</data>';
        const blob = new Blob([xml], { type: "application/xml" });
        const link = document.createElement("a");
        link.href = URL.createObjectURL(blob);
        link.download = "data.xml";
        document.body.appendChild(link);
        link.click();
        document.body.removeChild(link);
        Logger.log('XML 파일이 다운로드되었습니다.', 'success');
    }

    function downloadXLS() {
        const data = JSON.parse(GM_getValue('data', '[]'));
        if (!data.length) { Logger.log('다운로드할 데이터가 없습니다.', 'warning'); return; }
        const sheet = XLSX.utils.json_to_sheet(data.map(item => {
            const entry = {};
            if (Config.settings.dataFields.name) entry['사이트 이름'] = item.name;
            if (Config.settings.dataFields.url) entry['주소'] = item.url;
            if (Config.settings.dataFields.description) entry['설명'] = item.description;
            if (Config.settings.dataFields.html) entry['📄 HTML 내용'] = item.html;
            if (Config.settings.dataFields.category) entry['분류'] = item.category;
            if (Config.settings.dataFields.quality) entry['품질점수'] = item.quality;
            if (Config.settings.dataFields.language) entry['언어'] = item.language;
            if (Config.settings.dataFields.wordCount) entry['단어 수'] = item.wordCount;
            if (Config.settings.dataFields.metaTags) entry['메타 태그'] = JSON.stringify(item.metaTags);
            if (Config.settings.dataFields.headings) entry['헤딩'] = item.headings.join('; ');
            return entry;
        }));
        const wb = XLSX.utils.book_new();
        XLSX.utils.book_append_sheet(wb, sheet, "Data");
        XLSX.writeFile(wb, "data.xlsx");
        Logger.log('XLS 파일이 다운로드되었습니다.', 'success');
    }

    return {
        downloadCSV,
        downloadJSON,
        downloadXML,
        downloadXLS
    };
})();

// Data Visualization Module
const DataVisualization = (function() {
    let chartInstance = null;

    function renderChart() {
        if (!document.getElementById('data-chart').getContext) return;
        const ctx = document.getElementById('data-chart').getContext('2d');
        if (chartInstance) chartInstance.destroy();
        const data = JSON.parse(GM_getValue('data', '[]'));
        const categories = {};
        data.forEach(item => { const cat = item.category || '기타'; categories[cat] = (categories[cat] || 0) + 1; });
        const chartData = { labels: Object.keys(categories), datasets: [{ label: '카테고리별 데이터 수', data: Object.values(categories), backgroundColor: Utils.generateColors(Object.keys(categories).length) }] };
        chartInstance = new Chart(ctx, { type: 'pie', data: chartData, options: { responsive: true } });
    }

    return {
        renderChart
    };
})();

// Crawling Module
const Crawling = (function() {
    let isCrawling = GM_getValue('isCrawling', false);
    let currentPage = parseInt(GM_getValue('currentPage', 1), 10);
    let totalPages = parseInt(GM_getValue('totalPages', Config.settings.maxPages), 10);
    let progress = parseFloat(GM_getValue('progress', 0));
    let currentDepth = parseInt(GM_getValue('currentDepth', 0), 10);
    let activeRequests = 0;
    let requestQueue = [];
    let previousCrawls = JSON.parse(GM_getValue('previousCrawls', '[]'));
    const visitedUrls = new Set(JSON.parse(GM_getValue('visitedUrls', '[]')));
    let data = JSON.parse(GM_getValue('data', '[]'));

    const searchEngineHandlers = {
        google: {
            extractResults: function(doc) {
                const results = [];
                doc.querySelectorAll('div.g, div[data-hveid]').forEach(item => {
                    const name = item.querySelector('h3')?.innerText.trim() || 'No Title';
                    const url = item.querySelector('a')?.href || '';
                    const desc = item.querySelector('.aCOpRe, .VwiC3b')?.innerText.trim() || 'No Description';
                    if (isValidURL(url)) results.push({ name, url, description: desc });
                });
                return results;
            },
            getNextPageUrl: function(doc) {
                return doc.querySelector('a#pnnext')?.href || null;
            }
        },
        bing: {
            extractResults: function(doc) {
                const results = [];
                doc.querySelectorAll('.b_algo').forEach(item => {
                    const name = item.querySelector('h2')?.innerText.trim() || 'No Title';
                    const url = item.querySelector('h2 a')?.href || '';
                    const desc = item.querySelector('.b_caption p')?.innerText.trim() || 'No Description';
                    if (isValidURL(url)) results.push({ name, url, description: desc });
                });
                return results;
            },
            getNextPageUrl: function(doc) {
                return doc.querySelector('a.sb_pagN')?.href || null;
            }
        },
        yahoo: {
            extractResults: function(doc) {
                const results = [];
                doc.querySelectorAll('.Sr').forEach(item => {
                    const name = item.querySelector('h3')?.innerText.trim() || 'No Title';
                    const url = item.querySelector('a')?.href || '';
                    const desc = item.querySelector('.compText a')?.innerText.trim() || 'No Description';
                    if (isValidURL(url)) results.push({ name, url, description: desc });
                });
                return results;
            },
            getNextPageUrl: function(doc) {
                return doc.querySelector('a.next')?.href || null;
            }
        },
        duckduckgo: {
            extractResults: function(doc) {
                const results = [];
                doc.querySelectorAll('.result').forEach(item => {
                    const name = item.querySelector('.result__title')?.innerText.trim() || 'No Title';
                    const url = item.querySelector('.result__url')?.href || '';
                    const desc = item.querySelector('.result__snippet')?.innerText.trim() || 'No Description';
                    if (isValidURL(url)) results.push({ name, url, description: desc });
                });
                return results;
            },
            getNextPageUrl: function(doc) {
                return doc.querySelector('a.result--more__btn')?.href || null;
            }
        },
        // 추가 검색 엔진 핸들러...
    };

    function identifySearchEngine() {
        const host = window.location.hostname;
        if (host.includes('google.com')) return 'google';
        if (host.includes('bing.com')) return 'bing';
        if (host.includes('yahoo.com')) return 'yahoo';
        if (host.includes('duckduckgo.com')) return 'duckduckgo';
        // 다른 검색 엔진 추가
        return null;
    }

    async function startCrawling() {
        if (isCrawling) {
            Logger.log('이미 크롤링이 진행 중입니다.', 'warning');
            return;
        }
        if (!Object.values(Config.settings.searchEngines).some(v => v)) {
            Logger.log('사용할 검색 엔진이 하나도 선택되지 않았습니다.', 'error');
            return;
        }
        isCrawling = true;
        GM_setValue('isCrawling', isCrawling);
        Logger.log('크롤링을 시작합니다.', 'info');
        UI.updateLogs();
        currentPage = parseInt(GM_getValue('currentPage', 1), 10);
        totalPages = parseInt(GM_getValue('totalPages', Config.settings.maxPages), 10);
        GM_setValue('totalPages', totalPages);
        currentDepth = parseInt(GM_getValue('currentDepth', 0), 10);
        if (!previousCrawls.some(crawl => crawl.status === '진행 중')) {
            previousCrawls.push({ date: new Date().toLocaleString(), count: 0, status: '진행 중' });
            GM_setValue('previousCrawls', JSON.stringify(previousCrawls));
            UI.updatePreviousCrawlsUI();
        }
        await crawlPage(currentDepth);
    }

    function stopCrawling() {
        if (!isCrawling) {
            Logger.log('크롤링이 진행 중이지 않습니다.', 'warning');
            return;
        }
        isCrawling = false;
        GM_setValue('isCrawling', isCrawling);
        Logger.log('크롤링이 중지되었습니다.', 'warning');
        finalizeCrawl('중지됨');
        UI.updateLogs();
        setButtonStates();
    }

    function setButtonStates() {
        $('#start-crawling').prop('disabled', isCrawling);
        $('#stop-crawling').prop('disabled', !isCrawling);
    }

    async function crawlPage(depth) {
        try {
            while (isCrawling && currentPage <= totalPages && depth <= Config.settings.maxDepth) {
                Logger.log(`페이지 ${currentPage} 크롤링 시작. (깊이: ${depth})`, 'info');
                if (Config.settings.respectRobots && !(await Robots.checkRobotsTxt(window.location.href))) {
                    Logger.log(`robots.txt에 의해 크롤링이 금지된 페이지: ${window.location.href}`, 'warning');
                    break;
                }
                const searchEngine = identifySearchEngine();
                if (!searchEngine || !searchEngineHandlers[searchEngine]) {
                    Logger.log(`지원되지 않는 검색 엔진: ${window.location.hostname}`, 'error');
                    break;
                }
                const doc = document;
                const handler = searchEngineHandlers[searchEngine];
                const results = handler.extractResults(doc);
                if (!results) { Logger.log('검색 결과를 찾을 수 없습니다.', 'error'); break; }
                results.forEach(result => {
                    if (!isCrawling) return;
                    const { name, url, description } = result;
                    if (visitedUrls.has(url) || !isValidURL(url) ||
                        (Config.settings.whitelist.length && !Config.settings.whitelist.some(d => url.includes(d))) ||
                        Config.settings.blacklist.some(d => url.includes(d)) ||
                        (Config.settings.includeKeywords.length && !Config.settings.includeKeywords.some(k => url.includes(k) || description.includes(k))) ||
                        (Config.settings.excludeKeywords.some(k => url.includes(k) || description.includes(k))) ||
                        (Config.settings.regexInclude.length && !Config.settings.regexInclude.some(r => new RegExp(r).test(url) || new RegExp(r).test(description))) ||
                        (Config.settings.regexExclude.length && Config.settings.regexExclude.some(r => new RegExp(r).test(url) || new RegExp(r).test(description)))) {
                        return;
                    }
                    requestQueue.push({ url, depth });
                    if (activeRequests >= Config.settings.maxConcurrentRequests) return;
                });
                if (previousCrawls.length && previousCrawls[previousCrawls.length -1].status === '진행 중') {
                    previousCrawls[previousCrawls.length -1].count += results.length;
                    GM_setValue('previousCrawls', JSON.stringify(previousCrawls));
                    UI.updatePreviousCrawlsUI();
                }
                await processQueue();
                progress = (currentPage / totalPages) * 100;
                GM_setValue('progress', progress);
                UI.updateLogs();
                if (currentPage < totalPages && isCrawling) {
                    const nextUrl = handler.getNextPageUrl(doc);
                    if (nextUrl) {
                        currentPage++;
                        GM_setValue('currentPage', currentPage);
                        depth++;
                        GM_setValue('currentDepth', depth);
                        Logger.log(`다음 페이지로 이동: ${currentPage}`, 'info');
                        await Utils.delay(Config.settings.delay + Math.random() * Config.settings.delayRandom);
                        const nextPageContent = await Fetch.fetchPageContent(nextUrl);
                        if (nextPageContent) {
                            updatePageContent(nextPageContent);
                            continue;
                        } else {
                            Logger.log('다음 페이지 내용을 가져올 수 없습니다.', 'error');
                            break;
                        }
                    } else { Logger.log('다음 페이지 링크를 찾을 수 없습니다.', 'error'); break; }
                } else { break; }
            }
            if (isCrawling && currentPage > totalPages && activeRequests === 0 && requestQueue.length === 0) {
                Logger.log('크롤링이 완료되었습니다.', 'success');
                finalizeCrawl('완료됨');
                isCrawling = false;
                GM_setValue('isCrawling', isCrawling);
                setButtonStates();
                GM_setValue('progress', 100);
                UI.updateLogs();
            }
        } catch (error) {
            Logger.log(`크롤링 중 오류 발생: ${error.message}`, 'error');
            finalizeCrawl('오류 발생');
        }
    }

    async function processQueue() {
        while (requestQueue.length && activeRequests < Config.settings.maxConcurrentRequests && isCrawling) {
            const { url, depth } = requestQueue.shift();
            activeRequests++;
            handleRequest(url, depth).finally(() => { activeRequests--; processQueue(); });
        }
    }

    async function handleRequest(url, depth) {
        if (!isCrawling || visitedUrls.has(url)) return;
        try {
            const response = await Fetch.fetchWithRetry(url, Config.settings.retryAttempts, Config.settings.retryDelay);
            const contentType = Fetch.getContentType(response);
            if (!Fetch.isContentTypeAllowed(contentType, Config.settings.contentTypes)) {
                Logger.log(`허용되지 않은 콘텐츠 유형(${contentType})의 URL: ${url}`, 'warning');
                return;
            }
            const parser = new DOMParser();
            const doc = parser.parseFromString(response.responseText, 'text/html');
            const text = doc.body ? doc.body.innerText : '';
            if (Config.settings.enableCaptchas && Robots.detectCaptcha(doc)) { 
                await Robots.handleCaptcha(url); 
                return; 
            }
            const wc = text.split(/\s+/).length;
            if (wc < Config.settings.minWordCount || wc > Config.settings.maxWordCount) {
                Logger.log(`단어 수 조건에 맞지 않는 URL: ${url} (${wc} 단어)`, 'warning');
                return;
            }
            const quality = Evaluate.evaluateContentQuality(text);
            const lang = doc.documentElement.lang || 'unknown';
            const category = Classifier.classifyKeywords(text);
            const meta = Extractor.extractMetaTags(doc);
            const headings = Extractor.extractHeadings(doc);
            if (category !== '기타' || quality > 100) {
                const entry = { url, quality, language: lang, wordCount: wc, category };
                if (Config.settings.dataFields.name) entry.name = Extractor.extractName(doc) || 'No Title';
                if (Config.settings.dataFields.description) entry.description = Extractor.extractDescription(doc) || 'No Description';
                if (Config.settings.dataFields.html) entry.html = response.responseText;
                if (Config.settings.dataFields.metaTags) entry.metaTags = meta;
                if (Config.settings.dataFields.headings) entry.headings = headings;
                data.push(entry);
                GM_setValue('data', JSON.stringify(data));
                UI.updateDataTable();
                Logger.log(`데이터 수집 완료: ${url}`, 'success');
                if (Config.settings.dataVisualization) DataVisualization.renderChart();
            }
            if (depth < Config.settings.maxDepth) {
                const links = Extractor.extractLinks(doc);
                links.forEach(link => { if (!visitedUrls.has(link) && isValidURL(link)) requestQueue.push({ url: link, depth: depth + 1 }); });
            }
        } catch (error) {
            Logger.log(`데이터 수집 실패: ${url} - ${error.message}`, 'error');
        }
        visitedUrls.add(url);
        GM_setValue('visitedUrls', JSON.stringify([...visitedUrls]));
    }

    function finalizeCrawl(status) {
        if (previousCrawls.length) {
            previousCrawls[previousCrawls.length -1].status = status;
            GM_setValue('previousCrawls', JSON.stringify(previousCrawls));
            UI.updatePreviousCrawlsUI();
        }
    }

    function updateCrawlSettings() {
        // Placeholder for any settings update needed during crawling
    }

    function updatePageContent(content) {
        const parser = new DOMParser();
        const doc = parser.parseFromString(content, 'text/html');
        Object.defineProperty(document, 'body', {
            value: doc.body,
            configurable: true
        });
        Object.defineProperty(document, 'documentElement', {
            value: doc.documentElement,
            configurable: true
        });
    }

    return {
        startCrawling,
        stopCrawling,
        isCrawling
    };
})();

// Fetch Module
const Fetch = (function() {
    async function fetchWithRetry(url, attempts, delayMs) {
        for (let i = 1; i <= attempts; i++) {
            try {
                return await GM_xmlhttpRequestPromise(url);
            } catch (e) {
                Logger.log(`요청 실패 (${i}/${attempts}): ${url} - ${e.message}`, 'warning');
                if (i < attempts) await Utils.delay(delayMs);
            }
        }
        throw new Error('모든 재시도 시도가 실패했습니다.');
    }

    function GM_xmlhttpRequestPromise(url) {
        return new Promise((resolve, reject) => {
            let reqUrl = Config.settings.proxy ? `${Config.settings.proxy}?url=${encodeURIComponent(url)}` : url;
            GM_xmlhttpRequest({
                method: "GET",
                url: reqUrl,
                headers: { 'User-Agent': getCurrentUserAgent(), 'Accept': '*/*' },
                timeout: Config.settings.timeout,
                onload: res => (res.status >= 200 && res.status < 300) ? resolve(res) : reject(new Error(`HTTP ${res.status}`)),
                onerror: () => reject(new Error('Request failed')),
                ontimeout: () => reject(new Error('Request timed out'))
            });
        });
    }

    function getCurrentUserAgent() {
        return Config.settings.userAgents[Math.floor(Math.random() * Config.settings.userAgents.length)];
    }

    function getContentType(res) {
        return res.responseHeaders.split('\n').find(h => h.toLowerCase().startsWith('content-type'))?.split(':')[1].trim().split(';')[0] || '';
    }

    function isContentTypeAllowed(ct, contentTypes) {
        if (contentTypes.include.length && !contentTypes.include.includes(ct)) return false;
        if (contentTypes.exclude.includes(ct)) return false;
        return true;
    }

    async function fetchPageContent(url) {
        try {
            const response = await fetchWithRetry(url, Config.settings.retryAttempts, Config.settings.retryDelay);
            return response.responseText;
        } catch (error) {
            Logger.log(`페이지 내용 가져오기 실패: ${url} - ${error.message}`, 'error');
            return null;
        }
    }

    return {
        fetchWithRetry,
        getCurrentUserAgent,
        getContentType,
        isContentTypeAllowed,
        fetchPageContent
    };
})();

// Robots Module
const Robots = (function() {
    async function checkRobotsTxt(url) {
        try {
            const parsed = new URL(url);
            const robots = `${parsed.origin}/robots.txt`;
            const res = await Fetch.fetchWithRetry(robots, 2, 2000);
            return new RobotsParser(res.responseText).isAllowed(url, Fetch.getCurrentUserAgent());
        } catch {
            Logger.log('robots.txt 확인 실패, 기본 허용.', 'warning');
            return true;
        }
    }

    function detectCaptcha(doc) {
        return doc.querySelector('iframe[src*="captcha"], form[action*="captcha"]') !== null;
    }

    async function handleCaptcha(url) {
        Logger.log(`CAPTCHA 감지됨: ${url}`, 'warning');
        UI.showCaptchaModal(url);
    }

    class RobotsParser {
        constructor(text) { this.rules = []; this.parse(text); }
        parse(text) {
            let agents = [];
            text.split('\n').forEach(line => {
                const [key, ...val] = line.split(':').map(s => s.trim());
                if (!key) return;
                if (key.toLowerCase() === 'user-agent') { agents = [val.join(':').toLowerCase()]; }
                else if (['disallow','allow'].includes(key.toLowerCase())) this.rules.push({ agents, type: key.toLowerCase(), path: val.join(':') });
            });
        }
        isAllowed(url, ua) {
            const path = new URL(url).pathname;
            let allowed = true;
            this.rules.forEach(rule => {
                if (rule.agents.includes('*') || rule.agents.includes(ua.toLowerCase())) {
                    if (rule.type === 'disallow' && path.startsWith(rule.path)) allowed = false;
                    if (rule.type === 'allow' && path.startsWith(rule.path)) allowed = true;
                }
            });
            return allowed;
        }
    }

    return {
        checkRobotsTxt,
        detectCaptcha,
        handleCaptcha
    };
})();

// Extractor Module
const Extractor = (function() {
    function extractMetaTags(doc) {
        const metas = {};
        doc.querySelectorAll('meta[name], meta[property]').forEach(meta => {
            const name = meta.getAttribute('name') || meta.getAttribute('property');
            const content = meta.getAttribute('content');
            if (name && content) metas[name.toLowerCase()] = content;
        });
        return metas;
    }

    function extractHeadings(doc) {
        const hs = [];
        for (let i = 1; i <= 6; i++) doc.querySelectorAll(`h${i}`).forEach(h => hs.push(h.innerText.trim()));
        return hs;
    }

    function extractName(doc) { return doc.querySelector('title')?.innerText || 'No Title'; }
    function extractDescription(doc) { return doc.querySelector('meta[name="description"]')?.getAttribute('content') || 'No Description'; }
    function extractLinks(doc) { return Array.from(doc.querySelectorAll('a[href]')).map(a => a.href).filter(isValidURL); }

    return {
        extractMetaTags,
        extractHeadings,
        extractName,
        extractDescription,
        extractLinks
    };
})();

// Classifier Module
const Classifier = (function() {
    function classifyKeywords(text) {
        const kws = combineKeywords();
        for (const [cat, keywords] of Object.entries(kws)) {
            if (keywords.some(kw => text.includes(kw))) return cat;
        }
        return '기타';
    }

    function combineKeywords() {
        const standard = {
            '교육': ['학교', '교육', '학생', '교수', '강의'],
            '기술': ['기술', '소프트웨어', '하드웨어', '프로그래밍', '개발'],
            '뉴스': ['뉴스', '속보', '기사', '보도'],
            '엔터테인먼트': ['영화', '음악', '연예', '게임', '드라마'],
            '스포츠': ['축구', '야구', '농구', '경기', '토너먼트'],
            '비즈니스': ['비즈니스', '경제', '주식', '투자', '시장'],
            '건강': ['건강', '의료', '헬스', '운동', '영양'],
            '과학': ['과학', '연구', '실험', '발견', '이론'],
            '문학': ['소설', '시', '문학', '작가', '책'],
            '여행': ['여행', '관광', '숙박', '비행기', '여권']
        };
        const custom = Config.settings.customKeywords || {};
        for (const [cat, kws] of Object.entries(custom)) {
            standard[cat] = standard[cat] ? [...new Set([...standard[cat], ...kws])] : kws;
        }
        return standard;
    }

    return {
        classifyKeywords
    };
})();

// Evaluate Module
const Evaluate = (function() {
    function evaluateContentQuality(text) { return text.split(/\s+/).length; }

    return {
        evaluateContentQuality
    };
})();

// Robots Check Module
const Robots = (function() {
    return {};
})();

// Scheduler Module
const Scheduler = (function() {
    let scheduleInterval = null;

    function scheduleCrawling() {
        clearInterval(scheduleInterval);
        const now = new Date();
        const [startHour, startMinute] = Config.settings.schedule.startTime.split(':').map(Number);
        const startTime = new Date(now.getFullYear(), now.getMonth(), now.getDate(), startHour, startMinute, 0);
        if (startTime < now) startTime.setDate(startTime.getDate() + 1);
        const initialDelay = startTime - now;
        setTimeout(() => {
            Crawling.startCrawling();
            scheduleInterval = setInterval(Crawling.startCrawling, Config.settings.schedule.interval);
        }, initialDelay);
        Logger.log(`크롤링 스케줄이 설정되었습니다. 시작 시간: ${startTime.toLocaleString()}, 간격: ${Config.settings.schedule.interval / 60000}분`, 'info');
    }

    return {
        scheduleCrawling
    };
})();

// Initialization
function init() {
    UI.createUI();
    UI.updateLogs();
    UI.updateDataTable();
    UI.updatePreviousCrawlsUI();
    if (Config.settings.dataVisualization) DataVisualization.renderChart();
    if (Crawling.isCrawling) {
        Logger.log('크롤링 재개됨.', 'info');
        Crawling.startCrawling();
    } else {
        Logger.log('크롤러 초기화됨.', 'info');
    }
}

window.addEventListener('load', init);

// UI Module Extended Functionality
UI.showCaptchaModal = function(url) {
    const modal = $('#captcha-modal');
    if (modal.length === 0) {
        const captchaModal = $(`
            <div id="captcha-modal" class="modal">
                <div class="modal-content">
                    <span class="close">&times;</span>
                    <h2>CAPTCHA 감지됨</h2>
                    <p>URL: ${Utils.escapeHTML(url)}</p>
                    <p>CAPTCHA가 감지되어 크롤링이 중지되었습니다. 수동으로 해결하거나 설정을 변경하세요.</p>
                    <button id="captcha-ok" style="padding: 10px 20px; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px;">확인</button>
                </div>
            </div>
        `);
        $('body').append(captchaModal);
        $('#captcha-modal .close, #captcha-ok').on('click', () => $('#captcha-modal').hide());
    } else {
        modal.find('p').eq(0).text(`URL: ${Utils.escapeHTML(url)}`);
    }
    $('#captcha-modal').show();
};
})(); 코드에 문제점이나 평가 , 리뷰 피드백 , 코드 최적화등 기타 등등 뭔가 해줘봐