// ==UserScript==
// @name         고급 웹 크롤러 스크립트 프로
// @namespace    http://tampermonkey.net/
// @version      6.1
// @description  고급 웹 크롤러 스크립트 with Enhanced Settings, Improved Accuracy, Resizable UI, Additional Features, 및 크롤링 초기화 기능 추가
// @author
// @match        https://www.google.com/search*
// @match        https://www.bing.com/search*
// @match        https://search.yahoo.com/search*
// @match        https://duckduckgo.com/*
// @match        https://www.baidu.com/s*
// @match        https://www.yandex.com/search/*
// @match        https://search.naver.com/search*
// @match        https://search.daum.net/search*
// @match        https://www.ask.com/*
// @match        https://www.ecosia.org/search*
// @match        https://www.startpage.com/*
// @match        https://searx.me/*
// @match        https://www.dogpile.com/*
// @match        https://www.bing.com/images*
// @grant        GM_xmlhttpRequest
// @grant        GM_setValue
// @grant        GM_getValue
// @grant        GM_deleteValue
// @connect      *
// @require      https://cdnjs.cloudflare.com/ajax/libs/jquery/3.6.0/jquery.min.js
// @require      https://cdn.jsdelivr.net/npm/chart.js
// @require      https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.18.5/xlsx.full.min.js
// ==/UserScript==

(function() {
    'use strict';

    // 기본 설정
    const defaultSettings = {
        delay: 1000,
        delayRandom: 500,
        maxPages: 10,
        userAgents: [
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
        ],
        userAgentIndex: 0,
        contentTypes: {
            include: ['text/html'],
            exclude: []
        },
        timeout: 10000,
        includeKeywords: [],
        excludeKeywords: [],
        regexInclude: [],
        regexExclude: [],
        language: [],
        minWordCount: 100,
        maxWordCount: 10000,
        maxDepth: 2,
        proxy: '',
        dataFields: {
            name: true,
            url: true,
            description: true,
            html: false,
            category: true,
            quality: false,
            language: true,
            wordCount: true,
            metaTags: false,
            headings: false
        },
        retryAttempts: 3,
        retryDelay: 2000,
        schedule: {
            enabled: false,
            startTime: '00:00',
            interval: 60 * 60 * 1000 // 1시간
        },
        searchEngines: {
            google: true,
            bing: true,
            yahoo: true,
            duckduckgo: true,
            baidu: true,
            yandex: true,
            naver: true,
            daum: true,
            ask: true,
            ecosia: true,
            startpage: true,
            searx: true,
            dogpile: true,
            bingimages: true
        },
        respectRobots: true,
        blacklist: [],
        whitelist: [],
        maxConcurrentRequests: 5,
        dataVisualization: false,
        exportFormats: {
            csv: true,
            json: true,
            xml: true,
            xls: true
        },
        advancedLogging: false,
        enableCaptchas: false,
        customKeywords: {}
    };

    // 현재 설정 로드
    const settings = {
        delay: GM_getValue('delay', defaultSettings.delay),
        delayRandom: GM_getValue('delayRandom', defaultSettings.delayRandom),
        maxPages: GM_getValue('maxPages', defaultSettings.maxPages),
        userAgents: GM_getValue('userAgents', defaultSettings.userAgents),
        userAgentIndex: GM_getValue('userAgentIndex', defaultSettings.userAgentIndex),
        contentTypes: GM_getValue('contentTypes', defaultSettings.contentTypes),
        timeout: GM_getValue('timeout', defaultSettings.timeout),
        includeKeywords: GM_getValue('includeKeywords', defaultSettings.includeKeywords),
        excludeKeywords: GM_getValue('excludeKeywords', defaultSettings.excludeKeywords),
        regexInclude: GM_getValue('regexInclude', defaultSettings.regexInclude),
        regexExclude: GM_getValue('regexExclude', defaultSettings.regexExclude),
        language: GM_getValue('language', defaultSettings.language),
        minWordCount: GM_getValue('minWordCount', defaultSettings.minWordCount),
        maxWordCount: GM_getValue('maxWordCount', defaultSettings.maxWordCount),
        maxDepth: GM_getValue('maxDepth', defaultSettings.maxDepth),
        proxy: GM_getValue('proxy', defaultSettings.proxy),
        dataFields: GM_getValue('dataFields', defaultSettings.dataFields),
        retryAttempts: GM_getValue('retryAttempts', defaultSettings.retryAttempts),
        retryDelay: GM_getValue('retryDelay', defaultSettings.retryDelay),
        schedule: GM_getValue('schedule', defaultSettings.schedule),
        searchEngines: GM_getValue('searchEngines', defaultSettings.searchEngines),
        respectRobots: GM_getValue('respectRobots', defaultSettings.respectRobots),
        blacklist: GM_getValue('blacklist', defaultSettings.blacklist),
        whitelist: GM_getValue('whitelist', defaultSettings.whitelist),
        maxConcurrentRequests: GM_getValue('maxConcurrentRequests', defaultSettings.maxConcurrentRequests),
        dataVisualization: GM_getValue('dataVisualization', defaultSettings.dataVisualization),
        exportFormats: GM_getValue('exportFormats', defaultSettings.exportFormats),
        advancedLogging: GM_getValue('advancedLogging', defaultSettings.advancedLogging),
        enableCaptchas: GM_getValue('enableCaptchas', defaultSettings.enableCaptchas),
        customKeywords: GM_getValue('customKeywords', defaultSettings.customKeywords)
    };

    // 크롤링 상태 복원
    let visitedUrls = GM_getValue('visitedUrls', []);
    let logs = GM_getValue('logs', []);
    let data = GM_getValue('data', []);
    let isCrawling = GM_getValue('isCrawling', false);
    let currentPage = GM_getValue('currentPage', 1);
    let totalPages = GM_getValue('totalPages', settings.maxPages);
    let progress = GM_getValue('progress', 0);
    let currentDepth = GM_getValue('currentDepth', 0);
    let activeRequests = 0;
    let requestQueue = [];

    // UI 생성
    function createUI() {
        const style = `
            #crawler-container {
                position: fixed;
                bottom: 10px;
                right: 10px;
                width: 400px;
                background: #fff;
                border: 1px solid #ccc;
                border-radius: 8px;
                box-shadow: 0 2px 10px rgba(0,0,0,0.1);
                z-index: 10000;
                font-family: Arial, sans-serif;
            }
            #crawler-tabs {
                display: flex;
                border-bottom: 1px solid #ccc;
            }
            .crawler-tab {
                flex: 1;
                padding: 10px;
                text-align: center;
                cursor: pointer;
                background: #f1f1f1;
            }
            .crawler-tab.active {
                background: #fff;
                border-bottom: 2px solid #007BFF;
            }
            #crawler-content {
                padding: 10px;
                max-height: 500px;
                overflow-y: auto;
            }
            .progress-bar {
                width: 100%;
                background: #e0e0e0;
                border-radius: 4px;
                overflow: hidden;
                height: 20px;
                margin-bottom: 5px;
            }
            .progress-bar-fill {
                height: 100%;
                background: #007BFF;
                width: 0%;
                transition: width 0.2s;
            }
            .status-text {
                margin-bottom: 10px;
            }
            .control-buttons button {
                margin: 2px;
                padding: 5px 10px;
                border: none;
                border-radius: 4px;
                cursor: pointer;
            }
            .control-buttons button:disabled {
                background: #ccc;
                cursor: not-allowed;
            }
            #log-messages {
                height: 250px;
                overflow-y: auto;
                background: #f9f9f9;
                padding: 10px;
                border: 1px solid #ccc;
                border-radius: 4px;
            }
            .modal {
                display: none;
                position: fixed;
                z-index: 10001;
                left: 0;
                top: 0;
                width: 100%;
                height: 100%;
                overflow: auto;
                background-color: rgba(0,0,0,0.4);
            }
            .modal-content {
                background-color: #fefefe;
                margin: 15% auto;
                padding: 20px;
                border: 1px solid #888;
                width: 80%;
                max-height: 80%;
                overflow-y: auto;
            }
            .close {
                color: #aaa;
                float: right;
                font-size: 28px;
                font-weight: bold;
                cursor: pointer;
            }
        `;
        $('head').append(`<style>${style}</style>`);

        const container = $(`
            <div id="crawler-container">
                <div id="crawler-tabs">
                    <div class="crawler-tab ${currentTab() === 'logs' ? 'active' : ''}" data-tab="logs" title="로그 보기">📜 로그</div>
                    <div class="crawler-tab ${currentTab() === 'data' ? 'active' : ''}" data-tab="data" title="수집된 데이터 보기">📊 데이터</div>
                    <div class="crawler-tab ${currentTab() === 'settings' ? 'active' : ''}" data-tab="settings" title="설정 변경">⚙️ 설정</div>
                </div>
                <div id="crawler-content">
                    <div id="tab-logs" style="display: ${currentTab() === 'logs' ? 'block' : 'none'};">
                        <div class="progress-bar">
                            <div class="progress-bar-fill" style="width: ${progress.toFixed(2)}%;"></div>
                        </div>
                        <div class="status-text">진행률: ${progress.toFixed(2)}% (${currentPage}/${totalPages} 페이지)</div>
                        <div class="control-buttons">
                            <button id="start-crawling" title="크롤링을 시작합니다." ${isCrawling ? 'disabled' : ''}>▶ 시작</button>
                            <button id="stop-crawling" title="크롤링을 중지합니다." ${isCrawling ? '' : 'disabled'}>■ 중지</button>
                            ${settings.exportFormats.csv ? `<button id="download-csv" title="수집된 데이터를 CSV 파일로 다운로드합니다.">📥 CSV</button>` : ''}
                            ${settings.exportFormats.json ? `<button id="download-json" title="수집된 데이터를 JSON 파일로 다운로드합니다.">📥 JSON</button>` : ''}
                            ${settings.exportFormats.xml ? `<button id="download-xml" title="수집된 데이터를 XML 파일로 다운로드합니다.">📥 XML</button>` : ''}
                            ${settings.exportFormats.xls ? `<button id="download-xls" title="수집된 데이터를 XLS 파일로 다운로드합니다.">📥 XLS</button>` : ''}
                            <button id="reset-data" title="수집된 데이터와 로그를 초기화합니다." style="background-color: #f44336; color: white;">🗑️ 데이터 초기화</button>
                        </div>
                        <div id="log-messages"></div>
                    </div>
                    <div id="tab-data" style="display: ${currentTab() === 'data' ? 'block' : 'none'};">
                        <input type="text" id="search-filter" placeholder="검색 필터..." style="width: 100%; padding: 5px; margin-bottom: 10px;">
                        <table id="data-table" style="width: 100%; border-collapse: collapse;">
                            <thead>
                                <tr>
                                    ${settings.dataFields.name ? `<th>사이트 이름</th>` : ''}
                                    ${settings.dataFields.url ? `<th>주소</th>` : ''}
                                    ${settings.dataFields.description ? `<th>설명</th>` : ''}
                                    ${settings.dataFields.html ? `<th>HTML</th>` : ''}
                                    ${settings.dataFields.category ? `<th>분류</th>` : ''}
                                    ${settings.dataFields.quality ? `<th>품질점수</th>` : ''}
                                    ${settings.dataFields.language ? `<th>언어</th>` : ''}
                                    ${settings.dataFields.wordCount ? `<th>단어 수</th>` : ''}
                                    ${settings.dataFields.metaTags ? `<th>메타 태그</th>` : ''}
                                    ${settings.dataFields.headings ? `<th>헤딩</th>` : ''}
                                </tr>
                            </thead>
                            <tbody id="data-table-body"></tbody>
                        </table>
                        ${settings.dataVisualization ? `<canvas id="data-chart"></canvas>` : ''}
                    </div>
                    <div id="tab-settings" style="display: ${currentTab() === 'settings' ? 'block' : 'none'};">
                        <h3>설정</h3>
                        <div>
                            <label for="delay-input">딜레이 (ms):</label>
                            <input type="number" id="delay-input" value="${settings.delay}" style="width: 100%; padding: 5px; margin-bottom: 10px;">
                        </div>
                        <div>
                            <label for="delay-random-input">랜덤 딜레이 (ms):</label>
                            <input type="number" id="delay-random-input" value="${settings.delayRandom}" style="width: 100%; padding: 5px; margin-bottom: 10px;">
                        </div>
                        <div>
                            <label for="max-pages-input">최대 페이지:</label>
                            <input type="number" id="max-pages-input" value="${settings.maxPages}" style="width: 100%; padding: 5px; margin-bottom: 10px;">
                        </div>
                        <div>
                            <label for="user-agent-select">사용자 에이전트:</label>
                            <select id="user-agent-select" style="width: 100%; padding: 5px; margin-bottom: 10px;"></select>
                            <button id="add-user-agent" style="width: 48%; padding: 5px; margin-right: 4%;">추가</button>
                            <button id="remove-user-agent" style="width: 48%; padding: 5px;">제거</button>
                        </div>
                        <button id="save-settings" style="width: 100%; padding: 10px; background-color: #4CAF50; color: white; border: none; border-radius: 4px; cursor: pointer;">설정 저장</button>
                        <button id="reset-settings" style="width: 100%; padding: 10px; background-color: #f44336; color: white; border: none; border-radius: 4px; cursor: pointer; margin-top: 5px;">설정 초기화</button>
                    </div>
                </div>

                <div id="html-modal" class="modal">
                    <div class="modal-content">
                        <span class="close">&times;</span>
                        <pre id="modal-html-content"></pre>
                    </div>
                </div>
            </div>
        `);
        $('body').append(container);

        // 탭 전환 이벤트
        $('.crawler-tab').on('click', function() {
            const tab = $(this).data('tab');
            $('.crawler-tab').removeClass('active');
            $(this).addClass('active');
            $('#crawler-content > div').hide();
            $(`#tab-${tab}`).show();
            saveCurrentTab(tab);
        });

        // 단축키 이벤트
        $(document).on('keydown', function(e) {
            if (e.ctrlKey) {
                switch (e.key) {
                    case '1':
                        $('.crawler-tab[data-tab="logs"]').click();
                        break;
                    case '2':
                        $('.crawler-tab[data-tab="data"]').click();
                        break;
                    case '3':
                        $('.crawler-tab[data-tab="settings"]').click();
                        break;
                    case 's':
                        e.preventDefault();
                        saveSettings();
                        break;
                    case 'p':
                        e.preventDefault();
                        if (!isCrawling) startCrawling();
                        break;
                    case 'q':
                        e.preventDefault();
                        if (isCrawling) stopCrawling();
                        break;
                    case 'd':
                        e.preventDefault();
                        downloadCSV();
                        break;
                    case 'j':
                        e.preventDefault();
                        downloadJSON();
                        break;
                    case 'x':
                        e.preventDefault();
                        downloadXML();
                        break;
                    case 'l':
                        e.preventDefault();
                        downloadXLS();
                        break;
                }
            }
        });

        // 설정 저장 버튼 이벤트
        $('#save-settings').on('click', saveSettings);
        $('#reset-settings').on('click', resetSettings);

        // 추가/제거 사용자 에이전트 버튼 이벤트
        $('#add-user-agent').on('click', function() {
            const newUA = prompt("새 사용자 에이전트를 입력하세요:");
            if (newUA) {
                settings.userAgents.push(newUA);
                GM_setValue('userAgents', settings.userAgents);
                updateUserAgentSelect();
                log('새 사용자 에이전트가 추가되었습니다.', 'success');
            }
        });

        $('#remove-user-agent').on('click', function() {
            const selectedOptions = $('#user-agent-select').val();
            if (selectedOptions && selectedOptions.length > 0) {
                settings.userAgents = settings.userAgents.filter((_, idx) => !selectedOptions.includes(idx.toString()));
                GM_setValue('userAgents', settings.userAgents);
                settings.userAgentIndex = 0;
                GM_setValue('userAgentIndex', settings.userAgentIndex);
                updateUserAgentSelect();
                log('선택한 사용자 에이전트가 제거되었습니다.', 'success');
            } else {
                alert('제거할 사용자 에이전트를 선택하세요.');
            }
        });

        // 다운로드 버튼 이벤트
        $('#download-csv').on('click', downloadCSV);
        $('#download-json').on('click', downloadJSON);
        $('#download-xml').on('click', downloadXML);
        $('#download-xls').on('click', downloadXLS);

        // 데이터 초기화 버튼 이벤트 추가
        $('#reset-data').on('click', resetData);

        // 검색 필터 기능
        $('#search-filter').on('input', function() {
            filterTable($(this).val());
        });

        // 설정 초기화
        updateUserAgentSelect();
        populateSettingsTab();
        updateLogsUI();
        updateDataUI();
        if (settings.dataVisualization) {
            renderChart();
        }

        // 모달 창 기능
        const modal = document.getElementById("html-modal");
        const span = document.getElementsByClassName("close")[0];
        span.onclick = function() { modal.style.display = "none"; }
        window.onclick = function(event) {
            if (event.target == modal) {
                modal.style.display = "none";
            }
        }
    }

    function currentTab() {
        return GM_getValue('currentTab', 'logs');
    }

    function saveCurrentTab(tab) {
        GM_setValue('currentTab', tab);
    }

    function updateUserAgentSelect() {
        $('#user-agent-select').empty();
        settings.userAgents.forEach((ua, idx) => {
            const selected = idx === settings.userAgentIndex ? 'selected' : '';
            $('#user-agent-select').append(`<option value="${idx}" ${selected}>${ua}</option>`);
        });
    }

    // 사용자 지정 키워드 텍스트 생성 함수
    function getCustomKeywordsText() {
        const customKeywords = settings.customKeywords || {};
        return Object.entries(customKeywords).map(([category, kws]) => `${category}:${kws.join(',')}`).join('\n');
    }

    // 사용자 지정 키워드 저장 및 로드 함수
    function parseCustomKeywords(text) {
        const lines = text.split('\n').map(line => line.trim()).filter(line => line);
        const customKeywords = {};
        lines.forEach(line => {
            const [category, kws] = line.split(':');
            if (category && kws) {
                customKeywords[category.trim()] = kws.split(',').map(kw => kw.trim()).filter(kw => kw);
            }
        });
        return customKeywords;
    }

    function combineKeywords() {
        const standardKeywords = {
            '교육': ['학교', '교육', '학생', '교수', '강의'],
            '기술': ['기술', '소프트웨어', '하드웨어', '프로그래밍', '개발'],
            '뉴스': ['뉴스', '속보', '기사', '보도'],
            '엔터테인먼트': ['영화', '음악', '연예', '게임', '드라마'],
            '스포츠': ['축구', '야구', '농구', '경기', '토너먼트'],
            '비즈니스': ['비즈니스', '경제', '주식', '투자', '시장'],
            '건강': ['건강', '의료', '헬스', '운동', '영양'],
            '과학': ['과학', '연구', '실험', '발견', '이론'],
            '문학': ['소설', '시', '문학', '작가', '책'],
            '여행': ['여행', '관광', '숙박', '비행기', '여권']
            // 추가 키워드 및 카테고리
        };
        const customKeywords = settings.customKeywords || {};

        for (const [category, kws] of Object.entries(customKeywords)) {
            if (standardKeywords[category]) {
                standardKeywords[category] = [...new Set([...standardKeywords[category], ...kws])];
            } else {
                standardKeywords[category] = kws;
            }
        }

        return standardKeywords;
    }

    // 로깅 함수
    function log(message, type = 'info') {
        const timestamp = new Date().toLocaleString();
        logs.push({ timestamp, message, type });
        GM_setValue('logs', logs);
        updateLogsUI();

        if (settings.advancedLogging) {
            switch(type) {
                case 'info':
                    console.info(`[${timestamp}] [INFO] ${message}`);
                    break;
                case 'warning':
                    console.warn(`[${timestamp}] [WARNING] ${message}`);
                    break;
                case 'error':
                    console.error(`[${timestamp}] [ERROR] ${message}`);
                    break;
                case 'success':
                    console.log(`%c[${timestamp}] [SUCCESS] ${message}`, 'color: green; font-weight: bold;');
                    break;
                default:
                    console.log(`[${timestamp}] [${type.toUpperCase()}] ${message}`);
            }
        }
    }

    function updateLogsUI() {
        const logHtml = logs.map(log => `<div>[${log.timestamp}] [${log.type.toUpperCase()}] ${log.message}</div>`).join('');
        $('#log-messages').html(logHtml);
        $('.progress-bar-fill').css('width', `${progress.toFixed(2)}%`);
        $('.status-text').text(`진행률: ${progress.toFixed(2)}% (${currentPage}/${totalPages} 페이지)`);
    }

    // 데이터 업데이트 함수
    function updateDataUI() {
        const filter = $('#search-filter').val().toLowerCase();
        let filteredData = data;
        if (filter) {
            filteredData = data.filter(item =>
                (settings.dataFields.name && item.name.toLowerCase().includes(filter)) ||
                (settings.dataFields.url && item.url.toLowerCase().includes(filter)) ||
                (settings.dataFields.description && item.description.toLowerCase().includes(filter)) ||
                (settings.dataFields.html && item.html.toLowerCase().includes(filter)) ||
                (settings.dataFields.category && item.category.toLowerCase().includes(filter)) ||
                (settings.dataFields.language && item.language && item.language.toLowerCase().includes(filter)) ||
                (settings.dataFields.wordCount && item.wordCount.toString().includes(filter)) ||
                (settings.dataFields.metaTags && item.metaTags && Object.values(item.metaTags).some(v => v.toLowerCase().includes(filter))) ||
                (settings.dataFields.headings && item.headings && item.headings.some(h => h.toLowerCase().includes(filter)))
            );
        }

        let tableHtml = filteredData.map(item => `
            <tr>
                ${settings.dataFields.name ? `<td>${escapeHTML(item.name)}</td>` : ''}
                ${settings.dataFields.url ? `<td><a href="${escapeHTML(item.url)}" target="_blank">${escapeHTML(item.url)}</a></td>` : ''}
                ${settings.dataFields.description ? `<td>${escapeHTML(item.description)}</td>` : ''}
                ${settings.dataFields.html ? `<td><button class="view-html" data-html="${encodeURIComponent(item.html)}" title="HTML 내용을 봅니다.">📄 보기</button></td>` : ''}
                ${settings.dataFields.category ? `<td>${escapeHTML(item.category)}</td>` : ''}
                ${settings.dataFields.quality ? `<td>${escapeHTML(item.quality.toString())}</td>` : ''}
                ${settings.dataFields.language ? `<td>${escapeHTML(item.language)}</td>` : ''}
                ${settings.dataFields.wordCount ? `<td>${escapeHTML(item.wordCount.toString())}</td>` : ''}
                ${settings.dataFields.metaTags ? `<td>${escapeHTML(JSON.stringify(item.metaTags))}</td>` : ''}
                ${settings.dataFields.headings ? `<td>${escapeHTML(item.headings.join(', '))}</td>` : ''}
            </tr>
        `).join('');
        $('#data-table-body').html(tableHtml);

        // HTML 보기 모달
        $('.view-html').off('click').on('click', function() {
            const htmlContent = decodeURIComponent($(this).data('html'));
            $('#modal-html-content').text(htmlContent);
            $('#html-modal').show();
        });

        if (settings.dataVisualization) {
            renderChart();
        }
    }

    // 차트 렌더링 함수
    function renderChart() {
        if (!document.getElementById('data-chart').getContext) return;

        const ctx = document.getElementById('data-chart').getContext('2d');

        if (window.dataChartInstance) {
            window.dataChartInstance.destroy();
        }

        const categories = {};
        data.forEach(item => {
            const category = item.category || '기타';
            if (categories[category]) {
                categories[category]++;
            } else {
                categories[category] = 1;
            }
        });

        const chartData = {
            labels: Object.keys(categories),
            datasets: [{
                label: '카테고리별 데이터 수',
                data: Object.values(categories),
                backgroundColor: generateColors(Object.keys(categories).length)
            }]
        };

        window.dataChartInstance = new Chart(ctx, {
            type: 'pie',
            data: chartData,
            options: {
                responsive: true
            }
        });
    }

    // 색상 생성 함수
    function generateColors(count) {
        const colors = [];
        for(let i=0; i<count; i++) {
            const hue = i * (360 / count);
            colors.push(`hsl(${hue}, 70%, 50%)`);
        }
        return colors;
    }

    // HTML 이스케이프 함수
    function escapeHTML(str) {
        if (!str) return '';
        return str.replace(/&/g, "&amp;")
                  .replace(/</g, "&lt;")
                  .replace(/>/g, "&gt;")
                  .replace(/"/g, "&quot;")
                  .replace(/'/g, "&#039;");
    }

    // CSV, JSON, XML, XLS 다운로드 함수
    function downloadCSV() {
        if (data.length === 0) {
            log('다운로드할 데이터가 없습니다.', 'warning');
            return;
        }
        const headers = [];
        if (settings.dataFields.name) headers.push("사이트 이름");
        if (settings.dataFields.url) headers.push("주소");
        if (settings.dataFields.description) headers.push("설명");
        if (settings.dataFields.html) headers.push("📄 HTML 내용");
        if (settings.dataFields.category) headers.push("분류");
        if (settings.dataFields.quality) headers.push("품질점수");
        if (settings.dataFields.language) headers.push("언어");
        if (settings.dataFields.wordCount) headers.push("단어 수");
        if (settings.dataFields.metaTags) headers.push("메타 태그");
        if (settings.dataFields.headings) headers.push("헤딩");

        const csvRows = [
            headers.join(",")
        ];

        data.forEach(d => {
            const row = [];
            if (settings.dataFields.name) row.push(`"${escapeCSV(d.name)}"`);
            if (settings.dataFields.url) row.push(`"${escapeCSV(d.url)}"`);
            if (settings.dataFields.description) row.push(`"${escapeCSV(d.description)}"`);
            if (settings.dataFields.html) row.push(`"${escapeCSV(d.html)}"`);
            if (settings.dataFields.category) row.push(`"${escapeCSV(d.category)}"`);
            if (settings.dataFields.quality) row.push(`"${d.quality}"`);
            if (settings.dataFields.language) row.push(`"${escapeCSV(d.language)}"`);
            if (settings.dataFields.wordCount) row.push(`"${d.wordCount}"`);
            if (settings.dataFields.metaTags) row.push(`"${escapeCSV(JSON.stringify(d.metaTags))}"`);
            if (settings.dataFields.headings) row.push(`"${escapeCSV(d.headings.join('; '))}"`);
            csvRows.push(row.join(","));
        });

        const csvContent = "data:text/csv;charset=utf-8," + csvRows.join("\n");
        const encodedUri = encodeURI(csvContent);
        const link = document.createElement("a");
        link.setAttribute("href", encodedUri);
        const filename = "data.csv";
        link.setAttribute("download", filename);
        document.body.appendChild(link);
        link.click();
        document.body.removeChild(link);
        log('CSV 파일이 다운로드되었습니다.', 'success');
    }

    function downloadJSON() {
        if (data.length === 0) {
            log('다운로드할 데이터가 없습니다.', 'warning');
            return;
        }
        const jsonContent = JSON.stringify(data, null, 2);
        const blob = new Blob([jsonContent], { type: "application/json" });
        const url = URL.createObjectURL(blob);
        const link = document.createElement("a");
        link.setAttribute("href", url);
        const filename = "data.json";
        link.setAttribute("download", filename);
        document.body.appendChild(link);
        link.click();
        document.body.removeChild(link);
        log('JSON 파일이 다운로드되었습니다.', 'success');
    }

    function downloadXML() {
        if (data.length === 0) {
            log('다운로드할 데이터가 없습니다.', 'warning');
            return;
        }
        let xmlContent = '<?xml version="1.0" encoding="UTF-8"?>\n<data>\n';
        data.forEach(d => {
            xmlContent += `  <entry>\n`;
            if (settings.dataFields.name) xmlContent += `    <name>${escapeXML(d.name)}</name>\n`;
            if (settings.dataFields.url) xmlContent += `    <url>${escapeXML(d.url)}</url>\n`;
            if (settings.dataFields.description) xmlContent += `    <description>${escapeXML(d.description)}</description>\n`;
            if (settings.dataFields.html) xmlContent += `    <html><![CDATA[${d.html}]]></html>\n`;
            if (settings.dataFields.category) xmlContent += `    <category>${escapeXML(d.category)}</category>\n`;
            if (settings.dataFields.quality) xmlContent += `    <quality>${d.quality}</quality>\n`;
            if (settings.dataFields.language) xmlContent += `    <language>${escapeXML(d.language)}</language>\n`;
            if (settings.dataFields.wordCount) xmlContent += `    <wordCount>${d.wordCount}</wordCount>\n`;
            if (settings.dataFields.metaTags) xmlContent += `    <metaTags>${escapeXML(JSON.stringify(d.metaTags))}</metaTags>\n`;
            if (settings.dataFields.headings) xmlContent += `    <headings>${escapeXML(d.headings.join('; '))}</headings>\n`;
            xmlContent += `  </entry>\n`;
        });
        xmlContent += '</data>';

        const blob = new Blob([xmlContent], { type: "application/xml" });
        const url = URL.createObjectURL(blob);
        const link = document.createElement("a");
        link.setAttribute("href", url);
        const filename = "data.xml";
        link.setAttribute("download", filename);
        document.body.appendChild(link);
        link.click();
        document.body.removeChild(link);
        log('XML 파일이 다운로드되었습니다.', 'success');
    }

    function downloadXLS() {
        if (data.length === 0) {
            log('다운로드할 데이터가 없습니다.', 'warning');
            return;
        }
        const worksheet = XLSX.utils.json_to_sheet(data.map(d => {
            const entry = {};
            if (settings.dataFields.name) entry['사이트 이름'] = d.name;
            if (settings.dataFields.url) entry['주소'] = d.url;
            if (settings.dataFields.description) entry['설명'] = d.description;
            if (settings.dataFields.html) entry['📄 HTML 내용'] = d.html;
            if (settings.dataFields.category) entry['분류'] = d.category;
            if (settings.dataFields.quality) entry['품질점수'] = d.quality;
            if (settings.dataFields.language) entry['언어'] = d.language;
            if (settings.dataFields.wordCount) entry['단어 수'] = d.wordCount;
            if (settings.dataFields.metaTags) entry['메타 태그'] = JSON.stringify(d.metaTags);
            if (settings.dataFields.headings) entry['헤딩'] = d.headings.join('; ');
            return entry;
        }));
        const workbook = XLSX.utils.book_new();
        XLSX.utils.book_append_sheet(workbook, worksheet, "Data");
        XLSX.writeFile(workbook, "data.xlsx");
        log('XLS 파일이 다운로드되었습니다.', 'success');
    }

    // XML 이스케이프 함수
    function escapeXML(str) {
        if (!str) return '';
        return str.replace(/&/g, "&amp;")
                  .replace(/</g, "&lt;")
                  .replace(/>/g, "&gt;")
                  .replace(/"/g, "&quot;")
                  .replace(/'/g, "&#039;");
    }

    // CSV 이스케이프 함수
    function escapeCSV(str) {
        if (!str) return '';
        return str.replace(/"/g, '""');
    }

    // 테이블 필터링 함수
    function filterTable(keyword) {
        $('#data-table-body tr').each(function() {
            const rowText = $(this).text().toLowerCase();
            if (rowText.includes(keyword.toLowerCase())) {
                $(this).show();
            } else {
                $(this).hide();
            }
        });
    }

    // 설정 저장 함수
    function saveSettings() {
        const delay = parseInt($('#delay-input').val(), 10);
        settings.delay = isNaN(delay) ? defaultSettings.delay : delay;

        const delayRandom = parseInt($('#delay-random-input').val(), 10);
        settings.delayRandom = isNaN(delayRandom) ? defaultSettings.delayRandom : delayRandom;

        const maxPages = parseInt($('#max-pages-input').val(), 10);
        settings.maxPages = isNaN(maxPages) ? defaultSettings.maxPages : maxPages;

        const selectedUAIndex = parseInt($('#user-agent-select').val(), 10);
        settings.userAgentIndex = isNaN(selectedUAIndex) ? 0 : selectedUAIndex;

        const contentInclude = $('#content-include-input').val().split(',').map(ct => ct.trim()).filter(ct => ct);
        const contentExclude = $('#content-exclude-input').val().split(',').map(ct => ct.trim()).filter(ct => ct);
        settings.contentTypes.include = contentInclude.length > 0 ? contentInclude : defaultSettings.contentTypes.include;
        settings.contentTypes.exclude = contentExclude.length > 0 ? contentExclude : defaultSettings.contentTypes.exclude;

        const timeout = parseInt($('#timeout-input').val(), 10);
        settings.timeout = isNaN(timeout) ? defaultSettings.timeout : timeout;

        const includeKeywords = $('#include-keywords-input').val().split('\n').map(kw => kw.trim()).filter(kw => kw);
        settings.includeKeywords = includeKeywords;

        const excludeKeywords = $('#exclude-keywords-input').val().split('\n').map(kw => kw.trim()).filter(kw => kw);
        settings.excludeKeywords = excludeKeywords;

        const regexInclude = $('#regex-include-input').val().split('\n').map(rx => rx.trim()).filter(rx => rx);
        settings.regexInclude = regexInclude;

        const regexExclude = $('#regex-exclude-input').val().split('\n').map(rx => rx.trim()).filter(rx => rx);
        settings.regexExclude = regexExclude;

        const language = $('#language-select').val() || [];
        settings.language = language;

        const minWordCount = parseInt($('#min-word-count-input').val(), 10);
        settings.minWordCount = isNaN(minWordCount) ? defaultSettings.minWordCount : minWordCount;

        const maxWordCount = parseInt($('#max-word-count-input').val(), 10);
        settings.maxWordCount = isNaN(maxWordCount) ? defaultSettings.maxWordCount : maxWordCount;

        const maxDepth = parseInt($('#max-depth-input').val(), 10);
        settings.maxDepth = isNaN(maxDepth) ? defaultSettings.maxDepth : maxDepth;

        const proxy = $('#proxy-input').val().trim();
        settings.proxy = proxy || defaultSettings.proxy;

        settings.dataFields.name = $('#field-name').is(':checked');
        settings.dataFields.url = $('#field-url').is(':checked');
        settings.dataFields.description = $('#field-description').is(':checked');
        settings.dataFields.html = $('#field-html').is(':checked');
        settings.dataFields.category = $('#field-category').is(':checked');
        settings.dataFields.quality = $('#field-quality').is(':checked');
        settings.dataFields.language = $('#field-language').is(':checked');
        settings.dataFields.wordCount = $('#field-wordCount').is(':checked');
        settings.dataFields.metaTags = $('#field-metaTags').is(':checked');
        settings.dataFields.headings = $('#field-headings').is(':checked');

        const retryAttempts = parseInt($('#retry-attempts-input').val(), 10);
        settings.retryAttempts = isNaN(retryAttempts) ? defaultSettings.retryAttempts : retryAttempts;

        const retryDelay = parseInt($('#retry-delay-input').val(), 10);
        settings.retryDelay = isNaN(retryDelay) ? defaultSettings.retryDelay : retryDelay;

        settings.schedule.enabled = $('#schedule-enabled').is(':checked');
        const startTime = $('#start-time-input').val();
        settings.schedule.startTime = startTime || defaultSettings.schedule.startTime;
        const interval = parseInt($('#interval-input').val(), 10);
        settings.schedule.interval = isNaN(interval) ? defaultSettings.schedule.interval : interval * 60000; // 분을 밀리초로 변환

        const searchEngines = {
            google: $('#engine-google').is(':checked'),
            bing: $('#engine-bing').is(':checked'),
            yahoo: $('#engine-yahoo').is(':checked'),
            duckduckgo: $('#engine-duckduckgo').is(':checked'),
            baidu: $('#engine-baidu').is(':checked'),
            yandex: $('#engine-yandex').is(':checked'),
            naver: $('#engine-naver').is(':checked'),
            daum: $('#engine-daum').is(':checked'),
            ask: $('#engine-ask').is(':checked'),
            ecosia: $('#engine-ecosia').is(':checked'),
            startpage: $('#engine-startpage').is(':checked'),
            searx: $('#engine-searx').is(':checked'),
            dogpile: $('#engine-dogpile').is(':checked'),
            bingimages: $('#engine-bingimages').is(':checked')
            // 추가 검색 엔진 옵션
        };
        settings.searchEngines = searchEngines;

        settings.respectRobots = $('#respect-robots-checkbox').is(':checked');

        const blacklist = $('#blacklist-input').val().split('\n').map(d => d.trim()).filter(d => d);
        settings.blacklist = blacklist;

        const whitelist = $('#whitelist-input').val().split('\n').map(d => d.trim()).filter(d => d);
        settings.whitelist = whitelist;

        const maxConcurrentRequests = parseInt($('#max-concurrent-requests-input').val(), 10);
        settings.maxConcurrentRequests = isNaN(maxConcurrentRequests) ? defaultSettings.maxConcurrentRequests : maxConcurrentRequests;

        settings.dataVisualization = $('#data-visualization-checkbox').is(':checked');
        GM_setValue('dataVisualization', settings.dataVisualization);

        settings.enableCaptchas = $('#enable-captchas-checkbox').is(':checked');
        GM_setValue('enableCaptchas', settings.enableCaptchas);

        settings.exportFormats.csv = $('#export-csv').is(':checked');
        settings.exportFormats.json = $('#export-json').is(':checked');
        settings.exportFormats.xml = $('#export-xml').is(':checked');
        settings.exportFormats.xls = $('#export-xls').is(':checked');

        settings.advancedLogging = $('#advanced-logging-checkbox').is(':checked');
        GM_setValue('advancedLogging', settings.advancedLogging);

        const customKeywordsText = $('#custom-keywords-input').val();
        const customKeywords = parseCustomKeywords(customKeywordsText);
        settings.customKeywords = customKeywords;
        GM_setValue('customKeywords', customKeywords);

        GM_setValue('delay', settings.delay);
        GM_setValue('delayRandom', settings.delayRandom);
        GM_setValue('maxPages', settings.maxPages);
        GM_setValue('userAgentIndex', settings.userAgentIndex);
        GM_setValue('contentTypes', settings.contentTypes);
        GM_setValue('timeout', settings.timeout);
        GM_setValue('includeKeywords', settings.includeKeywords);
        GM_setValue('excludeKeywords', settings.excludeKeywords);
        GM_setValue('regexInclude', settings.regexInclude);
        GM_setValue('regexExclude', settings.regexExclude);
        GM_setValue('language', settings.language);
        GM_setValue('minWordCount', settings.minWordCount);
        GM_setValue('maxWordCount', settings.maxWordCount);
        GM_setValue('maxDepth', settings.maxDepth);
        GM_setValue('proxy', settings.proxy);
        GM_setValue('dataFields', settings.dataFields);
        GM_setValue('retryAttempts', settings.retryAttempts);
        GM_setValue('retryDelay', settings.retryDelay);
        GM_setValue('schedule', settings.schedule);
        GM_setValue('searchEngines', settings.searchEngines);
        GM_setValue('blacklist', settings.blacklist);
        GM_setValue('whitelist', settings.whitelist);
        GM_setValue('respectRobots', settings.respectRobots);
        GM_setValue('maxConcurrentRequests', settings.maxConcurrentRequests);
        GM_setValue('exportFormats', settings.exportFormats);

        totalPages = settings.maxPages;
        GM_setValue('totalPages', totalPages);

        log('설정이 저장되었습니다.', 'success');

        if (settings.schedule.enabled) {
            scheduleCrawling();
        }

        updateDataUI();
        if (settings.dataVisualization) {
            renderChart();
        }
    }

    // 설정 초기화 함수
    function resetSettings() {
        if (confirm("기본 설정으로 초기화하시겠습니까?")) {
            GM_deleteValue('delay');
            GM_deleteValue('delayRandom');
            GM_deleteValue('maxPages');
            GM_deleteValue('userAgents');
            GM_deleteValue('userAgentIndex');
            GM_deleteValue('blacklist');
            GM_deleteValue('whitelist');
            GM_deleteValue('contentTypes');
            GM_deleteValue('timeout');
            GM_deleteValue('includeKeywords');
            GM_deleteValue('excludeKeywords');
            GM_deleteValue('regexInclude');
            GM_deleteValue('regexExclude');
            GM_deleteValue('language');
            GM_deleteValue('minWordCount');
            GM_deleteValue('maxWordCount');
            GM_deleteValue('maxDepth');
            GM_deleteValue('proxy');
            GM_deleteValue('dataFields');
            GM_deleteValue('retryAttempts');
            GM_deleteValue('retryDelay');
            GM_deleteValue('schedule');
            GM_deleteValue('searchEngines');
            GM_deleteValue('exportFormats');
            GM_deleteValue('respectRobots');
            GM_deleteValue('maxConcurrentRequests');
            GM_deleteValue('dataVisualization');
            GM_deleteValue('advancedLogging');
            GM_deleteValue('enableCaptchas');
            GM_deleteValue('customKeywords');

            Object.assign(settings, defaultSettings);
            GM_setValue('userAgents', settings.userAgents);
            GM_setValue('userAgentIndex', settings.userAgentIndex);
            GM_setValue('contentTypes', settings.contentTypes);
            GM_setValue('dataFields', settings.dataFields);
            GM_setValue('exportFormats', settings.exportFormats);
            GM_setValue('advancedLogging', settings.advancedLogging);
            GM_setValue('enableCaptchas', settings.enableCaptchas);

            $('#delay-input').val(settings.delay);
            $('#delay-random-input').val(settings.delayRandom);
            $('#max-pages-input').val(settings.maxPages);
            updateUserAgentSelect();
            $('#content-include-input').val(settings.contentTypes.include.join(','));
            $('#content-exclude-input').val(settings.contentTypes.exclude.join(','));
            $('#timeout-input').val(settings.timeout);
            $('#include-keywords-input').val(settings.includeKeywords.join('\n'));
            $('#exclude-keywords-input').val(settings.excludeKeywords.join('\n'));
            $('#regex-include-input').val(settings.regexInclude.join('\n'));
            $('#regex-exclude-input').val(settings.regexExclude.join('\n'));
            $('#language-select').val(settings.language);
            $('#min-word-count-input').val(settings.minWordCount);
            $('#max-word-count-input').val(settings.maxWordCount);
            $('#max-depth-input').val(settings.maxDepth);
            $('#proxy-input').val(settings.proxy);
            $('#field-name').prop('checked', settings.dataFields.name);
            $('#field-url').prop('checked', settings.dataFields.url);
            $('#field-description').prop('checked', settings.dataFields.description);
            $('#field-html').prop('checked', settings.dataFields.html);
            $('#field-category').prop('checked', settings.dataFields.category);
            $('#field-quality').prop('checked', settings.dataFields.quality);
            $('#field-language').prop('checked', settings.dataFields.language);
            $('#field-wordCount').prop('checked', settings.dataFields.wordCount);
            $('#field-metaTags').prop('checked', settings.dataFields.metaTags);
            $('#field-headings').prop('checked', settings.dataFields.headings);
            $('#retry-attempts-input').val(settings.retryAttempts);
            $('#retry-delay-input').val(settings.retryDelay);
            $('#schedule-enabled').prop('checked', settings.schedule.enabled);
            $('#start-time-input').val(settings.schedule.startTime);
            $('#interval-input').val(settings.schedule.interval / 60000);
            $('#engine-google').prop('checked', settings.searchEngines.google);
            $('#engine-bing').prop('checked', settings.searchEngines.bing);
            $('#engine-yahoo').prop('checked', settings.searchEngines.yahoo);
            $('#engine-duckduckgo').prop('checked', settings.searchEngines.duckduckgo);
            $('#engine-baidu').prop('checked', settings.searchEngines.baidu);
            $('#engine-yandex').prop('checked', settings.searchEngines.yandex);
            $('#engine-naver').prop('checked', settings.searchEngines.naver);
            $('#engine-daum').prop('checked', settings.searchEngines.daum);
            $('#engine-ask').prop('checked', settings.searchEngines.ask);
            $('#engine-ecosia').prop('checked', settings.searchEngines.ecosia);
            $('#engine-startpage').prop('checked', settings.searchEngines.startpage);
            $('#engine-searx').prop('checked', settings.searchEngines.searx);
            $('#engine-dogpile').prop('checked', settings.searchEngines.dogpile);
            $('#engine-bingimages').prop('checked', settings.searchEngines.bingimages);
            $('#respect-robots-checkbox').prop('checked', settings.respectRobots);
            $('#blacklist-input').val(settings.blacklist.join('\n'));
            $('#whitelist-input').val(settings.whitelist.join('\n'));
            $('#max-concurrent-requests-input').val(settings.maxConcurrentRequests);
            $('#data-visualization-checkbox').prop('checked', settings.dataVisualization);
            $('#enable-captchas-checkbox').prop('checked', settings.enableCaptchas);
            $('#export-csv').prop('checked', settings.exportFormats.csv);
            $('#export-json').prop('checked', settings.exportFormats.json);
            $('#export-xml').prop('checked', settings.exportFormats.xml);
            $('#export-xls').prop('checked', settings.exportFormats.xls);
            $('#advanced-logging-checkbox').prop('checked', settings.advancedLogging);
            $('#custom-keywords-input').val(getCustomKeywordsText());

            updateUserAgentSelect();
            updateDataUI();
            renderChart();
            log('기본 설정으로 초기화되었습니다.', 'success');
        }
    }

    // 데이터 초기화 함수
    function resetData() {
        if (confirm("수집된 데이터와 로그를 초기화하시겠습니까? 이 작업은 되돌릴 수 없습니다.")) {
            GM_setValue('data', []);
            GM_setValue('logs', []);
            GM_setValue('visitedUrls', []);
            GM_setValue('progress', 0);
            GM_setValue('currentPage', 1);
            GM_setValue('currentDepth', 0);
            GM_setValue('isCrawling', false);

            data = [];
            logs = [];
            visitedUrls = [];
            progress = 0;
            currentPage = 1;
            currentDepth = 0;
            isCrawling = false;
            requestQueue = [];
            activeRequests = 0;

            updateLogsUI();
            updateDataUI();
            setButtonStates();

            log('수집된 데이터와 로그가 초기화되었습니다.', 'success');
        }
    }

    // 크롤링 시작 함수
    async function startCrawling() {
        if (isCrawling) {
            log('이미 크롤링이 진행 중입니다.', 'warning');
            return;
        }
        if (!isAnySearchEngineEnabled()) {
            log('사용할 검색 엔진이 하나도 선택되지 않았습니다.', 'error');
            return;
        }
        isCrawling = true;
        GM_setValue('isCrawling', isCrawling);
        log('크롤링을 시작합니다.', 'info');
        setButtonStates();
        currentPage = GM_getValue('currentPage', 1);
        totalPages = GM_getValue('totalPages', settings.maxPages);
        GM_setValue('totalPages', totalPages);
        currentDepth = GM_getValue('currentDepth', 0);
        await crawlPage(currentDepth);
    }

    // 크롤링 중지 함수
    function stopCrawling() {
        if (!isCrawling) {
            log('크롤링이 진행 중이지 않습니다.', 'warning');
            return;
        }
        isCrawling = false;
        GM_setValue('isCrawling', isCrawling);
        log('크롤링이 중지되었습니다.', 'warning');
        setButtonStates();
    }

    // 버튼 상태 설정 함수
    function setButtonStates() {
        if (isCrawling) {
            $('#start-crawling').prop('disabled', true);
            $('#stop-crawling').prop('disabled', false);
        } else {
            $('#start-crawling').prop('disabled', false);
            $('#stop-crawling').prop('disabled', true);
        }
    }

    // 크롤링 함수
    async function crawlPage(currentDepthLevel) {
        try {
            while (isCrawling && currentPage <= totalPages && currentDepthLevel <= settings.maxDepth) {
                log(`페이지 ${currentPage} 크롤링 시작. (깊이: ${currentDepthLevel})`, 'info');

                if (settings.respectRobots) {
                    const robotsAllowed = await checkRobotsTxt(window.location.href);
                    if (!robotsAllowed) {
                        log(`robots.txt에 의해 크롤링이 금지된 페이지: ${window.location.href}`, 'warning');
                        break;
                    }
                }

                const searchResults = extractSearchResults();
                if (!searchResults) {
                    log('검색 결과를 찾을 수 없습니다.', 'error');
                    break;
                }

                for (let result of searchResults) {
                    if (!isCrawling) break;

                    const name = result.name;
                    const link = result.url;
                    const description = result.description;

                    if (visitedUrls.includes(link)) {
                        log(`중복된 URL 발견: ${link}`, 'warning');
                        continue;
                    }

                    if (settings.whitelist.length > 0 && !settings.whitelist.some(domain => link.includes(domain))) {
                        log(`화이트리스트에 없는 도메인: ${link}`, 'warning');
                        continue;
                    }
                    if (settings.blacklist.some(domain => link.includes(domain))) {
                        log(`블랙리스트에 있는 도메인: ${link}`, 'warning');
                        continue;
                    }

                    if (settings.language.length > 0) {
                        const pageLanguage = await getPageLanguage(link);
                        if (!settings.language.includes(pageLanguage)) {
                            log(`선택된 언어에 맞지 않는 페이지: ${link} (${pageLanguage})`, 'warning');
                            continue;
                        }
                    }

                    if (settings.includeKeywords.length > 0 && !settings.includeKeywords.some(kw => link.includes(kw) || description.includes(kw))) {
                        log(`포함 키워드가 없는 URL: ${link}`, 'warning');
                        continue;
                    }
                    if (settings.excludeKeywords.some(kw => link.includes(kw) || description.includes(kw))) {
                        log(`제외 키워드가 포함된 URL: ${link}`, 'warning');
                        continue;
                    }

                    if (settings.regexInclude.length > 0) {
                        const matchesInclude = settings.regexInclude.some(pattern => new RegExp(pattern).test(link) || new RegExp(pattern).test(description));
                        if (!matchesInclude) {
                            log(`정규 표현식 포함 조건에 맞지 않는 URL: ${link}`, 'warning');
                            continue;
                        }
                    }
                    if (settings.regexExclude.length > 0) {
                        const matchesExclude = settings.regexExclude.some(pattern => new RegExp(pattern).test(link) || new RegExp(pattern).test(description));
                        if (matchesExclude) {
                            log(`정규 표현식 제외 조건에 맞는 URL: ${link}`, 'warning');
                            continue;
                        }
                    }

                    requestQueue.push({ url: link, depth: currentDepthLevel });

                    if (activeRequests >= settings.maxConcurrentRequests) {
                        break;
                    }
                }

                processQueue();

                progress = (currentPage / totalPages) * 100;
                GM_setValue('progress', progress);
                updateLogsUI();

                if (currentPage < totalPages && isCrawling) {
                    const nextPageUrl = getNextPageUrl();
                    if (nextPageUrl) {
                        currentPage++;
                        GM_setValue('currentPage', currentPage);
                        currentDepth++;
                        GM_setValue('currentDepth', currentDepth);
                        log(`다음 페이지로 이동: ${currentPage}`, 'info');
                        await dynamicDelay();
                        window.location.href = nextPageUrl;
                        return;
                    } else {
                        log('다음 페이지 링크를 찾을 수 없습니다.', 'error');
                        break;
                    }
                } else {
                    break;
                }
            }

            if (isCrawling && currentPage > totalPages && activeRequests === 0 && requestQueue.length === 0) {
                log('크롤링이 완료되었습니다.', 'success');
                isCrawling = false;
                GM_setValue('isCrawling', isCrawling);
                setButtonStates();
                GM_setValue('progress', 100);
                $('.progress-bar-fill').css('width', `100%`);
                $('.status-text').text(`진행률: 100% (${currentPage}/${totalPages} 페이지)`);
            }
        } catch (error) {
            log(`크롤링 중 오류 발생: ${error.message}`, 'error');
        } finally {
            if (!isCrawling) {
                GM_setValue('isCrawling', isCrawling);
                setButtonStates();
                log('크롤링이 중지되었습니다.', 'info');
            }
        }
    }

    // 동시 요청 제어 함수
    async function processQueue() {
        while (requestQueue.length > 0 && activeRequests < settings.maxConcurrentRequests) {
            const { url, depth } = requestQueue.shift();
            activeRequests++;
            try {
                await handleRequest(url, depth);
            } catch (error) {
                log(`요청 처리 중 오류 발생: ${url} - ${error.message}`, 'error');
            } finally {
                activeRequests--;
                processQueue();
            }
        }
    }

    async function handleRequest(url, depthLevel) {
        if (!isCrawling) return;

        if (visitedUrls.includes(url)) {
            log(`중복된 URL 발견: ${url}`, 'warning');
            return;
        }

        if (settings.whitelist.length > 0 && !settings.whitelist.some(domain => url.includes(domain))) {
            log(`화이트리스트에 없는 도메인: ${url}`, 'warning');
            return;
        }
        if (settings.blacklist.some(domain => url.includes(domain))) {
            log(`블랙리스트에 있는 도메인: ${url}`, 'warning');
            return;
        }

        if (settings.language.length > 0) {
            const pageLanguage = await getPageLanguage(url);
            if (!settings.language.includes(pageLanguage)) {
                log(`선택된 언어에 맞지 않는 페이지: ${url} (${pageLanguage})`, 'warning');
                return;
            }
        }

        if (settings.includeKeywords.length > 0 && !settings.includeKeywords.some(kw => url.includes(kw))) {
            log(`포함 키워드가 없는 URL: ${url}`, 'warning');
            return;
        }
        if (settings.excludeKeywords.some(kw => url.includes(kw))) {
            log(`제외 키워드가 포함된 URL: ${url}`, 'warning');
            return;
        }

        if (settings.regexInclude.length > 0) {
            const matchesInclude = settings.regexInclude.some(pattern => new RegExp(pattern).test(url));
            if (!matchesInclude) {
                log(`정규 표현식 포함 조건에 맞지 않는 URL: ${url}`, 'warning');
                return;
            }
        }
        if (settings.regexExclude.length > 0) {
            const matchesExclude = settings.regexExclude.some(pattern => new RegExp(pattern).test(url));
            if (matchesExclude) {
                log(`정규 표현식 제외 조건에 맞는 URL: ${url}`, 'warning');
                return;
            }
        }

        try {
            const response = await fetchWithRetry(url, settings.retryAttempts, settings.retryDelay);
            const contentType = getContentType(response);
            if (!isContentTypeAllowed(contentType)) {
                log(`허용되지 않은 콘텐츠 유형(${contentType})의 URL: ${url}`, 'warning');
                return;
            }

            const pageHtml = response.responseText;
            const parser = new DOMParser();
            const pageDoc = parser.parseFromString(pageHtml, 'text/html');
            const textContent = pageDoc.body ? pageDoc.body.innerText : '';

            if (settings.enableCaptchas && detectCaptcha(pageDoc)) {
                await handleCaptcha(url);
                return;
            }

            const wordCount = textContent.split(/\s+/).length;
            if (wordCount < settings.minWordCount) {
                log(`단어 수가 최소 기준 (${settings.minWordCount})에 미치지 못하는 URL: ${url} (${wordCount} 단어)`, 'warning');
                return;
            }
            if (wordCount > settings.maxWordCount) {
                log(`단어 수가 최대 기준 (${settings.maxWordCount})을 초과하는 URL: ${url} (${wordCount} 단어)`, 'warning');
                return;
            }

            const qualityScore = evaluateContentQuality(textContent);
            const language = pageDoc.documentElement.lang || 'unknown';
            const category = classifyKeywords(textContent);
            const metaTags = extractMetaTags(pageDoc);
            const headings = extractHeadings(pageDoc);

            if (category !== '기타' || qualityScore > 100) {
                const dataEntry = {};
                if (settings.dataFields.name) dataEntry.name = extractName(pageDoc) || 'No Title';
                if (settings.dataFields.url) dataEntry.url = url;
                if (settings.dataFields.description) dataEntry.description = extractDescription(pageDoc) || 'No Description';
                if (settings.dataFields.html) dataEntry.html = pageHtml;
                if (settings.dataFields.category) dataEntry.category = category;
                if (settings.dataFields.quality) dataEntry.quality = qualityScore;
                if (settings.dataFields.language) dataEntry.language = language;
                if (settings.dataFields.wordCount) dataEntry.wordCount = wordCount;
                if (settings.dataFields.metaTags) dataEntry.metaTags = metaTags;
                if (settings.dataFields.headings) dataEntry.headings = headings;

                data.push(dataEntry);
                GM_setValue('data', data);
                updateDataUI();
                log(`데이터 수집 완료: ${url}`, 'success');

                if (settings.dataVisualization) {
                    renderChart();
                }
            } else {
                log(`조건에 맞지 않아 데이터에 추가되지 않은 URL: ${url}`, 'info');
            }

            if (depthLevel < settings.maxDepth) {
                const additionalLinks = extractLinks(pageDoc);
                for (let additionalLink of additionalLinks) {
                    if (!visitedUrls.includes(additionalLink) && isValidURL(additionalLink)) {
                        requestQueue.push({ url: additionalLink, depth: depthLevel + 1 });
                    }
                }
            }

        } catch (error) {
            log(`데이터 수집 실패: ${url} - ${error.message}`, 'error');
        }

        visitedUrls.push(url);
        GM_setValue('visitedUrls', visitedUrls);
    }

    // 크롤링 함수에 재시도 추가
    async function fetchWithRetry(url, attempts, delayMs) {
        for (let attempt = 1; attempt <= attempts; attempt++) {
            try {
                const response = await GM_xmlhttpRequestPromise(url);
                return response;
            } catch (error) {
                log(`요청 실패 (${attempt}/${attempts}): ${url} - ${error.message}`, 'warning');
                if (attempt < attempts) {
                    await delay(delayMs);
                } else {
                    throw error;
                }
            }
        }
    }

    // 활성화된 검색 엔진이 있는지 확인
    function isAnySearchEngineEnabled() {
        return Object.values(settings.searchEngines).some(enabled => enabled);
    }

    // 사용자 지정 키워드 분류 통합
    function classifyKeywords(text) {
        const standardKeywords = combineKeywords();
        for (const [category, kws] of Object.entries(standardKeywords)) {
            if (kws.some(kw => text.includes(kw))) {
                return category;
            }
        }
        return '기타';
    }

    // 검색 결과 추출 함수
    function extractSearchResults() {
        let results = [];
        if (window.location.hostname.includes('google.com') && settings.searchEngines.google) {
            const items = document.querySelectorAll('div.g, div[data-hveid]');
            items.forEach(item => {
                const nameElement = item.querySelector('h3');
                const linkElement = item.querySelector('a');
                let descElement = item.querySelector('.aCOpRe');
                if (!descElement) {
                    descElement = item.querySelector('.VwiC3b');
                }

                const name = nameElement ? nameElement.innerText.trim() : 'No Title';
                const link = linkElement ? linkElement.href : '';
                const description = descElement ? descElement.innerText.trim() : 'No Description';

                if (link && isValidURL(link)) results.push({ name, url: link, description });
            });
        } else if (window.location.hostname.includes('bing.com') && settings.searchEngines.bing) {
            const items = document.querySelectorAll('.b_algo');
            items.forEach(item => {
                const nameElement = item.querySelector('h2');
                const linkElement = nameElement ? nameElement.querySelector('a') : null;
                const descElement = item.querySelector('.b_caption p');

                const name = nameElement ? nameElement.innerText.trim() : 'No Title';
                const link = linkElement ? linkElement.href : '';
                const description = descElement ? descElement.innerText.trim() : 'No Description';

                if (link && isValidURL(link)) results.push({ name, url: link, description });
            });
        } else if (window.location.hostname.includes('search.yahoo.com') && settings.searchEngines.yahoo) {
            const items = document.querySelectorAll('div.dd.algo');
            items.forEach(item => {
                const nameElement = item.querySelector('h3.title');
                const linkElement = nameElement ? nameElement.querySelector('a') : null;
                const descElement = item.querySelector('.compText p');

                const name = nameElement ? nameElement.innerText.trim() : 'No Title';
                const link = linkElement ? linkElement.href : '';
                const description = descElement ? descElement.innerText.trim() : 'No Description';

                if (link && isValidURL(link)) results.push({ name, url: link, description });
            });
        } else if (window.location.hostname.includes('naver.com') && settings.searchEngines.naver) {
            const items = document.querySelectorAll('.news.mynews.section._prs_nws');
            items.forEach(item => {
                const nameElement = item.querySelector('.news_area .news_tit');
                const linkElement = nameElement ? nameElement.href : '';
                const descElement = item.querySelector('.dsc_wrap');

                const name = nameElement ? nameElement.innerText.trim() : 'No Title';
                const link = linkElement ? linkElement : '';
                const description = descElement ? descElement.innerText.trim() : 'No Description';

                if (link && isValidURL(link)) results.push({ name, url: link, description });
            });
        } else if (window.location.hostname.includes('daum.net') && settings.searchEngines.daum) {
            const items = document.querySelectorAll('.wrap_cont div');
            items.forEach(item => {
                const nameElement = item.querySelector('a.f_link_b');
                const linkElement = nameElement ? nameElement.href : '';
                const descElement = item.querySelector('.desc');

                const name = nameElement ? nameElement.innerText.trim() : 'No Title';
                const link = linkElement ? linkElement : '';
                const description = descElement ? descElement.innerText.trim() : 'No Description';

                if (link && isValidURL(link)) results.push({ name, url: link, description });
            });
        } else if (window.location.hostname.includes('duckduckgo.com') && settings.searchEngines.duckduckgo) {
            const items = document.querySelectorAll('.result');
            items.forEach(item => {
                const nameElement = item.querySelector('h2 a.result__a');
                const linkElement = nameElement ? nameElement.href : '';
                const descElement = item.querySelector('.result__snippet');

                const name = nameElement ? nameElement.innerText.trim() : 'No Title';
                const link = linkElement ? linkElement : '';
                const description = descElement ? descElement.innerText.trim() : 'No Description';

                if (link && isValidURL(link)) results.push({ name, url: link, description });
            });
        } else if (window.location.hostname.includes('baidu.com') && settings.searchEngines.baidu) {
            const items = document.querySelectorAll('.result');
            items.forEach(item => {
                const nameElement = item.querySelector('h3 a');
                const linkElement = nameElement ? nameElement.href : '';
                const descElement = item.querySelector('.c-abstract');

                const name = nameElement ? nameElement.innerText.trim() : 'No Title';
                const link = linkElement ? linkElement : '';
                const description = descElement ? descElement.innerText.trim() : 'No Description';

                if (link && isValidURL(link)) results.push({ name, url: link, description });
            });
        } else if (window.location.hostname.includes('yandex.com') && settings.searchEngines.yandex) {
            const items = document.querySelectorAll('.serp-item');
            items.forEach(item => {
                const nameElement = item.querySelector('.organic__title-link');
                const linkElement = nameElement ? nameElement.href : '';
                const descElement = item.querySelector('.text-container');

                const name = nameElement ? nameElement.innerText.trim() : 'No Title';
                const link = linkElement ? linkElement : '';
                const description = descElement ? descElement.innerText.trim() : 'No Description';

                if (link && isValidURL(link)) results.push({ name, url: link, description });
            });
        } else if (window.location.hostname.includes('ask.com') && settings.searchEngines.ask) {
            const items = document.querySelectorAll('.PartialSearchResults-item');
            items.forEach(item => {
                const nameElement = item.querySelector('h2 a');
                const linkElement = nameElement ? nameElement.href : '';
                const descElement = item.querySelector('.PartialSearchResults-item-abstract');

                const name = nameElement ? nameElement.innerText.trim() : 'No Title';
                const link = linkElement ? linkElement : '';
                const description = descElement ? descElement.innerText.trim() : 'No Description';

                if (link && isValidURL(link)) results.push({ name, url: link, description });
            });
        } else if (window.location.hostname.includes('ecosia.org') && settings.searchEngines.ecosia) {
            const items = document.querySelectorAll('.result-item');
            items.forEach(item => {
                const nameElement = item.querySelector('.result-title a');
                const linkElement = nameElement ? nameElement.href : '';
                const descElement = item.querySelector('.result-description');

                const name = nameElement ? nameElement.innerText.trim() : 'No Title';
                const link = linkElement ? linkElement : '';
                const description = descElement ? descElement.innerText.trim() : 'No Description';

                if (link && isValidURL(link)) results.push({ name, url: link, description });
            });
        } else if (window.location.hostname.includes('startpage.com') && settings.searchEngines.startpage) {
            const items = document.querySelectorAll('.w-gl__result');
            items.forEach(item => {
                const nameElement = item.querySelector('.w-gl__result-title a');
                const linkElement = nameElement ? nameElement.href : '';
                const descElement = item.querySelector('.w-gl__result-text');

                const name = nameElement ? nameElement.innerText.trim() : 'No Title';
                const link = linkElement ? linkElement : '';
                const description = descElement ? descElement.innerText.trim() : 'No Description';

                if (link && isValidURL(link)) results.push({ name, url: link, description });
            });
        } else if (window.location.hostname.includes('searx') && settings.searchEngines.searx) {
            const items = document.querySelectorAll('.result');
            items.forEach(item => {
                const nameElement = item.querySelector('.result_title a');
                const linkElement = nameElement ? nameElement.href : '';
                const descElement = item.querySelector('.result_desc');

                const name = nameElement ? nameElement.innerText.trim() : 'No Title';
                const link = linkElement ? linkElement : '';
                const description = descElement ? descElement.innerText.trim() : 'No Description';

                if (link && isValidURL(link)) results.push({ name, url: link, description });
            });
        } else if (window.location.hostname.includes('dogpile.com') && settings.searchEngines.dogpile) {
            const items = document.querySelectorAll('.result_info');
            items.forEach(item => {
                const nameElement = item.querySelector('h3 a');
                const linkElement = nameElement ? nameElement.href : '';
                const descElement = item.querySelector('.result_snippet');

                const name = nameElement ? nameElement.innerText.trim() : 'No Title';
                const link = linkElement ? linkElement : '';
                const description = descElement ? descElement.innerText.trim() : 'No Description';

                if (link && isValidURL(link)) results.push({ name, url: link, description });
            });
        } else if (window.location.hostname.includes('bing.com') && settings.searchEngines.bingimages) {
            const items = document.querySelectorAll('.iusc');
            items.forEach(item => {
                const nameElement = item.querySelector('a.iusc');
                const linkElement = nameElement ? nameElement.href : '';
                const descElement = item.querySelector('.txt');

                const name = nameElement ? nameElement.innerText.trim() : 'No Title';
                const link = linkElement ? linkElement : '';
                const description = descElement ? descElement.innerText.trim() : 'No Description';

                if (link && isValidURL(link)) results.push({ name, url: link, description });
            });
        } else {
            log('지원되지 않거나 비활성화된 검색 엔진입니다.', 'warning');
        }

        return results.length > 0 ? results : null;
    }

    // 다음 페이지 URL 얻는 함수
    function getNextPageUrl() {
        let nextPageUrl = null;
        if (window.location.hostname.includes('google.com') && settings.searchEngines.google) {
            const nextLink = document.querySelector('a#pnnext');
            if (nextLink) nextPageUrl = nextLink.href;
        } else if (window.location.hostname.includes('bing.com') && settings.searchEngines.bing) {
            const nextLink = document.querySelector('a.sb_pagN');
            if (nextLink) nextPageUrl = nextLink.href;
        } else if (window.location.hostname.includes('search.yahoo.com') && settings.searchEngines.yahoo) {
            const nextLink = document.querySelector('a.next');
            if (nextLink) nextPageUrl = nextLink.href;
        } else if (window.location.hostname.includes('naver.com') && settings.searchEngines.naver) {
            const nextLink = document.querySelector('.sc_page_inner a.btn_next');
            if (nextLink) nextPageUrl = nextLink.href;
        } else if (window.location.hostname.includes('daum.net') && settings.searchEngines.daum) {
            const nextLink = document.querySelector('.paging a.btn_next');
            if (nextLink) nextPageUrl = nextLink.href;
        } else if (window.location.hostname.includes('duckduckgo.com') && settings.searchEngines.duckduckgo) {
            const nextLink = document.querySelector('a.js-next-page');
            if (nextLink) nextPageUrl = nextLink.href;
        } else if (window.location.hostname.includes('baidu.com') && settings.searchEngines.baidu) {
            const nextLink = document.querySelector('.n');
            if (nextLink) nextPageUrl = nextLink.href;
        } else if (window.location.hostname.includes('yandex.com') && settings.searchEngines.yandex) {
            const nextLink = document.querySelector('.pager__item_kind_next a');
            if (nextLink) nextPageUrl = nextLink.href;
        } else if (window.location.hostname.includes('ask.com') && settings.searchEngines.ask) {
            const nextLink = document.querySelector('.pagination a.next');
            if (nextLink) nextPageUrl = nextLink.href;
        } else if (window.location.hostname.includes('ecosia.org') && settings.searchEngines.ecosia) {
            const nextLink = document.querySelector('.pagination a.next');
            if (nextLink) nextPageUrl = nextLink.href;
        } else if (window.location.hostname.includes('startpage.com') && settings.searchEngines.startpage) {
            const nextLink = document.querySelector('.pagination a.next');
            if (nextLink) nextPageUrl = nextLink.href;
        } else if (window.location.hostname.includes('searx') && settings.searchEngines.searx) {
            const nextLink = document.querySelector('.pagination a.next');
            if (nextLink) nextPageUrl = nextLink.href;
        } else if (window.location.hostname.includes('dogpile.com') && settings.searchEngines.dogpile) {
            const nextLink = document.querySelector('.pagination a.next');
            if (nextLink) nextPageUrl = nextLink.href;
        } else if (window.location.hostname.includes('bing.com') && settings.searchEngines.bingimages) {
            const nextLink = document.querySelector('a.sb_pagN');
            if (nextLink) nextPageUrl = nextLink.href;
        }

        log(`다음 페이지 URL: ${nextPageUrl || '없음'}`, 'info');
        return nextPageUrl;
    }

    // GM_xmlhttpRequest를 Promise로 래핑
    function GM_xmlhttpRequestPromise(url) {
        return new Promise((resolve, reject) => {
            let requestUrl = url;
            if (settings.proxy) {
                requestUrl = `${settings.proxy}?url=${encodeURIComponent(url)}`;
            }

            GM_xmlhttpRequest({
                method: "GET",
                url: requestUrl,
                headers: {
                    'User-Agent': getCurrentUserAgent(),
                    'Accept': '*/*'
                },
                timeout: settings.timeout,
                onload: function(response) {
                    if (response.status >= 200 && response.status < 300) {
                        resolve(response);
                    } else {
                        reject(new Error(`HTTP Status ${response.status}`));
                    }
                },
                onerror: function(error) {
                    reject(new Error('Request failed'));
                },
                ontimeout: function() {
                    reject(new Error('Request timed out'));
                }
            });
        });
    }

    // 현재 사용자 에이전트 얻는 함수
    function getCurrentUserAgent() {
        return settings.userAgents[settings.userAgentIndex % settings.userAgents.length];
    }

    // 콘텐츠 유형 얻는 함수
    function getContentType(response) {
        const contentTypeHeader = response.responseHeaders.split('\n').find(header => header.toLowerCase().startsWith('content-type'));
        return contentTypeHeader ? contentTypeHeader.split(':')[1].trim().split(';')[0] : '';
    }

    // 콘텐츠 유형 허용 여부 검사
    function isContentTypeAllowed(contentType) {
        if (settings.contentTypes.include.length > 0 && !settings.contentTypes.include.includes(contentType)) {
            return false;
        }
        if (settings.contentTypes.exclude.length > 0 && settings.contentTypes.exclude.includes(contentType)) {
            return false;
        }
        return true;
    }

    // 콘텐츠 품질 평가 함수
    function evaluateContentQuality(text) {
        const wordCount = text.split(/\s+/).length;
        return wordCount;
    }

    // 메타 태그 추출 함수
    function extractMetaTags(pageDoc) {
        const metaTags = {};
        const metas = pageDoc.querySelectorAll('meta');
        metas.forEach(meta => {
            const name = meta.getAttribute('name') || meta.getAttribute('property');
            const content = meta.getAttribute('content');
            if (name && content) {
                metaTags[name.toLowerCase()] = content;
            }
        });
        return metaTags;
    }

    // 헤딩 추출 함수
    function extractHeadings(pageDoc) {
        const headings = [];
        for (let i = 1; i <= 6; i++) {
            const hTags = pageDoc.querySelectorAll(`h${i}`);
            hTags.forEach(h => {
                headings.push(h.innerText.trim());
            });
        }
        return headings;
    }

    // URL 유효성 검사 함수
    function isValidURL(url) {
        try {
            new URL(url);
            return true;
        } catch (_) {
            return false;
        }
    }

    // robots.txt 확인 함수
    async function checkRobotsTxt(url) {
        try {
            const parsedUrl = new URL(url);
            const robotsUrl = `${parsedUrl.origin}/robots.txt`;
            const response = await GM_xmlhttpRequestPromise(robotsUrl);
            const robotsText = response.responseText;
            const parser = new RobotsParser(robotsUrl, robotsText);
            return parser.isAllowed(url, getCurrentUserAgent());
        } catch (error) {
            log(`robots.txt 확인 실패: ${error.message}`, 'warning');
            return true;
        }
    }

    // robots-parser 라이브러리
    class RobotsParser {
        constructor(url, text) {
            this.rules = [];
            this.parse(text);
        }

        parse(text) {
            const lines = text.split('\n');
            let userAgents = [];
            for (let line of lines) {
                line = line.trim();
                if (line.startsWith('#') || line === '') continue;
                const parts = line.split(':');
                if (parts.length < 2) continue;
                const key = parts[0].trim().toLowerCase();
                const value = parts.slice(1).join(':').trim();
                if (key === 'user-agent') {
                    userAgents = [value.toLowerCase()];
                } else if (key === 'disallow') {
                    this.rules.push({ userAgents: [...userAgents], disallow: value });
                } else if (key === 'allow') {
                    this.rules.push({ userAgents: [...userAgents], allow: value });
                }
            }
        }

        isAllowed(url, ua) {
            const parsedUrl = new URL(url);
            const path = parsedUrl.pathname;
            let allowed = true;
            for (let rule of this.rules) {
                if (rule.userAgents.includes('*') || rule.userAgents.includes(ua.toLowerCase())) {
                    if (rule.disallow && path.startsWith(rule.disallow)) {
                        allowed = false;
                    }
                    if (rule.allow && path.startsWith(rule.allow)) {
                        allowed = true;
                    }
                }
            }
            return allowed;
        }
    }

    // 딜레이 함수
    function delay(ms) {
        return new Promise(resolve => setTimeout(resolve, ms));
    }

    // 스케줄링 크롤링 함수
    function scheduleCrawling() {
        const [hours, minutes] = settings.schedule.startTime.split(':').map(Number);
        const now = new Date();
        const start = new Date();
        start.setHours(hours, minutes, 0, 0);
        if (start < now) {
            start.setDate(start.getDate() + 1);
        }
        const initialDelay = start - now;
        setTimeout(() => {
            startCrawling();
            setInterval(startCrawling, settings.schedule.interval);
        }, initialDelay);
        log(`스케줄링 설정됨: 매 ${settings.schedule.interval / 60000}분마다 크롤링 시작`, 'info');
    }

    // CAPTCHA 감지 함수
    function detectCaptcha(pageDoc) {
        return !!pageDoc.querySelector('iframe[src*="captcha"], div[class*="captcha"], form[action*="captcha"]');
    }

    // CAPTCHA 처리 함수
    async function handleCaptcha(url) {
        log(`CAPTCHA가 감지되었습니다: ${url}`, 'warning');
        alert(`CAPTCHA가 감지되었습니다. 수동으로 해결한 후 다시 시작해주세요.`);
        stopCrawling();
    }

    // 링크 추출 함수
    function extractLinks(pageDoc) {
        const links = [];
        const anchorTags = pageDoc.querySelectorAll('a[href]');
        anchorTags.forEach(a => {
            const href = a.href;
            if (isValidURL(href)) {
                links.push(href);
            }
        });
        return links;
    }

    // 이름 추출 함수
    function extractName(pageDoc) {
        const title = pageDoc.querySelector('title');
        return title ? title.innerText.trim() : null;
    }

    // 설명 추출 함수
    function extractDescription(pageDoc) {
        const metaDesc = pageDoc.querySelector('meta[name="description"]');
        return metaDesc ? metaDesc.getAttribute('content').trim() : null;
    }

    // 동적 딜레이 함수
    async function dynamicDelay() {
        const baseDelay = settings.delay;
        const randomPart = Math.floor(Math.random() * settings.delayRandom);
        await delay(baseDelay + randomPart);
    }

    // 초기 설정 함수
    function init() {
        createUI();
        if (isCrawling) {
            log('크롤링이 이전에 중단된 상태에서 다시 시작됩니다.', 'info');
            setButtonStates();
            crawlPage(currentDepth);
        } else {
            log('크롤러가 초기화되었습니다.', 'info');
        }
    }

    // 페이지 로드 시 초기화
    window.addEventListener('load', init);

    // 기타 기능 이벤트 핸들러 추가
    $('#data-visualization-checkbox').on('change', function() {
        settings.dataVisualization = $(this).is(':checked');
        GM_setValue('dataVisualization', settings.dataVisualization);
        $('#data-chart').toggle(settings.dataVisualization);
        log('데이터 시각화 설정이 변경되었습니다.', 'success');
        if (settings.dataVisualization) {
            renderChart();
        }
    });

    $('#enable-captchas-checkbox').on('change', function() {
        settings.enableCaptchas = $(this).is(':checked');
        GM_setValue('enableCaptchas', settings.enableCaptchas);
        log('CAPTCHA 처리 설정이 변경되었습니다.', 'success');
    });

    $('#schedule-enabled').on('change', function() {
        settings.schedule.enabled = $(this).is(':checked');
        GM_setValue('schedule.enabled', settings.schedule.enabled);
        $('#schedule-options').toggle(settings.schedule.enabled);
        if (settings.schedule.enabled) {
            scheduleCrawling();
        }
        log('스케줄링 설정이 변경되었습니다.', 'success');
    });

    // 설정 탭에 추가 설정 요소 동적 생성
    function populateSettingsTab() {
        const settingsTab = $('#tab-settings');
        const additionalSettingsHtml = `
            <div>
                <label for="content-include-input">포함 콘텐츠 유형 (쉼표로 구분):</label>
                <input type="text" id="content-include-input" value="${settings.contentTypes.include.join(',')}" style="width: 100%; padding: 5px; margin-bottom: 10px;">
            </div>
            <div>
                <label for="content-exclude-input">제외 콘텐츠 유형 (쉼표로 구분):</label>
                <input type="text" id="content-exclude-input" value="${settings.contentTypes.exclude.join(',')}" style="width: 100%; padding: 5px; margin-bottom: 10px;">
            </div>
            <div>
                <label for="timeout-input">타임아웃 (ms):</label>
                <input type="number" id="timeout-input" value="${settings.timeout}" style="width: 100%; padding: 5px; margin-bottom: 10px;">
            </div>
            <div>
                <label for="include-keywords-input">포함 키워드 (한 줄에 하나씩):</label>
                <textarea id="include-keywords-input" rows="3" style="width: 100%; padding: 5px; margin-bottom: 10px;">${settings.includeKeywords.join('\n')}</textarea>
            </div>
            <div>
                <label for="exclude-keywords-input">제외 키워드 (한 줄에 하나씩):</label>
                <textarea id="exclude-keywords-input" rows="3" style="width: 100%; padding: 5px; margin-bottom: 10px;">${settings.excludeKeywords.join('\n')}</textarea>
            </div>
            <div>
                <label for="regex-include-input">정규 표현식 포함 (한 줄에 하나씩):</label>
                <textarea id="regex-include-input" rows="3" style="width: 100%; padding: 5px; margin-bottom: 10px;">${settings.regexInclude.join('\n')}</textarea>
            </div>
            <div>
                <label for="regex-exclude-input">정규 표현식 제외 (한 줄에 하나씩):</label>
                <textarea id="regex-exclude-input" rows="3" style="width: 100%; padding: 5px; margin-bottom: 10px;">${settings.regexExclude.join('\n')}</textarea>
            </div>
            <div>
                <label for="language-select">언어 선택:</label>
                <select id="language-select" multiple style="width: 100%; padding: 5px; margin-bottom: 10px;">
                    <option value="en" ${settings.language.includes('en') ? 'selected' : ''}>영어</option>
                    <option value="ko" ${settings.language.includes('ko') ? 'selected' : ''}>한국어</option>
                    <option value="zh" ${settings.language.includes('zh') ? 'selected' : ''}>중국어</option>
                    <option value="ja" ${settings.language.includes('ja') ? 'selected' : ''}>일본어</option>
                    <!-- 필요한 언어 옵션 추가 -->
                </select>
            </div>
            <div>
                <label for="min-word-count-input">최소 단어 수:</label>
                <input type="number" id="min-word-count-input" value="${settings.minWordCount}" style="width: 100%; padding: 5px; margin-bottom: 10px;">
            </div>
            <div>
                <label for="max-word-count-input">최대 단어 수:</label>
                <input type="number" id="max-word-count-input" value="${settings.maxWordCount}" style="width: 100%; padding: 5px; margin-bottom: 10px;">
            </div>
            <div>
                <label for="max-depth-input">최대 깊이:</label>
                <input type="number" id="max-depth-input" value="${settings.maxDepth}" style="width: 100%; padding: 5px; margin-bottom: 10px;">
            </div>
            <div>
                <label for="proxy-input">프록시 URL:</label>
                <input type="text" id="proxy-input" value="${settings.proxy}" style="width: 100%; padding: 5px; margin-bottom: 10px;">
            </div>
            <div>
                <label>검색 엔진 활성화:</label><br>
                <label><input type="checkbox" id="engine-google" ${settings.searchEngines.google ? 'checked' : ''}> Google</label><br>
                <label><input type="checkbox" id="engine-bing" ${settings.searchEngines.bing ? 'checked' : ''}> Bing</label><br>
                <label><input type="checkbox" id="engine-yahoo" ${settings.searchEngines.yahoo ? 'checked' : ''}> Yahoo</label><br>
                <label><input type="checkbox" id="engine-duckduckgo" ${settings.searchEngines.duckduckgo ? 'checked' : ''}> DuckDuckGo</label><br>
                <label><input type="checkbox" id="engine-baidu" ${settings.searchEngines.baidu ? 'checked' : ''}> Baidu</label><br>
                <label><input type="checkbox" id="engine-yandex" ${settings.searchEngines.yandex ? 'checked' : ''}> Yandex</label><br>
                <label><input type="checkbox" id="engine-naver" ${settings.searchEngines.naver ? 'checked' : ''}> Naver</label><br>
                <label><input type="checkbox" id="engine-daum" ${settings.searchEngines.daum ? 'checked' : ''}> Daum</label><br>
                <label><input type="checkbox" id="engine-ask" ${settings.searchEngines.ask ? 'checked' : ''}> Ask</label><br>
                <label><input type="checkbox" id="engine-ecosia" ${settings.searchEngines.ecosia ? 'checked' : ''}> Ecosia</label><br>
                <label><input type="checkbox" id="engine-startpage" ${settings.searchEngines.startpage ? 'checked' : ''}> Startpage</label><br>
                <label><input type="checkbox" id="engine-searx" ${settings.searchEngines.searx ? 'checked' : ''}> Searx</label><br>
                <label><input type="checkbox" id="engine-dogpile" ${settings.searchEngines.dogpile ? 'checked' : ''}> Dogpile</label><br>
                <label><input type="checkbox" id="engine-bingimages" ${settings.searchEngines.bingimages ? 'checked' : ''}> Bing Images</label><br>
                <!-- 추가 검색 엔진 옵션 -->
            </div>
            <div>
                <label><input type="checkbox" id="respect-robots-checkbox" ${settings.respectRobots ? 'checked' : ''}> robots.txt 준수</label><br>
            </div>
            <div>
                <label for="blacklist-input">블랙리스트 도메인 (한 줄에 하나씩):</label>
                <textarea id="blacklist-input" rows="3" style="width: 100%; padding: 5px; margin-bottom: 10px;">${settings.blacklist.join('\n')}</textarea>
            </div>
            <div>
                <label for="whitelist-input">화이트리스트 도메인 (한 줄에 하나씩):</label>
                <textarea id="whitelist-input" rows="3" style="width: 100%; padding: 5px; margin-bottom: 10px;">${settings.whitelist.join('\n')}</textarea>
            </div>
            <div>
                <label for="max-concurrent-requests-input">최대 동시 요청 수:</label>
                <input type="number" id="max-concurrent-requests-input" value="${settings.maxConcurrentRequests}" style="width: 100%; padding: 5px; margin-bottom: 10px;">
            </div>
            <div>
                <label><input type="checkbox" id="advanced-logging-checkbox" ${settings.advancedLogging ? 'checked' : ''}> 고급 로깅 (콘솔 출력)</label><br>
            </div>
            <div>
                <label for="custom-keywords-input">사용자 지정 키워드 (카테고리:키워드1,키워드2 ... 한 줄에 하나씩):</label>
                <textarea id="custom-keywords-input" rows="5" style="width: 100%; padding: 5px; margin-bottom: 10px;">${getCustomKeywordsText()}</textarea>
            </div>
        `;
        settingsTab.append(additionalSettingsHtml);
    }

    // 로깅 함수 재정의
    function updateLogsUI() {
        const logHtml = logs.map(log => `<div>[${log.timestamp}] [${log.type.toUpperCase()}] ${log.message}</div>`).join('');
        $('#log-messages').html(logHtml);
        $('.progress-bar-fill').css('width', `${progress.toFixed(2)}%`);
        $('.status-text').text(`진행률: ${progress.toFixed(2)}% (${currentPage}/${totalPages} 페이지)`);
    }

    // 초기화 함수 끝
    function init() {
        createUI();
        populateSettingsTab();
        if (isCrawling) {
            log('크롤링이 이전에 중단된 상태에서 다시 시작됩니다.', 'info');
            setButtonStates();
            crawlPage(currentDepth);
        } else {
            log('크롤러가 초기화되었습니다.', 'info');
        }
    }

    // 페이지 로드 시 초기화
    window.addEventListener('load', init);

    // 최적화
    // 불필요한 DOM 접근 줄이기 및 비동기 작업 최적화는 기존에 일부 진행됨
})();

여기서 이게  안 되는 문제가 있었는데 자 이제 밑에걸 반영해서 고쳐봐 ㄱㄱ 일단 최대한 아무말없이 써봐. 그냥 코드만 써. ㅇㅇ 그리고 니가 써도 되는 용량이 최대가 될때 파트 1 으로 끝어 알겠지?




1. 중복된 init() 함수 정의
스크립트 내에서 init() 함수가 두 번 정의되어 있습니다. 첫 번째 정의는 UI 생성과 초기 상태 설정을 담당하고 있으며, 두 번째 정의는 동일한 역할을 수행하려는 의도로 보입니다. 그러나 두 번째 정의가 첫 번째 정의를 덮어씌우게 되어, 초기 설정 과정이 예상대로 수행되지 않을 수 있습니다.


// 첫 번째 init 함수
function init() {
    createUI();
    // ... 기타 초기 설정
}

// 중복된 두 번째 init 함수
function init() {
    createUI();
    populateSettingsTab();
    if (isCrawling) {
        log('크롤링이 이전에 중단된 상태에서 다시 시작됩니다.', 'info');
        setButtonStates();
        crawlPage(currentDepth);
    } else {
        log('크롤러가 초기화되었습니다.', 'info');
    }
}
해결 방안:

init() 함수는 한 번만 정의하고, 모든 초기 설정 로직을 그 안에 포함시킵니다.
중복된 정의를 제거하여 초기화 과정이 올바르게 수행되도록 수정합니다.
2. UI 요소와 설정 간의 불일치
saveSettings() 함수에서는 다양한 설정 입력 요소(#content-include-input, #timeout-input 등)를 참조하지만, 이러한 요소들이 실제로 createUI() 또는 populateSettingsTab()에서 제대로 생성되지 않았을 가능성이 있습니다. 특히, populateSettingsTab()에서 동적으로 추가되는 입력 요소들이 실제로 DOM에 제대로 반영되고 있는지 확인이 필요합니다.


// saveSettings() 함수 예시
function saveSettings() {
    const delay = parseInt($('#delay-input').val(), 10);
    // ... 기타 설정 저장 로직
    const contentInclude = $('#content-include-input').val().split(',').map(ct => ct.trim()).filter(ct => ct);
    // ...
}
해결 방안:

populateSettingsTab() 함수 내에서 모든 필요한 입력 요소들이 정확히 생성되고 있는지 확인합니다.
UI 생성 로직과 설정 저장 로직이 일관되게 연결되도록 검토합니다.
3. 비동기 크롤링 로직의 문제
crawlPage() 함수 내에서 window.location.href = nextPageUrl;을 호출하여 페이지를 이동시키는데, 이는 현재 스크립트의 실행을 중단시키고 새로운 페이지로 이동하게 됩니다. 따라서 크롤링 로직이 완전히 완료되지 않을 수 있습니다.


// crawlPage() 함수 내 페이지 이동 예시
if (currentPage < totalPages && isCrawling) {
    const nextPageUrl = getNextPageUrl();
    if (nextPageUrl) {
        currentPage++;
        GM_setValue('currentPage', currentPage);
        currentDepth++;
        GM_setValue('currentDepth', currentDepth);
        log(`다음 페이지로 이동: ${currentPage}`, 'info');
        await dynamicDelay();
        window.location.href = nextPageUrl;
        return;
    } else {
        log('다음 페이지 링크를 찾을 수 없습니다.', 'error');
        break;
    }
}
해결 방안:

페이지 이동을 최소화하거나, 크롤링 로직을 백그라운드에서 지속적으로 수행할 수 있는 구조로 변경합니다.
Tampermonkey의 @include나 @match 메타데이터를 활용하여 크롤링이 필요한 페이지에서만 스크립트가 실행되도록 설정합니다.
4. 간단한 RobotsParser 구현
스크립트 내의 RobotsParser 클래스는 기본적인 robots.txt 규칙만을 처리하고 있으며, 복잡한 규칙이나 특정 조건을 정확히 반영하지 못할 수 있습니다. 예를 들어, 와일드카드(*)나 특정 User-Agent 규칙을 완벽히 처리하지 못할 가능성이 있습니다.


class RobotsParser {
    constructor(url, text) {
        this.rules = [];
        this.parse(text);
    }

    parse(text) {
        // 간단한 파서 구현
    }

    isAllowed(url, ua) {
        // 단순 규칙 적용
    }
}
해결 방안:

더 정교한 robots.txt 파서를 사용하거나, 외부 라이브러리를 통합하여 다양한 규칙을 정확히 처리할 수 있도록 개선합니다.
필요한 경우, Tampermonkey의 @require 메타데이터를 활용하여 추가적인 라이브러리를 로드할 수 있습니다.
5. 잠재적인 보안 및 성능 이슈
데이터 저장 및 처리: dataFields.html이 활성화되면 전체 페이지의 HTML 내용을 저장하게 되는데, 이는 상당한 양의 데이터를 저장하게 되어 저장 공간을 초과하거나 성능 저하를 초래할 수 있습니다.
에이전트 관리: 사용자 에이전트가 빈번하게 변경될 경우, 서버 측에서 비정상적인 트래픽으로 간주될 수 있습니다.
해결 방안:

필요하지 않은 데이터 필드는 기본적으로 비활성화하고, 사용자가 선택적으로 활성화할 수 있도록 안내합니다.
사용자 에이전트의 변경 빈도를 조절하고, 일관된 패턴을 유지하도록 설정합니다.
종합적인 해결책 예시
중복된 함수 제거 및 통합:

// 중복된 init 함수 제거 및 통합
function init() {
    createUI();
    populateSettingsTab();
    if (isCrawling) {
        log('크롤링이 이전에 중단된 상태에서 다시 시작됩니다.', 'info');
        setButtonStates();
        crawlPage(currentDepth);
    } else {
        log('크롤러가 초기화되었습니다.', 'info');
    }
}

window.addEventListener('load', init);
UI 요소 확인 및 일관성 유지:
populateSettingsTab()에서 모든 필요한 입력 요소들이 정확히 생성되는지 확인합니다.
saveSettings()에서 참조하는 모든 요소들이 실제로 존재하는지 검토합니다.
크롤링 로직 개선:
페이지 이동을 최소화하고, 가능한 경우 현재 페이지 내에서 크롤링을 완료하도록 로직을 재설계합니다.
필요 시, 페이지 이동 후 상태를 저장하고 이어서 크롤링을 계속할 수 있도록 구현합니다.

보안 및 성능 최적화:

사용자 에이전트 관리와 트래픽 패턴을 신중히 설정합니다.