innerHTML을 통한 전체 DOM 교체 지양:
설명: 전체 DOM을 교체하는 대신, 크롤링 대상 페이지의 콘텐츠를 별도의 함수나 모듈을 통해 비동기적으로 가져와 처리하는 방식으로 변경하는 것이 좋습니다. 이렇게 하면 현재 페이지의 UI가 유지되며, Tampermonkey 스크립트도 계속 활성화 상태로 유지됩니다.
제안: fetch API나 GM_xmlhttpRequest를 사용하여 필요한 데이터를 가져오고, 현재 페이지의 DOM을 변경하지 않도록 수정합니다.
UI를 지속적으로 유지하기 위한 재초기화 로직 추가:
설명: 만약 페이지가 완전히 리로드되거나 DOM이 교체되는 상황이라면, Tampermonkey 스크립트가 다시 초기화되고 UI를 재생성할 수 있도록 로직을 추가해야 합니다.
제안: MutationObserver를 활용하여 DOM 변경을 감지하고, 필요 시 UI를 재생성하는 로직을 추가합니다.
백그라운드 스크립트 활용:
설명: 크롤링 작업을 백그라운드에서 수행하고, UI는 별도의 고정된 위치(예: 브라우저 액션 팝업)에서 유지하는 방식을 고려할 수 있습니다.
제안: Tampermonkey 외부의 백그라운드 스크립트를 활용하거나, Tampermonkey의 @grant 기능을 사용하여 백그라운드 작업을 효율적으로 처리합니다.
페이지 내비게이션 감지 및 UI 재삽입:
설명: 페이지가 변경될 때마다 UI를 다시 삽입하는 로직을 추가하면 문제가 해결될 수 있습니다.
제안: 기존의 MutationObserver를 활용하여 페이지 변경 이벤트를 감지하고, 변경 시 UI를 다시 생성하도록 합니다.        이걸 바탕으로 밑에 코드를 개선해보셈                        // ==UserScript==
// @name         Advanced Web Crawler Enhanced Pro Optimized v6.3.2
// @namespace    http://tampermonkey.net/
// @version      6.3.2
// @description  Further optimized and enhanced version with responsive design and other improvements.
// @author
// @match        https://www.google.com/search*
// @match        https://www.bing.com/search*
// @match        https://search.yahoo.com/search*
// @match        https://duckduckgo.com/*
// @match        https://www.baidu.com/s*
// @match        https://www.yandex.com/search/*
// @match        https://search.naver.com/search*
// @match        https://search.daum.net/search*
// @match        https://www.ask.com/*
// @match        https://www.ecosia.org/search*
// @match        https://www.startpage.com/*
// @match        https://searx.me/*
// @match        https://www.dogpile.com/*
// @match        https://www.bing.com/images*
// @grant        GM_xmlhttpRequest
// @grant        GM_setValue
// @grant        GM_getValue
// @grant        GM_deleteValue
// @grant        GM_addStyle
// @connect      *
// @require      https://cdnjs.cloudflare.com/ajax/libs/jquery/3.6.0/jquery.min.js
// @require      https://cdn.jsdelivr.net/npm/chart.js
// @require      https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.18.5/xlsx.full.min.js
// @require      https://cdnjs.cloudflare.com/ajax/libs/mocha/10.2.0/mocha.min.js
// @require      https://cdnjs.cloudflare.com/ajax/libs/chai/4.3.7/chai.min.js
// ==/UserScript==

(function() {
    'use strict';

    const Config = (() => {
        const DEFAULT_SETTINGS = {
            delay: 2000,
            delayRandom: 500,
            maxPages: 5,
            userAgents: [
                'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)',
                'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15',
                'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)',
                'Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15',
                'Mozilla/5.0 (iPad; CPU OS 14_0 like Mac OS X) AppleWebKit/605.1.15',
                'AdvancedWebCrawler/6.3 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
            ],
            userAgentIndex: 0,
            blacklist: [],
            whitelist: [],
            contentTypes: {
                include: ['text/html'],
                exclude: ['application/pdf', 'image/jpeg', 'image/png', 'video/mp4']
            },
            timeout: 8000,
            includeKeywords: [],
            excludeKeywords: [],
            language: [],
            minWordCount: 100,
            maxWordCount: 10000,
            maxDepth: 2,
            proxy: '',
            regexInclude: [],
            regexExclude: [],
            dataFields: {
                name: true,
                url: true,
                description: true,
                html: true,
                category: true,
                quality: true,
                language: true,
                wordCount: true,
                metaTags: false,
                headings: false
            },
            schedule: {
                enabled: false,
                startTime: '00:00',
                interval: 3600000,
                specificDays: []
            },
            searchEngines: {
                google: true,
                bing: true,
                yahoo: true,
                duckduckgo: true,
                baidu: true,
                yandex: true,
                naver: true,
                daum: true,
                ask: true,
                ecosia: true,
                startpage: true,
                searx: true,
                dogpile: true,
                bingimages: true
            },
            retryAttempts: 3,
            retryDelay: 2000,
            respectRobots: true,
            maxConcurrentRequests: 3,
            dataVisualization: true,
            exportFormats: {
                csv: true,
                json: true,
                xml: true,
                xls: true
            },
            advancedLogging: true,
            enableCaptchas: false,
            customKeywords: {},
            templates: {},
            currentTab: 'logs',
            isCrawling: false,
            darkMode: false
        };

        let settings = { ...DEFAULT_SETTINGS, ...getSavedSettings() };

        function getSavedSettings() {
            return GM_getValue('settings', DEFAULT_SETTINGS);
        }

        function saveSettings(newSettings) {
            settings = { ...settings, ...newSettings };
            GM_setValue('settings', settings);
        }

        function resetSettings() {
            settings = { ...DEFAULT_SETTINGS };
            GM_setValue('settings', settings);
        }

        function saveTemplate(name) {
            settings.templates[name] = { ...settings };
            saveSettings({});
            Logger.log(`설정 템플릿 '${name}'이(가) 저장되었습니다.`, 'success');
        }

        function loadTemplate(name) {
            if (name in settings.templates) {
                settings = { ...settings, ...settings.templates[name] };
                saveSettings({});
                Logger.log(`설정 템플릿 '${name}'이(가) 불러와졌습니다.`, 'success');
            } else {
                Logger.log(`설정 템플릿 '${name}'을(를) 찾을 수 없습니다.`, 'error');
            }
        }

        function validateProxy(proxyUrl) {
            try {
                const url = new URL(proxyUrl);
                return ['http:', 'https:'].includes(url.protocol);
            } catch {
                return false;
            }
        }

        return {
            DEFAULT_SETTINGS,
            get settings() { return settings; },
            saveSettings,
            resetSettings,
            saveTemplate,
            loadTemplate,
            validateProxy
        };
    })();

    const Logger = (() => {
        let logs = JSON.parse(GM_getValue('logs', '[]'));

        function log(message, type = 'info') {
            const timestamp = new Date().toLocaleString();
            logs.push({ timestamp, message, type });
            if (logs.length > 1000) logs.shift();
            GM_setValue('logs', JSON.stringify(logs));
            UI.updateLogs();
            if (Config.settings.advancedLogging) {
                const styles = {
                    info: 'color: blue;',
                    warning: 'color: orange;',
                    error: 'color: red;',
                    success: 'color: green; font-weight: bold;',
                    debug: 'color: purple;'
                };
                console.log(`%c[${timestamp}] [${type.toUpperCase()}] ${message}`, styles[type] || '');
            }
        }

        function getLogs() {
            return logs;
        }

        function clearLogs() {
            logs = [];
            GM_setValue('logs', JSON.stringify(logs));
            UI.updateLogs();
        }

        return {
            log,
            getLogs,
            clearLogs
        };
    })();

    const Utils = (() => {
        const escapeMap = {
            html: { '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#039;' },
            csv: { '"': '""' },
            xml: { '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#039;' }
        };

        function escape(str, type = 'html') {
            return String(str).replace(new RegExp(`[${Object.keys(escapeMap[type]).join('')}]`, 'g'), match => escapeMap[type][match]);
        }

        function capitalize(str) {
            return str.charAt(0).toUpperCase() + str.slice(1);
        }

        function delay(ms) {
            return new Promise(res => setTimeout(res, ms));
        }

        function generateColors(count) {
            return Array.from({ length: count }, (_, i) => `hsl(${i * (360 / count)}, 70%, 50%)`);
        }

        function parseCustomKeywords(text) {
            const lines = text.split('\n').map(l => l.trim()).filter(l => l);
            return lines.reduce((acc, line) => {
                const [cat, kws] = line.split(':').map(s => s.trim());
                if (cat && kws) acc[cat] = kws.split(',').map(kw => kw.trim()).filter(Boolean);
                return acc;
            }, {});
        }

        function formatCustomKeywords(ck) {
            return Object.entries(ck).map(([c, kws]) => `${c}:${kws.join(',')}`).join('\n');
        }

        function isValidURL(url) {
            try { new URL(url); return true; }
            catch { return false; }
        }

        return {
            escape,
            capitalize,
            delay,
            generateColors,
            parseCustomKeywords,
            formatCustomKeywords,
            isValidURL
        };
    })();

    const UI = (() => {
        function createUI() {
            GM_addStyle(`
                #crawler-container {
                    position: fixed;
                    top: 10px;
                    right: 10px;
                    width: 90%;
                    max-width: 800px;
                    height: 80%;
                    background: var(--bg-color, #fff);
                    border: 1px solid #ccc;
                    z-index: 10000;
                    font-family: Arial, sans-serif;
                    box-shadow: 0 4px 8px rgba(0,0,0,0.2);
                    overflow: hidden;
                    resize: both;
                    border-radius: 8px;
                    min-width: 300px;
                    min-height: 300px;
                }
                #crawler-container.dark {
                    --bg-color: #2b2b2b;
                    color: #f1f1f1;
                    border-color: #555;
                }
                #crawler-header {
                    display: flex;
                    justify-content: space-between;
                    align-items: center;
                    background: var(--header-bg, #f1f1f1);
                    border-bottom: 1px solid #ccc;
                    padding: 10px;
                    cursor: move;
                }
                #crawler-container.dark #crawler-header {
                    --header-bg: #3c3c3c;
                    border-color: #555;
                }
                #crawler-minimize, #dark-mode-toggle {
                    background: none;
                    border: none;
                    font-size: 18px;
                    cursor: pointer;
                    color: inherit;
                }
                #crawler-tabs {
                    display: flex;
                    flex-wrap: wrap;
                    gap: 5px;
                }
                .crawler-tab { padding: 8px 12px; cursor: pointer; font-weight: bold; border-radius: 4px; }
                .crawler-tab.active { background: var(--active-tab-bg, #ddd); }
                #crawler-container.dark .crawler-tab.active { background: #555; }
                #crawler-content { padding: 10px; overflow-y: auto; height: calc(100% - 60px); }
                table { width: 100%; border-collapse: collapse; }
                th, td { padding: 8px; text-align: left; border-bottom: 1px solid #ddd; }
                th { background-color: var(--th-bg, #f2f2f2); cursor: pointer; }
                #crawler-container.dark th { background-color: #444; }
                tr:hover { background-color: #f5f5f5; }
                #crawler-container.dark tr:hover { background-color: #555; }
                .progress-bar { width: 100%; background-color: #f3f3f3; border: 1px solid #ccc; border-radius: 5px; overflow: hidden; margin-bottom: 10px; }
                .progress-bar-fill { height: 20px; background-color: #4caf50; width: 0%; transition: width 0.2s; }
                .modal { display: none; position: fixed; z-index: 10001; left: 0; top: 0; width: 100%; height: 100%; overflow: auto; background-color: rgba(0,0,0,0.4); }
                .modal-content { background-color: var(--modal-bg, #fff); margin: 5% auto; padding: 20px; border: 1px solid #888; width: 90%; max-width: 800px; max-height: 80%; overflow-y: auto; border-radius: 8px; }
                #crawler-container.dark .modal-content { --modal-bg: #3c3c3c; color: #f1f1f1; border-color: #555; }
                .close { color: #aaa; float: right; font-size: 28px; font-weight: bold; cursor: pointer; }
                .close:hover, .close:focus { color: black; text-decoration: none; cursor: pointer; }
                .control-buttons { margin: 10px 0; display: flex; flex-wrap: wrap; gap: 10px; }
                .control-buttons button { padding: 8px 12px; cursor: pointer; border: none; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px; flex: 1 1 auto; min-width: 80px; }
                .control-buttons button:disabled { background-color: #a5d6a7; cursor: not-allowed; }
                .status-text { margin-top: 10px; font-weight: bold; font-size: 14px; }
                .settings-section { margin-bottom: 15px; }
                .settings-section label { display: block; margin-bottom: 5px; font-weight: bold; }
                .settings-section input[type="text"], .settings-section input[type="number"], .settings-section textarea, .settings-section select {
                    width: 100%; padding: 5px; box-sizing: border-box; border: 1px solid #ccc; border-radius: 4px; font-size: 14px;
                }
                .settings-section.dark .settings-section input, .settings-section.dark .settings-section select, .settings-section.dark .settings-section textarea {
                    background-color: #555; color: #f1f1f1; border-color: #777;
                }
                .checkbox-group, .radio-group { display: flex; flex-wrap: wrap; gap: 10px; }
                .checkbox-group label, .radio-group label { font-size: 14px; }
                .tooltip { position: relative; display: inline-block; cursor: help; font-size: 14px; }
                .tooltip .tooltiptext {
                    visibility: hidden; width: 220px; background-color: #555; color: #fff; text-align: center; border-radius: 6px;
                    padding: 5px; position: absolute; z-index: 1; bottom: 125%; left: 50%; margin-left: -110px;
                    opacity: 0; transition: opacity 0.3s;
                }
                .tooltip:hover .tooltiptext { visibility: visible; opacity: 1; }
                #crawler-container::-webkit-resizer { background: #f1f1f1; border: none; }
                #data-visualization { margin-top: 20px; }
                #data-chart { width: 100%; height: 300px; }
                #modal-html-content { white-space: pre-wrap; word-wrap: break-word; background-color: #f9f9f9; padding: 10px; border-radius: 4px; }
                #previous-crawls table, #previous-crawls th, #previous-crawls td { width: 100%; padding: 5px; border: 1px solid #ddd; }
                #reset-data { background-color: #f44336; }
                .spinner {
                    border: 4px solid #f3f3f3; border-top: 4px solid #3498db; border-radius: 50%;
                    width: 24px; height: 24px; animation: spin 2s linear infinite; display: inline-block; vertical-align: middle;
                }
                @keyframes spin {
                    0% { transform: rotate(0deg); }
                    100% { transform: rotate(360deg); }
                }
                .error-message { color: red; font-weight: bold; margin-top: 10px; }
                .success-message { color: green; font-weight: bold; margin-top: 10px; }
                .dark-mode-toggle { background: none; border: none; cursor: pointer; font-size: 18px; color: inherit; }
                @media (max-width: 768px) {
                    #crawler-container { width: 100%; height: 100%; top: 0; right: 0; border-radius: 0; }
                    #crawler-header { flex-direction: row; align-items: center; }
                    #crawler-tabs { flex-direction: row; overflow-x: auto; }
                    .crawler-tab { flex: 0 0 auto; }
                }
            `);
            const container = $(`
                <div id="crawler-container">
                    <div id="crawler-header">
                        <div id="crawler-tabs">
                            <div class="crawler-tab" data-tab="logs" title="로그 보기">📜 로그</div>
                            <div class="crawler-tab" data-tab="data" title="수집된 데이터 보기">📊 데이터</div>
                            <div class="crawler-tab" data-tab="settings" title="설정 변경">⚙️ 설정</div>
                            <div class="crawler-tab" data-tab="previous-crawls" title="이전 크롤링 기록">📁 기록</div>
                            <div class="crawler-tab" data-tab="templates" title="설정 템플릿">📂 템플릿</div>
                            <div class="crawler-tab" data-tab="tests" title="테스트 실행">🧪 테스트</div>
                            <button id="dark-mode-toggle" class="dark-mode-toggle" title="테마 전환">🌙</button>
                        </div>
                        <button id="crawler-minimize" title="창 최소화">─</button>
                    </div>
                    <div id="crawler-content">
                        <div id="tab-logs">
                            <div class="progress-bar">
                                <div class="progress-bar-fill" style="width: 0%;"></div>
                            </div>
                            <div class="status-text">진행률: 0% (0/0 페이지)</div>
                            <div class="control-buttons">
                                <button id="start-crawling" title="크롤링을 시작합니다.">▶ 시작</button>
                                <button id="stop-crawling" title="크롤링을 중지합니다." disabled>■ 중지</button>
                                ${Config.settings.exportFormats.csv ? `<button id="download-csv" title="CSV 다운로드">📥 CSV</button>` : ''}
                                ${Config.settings.exportFormats.json ? `<button id="download-json" title="JSON 다운로드">📥 JSON</button>` : ''}
                                ${Config.settings.exportFormats.xml ? `<button id="download-xml" title="XML 다운로드">📥 XML</button>` : ''}
                                ${Config.settings.exportFormats.xls ? `<button id="download-xls" title="XLS 다운로드">📥 XLS</button>` : ''}
                                <button id="reset-data" title="모든 크롤링 데이터를 초기화합니다.">🗑️ 초기화</button>
                            </div>
                            <div id="log-messages" style="height: 200px; overflow-y: auto; background: var(--log-bg, #f9f9f9); padding: 10px; border: 1px solid #ccc; border-radius: 4px;">
                                로그 내용이 여기에 표시됩니다.
                            </div>
                            <div id="error-message" class="error-message" style="display: none;"></div>
                            <div id="success-message" class="success-message" style="display: none;"></div>
                        </div>
                        <div id="tab-data" style="display: none;">
                            <input type="text" id="search-filter" placeholder="검색 필터..." style="width: 100%; padding: 5px; margin-bottom: 10px; border: 1px solid #ccc; border-radius: 4px;">
                            <table>
                                <thead>
                                    <tr>
                                        ${Config.settings.dataFields.name ? '<th data-sort="name">사이트 이름</th>' : ''}
                                        ${Config.settings.dataFields.url ? '<th data-sort="url">주소</th>' : ''}
                                        ${Config.settings.dataFields.description ? '<th data-sort="description">설명</th>' : ''}
                                        ${Config.settings.dataFields.html ? '<th>📄 HTML 내용</th>' : ''}
                                        ${Config.settings.dataFields.category ? '<th data-sort="category">분류</th>' : ''}
                                        ${Config.settings.dataFields.quality ? '<th data-sort="quality">품질점수</th>' : ''}
                                        ${Config.settings.dataFields.language ? '<th data-sort="language">언어</th>' : ''}
                                        ${Config.settings.dataFields.wordCount ? '<th data-sort="wordCount">단어 수</th>' : ''}
                                        ${Config.settings.dataFields.metaTags ? '<th>메타 태그</th>' : ''}
                                        ${Config.settings.dataFields.headings ? '<th>헤딩</th>' : ''}
                                    </tr>
                                </thead>
                                <tbody id="data-table-body"></tbody>
                            </table>
                            ${Config.settings.dataVisualization ? `
                                <div id="data-visualization">
                                    <h3>데이터 시각화</h3>
                                    <canvas id="data-chart"></canvas>
                                </div>
                            ` : ''}
                        </div>
                        <div id="tab-settings" style="display: none;">
                            <h3>설정</h3>
                            ${createSettingsSections()}
                            <div class="settings-section">
                                <button id="save-settings" style="margin-top: 10px; padding: 10px 20px; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px;">저장</button>
                                <button id="reset-settings" style="margin-top: 10px; padding: 10px 20px; background-color: #f44336; color: white; border-radius: 4px; font-size: 14px;">초기화</button>
                            </div>
                        </div>
                        <div id="tab-templates" style="display: none;">
                            <h3>설정 템플릿</h3>
                            <div class="settings-section">
                                <label for="new-template-name">새 템플릿 이름:</label>
                                <input type="text" id="new-template-name" placeholder="템플릿 이름을 입력하세요">
                                <button id="save-template" style="margin-top: 5px; padding: 8px 12px; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px;">📁 템플릿 저장</button>
                            </div>
                            <div class="settings-section">
                                <label for="template-select">기존 템플릿:</label>
                                <select id="template-select" style="width: 100%; padding: 5px; margin-top: 10px; border: 1px solid #ccc; border-radius: 4px;">
                                    <option value="" disabled selected>템플릿 선택</option>
                                    ${Object.keys(Config.settings.templates).map(t => `<option value="${t}">${t}</option>`).join('')}
                                </select>
                                <button id="load-template" style="margin-top: 10px; padding: 8px 12px; background-color: #2196F3; color: white; border-radius: 4px; font-size: 14px;">📂 템플릿 불러오기</button>
                                <button id="delete-template" style="margin-top: 5px; padding: 8px 12px; background-color: #f44336; color: white; border-radius: 4px; font-size: 14px;">템플릿 삭제</button>
                            </div>
                        </div>
                        <div id="tab-previous-crawls" style="display: none;">
                            <h3>이전 크롤링 기록</h3>
                            <button id="clear-previous-crawls" style="padding: 8px 12px; background-color: #f44336; color: white; border-radius: 4px; font-size: 14px; margin-bottom: 10px;">기록 초기화</button>
                            <table>
                                <thead>
                                    <tr>
                                        <th>날짜</th>
                                        <th>크롤링 수</th>
                                        <th>상태</th>
                                    </tr>
                                </thead>
                                <tbody id="previous-crawls-body"></tbody>
                            </table>
                        </div>
                        <div id="tab-tests" style="display: none;">
                            <button id="run-tests" style="padding: 10px 20px; background-color: #FFC107; color: white; border-radius: 4px; font-size: 14px;">테스트 실행</button>
                        </div>
                        <div id="html-modal" class="modal">
                            <div class="modal-content">
                                <span class="close">&times;</span>
                                <pre id="modal-html-content"></pre>
                            </div>
                        </div>
                        <div id="test-modal" class="modal">
                            <div class="modal-content">
                                <span class="close">&times;</span>
                                <div id="mocha"></div>
                            </div>
                        </div>
                    </div>
                </div>
            `);
            $('body').append(container);

            bindEvents();
            makeDraggable();
        }

        function createSettingsSections() {
            const sections = [
                {
                    label: '크롤링 딜레이 (ms)',
                    id: 'delay-input',
                    type: 'number',
                    value: Config.settings.delay,
                    min: 500,
                    step: 500,
                    tooltip: '기본 딜레이과 딜레이 랜덤 값을 조합하여 요청 사이의 딜레이를 설정합니다.'
                },
                {
                    label: '딜레이 랜덤 범위 (ms)',
                    id: 'delay-random-input',
                    type: 'number',
                    value: Config.settings.delayRandom,
                    min: 0,
                    step: 500,
                    tooltip: '딜레이에 무작위 요소를 추가하여 보다 자연스러운 요청을 합니다.'
                },
                {
                    label: '최대 크롤링 페이지 수',
                    id: 'max-pages-input',
                    type: 'number',
                    value: Config.settings.maxPages,
                    min: 1,
                    step: 1,
                    tooltip: '크롤링할 최대 페이지 수를 설정하세요.'
                },
                {
                    label: '사용자 에이전트(User-Agent)',
                    id: 'user-agent-select',
                    type: 'select-multiple',
                    options: Config.settings.userAgents.map((ua, idx) => ({ value: idx, text: ua, selected: idx === Config.settings.userAgentIndex })),
                    buttons: [
                        { id: 'add-user-agent', text: '추가', bgColor: '#2196F3' },
                        { id: 'remove-user-agent', text: '제거', bgColor: '#f44336' }
                    ],
                    tooltip: '크롤링에 사용할 사용자 에이전트를 선택하거나 추가할 수 있습니다.'
                },
                {
                    label: '콘텐츠 유형 포함',
                    id: 'content-include-input',
                    type: 'text',
                    value: Config.settings.contentTypes.include.join(', '),
                    tooltip: '크롤링할 콘텐츠 유형을 쉼표로 구분하여 입력하세요.'
                },
                {
                    label: '콘텐츠 유형 제외',
                    id: 'content-exclude-input',
                    type: 'text',
                    value: Config.settings.contentTypes.exclude.join(', '),
                    tooltip: '크롤링에서 제외할 콘텐츠 유형을 쉼표로 구분하여 입력하세요.'
                },
                {
                    label: '크롤링 타임아웃 (ms)',
                    id: 'timeout-input',
                    type: 'number',
                    value: Config.settings.timeout,
                    min: 1000,
                    step: 1000,
                    tooltip: '요청이 완료되기까지 기다릴 최대 시간을 밀리초 단위로 설정합니다.'
                },
                {
                    label: '포함할 키워드',
                    id: 'include-keywords-input',
                    type: 'textarea',
                    value: Config.settings.includeKeywords.join('\n'),
                    tooltip: '크롤링할 페이지에 반드시 포함되어야 하는 키워드를 입력하세요. 각 키워드는 줄바꿈으로 구분됩니다.'
                },
                {
                    label: '제외할 키워드',
                    id: 'exclude-keywords-input',
                    type: 'textarea',
                    value: Config.settings.excludeKeywords.join('\n'),
                    tooltip: '크롤링할 페이지에 포함되면 안 되는 키워드를 입력하세요. 각 키워드는 줄바꿈으로 구분됩니다.'
                },
                {
                    label: '포함할 키워드 (정규 표현식)',
                    id: 'regex-include-input',
                    type: 'textarea',
                    value: Config.settings.regexInclude.join('\n'),
                    tooltip: '포함할 키워드의 정규 표현식을 입력하세요. 각 패턴은 줄바꿈으로 구분됩니다.'
                },
                {
                    label: '제외할 키워드 (정규 표현식)',
                    id: 'regex-exclude-input',
                    type: 'textarea',
                    value: Config.settings.regexExclude.join('\n'),
                    tooltip: '제외할 키워드의 정규 표현식을 입력하세요. 각 패턴은 줄바꿈으로 구분됩니다.'
                },
                {
                    label: '언어 필터링',
                    id: 'language-select',
                    type: 'select-multiple',
                    options: [
                        { value: 'en', text: '영어 (en)' }, { value: 'ko', text: '한국어 (ko)' },
                        { value: 'zh', text: '중국어 (zh)' }, { value: 'ja', text: '일본어 (ja)' },
                        { value: 'ru', text: '러시아어 (ru)' }, { value: 'es', text: '스페인어 (es)' },
                        { value: 'fr', text: '프랑스어 (fr)' }, { value: 'de', text: '독일어 (de)' },
                        { value: 'it', text: '이탈리아어 (it)' }, { value: 'pt', text: '포르투갈어 (pt)' },
                        { value: 'ar', text: '아랍어 (ar)' }, { value: 'hi', text: '힌디어 (hi)' },
                        { value: 'bn', text: '벵골어 (bn)' }, { value: 'pa', text: '펀자브어 (pa)' },
                        { value: 'nl', text: '네덜란드어 (nl)' }, { value: 'sv', text: '스웨덴어 (sv)' },
                        { value: 'tr', text: '터키어 (tr)' }, { value: 'vi', text: '베트남어 (vi)' },
                        { value: 'pl', text: '폴란드어 (pl)' }, { value: 'th', text: '태국어 (th)' }
                    ].map(lang => ({ value: lang.value, text: lang.text, selected: Config.settings.language.includes(lang.value) }))
                },
                {
                    label: '최소 단어 수',
                    id: 'min-word-count-input',
                    type: 'number',
                    value: Config.settings.minWordCount,
                    min: 10,
                    step: 10,
                    tooltip: '수집할 페이지의 최소 단어 수를 설정하세요.'
                },
                {
                    label: '최대 단어 수',
                    id: 'max-word-count-input',
                    type: 'number',
                    value: Config.settings.maxWordCount,
                    min: 100,
                    step: 100,
                    tooltip: '수집할 페이지의 최대 단어 수를 설정하세요.'
                },
                {
                    label: '최대 크롤링 깊이',
                    id: 'max-depth-input',
                    type: 'number',
                    value: Config.settings.maxDepth,
                    min: 1,
                    step: 1,
                    tooltip: '링크를 추적할 최대 깊이를 설정하세요.'
                },
                {
                    label: '프록시 서버',
                    id: 'proxy-input',
                    type: 'text',
                    value: Config.settings.proxy,
                    placeholder: 'http://proxyserver:port',
                    tooltip: '프록시 서버 주소를 입력하세요. 예: http://proxyserver:port'
                },
                {
                    label: '수집할 데이터 필드',
                    id: '',
                    type: 'checkbox-group',
                    options: Object.keys(Config.settings.dataFields).map(field => ({
                        id: `field-${field}`,
                        text: Utils.capitalize(getFieldName(field)),
                        checked: Config.settings.dataFields[field]
                    }))
                },
                {
                    label: '재시도 횟수',
                    id: 'retry-attempts-input',
                    type: 'number',
                    value: Config.settings.retryAttempts,
                    min: 0,
                    step: 1,
                    tooltip: '요청 실패 시 재시도할 횟수를 설정하세요.'
                },
                {
                    label: '재시도 딜레이 (ms)',
                    id: 'retry-delay-input',
                    type: 'number',
                    value: Config.settings.retryDelay,
                    min: 1000,
                    step: 500,
                    tooltip: '재시도 시 대기할 시간을 밀리초 단위로 설정하세요.'
                },
                {
                    label: '크롤링 스케줄링',
                    id: 'schedule-enabled',
                    type: 'checkbox',
                    checked: Config.settings.schedule.enabled,
                    tooltip: '크롤링 스케줄링을 활성화합니다.'
                },
                {
                    label: '시작 시간 (HH:MM)',
                    id: 'start-time-input',
                    type: 'time',
                    value: Config.settings.schedule.startTime,
                    display: Config.settings.schedule.enabled,
                    tooltip: '크롤링을 시작할 시간을 설정하세요.'
                },
                {
                    label: '크롤링 간격 (분)',
                    id: 'interval-input',
                    type: 'number',
                    value: Config.settings.schedule.interval / 60000,
                    min: 1,
                    step: 1,
                    display: Config.settings.schedule.enabled,
                    tooltip: '크롤링 작업 간의 간격을 분 단위로 설정하세요.'
                },
                {
                    label: '특정 요일',
                    id: 'specific-days-input',
                    type: 'select-multiple',
                    options: [
                        { value: '0', text: '일요일' },
                        { value: '1', text: '월요일' },
                        { value: '2', text: '화요일' },
                        { value: '3', text: '수요일' },
                        { value: '4', text: '목요일' },
                        { value: '5', text: '금요일' },
                        { value: '6', text: '토요일' }
                    ].map(day => ({
                        value: day.value,
                        text: day.text,
                        selected: Config.settings.schedule.specificDays.includes(parseInt(day.value))
                    })),
                    display: Config.settings.schedule.enabled,
                    tooltip: '크롤링을 수행할 특정 요일을 선택하세요. (복수 선택 가능)'
                },
                {
                    label: '크롤링할 검색 엔진 선택',
                    id: '',
                    type: 'checkbox-group',
                    options: Object.keys(Config.settings.searchEngines).map(engine => ({
                        id: `engine-${engine}`,
                        text: Utils.capitalize(engine),
                        checked: Config.settings.searchEngines[engine]
                    }))
                },
                {
                    label: '블랙리스트 도메인',
                    id: 'blacklist-input',
                    type: 'textarea',
                    value: Config.settings.blacklist.join('\n'),
                    tooltip: '크롤링에서 제외할 도메인을 입력하세요. 각 도메인은 줄바꿈으로 구분됩니다.'
                },
                {
                    label: '화이트리스트 도메인',
                    id: 'whitelist-input',
                    type: 'textarea',
                    value: Config.settings.whitelist.join('\n'),
                    tooltip: '크롤링을 허용할 도메인을 입력하세요. 각 도메인은 줄바꿈으로 구분됩니다.'
                },
                {
                    label: 'robots.txt 준수',
                    id: 'respect-robots-checkbox',
                    type: 'checkbox',
                    checked: Config.settings.respectRobots,
                    tooltip: '대상 사이트의 robots.txt 규칙을 준수하여 크롤링을 진행합니다.'
                },
                {
                    label: '동시 요청 최대 수',
                    id: 'max-concurrent-requests-input',
                    type: 'number',
                    value: Config.settings.maxConcurrentRequests,
                    min: 1,
                    step: 1,
                    tooltip: '한 번에 요청할 수 있는 최대 동시 요청 수를 설정하세요.'
                },
                {
                    label: '데이터 시각화 활성화',
                    id: 'data-visualization-checkbox',
                    type: 'checkbox',
                    checked: Config.settings.dataVisualization,
                    tooltip: '수집된 데이터를 시각화하여 그래프로 표시합니다.'
                },
                {
                    label: 'CAPTCHA 처리 활성화',
                    id: 'enable-captchas-checkbox',
                    type: 'checkbox',
                    checked: Config.settings.enableCaptchas,
                    tooltip: 'CAPTCHA가 감지되었을 때 자동 처리를 활성화합니다. (제한적)'
                },
                {
                    label: '데이터 내보내기 형식',
                    id: '',
                    type: 'checkbox-group',
                    options: Object.keys(Config.settings.exportFormats).map(format => ({
                        id: `export-${format}`,
                        text: format.toUpperCase(),
                        checked: Config.settings.exportFormats[format]
                    }))
                },
                {
                    label: '고급 로깅 활성화',
                    id: 'advanced-logging-checkbox',
                    type: 'checkbox',
                    checked: Config.settings.advancedLogging,
                    tooltip: '추가적인 로깅 정보를 활성화합니다.'
                },
                {
                    label: '사용자 지정 키워드 분류',
                    id: 'custom-keywords-input',
                    type: 'textarea',
                    value: Utils.formatCustomKeywords(Config.settings.customKeywords),
                    tooltip: '사용자 지정 키워드를 추가하여 데이터 분류를 향상시킵니다. 형식: 카테고리:키워드1,키워드2,...'
                }
            ];

            return sections.map(sec => {
                if (sec.type === 'checkbox-group') {
                    return `
                        <div class="settings-section">
                            <label>${sec.label}:</label>
                            <div class="checkbox-group">
                                ${sec.options.map(opt => `<label><input type="checkbox" id="${opt.id}" ${opt.checked ? 'checked' : ''}> ${opt.text}</label>`).join('')}
                            </div>
                        </div>
                    `;
                }
                if (sec.type === 'select-multiple') {
                    return `
                        <div class="settings-section">
                            <label for="${sec.id}">${sec.label}:
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">${sec.tooltip}</span>
                                </span>
                            </label>
                            <select id="${sec.id}" multiple size="${sec.options.length || 5}">
                                ${sec.options.map(opt => `<option value="${opt.value}" ${opt.selected ? 'selected' : ''}>${opt.text}</option>`).join('')}
                            </select>
                        </div>
                    `;
                }
                if (sec.type === 'checkbox') {
                    return `
                        <div class="settings-section">
                            <label for="${sec.id}">${sec.label}:
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">${sec.tooltip}</span>
                                </span>
                            </label>
                            <input type="checkbox" id="${sec.id}" ${sec.checked ? 'checked' : ''}>
                        </div>
                    `;
                }
                if (sec.type === 'textarea') {
                    return `
                        <div class="settings-section">
                            <label for="${sec.id}">${sec.label}:
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">${sec.tooltip}</span>
                                </span>
                            </label>
                            <textarea id="${sec.id}" rows="3">${sec.value}</textarea>
                        </div>
                    `;
                }
                if (sec.type === 'select') {
                    return `
                        <div class="settings-section">
                            <label for="${sec.id}">${sec.label}:</label>
                            <select id="${sec.id}" ${sec.multiple ? 'multiple' : ''}>
                                ${sec.options.map(opt => `<option value="${opt.value}" ${opt.selected ? 'selected' : ''}>${opt.text}</option>`).join('')}
                            </select>
                        </div>
                    `;
                }
                return `
                    <div class="settings-section">
                        <label for="${sec.id}">${sec.label}:
                            <span class="tooltip">ℹ️
                                <span class="tooltiptext">${sec.tooltip}</span>
                            </span>
                        </label>
                        <input type="${sec.type}" id="${sec.id}" value="${sec.value}" ${sec.min ? `min="${sec.min}"` : ''} ${sec.step ? `step="${sec.step}"` : ''} ${sec.placeholder ? `placeholder="${sec.placeholder}"` : ''} ${sec.display === false ? 'style="display: none;"' : ''}>
                        ${sec.buttons ? sec.buttons.map(btn => `<button id="${btn.id}" style="margin-top: 5px; padding: 5px 10px; background-color: ${btn.bgColor}; color: white; border-radius: 4px; font-size: 12px;">${btn.text}</button>`).join('') : ''}
                    </div>
                `;
            }).join('');
        }

        function bindEvents() {
            $('#crawler-minimize').on('click', toggleMinimize);
            $('.crawler-tab').on('click', switchTab);
            $('#save-settings').on('click', saveSettingsHandler);
            $('#reset-settings').on('click', resetSettingsHandler);
            $('#add-user-agent').on('click', addUserAgent);
            $('#remove-user-agent').on('click', removeUserAgent);
            $('#download-csv').on('click', Download.downloadCSV);
            $('#download-json').on('click', Download.downloadJSON);
            $('#download-xml').on('click', Download.downloadXML);
            $('#download-xls').on('click', Download.downloadXLS);
            $('#search-filter').on('input', filterData);
            $('#schedule-enabled').on('change', toggleScheduleOptions);
            $('#data-visualization-checkbox').on('change', toggleDataVisualization);
            $('#enable-captchas-checkbox').on('change', toggleCaptchaHandling);
            $('#save-template').on('click', saveTemplateHandler);
            $('#load-template').on('click', loadTemplateHandler);
            $('#delete-template').on('click', deleteTemplateHandler);
            $('#template-select').on('change', selectTemplateHandler);
            $('#run-tests').on('click', Testing.runTests);
            $('#clear-previous-crawls').on('click', clearPreviousCrawls);
            $('#reset-data').on('click', resetData);
            $('#dark-mode-toggle').on('click', toggleDarkMode);
            $('.close').on('click', () => $('.modal').hide());
            $(window).on('click', e => { if ($(e.target).hasClass('modal')) $('.modal').hide(); });
            $('#start-crawling').on('click', Crawling.startCrawling);
            $('#stop-crawling').on('click', Crawling.stopCrawling);
            $(document).on('keydown', handleShortcuts);
            $('#proxy-input').on('input', validateProxyInput);
            $('input[type="checkbox"], input[type="radio"]').on('change', toggleDataField);
        }

        function toggleMinimize() {
            $('#crawler-content').toggle();
            $('#crawler-minimize').text($('#crawler-content').is(':hidden') ? '☐' : '─');
            Logger.log(`크롤러 창이 ${$('#crawler-content').is(':hidden') ? '최소화' : '확대'}되었습니다.`, 'info');
        }

        function switchTab() {
            const tab = $(this).data('tab');
            $('.crawler-tab').removeClass('active');
            $(this).addClass('active');
            $('#crawler-content > div').hide();
            $(`#tab-${tab}`).show();
            Config.saveSettings({ currentTab: tab });
        }

        function updateUI() {
            const currentTab = Config.settings.currentTab || 'logs';
            $(`.crawler-tab[data-tab="${currentTab}"]`).addClass('active');
            $('#crawler-content > div').hide();
            $(`#tab-${currentTab}`).show();
            updateLogs();
            updateDataTable();
            updatePreviousCrawlsUI();
            updateTemplatesUI();
            if (Config.settings.dataVisualization) DataVisualization.renderChart();
            applyDarkMode();
        }

        function updateLogs() {
            const logs = Logger.getLogs();
            $('#log-messages').html(logs.map(l => `<div>[${l.timestamp}] [${l.type.toUpperCase()}] ${Utils.escape(l.message)}</div>`).join(''));
            const progress = GM_getValue('progress', 0);
            const currentPage = GM_getValue('currentPage', 0);
            const totalPages = Config.settings.maxPages;
            $('.progress-bar-fill').css('width', `${progress.toFixed(2)}%`);
            $('.status-text').text(`진행률: ${progress.toFixed(2)}% (${currentPage}/${totalPages} 페이지)`);
            const latestLog = logs[logs.length -1];
            if (latestLog) {
                if (['error', 'warning'].includes(latestLog.type)) {
                    $('#error-message').text(latestLog.message).show().delay(5000).fadeOut();
                }
                if (['success', 'info'].includes(latestLog.type)) {
                    $('#success-message').text(latestLog.message).show().delay(5000).fadeOut();
                }
            }
        }

        function updateDataTable() {
            const data = JSON.parse(GM_getValue('data', '[]'));
            const filter = $('#search-filter').val().toLowerCase();
            const filtered = filter ? data.filter(item => Object.keys(Config.settings.dataFields).some(f => Config.settings.dataFields[f] && String(item[f]).toLowerCase().includes(filter))) : data;
            $('#data-table-body').html(filtered.map(item => `
                <tr>
                    ${Config.settings.dataFields.name ? `<td>${Utils.escape(item.name)}</td>` : ''}
                    ${Config.settings.dataFields.url ? `<td><a href="${Utils.escape(item.url)}" target="_blank">${Utils.escape(item.url)}</a></td>` : ''}
                    ${Config.settings.dataFields.description ? `<td>${Utils.escape(item.description)}</td>` : ''}
                    ${Config.settings.dataFields.html ? `<td><button class="view-html" data-html="${encodeURIComponent(item.html)}">📄 보기</button></td>` : ''}
                    ${Config.settings.dataFields.category ? `<td>${Utils.escape(item.category)}</td>` : ''}
                    ${Config.settings.dataFields.quality ? `<td>${Utils.escape(item.quality.toString())}</td>` : ''}
                    ${Config.settings.dataFields.language ? `<td>${Utils.escape(item.language)}</td>` : ''}
                    ${Config.settings.dataFields.wordCount ? `<td>${Utils.escape(item.wordCount.toString())}</td>` : ''}
                    ${Config.settings.dataFields.metaTags ? `<td>${Utils.escape(JSON.stringify(item.metaTags))}</td>` : ''}
                    ${Config.settings.dataFields.headings ? `<td>${Utils.escape(item.headings.join(', '))}</td>` : ''}
                </tr>
            `).join(''));
            $('.view-html').off('click').on('click', function() {
                $('#modal-html-content').text(decodeURIComponent($(this).data('html')));
                $('#html-modal').show();
            });
            if (Config.settings.dataVisualization) DataVisualization.renderChart();
        }

        function updatePreviousCrawlsUI() {
            const previousCrawls = JSON.parse(GM_getValue('previousCrawls', '[]'));
            $('#previous-crawls-body').html(previousCrawls.map(crawl => `
                <tr>
                    <td>${Utils.escape(crawl.date)}</td>
                    <td>${Utils.escape(crawl.count.toString())}</td>
                    <td>${Utils.escape(crawl.status)}</td>
                </tr>
            `).join(''));
        }

        function updateTemplatesUI() {
            $('#template-select').empty().append(`<option value="" disabled selected>템플릿 선택</option>`);
            Object.keys(Config.settings.templates).forEach(t => $('#template-select').append(`<option value="${t}">${t}</option>`));
        }

        function addUserAgent() {
            const ua = prompt("새 사용자 에이전트를 입력하세요:");
            if (ua) {
                Config.settings.userAgents.push(ua);
                Config.saveSettings({ userAgents: Config.settings.userAgents });
                updateUserAgentSelect();
                Logger.log('새 사용자 에이전트가 추가되었습니다.', 'success');
            }
        }

        function removeUserAgent() {
            const selected = $('#user-agent-select').val();
            if (selected && selected.length) {
                selected.sort((a, b) => b - a).forEach(idx => Config.settings.userAgents.splice(idx, 1));
                Config.settings.userAgentIndex = 0;
                Config.saveSettings({ userAgents: Config.settings.userAgents, userAgentIndex: Config.settings.userAgentIndex });
                updateUserAgentSelect();
                Logger.log('선택한 사용자 에이전트가 제거되었습니다.', 'success');
            } else {
                alert('제거할 사용자 에이전트를 선택하세요.');
            }
        }

        function updateUserAgentSelect() {
            $('#user-agent-select').empty();
            Config.settings.userAgents.forEach((ua, idx) => {
                $('#user-agent-select').append(`<option value="${idx}" ${idx === Config.settings.userAgentIndex ? 'selected' : ''}>${ua}</option>`);
            });
        }

        function saveSettingsHandler() {
            const newSettings = {
                delay: parseInt($('#delay-input').val(), 10) || Config.DEFAULT_SETTINGS.delay,
                delayRandom: parseInt($('#delay-random-input').val(), 10) || Config.DEFAULT_SETTINGS.delayRandom,
                maxPages: parseInt($('#max-pages-input').val(), 10) || Config.DEFAULT_SETTINGS.maxPages,
                userAgentIndex: parseInt($('#user-agent-select').val(), 10) || 0,
                contentTypes: {
                    include: $('#content-include-input').val().split(',').map(ct => ct.trim()).filter(Boolean),
                    exclude: $('#content-exclude-input').val().split(',').map(ct => ct.trim()).filter(Boolean)
                },
                timeout: parseInt($('#timeout-input').val(), 10) || Config.DEFAULT_SETTINGS.timeout,
                includeKeywords: $('#include-keywords-input').val().split('\n').map(k => k.trim()).filter(Boolean),
                excludeKeywords: $('#exclude-keywords-input').val().split('\n').map(k => k.trim()).filter(Boolean),
                regexInclude: $('#regex-include-input').val().split('\n').map(r => r.trim()).filter(Boolean),
                regexExclude: $('#regex-exclude-input').val().split('\n').map(r => r.trim()).filter(Boolean),
                language: $('#language-select').val() || [],
                minWordCount: parseInt($('#min-word-count-input').val(), 10) || Config.DEFAULT_SETTINGS.minWordCount,
                maxWordCount: parseInt($('#max-word-count-input').val(), 10) || Config.DEFAULT_SETTINGS.maxWordCount,
                maxDepth: parseInt($('#max-depth-input').val(), 10) || Config.DEFAULT_SETTINGS.maxDepth,
                proxy: $('#proxy-input').val().trim(),
                dataFields: Object.fromEntries(['name','url','description','html','category','quality','language','wordCount','metaTags','headings'].map(f => [`${f}`, $(`#field-${f}`).is(':checked') ])),
                retryAttempts: parseInt($('#retry-attempts-input').val(), 10) || Config.DEFAULT_SETTINGS.retryAttempts,
                retryDelay: parseInt($('#retry-delay-input').val(), 10) || Config.DEFAULT_SETTINGS.retryDelay,
                schedule: {
                    enabled: $('#schedule-enabled').is(':checked'),
                    startTime: $('#start-time-input').val() || Config.DEFAULT_SETTINGS.schedule.startTime,
                    interval: (parseInt($('#interval-input').val(), 10) || (Config.DEFAULT_SETTINGS.schedule.interval / 60000)) * 60000,
                    specificDays: $('#specific-days-input').val() ? $('#specific-days-input').val().map(d => parseInt(d, 10)) : []
                },
                searchEngines: Object.fromEntries(Object.keys(Config.settings.searchEngines).map(engine => [`${engine}`, $(`#engine-${engine}`).is(':checked') ])),
                blacklist: $('#blacklist-input').val().split('\n').map(d => d.trim()).filter(Boolean),
                whitelist: $('#whitelist-input').val().split('\n').map(d => d.trim()).filter(Boolean),
                respectRobots: $('#respect-robots-checkbox').is(':checked'),
                maxConcurrentRequests: parseInt($('#max-concurrent-requests-input').val(), 10) || Config.DEFAULT_SETTINGS.maxConcurrentRequests,
                dataVisualization: $('#data-visualization-checkbox').is(':checked'),
                exportFormats: Object.fromEntries(['csv','json','xml','xls'].map(fmt => [`${fmt}`, $(`#export-${fmt}`).is(':checked') ])),
                advancedLogging: $('#advanced-logging-checkbox').is(':checked'),
                enableCaptchas: $('#enable-captchas-checkbox').is(':checked'),
                customKeywords: Utils.parseCustomKeywords($('#custom-keywords-input').val())
            };

            if (newSettings.proxy && !Config.validateProxy(newSettings.proxy)) {
                $('#error-message').text('유효하지 않은 프록시 URL입니다.').show();
                setTimeout(() => $('#error-message').fadeOut(), 5000);
                return;
            }

            Config.saveSettings(newSettings);
            Crawling.updateCrawlSettings();
            Logger.log('설정이 저장되었습니다.', 'success');
            if (newSettings.schedule.enabled) Scheduler.scheduleCrawling();
            updateDataTable();
            if (newSettings.dataVisualization) DataVisualization.renderChart();
        }

        function resetSettingsHandler() {
            if (confirm("기본 설정으로 초기화하시겠습니까?")) {
                Config.resetSettings();
                location.reload();
            }
        }

        function toggleScheduleOptions() {
            $('#start-time-input, #interval-input, #specific-days-input').toggle($('#schedule-enabled').is(':checked'));
        }

        function toggleDataVisualization() {
            const enabled = $('#data-visualization-checkbox').is(':checked');
            Config.saveSettings({ dataVisualization: enabled });
            $('#data-visualization').toggle(enabled);
            Logger.log(`데이터 시각화가 ${enabled ? '활성화' : '비활성화'}되었습니다.`, 'success');
            if (enabled) DataVisualization.renderChart();
        }

        function toggleCaptchaHandling() {
            const enabled = $('#enable-captchas-checkbox').is(':checked');
            Config.saveSettings({ enableCaptchas: enabled });
            Logger.log(`CAPTCHA 처리 ${enabled ? '활성화' : '비활성화'}되었습니다.`, 'success');
        }

        function filterData() {
            updateDataTable();
        }

        function clearPreviousCrawls() {
            if (confirm("이전 크롤링 기록을 모두 초기화하시겠습니까?")) {
                GM_setValue('previousCrawls', JSON.stringify([]));
                updatePreviousCrawlsUI();
                Logger.log('이전 크롤링 기록이 초기화되었습니다.', 'success');
            }
        }

        function resetData() {
            if (confirm("모든 크롤링 데이터를 초기화하시겠습니까?")) {
                GM_setValue('data', JSON.stringify([]));
                GM_setValue('visitedUrls', JSON.stringify([]));
                updateDataTable();
                Logger.log('크롤링 데이터가 초기화되었습니다.', 'success');
            }
        }

        function handleShortcuts(e) {
            if (e.ctrlKey) {
                const actions = { '1': 'logs', '2': 'data', '3': 'settings', '4': 'previous-crawls', '5': 'templates', '6': 'tests', 's': 'save', 'r': 'reset', 'p': 'start', 'q': 'stop', 'c': 'csv', 'j': 'json', 'x': 'xml', 'l': 'xls', 'd': 'dark-mode' };
                const action = actions[e.key];
                if (action) {
                    e.preventDefault();
                    switch(action) {
                        case 'logs': $('.crawler-tab[data-tab="logs"]').click(); break;
                        case 'data': $('.crawler-tab[data-tab="data"]').click(); break;
                        case 'settings': $('.crawler-tab[data-tab="settings"]').click(); break;
                        case 'previous-crawls': $('.crawler-tab[data-tab="previous-crawls"]').click(); break;
                        case 'templates': $('.crawler-tab[data-tab="templates"]').click(); break;
                        case 'tests': $('.crawler-tab[data-tab="tests"]').click(); break;
                        case 'save': saveSettingsHandler(); break;
                        case 'reset': resetSettingsHandler(); break;
                        case 'start': if (!Crawling.isCrawling()) Crawling.startCrawling(); break;
                        case 'stop': if (Crawling.isCrawling()) Crawling.stopCrawling(); break;
                        case 'csv': Config.settings.exportFormats.csv && Download.downloadCSV(); break;
                        case 'json': Config.settings.exportFormats.json && Download.downloadJSON(); break;
                        case 'xml': Config.settings.exportFormats.xml && Download.downloadXML(); break;
                        case 'xls': Config.settings.exportFormats.xls && Download.downloadXLS(); break;
                        case 'dark-mode': toggleDarkMode(); break;
                    }
                }
            }
        }

        function makeDraggable() {
            const container = document.getElementById('crawler-container');
            const header = document.getElementById('crawler-header');
            let isDragging = false, startX, startY, initialX, initialY;
            header.onmousedown = function(e) {
                isDragging = true;
                startX = e.clientX;
                startY = e.clientY;
                initialX = container.offsetLeft;
                initialY = container.offsetTop;
                document.onmousemove = drag;
                document.onmouseup = stopDrag;
                return false;
            };
            function drag(e) {
                if (isDragging) {
                    container.style.left = (initialX + e.clientX - startX) + 'px';
                    container.style.top = (initialY + e.clientY - startY) + 'px';
                }
            }
            function stopDrag() {
                isDragging = false;
                document.onmousemove = null;
                document.onmouseup = null;
            }
        }

        function getFieldName(field) {
            const names = {
                name: '이름',
                url: '주소',
                description: '설명',
                html: 'HTML 내용',
                category: '분류',
                quality: '품질점수',
                language: '언어',
                wordCount: '단어 수',
                metaTags: '메타 태그',
                headings: '헤딩'
            };
            return names[field] || field;
        }

        function saveTemplateHandler() {
            const templateName = $('#new-template-name').val().trim();
            if (!templateName) {
                alert('템플릿 이름을 입력하세요.');
                return;
            }
            Config.saveTemplate(templateName);
            updateTemplatesUI();
            $('#new-template-name').val('');
        }

        function loadTemplateHandler() {
            const selected = $('#template-select').val();
            if (!selected) {
                alert('불러올 템플릿을 선택하세요.');
                return;
            }
            Config.loadTemplate(selected);
            updateUI();
            $('.crawler-tab[data-tab="settings"]').click();
        }

        function deleteTemplateHandler() {
            const selected = $('#template-select').val();
            if (!selected) {
                alert('삭제할 템플릿을 선택하세요.');
                return;
            }
            delete Config.settings.templates[selected];
            Config.saveSettings({ templates: Config.settings.templates });
            Logger.log(`템플릿 ${selected}이(가) 삭제되었습니다.`, 'success');
            updateTemplatesUI();
        }

        function selectTemplateHandler() {
            const selected = $(this).val();
            if (selected) {
                Config.loadTemplate(selected);
                updateUI();
                $('.crawler-tab[data-tab="settings"]').click();
            }
        }

        function toggleDarkMode() {
            $('#crawler-container').toggleClass('dark');
            const isDark = $('#crawler-container').hasClass('dark');
            $('#dark-mode-toggle').text(isDark ? '☀️' : '🌙');
            Config.saveSettings({ darkMode: isDark });
            Logger.log(`다크 모드가 ${isDark ? '활성화' : '비활성화'}되었습니다.`, 'success');
        }

        function applyDarkMode() {
            if (Config.settings.darkMode) {
                $('#crawler-container').addClass('dark');
                $('#dark-mode-toggle').text('☀️');
            } else {
                $('#crawler-container').removeClass('dark');
                $('#dark-mode-toggle').text('🌙');
            }
        }

        function toggleDataField() {
            const field = $(this).attr('id').replace('field-', '');
            Config.settings.dataFields[field] = $(this).is(':checked');
            Config.saveSettings({ dataFields: Config.settings.dataFields });
            Download.updateExportButtons();
        }

        function validateProxyInput() {
            const proxy = $('#proxy-input').val().trim();
            if (proxy && !Config.validateProxy(proxy)) {
                $('#error-message').text('유효하지 않은 프록시 URL입니다.').show();
            } else {
                $('#error-message').hide();
            }
        }

        return {
            createUI,
            updateUI,
            updateLogs,
            updateDataTable,
            updatePreviousCrawlsUI,
            updateTemplatesUI,
            applyDarkMode
        };
    })();

    const Download = (() => {
        function downloadCSV() {
            const data = JSON.parse(GM_getValue('data', '[]'));
            if (!data.length) { Logger.log('다운로드할 데이터가 없습니다.', 'warning'); return; }
            const headers = Object.keys(Config.settings.dataFields).filter(f => Config.settings.dataFields[f]).map(f => Utils.capitalize(getFieldName(f)));
            const csvContent = [headers.join(",")]
                .concat(data.map(item => Object.keys(Config.settings.dataFields).filter(f => Config.settings.dataFields[f]).map(f => `"${Utils.escape(item[f], 'csv')}"`).join(",")))
                .join("\n");
            const blob = new Blob([csvContent], { type: "text/csv;charset=utf-8;" });
            const link = document.createElement("a");
            link.href = URL.createObjectURL(blob);
            link.download = "data.csv";
            document.body.appendChild(link);
            link.click();
            document.body.removeChild(link);
            Logger.log('CSV 파일이 다운로드되었습니다.', 'success');
        }

        function downloadJSON() {
            const data = JSON.parse(GM_getValue('data', '[]'));
            if (!data.length) { Logger.log('다운로드할 데이터가 없습니다.', 'warning'); return; }
            const jsonContent = JSON.stringify(data, null, 2);
            const blob = new Blob([jsonContent], { type: "application/json" });
            const link = document.createElement("a");
            link.href = URL.createObjectURL(blob);
            link.download = "data.json";
            document.body.appendChild(link);
            link.click();
            document.body.removeChild(link);
            Logger.log('JSON 파일이 다운로드되었습니다.', 'success');
        }

        function downloadXML() {
            const data = JSON.parse(GM_getValue('data', '[]'));
            if (!data.length) { Logger.log('다운로드할 데이터가 없습니다.', 'warning'); return; }
            let xmlContent = '<?xml version="1.0" encoding="UTF-8"?><data>';
            data.forEach(item => {
                xmlContent += '<entry>';
                if (Config.settings.dataFields.name) xmlContent += `<name>${Utils.escape(item.name || 'No Title', 'xml')}</name>`;
                if (Config.settings.dataFields.url) xmlContent += `<url>${Utils.escape(item.url || '', 'xml')}</url>`;
                if (Config.settings.dataFields.description) xmlContent += `<description>${Utils.escape(item.description || 'No Description', 'xml')}</description>`;
                if (Config.settings.dataFields.html) xmlContent += `<html><![CDATA[${item.html || ''}]]></html>`;
                if (Config.settings.dataFields.category) xmlContent += `<category>${Utils.escape(item.category || '', 'xml')}</category>`;
                if (Config.settings.dataFields.quality) xmlContent += `<quality>${Utils.escape(item.quality)}</quality>`;
                if (Config.settings.dataFields.language) xmlContent += `<language>${Utils.escape(item.language || 'unknown', 'xml')}</language>`;
                if (Config.settings.dataFields.wordCount) xmlContent += `<wordCount>${Utils.escape(item.wordCount)}</wordCount>`;
                if (Config.settings.dataFields.metaTags) xmlContent += `<metaTags>${Utils.escape(JSON.stringify(item.metaTags), 'xml')}</metaTags>`;
                if (Config.settings.dataFields.headings) xmlContent += `<headings>${Utils.escape(item.headings.join('; '), 'xml')}</headings>`;
                xmlContent += '</entry>';
            });
            xmlContent += '</data>';
            const blob = new Blob([xmlContent], { type: "application/xml" });
            const link = document.createElement("a");
            link.href = URL.createObjectURL(blob);
            link.download = "data.xml";
            document.body.appendChild(link);
            link.click();
            document.body.removeChild(link);
            Logger.log('XML 파일이 다운로드되었습니다.', 'success');
        }

        function downloadXLS() {
            const data = JSON.parse(GM_getValue('data', '[]'));
            if (!data.length) { Logger.log('다운로드할 데이터가 없습니다.', 'warning'); return; }
            const worksheetData = data.map(item => {
                const entry = {};
                Object.keys(Config.settings.dataFields).forEach(f => {
                    if (Config.settings.dataFields[f]) {
                        if (f === 'metaTags') entry['메타 태그'] = JSON.stringify(item[f]);
                        else if (f === 'headings') entry['헤딩'] = item[f].join('; ');
                        else entry[Utils.capitalize(getFieldName(f))] = item[f];
                    }
                });
                return entry;
            });
            const worksheet = XLSX.utils.json_to_sheet(worksheetData);
            const workbook = XLSX.utils.book_new();
            XLSX.utils.book_append_sheet(workbook, worksheet, "Data");
            XLSX.writeFile(workbook, "data.xlsx");
            Logger.log('XLS 파일이 다운로드되었습니다.', 'success');
        }

        function getFieldName(field) {
            const names = {
                name: '이름',
                url: '주소',
                description: '설명',
                html: 'HTML 내용',
                category: '분류',
                quality: '품질점수',
                language: '언어',
                wordCount: '단어 수',
                metaTags: '메타 태그',
                headings: '헤딩'
            };
            return names[field] || field;
        }

        function updateExportButtons() {
            // Placeholder for dynamic enabling/disabling of export buttons based on data fields
        }

        return {
            downloadCSV,
            downloadJSON,
            downloadXML,
            downloadXLS,
            updateExportButtons
        };
    })();

    const DataVisualization = (() => {
        let chartInstance = null;

        function renderChart() {
            const ctx = document.getElementById('data-chart').getContext('2d');
            if (chartInstance) chartInstance.destroy();
            const data = JSON.parse(GM_getValue('data', '[]'));
            const categories = data.reduce((acc, item) => {
                const cat = item.category || '기타';
                acc[cat] = (acc[cat] || 0) + 1;
                return acc;
            }, {});
            const chartData = {
                labels: Object.keys(categories),
                datasets: [{
                    label: '카테고리별 데이터 수',
                    data: Object.values(categories),
                    backgroundColor: Utils.generateColors(Object.keys(categories).length)
                }]
            };
            chartInstance = new Chart(ctx, { type: 'pie', data: chartData, options: { responsive: true } });
        }

        return {
            renderChart
        };
    })();

    const Crawling = (() => {
        let isCrawlingFlag = Config.settings.isCrawling;
        let currentPage = parseInt(GM_getValue('currentPage', 1), 10);
        let totalPages = Config.settings.maxPages;
        let progress = parseFloat(GM_getValue('progress', 0));
        let currentDepth = parseInt(GM_getValue('currentDepth', 0), 10);
        let activeRequests = 0;
        let requestQueue = [];
        let previousCrawls = JSON.parse(GM_getValue('previousCrawls', '[]'));
        let visitedUrls = new Set(JSON.parse(GM_getValue('visitedUrls', '[]')));
        let data = JSON.parse(GM_getValue('data', '[]'));
        let navigationObserver = null;

        const searchEngineHandlers = {
            google: {
                extractResults: doc => {
                    const results = [];
                    doc.querySelectorAll('div.g, div[data-hveid]').forEach(item => {
                        const name = item.querySelector('h3')?.innerText.trim() || 'No Title';
                        const url = item.querySelector('a')?.href || '';
                        const desc = item.querySelector('.aCOpRe, .VwiC3b')?.innerText.trim() || 'No Description';
                        if (Utils.isValidURL(url)) results.push({ name, url, description: desc });
                    });
                    return results;
                },
                getNextPageUrl: doc => doc.querySelector('a#pnnext')?.href || null,
                getContentContainer: () => '#search'
            },
            bing: {
                extractResults: doc => {
                    const results = [];
                    doc.querySelectorAll('.b_algo').forEach(item => {
                        const name = item.querySelector('h2')?.innerText.trim() || 'No Title';
                        const url = item.querySelector('h2 a')?.href || '';
                        const desc = item.querySelector('.b_caption p')?.innerText.trim() || 'No Description';
                        if (Utils.isValidURL(url)) results.push({ name, url, description: desc });
                    });
                    return results;
                },
                getNextPageUrl: doc => doc.querySelector('a.sb_pagN')?.href || null,
                getContentContainer: () => '#b_content'
            },
            yahoo: {
                extractResults: doc => {
                    const results = [];
                    doc.querySelectorAll('.dd.algo').forEach(item => {
                        const name = item.querySelector('h3')?.innerText.trim() || 'No Title';
                        const url = item.querySelector('a')?.href || '';
                        const desc = item.querySelector('.compText a')?.innerText.trim() || 'No Description';
                        if (Utils.isValidURL(url)) results.push({ name, url, description: desc });
                    });
                    return results;
                },
                getNextPageUrl: doc => doc.querySelector('a.next')?.href || null,
                getContentContainer: () => '#web'
            },
            duckduckgo: {
                extractResults: doc => {
                    const results = [];
                    doc.querySelectorAll('.result').forEach(item => {
                        const name = item.querySelector('h2')?.innerText.trim() || 'No Title';
                        const url = item.querySelector('a.result__a')?.href || '';
                        const desc = item.querySelector('.result__snippet')?.innerText.trim() || 'No Description';
                        if (Utils.isValidURL(url)) results.push({ name, url, description: desc });
                    });
                    return results;
                },
                getNextPageUrl: doc => doc.querySelector('a.result--more__btn')?.href || null,
                getContentContainer: () => '#links'
            },
            // Additional search engines can be added here...
        };

        function identifySearchEngine() {
            const host = window.location.hostname;
            if (host.includes('google.com')) return 'google';
            if (host.includes('bing.com') && !window.location.pathname.includes('/images')) return 'bing';
            if (host.includes('yahoo.com')) return 'yahoo';
            if (host.includes('duckduckgo.com')) return 'duckduckgo';
            // Add more as needed
            return null;
        }

        async function startCrawling() {
            if (isCrawlingFlag) {
                Logger.log('이미 크롤링이 진행 중입니다.', 'warning');
                return;
            }
            if (!Object.values(Config.settings.searchEngines).some(v => v)) {
                Logger.log('사용할 검색 엔진이 하나도 선택되지 않았습니다.', 'error');
                return;
            }
            isCrawlingFlag = true;
            Config.saveSettings({ isCrawling: isCrawlingFlag });
            Logger.log('크롤링을 시작합니다.', 'info');
            UI.updateLogs();
            if (!previousCrawls.some(crawl => crawl.status === '진행 중')) {
                previousCrawls.push({ date: new Date().toLocaleString(), count: 0, status: '진행 중' });
                GM_setValue('previousCrawls', JSON.stringify(previousCrawls));
                UI.updatePreviousCrawlsUI();
            }
            setButtonStates();
            await crawlPage(currentDepth);
            monitorPageNavigation();
        }

        function stopCrawling() {
            if (!isCrawlingFlag) {
                Logger.log('크롤링이 진행 중이지 않습니다.', 'warning');
                return;
            }
            isCrawlingFlag = false;
            Config.saveSettings({ isCrawling: isCrawlingFlag });
            Logger.log('크롤링이 중지되었습니다.', 'warning');
            finalizeCrawl('중지됨');
            UI.updateLogs();
            setButtonStates();
        }

        function setButtonStates() {
            $('#start-crawling').prop('disabled', isCrawlingFlag);
            $('#stop-crawling').prop('disabled', !isCrawlingFlag);
        }

        async function crawlPage(depth) {
            try {
                while (isCrawlingFlag && currentPage <= totalPages && depth <= Config.settings.maxDepth) {
                    Logger.log(`페이지 ${currentPage} 크롤링 시작. (깊이: ${depth})`, 'info');
                    if (Config.settings.respectRobots && !(await Robots.checkRobotsTxt(window.location.href))) {
                        Logger.log(`robots.txt에 의해 크롤링이 금지된 페이지: ${window.location.href}`, 'warning');
                        break;
                    }
                    const searchEngine = identifySearchEngine();
                    if (!searchEngine || !searchEngineHandlers[searchEngine]) {
                        Logger.log(`지원되지 않는 검색 엔진: ${window.location.hostname}`, 'error');
                        break;
                    }
                    const doc = document;
                    const handler = searchEngineHandlers[searchEngine];
                    const results = handler.extractResults(doc);
                    if (!results) { Logger.log('검색 결과를 찾을 수 없습니다.', 'error'); break; }
                    results.forEach(result => {
                        if (!isCrawlingFlag) return;
                        const { name, url, description } = result;
                        if (shouldSkip(url, description)) return;
                        requestQueue.push({ url, depth });
                        if (activeRequests >= Config.settings.maxConcurrentRequests) return;
                    });
                    if (previousCrawls.length && previousCrawls[previousCrawls.length -1].status === '진행 중') {
                        previousCrawls[previousCrawls.length -1].count += results.length;
                        GM_setValue('previousCrawls', JSON.stringify(previousCrawls));
                        UI.updatePreviousCrawlsUI();
                    }
                    await processQueue();
                    progress = (currentPage / totalPages) * 100;
                    Config.saveSettings({ progress });
                    UI.updateLogs();
                    if (currentPage < totalPages && isCrawlingFlag) {
                        const nextUrl = handler.getNextPageUrl(doc);
                        if (nextUrl) {
                            currentPage++;
                            Config.saveSettings({ currentPage });
                            depth++;
                            Config.saveSettings({ currentDepth: depth });
                            Logger.log(`다음 페이지로 이동: ${currentPage}`, 'info');
                            await Utils.delay(Config.settings.delay + Math.random() * Config.settings.delayRandom);
                            const nextPageContent = await Fetch.fetchPageContent(nextUrl);
                            if (nextPageContent) {
                                updatePageContent(nextPageContent);
                                continue;
                            } else {
                                Logger.log('다음 페이지 내용을 가져올 수 없습니다.', 'error');
                                break;
                            }
                        } else { Logger.log('다음 페이지 링크를 찾을 수 없습니다.', 'error'); break; }
                    } else { break; }
                }
                if (isCrawlingFlag && currentPage > totalPages && activeRequests === 0 && requestQueue.length === 0) {
                    Logger.log('크롤링이 완료되었습니다.', 'success');
                    finalizeCrawl('완료됨');
                    isCrawlingFlag = false;
                    Config.saveSettings({ isCrawling: isCrawlingFlag });
                    setButtonStates();
                    Config.saveSettings({ progress: 100 });
                    UI.updateLogs();
                }
            } catch (error) {
                Logger.log(`크롤링 중 오류 발생: ${error.message}`, 'error');
                finalizeCrawl('오류 발생');
            }
        }

        function shouldSkip(url, desc) {
            if (visitedUrls.has(url) || !Utils.isValidURL(url)) return true;
            if (Config.settings.whitelist.length && !Config.settings.whitelist.some(d => url.includes(d))) return true;
            if (Config.settings.blacklist.some(d => url.includes(d))) return true;
            if (Config.settings.includeKeywords.length && !Config.settings.includeKeywords.some(k => url.includes(k) || desc.includes(k))) return true;
            if (Config.settings.excludeKeywords.some(k => url.includes(k) || desc.includes(k))) return true;
            if (Config.settings.regexInclude.length && !Config.settings.regexInclude.some(r => new RegExp(r).test(url) || new RegExp(r).test(desc))) return true;
            if (Config.settings.regexExclude.length && Config.settings.regexExclude.some(r => new RegExp(r).test(url) || new RegExp(r).test(desc))) return true;
            return false;
        }

        async function processQueue() {
            while (requestQueue.length && activeRequests < Config.settings.maxConcurrentRequests && isCrawlingFlag) {
                const { url, depth } = requestQueue.shift();
                activeRequests++;
                handleRequest(url, depth).finally(() => { activeRequests--; processQueue(); });
            }
        }

        async function handleRequest(url, depth) {
            if (!isCrawlingFlag || visitedUrls.has(url)) return;
            try {
                const response = await Fetch.fetchWithRetry(url, Config.settings.retryAttempts, Config.settings.retryDelay);
                const contentType = Fetch.getContentType(response);
                if (!Fetch.isContentTypeAllowed(contentType, Config.settings.contentTypes)) {
                    Logger.log(`허용되지 않은 콘텐츠 유형(${contentType})의 URL: ${url}`, 'warning');
                    return;
                }
                const parser = new DOMParser();
                const doc = parser.parseFromString(response.responseText, 'text/html');
                const text = doc.body?.innerText || '';
                if (Config.settings.enableCaptchas && Robots.detectCaptcha(doc)) { await Robots.handleCaptcha(url); return; }
                const wc = text.split(/\s+/).length;
                if (wc < Config.settings.minWordCount || wc > Config.settings.maxWordCount) {
                    Logger.log(`단어 수 조건에 맞지 않는 URL: ${url} (${wc} 단어)`, 'warning');
                    return;
                }
                const quality = Evaluate.evaluateContentQuality(text);
                const lang = doc.documentElement.lang || 'unknown';
                const category = Classifier.classifyKeywords(text);
                const meta = Extractor.extractMetaTags(doc);
                const headings = Extractor.extractHeadings(doc);
                if (category !== '기타' || quality > 100) {
                    const entry = { url, quality, language: lang, wordCount: wc, category };
                    Config.settings.dataFields.name && (entry.name = Extractor.extractName(doc) || 'No Title');
                    Config.settings.dataFields.description && (entry.description = Extractor.extractDescription(doc) || 'No Description');
                    Config.settings.dataFields.html && (entry.html = response.responseText);
                    Config.settings.dataFields.metaTags && (entry.metaTags = meta);
                    Config.settings.dataFields.headings && (entry.headings = headings);
                    data.push(entry);
                    GM_setValue('data', JSON.stringify(data));
                    UI.updateDataTable();
                    Logger.log(`데이터 수집 완료: ${url}`, 'success');
                    Config.settings.dataVisualization && DataVisualization.renderChart();
                }
                if (depth < Config.settings.maxDepth) {
                    Extractor.extractLinks(doc).forEach(link => { if (!visitedUrls.has(link) && Utils.isValidURL(link)) requestQueue.push({ url: link, depth: depth + 1 }); });
                }
            } catch (error) {
                Logger.log(`데이터 수집 실패: ${url} - ${error.message}`, 'error');
            }
            visitedUrls.add(url);
            GM_setValue('visitedUrls', JSON.stringify([...visitedUrls]));
        }

        function finalizeCrawl(status) {
            if (previousCrawls.length) {
                previousCrawls[previousCrawls.length -1].status = status;
                GM_setValue('previousCrawls', JSON.stringify(previousCrawls));
                UI.updatePreviousCrawlsUI();
            }
            if (navigationObserver) {
                navigationObserver.disconnect();
                navigationObserver = null;
            }
        }

        function updateCrawlSettings() {
            // Placeholder for any settings update needed during crawling
        }

        function updatePageContent(content) {
            const parser = new DOMParser();
            const doc = parser.parseFromString(content, 'text/html');
            document.documentElement.innerHTML = doc.documentElement.innerHTML;
        }

        function monitorPageNavigation() {
            const searchEngine = identifySearchEngine();
            if (!searchEngine || !searchEngineHandlers[searchEngine]) return;
            const containerSelector = searchEngineHandlers[searchEngine].getContentContainer();
            const targetNode = document.querySelector(containerSelector);
            if (!targetNode) {
                Logger.log('페이지 내비게이션 감시 대상 노드를 찾을 수 없습니다.', 'warning');
                return;
            }
            navigationObserver = new MutationObserver(mutations => {
                for (let mutation of mutations) {
                    if (mutation.type === 'childList' && isCrawlingFlag) {
                        Logger.log('페이지 내비게이션 감지됨. 크롤링을 중지합니다.', 'warning');
                        Crawling.stopCrawling();
                        break;
                    }
                }
            });
            navigationObserver.observe(targetNode, { childList: true, subtree: true });
            Logger.log('페이지 내비게이션 감시가 활성화되었습니다.', 'info');
        }

        return {
            startCrawling,
            stopCrawling,
            isCrawling: () => isCrawlingFlag,
            updateCrawlSettings
        };
    })();

    const Fetch = (() => {
        async function fetchWithRetry(url, attempts, delayMs) {
            for (let i = 1; i <= attempts; i++) {
                try {
                    return await GM_xmlhttpRequestPromise(url);
                } catch (e) {
                    Logger.log(`요청 실패 (${i}/${attempts}): ${url} - ${e.message}`, 'warning');
                    if (i < attempts) await Utils.delay(delayMs);
                }
            }
            throw new Error('모든 재시도 시도가 실패했습니다.');
        }

        function GM_xmlhttpRequestPromise(url) {
            return new Promise((resolve, reject) => {
                let reqUrl = Config.settings.proxy ? `${Config.settings.proxy}?url=${encodeURIComponent(url)}` : url;
                GM_xmlhttpRequest({
                    method: "GET",
                    url: reqUrl,
                    headers: { 'User-Agent': getCurrentUserAgent(), 'Accept': '*/*' },
                    timeout: Config.settings.timeout,
                    onload: res => (res.status >= 200 && res.status < 300) ? resolve(res) : reject(new Error(`HTTP ${res.status}`)),
                    onerror: () => reject(new Error('Request failed')),
                    ontimeout: () => reject(new Error('Request timed out'))
                });
            });
        }

        function getCurrentUserAgent() {
            const index = Config.settings.userAgentIndex % Config.settings.userAgents.length;
            return Config.settings.userAgents[index];
        }

        function getContentType(res) {
            return res.responseHeaders.split('\n').find(h => h.toLowerCase().startsWith('content-type'))?.split(':')[1].trim().split(';')[0] || '';
        }

        function isContentTypeAllowed(ct, contentTypes) {
            if (contentTypes.include.length && !contentTypes.include.includes(ct)) return false;
            if (contentTypes.exclude.includes(ct)) return false;
            return true;
        }

        async function fetchPageContent(url) {
            try {
                const response = await fetchWithRetry(url, Config.settings.retryAttempts, Config.settings.retryDelay);
                return response.responseText;
            } catch (error) {
                Logger.log(`페이지 내용 가져오기 실패: ${url} - ${error.message}`, 'error');
                return null;
            }
        }

        return {
            fetchWithRetry,
            getCurrentUserAgent,
            getContentType,
            isContentTypeAllowed,
            fetchPageContent
        };
    })();

    const Robots = (() => {
        async function checkRobotsTxt(url) {
            try {
                const parsed = new URL(url);
                const robotsUrl = `${parsed.origin}/robots.txt`;
                const res = await Fetch.fetchWithRetry(robotsUrl, 2, 2000);
                return new RobotsParser(res.responseText).isAllowed(url, Fetch.getCurrentUserAgent());
            } catch {
                Logger.log('robots.txt 확인 실패, 기본 허용.', 'warning');
                return true;
            }
        }

        function detectCaptcha(doc) {
            return !!doc.querySelector('iframe[src*="captcha"], form[action*="captcha"]');
        }

        async function handleCaptcha(url) {
            Logger.log(`CAPTCHA 감지됨: ${url}`, 'warning');
            await Utils.delay(5000);
            Logger.log(`CAPTCHA 처리 완료: ${url}`, 'info');
        }

        class RobotsParser {
            constructor(text) { this.rules = []; this.parse(text); }
            parse(text) {
                let agents = [];
                text.split('\n').forEach(line => {
                    const [key, ...val] = line.split(':').map(s => s.trim());
                    if (!key) return;
                    if (key.toLowerCase() === 'user-agent') agents = [val.join(':').toLowerCase()];
                    else if (['disallow','allow'].includes(key.toLowerCase())) this.rules.push({ agents, type: key.toLowerCase(), path: val.join(':') });
                });
            }
            isAllowed(url, ua) {
                const parsedUrl = new URL(url);
                const path = parsedUrl.pathname;
                let allowed = true;
                this.rules.forEach(rule => {
                    if (rule.agents.includes('*') || rule.agents.includes(ua.toLowerCase())) {
                        if (rule.type === 'disallow' && path.startsWith(rule.path)) allowed = false;
                        if (rule.type === 'allow' && path.startsWith(rule.path)) allowed = true;
                    }
                });
                return allowed;
            }
        }

        return {
            checkRobotsTxt,
            detectCaptcha,
            handleCaptcha
        };
    })();

    const Extractor = (() => {
        function extractMetaTags(doc) {
            const metas = {};
            doc.querySelectorAll('meta[name], meta[property]').forEach(meta => {
                const name = meta.getAttribute('name') || meta.getAttribute('property');
                const content = meta.getAttribute('content');
                if (name && content) metas[name.toLowerCase()] = content;
            });
            return metas;
        }

        function extractHeadings(doc) {
            const hs = [];
            for (let i = 1; i <= 6; i++) doc.querySelectorAll(`h${i}`).forEach(h => hs.push(h.innerText.trim()));
            return hs;
        }

        function extractName(doc) { return doc.querySelector('title')?.innerText || 'No Title'; }
        function extractDescription(doc) { return doc.querySelector('meta[name="description"]')?.getAttribute('content') || 'No Description'; }
        function extractLinks(doc) { return Array.from(doc.querySelectorAll('a[href]')).map(a => a.href).filter(Utils.isValidURL); }

        return {
            extractMetaTags,
            extractHeadings,
            extractName,
            extractDescription,
            extractLinks
        };
    })();

    const Classifier = (() => {
        function classifyKeywords(text) {
            const kws = combineKeywords();
            for (const [cat, keywords] of Object.entries(kws)) {
                if (keywords.some(kw => text.includes(kw))) return cat;
            }
            return '기타';
        }

        function combineKeywords() {
            const standard = {
                '교육': ['학교', '교육', '학생', '교수', '강의'],
                '기술': ['기술', '소프트웨어', '하드웨어', '프로그래밍', '개발'],
                '뉴스': ['뉴스', '속보', '기사', '보도'],
                '엔터테인먼트': ['영화', '음악', '연예', '게임', '드라마'],
                '스포츠': ['축구', '야구', '농구', '경기', '토너먼트'],
                '비즈니스': ['비즈니스', '경제', '주식', '투자', '시장'],
                '건강': ['건강', '의료', '헬스', '운동', '영양'],
                '과학': ['과학', '연구', '실험', '발견', '이론'],
                '문학': ['소설', '시', '문학', '작가', '책'],
                '여행': ['여행', '관광', '숙박', '비행기', '여권']
            };
            const custom = Config.settings.customKeywords || {};
            for (const [cat, kws] of Object.entries(custom)) {
                standard[cat] = standard[cat] ? [...new Set([...standard[cat], ...kws])] : kws;
            }
            return standard;
        }

        return {
            classifyKeywords
        };
    })();

    const Evaluate = (() => {
        function evaluateContentQuality(text) { return text.split(/\s+/).length; }

        return {
            evaluateContentQuality
        };
    })();

    const Scheduler = (() => {
        let scheduleInterval = null;

        function scheduleCrawling() {
            clearInterval(scheduleInterval);
            const now = new Date();
            const [startHour, startMinute] = Config.settings.schedule.startTime.split(':').map(Number);
            const startTime = new Date(now.getFullYear(), now.getMonth(), now.getDate(), startHour, startMinute, 0);
            if (startTime < now) startTime.setDate(startTime.getDate() + 1);
            const initialDelay = startTime - now;
            setTimeout(() => {
                if (Config.settings.schedule.specificDays.length === 0 || Config.settings.schedule.specificDays.includes(startTime.getDay())) {
                    Crawling.startCrawling();
                }
                scheduleInterval = setInterval(() => {
                    const currentDay = new Date().getDay();
                    if (Config.settings.schedule.specificDays.length === 0 || Config.settings.schedule.specificDays.includes(currentDay)) {
                        Crawling.startCrawling();
                    }
                }, Config.settings.schedule.interval);
            }, initialDelay);
            Logger.log(`크롤링 스케줄이 설정되었습니다. 시작 시간: ${startTime.toLocaleString()}, 간격: ${Config.settings.schedule.interval / 60000}분`, 'info');
        }

        return {
            scheduleCrawling
        };
    })();

    const Testing = (() => {
        function runTests() {
            $('#test-modal').show();
            $('#mocha').html('');
            $('head').append('<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/mocha/10.2.0/mocha.min.css" />');
            mocha.setup('bdd');
            const assert = chai.assert;

            describe('Utils Module', function() {
                it('should escape HTML correctly', function() {
                    const input = '<div>Hello & "World"!</div>';
                    const expected = '&lt;div&gt;Hello &amp; &quot;World&quot;!&lt;/div&gt;';
                    assert.equal(Utils.escape(input, 'html'), expected);
                });
                it('should capitalize strings correctly', function() {
                    const input = 'test';
                    const expected = 'Test';
                    assert.equal(Utils.capitalize(input), expected);
                });
                it('should validate URLs correctly', function() {
                    assert.isTrue(Utils.isValidURL('https://www.example.com'));
                    assert.isFalse(Utils.isValidURL('htp:/invalid-url'));
                });
            });

            describe('Config Module', function() {
                it('should save and load settings correctly', function() {
                    const originalDelay = Config.settings.delay;
                    Config.saveSettings({ delay: 3000 });
                    assert.equal(Config.settings.delay, 3000);
                    Config.saveSettings({ delay: originalDelay });
                });
                it('should add and remove templates correctly', function() {
                    const templateName = 'TestTemplate';
                    Config.saveTemplate(templateName);
                    assert.property(Config.settings.templates, templateName);
                    Config.loadTemplate(templateName);
                    Config.settings.templates[templateName] = undefined;
                    delete Config.settings.templates[templateName];
                    assert.notProperty(Config.settings.templates, templateName);
                });
            });

            describe('Logger Module', function() {
                it('should log messages correctly', function() {
                    Logger.log('Test log message', 'info');
                    const logs = Logger.getLogs();
                    assert.isAtLeast(logs.length, 1);
                    assert.equal(logs[logs.length -1].message, 'Test log message');
                });
                it('should clear logs correctly', function() {
                    Logger.clearLogs();
                    assert.equal(Logger.getLogs().length, 0);
                });
            });

            mocha.run();
        }

        return {
            runTests
        };
    })();

    function init() {
        UI.createUI();
        UI.updateUI();
        if (Config.settings.dataVisualization) DataVisualization.renderChart();
        if (Config.settings.schedule.enabled) Scheduler.scheduleCrawling();
        if (Config.settings.isCrawling) {
            Logger.log('크롤링 재개됨.', 'info');
            Crawling.startCrawling();
        } else {
            Logger.log('크롤러 초기화됨.', 'info');
        }
    }

    window.addEventListener('load', init);

})();