// ==UserScript==
// @name         Advanced Web Crawler Enhanced Pro Optimized
// @namespace    http://tampermonkey.net/
// @version      6.0.1
// @description  Optimized version of Advanced Web Crawler Enhanced Pro
// @author       Your Name
// @match        https://www.google.com/search*
// @match        https://www.bing.com/search*
// @match        https://search.yahoo.com/search*
// @match        https://duckduckgo.com/*
// @match        https://www.baidu.com/s*
// @match        https://www.yandex.com/search/*
// @match        https://search.naver.com/search*
// @match        https://search.daum.net/search*
// @match        https://www.ask.com/*
// @match        https://www.ecosia.org/search*
// @match        https://www.startpage.com/*
// @match        https://searx.me/*
// @match        https://www.dogpile.com/*
// @match        https://www.bing.com/images*
// @grant        GM_xmlhttpRequest
// @grant        GM_setValue
// @grant        GM_getValue
// @grant        GM_deleteValue
// @connect      *
// @require      https://cdnjs.cloudflare.com/ajax/libs/jquery/3.6.0/jquery.min.js
// @require      https://cdn.jsdelivr.net/npm/chart.js
// @require      https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.18.5/xlsx.full.min.js
// ==/UserScript==

(function() {
    'use strict';
    const defaultSettings = {
        delay: 2000,
        delayRandom: 500,
        maxPages: 3,
        userAgents: [
            'AdvancedWebCrawler/6.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15',
            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)',
            'Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15',
            'Mozilla/5.0 (iPad; CPU OS 14_0 like Mac OS X) AppleWebKit/605.1.15',
        ],
        userAgentIndex: 0,
        blacklist: [],
        whitelist: [],
        contentTypes: {
            include: ['text/html'],
            exclude: ['application/pdf', 'image/jpeg', 'image/png', 'video/mp4']
        },
        timeout: 8000,
        includeKeywords: [],
        excludeKeywords: [],
        language: [],
        minWordCount: 100,
        maxWordCount: 10000,
        maxDepth: 2,
        proxy: '',
        regexInclude: [],
        regexExclude: [],
        dataFields: {
            name: true,
            url: true,
            description: true,
            html: true,
            category: true,
            quality: true,
            language: true,
            wordCount: true,
            metaTags: false,
            headings: false
        },
        schedule: {
            enabled: false,
            startTime: '00:00',
            interval: 3600000
        },
        searchEngines: {
            google: true,
            bing: true,
            yahoo: true,
            duckduckgo: true,
            baidu: true,
            yandex: true,
            naver: true,
            daum: true,
            ask: true,
            ecosia: true,
            startpage: true,
            searx: true,
            dogpile: true,
            bingimages: true
        },
        retryAttempts: 3,
        retryDelay: 2000,
        respectRobots: true,
        maxConcurrentRequests: 3,
        dataVisualization: true,
        exportFormats: {
            csv: true,
            json: true,
            xml: true,
            xls: true
        },
        advancedLogging: true,
        enableCaptchas: false
    };
    const settings = Object.assign({}, defaultSettings, {
        delay: parseInt(GM_getValue('delay', defaultSettings.delay), 10),
        delayRandom: parseInt(GM_getValue('delayRandom', defaultSettings.delayRandom), 10),
        maxPages: parseInt(GM_getValue('maxPages', defaultSettings.maxPages), 10),
        userAgents: GM_getValue('userAgents', defaultSettings.userAgents),
        userAgentIndex: parseInt(GM_getValue('userAgentIndex', defaultSettings.userAgentIndex), 10),
        blacklist: GM_getValue('blacklist', defaultSettings.blacklist),
        whitelist: GM_getValue('whitelist', defaultSettings.whitelist),
        contentTypes: GM_getValue('contentTypes', defaultSettings.contentTypes),
        timeout: parseInt(GM_getValue('timeout', defaultSettings.timeout), 10),
        includeKeywords: GM_getValue('includeKeywords', defaultSettings.includeKeywords),
        excludeKeywords: GM_getValue('excludeKeywords', defaultSettings.excludeKeywords),
        regexInclude: GM_getValue('regexInclude', defaultSettings.regexInclude),
        regexExclude: GM_getValue('regexExclude', defaultSettings.regexExclude),
        language: GM_getValue('language', defaultSettings.language),
        minWordCount: parseInt(GM_getValue('minWordCount', defaultSettings.minWordCount), 10),
        maxWordCount: parseInt(GM_getValue('maxWordCount', defaultSettings.maxWordCount), 10),
        maxDepth: parseInt(GM_getValue('maxDepth', defaultSettings.maxDepth), 10),
        proxy: GM_getValue('proxy', defaultSettings.proxy),
        dataFields: GM_getValue('dataFields', defaultSettings.dataFields),
        schedule: GM_getValue('schedule', defaultSettings.schedule),
        searchEngines: GM_getValue('searchEngines', defaultSettings.searchEngines),
        retryAttempts: parseInt(GM_getValue('retryAttempts', defaultSettings.retryAttempts), 10),
        retryDelay: parseInt(GM_getValue('retryDelay', defaultSettings.retryDelay), 10),
        respectRobots: GM_getValue('respectRobots', defaultSettings.respectRobots),
        maxConcurrentRequests: parseInt(GM_getValue('maxConcurrentRequests', defaultSettings.maxConcurrentRequests), 10),
        dataVisualization: GM_getValue('dataVisualization', defaultSettings.dataVisualization),
        exportFormats: GM_getValue('exportFormats', defaultSettings.exportFormats),
        advancedLogging: GM_getValue('advancedLogging', defaultSettings.advancedLogging),
        enableCaptchas: GM_getValue('enableCaptchas', defaultSettings.enableCaptchas),
        customKeywords: GM_getValue('customKeywords', {})
    });
    let visitedUrls = GM_getValue('visitedUrls', []);
    let logs = GM_getValue('logs', []);
    let data = GM_getValue('data', []);
    let isCrawling = GM_getValue('isCrawling', false);
    let currentPage = GM_getValue('currentPage', 1);
    let totalPages = GM_getValue('totalPages', settings.maxPages);
    let progress = GM_getValue('progress', 0);
    let currentDepth = GM_getValue('currentDepth', 0);
    let activeRequests = 0;
    let requestQueue = [];

    function createUI() {
        const style = `
            #crawler-container { position: fixed; top: 20px; right: 20px; width: 800px; height: 800px; background: #fff; border: 1px solid #ccc; z-index: 10000; font-family: Arial, sans-serif; box-shadow: 0 4px 8px rgba(0,0,0,0.2); overflow: hidden; resize: both; overflow: auto; border-radius: 8px; min-width: 500px; min-height: 600px; }
            #crawler-tabs { display: flex; background: #f1f1f1; border-bottom: 1px solid #ccc; }
            .crawler-tab { flex: 1; padding: 10px; text-align: center; cursor: pointer; font-weight: bold; }
            .crawler-tab.active { background: #ddd; }
            #crawler-content { padding: 10px; overflow: auto; height: calc(100% - 50px); }
            table { width: 100%; border-collapse: collapse; }
            th, td { padding: 8px; text-align: left; border-bottom: 1px solid #ddd; }
            th { background-color: #f2f2f2; cursor: pointer; }
            tr:hover { background-color: #f5f5f5; }
            .progress-bar { width: 100%; background-color: #f3f3f3; border: 1px solid #ccc; border-radius: 5px; overflow: hidden; margin-bottom: 10px; }
            .progress-bar-fill { height: 20px; background-color: #4caf50; width: 0%; transition: width 0.2s; }
            .modal { display: none; position: fixed; z-index: 10001; left: 0; top: 0; width: 100%; height: 100%; overflow: auto; background-color: rgba(0,0,0,0.4); }
            .modal-content { background-color: #fff; margin: 5% auto; padding: 20px; border: 1px solid #888; width: 80%; max-height: 80%; overflow-y: auto; border-radius: 8px; }
            .close { color: #aaa; float: right; font-size: 28px; font-weight: bold; cursor: pointer; }
            .control-buttons { margin: 10px 0; }
            .control-buttons button { padding: 8px 12px; margin-right: 8px; cursor: pointer; border: none; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px; }
            .control-buttons button:disabled { background-color: #a5d6a7; cursor: not-allowed; }
            .status-text { margin-top: 10px; font-weight: bold; font-size: 14px; }
            .settings-section { margin-bottom: 15px; }
            .settings-section label { display: block; margin-bottom: 5px; font-weight: bold; }
            .settings-section input[type="text"], .settings-section input[type="number"], .settings-section textarea, .settings-section select { width: 100%; padding: 5px; box-sizing: border-box; border: 1px solid #ccc; border-radius: 4px; font-size: 14px; }
            .settings-section .checkbox-group { display: flex; flex-wrap: wrap; }
            .settings-section .checkbox-group label { margin-right: 10px; font-size: 14px; }
            .settings-section .radio-group { display: flex; flex-wrap: wrap; }
            .settings-section .radio-group label { margin-right: 10px; font-size: 14px; }
            .tooltip { position: relative; display: inline-block; cursor: help; font-size: 14px; }
            .tooltip .tooltiptext { visibility: hidden; width: 220px; background-color: #555; color: #fff; text-align: center; border-radius: 6px; padding: 5px; position: absolute; z-index: 1; bottom: 125%; left: 50%; margin-left: -110px; opacity: 0; transition: opacity 0.3s; }
            .tooltip:hover .tooltiptext { visibility: visible; opacity: 1; }
            #crawler-container::-webkit-resizer { background: #f1f1f1; border: none; }
            #data-visualization { margin-top: 20px; }
            #data-chart { width: 100%; height: 300px; }
            #modal-html-content { white-space: pre-wrap; word-wrap: break-word; background-color: #f9f9f9; padding: 10px; border-radius: 4px; }
        `;
        $('head').append(`<style>${style}</style>`);
        const container = $(`
            <div id="crawler-container">
                <div id="crawler-tabs">
                    <div class="crawler-tab ${currentTab() === 'logs' ? 'active' : ''}" data-tab="logs" title="로그 보기">📜 로그</div>
                    <div class="crawler-tab ${currentTab() === 'data' ? 'active' : ''}" data-tab="data" title="수집된 데이터 보기">📊 데이터</div>
                    <div class="crawler-tab ${currentTab() === 'settings' ? 'active' : ''}" data-tab="settings" title="설정 변경">⚙️ 설정</div>
                </div>
                <div id="crawler-content">
                    <div id="tab-logs" style="display: ${currentTab() === 'logs' ? 'block' : 'none'};">
                        <div class="progress-bar">
                            <div class="progress-bar-fill" style="width: ${progress.toFixed(2)}%;"></div>
                        </div>
                        <div class="status-text">진행률: ${progress.toFixed(2)}% (${currentPage}/${totalPages} 페이지)</div>
                        <div class="control-buttons">
                            <button id="start-crawling" title="크롤링을 시작합니다." ${isCrawling ? 'disabled' : ''}>▶ 시작</button>
                            <button id="stop-crawling" title="크롤링을 중지합니다." ${isCrawling ? '' : 'disabled'}>■ 중지</button>
                            ${settings.exportFormats.csv ? `<button id="download-csv" title="CSV 다운로드">📥 CSV</button>` : ''}
                            ${settings.exportFormats.json ? `<button id="download-json" title="JSON 다운로드">📥 JSON</button>` : ''}
                            ${settings.exportFormats.xml ? `<button id="download-xml" title="XML 다운로드">📥 XML</button>` : ''}
                            ${settings.exportFormats.xls ? `<button id="download-xls" title="XLS 다운로드">📥 XLS</button>` : ''}
                        </div>
                        <div id="log-messages" style="height: 250px; overflow-y: auto; background: #f9f9f9; padding: 10px; border: 1px solid #ccc; border-radius: 4px;">로그 내용이 여기에 표시됩니다.</div>
                    </div>
                    <div id="tab-data" style="display: ${currentTab() === 'data' ? 'block' : 'none'};">
                        <input type="text" id="search-filter" placeholder="검색 필터..." style="width: 100%; padding: 5px; margin-bottom: 10px; border: 1px solid #ccc; border-radius: 4px;">
                        <table>
                            <thead>
                                <tr>
                                    ${settings.dataFields.name ? '<th data-sort="name">사이트 이름</th>' : ''}
                                    ${settings.dataFields.url ? '<th data-sort="url">주소</th>' : ''}
                                    ${settings.dataFields.description ? '<th data-sort="description">설명</th>' : ''}
                                    ${settings.dataFields.html ? '<th>📄 HTML 내용</th>' : ''}
                                    ${settings.dataFields.category ? '<th data-sort="category">분류</th>' : ''}
                                    ${settings.dataFields.quality ? '<th data-sort="quality">품질점수</th>' : ''}
                                    ${settings.dataFields.language ? '<th data-sort="language">언어</th>' : ''}
                                    ${settings.dataFields.wordCount ? '<th data-sort="wordCount">단어 수</th>' : ''}
                                    ${settings.dataFields.metaTags ? '<th>메타 태그</th>' : ''}
                                    ${settings.dataFields.headings ? '<th>헤딩</th>' : ''}
                                </tr>
                            </thead>
                            <tbody id="data-table-body"></tbody>
                        </table>
                        ${settings.dataVisualization ? `
                            <div id="data-visualization">
                                <h3>데이터 시각화</h3>
                                <canvas id="data-chart"></canvas>
                            </div>
                        ` : ''}
                    </div>
                    <div id="tab-settings" style="display: ${currentTab() === 'settings' ? 'block' : 'none'};">
                        <h3>설정</h3>
                        <div class="settings-section">
                            <label for="delay-input">크롤링 딜레이 (ms):
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">기본 딜레이과 딜레이 랜덤 값을 조합하여 요청 사이의 딜레이를 설정합니다.</span>
                                </span>
                            </label>
                            <input type="number" id="delay-input" value="${settings.delay}" min="500" step="500">
                            <label for="delay-random-input">딜레이 랜덤 범위 (ms):
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">딜레이에 무작위 요소를 추가하여 보다 자연스러운 요청을 합니다.</span>
                                </span>
                            </label>
                            <input type="number" id="delay-random-input" value="${settings.delayRandom}" min="0" step="500">
                        </div>
                        <div class="settings-section">
                            <label for="max-pages-input">최대 크롤링 페이지 수:
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">크롤링할 최대 페이지 수를 설정하세요.</span>
                                </span>
                            </label>
                            <input type="number" id="max-pages-input" value="${settings.maxPages}" min="1">
                        </div>
                        <div class="settings-section">
                            <label for="user-agent-select">사용자 에이전트(User-Agent):
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">크롤링에 사용할 사용자 에이전트를 선택하거나 추가할 수 있습니다.</span>
                                </span>
                            </label>
                            <select id="user-agent-select" multiple size="6">
                                ${settings.userAgents.map((ua, idx) => `<option value="${idx}" ${idx === settings.userAgentIndex ? 'selected' : ''}>${ua}</option>`).join('')}
                            </select>
                            <button id="add-user-agent" style="margin-top: 5px; padding: 5px 10px; background-color: #2196F3; color: white; border-radius: 4px; font-size: 12px;">추가</button>
                            <button id="remove-user-agent" style="margin-top: 5px; padding: 5px 10px; background-color: #f44336; color: white; border-radius: 4px; font-size: 12px;">제거</button>
                        </div>
                        <div class="settings-section">
                            <label>콘텐츠 유형:</label>
                            <div>
                                <strong>포함할 유형:</strong>
                                <textarea id="content-include-input" rows="3">${settings.contentTypes.include.join(', ')}</textarea>
                            </div>
                            <div>
                                <strong>제외할 유형:</strong>
                                <textarea id="content-exclude-input" rows="3">${settings.contentTypes.exclude.join(', ')}</textarea>
                            </div>
                        </div>
                        <div class="settings-section">
                            <label for="timeout-input">크롤링 타임아웃 (ms):
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">요청이 완료되기까지 기다릴 최대 시간을 밀리초 단위로 설정합니다.</span>
                                </span>
                            </label>
                            <input type="number" id="timeout-input" value="${settings.timeout}" min="1000" step="1000">
                        </div>
                        <div class="settings-section">
                            <label for="include-keywords-input">포함할 키워드:
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">크롤링할 페이지에 반드시 포함되어야 하는 키워드를 입력하세요. 각 키워드는 줄바꿈으로 구분됩니다.</span>
                                </span>
                            </label>
                            <textarea id="include-keywords-input" rows="3">${settings.includeKeywords.join('\n')}</textarea>
                        </div>
                        <div class="settings-section">
                            <label for="exclude-keywords-input">제외할 키워드:
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">크롤링할 페이지에 포함되면 안 되는 키워드를 입력하세요. 각 키워드는 줄바꿈으로 구분됩니다.</span>
                                </span>
                            </label>
                            <textarea id="exclude-keywords-input" rows="3">${settings.excludeKeywords.join('\n')}</textarea>
                        </div>
                        <div class="settings-section">
                            <label for="regex-include-input">포함할 키워드 (정규 표현식):
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">포함할 키워드의 정규 표현식을 입력하세요. 각 패턴은 줄바꿈으로 구분됩니다.</span>
                                </span>
                            </label>
                            <textarea id="regex-include-input" rows="3">${settings.regexInclude.join('\n')}</textarea>
                        </div>
                        <div class="settings-section">
                            <label for="regex-exclude-input">제외할 키워드 (정규 표현식):
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">제외할 키워드의 정규 표현식을 입력하세요. 각 패턴은 줄바꿈으로 구분됩니다.</span>
                                </span>
                            </label>
                            <textarea id="regex-exclude-input" rows="3">${settings.regexExclude.join('\n')}</textarea>
                        </div>
                        <div class="settings-section">
                            <label for="language-select">언어 필터링:</label>
                            <select id="language-select" multiple size="10">
                                ${[
                                    { value: 'en', text: '영어 (en)' }, { value: 'ko', text: '한국어 (ko)' },
                                    { value: 'zh', text: '중국어 (zh)' }, { value: 'ja', text: '일본어 (ja)' },
                                    { value: 'ru', text: '러시아어 (ru)' }, { value: 'es', text: '스페인어 (es)' },
                                    { value: 'fr', text: '프랑스어 (fr)' }, { value: 'de', text: '독일어 (de)' },
                                    { value: 'it', text: '이탈리아어 (it)' }, { value: 'pt', text: '포르투갈어 (pt)' },
                                    { value: 'ar', text: '아랍어 (ar)' }, { value: 'hi', text: '힌디어 (hi)' },
                                    { value: 'bn', text: '벵골어 (bn)' }, { value: 'pa', text: '펀자브어 (pa)' },
                                    { value: 'nl', text: '네덜란드어 (nl)' }, { value: 'sv', text: '스웨덴어 (sv)' },
                                    { value: 'tr', text: '터키어 (tr)' }, { value: 'vi', text: '베트남어 (vi)' },
                                    { value: 'pl', text: '폴란드어 (pl)' }, { value: 'th', text: '태국어 (th)' }
                                ].map(lang => `<option value="${lang.value}" ${settings.language.includes(lang.value) ? 'selected' : ''}>${lang.text}</option>`).join('')}
                            </select>
                        </div>
                        <div class="settings-section">
                            <label for="min-word-count-input">최소 단어 수:
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">수집할 페이지의 최소 단어 수를 설정하세요.</span>
                                </span>
                            </label>
                            <input type="number" id="min-word-count-input" value="${settings.minWordCount}" min="10" step="10">
                            <label for="max-word-count-input">최대 단어 수:
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">수집할 페이지의 최대 단어 수를 설정하세요.</span>
                                </span>
                            </label>
                            <input type="number" id="max-word-count-input" value="${settings.maxWordCount}" min="100" step="100">
                        </div>
                        <div class="settings-section">
                            <label for="max-depth-input">최대 크롤링 깊이:
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">링크를 추적할 최대 깊이를 설정하세요.</span>
                                </span>
                            </label>
                            <input type="number" id="max-depth-input" value="${settings.maxDepth}" min="1" step="1">
                        </div>
                        <div class="settings-section">
                            <label for="proxy-input">프록시 서버:
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">프록시 서버 주소를 입력하세요. 예: http://proxyserver:port</span>
                                </span>
                            </label>
                            <input type="text" id="proxy-input" value="${settings.proxy}" placeholder="http://proxyserver:port">
                        </div>
                        <div class="settings-section">
                            <label>수집할 데이터 필드:</label>
                            <div class="checkbox-group">
                                ${Object.keys(settings.dataFields).map(field => `<label><input type="checkbox" id="field-${field}" ${settings.dataFields[field] ? 'checked' : ''}> ${getFieldName(field)}</label>`).join('')}
                            </div>
                        </div>
                        <div class="settings-section">
                            <label for="retry-attempts-input">재시도 횟수:
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">요청 실패 시 재시도할 횟수를 설정하세요.</span>
                                </span>
                            </label>
                            <input type="number" id="retry-attempts-input" value="${settings.retryAttempts}" min="0" step="1">
                            <label for="retry-delay-input">재시도 딜레이 (ms):
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">재시도 시 대기할 시간을 밀리초 단위로 설정하세요.</span>
                                </span>
                            </label>
                            <input type="number" id="retry-delay-input" value="${settings.retryDelay}" min="1000" step="500">
                        </div>
                        <div class="settings-section">
                            <label>크롤링 스케줄링:</label>
                            <div class="checkbox-group">
                                <label><input type="checkbox" id="schedule-enabled" ${settings.schedule.enabled ? 'checked' : ''}> 스케줄링 활성화</label>
                            </div>
                            <div id="schedule-options" style="display: ${settings.schedule.enabled ? 'block' : 'none'}; margin-top: 10px;">
                                <label for="start-time-input">시작 시간 (HH:MM):
                                    <span class="tooltip">ℹ️
                                        <span class="tooltiptext">크롤링을 시작할 시간을 설정하세요.</span>
                                    </span>
                                </label>
                                <input type="time" id="start-time-input" value="${settings.schedule.startTime}">
                                <label for="interval-input">크롤링 간격 (분):
                                    <span class="tooltip">ℹ️
                                        <span class="tooltiptext">크롤링 작업 간의 간격을 분 단위로 설정하세요.</span>
                                    </span>
                                </label>
                                <input type="number" id="interval-input" value="${settings.schedule.interval / 60000}" min="1" step="1">
                            </div>
                        </div>
                        <div class="settings-section">
                            <label>크롤링할 검색 엔진 선택:</label>
                            <div class="checkbox-group">
                                ${Object.keys(settings.searchEngines).map(engine => `<label><input type="checkbox" id="engine-${engine}" ${settings.searchEngines[engine] ? 'checked' : ''}> ${capitalize(engine)}</label>`).join('')}
                            </div>
                        </div>
                        <div class="settings-section">
                            <label for="blacklist-input">블랙리스트 도메인:
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">크롤링에서 제외할 도메인을 입력하세요. 각 도메인은 줄바꿈으로 구분됩니다.</span>
                                </span>
                            </label>
                            <textarea id="blacklist-input" rows="5">${settings.blacklist.join('\n')}</textarea>
                        </div>
                        <div class="settings-section">
                            <label for="whitelist-input">화이트리스트 도메인:
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">크롤링을 허용할 도메인을 입력하세요. 각 도메인은 줄바꿈으로 구분됩니다.</span>
                                </span>
                            </label>
                            <textarea id="whitelist-input" rows="5">${settings.whitelist.join('\n')}</textarea>
                        </div>
                        <div class="settings-section">
                            <label for="respect-robots-checkbox">robots.txt 준수:
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">대상 사이트의 robots.txt 규칙을 준수하여 크롤링을 진행합니다.</span>
                                </span>
                            </label>
                            <input type="checkbox" id="respect-robots-checkbox" ${settings.respectRobots ? 'checked' : ''}>
                        </div>
                        <div class="settings-section">
                            <label for="max-concurrent-requests-input">동시 요청 최대 수:
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">한 번에 요청할 수 있는 최대 동시 요청 수를 설정하세요.</span>
                                </span>
                            </label>
                            <input type="number" id="max-concurrent-requests-input" value="${settings.maxConcurrentRequests}" min="1" step="1">
                        </div>
                        <div class="settings-section">
                            <label for="data-visualization-checkbox">데이터 시각화 활성화:
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">수집된 데이터를 시각화하여 그래프로 표시합니다.</span>
                                </span>
                            </label>
                            <input type="checkbox" id="data-visualization-checkbox" ${settings.dataVisualization ? 'checked' : ''}>
                        </div>
                        <div class="settings-section">
                            <label for="enable-captchas-checkbox">CAPTCHA 처리 활성화:
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">CAPTCHA가 감지되었을 때 자동 처리를 활성화합니다. (제한적)</span>
                                </span>
                            </label>
                            <input type="checkbox" id="enable-captchas-checkbox" ${settings.enableCaptchas ? 'checked' : ''}>
                        </div>
                        <div class="settings-section">
                            <label>데이터 내보내기 형식:</label>
                            <div class="checkbox-group">
                                ${Object.keys(settings.exportFormats).map(format => `<label><input type="checkbox" id="export-${format}" ${settings.exportFormats[format] ? 'checked' : ''}> ${format.toUpperCase()}</label>`).join('')}
                            </div>
                        </div>
                        <div class="settings-section">
                            <label for="advanced-logging-checkbox">고급 로깅 활성화:
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">추가적인 로깅 정보를 활성화합니다.</span>
                                </span>
                            </label>
                            <input type="checkbox" id="advanced-logging-checkbox" ${settings.advancedLogging ? 'checked' : ''}>
                        </div>
                        <div class="settings-section">
                            <label for="custom-keywords-input">사용자 지정 키워드 분류:
                                <span class="tooltip">ℹ️
                                    <span class="tooltiptext">사용자 지정 키워드를 추가하여 데이터 분류를 향상시킵니다. 형식: 카테고리:키워드1,키워드2,...</span>
                                </span>
                            </label>
                            <textarea id="custom-keywords-input" rows="5">${formatCustomKeywords(settings.customKeywords)}</textarea>
                        </div>
                        <button id="save-settings" style="margin-top: 10px; padding: 10px 20px; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px;">저장</button>
                        <button id="reset-settings" style="margin-top: 10px; padding: 10px 20px; background-color: #f44336; color: white; border-radius: 4px; font-size: 14px;">초기화</button>
                    </div>
                </div>
                <div id="html-modal" class="modal">
                    <div class="modal-content">
                        <span class="close">&times;</span>
                        <pre id="modal-html-content"></pre>
                    </div>
                </div>
            </div>
        `);
        $('body').append(container);
        $('.crawler-tab').on('click', function() {
            const tab = $(this).data('tab');
            $('.crawler-tab').removeClass('active');
            $(this).addClass('active');
            $('#crawler-content > div').hide();
            $(`#tab-${tab}`).show();
            saveCurrentTab(tab);
        });
        $(document).on('keydown', function(e) {
            if (e.ctrlKey) {
                const actions = { '1': 'logs', '2': 'data', '3': 'settings', 's': 'save', 'p': 'start', 'q': 'stop', 'd': 'csv', 'j': 'json', 'x': 'xml', 'l': 'xls' };
                const action = actions[e.key];
                if (action) {
                    e.preventDefault();
                    switch(action) {
                        case 'logs': $('.crawler-tab[data-tab="logs"]').click(); break;
                        case 'data': $('.crawler-tab[data-tab="data"]').click(); break;
                        case 'settings': $('.crawler-tab[data-tab="settings"]').click(); break;
                        case 'save': saveSettings(); break;
                        case 'start': if (!isCrawling) startCrawling(); break;
                        case 'stop': if (isCrawling) stopCrawling(); break;
                        case 'csv': settings.exportFormats.csv && downloadCSV(); break;
                        case 'json': settings.exportFormats.json && downloadJSON(); break;
                        case 'xml': settings.exportFormats.xml && downloadXML(); break;
                        case 'xls': settings.exportFormats.xls && downloadXLS(); break;
                    }
                }
            }
        });
        $('#save-settings').on('click', saveSettings);
        $('#reset-settings').on('click', resetSettings);
        $('#add-user-agent').on('click', function() {
            const ua = prompt("새 사용자 에이전트를 입력하세요:");
            if (ua) {
                settings.userAgents.push(ua);
                GM_setValue('userAgents', settings.userAgents);
                updateUserAgentSelect();
                log('새 사용자 에이전트가 추가되었습니다.', 'success');
            }
        });
        $('#remove-user-agent').on('click', function() {
            const selected = $('#user-agent-select').val();
            if (selected && selected.length) {
                settings.userAgents = settings.userAgents.filter((_, idx) => !selected.includes(idx.toString()));
                GM_setValue('userAgents', settings.userAgents);
                settings.userAgentIndex = 0;
                GM_setValue('userAgentIndex', settings.userAgentIndex);
                updateUserAgentSelect();
                log('선택한 사용자 에이전트가 제거되었습니다.', 'success');
            } else alert('제거할 사용자 에이전트를 선택하세요.');
        });
        $('#download-csv').on('click', downloadCSV);
        $('#download-json').on('click', downloadJSON);
        $('#download-xml').on('click', downloadXML);
        $('#download-xls').on('click', downloadXLS);
        $('#search-filter').on('input', function() { filterTable($(this).val()); });
        $('#schedule-enabled').on('change', function() { $('#schedule-options').toggle($(this).is(':checked')); });
        $('#data-visualization-checkbox').on('change', function() {
            settings.dataVisualization = $(this).is(':checked');
            GM_setValue('dataVisualization', settings.dataVisualization);
            $('#data-visualization').toggle(settings.dataVisualization);
            log('데이터 시각화 설정이 변경되었습니다.', 'success');
            if (settings.dataVisualization) renderChart();
        });
        $('#enable-captchas-checkbox').on('change', function() {
            settings.enableCaptchas = $(this).is(':checked');
            GM_setValue('enableCaptchas', settings.enableCaptchas);
            log('CAPTCHA 처리 설정이 변경되었습니다.', 'success');
        });
        $('.view-html').on('click', function() {
            const html = decodeURIComponent($(this).data('html'));
            $('#modal-html-content').text(html);
            $('#html-modal').show();
        });
        $('.close').on('click', function() { $('#html-modal').hide(); });
        $(window).on('click', function(e) { if (e.target.id === 'html-modal') $('#html-modal').hide(); });
        $('#start-crawling').on('click', startCrawling);
        $('#stop-crawling').on('click', stopCrawling);
        updateLogsUI();
        updateDataUI();
        if (settings.dataVisualization) renderChart();
    }

    function currentTab() { return GM_getValue('currentTab', 'logs'); }
    function saveCurrentTab(tab) { GM_setValue('currentTab', tab); }
    function updateUserAgentSelect() {
        $('#user-agent-select').empty();
        settings.userAgents.forEach((ua, idx) => {
            $('#user-agent-select').append(`<option value="${idx}" ${idx === settings.userAgentIndex ? 'selected' : ''}>${ua}</option>`);
        });
    }
    function getCustomKeywordsText() {
        const ck = settings.customKeywords || {};
        return Object.entries(ck).map(([cat, kws]) => `${cat}:${kws.join(',')}`).join('\n');
    }
    function parseCustomKeywords(text) {
        const lines = text.split('\n').map(l => l.trim()).filter(l => l);
        const ck = {};
        lines.forEach(line => {
            const [cat, kws] = line.split(':');
            if (cat && kws) ck[cat.trim()] = kws.split(',').map(kw => kw.trim()).filter(kw => kw);
        });
        return ck;
    }
    function combineKeywords() {
        const standard = {
            '교육': ['학교', '교육', '학생', '교수', '강의'],
            '기술': ['기술', '소프트웨어', '하드웨어', '프로그래밍', '개발'],
            '뉴스': ['뉴스', '속보', '기사', '보도'],
            '엔터테인먼트': ['영화', '음악', '연예', '게임', '드라마'],
            '스포츠': ['축구', '야구', '농구', '경기', '토너먼트'],
            '비즈니스': ['비즈니스', '경제', '주식', '투자', '시장'],
            '건강': ['건강', '의료', '헬스', '운동', '영양'],
            '과학': ['과학', '연구', '실험', '발견', '이론'],
            '문학': ['소설', '시', '문학', '작가', '책'],
            '여행': ['여행', '관광', '숙박', '비행기', '여권']
        };
        const custom = settings.customKeywords || {};
        for (const [cat, kws] of Object.entries(custom)) {
            standard[cat] = standard[cat] ? [...new Set([...standard[cat], ...kws])] : kws;
        }
        return standard;
    }
    function log(message, type = 'info') {
        const timestamp = new Date().toLocaleString();
        logs.push({ timestamp, message, type });
        GM_setValue('logs', logs);
        updateLogsUI();
        if (settings.advancedLogging) {
            const styles = { 'info': 'color: blue;', 'warning': 'color: orange;', 'error': 'color: red;', 'success': 'color: green; font-weight: bold;' };
            console.log(`%c[${timestamp}] [${type.toUpperCase()}] ${message}`, styles[type] || '');
        }
    }
    function updateLogsUI() {
        $('#log-messages').html(logs.map(l => `<div>[${l.timestamp}] [${l.type.toUpperCase()}] ${l.message}</div>`).join(''));
        $('.progress-bar-fill').css('width', `${progress.toFixed(2)}%`);
        $('.status-text').text(`진행률: ${progress.toFixed(2)}% (${currentPage}/${totalPages} 페이지)`);
    }
    function updateDataUI() {
        const filter = $('#search-filter').val().toLowerCase();
        const filtered = filter ? data.filter(item => Object.keys(settings.dataFields).some(field => settings.dataFields[field] && String(item[field]).toLowerCase().includes(filter))) : data;
        const rows = filtered.map(item => `
            <tr>
                ${settings.dataFields.name ? `<td>${escapeHTML(item.name)}</td>` : ''}
                ${settings.dataFields.url ? `<td><a href="${escapeHTML(item.url)}" target="_blank">${escapeHTML(item.url)}</a></td>` : ''}
                ${settings.dataFields.description ? `<td>${escapeHTML(item.description)}</td>` : ''}
                ${settings.dataFields.html ? `<td><button class="view-html" data-html="${encodeURIComponent(item.html)}">📄 보기</button></td>` : ''}
                ${settings.dataFields.category ? `<td>${escapeHTML(item.category)}</td>` : ''}
                ${settings.dataFields.quality ? `<td>${escapeHTML(item.quality.toString())}</td>` : ''}
                ${settings.dataFields.language ? `<td>${escapeHTML(item.language)}</td>` : ''}
                ${settings.dataFields.wordCount ? `<td>${escapeHTML(item.wordCount.toString())}</td>` : ''}
                ${settings.dataFields.metaTags ? `<td>${escapeHTML(JSON.stringify(item.metaTags))}</td>` : ''}
                ${settings.dataFields.headings ? `<td>${escapeHTML(item.headings.join(', '))}</td>` : ''}
            </tr>
        `).join('');
        $('#data-table-body').html(rows);
        $('.view-html').off('click').on('click', function() {
            $('#modal-html-content').text(decodeURIComponent($(this).data('html')));
            $('#html-modal').show();
        });
        if (settings.dataVisualization) renderChart();
    }
    function renderChart() {
        if (!document.getElementById('data-chart').getContext) return;
        const ctx = document.getElementById('data-chart').getContext('2d');
        if (window.dataChartInstance) window.dataChartInstance.destroy();
        const categories = {};
        data.forEach(item => { const cat = item.category || '기타'; categories[cat] = (categories[cat] || 0) + 1; });
        const chartData = { labels: Object.keys(categories), datasets: [{ label: '카테고리별 데이터 수', data: Object.values(categories), backgroundColor: generateColors(Object.keys(categories).length) }] };
        window.dataChartInstance = new Chart(ctx, { type: 'pie', data: chartData, options: { responsive: true } });
    }
    function generateColors(count) {
        return Array.from({length: count}, (_, i) => `hsl(${i * (360 / count)}, 70%, 50%)`);
    }
    function escapeHTML(str) {
        return String(str).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#039;");
    }
    function downloadCSV() {
        if (!data.length) { log('다운로드할 데이터가 없습니다.', 'warning'); return; }
        const headers = Object.keys(settings.dataFields).filter(f => settings.dataFields[f]).map(f => getFieldName(f));
        const csv = [headers.join(",")].concat(data.map(item => Object.keys(settings.dataFields).filter(f => settings.dataFields[f]).map(f => `"${escapeCSV(item[f] || '')}"`).join(","))).join("\n");
        const link = document.createElement("a");
        link.href = `data:text/csv;charset=utf-8,${encodeURIComponent(csv)}`;
        link.download = "data.csv";
        document.body.appendChild(link);
        link.click();
        document.body.removeChild(link);
        log('CSV 파일이 다운로드되었습니다.', 'success');
    }
    function downloadJSON() {
        if (!data.length) { log('다운로드할 데이터가 없습니다.', 'warning'); return; }
        const json = JSON.stringify(data, null, 2);
        const blob = new Blob([json], { type: "application/json" });
        const link = document.createElement("a");
        link.href = URL.createObjectURL(blob);
        link.download = "data.json";
        document.body.appendChild(link);
        link.click();
        document.body.removeChild(link);
        log('JSON 파일이 다운로드되었습니다.', 'success');
    }
    function downloadXML() {
        if (!data.length) { log('다운로드할 데이터가 없습니다.', 'warning'); return; }
        let xml = '<?xml version="1.0" encoding="UTF-8"?><data>';
        data.forEach(item => {
            xml += '<entry>';
            if (settings.dataFields.name) xml += `<name>${escapeXML(item.name || 'No Title')}</name>`;
            if (settings.dataFields.url) xml += `<url>${escapeXML(item.url || '')}</url>`;
            if (settings.dataFields.description) xml += `<description>${escapeXML(item.description || 'No Description')}</description>`;
            if (settings.dataFields.html) xml += `<html><![CDATA[${item.html || ''}]]></html>`;
            if (settings.dataFields.category) xml += `<category>${escapeXML(item.category || '')}</category>`;
            if (settings.dataFields.quality) xml += `<quality>${escapeXML(item.quality)}</quality>`;
            if (settings.dataFields.language) xml += `<language>${escapeXML(item.language || 'unknown')}</language>`;
            if (settings.dataFields.wordCount) xml += `<wordCount>${escapeXML(item.wordCount)}</wordCount>`;
            if (settings.dataFields.metaTags) xml += `<metaTags>${escapeXML(JSON.stringify(item.metaTags))}</metaTags>`;
            if (settings.dataFields.headings) xml += `<headings>${escapeXML(item.headings.join('; '))}</headings>`;
            xml += '</entry>';
        });
        xml += '</data>';
        const blob = new Blob([xml], { type: "application/xml" });
        const link = document.createElement("a");
        link.href = URL.createObjectURL(blob);
        link.download = "data.xml";
        document.body.appendChild(link);
        link.click();
        document.body.removeChild(link);
        log('XML 파일이 다운로드되었습니다.', 'success');
    }
    function downloadXLS() {
        if (!data.length) { log('다운로드할 데이터가 없습니다.', 'warning'); return; }
        const sheet = XLSX.utils.json_to_sheet(data.map(item => {
            const entry = {};
            if (settings.dataFields.name) entry['사이트 이름'] = item.name;
            if (settings.dataFields.url) entry['주소'] = item.url;
            if (settings.dataFields.description) entry['설명'] = item.description;
            if (settings.dataFields.html) entry['📄 HTML 내용'] = item.html;
            if (settings.dataFields.category) entry['분류'] = item.category;
            if (settings.dataFields.quality) entry['품질점수'] = item.quality;
            if (settings.dataFields.language) entry['언어'] = item.language;
            if (settings.dataFields.wordCount) entry['단어 수'] = item.wordCount;
            if (settings.dataFields.metaTags) entry['메타 태그'] = JSON.stringify(item.metaTags);
            if (settings.dataFields.headings) entry['헤딩'] = item.headings.join('; ');
            return entry;
        }));
        const wb = XLSX.utils.book_new();
        XLSX.utils.book_append_sheet(wb, sheet, "Data");
        XLSX.writeFile(wb, "data.xlsx");
        log('XLS 파일이 다운로드되었습니다.', 'success');
    }
    function escapeCSV(str) { return String(str).replace(/"/g, '""'); }
    function filterTable(keyword) {
        $('#data-table-body tr').each(function() {
            $(this).toggle($(this).text().toLowerCase().includes(keyword.toLowerCase()));
        });
    }
    function saveSettings() {
        settings.delay = parseInt($('#delay-input').val(), 10) || defaultSettings.delay;
        settings.delayRandom = parseInt($('#delay-random-input').val(), 10) || defaultSettings.delayRandom;
        settings.maxPages = parseInt($('#max-pages-input').val(), 10) || defaultSettings.maxPages;
        const selectedUA = parseInt($('#user-agent-select').val(), 10);
        settings.userAgentIndex = isNaN(selectedUA) ? 0 : selectedUA;
        settings.contentTypes.include = $('#content-include-input').val().split(',').map(ct => ct.trim()).filter(ct => ct) || defaultSettings.contentTypes.include;
        settings.contentTypes.exclude = $('#content-exclude-input').val().split(',').map(ct => ct.trim()).filter(ct => ct) || defaultSettings.contentTypes.exclude;
        settings.timeout = parseInt($('#timeout-input').val(), 10) || defaultSettings.timeout;
        settings.includeKeywords = $('#include-keywords-input').val().split('\n').map(kw => kw.trim()).filter(kw => kw) || defaultSettings.includeKeywords;
        settings.excludeKeywords = $('#exclude-keywords-input').val().split('\n').map(kw => kw.trim()).filter(kw => kw) || defaultSettings.excludeKeywords;
        settings.regexInclude = $('#regex-include-input').val().split('\n').map(rx => rx.trim()).filter(rx => rx) || defaultSettings.regexInclude;
        settings.regexExclude = $('#regex-exclude-input').val().split('\n').map(rx => rx.trim()).filter(rx => rx) || defaultSettings.regexExclude;
        settings.language = $('#language-select').val() || [];
        settings.minWordCount = parseInt($('#min-word-count-input').val(), 10) || defaultSettings.minWordCount;
        settings.maxWordCount = parseInt($('#max-word-count-input').val(), 10) || defaultSettings.maxWordCount;
        settings.maxDepth = parseInt($('#max-depth-input').val(), 10) || defaultSettings.maxDepth;
        settings.proxy = $('#proxy-input').val().trim() || defaultSettings.proxy;
        ['name','url','description','html','category','quality','language','wordCount','metaTags','headings'].forEach(field => {
            settings.dataFields[field] = $(`#field-${field}`).is(':checked');
        });
        settings.retryAttempts = parseInt($('#retry-attempts-input').val(), 10) || defaultSettings.retryAttempts;
        settings.retryDelay = parseInt($('#retry-delay-input').val(), 10) || defaultSettings.retryDelay;
        settings.schedule.enabled = $('#schedule-enabled').is(':checked');
        settings.schedule.startTime = $('#start-time-input').val() || defaultSettings.schedule.startTime;
        settings.schedule.interval = parseInt($('#interval-input').val(), 10) ? parseInt($('#interval-input').val(), 10) * 60000 : defaultSettings.schedule.interval;
        Object.keys(settings.searchEngines).forEach(engine => { settings.searchEngines[engine] = $(`#engine-${engine}`).is(':checked'); });
        settings.blacklist = $('#blacklist-input').val().split('\n').map(d => d.trim()).filter(d => d) || defaultSettings.blacklist;
        settings.whitelist = $('#whitelist-input').val().split('\n').map(d => d.trim()).filter(d => d) || defaultSettings.whitelist;
        settings.respectRobots = $('#respect-robots-checkbox').is(':checked');
        settings.maxConcurrentRequests = parseInt($('#max-concurrent-requests-input').val(), 10) || defaultSettings.maxConcurrentRequests;
        settings.dataVisualization = $('#data-visualization-checkbox').is(':checked');
        settings.exportFormats.csv = $('#export-csv').is(':checked');
        settings.exportFormats.json = $('#export-json').is(':checked');
        settings.exportFormats.xml = $('#export-xml').is(':checked');
        settings.exportFormats.xls = $('#export-xls').is(':checked');
        settings.advancedLogging = $('#advanced-logging-checkbox').is(':checked');
        settings.enableCaptchas = $('#enable-captchas-checkbox').is(':checked');
        const customKeywords = parseCustomKeywords($('#custom-keywords-input').val());
        settings.customKeywords = customKeywords;
        Object.assign(settings, {
            delay, delayRandom, maxPages, userAgentIndex: settings.userAgentIndex,
            contentTypes, timeout, includeKeywords, excludeKeywords, regexInclude, regexExclude, language,
            minWordCount, maxWordCount, maxDepth, proxy, dataFields: settings.dataFields,
            retryAttempts, retryDelay, schedule: settings.schedule, searchEngines: settings.searchEngines,
            blacklist, whitelist, respectRobots, maxConcurrentRequests, dataVisualization,
            exportFormats: settings.exportFormats, advancedLogging, enableCaptchas, customKeywords
        });
        Object.keys(defaultSettings).forEach(key => GM_setValue(key, settings[key]));
        GM_setValue('customKeywords', customKeywords);
        totalPages = settings.maxPages;
        GM_setValue('totalPages', totalPages);
        log('설정이 저장되었습니다.', 'success');
        if (settings.schedule.enabled) scheduleCrawling();
        updateDataUI();
        if (settings.dataVisualization) renderChart();
    }
    function resetSettings() {
        if (confirm("기본 설정으로 초기화하시겠습니까?")) {
            Object.keys(defaultSettings).forEach(key => GM_deleteValue(key));
            Object.assign(settings, defaultSettings);
            ['userAgents','userAgentIndex','contentTypes','dataFields','exportFormats','advancedLogging','enableCaptchas'].forEach(key => GM_setValue(key, settings[key]));
            $('#custom-keywords-input').val('');
            updateUserAgentSelect();
            updateDataUI();
            renderChart();
            log('기본 설정으로 초기화되었습니다.', 'success');
        }
    }
    async function startCrawling() {
        if (isCrawling) { log('이미 크롤링이 진행 중입니다.', 'warning'); return; }
        if (!Object.values(settings.searchEngines).some(v => v)) { log('사용할 검색 엔진이 하나도 선택되지 않았습니다.', 'error'); return; }
        isCrawling = true;
        GM_setValue('isCrawling', isCrawling);
        log('크롤링을 시작합니다.', 'info');
        setButtonStates();
        currentPage = GM_getValue('currentPage', 1);
        totalPages = settings.maxPages;
        GM_setValue('totalPages', totalPages);
        currentDepth = GM_getValue('currentDepth', 0);
        await crawlPage(currentDepth);
    }
    function stopCrawling() {
        if (!isCrawling) { log('크롤링이 진행 중이지 않습니다.', 'warning'); return; }
        isCrawling = false;
        GM_setValue('isCrawling', isCrawling);
        log('크롤링이 중지되었습니다.', 'warning');
        setButtonStates();
    }
    function setButtonStates() {
        $('#start-crawling').prop('disabled', isCrawling);
        $('#stop-crawling').prop('disabled', !isCrawling);
    }
    async function crawlPage(depth) {
        try {
            while (isCrawling && currentPage <= totalPages && depth <= settings.maxDepth) {
                log(`페이지 ${currentPage} 크롤링 시작. (깊이: ${depth})`, 'info');
                if (settings.respectRobots && !(await checkRobotsTxt(window.location.href))) {
                    log(`robots.txt에 의해 크롤링이 금지된 페이지: ${window.location.href}`, 'warning');
                    break;
                }
                const results = extractSearchResults();
                if (!results) { log('검색 결과를 찾을 수 없습니다.', 'error'); break; }
                results.forEach(result => {
                    if (!isCrawling) return;
                    const { name, url, description } = result;
                    if (visitedUrls.includes(url) || !isValidURL(url) ||
                        (settings.whitelist.length && !settings.whitelist.some(d => url.includes(d))) ||
                        settings.blacklist.some(d => url.includes(d)) ||
                        (settings.includeKeywords.length && !settings.includeKeywords.some(k => url.includes(k) || description.includes(k))) ||
                        (settings.excludeKeywords.some(k => url.includes(k) || description.includes(k))) ||
                        (settings.regexInclude.length && !settings.regexInclude.some(r => new RegExp(r).test(url) || new RegExp(r).test(description))) ||
                        (settings.regexExclude.length && settings.regexExclude.some(r => new RegExp(r).test(url) || new RegExp(r).test(description)))) {
                        return;
                    }
                    requestQueue.push({ url, depth });
                    if (activeRequests >= settings.maxConcurrentRequests) return;
                });
                await processQueue();
                progress = (currentPage / totalPages) * 100;
                GM_setValue('progress', progress);
                updateLogsUI();
                if (currentPage < totalPages && isCrawling) {
                    const nextUrl = getNextPageUrl();
                    if (nextUrl) {
                        currentPage++;
                        GM_setValue('currentPage', currentPage);
                        depth++;
                        GM_setValue('currentDepth', depth);
                        log(`다음 페이지로 이동: ${currentPage}`, 'info');
                        await dynamicDelay();
                        window.location.href = nextUrl;
                        return;
                    } else { log('다음 페이지 링크를 찾을 수 없습니다.', 'error'); break; }
                } else { break; }
            }
            if (isCrawling && currentPage > totalPages && activeRequests === 0 && requestQueue.length === 0) {
                log('크롤링이 완료되었습니다.', 'success');
                isCrawling = false;
                GM_setValue('isCrawling', isCrawling);
                setButtonStates();
                GM_setValue('progress', 100);
                $('.progress-bar-fill').css('width', `100%`);
                $('.status-text').text(`진행률: 100% (${currentPage}/${totalPages} 페이지)`);
            }
        } catch (error) {
            log(`크롤링 중 오류 발생: ${error.message}`, 'error');
        }
    }
    async function processQueue() {
        while (requestQueue.length && activeRequests < settings.maxConcurrentRequests && isCrawling) {
            const { url, depth } = requestQueue.shift();
            activeRequests++;
            handleRequest(url, depth).finally(() => { activeRequests--; processQueue(); });
        }
    }
    async function handleRequest(url, depth) {
        if (!isCrawling || visitedUrls.includes(url)) return;
        try {
            const response = await fetchWithRetry(url, settings.retryAttempts, settings.retryDelay);
            const contentType = getContentType(response);
            if (!isContentTypeAllowed(contentType)) { log(`허용되지 않은 콘텐츠 유형(${contentType})의 URL: ${url}`, 'warning'); return; }
            const parser = new DOMParser();
            const doc = parser.parseFromString(response.responseText, 'text/html');
            const text = doc.body ? doc.body.innerText : '';
            if (settings.enableCaptchas && detectCaptcha(doc)) { await handleCaptcha(url); return; }
            const wc = text.split(/\s+/).length;
            if (wc < settings.minWordCount || wc > settings.maxWordCount) {
                log(`단어 수 조건에 맞지 않는 URL: ${url} (${wc} 단어)`, 'warning');
                return;
            }
            const quality = evaluateContentQuality(text);
            const lang = doc.documentElement.lang || 'unknown';
            const category = classifyKeywords(text);
            const meta = extractMetaTags(doc);
            const headings = extractHeadings(doc);
            if (category !== '기타' || quality > 100) {
                const entry = { url, quality, language: lang, wordCount: wc, category };
                if (settings.dataFields.name) entry.name = extractName(doc) || 'No Title';
                if (settings.dataFields.description) entry.description = extractDescription(doc) || 'No Description';
                if (settings.dataFields.html) entry.html = response.responseText;
                if (settings.dataFields.metaTags) entry.metaTags = meta;
                if (settings.dataFields.headings) entry.headings = headings;
                data.push(entry);
                GM_setValue('data', data);
                updateDataUI();
                log(`데이터 수집 완료: ${url}`, 'success');
                if (settings.dataVisualization) renderChart();
            }
            if (depth < settings.maxDepth) {
                const links = extractLinks(doc);
                links.forEach(link => { if (!visitedUrls.includes(link) && isValidURL(link)) requestQueue.push({ url: link, depth: depth + 1 }); });
            }
        } catch (error) {
            log(`데이터 수집 실패: ${url} - ${error.message}`, 'error');
        }
        visitedUrls.push(url);
        GM_setValue('visitedUrls', visitedUrls);
    }
    async function fetchWithRetry(url, attempts, delayMs) {
        for (let i = 1; i <= attempts; i++) {
            try { return await GM_xmlhttpRequestPromise(url); }
            catch (e) { log(`요청 실패 (${i}/${attempts}): ${url} - ${e.message}`, 'warning'); if (i < attempts) await delay(delayMs); }
        }
        throw new Error('All retry attempts failed');
    }
    function isAnySearchEngineEnabled() { return Object.values(settings.searchEngines).some(v => v); }
    function classifyKeywords(text) {
        const kws = combineKeywords();
        for (const [cat, keywords] of Object.entries(kws)) { if (keywords.some(kw => text.includes(kw))) return cat; }
        return '기타';
    }
    function extractSearchResults() {
        let results = [];
        const host = window.location.hostname;
        if (host.includes('google.com') && settings.searchEngines.google) {
            document.querySelectorAll('div.g, div[data-hveid]').forEach(item => {
                const name = item.querySelector('h3')?.innerText.trim() || 'No Title';
                const url = item.querySelector('a')?.href || '';
                const desc = item.querySelector('.aCOpRe, .VwiC3b')?.innerText.trim() || 'No Description';
                if (isValidURL(url)) results.push({ name, url, description: desc });
            });
        }
        else if (host.includes('bing.com') && settings.searchEngines.bing) {
            document.querySelectorAll('.b_algo').forEach(item => {
                const name = item.querySelector('h2')?.innerText.trim() || 'No Title';
                const url = item.querySelector('h2 a')?.href || '';
                const desc = item.querySelector('.b_caption p')?.innerText.trim() || 'No Description';
                if (isValidURL(url)) results.push({ name, url, description: desc });
            });
        }
        // Add similar blocks for other search engines...
        return results.length ? results : null;
    }
    function getNextPageUrl() {
        const host = window.location.hostname;
        let next = null;
        if (host.includes('google.com')) next = document.querySelector('a#pnnext')?.href;
        else if (host.includes('bing.com')) next = document.querySelector('a.sb_pagN')?.href;
        // Add similar blocks for other search engines...
        log(`다음 페이지 URL: ${next || '없음'}`, 'info');
        return next;
    }
    function GM_xmlhttpRequestPromise(url) {
        return new Promise((resolve, reject) => {
            let reqUrl = settings.proxy ? `${settings.proxy}?url=${encodeURIComponent(url)}` : url;
            GM_xmlhttpRequest({
                method: "GET",
                url: reqUrl,
                headers: { 'User-Agent': getCurrentUserAgent(), 'Accept': '*/*' },
                timeout: settings.timeout,
                onload: res => (res.status >= 200 && res.status < 300) ? resolve(res) : reject(new Error(`HTTP ${res.status}`)),
                onerror: () => reject(new Error('Request failed')),
                ontimeout: () => reject(new Error('Request timed out'))
            });
        });
    }
    function getCurrentUserAgent() { return settings.userAgents[settings.userAgentIndex % settings.userAgents.length]; }
    function getContentType(res) { return res.responseHeaders.split('\n').find(h => h.toLowerCase().startsWith('content-type'))?.split(':')[1].trim().split(';')[0] || ''; }
    function isContentTypeAllowed(ct) {
        if (settings.contentTypes.include.length && !settings.contentTypes.include.includes(ct)) return false;
        if (settings.contentTypes.exclude.includes(ct)) return false;
        return true;
    }
    function evaluateContentQuality(text) { return text.split(/\s+/).length; }
    function extractMetaTags(doc) {
        const metas = {};
        doc.querySelectorAll('meta[name], meta[property]').forEach(meta => {
            const name = meta.getAttribute('name') || meta.getAttribute('property');
            const content = meta.getAttribute('content');
            if (name && content) metas[name.toLowerCase()] = content;
        });
        return metas;
    }
    function extractHeadings(doc) {
        const hs = [];
        for (let i = 1; i <= 6; i++) doc.querySelectorAll(`h${i}`).forEach(h => hs.push(h.innerText.trim()));
        return hs;
    }
    function isValidURL(url) { try { new URL(url); return true; } catch { return false; } }
    async function checkRobotsTxt(url) {
        try {
            const parsed = new URL(url);
            const robots = `${parsed.origin}/robots.txt`;
            const res = await GM_xmlhttpRequestPromise(robots);
            return new RobotsParser(robots, res.responseText).isAllowed(url, getCurrentUserAgent());
        } catch { log('robots.txt 확인 실패, 기본 허용.', 'warning'); return true; }
    }
    class RobotsParser {
        constructor(url, text) { this.rules = []; this.parse(text); }
        parse(text) {
            let agents = [];
            text.split('\n').forEach(line => {
                const [key, ...val] = line.split(':').map(s => s.trim());
                if (!key) return;
                if (key.toLowerCase() === 'user-agent') { agents = [val.join(':').toLowerCase()]; }
                else if (['disallow','allow'].includes(key.toLowerCase())) this.rules.push({ agents, type: key.toLowerCase(), path: val.join(':') });
            });
        }
        isAllowed(url, ua) {
            const path = new URL(url).pathname;
            let allowed = true;
            this.rules.forEach(rule => {
                if (rule.agents.includes('*') || rule.agents.includes(ua.toLowerCase())) {
                    if (rule.type === 'disallow' && path.startsWith(rule.path)) allowed = false;
                    if (rule.type === 'allow' && path.startsWith(rule.path)) allowed = true;
                }
            });
            return allowed;
        }
    }
    function delay(ms) { return new Promise(res => setTimeout(res, ms)); }
    function formatCustomKeywords(ck) { return Object.entries(ck).map(([c, kws]) => `${c}:${kws.join(',')}`).join('\n'); }
    function getFieldName(field) { return { name: '사이트 이름', url: '주소', description: '설명', html: '📄 HTML 내용', category: '분류', quality: '품질점수', language: '언어', wordCount: '단어 수', metaTags: '메타 태그', headings: '헤딩' }[field]; }
    function capitalize(str) { return str.charAt(0).toUpperCase() + str.slice(1); }
    async function handleCaptcha(url) { log(`CAPTCHA 감지됨: ${url}`, 'warning'); /* CAPTCHA 처리 로직 추가 */ }
    function extractName(doc) { return doc.querySelector('title')?.innerText || 'No Title'; }
    function extractDescription(doc) { return doc.querySelector('meta[name="description"]')?.getAttribute('content') || 'No Description'; }
    function extractLinks(doc) { return Array.from(doc.querySelectorAll('a[href]')).map(a => a.href).filter(isValidURL); }
    function escapeXML(str) { return String(str).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#039;"); }

    function init() {
        createUI();
        if (isCrawling) { log('크롤링 재개됨.', 'info'); setButtonStates(); crawlPage(currentDepth); }
        else log('크롤러 초기화됨.', 'info');
    }
    window.addEventListener('load', init);
})();                                                                                                                             참 좀 작게나 내가 조절 할 수 있게 해줘 ㅇㅇ