// ==UserScript==
// @name         종합 웹 크롤러 7.1
// @namespace    http://tampermonkey.net/
// @version      7.1
// @description  고급 기능을 갖춘 웹 크롤러로 검색 결과를 실시간으로 수집하고 표시 (개선된 버전)
// @author       YourName
// @match        *://www.google.com/search*
// @match        *://www.bing.com/search*
// @match        *://search.daum.net/search*
// @match        *://search.naver.com/search*
// @match        *://search.yahoo.com/search*
// @grant        GM_xmlhttpRequest
// @grant        GM_addStyle
// @grant        GM_setValue
// @grant        GM_getValue
// @grant        GM_notification
// @connect      *
// @require      https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.18.5/xlsx.full.min.js
// @require      https://cdn.jsdelivr.net/npm/chart.js
// ==/UserScript==

(function() {
    'use strict';

    // CSS 스타일 추가
    GM_addStyle(`
        #crawler-container {
            position: fixed;
            top: 10px;
            right: 10px;
            background-color: #f9f9f9;
            border: 1px solid #4CAF50;
            padding: 15px;
            z-index: 10000;
            width: 600px; /* 창 너비 축소 */
            max-height: 80vh; /* 최대 높이 조정 */
            overflow: hidden;
            box-shadow: 0 4px 15px rgba(0,0,0,0.2);
            border-radius: 8px;
            font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
            font-size: 14px;
            resize: both;
            display: flex;
            flex-direction: column;
        }
        #crawler-container header {
            cursor: move;
            background-color: #4CAF50;
            color: white;
            padding: 8px 10px;
            border-radius: 6px 6px 0 0;
            text-align: center;
            font-size: 18px;
            font-weight: bold;
            display: flex;
            justify-content: space-between;
            align-items: center;
        }
        #crawler-container header .title {
            flex-grow: 1;
        }
        #crawler-container header .close-btn {
            cursor: pointer;
            font-size: 20px;
            color: #ff4d4d;
            margin-left: 10px;
        }
        #crawler-container .tabs {
            display: flex;
            border-bottom: 1px solid #ccc;
            margin-top: 10px;
        }
        #crawler-container .tab {
            flex: 1;
            padding: 8px;
            text-align: center;
            cursor: pointer;
            background-color: #e0e0e0;
            border-right: 1px solid #ccc;
            font-weight: bold;
            transition: background-color 0.3s;
        }
        #crawler-container .tab:last-child {
            border-right: none;
        }
        #crawler-container .tab.active {
            background-color: #ffffff;
            border-bottom: 2px solid #4CAF50;
        }
        #crawler-container .tab-content {
            display: none;
            padding: 10px;
            overflow-y: auto;
            flex-grow: 1;
        }
        #crawler-container .tab-content.active {
            display: block;
        }
        /* 버튼 스타일 */
        #crawler-container button {
            padding: 6px 10px;
            margin: 5px 2px;
            border: none;
            border-radius: 4px;
            background-color: #4CAF50;
            color: white;
            cursor: pointer;
            transition: background-color 0.3s;
            font-size: 13px;
        }
        #crawler-container button:hover {
            background-color: #45a049;
        }
        /* 설정 섹션 스타일 */
        #crawler-container .options,
        #crawler-container .filters,
        #crawler-container .exclude-domains,
        #crawler-container .advanced-options,
        #crawler-container .controls {
            margin-bottom: 10px;
        }
        #crawler-container .options label,
        #crawler-container .advanced-options label,
        #crawler-container .controls label {
            margin-right: 10px;
            display: inline-flex;
            align-items: center;
            font-weight: normal;
        }
        #crawler-container .options input[type="checkbox"],
        #crawler-container .advanced-options input[type="checkbox"],
        #crawler-container .controls input[type="number"] {
            margin-right: 5px;
        }
        #crawler-container input[type="text"],
        #crawler-container textarea,
        #crawler-container input[type="number"],
        #crawler-container select {
            width: 100%;
            padding: 6px;
            margin: 5px 0 8px 0;
            border: 1px solid #ccc;
            border-radius: 4px;
            font-size: 13px;
        }
        #crawler-container table {
            border-collapse: collapse;
            width: 100%;
            margin-top: 10px;
            font-size: 12px;
        }
        #crawler-container th, #crawler-container td {
            padding: 8px;
            border: 1px solid #ddd;
            text-align: left;
            vertical-align: top;
        }
        #crawler-container th {
            background-color: #4CAF50;
            color: white;
            position: sticky;
            top: 0;
            z-index: 1;
        }
        #crawler-container .log {
            max-height: 150px;
            overflow-y: auto;
            background-color: #eef;
            padding: 8px;
            border: 1px solid #ccc;
            border-radius: 4px;
            font-size: 12px;
            white-space: pre-wrap;
        }
        #crawler-container .controls,
        #crawler-container .options,
        #crawler-container .advanced-options,
        #crawler-container .notification-area {
            display: flex;
            flex-wrap: wrap;
            align-items: center;
            justify-content: space-between;
        }
        #crawler-container .filters label,
        #crawler-container .exclude-domains label {
            display: block;
            margin-bottom: 5px;
            font-weight: bold;
        }
        #crawler-container progress {
            width: 70%;
            height: 15px;
            margin-right: 5px;
        }
        #crawler-container .status {
            margin-top: 8px;
            font-weight: bold;
            font-size: 13px;
        }
        #crawler-container .export-btn {
            background-color: #2196F3;
        }
        #crawler-container .export-btn:hover {
            background-color: #0b7dda;
        }
        #crawler-container .pause-btn {
            background-color: #FF9800;
        }
        #crawler-container .pause-btn:hover {
            background-color: #e68a00;
        }
        #crawler-container .chart-container {
            position: relative;
            height: 200px;
            width: 100%;
            margin-top: 15px;
        }
        #crawler-container .notification-area {
            flex-direction: row;
            align-items: center;
        }
        /* 반응형 디자인 */
        @media (max-width: 650px) {
            #crawler-container {
                width: 95%;
                right: 2.5%;
            }
        }
    `);

    // UI 생성
    const container = document.createElement('div');
    container.id = 'crawler-container';
    container.innerHTML = `
        <header>
            <span class="title">🔥 웹 크롤러 7.1</span>
            <span class="close-btn" title="닫기">&times;</span>
        </header>
        <div class="tabs">
            <div class="tab active" data-tab="settings">설정</div>
            <div class="tab" data-tab="results">결과</div>
            <div class="tab" data-tab="logs">로그</div>
        </div>
        <div class="tab-content active" id="settings">
            <div class="notification-area">
                <button id="startBtn" title="크롤링 시작"><i>▶️</i> 시작</button>
                <button id="pauseBtn" class="pause-btn" title="크롤링 일시정지/재개"><i>⏸️</i> 일시정지</button>
                <button id="stopBtn" title="크롤링 중지"><i>⏹️</i> 중지</button>
                <button id="resetBtn" title="크롤링 초기화"><i>🔄</i> 초기화</button>
                <button id="exportBtn" class="export-btn" title="결과 내보내기"><i>💾</i> 내보내기</button>
            </div>
            <div class="options">
                <label><input type="checkbox" id="toggleHTML" checked> HTML 가져오기</label>
                <label><input type="checkbox" id="toggleImages" checked> 이미지 가져오기</label>
                <label><input type="checkbox" id="toggleAds" checked> 광고 제외</label>
            </div>
            <div class="filters">
                <label for="filterInput">필터 단어 (정규식, 쉼표로 구분):</label>
                <input type="text" id="filterInput" placeholder="예: 기부, 광고, /promo/i">
            </div>
            <div class="exclude-domains">
                <label for="excludeDomainsInput">제외할 도메인 (쉼표로 구분):</label>
                <input type="text" id="excludeDomainsInput" placeholder="예: facebook.com, youtube.com">
            </div>
            <div class="advanced-options">
                <label for="duplicateThreshold">중복 허용 횟수:</label>
                <input type="number" id="duplicateThreshold" value="2" min="1" style="width: 60px;">
            </div>
            <div class="controls">
                <label for="retryAttempts">재시도 횟수:</label>
                <input type="number" id="retryAttempts" value="3" min="1" style="width: 60px;">
                <label for="retryInterval">재시도 간격 (ms):</label>
                <input type="number" id="retryInterval" value="500" min="100" style="width: 80px;">
            </div>
            <div style="margin-top:10px;">
                <label for="progressBar">진행률:</label>
                <progress id="progressBar" value="0" max="100"></progress>
                <span id="progressText">0%</span>
            </div>
            <div class="status">
                현재 페이지: <span id="currentPage">0</span> /
                총 페이지: <span id="totalPages">0</span>
                <br>
                방문된 사이트: <span id="visitedCount">0</span>
                <br>
                경과 시간: <span id="elapsedTime">00:00:00</span>
                <br>
                예상 남은 시간: <span id="remainingTime">00:00:00</span>
            </div>
            <div class="chart-container">
                <canvas id="crawlChart"></canvas>
            </div>
        </div>
        <div class="tab-content" id="results">
            <div id="results" style="margin-top:10px; max-height: 280px; overflow: auto;">
                <table id="resultsTable">
                    <thead>
                        <tr>
                            <th>사이트 이름</th>
                            <th>주소</th>
                            <th>설명</th>
                            <th>이미지</th>
                            <th>HTML 링크</th>
                            <th>WHOIS 정보</th>
                            <th>페이지 로딩 시간 (ms)</th>
                        </tr>
                    </thead>
                    <tbody id="resultsBody">
                    </tbody>
                </table>
            </div>
        </div>
        <div class="tab-content" id="logs">
            <div class="log" id="log">
                <strong>로그:</strong>
            </div>
        </div>
    `;
    document.body.appendChild(container);

    // 드래그 가능하게 설정
    (function makeDraggable(elmnt) {
        const header = elmnt.querySelector('header');
        const closeBtn = elmnt.querySelector('.close-btn');
        let pos1 = 0, pos2 = 0, pos3 = 0, pos4 = 0;
        header.onmousedown = dragMouseDown;

        closeBtn.onclick = () => {
            elmnt.style.display = 'none';
        }

        function dragMouseDown(e) {
            e = e || window.event;
            e.preventDefault();
            pos3 = e.clientX;
            pos4 = e.clientY;
            document.onmouseup = closeDragElement;
            document.onmousemove = elementDrag;
        }

        function elementDrag(e) {
            e = e || window.event;
            e.preventDefault();
            pos1 = pos3 - e.clientX;
            pos2 = pos4 - e.clientY;
            pos3 = e.clientX;
            pos4 = e.clientY;
            elmnt.style.top = (elmnt.offsetTop - pos2) + "px";
            elmnt.style.left = (elmnt.offsetLeft - pos1) + "px";
        }

        function closeDragElement() {
            document.onmouseup = null;
            document.onmousemove = null;
        }
    })(container);

    // 탭 기능 구현
    (function setupTabs() {
        const tabs = container.querySelectorAll('.tab');
        const tabContents = container.querySelectorAll('.tab-content');

        tabs.forEach(tab => {
            tab.addEventListener('click', () => {
                tabs.forEach(t => t.classList.remove('active'));
                tab.classList.add('active');

                const target = tab.getAttribute('data-tab');
                tabContents.forEach(content => {
                    if(content.id === target){
                        content.classList.add('active');
                    } else {
                        content.classList.remove('active');
                    }
                });
            });
        });
    })();

    // 변수 초기화
    let isRunning = false;
    let isPaused = false;
    let currentPage = 1;
    let totalPages = 0;
    let results = [];
    let searchQuery = '';
    let searchEngine = '';
    let visitedLinks = new Map(); // 도메인별 방문 횟수 추적
    let logMessages = [];
    let filterPatterns = [];
    let excludeDomains = [];
    let duplicateThreshold = 2; // 중복 허용 횟수
    let retryAttempts = 3; // 재시도 횟수
    let retryInterval = 500; // 재시도 간격 (ms)
    let startTime = null;
    let elapsedInterval = null;
    let crawlChart = null;
    let crawlData = {
        labels: [],  // 도메인 이름
        datasets: [{
            label: '방문 횟수',
            data: [],
            backgroundColor: 'rgba(75, 192, 192, 0.6)',
            borderColor: 'rgba(75, 192, 192, 1)',
            borderWidth: 1
        }]
    };

    // 버튼 및 요소 가져오기
    const startBtn = document.getElementById('startBtn');
    const pauseBtn = document.getElementById('pauseBtn');
    const stopBtn = document.getElementById('stopBtn');
    const resetBtn = document.getElementById('resetBtn');
    const exportBtn = document.getElementById('exportBtn');
    const progressBar = document.getElementById('progressBar');
    const progressText = document.getElementById('progressText');
    const resultsBody = document.getElementById('resultsBody');
    const logContainer = document.getElementById('log');
    const currentPageElem = document.getElementById('currentPage');
    const totalPagesElem = document.getElementById('totalPages');
    const toggleHTML = document.getElementById('toggleHTML');
    const toggleImages = document.getElementById('toggleImages');
    const toggleAds = document.getElementById('toggleAds');
    const filterInput = document.getElementById('filterInput');
    const excludeDomainsInput = document.getElementById('excludeDomainsInput');
    const duplicateThresholdInput = document.getElementById('duplicateThreshold');
    const retryAttemptsInput = document.getElementById('retryAttempts');
    const retryIntervalInput = document.getElementById('retryInterval');
    const visitedCountElem = document.getElementById('visitedCount');
    const elapsedTimeElem = document.getElementById('elapsedTime');
    const remainingTimeElem = document.getElementById('remainingTime');
    const crawlChartCtx = document.getElementById('crawlChart').getContext('2d');

    // 이벤트 리스너
    startBtn.addEventListener('click', startCrawling);
    pauseBtn.addEventListener('click', togglePause);
    stopBtn.addEventListener('click', stopCrawling);
    resetBtn.addEventListener('click', resetCrawling);
    exportBtn.addEventListener('click', exportResults);
    filterInput.addEventListener('input', updateFilterPatterns);
    excludeDomainsInput.addEventListener('input', updateExcludeDomains);
    duplicateThresholdInput.addEventListener('change', updateDuplicateThreshold);
    retryAttemptsInput.addEventListener('change', updateRetryAttempts);
    retryIntervalInput.addEventListener('change', updateRetryInterval);

    // 페이지 새로 고침 시 크롤링 초기화
    window.addEventListener('beforeunload', () => {
        resetCrawling();
    });

    // 초기화 로드
    loadLogs();
    loadCrawlData();

    // 차트 초기화
    function initializeChart() {
        crawlChart = new Chart(crawlChartCtx, {
            type: 'bar',
            data: crawlData,
            options: {
                responsive: true,
                maintainAspectRatio: false,
                scales: {
                    y: { beginAtZero: true }
                },
                plugins: {
                    legend: {
                        display: false
                    },
                    tooltip: {
                        enabled: true
                    }
                }
            }
        });
    }
    initializeChart();

    function log(message) {
        const time = new Date().toLocaleTimeString();
        const entry = `[${time}] ${message}`;
        logMessages.push(entry);
        logContainer.innerHTML += `${entry}<br>`;
        logContainer.scrollTop = logContainer.scrollHeight;
        // 저장
        GM_setValue('crawlerLogs', JSON.stringify(logMessages));
    }

    function loadLogs() {
        const storedLogs = GM_getValue('crawlerLogs', []);
        if (storedLogs.length > 0) {
            logMessages = JSON.parse(storedLogs);
            logContainer.innerHTML = `<strong>로그:</strong><br>` + logMessages.join('<br>');
            logContainer.scrollTop = logContainer.scrollHeight;
        }
    }

    function updateFilterPatterns() {
        const input = filterInput.value.trim();
        filterPatterns = input ? input.split(',').map(word => {
            word = word.trim();
            if (word.startsWith('/') && word.endsWith('/')) {
                try {
                    return new RegExp(word.slice(1, -1), 'i');
                } catch (e) {
                    log(`⚠️ 유효하지 않은 정규식: ${word}`);
                    return null;
                }
            } else {
                return new RegExp(word, 'i');
            }
        }).filter(regex => regex !== null) : [];
        log(`🛠 필터 패턴 업데이트: [${filterPatterns.map(r => r.toString()).join(', ')}]`);
    }

    function updateExcludeDomains() {
        const input = excludeDomainsInput.value.trim();
        excludeDomains = input ? input.split(',').map(domain => domain.trim().toLowerCase()) : [];
        log(`🚫 제외 도메인 업데이트: [${excludeDomains.join(', ')}]`);
    }

    function updateDuplicateThreshold() {
        const value = parseInt(duplicateThresholdInput.value);
        duplicateThreshold = isNaN(value) || value < 1 ? 2 : value;
        log(`🔄 중복 허용 횟수 설정: ${duplicateThreshold}`);
    }

    function updateRetryAttempts() {
        const value = parseInt(retryAttemptsInput.value);
        retryAttempts = isNaN(value) || value < 1 ? 3 : value;
        log(`🔄 재시도 횟수 설정: ${retryAttempts}`);
    }

    function updateRetryInterval() {
        const value = parseInt(retryIntervalInput.value);
        retryInterval = isNaN(value) || value < 100 ? 500 : value;
        log(`🔄 재시도 간격 설정: ${retryInterval}ms`);
    }

    function getSearchQueryAndEngine() {
        const url = window.location.href;
        if (url.includes('google.com/search')) {
            const params = new URLSearchParams(window.location.search);
            searchQuery = params.get('q') || '';
            searchEngine = 'google';
        } else if (url.includes('bing.com/search')) {
            const params = new URLSearchParams(window.location.search);
            searchQuery = params.get('q') || '';
            searchEngine = 'bing';
        } else if (url.includes('daum.net/search')) {
            const params = new URLSearchParams(window.location.search);
            searchQuery = params.get('q') || '';
            searchEngine = 'daum';
        } else if (url.includes('search.naver.com/search')) {
            const params = new URLSearchParams(window.location.search);
            searchQuery = params.get('query') || '';
            searchEngine = 'naver';
        } else if (url.includes('search.yahoo.com/search')) {
            const params = new URLSearchParams(window.location.search);
            searchQuery = params.get('p') || '';
            searchEngine = 'yahoo';
        } else {
            searchQuery = '';
            searchEngine = '';
        }
        log(`🔍 검색 엔진: ${searchEngine}, 검색어: "${searchQuery}"`);
    }

    async function startCrawling() {
        if (isRunning) {
            alert('이미 크롤링이 진행 중입니다.');
            return;
        }
        getSearchQueryAndEngine();
        if (!searchQuery) {
            alert('현재 페이지에서 검색어를 찾을 수 없습니다.');
            log('⚠️ 검색어 미입력 또는 인식 실패.');
            return;
        }
        isRunning = true;
        isPaused = false;
        currentPage = 1;
        totalPages = 0;
        results = [];
        visitedLinks.clear();
        resultsBody.innerHTML = '';
        visitedCountElem.innerText = '0';
        elapsedTimeElem.innerText = '00:00:00';
        remainingTimeElem.innerText = '00:00:00';
        logMessages = [];
        GM_setValue('crawlerLogs', JSON.stringify(logMessages));
        log('✅ 크롤링을 시작합니다.');
        updateDuplicateThreshold();
        updateRetryAttempts();
        updateRetryInterval();
        await crawlPage();
        startTime = Date.now();
        elapsedInterval = setInterval(updateElapsedTime, 1000);
    }

    function togglePause() {
        if (!isRunning) {
            alert('일시정지/재개할 크롤링이 없습니다.');
            return;
        }
        isPaused = !isPaused;
        if (isPaused) {
            log('⏸️ 크롤링을 일시정지하였습니다.');
            pauseBtn.innerText = '▶️ 재개';
        } else {
            log('▶️ 크롤링을 재개하였습니다.');
            pauseBtn.innerText = '⏸️ 일시정지';
            crawlPage();
        }
    }

    function stopCrawling() {
        if (!isRunning) {
            alert('중지할 크롤링이 없습니다.');
            return;
        }
        isRunning = false;
        isPaused = false;
        pauseBtn.innerText = '⏸️ 일시정지';
        clearInterval(elapsedInterval);
        updateProgress();
        log('🚫 크롤링을 중지하였습니다.');
        GM_notification({
            text: '크롤링이 중지되었습니다.',
            title: '웹 크롤러 7.1',
            timeout: 3000
        });
    }

    async function crawlPage() {
        if (!isRunning || isPaused) {
            return;
        }

        // 검색 엔진에 따른 URL 구성
        let url = '';
        if (searchEngine === 'google') {
            url = `https://www.google.com/search?q=${encodeURIComponent(searchQuery)}&start=${(currentPage-1)*10}`;
        } else if (searchEngine === 'bing') {
            url = `https://www.bing.com/search?q=${encodeURIComponent(searchQuery)}&first=${(currentPage-1)*10 +1}`;
        } else if (searchEngine === 'daum') {
            url = `https://search.daum.net/search?q=${encodeURIComponent(searchQuery)}&page=${currentPage}`;
        } else if (searchEngine === 'naver') {
            url = `https://search.naver.com/search.naver?query=${encodeURIComponent(searchQuery)}&start=${(currentPage-1)*10 +1}`;
        } else if (searchEngine === 'yahoo') {
            url = `https://search.yahoo.com/search?p=${encodeURIComponent(searchQuery)}&b=${(currentPage-1)*10 +1}`;
        } else {
            alert('지원하지 않는 검색 엔진입니다.');
            log('❌ 지원하지 않는 검색 엔진.');
            isRunning = false;
            return;
        }

        log(`📄 페이지 ${currentPage} 크롤링 중: ${url}`);

        GM_xmlhttpRequest({
            method: "GET",
            url: url,
            headers: {
                "User-Agent": navigator.userAgent,
                "Accept": "text/html"
            },
            onload: function(response) {
                if (response.status !== 200) {
                    log(`❌ 페이지 ${currentPage} 로드 실패: 상태 ${response.status}`);
                    handleCrawlError(currentPage, retryAttempts);
                    return;
                }

                const parser = new DOMParser();
                const doc = parser.parseFromString(response.responseText, 'text/html');
                let items = [];
                if (searchEngine === 'google') {
                    items = doc.querySelectorAll('div.g');
                    if (currentPage === 1) {
                        const pager = doc.querySelectorAll('a[href*="start="]');
                        totalPages = pager.length || 1;
                        totalPagesElem.innerText = totalPages;
                    }
                } else if (searchEngine === 'bing') {
                    items = doc.querySelectorAll('li.b_algo');
                    if (currentPage === 1) {
                        const pager = doc.querySelectorAll('a.sb_pagS');
                        totalPages = pager.length || 1;
                        totalPagesElem.innerText = totalPages;
                    }
                } else if (searchEngine === 'daum') {
                    items = doc.querySelectorAll('div.wrap_cont');
                    if (currentPage === 1) {
                        const pager = doc.querySelectorAll('a.pagination__link');
                        totalPages = pager.length || 1;
                        totalPagesElem.innerText = totalPages;
                    }
                } else if (searchEngine === 'naver') {
                    items = doc.querySelectorAll('div.basic1');
                    if (currentPage === 1) {
                        const pager = doc.querySelectorAll('a.nav');
                        totalPages = pager.length || 1;
                        totalPagesElem.innerText = totalPages;
                    }
                } else if (searchEngine === 'yahoo') {
                    items = doc.querySelectorAll('div.compTitle');
                    if (currentPage === 1) {
                        const pager = doc.querySelectorAll('a.fz-l.tco-pagination__anchor');
                        totalPages = pager.length || 1;
                        totalPagesElem.innerText = totalPages;
                    }
                }

                log(`📊 페이지 ${currentPage}에서 ${items.length}개 항목 발견.`);

                items.forEach(item => {
                    let title = 'N/A';
                    let link = 'N/A';
                    let description = 'N/A';
                    let image = 'N/A';
                    let htmlLink = 'N/A';
                    let whoisInfo = 'N/A';
                    let pageLoadTime = 'N/A';

                    if (searchEngine === 'google') {
                        const titleElem = item.querySelector('h3');
                        const linkElem = item.querySelector('a');
                        const descElem = item.querySelector('.IsZvec') || item.querySelector('.VwiC3b');
                        if (titleElem) title = titleElem.innerText;
                        if (linkElem) link = linkElem.href;
                        if (descElem) description = descElem.innerText;

                        // 이미지 가져오기 (가능한 경우)
                        const imgElem = item.querySelector('img');
                        if (imgElem && imgElem.src) {
                            image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
                        }

                        // HTML 보기 링크
                        if (toggleHTML.checked && link !== 'N/A') {
                            htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`;
                        }
                    } else if (searchEngine === 'bing') {
                        const titleElem = item.querySelector('h2 a');
                        const descElem = item.querySelector('p');
                        if (titleElem) {
                            title = titleElem.innerText;
                            link = titleElem.href;
                        }
                        if (descElem) description = descElem.innerText;

                        // 이미지 가져오기 (가능한 경우)
                        const imgElem = item.querySelector('img');
                        if (imgElem && imgElem.src) {
                            image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
                        }

                        // HTML 보기 링크
                        if (toggleHTML.checked && link !== 'N/A') {
                            htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`;
                        }
                    } else if (searchEngine === 'daum') {
                        const titleElem = item.querySelector('a.link_fax');
                        const descElem = item.querySelector('div.cont_inner');
                        if (titleElem) {
                            title = titleElem.innerText.trim();
                            link = titleElem.href;
                        }
                        if (descElem) description = descElem.innerText.trim();

                        // 이미지 가져오기 (가능한 경우)
                        const imgElem = item.querySelector('img');
                        if (imgElem && imgElem.src) {
                            image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
                        }

                        // HTML 보기 링크
                        if (toggleHTML.checked && link !== 'N/A') {
                            htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`;
                        }
                    } else if (searchEngine === 'naver') {
                        const titleElem = item.querySelector('a.tit');
                        const descElem = item.querySelector('div.desc');
                        if (titleElem) {
                            title = titleElem.innerText;
                            link = titleElem.href;
                        }
                        if (descElem) description = descElem.innerText;

                        // 이미지 가져오기 (가능한 경우)
                        const imgElem = item.querySelector('img.thumb');
                        if (imgElem && imgElem.src) {
                            image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
                        }

                        // HTML 보기 링크
                        if (toggleHTML.checked && link !== 'N/A') {
                            htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`;
                        }
                    } else if (searchEngine === 'yahoo') {
                        const titleElem = item.querySelector('h3');
                        const linkElem = item.querySelector('a');
                        const descElem = item.querySelector('div.compText');
                        if (titleElem) title = titleElem.innerText;
                        if (linkElem) link = linkElem.href;
                        if (descElem) description = descElem.innerText;

                        // 이미지 가져오기 (가능한 경우)
                        const imgElem = item.querySelector('img');
                        if (imgElem && imgElem.src) {
                            image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
                        }

                        // HTML 보기 링크
                        if (toggleHTML.checked && link !== 'N/A') {
                            htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`;
                        }
                    }

                    // 광고 제외 기능 적용
                    if (toggleAds.checked) {
                        const combinedText = `${title} ${description}`.toLowerCase();
                        const isAd = combinedText.includes('advertisement') || combinedText.includes('광고');
                        if (isAd) {
                            log(`🚫 광고 항목 제외: "${title}"`);
                            return;
                        }
                    }

                    // 필터 단어 적용
                    if (filterPatterns.length > 0) {
                        const combinedText = `${title} ${description}`.toLowerCase();
                        const isFiltered = filterPatterns.some(regex => regex.test(combinedText));
                        if (isFiltered) {
                            log(`❌ 필터 단어 포함 항목 제외: "${title}"`);
                            return;
                        }
                    }

                    // 도메인 추출
                    const domain = extractDomain(link);
                    if (!domain) return;

                    // 제외 도메인 검사
                    if (excludeDomains.includes(domain.toLowerCase())) {
                        log(`🚫 제외 도메인 항목 제외: "${link}"`);
                        return;
                    }

                    // 중복 검사 (도메인별 방문 횟수)
                    let visitCount = visitedLinks.get(domain) || 0;
                    if (visitCount >= duplicateThreshold) {
                        log(`🔄 중복 방문 제한: "${domain}"`);
                        return;
                    }

                    // 방문 횟수 업데이트
                    visitedLinks.set(domain, visitCount + 1);
                    visitedCountElem.innerText = visitedLinks.size.toString();

                    // 페이지 로딩 시간 측정 (추가 기능)
                    const startLoad = performance.now();
                    fetch(link, { mode: 'no-cors' })
                        .then(response => {
                            const endLoad = performance.now();
                            pageLoadTime = Math.round(endLoad - startLoad);
                            return Promise.resolve();
                        })
                        .catch(err => {
                            pageLoadTime = '계산 불가';
                        });

                    // WHOIS 정보 가져오기 (추가 기능: 외부 API 필요)
                    whoisInfo = 'N/A'; // 기본값
                    if (toggleHTML.checked && link !== 'N/A') {
                        // 외부 API를 통해 WHOIS 정보 가져오기 (예: jsonwhois.com 사용)
                        // 여기서는 예시로 간단히 처리
                        // 실제로는 WHOIS API 키와 요청이 필요
                        whoisInfo = `<a href="https://jsonwhois.com/whois/${encodeURIComponent(domain)}" target="_blank">보기</a>`;
                    }

                    // 이미지 옵션
                    if (!toggleImages.checked) {
                        image = '비활성화';
                    }

                    results.push({ title, link, description, image, htmlLink, whoisInfo, pageLoadTime });

                    // 결과 테이블에 추가
                    const row = document.createElement('tr');
                    row.innerHTML = `
                        <td>${escapeHTML(title)}</td>
                        <td><a href="${escapeHTML(link)}" target="_blank">${escapeHTML(link)}</a></td>
                        <td>${escapeHTML(description)}</td>
                        <td>${image}</td>
                        <td>${htmlLink}</td>
                        <td>${whoisInfo}</td>
                        <td>${pageLoadTime}</td>
                    `;
                    resultsBody.appendChild(row);
                });

                // 데이터 시각화 업데이트
                updateChart();

                // 진행률 업데이트
                updateProgress();

                // 다음 페이지로 이동
                currentPage++;
                currentPageElem.innerText = currentPage;

                // 딜레이 100ms 추가
                setTimeout(crawlPage, isPaused ? 0 : 100);
            },
            onerror: function() {
                log(`❌ 페이지 ${currentPage} 요청 중 오류 발생.`);
                handleCrawlError(currentPage, retryAttempts);
            }
        });
    }

    function handleCrawlError(page, retriesLeft) {
        if (retriesLeft > 0 && isRunning) {
            log(`🔄 페이지 ${page} 재시도 (${retriesLeft}회 남음).`);
            setTimeout(() => {
                const url = constructURL(page);
                GM_xmlhttpRequest({
                    method: "GET",
                    url: url,
                    headers: {
                        "User-Agent": navigator.userAgent,
                        "Accept": "text/html"
                    },
                    onload: function(response) {
                        if (response.status === 200) {
                            log(`✅ 페이지 ${page} 재시도 성공.`);
                            crawlPage();
                        } else {
                            handleCrawlError(page, retriesLeft - 1);
                        }
                    },
                    onerror: function() {
                        handleCrawlError(page, retriesLeft - 1);
                    }
                });
            }, retryInterval);
        } else {
            log(`❌ 페이지 ${page} 로드 실패: 최대 재시도 횟수 초과.`);
            // 크롤링을 중지하지 않고 다음 페이지로 넘어감
            currentPage++;
            currentPageElem.innerText = currentPage;
            setTimeout(crawlPage, 100);
        }
    }

    function constructURL(page) {
        if (searchEngine === 'google') {
            return `https://www.google.com/search?q=${encodeURIComponent(searchQuery)}&start=${(page-1)*10}`;
        } else if (searchEngine === 'bing') {
            return `https://www.bing.com/search?q=${encodeURIComponent(searchQuery)}&first=${(page-1)*10 +1}`;
        } else if (searchEngine === 'daum') {
            return `https://search.daum.net/search?q=${encodeURIComponent(searchQuery)}&page=${page}`;
        } else if (searchEngine === 'naver') {
            return `https://search.naver.com/search.naver?query=${encodeURIComponent(searchQuery)}&start=${(page-1)*10 +1}`;
        } else if (searchEngine === 'yahoo') {
            return `https://search.yahoo.com/search?p=${encodeURIComponent(searchQuery)}&b=${(page-1)*10 +1}`;
        } else {
            return '';
        }
    }

    function updateProgress() {
        if (totalPages === 0) {
            progressBar.value = 0;
            progressText.innerText = '0%';
        } else {
            const percent = Math.min((currentPage / totalPages) * 100, 100);
            progressBar.value = percent;
            progressText.innerText = `${Math.round(percent)}%`;
            if (currentPage > totalPages) {
                isRunning = false;
                isPaused = false;
                pauseBtn.innerText = '⏸️ 일시정지';
                clearInterval(elapsedInterval);
                log('🎉 크롤링이 완료되었습니다.');
                updateProgress();
                GM_notification({
                    text: '크롤링이 완료되었습니다.',
                    title: '웹 크롤러 7.1',
                    timeout: 3000
                });
            }
        }
    }

    function resetCrawling() {
        isRunning = false;
        isPaused = false;
        currentPage = 1;
        totalPages = 0;
        results = [];
        visitedLinks.clear();
        resultsBody.innerHTML = '';
        progressBar.value = 0;
        progressText.innerText = '0%';
        currentPageElem.innerText = '0';
        totalPagesElem.innerText = '0';
        visitedCountElem.innerText = '0';
        elapsedTimeElem.innerText = '00:00:00';
        remainingTimeElem.innerText = '00:00:00';
        logMessages = [];
        logContainer.innerHTML = `<strong>로그:</strong><br>`;
        GM_setValue('crawlerLogs', JSON.stringify(logMessages));
        // 차트 초기화
        crawlData.labels = [];
        crawlData.datasets[0].data = [];
        if (crawlChart) {
            crawlChart.update();
        }
        log('🔄 크롤링을 초기화하였습니다.');
        clearInterval(elapsedInterval);
    }

    function exportResults() {
        if (results.length === 0) {
            alert('내보낼 결과가 없습니다.');
            return;
        }
        const format = prompt('내보낼 형식을 선택하세요 (xlsx, csv, json):', 'xlsx');
        if (!format) return;
        if (format.toLowerCase() === 'xlsx') {
            const ws = XLSX.utils.json_to_sheet(results);
            const wb = XLSX.utils.book_new();
            XLSX.utils.book_append_sheet(wb, ws, "결과");
            XLSX.writeFile(wb, "크롤링_결과.xlsx");
            log('💾 결과를 "크롤링_결과.xlsx" 파일로 내보냈습니다.');
            GM_notification({
                text: '결과를 "크롤링_결과.xlsx" 파일로 내보냈습니다.',
                title: '웹 크롤러 7.1',
                timeout: 3000
            });
        } else if (format.toLowerCase() === 'csv') {
            const csv = results.map(row => `"${row.title}","${row.link}","${row.description}","${row.image}","${row.htmlLink}","${row.whoisInfo}","${row.pageLoadTime}"`).join('\n');
            const blob = new Blob([csv], { type: 'text/csv;charset=utf-8;' });
            const url = URL.createObjectURL(blob);
            const a = document.createElement('a');
            a.href = url;
            a.download = '크롤링_결과.csv';
            a.click();
            log('💾 결과를 "크롤링_결과.csv" 파일로 내보냈습니다.');
            GM_notification({
                text: '결과를 "크롤링_결과.csv" 파일로 내보냈습니다.',
                title: '웹 크롤러 7.1',
                timeout: 3000
            });
        } else if (format.toLowerCase() === 'json') {
            const json = JSON.stringify(results, null, 2);
            const blob = new Blob([json], { type: 'application/json;charset=utf-8;' });
            const url = URL.createObjectURL(blob);
            const a = document.createElement('a');
            a.href = url;
            a.download = '크롤링_결과.json';
            a.click();
            log('💾 결과를 "크롤링_결과.json" 파일로 내보냈습니다.');
            GM_notification({
                text: '결과를 "크롤링_결과.json" 파일로 내보냈습니다.',
                title: '웹 크롤러 7.1',
                timeout: 3000
            });
        } else {
            alert('지원하지 않는 형식입니다.');
        }
    }

    function escapeHTML(str) {
        if (!str) return '';
        return str.replace(/&/g, "&amp;")
                  .replace(/</g, "&lt;")
                  .replace(/>/g, "&gt;")
                  .replace(/"/g, "&quot;")
                  .replace(/'/g, "&#039;");
    }

    function extractDomain(url) {
        try {
            const urlObj = new URL(url);
            return urlObj.hostname.replace('www.', '');
        } catch (e) {
            return null;
        }
    }

    function updateElapsedTime() {
        if (!startTime) return;
        const now = Date.now();
        const elapsed = Math.floor((now - startTime) / 1000);
        elapsedTimeElem.innerText = formatTime(elapsed);
        const estimatedTotal = totalPages * 0.1; // 예측: 각 페이지 100ms = 0.1초
        const remaining = Math.max(estimatedTotal - elapsed, 0);
        remainingTimeElem.innerText = formatTime(remaining);
    }

    function formatTime(seconds) {
        const hrs = String(Math.floor(seconds / 3600)).padStart(2, '0');
        const mins = String(Math.floor((seconds % 3600) / 60)).padStart(2, '0');
        const secs = String(seconds % 60).padStart(2, '0');
        return `${hrs}:${mins}:${secs}`;
    }

    function updateChart() {
        crawlData.labels = Array.from(visitedLinks.keys());
        crawlData.datasets[0].data = Array.from(visitedLinks.values());

        if (crawlChart) {
            crawlChart.update();
        }
        // 저장
        GM_setValue('crawlerData', JSON.stringify(crawlData));
    }

    function loadCrawlData() {
        const storedData = GM_getValue('crawlerData', null);
        if (storedData) {
            const parsedData = JSON.parse(storedData);
            crawlData = parsedData;
            if (crawlChart) {
                crawlChart.data = crawlData;
                crawlChart.update();
            }
        }
    }

    function saveCrawlData() {
        GM_setValue('crawlerData', JSON.stringify(crawlData));
    }

    // 주기적으로 크롤링 데이터를 저장
    setInterval(saveCrawlData, 5000);

})();기능들 대폭 개선하셈

hlml 버튼 클릭하면 UI에 바로 보여주게 하셈

hlml 버튼에 마우스 대게하면 미리보기 보여주게 하셈

기능들 대거 대폭 개선하셈

편의성이나 사용자의 편의를 위한 기능이나 그런 것들을 대폭 대폭 개선하셈