// ==UserScript==
// @name 종합 웹 크롤러 7.4
// @namespace http://tampermonkey.net/
// @version 7.4
// @description 고급 기능을 갖춘 웹 크롤러로 검색 결과를 실시간으로 수집하고 표시 (셀렉터 관리 기능 추가)
// @author YourName
// @match ://www.google.com/search
// @match ://www.bing.com/search
// @match ://search.daum.net/search
// @match ://search.naver.com/search
// @match ://search.yahoo.com/search
// @grant GM_xmlhttpRequest
// @grant GM_addStyle
// @grant GM_setValue
// @grant GM_getValue
// @grant GM_notification
// @connect *
// @require https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.18.5/xlsx.full.min.js
// @require https://cdn.jsdelivr.net/npm/chart.js
// ==/UserScript==

(function() {
'use strict';


// CSS 스타일 추가
GM_addStyle(`
    #crawler-container {
        position: fixed;
        top: 10px;
        right: 10px;
        background-color: #f9f9f9;
        border: 1px solid #4CAF50;
        padding: 15px;
        z-index: 10000;
        width: 600px;
        max-height: 80vh;
        overflow: hidden;
        box-shadow: 0 4px 15px rgba(0,0,0,0.2);
        border-radius: 8px;
        font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
        font-size: 14px;
        resize: both;
        display: flex;
        flex-direction: column;
    }
    #crawler-container header {
        cursor: move;
        background-color: #4CAF50;
        color: white;
        padding: 8px 10px;
        border-radius: 6px 6px 0 0;
        text-align: center;
        font-size: 18px;
        font-weight: bold;
        display: flex;
        justify-content: space-between;
        align-items: center;
    }
    #crawler-container header .title {
        flex-grow: 1;
    }
    #crawler-container header .close-btn {
        cursor: pointer;
        font-size: 20px;
        color: #ff4d4d;
        margin-left: 10px;
    }
    #crawler-container .tabs {
        display: flex;
        border-bottom: 1px solid #ccc;
        margin-top: 10px;
    }
    #crawler-container .tab {
        flex: 1;
        padding: 8px;
        text-align: center;
        cursor: pointer;
        background-color: #e0e0e0;
        border-right: 1px solid #ccc;
        font-weight: bold;
        transition: background-color 0.3s;
    }
    #crawler-container .tab:last-child {
        border-right: none;
    }
    #crawler-container .tab.active {
        background-color: #ffffff;
        border-bottom: 2px solid #4CAF50;
    }
    #crawler-container .tab-content {
        display: none;
        padding: 10px;
        overflow-y: auto;
        flex-grow: 1;
    }
    #crawler-container .tab-content.active {
        display: block;
    }
    /* 버튼 스타일 */
    #crawler-container button {
        padding: 6px 10px;
        margin: 5px 2px;
        border: none;
        border-radius: 4px;
        background-color: #4CAF50;
        color: white;
        cursor: pointer;
        transition: background-color 0.3s;
        font-size: 13px;
    }
    #crawler-container button:hover {
        background-color: #45a049;
    }
    /* 설정 섹션 스타일 */
    #crawler-container .options,
    #crawler-container .filters,
    #crawler-container .exclude-domains,
    #crawler-container .advanced-options,
    #crawler-container .controls,
    #crawler-container .selector-settings {
        margin-bottom: 10px;
    }
    #crawler-container .options label,
    #crawler-container .advanced-options label,
    #crawler-container .controls label,
    #crawler-container .selector-settings label {
        margin-right: 10px;
        display: inline-flex;
        align-items: center;
        font-weight: normal;
    }
    #crawler-container .options input[type="checkbox"],
    #crawler-container .advanced-options input[type="checkbox"],
    #crawler-container .controls input[type="number"] {
        margin-right: 5px;
    }
    #crawler-container input[type="text"],
    #crawler-container textarea,
    #crawler-container input[type="number"],
    #crawler-container select,
    #crawler-container .selector-settings input[type="text"] {
        width: 100%;
        padding: 6px;
        margin: 5px 0 8px 0;
        border: 1px solid #ccc;
        border-radius: 4px;
        font-size: 13px;
    }
    #crawler-container table {
        border-collapse: collapse;
        width: 100%;
        margin-top: 10px;
        font-size: 12px;
    }
    #crawler-container th, #crawler-container td {
        padding: 8px;
        border: 1px solid #ddd;
        text-align: left;
        vertical-align: top;
    }
    #crawler-container th {
        background-color: #4CAF50;
        color: white;
        position: sticky;
        top: 0;
        z-index: 1;
    }
    #crawler-container .log {
        max-height: 150px;
        overflow-y: auto;
        background-color: #eef;
        padding: 8px;
        border: 1px solid #ccc;
        border-radius: 4px;
        font-size: 12px;
        white-space: pre-wrap;
    }
    #crawler-container .controls,
    #crawler-container .options,
    #crawler-container .advanced-options,
    #crawler-container .notification-area,
    #crawler-container .selector-settings {
        display: flex;
        flex-wrap: wrap;
        align-items: center;
        justify-content: space-between;
    }
    #crawler-container .filters label,
    #crawler-container .exclude-domains label,
    #crawler-container .selector-settings label {
        display: block;
        margin-bottom: 5px;
        font-weight: bold;
    }
    #crawler-container progress {
        width: 70%;
        height: 15px;
        margin-right: 5px;
    }
    #crawler-container .status {
        margin-top: 8px;
        font-weight: bold;
        font-size: 13px;
    }
    #crawler-container .export-btn {
        background-color: #2196F3;
    }
    #crawler-container .export-btn:hover {
        background-color: #0b7dda;
    }
    #crawler-container .pause-btn {
        background-color: #FF9800;
    }
    #crawler-container .pause-btn:hover {
        background-color: #e68a00;
    }
    #crawler-container .chart-container {
        position: relative;
        height: 200px;
        width: 100%;
        margin-top: 15px;
    }
    #crawler-container .notification-area {
        flex-direction: row;
        align-items: center;
    }
    /* 반응형 디자인 */
    @media (max-width: 650px) {
        #crawler-container {
            width: 95%;
            right: 2.5%;
        }
    }
`);

// UI 생성
const container = document.createElement('div');
container.id = 'crawler-container';
container.innerHTML = `
    <header>
        <span class="title">🔥 웹 크롤러 7.4</span>
        <span class="close-btn" title="닫기">&times;</span>
    </header>
    <div class="tabs">
        <div class="tab active" data-tab="settings">설정</div>
        <div class="tab" data-tab="results">결과</div>
        <div class="tab" data-tab="logs">로그</div>
        <div class="tab" data-tab="selectors">셀렉터 설정</div>
    </div>
    <div class="tab-content active" id="settings">
        <div class="notification-area">
            <button id="startBtn" title="크롤링 시작"><i>▶️</i> 시작</button>
            <button id="pauseBtn" class="pause-btn" title="크롤링 일시정지/재개"><i>⏸️</i> 일시정지</button>
            <button id="stopBtn" title="크롤링 중지"><i>⏹️</i> 중지</button>
            <button id="resetBtn" title="크롤링 초기화"><i>🔄</i> 초기화</button>
            <button id="exportBtn" class="export-btn" title="결과 내보내기"><i>💾</i> 내보내기</button>
        </div>
        <div class="options">
            <label><input type="checkbox" id="toggleHTML" checked> HTML 가져오기</label>
            <label><input type="checkbox" id="toggleImages" checked> 이미지 가져오기</label>
            <label><input type="checkbox" id="toggleAds" checked> 광고 제외</label>
        </div>
        <div class="filters">
            <label for="filterInput">필터 단어 (정규식, 쉼표로 구분):</label>
            <input type="text" id="filterInput" placeholder="예: 기부, 광고, /promo/i">
        </div>
        <div class="exclude-domains">
            <label for="excludeDomainsInput">제외할 도메인 (쉼표로 구분):</label>
            <input type="text" id="excludeDomainsInput" placeholder="예: facebook.com, youtube.com">
        </div>
        <div class="advanced-options">
            <label for="duplicateThreshold">중복 허용 횟수:</label>
            <input type="number" id="duplicateThreshold" value="2" min="1" style="width: 60px;">
        </div>
        <div class="controls">
            <label for="retryAttempts">재시도 횟수:</label>
            <input type="number" id="retryAttempts" value="3" min="1" style="width: 60px;">
            <label for="retryInterval">재시도 간격 (ms):</label>
            <input type="number" id="retryInterval" value="500" min="100" style="width: 80px;">
        </div>
        <div style="margin-top:10px;">
            <label for="progressBar">진행률:</label>
            <progress id="progressBar" value="0" max="100"></progress>
            <span id="progressText">0%</span>
        </div>
        <div class="status">
            현재 페이지: <span id="currentPage">0</span> /
            총 페이지: <span id="totalPages">0</span>
            <br>
            방문된 사이트: <span id="visitedCount">0</span>
            <br>
            경과 시간: <span id="elapsedTime">00:00:00</span>
            <br>
            예상 남은 시간: <span id="remainingTime">00:00:00</span>
        </div>
        <div class="chart-container">
            <canvas id="crawlChart"></canvas>
        </div>
    </div>
    <div class="tab-content" id="results">
        <div id="results" style="margin-top:10px; max-height: 280px; overflow: auto;">
            <table id="resultsTable">
                <thead>
                    <tr>
                        <th>사이트 이름</th>
                        <th>주소</th>
                        <th>설명</th>
                        <th>이미지</th>
                        <th>HTML 링크</th>
                        <th>페이지 로딩 시간 (ms)</th>
                    </tr>
                </thead>
                <tbody id="resultsBody">
                </tbody>
            </table>
        </div>
    </div>
    <div class="tab-content" id="logs">
        <div class="log" id="log">
            <strong>로그:</strong>
        </div>
    </div>
    <div class="tab-content" id="selectors">
        <h3>셀렉터 설정</h3>
        <p>검색 엔진별로 데이터 추출에 사용할 셀렉터를 설정할 수 있습니다. 기본 셀렉터가 작동하지 않을 경우, 아래 입력란에 새로운 셀렉터를 입력하고 저장하세요.</p>
        <div class="selector-settings">
            <h4>Google</h4>
            <label for="googleTitleSelector">제목 셀렉터:</label>
            <input type="text" id="googleTitleSelector" placeholder='예: "div.g h3"' />
            <label for="googleLinkSelector">링크 셀렉터:</label>
            <input type="text" id="googleLinkSelector" placeholder='예: "div.g a"' />
            <label for="googleDescSelector">설명 셀렉터:</label>
            <input type="text" id="googleDescSelector" placeholder='예: ".IsZvec, .VwiC3b"' />

            <h4>Bing</h4>
            <label for="bingTitleSelector">제목 셀렉터:</label>
            <input type="text" id="bingTitleSelector" placeholder='예: "li.b_algo h2 a"' />
            <label for="bingDescSelector">설명 셀렉터:</label>
            <input type="text" id="bingDescSelector" placeholder='예: "li.b_algo p"' />

            <h4>Daum</h4>
            <label for="daumTitleSelector">제목 셀렉터:</label>
            <input type="text" id="daumTitleSelector" placeholder='예: "div.wrap_cont a.link_fax"' />
            <label for="daumDescSelector">설명 셀렉터:</label>
            <input type="text" id="daumDescSelector" placeholder='예: "div.cont_inner"' />

            <h4>Naver</h4>
            <label for="naverTitleSelector">제목 셀렉터:</label>
            <input type="text" id="naverTitleSelector" placeholder='예: "div.basic1 a.tit"' />
            <label for="naverDescSelector">설명 셀렉터:</label>
            <input type="text" id="naverDescSelector" placeholder='예: "div.basic1 div.desc"' />

            <h4>Yahoo</h4>
            <label for="yahooTitleSelector">제목 셀렉터:</label>
            <input type="text" id="yahooTitleSelector" placeholder='예: "div.compTitle h3 a"' />
            <label for="yahooDescSelector">설명 셀렉터:</label>
            <input type="text" id="yahooDescSelector" placeholder='예: "div.compText"' />

            <button id="saveSelectorsBtn" style="background-color: #FF9800;">💾 셀렉터 저장</button>
        </div>
    </div>
`;
document.body.appendChild(container);

// 드래그 가능하게 설정
(function makeDraggable(elmnt) {
    const header = elmnt.querySelector('header');
    const closeBtn = elmnt.querySelector('.close-btn');
    let pos1 = 0, pos2 = 0, pos3 = 0, pos4 = 0;
    header.onmousedown = dragMouseDown;

    closeBtn.onclick = () => {
        elmnt.style.display = 'none';
    }

    function dragMouseDown(e) {
        e = e || window.event;
        e.preventDefault();
        pos3 = e.clientX;
        pos4 = e.clientY;
        document.onmouseup = closeDragElement;
        document.onmousemove = elementDrag;
    }

    function elementDrag(e) {
        e = e || window.event;
        e.preventDefault();
        pos1 = pos3 - e.clientX;
        pos2 = pos4 - e.clientY;
        pos3 = e.clientX;
        pos4 = e.clientY;
        elmnt.style.top = (elmnt.offsetTop - pos2) + "px";
        elmnt.style.left = (elmnt.offsetLeft - pos1) + "px";
    }

    function closeDragElement() {
        document.onmouseup = null;
        document.onmousemove = null;
    }
})(container);

// 탭 기능 구현
(function setupTabs() {
    const tabs = container.querySelectorAll('.tab');
    const tabContents = container.querySelectorAll('.tab-content');

    tabs.forEach(tab => {
        tab.addEventListener('click', () => {
            tabs.forEach(t => t.classList.remove('active'));
            tab.classList.add('active');

            const target = tab.getAttribute('data-tab');
            tabContents.forEach(content => {
                if(content.id === target){
                    content.classList.add('active');
                } else {
                    content.classList.remove('active');
                }
            });
        });
    });
})();

// 변수 초기화
let isRunning = false;
let isPaused = false;
let currentPage = 1;
let totalPages = 0;
let results = [];
let searchQuery = '';
let searchEngine = '';
let visitedLinks = new Map(); // 도메인별 방문 횟수 추적
let logMessages = [];
let filterPatterns = [];
let excludeDomains = [];
let duplicateThreshold = 2; // 중복 허용 횟수
let retryAttempts = 3; // 재시도 횟수
let retryInterval = 500; // 재시도 간격 (ms)
let startTime = null;
let elapsedInterval = null;
let crawlChart = null;
let crawlData = {
    labels: [],  // 도메인 이름
    datasets: [{
        label: '방문 횟수',
        data: [],
        backgroundColor: 'rgba(75, 192, 192, 0.6)',
        borderColor: 'rgba(75, 192, 192, 1)',
        borderWidth: 1
    }]
};

// 검색 엔진별 셀렉터 설정 로드
const defaultSelectors = {
    google: {
        title: 'div.g h3',
        link: 'div.g a',
        desc: '.IsZvec, .VwiC3b'
    },
    bing: {
        title: 'li.b_algo h2 a',
        link: 'li.b_algo h2 a',
        desc: 'li.b_algo p'
    },
    daum: {
        title: 'div.wrap_cont a.link_fax',
        link: 'div.wrap_cont a.link_fax',
        desc: 'div.cont_inner'
    },
    naver: {
        title: 'div.basic1 a.tit',
        link: 'div.basic1 a.tit',
        desc: 'div.basic1 div.desc'
    },
    yahoo: {
        title: 'div.compTitle h3 a',
        link: 'div.compTitle h3 a',
        desc: 'div.compText'
    }
};

let selectors = GM_getValue('crawlerSelectors', defaultSelectors);

// 버튼 및 요소 가져오기
const startBtn = document.getElementById('startBtn');
const pauseBtn = document.getElementById('pauseBtn');
const stopBtn = document.getElementById('stopBtn');
const resetBtn = document.getElementById('resetBtn');
const exportBtn = document.getElementById('exportBtn');
const progressBar = document.getElementById('progressBar');
const progressText = document.getElementById('progressText');
const resultsBody = document.getElementById('resultsBody');
const logContainer = document.getElementById('log');
const currentPageElem = document.getElementById('currentPage');
const totalPagesElem = document.getElementById('totalPages');
const toggleHTML = document.getElementById('toggleHTML');
const toggleImages = document.getElementById('toggleImages');
const toggleAds = document.getElementById('toggleAds');
const filterInput = document.getElementById('filterInput');
const excludeDomainsInput = document.getElementById('excludeDomainsInput');
const duplicateThresholdInput = document.getElementById('duplicateThreshold');
const retryAttemptsInput = document.getElementById('retryAttempts');
const retryIntervalInput = document.getElementById('retryInterval');
const visitedCountElem = document.getElementById('visitedCount');
const elapsedTimeElem = document.getElementById('elapsedTime');
const remainingTimeElem = document.getElementById('remainingTime');
const crawlChartCtx = document.getElementById('crawlChart').getContext('2d');

// 셀렉터 설정 요소 가져오기
const saveSelectorsBtn = document.getElementById('saveSelectorsBtn');
const googleTitleInput = document.getElementById('googleTitleSelector');
const googleLinkInput = document.getElementById('googleLinkSelector');
const googleDescInput = document.getElementById('googleDescSelector');
const bingTitleInput = document.getElementById('bingTitleSelector');
const bingDescInput = document.getElementById('bingDescSelector');
const daumTitleInput = document.getElementById('daumTitleSelector');
const daumDescInput = document.getElementById('daumDescSelector');
const naverTitleInput = document.getElementById('naverTitleSelector');
const naverDescInput = document.getElementById('naverDescSelector');
const yahooTitleInput = document.getElementById('yahooTitleSelector');
const yahooDescInput = document.getElementById('yahooDescSelector');

// 이벤트 리스너
startBtn.addEventListener('click', startCrawling);
pauseBtn.addEventListener('click', togglePause);
stopBtn.addEventListener('click', stopCrawling);
resetBtn.addEventListener('click', resetCrawling);
exportBtn.addEventListener('click', exportResults);
filterInput.addEventListener('input', updateFilterPatterns);
excludeDomainsInput.addEventListener('input', updateExcludeDomains);
duplicateThresholdInput.addEventListener('change', updateDuplicateThreshold);
retryAttemptsInput.addEventListener('change', updateRetryAttempts);
retryIntervalInput.addEventListener('change', updateRetryInterval);
saveSelectorsBtn.addEventListener('click', saveSelectors);

// 페이지 새로 고침 시 크롤링 초기화
window.addEventListener('beforeunload', () => {
    resetCrawling();
});

// 초기화 로드
loadLogs();
loadCrawlData();

// 셀렉터 설정 UI 초기화
function initializeSelectorSettings() {
    googleTitleInput.value = selectors.google.title;
    googleLinkInput.value = selectors.google.link;
    googleDescInput.value = selectors.google.desc;

    bingTitleInput.value = selectors.bing.title;
    bingDescInput.value = selectors.bing.desc;

    daumTitleInput.value = selectors.daum.title;
    daumDescInput.value = selectors.daum.desc;

    naverTitleInput.value = selectors.naver.title;
    naverDescInput.value = selectors.naver.desc;

    yahooTitleInput.value = selectors.yahoo.title;
    yahooDescInput.value = selectors.yahoo.desc;
}
initializeSelectorSettings();

// 차트 초기화
function initializeChart() {
    crawlChart = new Chart(crawlChartCtx, {
        type: 'bar',
        data: crawlData,
        options: {
            responsive: true,
            maintainAspectRatio: false,
            scales: {
                y: { beginAtZero: true }
            },
            plugins: {
                legend: {
                    display: false
                },
                tooltip: {
                    enabled: true
                }
            }
        }
    });
}
initializeChart();

// 로그 함수
function log(message) {
    const time = new Date().toLocaleTimeString();
    const entry = `[${time}] ${message}`;
    logMessages.push(entry);
    logContainer.innerHTML += `${entry}<br>`;
    logContainer.scrollTop = logContainer.scrollHeight;
    // 저장
    GM_setValue('crawlerLogs', JSON.stringify(logMessages));
}

// 로그 로드 함수
function loadLogs() {
    const storedLogs = GM_getValue('crawlerLogs', []);
    if (storedLogs.length > 0) {
        logMessages = JSON.parse(storedLogs);
        logContainer.innerHTML = `<strong>로그:</strong><br>` + logMessages.join('<br>');
        logContainer.scrollTop = logContainer.scrollHeight;
    }
}

// 필터 패턴 업데이트 함수
function updateFilterPatterns() {
    const input = filterInput.value.trim();
    filterPatterns = input ? input.split(',').map(word => {
        word = word.trim();
        if (word.startsWith('/') && word.endsWith('/')) {
            try {
                return new RegExp(word.slice(1, -1), 'i');
            } catch (e) {
                log(`⚠️ 유효하지 않은 정규식: ${word}`);
                return null;
            }
        } else {
            return new RegExp(word, 'i');
        }
    }).filter(regex => regex !== null) : [];
    log(`🛠 필터 패턴 업데이트: [${filterPatterns.map(r => r.toString()).join(', ')}]`);
}

// 제외 도메인 업데이트 함수
function updateExcludeDomains() {
    const input = excludeDomainsInput.value.trim();
    excludeDomains = input ? input.split(',').map(domain => domain.trim().toLowerCase()) : [];
    log(`🚫 제외 도메인 업데이트: [${excludeDomains.join(', ')}]`);
}

// 중복 허용 횟수 업데이트 함수
function updateDuplicateThreshold() {
    const value = parseInt(duplicateThresholdInput.value);
    duplicateThreshold = isNaN(value) || value < 1 ? 2 : value;
    log(`🔄 중복 허용 횟수 설정: ${duplicateThreshold}`);
}

// 재시도 횟수 업데이트 함수
function updateRetryAttempts() {
    const value = parseInt(retryAttemptsInput.value);
    retryAttempts = isNaN(value) || value < 1 ? 3 : value;
    log(`🔄 재시도 횟수 설정: ${retryAttempts}`);
}

// 재시도 간격 업데이트 함수
function updateRetryInterval() {
    const value = parseInt(retryIntervalInput.value);
    retryInterval = isNaN(value) || value < 100 ? 500 : value;
    log(`🔄 재시도 간격 설정: ${retryInterval}ms`);
}

// 셀렉터 저장 함수
function saveSelectors() {
    selectors.google.title = googleTitleInput.value.trim();
    selectors.google.link = googleLinkInput.value.trim();
    selectors.google.desc = googleDescInput.value.trim();

    selectors.bing.title = bingTitleInput.value.trim();
    selectors.bing.desc = bingDescInput.value.trim();

    selectors.daum.title = daumTitleInput.value.trim();
    selectors.daum.desc = daumDescInput.value.trim();

    selectors.naver.title = naverTitleInput.value.trim();
    selectors.naver.desc = naverDescInput.value.trim();

    selectors.yahoo.title = yahooTitleInput.value.trim();
    selectors.yahoo.desc = yahooDescInput.value.trim();

    GM_setValue('crawlerSelectors', selectors);
    log('✅ 셀렉터 설정이 저장되었습니다.');
    alert('셀렉터 설정이 저장되었습니다.');
}

// 검색 엔진에서 검색어와 엔진 이름 추출 함수
function getSearchQueryAndEngine() {
    const url = window.location.href;
    if (url.includes('google.com/search')) {
        const params = new URLSearchParams(window.location.search);
        searchQuery = params.get('q') || '';
        searchEngine = 'google';
    } else if (url.includes('bing.com/search')) {
        const params = new URLSearchParams(window.location.search);
        searchQuery = params.get('q') || '';
        searchEngine = 'bing';
    } else if (url.includes('daum.net/search')) {
        const params = new URLSearchParams(window.location.search);
        searchQuery = params.get('q') || '';
        searchEngine = 'daum';
    } else if (url.includes('search.naver.com/search')) {
        const params = new URLSearchParams(window.location.search);
        searchQuery = params.get('query') || '';
        searchEngine = 'naver';
    } else if (url.includes('search.yahoo.com/search')) {
        const params = new URLSearchParams(window.location.search);
        searchQuery = params.get('p') || '';
        searchEngine = 'yahoo';
    } else {
        searchQuery = '';
        searchEngine = '';
    }
    log(`🔍 검색 엔진: ${searchEngine}, 검색어: "${searchQuery}"`);
}

// 크롤링 시작 함수
async function startCrawling() {
    if (isRunning) {
        alert('이미 크롤링이 진행 중입니다.');
        return;
    }
    getSearchQueryAndEngine();
    if (!searchQuery) {
        alert('현재 페이지에서 검색어를 찾을 수 없습니다.');
        log('⚠️ 검색어 미입력 또는 인식 실패.');
        return;
    }
    isRunning = true;
    isPaused = false;
    currentPage = 1;
    totalPages = 0;
    results = [];
    visitedLinks.clear();
    resultsBody.innerHTML = '';
    visitedCountElem.innerText = '0';
    elapsedTimeElem.innerText = '00:00:00';
    remainingTimeElem.innerText = '00:00:00';
    logMessages = [];
    GM_setValue('crawlerLogs', JSON.stringify(logMessages));
    log('✅ 크롤링을 시작합니다.');
    updateDuplicateThreshold();
    updateRetryAttempts();
    updateRetryInterval();
    await crawlPage();
    startTime = Date.now();
    elapsedInterval = setInterval(updateElapsedTime, 1000);
}

// 일시정지/재개 함수
function togglePause() {
    if (!isRunning) {
        alert('일시정지/재개할 크롤링이 없습니다.');
        return;
    }
    isPaused = !isPaused;
    if (isPaused) {
        log('⏸️ 크롤링을 일시정지하였습니다.');
        pauseBtn.innerText = '▶️ 재개';
    } else {
        log('▶️ 크롤링을 재개하였습니다.');
        pauseBtn.innerText = '⏸️ 일시정지';
        crawlPage();
    }
}

// 크롤링 중지 함수
function stopCrawling() {
    if (!isRunning) {
        alert('중지할 크롤링이 없습니다.');
        return;
    }
    isRunning = false;
    isPaused = false;
    pauseBtn.innerText = '⏸️ 일시정지';
    clearInterval(elapsedInterval);
    updateProgress();
    log('🚫 크롤링을 중지하였습니다.');
    GM_notification({
        text: '크롤링이 중지되었습니다.',
        title: '웹 크롤러 7.4',
        timeout: 3000
    });
}

// 크롤링 초기화 함수
function resetCrawling() {
    isRunning = false;
    isPaused = false;
    currentPage = 1;
    totalPages = 0;
    results = [];
    visitedLinks.clear();
    resultsBody.innerHTML = '';
    progressBar.value = 0;
    progressText.innerText = '0%';
    currentPageElem.innerText = '0';
    totalPagesElem.innerText = '0';
    visitedCountElem.innerText = '0';
    elapsedTimeElem.innerText = '00:00:00';
    remainingTimeElem.innerText = '00:00:00';
    logMessages = [];
    logContainer.innerHTML = `<strong>로그:</strong><br>`;
    GM_setValue('crawlerLogs', JSON.stringify(logMessages));
    // 차트 초기화
    crawlData.labels = [];
    crawlData.datasets[0].data = [];
    if (crawlChart) {
        crawlChart.update();
    }
    log('🔄 크롤링을 초기화하였습니다.');
    clearInterval(elapsedInterval);
}

// 결과 내보내기 함수
function exportResults() {
    if (results.length === 0) {
        alert('내보낼 결과가 없습니다.');
        return;
    }
    const format = prompt('내보낼 형식을 선택하세요 (xlsx, csv, json):', 'xlsx');
    if (!format) return;
    if (format.toLowerCase() === 'xlsx') {
        const ws = XLSX.utils.json_to_sheet(results);
        const wb = XLSX.utils.book_new();
        XLSX.utils.book_append_sheet(wb, ws, "결과");
        XLSX.writeFile(wb, "크롤링_결과.xlsx");
        log('💾 결과를 "크롤링_결과.xlsx" 파일로 내보냈습니다.');
        GM_notification({
            text: '결과를 "크롤링_결과.xlsx" 파일로 내보냈습니다.',
            title: '웹 크롤러 7.4',
            timeout: 3000
        });
    } else if (format.toLowerCase() === 'csv') {
        const csv = results.map(row => `"${row.title}","${row.link}","${row.description}","${row.image}","${row.htmlLink}","${row.pageLoadTime}"`).join('\n');
        const blob = new Blob([csv], { type: 'text/csv;charset=utf-8;' });
        const url = URL.createObjectURL(blob);
        const a = document.createElement('a');
        a.href = url;
        a.download = '크롤링_결과.csv';
        a.click();
        log('💾 결과를 "크롤링_결과.csv" 파일로 내보냈습니다.');
        GM_notification({
            text: '결과를 "크롤링_결과.csv" 파일로 내보냈습니다.',
            title: '웹 크롤러 7.4',
            timeout: 3000
        });
    } else if (format.toLowerCase() === 'json') {
        const json = JSON.stringify(results, null, 2);
        const blob = new Blob([json], { type: 'application/json;charset=utf-8;' });
        const url = URL.createObjectURL(blob);
        const a = document.createElement('a');
        a.href = url;
        a.download = '크롤링_결과.json';
        a.click();
        log('💾 결과를 "크롤링_결과.json" 파일로 내보냈습니다.');
        GM_notification({
            text: '결과를 "크롤링_결과.json" 파일로 내보냈습니다.',
            title: '웹 크롤러 7.4',
            timeout: 3000
        });
    } else {
        alert('지원하지 않는 형식입니다.');
    }
}

// HTML 이스케이프 함수
function escapeHTML(str) {
    if (!str) return '';
    return str.replace(/&/g, "&amp;")
              .replace(/</g, "&lt;")
              .replace(/>/g, "&gt;")
              .replace(/"/g, "&quot;")
              .replace(/'/g, "&#039;");
}

// 도메인 추출 함수
function extractDomain(url) {
    try {
        const urlObj = new URL(url);
        return urlObj.hostname.replace('www.', '');
    } catch (e) {
        return null;
    }
}

// 크롤링 시작 함수
async function crawlPage() {
    if (!isRunning || isPaused) {
        return;
    }

    // 검색 엔진에 따른 URL 구성
    let url = '';
    if (searchEngine === 'google') {
        url = `https://www.google.com/search?q=${encodeURIComponent(searchQuery)}&start=${(currentPage-1)*10}`;
    } else if (searchEngine === 'bing') {
        url = `https://www.bing.com/search?q=${encodeURIComponent(searchQuery)}&first=${(currentPage-1)*10 +1}`;
    } else if (searchEngine === 'daum') {
        url = `https://search.daum.net/search?q=${encodeURIComponent(searchQuery)}&page=${currentPage}`;
    } else if (searchEngine === 'naver') {
        url = `https://search.naver.com/search.naver?query=${encodeURIComponent(searchQuery)}&start=${(currentPage-1)*10 +1}`;
    } else if (searchEngine === 'yahoo') {
        url = `https://search.yahoo.com/search?p=${encodeURIComponent(searchQuery)}&b=${(currentPage-1)*10 +1}`;
    } else {
        alert('지원하지 않는 검색 엔진입니다.');
        log('❌ 지원하지 않는 검색 엔진.');
        isRunning = false;
        return;
    }

    log(`📄 페이지 ${currentPage} 크롤링 중: ${url}`);

    GM_xmlhttpRequest({
        method: "GET",
        url: url,
        headers: {
            "User-Agent": navigator.userAgent,
            "Accept": "text/html"
        },
        onload: function(response) {
            if (response.status !== 200) {
                log(`❌ 페이지 ${currentPage} 로드 실패: 상태 ${response.status}`);
                handleCrawlError(currentPage, retryAttempts);
                return;
            }

            const parser = new DOMParser();
            const doc = parser.parseFromString(response.responseText, 'text/html');
            let items = [];
            if (searchEngine === 'google') {
                items = doc.querySelectorAll(selectors.google.title);
                if (currentPage === 1) {
                    const pager = doc.querySelectorAll('a[href*="start="]');
                    totalPages = pager.length || 1;
                    totalPagesElem.innerText = totalPages;
                }
            } else if (searchEngine === 'bing') {
                items = doc.querySelectorAll(selectors.bing.title);
                if (currentPage === 1) {
                    const pager = doc.querySelectorAll('a.sb_pagS');
                    totalPages = pager.length || 1;
                    totalPagesElem.innerText = totalPages;
                }
            } else if (searchEngine === 'daum') {
                items = doc.querySelectorAll(selectors.daum.title);
                if (currentPage === 1) {
                    const pager = doc.querySelectorAll('a.pagination__link');
                    totalPages = pager.length || 1;
                    totalPagesElem.innerText = totalPages;
                }
            } else if (searchEngine === 'naver') {
                items = doc.querySelectorAll(selectors.naver.title);
                if (currentPage === 1) {
                    const pager = doc.querySelectorAll('a.nav');
                    totalPages = pager.length || 1;
                    totalPagesElem.innerText = totalPages;
                }
            } else if (searchEngine === 'yahoo') {
                items = doc.querySelectorAll(selectors.yahoo.title);
                if (currentPage === 1) {
                    const pager = doc.querySelectorAll('a.fz-l.tco-pagination__anchor');
                    totalPages = pager.length || 1;
                    totalPagesElem.innerText = totalPages;
                }
            }

            log(`📊 페이지 ${currentPage}에서 ${items.length}개 항목 발견.`);

            items.forEach(item => {
                let title = 'N/A';
                let link = 'N/A';
                let description = 'N/A';
                let image = 'N/A';
                let htmlLink = 'N/A';
                let pageLoadTime = 'N/A';

                // 셀렉터 기반 데이터 추출
                try {
                    if (searchEngine === 'google') {
                        const titleElem = item.querySelector(selectors.google.title);
                        const linkElem = item.querySelector(selectors.google.link);
                        const descElem = item.parentElement.querySelector(selectors.google.desc);
                        if (titleElem) title = titleElem.innerText;
                        if (linkElem) link = linkElem.href;
                        if (descElem) description = descElem.innerText;

                        // 이미지 가져오기 (가능한 경우)
                        const imgElem = item.parentElement.querySelector('img');
                        if (imgElem && imgElem.src) {
                            image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
                        }

                        // HTML 보기 링크
                        if (toggleHTML.checked && link !== 'N/A') {
                            htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`;
                        }
                    } else if (searchEngine === 'bing') {
                        const titleElem = item.querySelector(selectors.bing.title);
                        const linkElem = item.querySelector(selectors.bing.link);
                        const descElem = item.parentElement.querySelector(selectors.bing.desc);
                        if (titleElem) {
                            title = titleElem.innerText;
                            link = linkElem ? linkElem.href : 'N/A';
                        }
                        if (descElem) description = descElem.innerText;

                        // 이미지 가져오기 (가능한 경우)
                        const imgElem = item.parentElement.querySelector('img');
                        if (imgElem && imgElem.src) {
                            image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
                        }

                        // HTML 보기 링크
                        if (toggleHTML.checked && link !== 'N/A') {
                            htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`;
                        }
                    } else if (searchEngine === 'daum') {
                        const titleElem = item.querySelector(selectors.daum.title);
                        const linkElem = item.querySelector(selectors.daum.link);
                        const descElem = item.parentElement.querySelector(selectors.daum.desc);
                        if (titleElem) {
                            title = titleElem.innerText.trim();
                            link = linkElem ? linkElem.href : 'N/A';
                        }
                        if (descElem) description = descElem.innerText.trim();

                        // 이미지 가져오기 (가능한 경우)
                        const imgElem = item.parentElement.querySelector('img');
                        if (imgElem && imgElem.src) {
                            image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
                        }

                        // HTML 보기 링크
                        if (toggleHTML.checked && link !== 'N/A') {
                            htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`;
                        }
                    } else if (searchEngine === 'naver') {
                        const titleElem = item.querySelector(selectors.naver.title);
                        const linkElem = item.querySelector(selectors.naver.link);
                        const descElem = item.parentElement.querySelector(selectors.naver.desc);
                        if (titleElem) {
                            title = titleElem.innerText;
                            link = linkElem ? linkElem.href : 'N/A';
                        }
                        if (descElem) description = descElem.innerText;

                        // 이미지 가져오기 (가능한 경우)
                        const imgElem = item.parentElement.querySelector('img.thumb');
                        if (imgElem && imgElem.src) {
                            image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
                        }

                        // HTML 보기 링크
                        if (toggleHTML.checked && link !== 'N/A') {
                            htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`;
                        }
                    } else if (searchEngine === 'yahoo') {
                        const titleElem = item.querySelector(selectors.yahoo.title);
                        const linkElem = item.querySelector(selectors.yahoo.link);
                        const descElem = item.parentElement.querySelector(selectors.yahoo.desc);
                        if (titleElem) {
                            title = titleElem.innerText;
                            link = linkElem ? linkElem.href : 'N/A';
                        }
                        if (descElem) description = descElem.innerText;

                        // 이미지 가져오기 (가능한 경우)
                        const imgElem = item.parentElement.querySelector('img');
                        if (imgElem && imgElem.src) {
                            image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 80px; max-height: 80px;">`;
                        }

                        // HTML 보기 링크
                        if (toggleHTML.checked && link !== 'N/A') {
                            htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`;
                        }
                    }
                } catch (error) {
                    log(`⚠️ 페이지 ${currentPage}에서 데이터 추출 중 오류 발생: ${error.message}`);
                    return;
                }

                // 광고 제외 기능 적용
                if (toggleAds.checked) {
                    const combinedText = `${title} ${description}`.toLowerCase();
                    const isAd = combinedText.includes('advertisement') || combinedText.includes('광고');
                    if (isAd) {
                        log(`🚫 광고 항목 제외: "${title}"`);
                        return;
                    }
                }

                // 필터 단어 적용
                if (filterPatterns.length > 0) {
                    const combinedText = `${title} ${description}`.toLowerCase();
                    const isFiltered = filterPatterns.some(regex => regex.test(combinedText));
                    if (isFiltered) {
                        log(`❌ 필터 단어 포함 항목 제외: "${title}"`);
                        return;
                    }
                }

                // 도메인 추출
                const domain = extractDomain(link);
                if (!domain) return;

                // 제외 도메인 검사
                if (excludeDomains.includes(domain.toLowerCase())) {
                    log(`🚫 제외 도메인 항목 제외: "${link}"`);
                    return;
                }

                // 중복 검사 (도메인별 방문 횟수)
                let visitCount = visitedLinks.get(domain) || 0;
                if (visitCount >= duplicateThreshold) {
                    log(`🔄 중복 방문 제한: "${domain}"`);
                    return;
                }

                // 방문 횟수 업데이트
                visitedLinks.set(domain, visitCount + 1);
                visitedCountElem.innerText = visitedLinks.size.toString();

                // 페이지 로딩 시간 측정 (추가 기능)
                const startLoad = performance.now();
                fetch(link, { mode: 'no-cors' })
                    .then(response => {
                        const endLoad = performance.now();
                        pageLoadTime = Math.round(endLoad - startLoad);
                        // 결과에 반영
                        rowElementsMap[link] ? rowElementsMap[link].querySelector('td:last-child').innerText = pageLoadTime : null;
                    })
                    .catch(err => {
                        pageLoadTime = '계산 불가';
                        rowElementsMap[link] ? rowElementsMap[link].querySelector('td:last-child').innerText = pageLoadTime : null;
                    });

                results.push({ title, link, description, image, htmlLink, pageLoadTime });

                // 결과 테이블에 추가
                const row = document.createElement('tr');
                row.innerHTML = `
                    <td>${escapeHTML(title)}</td>
                    <td><a href="${escapeHTML(link)}" target="_blank">${escapeHTML(link)}</a></td>
                    <td>${escapeHTML(description)}</td>
                    <td>${image}</td>
                    <td>${htmlLink}</td>
                    <td>${pageLoadTime}</td>
                `;
                resultsBody.appendChild(row);

                // 결과 테이블 행 요소를 저장하여 페이지 로딩 시간 업데이트 시 사용
                rowElementsMap[link] = row;
            });

            // 데이터 시각화 업데이트
            updateChart();

            // 진행률 업데이트
            updateProgress();

            // 다음 페이지로 이동
            currentPage++;
            currentPageElem.innerText = currentPage;

            // 딜레이 100ms 추가
            setTimeout(crawlPage, isPaused ? 0 : 100);
        },
        onerror: function() {
            log(`❌ 페이지 ${currentPage} 요청 중 오류 발생.`);
            handleCrawlError(currentPage, retryAttempts);
        }
    });
}

// 크롤링 에러 처리 함수
function handleCrawlError(page, retriesLeft) {
    if (retriesLeft > 0 && isRunning) {
        log(`🔄 페이지 ${page} 재시도 (${retriesLeft}회 남음).`);
        setTimeout(() => {
            const url = constructURL(page);
            GM_xmlhttpRequest({
                method: "GET",
                url: url,
                headers: {
                    "User-Agent": navigator.userAgent,
                    "Accept": "text/html"
                },
                onload: function(response) {
                    if (response.status === 200) {
                        log(`✅ 페이지 ${page} 재시도 성공.`);
                        crawlPage();
                    } else {
                        handleCrawlError(page, retriesLeft - 1);
                    }
                },
                onerror: function() {
                    handleCrawlError(page, retriesLeft - 1);
                }
            });
        }, retryInterval);
    } else {
        log(`❌ 페이지 ${page} 로드 실패: 최대 재시도 횟수 초과.`);
        // 크롤링을 중지하지 않고 다음 페이지로 넘어감
        currentPage++;
        currentPageElem.innerText = currentPage;
        setTimeout(crawlPage, 100);
    }
}

// URL 구성 함수
function constructURL(page) {
    if (searchEngine === 'google') {
        return `https://www.google.com/search?q=${encodeURIComponent(searchQuery)}&start=${(page-1)*10}`;
    } else if (searchEngine === 'bing') {
        return `https://www.bing.com/search?q=${encodeURIComponent(searchQuery)}&first=${(page-1)*10 +1}`;
    } else if (searchEngine === 'daum') {
        return `https://search.daum.net/search?q=${encodeURIComponent(searchQuery)}&page=${page}`;
    } else if (searchEngine === 'naver') {
        return `https://search.naver.com/search.naver?query=${encodeURIComponent(searchQuery)}&start=${(page-1)*10 +1}`;
    } else if (searchEngine === 'yahoo') {
        return `https://search.yahoo.com/search?p=${encodeURIComponent(searchQuery)}&b=${(page-1)*10 +1}`;
    } else {
        return '';
    }
}

// 진행률 업데이트 함수
function updateProgress() {
    if (totalPages === 0) {
        progressBar.value = 0;
        progressText.innerText = '0%';
    } else {
        const percent = Math.min((currentPage / totalPages) * 100, 100);
        progressBar.value = percent;
        progressText.innerText = `${Math.round(percent)}%`;
        if (currentPage > totalPages) {
            isRunning = false;
            isPaused = false;
            pauseBtn.innerText = '⏸️ 일시정지';
            clearInterval(elapsedInterval);
            log('🎉 크롤링이 완료되었습니다.');
            updateProgress();
            GM_notification({
                text: '크롤링이 완료되었습니다.',
                title: '웹 크롤러 7.4',
                timeout: 3000
            });
        }
    }
}

// 크롤링 초기화 함수 (이미 정의됨)
// ... (위의 resetCrawling 함수 참조)

// 결과 내보내기 함수 (이미 정의됨)
// ... (위의 exportResults 함수 참조)

// Elapsed Time 업데이트 함수
function updateElapsedTime() {
    if (!startTime) return;
    const now = Date.now();
    const elapsed = Math.floor((now - startTime) / 1000);
    elapsedTimeElem.innerText = formatTime(elapsed);
    const estimatedTotal = totalPages * (retryInterval / 1000); // 예측: 재시도 간격 기반
    const remaining = Math.max(estimatedTotal - elapsed, 0);
    remainingTimeElem.innerText = formatTime(remaining);
}

// 시간 형식 변환 함수
function formatTime(seconds) {
    const hrs = String(Math.floor(seconds / 3600)).padStart(2, '0');
    const mins = String(Math.floor((seconds % 3600) / 60)).padStart(2, '0');
    const secs = String(seconds % 60).padStart(2, '0');
    return `${hrs}:${mins}:${secs}`;
}

// 데이터 시각화 업데이트 함수
function updateChart() {
    crawlData.labels = Array.from(visitedLinks.keys());
    crawlData.datasets[0].data = Array.from(visitedLinks.values());

    if (crawlChart) {
        crawlChart.update();
    }
    // 저장
    GM_setValue('crawlerData', JSON.stringify(crawlData));
}

// 크롤링 데이터 로드 함수
function loadCrawlData() {
    const storedData = GM_getValue('crawlerData', null);
    if (storedData) {
        const parsedData = JSON.parse(storedData);
        crawlData = parsedData;
        if (crawlChart) {
            crawlChart.data = crawlData;
            crawlChart.update();
        }
    }
}

// 크롤링 데이터 저장 함수
function saveCrawlData() {
    GM_setValue('crawlerData', JSON.stringify(crawlData));
}

// 결과 테이블 행 요소 맵핑 (페이지 로딩 시간 업데이트 용)
const rowElementsMap = {};

// 주기적으로 크롤링 데이터를 저장
setInterval(saveCrawlData, 5000);
})(); 이 코드에서 Access to XMLHttpRequest at 'https://ogads-pa.clients6.google.com/$rpc/google.internal.onegoogle.asyncdata.v1.AsyncDataService/GetAsyncData' from origin 'https://www.google.com' has been blocked by CORS policy: The value of the 'Access-Control-Allow-Origin' header in the response must not be the wildcard '*' when the request's credentials mode is 'include'. The credentials mode of requests initiated by the XMLHttpRequest is controlled by the withCredentials attribute.
ogads-pa.clients6.google.com/$rpc/google.internal.onegoogle.asyncdata.v1.AsyncDataService/GetAsyncData:1


   Failed to load resource: net::ERR_FAILED
search:1 Access to XMLHttpRequest at 'https://play.google.com/log?format=json&hasfast=true&authuser=0' from origin 'https://www.google.com' has been blocked by CORS policy: The value of the 'Access-Control-Allow-Origin' header in the response must not be the wildcard '*' when the request's credentials mode is 'include'. The credentials mode of requests initiated by the XMLHttpRequest is controlled by the withCredentials attribute.
play.google.com/log?format=json&hasfast=true&authuser=0:1


   Failed to load resource: net::ERR_FAILED
search:1 Access to XMLHttpRequest at 'https://play.google.com/log?format=json&hasfast=true&authuser=0' from origin 'https://www.google.com' has been blocked by CORS policy: The value of the 'Access-Control-Allow-Origin' header in the response must not be the wildcard '*' when the request's credentials mode is 'include'. The credentials mode of requests initiated by the XMLHttpRequest is controlled by the withCredentials attribute.
play.google.com/log?format=json&hasfast=true&authuser=0:1


   Failed to load resource: net::ERR_FAILED
search:1 Access to XMLHttpRequest at 'https://play.google.com/log?format=json&hasfast=true&authuser=0' from origin 'https://www.google.com' has been blocked by CORS policy: The value of the 'Access-Control-Allow-Origin' header in the response must not be the wildcard '*' when the request's credentials mode is 'include'. The credentials mode of requests initiated by the XMLHttpRequest is controlled by the withCredentials attribute.
play.google.com/log?format=json&hasfast=true&authuser=0:1


   Failed to load resource: net::ERR_FAILED
search:1 Access to XMLHttpRequest at 'https://play.google.com/log?format=json&hasfast=true&authuser=0' from origin 'https://www.google.com' has been blocked by CORS policy: The value of the 'Access-Control-Allow-Origin' header in the response must not be the wildcard '*' when the request's credentials mode is 'include'. The credentials mode of requests initiated by the XMLHttpRequest is controlled by the withCredentials attribute.
play.google.com/log?format=json&hasfast=true&authuser=0:1


   Failed to load resource: net::ERR_FAILED 이런 문제가 떴음 해결한 코드를 나한테 주셈 . 크롤링 속도 제한이나 페이지 수 제한은 절대 절대 하지말고