// ==UserScript==
// @name         고도화된 다중 검색 엔진 크롤러
// @namespace    http://tampermonkey.net/
// @version      3.1
// @description  다양한 검색 엔진의 결과를 크롤링하여 데이터를 수집하고 표시합니다. UI 개선, 실시간 데이터 필터링 및 업데이트, 품질 평가 등 고도화된 기능을 포함합니다.
// @author       YourName
// @match        *://www.google.com/*
// @match        *://www.bing.com/*
// @match        *://duckduckgo.com/*
// @match        *://뉴토끼.site/*
// @match        *://lettercounter.net/*
// @match        *://tv.54.wiKi/*
// @grant        GM_setValue
// @grant        GM_getValue
// @grant        GM_addValueChangeListener
// @grant        GM_xmlhttpRequest
// @grant        GM_addStyle
// @require      https://code.jquery.com/jquery-3.6.0.min.js
// @require      https://cdn.datatables.net/1.13.4/js/jquery.dataTables.min.js
// @require      https://cdn.jsdelivr.net/npm/chart.js@4.3.0/dist/chart.umd.min.js
// @require      https://cdnjs.cloudflare.com/ajax/libs/lodash.js/4.17.21/lodash.min.js
// @require      https://cdn.jsdelivr.net/npm/sweetalert2@11
// @require      https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.29.4/moment.min.js
// @require      https://cdnjs.cloudflare.com/ajax/libs/popper.js/2.11.6/umd/popper.min.js
// @require      https://maxcdn.bootstrapcdn.com/bootstrap/4.5.2/js/bootstrap.min.js
// @require      https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/js/all.min.js
// @require      https://cdnjs.cloudflare.com/ajax/libs/bootstrap-toggle/2.2.2/js/bootstrap-toggle.min.js
// @require      https://cdnjs.cloudflare.com/ajax/libs/select2/4.0.13/js/select2.min.js
// @require      https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.11/clipboard.min.js
// @require      https://cdnjs.cloudflare.com/ajax/libs/toastr.js/latest/toastr.min.js
// @require      https://cdnjs.cloudflare.com/ajax/libs/axios/1.3.4/axios.min.js
// @connect      *
// @run-at       document-end
// ==/UserScript==

(function() {
    'use strict';

    // Inject CSS libraries
    const cssLinks = [
        'https://maxcdn.bootstrapcdn.com/bootstrap/4.5.2/css/bootstrap.min.css',
        'https://cdn.datatables.net/1.13.4/css/jquery.dataTables.min.css',
        'https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css',
        'https://cdnjs.cloudflare.com/ajax/libs/bootstrap-toggle/2.2.2/css/bootstrap-toggle.min.css',
        'https://cdnjs.cloudflare.com/ajax/libs/select2/4.0.13/css/select2.min.css',
        'https://cdnjs.cloudflare.com/ajax/libs/toastr.js/latest/toastr.min.css'
    ];

    cssLinks.forEach(link => {
        const el = document.createElement('link');
        el.rel = 'stylesheet';
        el.href = link;
        document.head.appendChild(el);
    });

    // Initialize stored values if undefined
    const initialValues = {
        'isCrawling': false,
        'crawlData': JSON.stringify([]),
        'currentPage': 1,
        'crawlDelay': 1000,
        'logData': JSON.stringify([]),
        'tvPage': 54,
        'blacklist': JSON.stringify([]),
        'whitelist': JSON.stringify([])
    };

    for (const [key, value] of Object.entries(initialValues)) {
        if (GM_getValue(key) === undefined) {
            GM_setValue(key, value);
        }
    }

    // Function to get list from storage
    function getList(key) {
        return JSON.parse(GM_getValue(key));
    }

    // Function to set list in storage
    function setList(key, list) {
        GM_setValue(key, JSON.stringify(list));
    }

    // Search Engines configuration with actual selectors
    const searchEngines = {
        'www.google.com': {
            name: 'Google',
            resultSelector: 'div.g',
            titleSelector: 'h3',
            linkSelector: 'a',
            descriptionSelector: 'span.aCOpRe, .VwiC3b',
            nextButtonSelector: '#pnnext'
        },
        'www.bing.com': {
            name: 'Bing',
            resultSelector: 'li.b_algo',
            titleSelector: 'h2',
            linkSelector: 'a',
            descriptionSelector: 'p',
            nextButtonSelector: 'a.sb_pagN'
        },
        'duckduckgo.com': {
            name: 'DuckDuckGo',
            resultSelector: 'div.result',
            titleSelector: 'h2 > a.result__a',
            linkSelector: 'a.result__a',
            descriptionSelector: '.result__snippet',
            nextButtonSelector: 'a.result--more__btn'
        },
        '뉴토끼.site': {
            name: '뉴토끼',
            resultSelector: '.search-result', // 실제 검색 결과 컨테이너 클래스
            titleSelector: '.result-title > a', // 실제 제목 링크 클래스
            linkSelector: '.result-title > a',
            descriptionSelector: '.result-description',
            nextButtonSelector: '.pagination-next'
        },
        'lettercounter.net': {
            name: 'LetterCounter',
            resultSelector: '.search-results .result-item',
            titleSelector: '.result-title > a',
            linkSelector: '.result-title > a',
            descriptionSelector: '.result-snippet',
            nextButtonSelector: '.pagination-next'
        },
        'tv.54.wiKi': {
            name: 'TV54',
            resultSelector: '.tv-search-result',
            titleSelector: '.tv-result-title > a',
            linkSelector: '.tv-result-title > a',
            descriptionSelector: '.tv-result-description',
            nextButtonSelector: '.tv-pagination-next'
        }
    };

    // Detect current search engine
    const currentHost = window.location.host;
    let engine = searchEngines[currentHost];
    if (!engine) {
        console.log('지원되지 않는 검색 엔진입니다.');
        return;
    }

    // Create UI
    if ($('#crawlUI').length === 0) {
        createUI();
    }

    // Initialize DataTable
    const table = $('#crawlResults').DataTable({
        "paging": true,
        "searching": false,
        "info": false,
        "autoWidth": false,
        "order": [[4, "desc"]], // Order by quality score descending
        "columns": [
            { "data": "name" },
            { "data": "url" },
            { "data": "description" },
            { "data": "htmlButton" },
            { "data": "qualityScore" },
            { "data": "category" }
        ]
    });

    // Initialize Toastr
    toastr.options = {
        "positionClass": "toast-bottom-right",
        "timeOut": "3000"
    };

    // Event Listeners
    setupEventListeners();

    // Update UI based on initial values
    updateUI();

    // Start crawling if already in progress
    if (GM_getValue('isCrawling')) {
        setTimeout(() => {
            crawlPage();
        }, GM_getValue('crawlDelay'));
    }

    // Function to create the UI
    function createUI() {
        const container = $(`
            <div id="crawlUI" class="p-3 mb-2 bg-light border" style="position:fixed; top:20px; right:20px; width:400px; z-index:10000; overflow-y:auto; max-height:90vh;">
                <h2 class="text-success">🔍 검색 엔진 크롤러</h2>
                
                <div class="form-group">
                    <label for="crawlDelayInput">🕒 크롤링 속도 (ms):</label>
                    <input type="number" class="form-control" id="crawlDelayInput" min="100" value="${GM_getValue('crawlDelay')}">
                </div>
                
                <div class="form-group">
                    <label for="searchFilterInput">🔎 실시간 필터:</label>
                    <input type="text" class="form-control" id="searchFilterInput" placeholder="키워드 입력">
                </div>
                
                <div class="form-group">
                    <button id="startButton" class="btn btn-success">▶️ 시작</button>
                    <button id="stopButton" class="btn btn-danger" disabled>⏹️ 중지</button>
                    <button id="resetButton" class="btn btn-secondary">🔄 초기화</button>
                </div>
                
                <div class="form-group">
                    <progress id="progressBar" value="0" max="100" class="w-100"></progress>
                    <span id="progressText">0% 완료</span>
                </div>
                
                <div class="form-group">
                    <div>📄 현재 페이지: <span id="currentPageIndicator">1</span></div>
                    <div>📺 현재 TV 페이지: <span id="currentTVPageIndicator">54</span></div>
                </div>
                
                <div class="form-group">
                    <h3 class="text-primary">📝 로그</h3>
                    <textarea id="crawlLog" class="form-control" rows="5" readonly></textarea>
                </div>
                
                <div class="form-group">
                    <h3 class="text-primary">📰 크롤링 결과</h3>
                    <table id="crawlResults" class="display" style="width:100%">
                        <thead>
                            <tr>
                                <th>사이트 이름</th>
                                <th>주소</th>
                                <th>설명</th>
                                <th>📄 HTML 보기</th>
                                <th>🗨️ 품질 점수</th>
                                <th>🗂️ 분류</th>
                            </tr>
                        </thead>
                        <tbody></tbody>
                    </table>
                </div>
                
                <div class="form-group">
                    <h3 class="text-primary">🎬 애니메이션 보기</h3>
                    <iframe src="https://linkkf.net/" width="100%" height="300px" style="border:1px solid #ccc; border-radius:4px;"></iframe>
                </div>
                
                <div class="form-group">
                    <h3 class="text-primary">📺 TV 사이트 보기</h3>
                    <iframe id="tvViewerFrame" src="https://tv.${GM_getValue('tvPage')}.wiKi/" width="100%" height="300px" style="border:1px solid #ccc; border-radius:4px;"></iframe>
                    <button id="refreshTVButton" class="btn btn-primary btn-block mt-2">🔄 TV 사이트 새로 고침</button>
                </div>
                
                <div class="form-group">
                    <h3 class="text-primary">⚙️ 설정</h3>
                    <div class="form-group">
                        <label for="blacklistInput">🚫 블랙리스트 도메인 (콤마로 구분):</label>
                        <input type="text" class="form-control" id="blacklistInput" value="${getList('blacklist').join(',')}">
                    </div>
                    <div class="form-group">
                        <label for="whitelistInput">✅ 화이트리스트 도메인 (콤마로 구분):</label>
                        <input type="text" class="form-control" id="whitelistInput" value="${getList('whitelist').join(',')}">
                    </div>
                    <button id="saveSettingsButton" class="btn btn-primary btn-block">🔧 설정 저장</button>
                </div>
            </div>
        `);

        $('body').append(container);
    }

    // Function to setup event listeners
    function setupEventListeners() {
        $('#startButton').on('click', startCrawling);
        $('#stopButton').on('click', stopCrawling);
        $('#resetButton').on('click', resetCrawling);
        $('#searchFilterInput').on('keyup', filterTable);
        $('#refreshTVButton').on('click', refreshTVSite);
        $('#saveSettingsButton').on('click', saveSettings);
        $('#crawlDelayInput').on('change', function() {
            const val = parseInt($(this).val());
            if (isNaN(val) || val < 100) {
                Swal.fire('유효한 숫자를 입력하세요 (최소 100ms).');
                $(this).val(GM_getValue('crawlDelay'));
            } else {
                GM_setValue('crawlDelay', val);
                log(`🕒 크롤링 속도 변경: ${val}ms`);
            }
        });

        // Event delegation for dynamic HTML 보기 buttons
        $(document).on('click', '.view-html', function() {
            const url = decodeURIComponent($(this).data('url'));
            viewHTML(url);
        });
    }

    // Function to save settings
    function saveSettings() {
        const blacklist = $('#blacklistInput').val().split(',').map(item => item.trim()).filter(item => item);
        const whitelist = $('#whitelistInput').val().split(',').map(item => item.trim()).filter(item => item);
        setList('blacklist', blacklist);
        setList('whitelist', whitelist);
        log('⚙️ 블랙리스트와 화이트리스트를 저장했습니다.');
        Swal.fire('저장 완료', '블랙리스트와 화이트리스트가 저장되었습니다.', 'success');
    }

    // Function to filter the DataTable
    function filterTable() {
        const keyword = $('#searchFilterInput').val().toLowerCase();
        table.search(keyword).draw();
    }

    // Function to update the UI
    function updateUI() {
        $('#startButton').prop('disabled', GM_getValue('isCrawling'));
        $('#stopButton').prop('disabled', !GM_getValue('isCrawling'));
        $('#progressBar').val(getProgressPercentage());
        $('#progressText').text(`${getProgressPercentage()}% 완료`);
        $('#currentPageIndicator').text(GM_getValue('currentPage'));
        $('#currentTVPageIndicator').text(GM_getValue('tvPage'));
    }

    // Function to evaluate content quality
    function evaluateContentQuality(description) {
        const wordCount = description.split(/\s+/).length;
        if (wordCount > 100) return 10;
        if (wordCount > 50) return 7;
        if (wordCount > 20) return 5;
        return 2;
    }

    // Function to classify keywords
    function classifyKeywords(description) {
        const categories = {
            '기술': ['tech', 'software', 'hardware', '기술', '소프트웨어', '하드웨어'],
            '엔터테인먼트': ['anime', 'music', 'movie', '애니', '음악', '영화'],
            '과학': ['physics', 'chemistry', 'biology', '물리학', '화학', '생물학'],
            '교육': ['learning', 'education', 'school', '학습', '교육', '학교'],
            '스포츠': ['football', 'basketball', 'tennis', '축구', '농구', '테니스']
        };
        const descLower = description.toLowerCase();
        const matchedCategories = [];
        for (const [category, keywords] of Object.entries(categories)) {
            for (const keyword of keywords) {
                if (descLower.includes(keyword.toLowerCase())) {
                    matchedCategories.push(category);
                    break;
                }
            }
        }
        return matchedCategories.length > 0 ? matchedCategories.join(', ') : '기타';
    }

    // Function to check if URL is allowed based on blacklist and whitelist
    function isAllowed(url) {
        const blacklist = getList('blacklist').map(d => d.toLowerCase());
        const whitelist = getList('whitelist').map(d => d.toLowerCase());
        const host = (new URL(url)).host.toLowerCase();
        if (whitelist.length > 0 && !whitelist.includes(host)) {
            return false;
        }
        if (blacklist.includes(host)) {
            return false;
        }
        return true;
    }

    // Function to start crawling
    function startCrawling() {
        if (GM_getValue('isCrawling')) return;
        GM_setValue('isCrawling', true);
        log('🔍 크롤링 시작');
        if (GM_getValue('currentPage') === 1) {
            GM_setValue('crawlData', JSON.stringify([]));
            clearTable();
            clearLog();
        }
        crawlPage();
    }

    // Function to stop crawling
    function stopCrawling() {
        if (!GM_getValue('isCrawling')) return;
        GM_setValue('isCrawling', false);
        log('⏹️ 크롤링 중지');
    }

    // Function to reset crawling
    function resetCrawling() {
        stopCrawling();
        GM_setValue('crawlData', JSON.stringify([]));
        GM_setValue('currentPage', 1);
        GM_setValue('tvPage', 54);
        clearTable();
        clearLog();
        $('#tvViewerFrame').attr('src', `https://tv.${GM_getValue('tvPage')}.wiKi/`);
        log('🔄 크롤링 초기화 완료');
    }

    // Function to clear table
    function clearTable() {
        table.clear().draw();
    }

    // Function to clear logs
    function clearLog() {
        GM_setValue('logData', JSON.stringify([]));
        $('#crawlLog').val('');
    }

    // Function to crawl a page
    function crawlPage() {
        if (!GM_getValue('isCrawling')) return;
        log(`📄 페이지 ${GM_getValue('currentPage')} 크롤링 중...`);
        const crawlData = JSON.parse(GM_getValue('crawlData'));
        const results = document.querySelectorAll(engine.resultSelector);
        results.forEach(result => {
            if (isAd(result)) return;
            const titleElement = result.querySelector(engine.titleSelector);
            const linkElement = result.querySelector(engine.linkSelector);
            const descriptionElement = result.querySelector(engine.descriptionSelector);
            if (titleElement && linkElement) {
                const title = titleElement.innerText.trim();
                const url = linkElement.href;
                const description = descriptionElement ? descriptionElement.innerText.trim() : '';
                if (!isAllowed(url)) return;
                if (!crawlData.some(item => item.url === url)) {
                    const qualityScore = evaluateContentQuality(description);
                    const category = classifyKeywords(description);
                    crawlData.push({ name: engine.name, url, description, qualityScore, category });
                    addRowToTable(engine.name, url, description, qualityScore, category);
                    log(`✅ 데이터 수집: ${title} - ${url}`);
                }
            }
        });
        GM_setValue('crawlData', JSON.stringify(crawlData));
        updateUI();

        // Proceed to next page
        const nextButton = document.querySelector(engine.nextButtonSelector);
        if (nextButton && GM_getValue('currentPage') < 100 && GM_getValue('isCrawling')) {
            GM_setValue('currentPage', GM_getValue('currentPage') + 1);
            updateUI();
            setTimeout(() => {
                nextButton.click();
            }, GM_getValue('crawlDelay'));
            log(`➡️ 다음 페이지로 이동: ${GM_getValue('currentPage')}`);
        } else {
            GM_setValue('isCrawling', false);
            log('🏁 크롤링 완료');
            Swal.fire('크롤링 완료', '크롤링이 완료되었습니다!', 'success');
        }
    }

    // Function to add a row to table
    function addRowToTable(name, url, description, qualityScore, category) {
        const htmlButton = `<button class="btn btn-info btn-sm view-html" data-url="${encodeURIComponent(url)}">📄 HTML 보기</button>`;
        table.row.add({
            name,
            url: `<a href="${url}" target="_blank">${url}</a>`,
            description,
            htmlButton,
            qualityScore,
            category
        }).draw(false);
    }

    // Function to view HTML
    function viewHTML(url) {
        log(`🔗 HTML 요청: ${url}`);
        GM_xmlhttpRequest({
            method: 'GET',
            url: url,
            onload: function(response) {
                if (response.status >= 200 && response.status < 300) {
                    displayHTML(response.responseText, url);
                    log(`✅ HTML 로드 완료: ${url}`);
                } else {
                    log(`❌ HTML 로드 실패 (${response.status}): ${url}`);
                    Swal.fire('HTML 로드 실패', `Status: ${response.status}`, 'error');
                }
            },
            onerror: function(err) {
                log(`⚠️ HTML 요청 에러: ${url}`);
                Swal.fire('HTML 요청 에러', `Error: ${err}`, 'error');
            }
        });
    }

    // Function to display HTML in viewer
    function displayHTML(htmlContent, url) {
        const htmlViewerContainer = $(`
            <div id="htmlViewerContainer" class="modal fade" tabindex="-1" role="dialog">
              <div class="modal-dialog modal-lg" role="document">
                <div class="modal-content">
                  <div class="modal-header">
                    <h5 class="modal-title">📄 HTML 보기 - ${url}</h5>
                    <button type="button" class="close" data-dismiss="modal" aria-label="닫기">
                      <span aria-hidden="true">&times;</span>
                    </button>
                  </div>
                  <div class="modal-body">
                    <iframe src="${URL.createObjectURL(new Blob([htmlContent], {type: 'text/html'}))}" width="100%" height="600px" style="border:none;"></iframe>
                  </div>
                  <div class="modal-footer">
                    <button type="button" class="btn btn-secondary" data-dismiss="modal">닫기</button>
                  </div>
                </div>
              </div>
            </div>
        `);
        $('body').append(htmlViewerContainer);
        htmlViewerContainer.modal('show');
        htmlViewerContainer.on('hidden.bs.modal', function () {
            $(this).remove();
        });
    }

    // Function to refresh TV site
    function refreshTVSite() {
        let currentTVPage = GM_getValue('tvPage');
        currentTVPage += 1;
        GM_setValue('tvPage', currentTVPage);
        $('#tvViewerFrame').attr('src', `https://tv.${currentTVPage}.wiKi/`);
        log(`📺 TV 사이트 새로 고침: https://tv.${currentTVPage}.wiKi/`);
    }

    // Function to save settings
    function saveSettings() {
        const blacklist = $('#blacklistInput').val().split(',').map(item => item.trim()).filter(item => item);
        const whitelist = $('#whitelistInput').val().split(',').map(item => item.trim()).filter(item => item);
        setList('blacklist', blacklist);
        setList('whitelist', whitelist);
        log('⚙️ 블랙리스트와 화이트리스트를 저장했습니다.');
        Swal.fire('저장 완료', '블랙리스트와 화이트리스트가 저장되었습니다.', 'success');
    }

    // Function to filter the DataTable
    function filterTable() {
        const keyword = $('#searchFilterInput').val().toLowerCase();
        table.search(keyword).draw();
    }

    // Function to update the UI
    function updateUI() {
        $('#startButton').prop('disabled', GM_getValue('isCrawling'));
        $('#stopButton').prop('disabled', !GM_getValue('isCrawling'));
        $('#progressBar').val(getProgressPercentage());
        $('#progressText').text(`${getProgressPercentage()}% 완료`);
        $('#currentPageIndicator').text(GM_getValue('currentPage'));
        $('#currentTVPageIndicator').text(GM_getValue('tvPage'));
    }

    // Function to evaluate content quality
    function evaluateContentQuality(description) {
        const wordCount = description.split(/\s+/).length;
        if (wordCount > 100) return 10;
        if (wordCount > 50) return 7;
        if (wordCount > 20) return 5;
        return 2;
    }

    // Function to classify keywords
    function classifyKeywords(description) {
        const categories = {
            '기술': ['tech', 'software', 'hardware', '기술', '소프트웨어', '하드웨어'],
            '엔터테인먼트': ['anime', 'music', 'movie', '애니', '음악', '영화'],
            '과학': ['physics', 'chemistry', 'biology', '물리학', '화학', '생물학'],
            '교육': ['learning', 'education', 'school', '학습', '교육', '학교'],
            '스포츠': ['football', 'basketball', 'tennis', '축구', '농구', '테니스']
        };
        const descLower = description.toLowerCase();
        const matchedCategories = [];
        for (const [category, keywords] of Object.entries(categories)) {
            for (const keyword of keywords) {
                if (descLower.includes(keyword.toLowerCase())) {
                    matchedCategories.push(category);
                    break;
                }
            }
        }
        return matchedCategories.length > 0 ? matchedCategories.join(', ') : '기타';
    }

    // Function to check if URL is allowed based on blacklist and whitelist
    function isAllowed(url) {
        const blacklist = getList('blacklist').map(d => d.toLowerCase());
        const whitelist = getList('whitelist').map(d => d.toLowerCase());
        const host = (new URL(url)).host.toLowerCase();
        if (whitelist.length > 0 && !whitelist.includes(host)) {
            return false;
        }
        if (blacklist.includes(host)) {
            return false;
        }
        return true;
    }

    // Function to start crawling
    function startCrawling() {
        if (GM_getValue('isCrawling')) return;
        GM_setValue('isCrawling', true);
        log('🔍 크롤링 시작');
        if (GM_getValue('currentPage') === 1) {
            GM_setValue('crawlData', JSON.stringify([]));
            clearTable();
            clearLog();
        }
        crawlPage();
    }

    // Function to stop crawling
    function stopCrawling() {
        if (!GM_getValue('isCrawling')) return;
        GM_setValue('isCrawling', false);
        log('⏹️ 크롤링 중지');
    }

    // Function to reset crawling
    function resetCrawling() {
        stopCrawling();
        GM_setValue('crawlData', JSON.stringify([]));
        GM_setValue('currentPage', 1);
        GM_setValue('tvPage', 54);
        clearTable();
        clearLog();
        $('#tvViewerFrame').attr('src', `https://tv.${GM_getValue('tvPage')}.wiKi/`);
        log('🔄 크롤링 초기화 완료');
    }

    // Function to clear table
    function clearTable() {
        table.clear().draw();
    }

    // Function to clear logs
    function clearLog() {
        GM_setValue('logData', JSON.stringify([]));
        $('#crawlLog').val('');
    }

    // Function to crawl a page
    function crawlPage() {
        if (!GM_getValue('isCrawling')) return;
        log(`📄 페이지 ${GM_getValue('currentPage')} 크롤링 중...`);
        const crawlData = JSON.parse(GM_getValue('crawlData'));
        const results = document.querySelectorAll(engine.resultSelector);
        results.forEach(result => {
            if (isAd(result)) return;
            const titleElement = result.querySelector(engine.titleSelector);
            const linkElement = result.querySelector(engine.linkSelector);
            const descriptionElement = result.querySelector(engine.descriptionSelector);
            if (titleElement && linkElement) {
                const title = titleElement.innerText.trim();
                const url = linkElement.href;
                const description = descriptionElement ? descriptionElement.innerText.trim() : '';
                if (!isAllowed(url)) return;
                if (!crawlData.some(item => item.url === url)) {
                    const qualityScore = evaluateContentQuality(description);
                    const category = classifyKeywords(description);
                    crawlData.push({ name: engine.name, url, description, qualityScore, category });
                    addRowToTable(engine.name, url, description, qualityScore, category);
                    log(`✅ 데이터 수집: ${title} - ${url}`);
                }
            }
        });
        GM_setValue('crawlData', JSON.stringify(crawlData));
        updateUI();

        // Proceed to next page
        const nextButton = document.querySelector(engine.nextButtonSelector);
        if (nextButton && GM_getValue('currentPage') < 100 && GM_getValue('isCrawling')) {
            GM_setValue('currentPage', GM_getValue('currentPage') + 1);
            updateUI();
            setTimeout(() => {
                nextButton.click();
                setTimeout(crawlPage, GM_getValue('crawlDelay'));
            }, GM_getValue('crawlDelay'));
            log(`➡️ 다음 페이지로 이동: ${GM_getValue('currentPage')}`);
        } else {
            GM_setValue('isCrawling', false);
            log('🏁 크롤링 완료');
            Swal.fire('크롤링 완료', '크롤링이 완료되었습니다!', 'success');
        }
    }

    // Function to add a row to table
    function addRowToTable(name, url, description, qualityScore, category) {
        const htmlButton = `<button class="btn btn-info btn-sm view-html" data-url="${encodeURIComponent(url)}">📄 HTML 보기</button>`;
        table.row.add({
            name,
            url: `<a href="${url}" target="_blank">${url}</a>`,
            description,
            htmlButton,
            qualityScore,
            category
        }).draw(false);
    }

    // Function to view HTML
    function viewHTML(url) {
        log(`🔗 HTML 요청: ${url}`);
        GM_xmlhttpRequest({
            method: 'GET',
            url: url,
            onload: function(response) {
                if (response.status >= 200 && response.status < 300) {
                    displayHTML(response.responseText, url);
                    log(`✅ HTML 로드 완료: ${url}`);
                } else {
                    log(`❌ HTML 로드 실패 (${response.status}): ${url}`);
                    Swal.fire('HTML 로드 실패', `Status: ${response.status}`, 'error');
                }
            },
            onerror: function(err) {
                log(`⚠️ HTML 요청 에러: ${url}`);
                Swal.fire('HTML 요청 에러', `Error: ${err}`, 'error');
            }
        });
    }

    // Function to display HTML in viewer
    function displayHTML(htmlContent, url) {
        const htmlViewerContainer = $(`
            <div id="htmlViewerContainer" class="modal fade" tabindex="-1" role="dialog">
              <div class="modal-dialog modal-xl" role="document">
                <div class="modal-content">
                  <div class="modal-header">
                    <h5 class="modal-title">📄 HTML 보기 - ${url}</h5>
                    <button type="button" class="close" data-dismiss="modal" aria-label="닫기">
                      <span aria-hidden="true">&times;</span>
                    </button>
                  </div>
                  <div class="modal-body">
                    <iframe src="${URL.createObjectURL(new Blob([htmlContent], {type: 'text/html'}))}" width="100%" height="600px" style="border:none;"></iframe>
                  </div>
                  <div class="modal-footer">
                    <button type="button" class="btn btn-secondary" data-dismiss="modal">닫기</button>
                  </div>
                </div>
              </div>
            </div>
        `);
        $('body').append(htmlViewerContainer);
        htmlViewerContainer.modal('show');
        htmlViewerContainer.on('hidden.bs.modal', function () {
            $(this).remove();
        });
    }

    // Function to refresh TV site
    function refreshTVSite() {
        let currentTVPage = GM_getValue('tvPage');
        currentTVPage += 1;
        GM_setValue('tvPage', currentTVPage);
        $('#tvViewerFrame').attr('src', `https://tv.${currentTVPage}.wiKi/`);
        log(`📺 TV 사이트 새로 고침: https://tv.${currentTVPage}.wiKi/`);
    }

    // Function to save settings
    function saveSettings() {
        const blacklist = $('#blacklistInput').val().split(',').map(item => item.trim()).filter(item => item);
        const whitelist = $('#whitelistInput').val().split(',').map(item => item.trim()).filter(item => item);
        setList('blacklist', blacklist);
        setList('whitelist', whitelist);
        log('⚙️ 블랙리스트와 화이트리스트를 저장했습니다.');
        Swal.fire('저장 완료', '블랙리스트와 화이트리스트가 저장되었습니다.', 'success');
    }

    // Function to log messages
    function log(message) {
        const timestamp = new Date().toLocaleTimeString();
        const logEntry = `[${timestamp}] ${message}`;
        let logData = JSON.parse(GM_getValue('logData'));
        logData.push(logEntry);
        GM_setValue('logData', JSON.stringify(logData));
        $('#crawlLog').val($('#crawlLog').val() + logEntry + '\n');
        $('#crawlLog').scrollTop($('#crawlLog')[0].scrollHeight);
        toastr.info(message);
    }

    // Function to check if element is an ad
    function isAd(element) {
        const adClasses = ['ad', 'ads', 'commercial'];
        return adClasses.some(cls => element.classList.contains(cls));
    }

    // Function to get progress percentage
    function getProgressPercentage() {
        const maxPages = 100;
        const currentPage = GM_getValue('currentPage');
        return Math.min(Math.floor((currentPage / maxPages) * 100), 100);
    }

    // Function to classify keywords
    // Already defined above

    // Function to evaluate content quality
    // Already defined above

})();                        ui개선해!!!!