const express = require('express');
const sql = require('mssql');
const cors = require('cors');
const NodeCache = require('node-cache');
const fs = require('fs').promises;
const path = require('path');
const pdfParse = require('pdf-parse');
const { Worker } = require('worker_threads');
const WebSocket = require('ws');
const rateLimit = require('express-rate-limit');
const winston = require('winston');

// Configure logging
const logger = winston.createLogger({
    level: 'info',
    format: winston.format.combine(
        winston.format.timestamp(),
        winston.format.json()
    ),
    transports: [
        new winston.transports.File({ filename: 'error.log', level: 'error' }),
        new winston.transports.File({ filename: 'combined.log' })
    ]
});

if (process.env.NODE_ENV !== 'production') {
    logger.add(new winston.transports.Console({
        format: winston.format.simple()
    }));
}

// Initialize Express app
const app = express();
const wss = new WebSocket.Server({ port: 8080 });

// Middleware
app.use(cors());
app.use(express.json());
app.use(rateLimit({
    windowMs: 15 * 60 * 1000, // 15 minutes
    max: 100 // limit each IP to 100 requests per windowMs
}));

// Database configuration
const sqlconfig = {
    server: '192.168.2.5',
    database: 'ocr',
    user: 'sa',
    password: 'sa@12345',
    options: {
        encrypt: false,
        trustServerCertificate: true,
    },
    pool: {
        max: 20, // Increased pool size
        min: 0,
        idleTimeoutMillis: 30000,
    },
    requestTimeout: 300000, // Increased timeout
};

// Caching configuration
const searchCache = new NodeCache({
    stdTTL: 432000,
    checkperiod: 600,
});

// Active search sessions storage
const searchSessions = new Map();

// Database connection
const pool = new sql.ConnectionPool(sqlconfig);
const poolConnect = pool.connect().then(() => {
    logger.info('Database connected successfully');
}).catch(err => {
    logger.error('Database connection error:', err);
});

// PDF Worker Pool Implementation
class PdfWorkerPool {
    constructor(numWorkers = 4) {
        this.numWorkers = numWorkers;
        this.workers = [];
        this.taskQueue = [];
        this.initializeWorkers();
    }

    initializeWorkers() {
        for (let i = 0; i < this.numWorkers; i++) {
            const worker = new Worker(`
                const { parentPort } = require('worker_threads');
                const fs = require('fs').promises;
                const pdfParse = require('pdf-parse');
                const path = require('path');

                const getSnippet = (text, keyword, contextLength = 100) => {
                    const lowerText = text.toLowerCase();
                    const lowerKeyword = keyword.toLowerCase();
                    const startIndex = lowerText.indexOf(lowerKeyword);
                    if (startIndex === -1) return null;
                    const snippetStart = Math.max(0, startIndex - contextLength);
                    const snippetEnd = Math.min(text.length, startIndex + keyword.length + contextLength);
                    return \`...\${text.substring(snippetStart, snippetEnd)}...\`;
                };

                async function processFile(pdfPath, searchTerms) {
                    try {
                        const dataBuffer = await fs.readFile(pdfPath);
                        const pdfData = await pdfParse(dataBuffer);
                        const matches = [];
                        const pages = pdfData.text.split('\\n\\n');

                        pages.forEach((pageText, pageIndex) => {
                            searchTerms.forEach(({ term, isExact }) => {
                                if (isExact) {
                                    const regex = new RegExp(\`\\\\b\${term}\\\\b\`, 'i');
                                    if (regex.test(pageText)) {
                                        const snippet = getSnippet(pageText, term, 100);
                                        matches.push({ page: pageIndex, keyword: term, snippet, isExact: true });
                                    }
                                } else if (pageText.toLowerCase().includes(term.toLowerCase())) {
                                    const snippet = getSnippet(pageText, term, 100);
                                    matches.push({ page: pageIndex, keyword: term, snippet, isExact: false });
                                }
                            });
                        });

                        return matches;
                    } catch (error) {
                        throw error;
                    }
                }

                parentPort.on('message', async ({ pdfPath, searchTerms, taskId }) => {
                    try {
                        const matches = await processFile(pdfPath, searchTerms);
                        parentPort.postMessage({ taskId, matches, error: null });
                    } catch (error) {
                        parentPort.postMessage({ taskId, matches: [], error: error.message });
                    }
                });
            `, { eval: true });

            worker.busy = false;
            worker.on('error', error => {
                logger.error('Worker error:', error);
            });

            this.workers.push(worker);
        }
    }

    async processFile(pdfPath, searchTerms) {
        return new Promise((resolve, reject) => {
            const availableWorker = this.workers.find(w => !w.busy);
            if (!availableWorker) {
                this.taskQueue.push({ pdfPath, searchTerms, resolve, reject });
                return;
            }

            this.executeTask(availableWorker, { pdfPath, searchTerms, resolve, reject });
        });
    }

    executeTask(worker, { pdfPath, searchTerms, resolve, reject }) {
        worker.busy = true;
        const taskId = Math.random().toString(36).substring(7);

        worker.once('message', ({ taskId: responseId, matches, error }) => {
            worker.busy = false;
            if (error) {
                reject(error);
            } else {
                resolve(matches);
            }

            // Process next task in queue if any
            if (this.taskQueue.length > 0) {
                const nextTask = this.taskQueue.shift();
                this.executeTask(worker, nextTask);
            }
        });

        worker.postMessage({ pdfPath, searchTerms, taskId });
    }
}

const pdfWorkerPool = new PdfWorkerPool();

// WebSocket connection handler
wss.on('connection', (ws) => {
    logger.info('New WebSocket connection established');
    
    ws.on('message', (message) => {
        try {
            const { searchId, action } = JSON.parse(message);
            if (action === 'cancel' && searchSessions.has(searchId)) {
                const session = searchSessions.get(searchId);
                session.cancelled = true;
                searchSessions.delete(searchId);
                ws.send(JSON.stringify({ searchId, status: 'cancelled' }));
                logger.info(`Search ${searchId} cancelled by user`);
            }
        } catch (error) {
            logger.error('WebSocket message error:', error);
        }
    });
});

// Helper functions
const parseSearchTerms = (input) => {
    const terms = [];
    const regex = /"([^"]+)"|(\S+)/g;
    let match;
    
    while ((match = regex.exec(input)) !== null) {
        const term = match[1] || match[2];
        terms.push({
            term: term.trim(),
            isExact: match[1] !== undefined
        });
    }
    return terms;
};

const broadcastProgress = (searchId, processed, total, matchCount = 0) => {
    const message = JSON.stringify({
        searchId,
        status: 'progress',
        processed,
        total,
        matchCount,
        percentComplete: Math.round((processed / total) * 100)
    });

    wss.clients.forEach(client => {
        if (client.readyState === WebSocket.OPEN) {
            client.send(message);
        }
    });
};

const broadcastComplete = (searchId, results) => {
    const message = JSON.stringify({
        searchId,
        status: 'complete',
        results
    });

    wss.clients.forEach(client => {
        if (client.readyState === WebSocket.OPEN) {
            client.send(message);
        }
    });
};

// Search endpoint
app.get('/search', async (req, res) => {
    const { keyword, exclude } = req.query;
    const searchId = Math.random().toString(36).substring(7);

    if (!keyword || keyword.trim() === '') {
        return res.status(400).json({ error: 'No search keyword provided.' });
    }

    logger.info(`New search initiated - ID: ${searchId}, Keyword: ${keyword}`);

    // Initial response
    res.json({ 
        searchId,
        status: 'initiated',
        message: 'Search started. Connect to WebSocket for progress updates.'
    });

    // Create search session
    searchSessions.set(searchId, {
        cancelled: false,
        startTime: Date.now(),
        totalFiles: 0,
        processedFiles: 0
    });

    try {
        await poolConnect;
        const request = pool.request();
        const searchTerms = parseSearchTerms(keyword);
        let query = 'SELECT Filename FROM files WHERE 1=1';

        searchTerms.forEach((term, index) => {
            if (term.isExact) {
                query += ` AND (
                    PdfData LIKE @exactStart${index} OR 
                    PdfData LIKE @exactMiddle${index} OR 
                    PdfData LIKE @exactEnd${index}
                )`;
                request.input(`exactStart${index}`, sql.NVarChar, `% ${term.term} %`);
                request.input(`exactMiddle${index}`, sql.NVarChar, `% ${term.term} %`);
                request.input(`exactEnd${index}`, sql.NVarChar, `% ${term.term}`);
            } else {
                query += ` AND PdfData LIKE @term${index}`;
                request.input(`term${index}`, sql.NVarChar, `%${term.term}%`);
            }
        });

        if (exclude && exclude.trim() !== '') {
            const excludedWords = exclude.split(/\s+/).filter(Boolean);
            excludedWords.forEach((word, index) => {
                const sanitizedExclude = word.replace(/[^\w\s]/g, '').trim();
                if (sanitizedExclude) {
                    query += ` AND PdfData NOT LIKE @exclude${index}`;
                    request.input(`exclude${index}`, sql.NVarChar, `%${sanitizedExclude}%`);
                }
            });
        }

        logger.info(`Executing SQL Query for search ${searchId}`);
        const result = await request.query(query);
        const session = searchSessions.get(searchId);
        
        if (!session || session.cancelled) {
            logger.info(`Search ${searchId} was cancelled before processing`);
            return;
        }

        session.totalFiles = result.recordset.length;
        broadcastProgress(searchId, 0, session.totalFiles);

        // Process files in parallel batches
        const BATCH_SIZE = 20;
        const results = [];
        
        for (let i = 0; i < result.recordset.length; i += BATCH_SIZE) {
            if (session.cancelled) {
                logger.info(`Search ${searchId} cancelled during processing`);
                break;
            }

            const batch = result.recordset.slice(i, i + BATCH_SIZE);
            const batchPromises = batch.map(async record => {
                const fullPath = record.Filename;
                if (path.extname(fullPath).toLowerCase() !== '.pdf') return null;

                const relativePath = fullPath.replace('\\\\192.168.2.5\\c$\\GGN Office Repository\\', '');
                const absolutePath = `\\\\192.168.13.30\\c$\\GGN Office Repository\\${relativePath}`;

                try {
                    const matches = await pdfWorkerPool.processFile(absolutePath, searchTerms);
                    return matches.length > 0 ? { 
                        Filename: fullPath, 
                        relativePath, 
                        matches 
                    } : null;
                } catch (error) {
                    logger.error(`Error processing ${absolutePath}:`, error);
                    return null;
                }
            });
            
            const batchResults = await Promise.allSettled(batchPromises);
            const validResults = batchResults
                .filter(r => r.status === 'fulfilled' && r.value)
                .map(r => r.value);
            
            results.push(...validResults);
            
            session.processedFiles += batch.length;
            broadcastProgress(searchId, session.processedFiles, session.totalFiles, results.length);
        }

        broadcastComplete(searchId, results);
        searchSessions.delete(searchId);
        logger.info(`Search ${searchId} completed successfully`);

    } catch (error) {
        logger.error(`Error in search ${searchId}:`, error);
        wss.clients.forEach(client => {
            if (client.readyState === WebSocket.OPEN) {
                client.send(JSON.stringify({
                    searchId,
                    status: 'error',
                    error: error.message
                }));
            }
        });
        searchSessions.delete(searchId);
    }
});

// File serving endpoint
app.get('/files/*', (req, res) => {
    const relativePath = req.params[0];
    const basePath = '\\\\192.168.13.30\\c$\\GGN Office Repository';
    const filePath = path.join(basePath, relativePath);

    logger.info(`File request: ${filePath}`);

    fs.stat(filePath)
        .then(stats => {
            if (stats.isDirectory()) {
                return fs.readdir(filePath)
                    .then(files => {
                        const fileLinks = files.map(file => {
                            return `<li><a href="/files/${path.join(relativePath, file).replace(/\\/g, '/')}">${file}</a></li>`;
                        }).join('');
                        
                        res.send(`
                            <h1>Contents of ${relativePath}</h1>
                            <ul>${fileLinks}</ul>
                        `);
                    });
            } else if (stats.isFile()) {
                res.sendFile(filePath, (err) => {
                    if (err) {
                        logger.error(`Error sending file: ${filePath}`, err);
                        res.status(500).json({ error: 'Error sending file' });
                    }
                });
            } else {
                res.status(400).json({ error: 'Neither a file nor a directory' });
            }
        })
        .catch(err => {
            logger.error(`Error accessing path: ${filePath}`, err);
            res.status(404).json({ error: 'Path not found or inaccessible' });
        });
});

// Server initialization
const PORT = process.env.PORT || 3006;
app.listen(PORT, '192.168.7.221', () => {
    logger.info(`Server running at http://192.168.2.5:${PORT}`);   expalin what is happening through this cod