import os
import re
import PyPDF2
import spacy
from neo4j import GraphDatabase
from apscheduler.schedulers.background import BackgroundScheduler
from flask import Flask, request, jsonify
from concurrent_log_handler import ConcurrentRotatingFileHandler
import logging
import pickle
import openai  # Install the OpenAI Python package to use GPT-3.5 API

# Set your GPT-3.5 API key
openai.api_key = "YOUR_GPT_3.5_API_KEY"

# Initialize spaCy with the custom NER model for domain-specific entities
nlp = spacy.load("en_core_web_sm")  # Replace with fine-tuned model if available

# Initialize Neo4j database connection
uri = "bolt://localhost:7687"
user = "your_username"
password = "your_password"
driver = GraphDatabase.driver(uri, auth=(user, password))

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s]: %(message)s",
    handlers=[logging.StreamHandler(), ConcurrentRotatingFileHandler("processing_log.log")]
)
logger = logging.getLogger(__name__)

app = Flask(__name__)

def extract_text_from_pdf(file_path):
    with open(file_path, "rb") as file:
        pdf_reader = PyPDF2.PdfFileReader(file)
        text = ""
        for page_num in range(pdf_reader.numPages):
            page = pdf_reader.getPage(page_num)
            text += page.extract_text()
        return text

def preprocess_text(text):
    # Perform text cleaning and preprocessing (e.g., lowercasing, punctuation removal)
    text = text.lower()
    text = re.sub(r"[^\w\s]", "", text)
    return text

def extract_entities(text):
    doc = nlp(text)
    entities = [(ent.text, ent.label_) for ent in doc.ents]
    return entities

def extract_relationships(text, entities):
    # Implement advanced relationship extraction logic using dependency parsing
    relationships = []
    return relationships

def populate_knowledge_graph(entities, relationships):
    with driver.session() as session:
        for entity, entity_type in entities:
            # Create nodes for entities in the knowledge graph with entity type
            cypher_query = (
                f"MERGE (e:Entity {{name: '{entity}', type: '{entity_type}'}})"
            )
            session.run(cypher_query)

        for source, target, relation_type in relationships:
            # Create relationships between entities in the knowledge graph
            cypher_query = (
                f"MATCH (source:Entity {{name: '{source}'}}), "
                f"(target:Entity {{name: '{target}'}}) "
                f"MERGE (source)-[:{relation_type}]->(target)"
            )
            session.run(cypher_query)

processed_pdfs = set()  # Create an empty set to hold names of processed PDFs



def process_pdf_file(file_path):
    try:
        logger.info(f"Processing {file_path}")
        text = extract_text_from_pdf(file_path)
        preprocessed_text = preprocess_text(text)
        entities = extract_entities(preprocessed_text)
        relationships = extract_relationships(preprocessed_text, entities)
        populate_knowledge_graph(entities, relationships)
        logger.info(f"Completed processing {file_path}")
    except Exception as e:
        logger.error(f"Error processing {file_path}: {e}")
try:
    with open("processed_pdfs.pkl", "rb") as f:
        processed_pdfs = pickle.load(f)
except (FileNotFoundError, EOFError):
    processed_pdfs = set()
    
def process_pdf_files(pdf_folder):
    global processed_pdfs
    all_pdfs = set(os.listdir(pdf_folder))
    
    # Filter out non-PDF files and previously processed PDFs
    new_pdfs = {f for f in all_pdfs if f.endswith(".pdf") and f not in processed_pdfs}
    
    for file_name in new_pdfs:
        file_path = os.path.join(pdf_folder, file_name)
        process_pdf_file(file_path)
        processed_pdfs.add(file_name)  # Add the processed PDF's name to the set
    
    # Save processed PDFs to a file after processing new ones
    with open("processed_pdfs.pkl", "wb") as f:
        pickle.dump(processed_pdfs, f)

    
def query_knowledge_graph(query):
    with driver.session() as session:
        cypher_query = (
            f"MATCH (e:Entity) WHERE e.name CONTAINS '{query}' "
            f"WITH e, rand() as r ORDER BY r LIMIT 5 "  # Limit to 5 random results for variety
            f"RETURN e.name, e.type"
        )
        result = session.run(cypher_query)
        entities = [{"name": record["e.name"], "type": record["e.type"]} for record in result]
        return entities

def generate_text(prompt):
    try:
        response = openai.Completion.create(
            engine="text-davinci-003",  # Use GPT-3.5 engine
            prompt=prompt,
            max_tokens=200  # Control the length of generated text
        )
        return response.choices[0].text.strip()
    except Exception as e:
        logger.error(f"Error generating text: {e}")
        return "Error generating text"

@app.route("/add_data", methods=["POST"])
def add_data():
    try:
        data = request.get_json()
        text = data.get("text", "")
        preprocessed_text = preprocess_text(text)
        entities = extract_entities(preprocessed_text)
        relationships = extract_relationships(preprocessed_text, entities)
        populate_knowledge_graph(entities, relationships)
        return jsonify({"message": "Data added successfully"})
    except Exception as e:
        logger.error(f"Error adding data: {e}")
        return jsonify({"message": "Error adding data", "error": str(e)}), 500

@app.route("/query", methods=["POST"])
def query():
    try:
        data = request.get_json()
        user_query = data.get("query", "")
        entities = query_knowledge_graph(user_query)
        return jsonify({"entities": entities})
    except Exception as e:
        logger.error(f"Error processing query: {e}")
        return jsonify({"message": "Error processing query", "error": str(e)}), 500

@app.route("/generate_text", methods=["POST"])
def generate_text_api():
    try:
        data = request.get_json()
        prompt = data.get("prompt", "")
        generated_text = generate_text(prompt)
        return jsonify({"generated_text": generated_text})
    except Exception as e:
        logger.error(f"Error generating text: {e}")
        return jsonify({"message": "Error generating text", "error": str(e)}), 500

if __name__ == "__main__":
    pdf_folder = "path/to/your/pdf/folder"

       # Initial processing of existing PDF files
    process_pdf_files(pdf_folder)

    # Schedule to process new PDF files every hour (you can adjust the interval as needed)
    scheduler = BackgroundScheduler()
    scheduler.add_job(process_pdf_files, "interval", args=[pdf_folder], hours=1)
    scheduler.start()

    app.run(debug=True)