import logging
import numpy as np
from pathlib import Path
from typing import Optional
from sentence_transformers import SentenceTransformer
from transformers import AutoTokenizer, pipeline
import tabula
import pytesseract
from pdf2image import convert_from_path
import re
import faiss
import pickle
from tqdm import tqdm
import torch

# Correct imports for your version of llama_index
from llama_index.core import StorageContext, Document
from llama_index.core.storage.docstore import SimpleDocumentStore
from llama_index.core.vector_stores import SimpleVectorStore

# Configure logging
logging.basicConfig(
    level=logging.DEBUG,  # Set to DEBUG for detailed logs
    format='%(asctime)s - %(levelname)s - %(message)s',
    handlers=[
        logging.StreamHandler()
    ]
)

# Initialize Sentence-BERT model and tokenizer
model_name = 'all-MiniLM-L6-v2'
logging.info(f"Loading SentenceTransformer model: {model_name}")
sentence_model = SentenceTransformer(model_name)
tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")

# Define directories
PDFS_DIRECTORY = Path('D:/Contracts')  # Replace with your actual path
STORAGE_PERSIST_DIR = Path("D:/Storage_test4")  # Replace with your actual path
FAISS_INDEX_PATH = STORAGE_PERSIST_DIR / "faiss_index.bin"
EMBEDDINGS_FILE = STORAGE_PERSIST_DIR / "document_embeddings.pkl"
DOC_METADATA_FILE = STORAGE_PERSIST_DIR / "document_metadata.pkl"

# Ensure the storage directory exists
STORAGE_PERSIST_DIR.mkdir(parents=True, exist_ok=True)

# Initialize variables to store embeddings and document metadata
all_embeddings = []
all_metadata = []

# Initialize summarizer (BART model)
logging.info("Initializing BART summarizer")
summarizer = pipeline(
    "summarization",
    model="facebook/bart-large-cnn",
    tokenizer="facebook/bart-large-cnn",
    device=0 if torch.cuda.is_available() else -1
)

# Define the embedding dimension
dimension = sentence_model.get_sentence_embedding_dimension()

# Summarize text using BART summarizer with detailed debugging
def summarize_text(text, max_length=200, min_length=30):
    if len(text) < min_length:
        logging.warning("Text too short for summarization.")
        return ""
    try:
        if not text.strip():
            logging.warning("Text is empty or whitespace-only, skipping summarization.")
            return ""

        # Set explicit maximum input length
        max_input_length = 1024  # Maximum tokens for BART

        # Tokenize the text
        tokens = summarizer.tokenizer.encode(text, truncation=False)
        total_tokens = len(tokens)

        if total_tokens > max_input_length:
            logging.debug(f"Text too long ({total_tokens} tokens), splitting into chunks.")

            # Split the tokens into manageable chunks
            chunks = []
            start_idx = 0
            while start_idx < total_tokens:
                end_idx = min(start_idx + max_input_length, total_tokens)
                chunk_tokens = tokens[start_idx:end_idx]
                chunk_text = summarizer.tokenizer.decode(chunk_tokens, skip_special_tokens=True)
                chunks.append(chunk_text)
                start_idx = end_idx

            # Summarize each chunk and combine the summaries
            summaries = []
            for chunk in chunks:
                summary = summarizer(
                    chunk,
                    max_length=max_length,
                    min_length=min_length,
                    do_sample=False
                )[0]['summary_text']
                summaries.append(summary)
            combined_summary = ' '.join(summaries)
            logging.info("Generated combined summary for text.")
            return combined_summary
        else:
            # Text is within the acceptable length
            summary = summarizer(
                text,
                max_length=max_length,
                min_length=min_length,
                do_sample=False
            )[0]['summary_text']
            logging.info("Generated summary for text.")
            return summary
    except Exception as e:
        logging.error(f"Error generating summary: {e}")
        return ""

# ... Rest of your code remains the same ...

# Remember to update any location where summarization is called, which should already be compatible.

if __name__ == "__main__":
    main()


2024-09-27 15:17:06,913 - INFO - Loading SentenceTransformer model: all-MiniLM-L6-v2
2024-09-27 15:17:06,924 - INFO - Use pytorch device_name: cpu
2024-09-27 15:17:06,925 - INFO - Load pretrained SentenceTransformer: all-MiniLM-L6-v2
C:\Users\FourieH\AppData\Local\anaconda3\Lib\site-packages\transformers\tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884
  warnings.warn(
2024-09-27 15:17:10,214 - INFO - Initializing BART summarizer
2024-09-27 15:17:16,982 - INFO - Found 2 client directories to process.
Processing Clients:   0%|          | 0/2 [00:00<?, ?it/s]2024-09-27 15:17:16,988 - INFO - Processing client: Absa
2024-09-27 15:17:16,992 - INFO - Processing PDF: D:\Contracts\Absa\ABSA 37 OCCUPATIONAL HEALTH AND SAFETY AGREEMENT MOBILISATION_v3.0.pdf
2024-09-27 15:17:16,992 - INFO - Converting PDF to images: D:\Contracts\Absa\ABSA 37 OCCUPATIONAL HEALTH AND SAFETY AGREEMENT MOBILISATION_v3.0.pdf
2024-09-27 15:17:23,201 - INFO - Extracting text from page 1
2024-09-27 15:17:24,714 - INFO - Extracted text from image.
2024-09-27 15:17:24,714 - INFO - Extracting text from page 2
2024-09-27 15:17:29,763 - INFO - Extracted text from image.
2024-09-27 15:17:29,778 - INFO - Extracting text from page 3
2024-09-27 15:17:35,199 - INFO - Extracted text from image.
2024-09-27 15:17:35,199 - INFO - Extracting text from page 4
2024-09-27 15:17:40,886 - INFO - Extracted text from image.
2024-09-27 15:17:40,886 - INFO - Extracting text from page 5
2024-09-27 15:17:46,280 - INFO - Extracted text from image.
2024-09-27 15:17:46,280 - INFO - Extracting text from page 6
2024-09-27 15:17:48,403 - INFO - Extracted text from image.
2024-09-27 15:17:48,419 - INFO - Extracted text from PDF: D:\Contracts\Absa\ABSA 37 OCCUPATIONAL HEALTH AND SAFETY AGREEMENT MOBILISATION_v3.0.pdf
2024-09-27 15:17:48,435 - INFO - Extracting tables from PDF: D:\Contracts\Absa\ABSA 37 OCCUPATIONAL HEALTH AND SAFETY AGREEMENT MOBILISATION_v3.0.pdf
2024-09-27 15:17:49,793 - WARNING - No tables found.
Token indices sequence length is longer than the specified maximum sequence length for this model (5644 > 512). Running this sequence through the model will result in indexing errors
2024-09-27 15:17:51,074 - INFO - Created 13 text chunks with overlapping.
Error displaying widget
Error displaying widget
Error displaying widget
Error displaying widget
Error displaying widget
Error displaying widget
Error displaying widget
Error displaying widget
Error displaying widget
Error displaying widget
Error displaying widget
Error displaying widget
Error displaying widget
2024-09-27 15:17:52,618 - INFO - Generating document-level summary for: ABSA 37 OCCUPATIONAL HEALTH AND SAFETY AGREEMENT MOBILISATION_v3.0
2024-09-27 15:17:52,761 - ERROR - Error generating summary: index out of range in self
2024-09-27 15:17:52,764 - INFO - Document-level summary generated successfully for: ABSA 37 OCCUPATIONAL HEALTH AND SAFETY AGREEMENT MOBILISATION_v3.0
2024-09-27 15:17:52,764 - INFO - Generating full document summary for: ABSA 37 OCCUPATIONAL HEALTH AND SAFETY AGREEMENT MOBILISATION_v3.0
2024-09-27 15:17:52,957 - ERROR - Error generating summary: index out of range in self
2024-09-27 15:17:52,957 - INFO - Full document summary generated successfully for: ABSA 37 OCCUPATIONAL HEALTH AND SAFETY AGREEMENT MOBILISATION_v3.0
2024-09-27 15:17:52,957 - WARNING - Received empty text for embedding.
2024-09-27 15:17:52,957 - WARNING - Received empty text for embedding.
2024-09-27 15:17:53,168 - INFO - Created 13 text chunks with overlapping.
2024-09-27 15:17:53,175 - INFO - Number of text chunks: 13
2024-09-27 15:17:53,191 - INFO - Number of tables: 0
2024-09-27 15:17:53,191 - INFO - Document ABSA 37 OCCUPAT