Spaces:
Runtime error
Runtime error
File size: 3,203 Bytes
aaa7449 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 | import PyPDF2
from docx import Document
import os
from typing import List
class DocumentProcessor:
def __init__(self):
self.chunk_size = 200 # Smaller chunks for better matching
self.chunk_overlap = 50
def process_document(self, file_path: str) -> List[str]:
"""Process document and return text chunks"""
try:
file_extension = os.path.splitext(file_path)[1].lower()
if file_extension == '.pdf':
text = self._extract_pdf_text(file_path)
elif file_extension in ['.docx', '.doc']:
text = self._extract_docx_text(file_path)
elif file_extension == '.txt':
text = self._extract_txt_text(file_path)
else:
raise ValueError(f"Unsupported file type: {file_extension}")
# Split into chunks
chunks = self._split_text(text)
# Clean up temp file (but keep sample_policy.txt)
if os.path.exists(file_path) and "sample_policy" not in file_path:
os.remove(file_path)
return chunks
except Exception as e:
print(f"Error processing document: {e}")
return []
def _extract_txt_text(self, file_path: str) -> str:
"""Extract text from .txt file"""
try:
with open(file_path, 'r', encoding='utf-8') as file:
return file.read()
except Exception as e:
print(f"Error reading TXT: {e}")
return ""
def _extract_pdf_text(self, file_path: str) -> str:
"""Extract text from PDF"""
text = ""
try:
with open(file_path, 'rb') as file:
pdf_reader = PyPDF2.PdfReader(file)
for page in pdf_reader.pages:
text += page.extract_text() + "\n"
except Exception as e:
print(f"Error reading PDF: {e}")
return text
def _extract_docx_text(self, file_path: str) -> str:
"""Extract text from Word document"""
text = ""
try:
doc = Document(file_path)
for paragraph in doc.paragraphs:
text += paragraph.text + "\n"
except Exception as e:
print(f"Error reading DOCX: {e}")
return text
def _split_text(self, text: str) -> List[str]:
"""Split text into smaller, more focused chunks"""
# Split by lines first, then combine
lines = [line.strip() for line in text.split('\n') if line.strip()]
chunks = []
current_chunk = ""
for line in lines:
if len(current_chunk + line) < self.chunk_size:
current_chunk += line + " "
else:
if current_chunk.strip():
chunks.append(current_chunk.strip())
current_chunk = line + " "
# Add the last chunk
if current_chunk.strip():
chunks.append(current_chunk.strip())
return chunks |