import fitz import os import logging import random from models import Paper, PaperProcessor import concurrent.futures from concurrent.futures import ThreadPoolExecutor def extract_text_from_pdf(filename): with fitz.open(filename) as pdf_document: text = "" for page in pdf_document: text += page.get_text() return text.encode('latin-1', 'replace').decode('latin-1') def process_paper(pdf_file, paper_dir, prompt_dir, api_keys): logging.info(f"Processing file type in process_paper: {type(pdf_file)}") logging.debug(f"Starting to process paper: {pdf_file}") os.makedirs(paper_dir, exist_ok=True) if isinstance(pdf_file, str): pdf_path = pdf_file elif hasattr(pdf_file, 'name') and hasattr(pdf_file, 'read'): pdf_path = os.path.join(paper_dir, pdf_file.name) with open(pdf_path, "wb") as f: f.write(pdf_file.read()) else: logging.error( "Received object is neither a path nor a file-like object.") return [], [] extracted_text = extract_text_from_pdf(pdf_path) paper = Paper(pdf_file.name if hasattr(pdf_file, 'name') else os.path.basename(pdf_path), extracted_text) models = ['gpt-4-turbo-2024-04-09', 'gpt-4o', 'claude-3-opus-20240229', 'gemini-pro', 'command-r-plus'] selected_models = random.sample(models, 2) reviews = [] def process_with_model(model): processor = PaperProcessor(prompt_dir, model, **api_keys) return processor.process_paper(paper) with ThreadPoolExecutor() as executor: future_to_model = {executor.submit(process_with_model, model): model for model in selected_models} for future in concurrent.futures.as_completed(future_to_model): model = future_to_model[future] try: review_text = future.result() reviews.append(review_text) except Exception as exc: logging.error(f"Model {model} generated an exception: {exc}") logging.debug(f"Reviews generated: {reviews}") return reviews, selected_models