studyvariables

.gitignore

@@ -171,7 +171,4 @@ poetry.toml
 
 # LSP config files
 pyrightconfig.json
-
-# data
-# data/
-study_export_*
+phase-1.png

.gradio/certificate.pem

@@ -1,31 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
-TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
-cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
-WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
-ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
-MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
-h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
-0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
-A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
-T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
-B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
-B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
-KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
-OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
-jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
-qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
-rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
-HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
-hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
-ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
-3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
-NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
-ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
-TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
-jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
-oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
-4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
-mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
-emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
------END CERTIFICATE-----

app.py

@@ -1,146 +1,32 @@
-# app.py
-
-import csv
-
-import datetime
-
-# from datetime import datetime
-import io
 import json
-import logging
+from typing import List, Tuple
 import os
-from typing import Tuple, List, Any
 
 import gradio as gr
-import openai
 from dotenv import load_dotenv
 from slugify import slugify
 
-from config import STUDY_FILES, OPENAI_API_KEY
+from config import STUDY_FILES
 from rag.rag_pipeline import RAGPipeline
-from utils.helpers import (
-    append_to_study_files,
-    add_study_files_to_chromadb,
-    chromadb_client,
+from utils.helpers import generate_follow_up_questions, append_to_study_files
+from utils.prompts import (
+    highlight_prompt,
+    evidence_based_prompt,
+    sample_questions,
 )
-from utils.prompts import highlight_prompt, evidence_based_prompt
-from utils.zotero_manager import ZoteroManager
+import openai
 
-from interface import create_chat_interface
-from utils.pdf_processor import PDFProcessor
+from config import STUDY_FILES, OPENAI_API_KEY
+from utils.zotero_manager import ZoteroManager
 
-# Configure logging
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
 load_dotenv()
 
 openai.api_key = OPENAI_API_KEY
 
-# Initialize ChromaDB with study files
-add_study_files_to_chromadb("study_files.json", "study_files_collection")
-
 # Cache for RAG pipelines
 rag_cache = {}
 
-
-def get_rag_pipeline(study_name: str) -> RAGPipeline:
-    """Get or create a RAGPipeline instance for the given study by querying ChromaDB."""
-    if study_name not in rag_cache:
-        collection = chromadb_client.get_or_create_collection("study_files_collection")
-        result = collection.get(ids=[study_name])  # Retrieve document by ID
-
-        if not result or len(result["metadatas"]) == 0:
-            raise ValueError(f"Invalid study name: {study_name}")
-
-        study_file = result["metadatas"][0].get("file_path")
-        if not study_file:
-            raise ValueError(f"File path not found for study name: {study_name}")
-
-        rag_cache[study_name] = RAGPipeline(study_file)
-
-    return rag_cache[study_name]
-
-
-def get_study_info(study_name: str) -> str:
-    """Retrieve information about the specified study."""
-
-    collection = chromadb_client.get_or_create_collection("study_files_collection")
-    result = collection.get(ids=[study_name])  # Query by study name (as a list)
-    logging.info(f"Result: ======> {result}")
-
-    if not result or len(result["metadatas"]) == 0:
-        raise ValueError(f"Invalid study name: {study_name}")
-
-    study_file = result["metadatas"][0].get("file_path")
-    logging.info(f"study_file: =======> {study_file}")
-    if not study_file:
-        raise ValueError(f"File path not found for study name: {study_name}")
-
-    with open(study_file, "r") as f:
-        data = json.load(f)
-    return f"### Number of documents: {len(data)}"
-
-
-def markdown_table_to_csv(markdown_text: str) -> str:
-    """Convert a markdown table to CSV format."""
-    lines = [line.strip() for line in markdown_text.split("\n") if line.strip()]
-    table_lines = [line for line in lines if line.startswith("|")]
-
-    if not table_lines:
-        return ""
-
-    csv_data = []
-    for line in table_lines:
-        if "---" in line:
-            continue
-        # Split by |, remove empty strings, and strip whitespace
-        cells = [cell.strip() for cell in line.split("|") if cell.strip()]
-        csv_data.append(cells)
-
-    output = io.StringIO()
-    writer = csv.writer(output)
-    writer.writerows(csv_data)
-    return output.getvalue()
-
-
-def cleanup_temp_files():
-    """Clean up old temporary files."""
-    try:
-        current_time = datetime.datetime.now()
-        for file in os.listdir():
-            if file.startswith("study_export_") and file.endswith(".csv"):
-                file_time = datetime.datetime.fromtimestamp(os.path.getmtime(file))
-                # Calculate the time difference in seconds
-                time_difference = (current_time - file_time).total_seconds()
-                if time_difference > 20:  # 5 minutes in seconds
-                    try:
-                        os.remove(file)
-                    except Exception as e:
-                        logging.warning(f"Failed to remove temp file {file}: {e}")
-    except Exception as e:
-        logging.warning(f"Error during cleanup: {e}")
-
-
-def chat_function(message: str, study_name: str, prompt_type: str) -> str:
-    """Process a chat message and generate a response using the RAG pipeline."""
-
-    if not message.strip():
-        return "Please enter a valid query."
-
-    rag = get_rag_pipeline(study_name)
-    logging.info(f"rag: ==> {rag}")
-    prompt = {
-        "Highlight": highlight_prompt,
-        "Evidence-based": evidence_based_prompt,
-    }.get(prompt_type)
-
-    response, _ = rag.query(message, prompt_template=prompt)  # Unpack the tuple
-    return response
-
-
-def process_zotero_library_items(
-    zotero_library_id: str, zotero_api_access_key: str
-) -> str:
+def process_zotero_library_items(zotero_library_id: str, zotero_api_access_key: str) -> str:
     if not zotero_library_id or not zotero_api_access_key:
         return "Please enter your zotero library Id and API Access Key"
 
@@ -156,17 +42,11 @@ def process_zotero_library_items(
         )
 
         zotero_collections = zotero_manager.get_collections()
-        zotero_collection_lists = zotero_manager.list_zotero_collections(
-            zotero_collections
-        )
+        zotero_collection_lists = zotero_manager.list_zotero_collections(zotero_collections)
         filtered_zotero_collection_lists = (
-            zotero_manager.filter_and_return_collections_with_items(
-                zotero_collection_lists
-            )
+            zotero_manager.filter_and_return_collections_with_items(zotero_collection_lists)
         )
 
-        study_files_data = {}  # Dictionary to collect items for ChromaDB
-
         for collection in filtered_zotero_collection_lists:
             collection_name = collection.get("name")
             if collection_name not in STUDY_FILES:
@@ -175,362 +55,199 @@ def process_zotero_library_items(
                 zotero_collection_items = (
                     zotero_manager.get_collection_zotero_items_by_key(collection_key)
                 )
-                # Export zotero collection items to json
-                zotero_items_json = zotero_manager.zotero_items_to_json(
-                    zotero_collection_items
-                )
+                #### Export zotero collection items to json ####
+                zotero_items_json = zotero_manager.zotero_items_to_json(zotero_collection_items)
                 export_file = f"{slugify(collection_name)}_zotero_items.json"
                 zotero_manager.write_zotero_items_to_json_file(
                     zotero_items_json, f"data/{export_file}"
                 )
-                append_to_study_files(
-                    "study_files.json", collection_name, f"data/{export_file}"
-                )
-
-                # Collect for ChromaDB
-                study_files_data[collection_name] = f"data/{export_file}"
-
-                # Update in-memory STUDY_FILES for reference in current session
-                STUDY_FILES.update({collection_name: f"data/{export_file}"})
-                logging.info(f"STUDY_FILES: {STUDY_FILES}")
-
-        # After loop, add all collected data to ChromaDB
-        add_study_files_to_chromadb("study_files.json", "study_files_collection")
+                append_to_study_files("study_files.json", collection_name, f"data/{export_file}")
         message = "Successfully processed items in your zotero library"
     except Exception as e:
         message = f"Error process your zotero library: {str(e)}"
-
+    
     return message
 
 
-def process_multi_input(text, study_name, prompt_type):
-    # Split input based on commas and strip any extra spaces
-    variable_list = [word.strip().upper() for word in text.split(",")]
-    user_message = f"Extract and present in a tabular format the following variables for each {study_name} study: {', '.join(variable_list)}"
-    logging.info(f"User message: ==> {user_message}")
-    response = chat_function(user_message, study_name, prompt_type)
-    return [response, gr.update(visible=True)]
-
+def get_rag_pipeline(study_name: str) -> RAGPipeline:
+    """Get or create a RAGPipeline instance for the given study."""
+    if study_name not in rag_cache:
+        study_file = STUDY_FILES.get(study_name)
+        if not study_file:
+            raise ValueError(f"Invalid study name: {study_name}")
+        rag_cache[study_name] = RAGPipeline(study_file)
+    return rag_cache[study_name]
 
-def download_as_csv(markdown_content):
-    """Convert markdown table to CSV and provide for download."""
-    if not markdown_content:
-        return None
 
-    csv_content = markdown_table_to_csv(markdown_content)
-    if not csv_content:
-        return None
+def chat_function(
+    message: str, study_name: str, prompt_type: str
+) -> str:
+    """Process a chat message and generate a response using the RAG pipeline."""
 
-    # Create temporary file with actual content
-    timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
-    temp_path = f"study_export_{timestamp}.csv"
+    if not message.strip():
+        return "Please enter a valid query."
 
-    with open(temp_path, "w", newline="", encoding="utf-8") as f:
-        f.write(csv_content)
+    rag = get_rag_pipeline(study_name)
+    prompt = {
+        "Highlight": highlight_prompt,
+        "Evidence-based": evidence_based_prompt,
+    }.get(prompt_type)
 
-    return temp_path
+    response = rag.query(message, prompt_template=prompt)
+    return response.response
 
 
-# PDF Support
+def get_study_info(study_name: str) -> str:
+    """Retrieve information about the specified study."""
 
+    study_file = STUDY_FILES.get(study_name)
+    if not study_file:
+        return "Invalid study name"
 
-def process_pdf_uploads(files: List[gr.File], collection_name: str) -> str:
-    """Process uploaded PDF files and add them to the system."""
-    if not files or not collection_name:
-        return "Please upload PDF files and provide a collection name"
+    with open(study_file, "r") as f:
+        data = json.load(f)
+    return f"### Number of documents: {len(data)}"
 
-    try:
-        processor = PDFProcessor()
-
-        # Save uploaded files temporarily
-        file_paths = []
-        for file in files:
-            # Get the actual file path from the Gradio File object
-            if hasattr(file, "name"):  # If it's already a path
-                temp_path = file.name
-            else:  # If it needs to be saved
-                temp_path = os.path.join(processor.upload_dir, file.orig_name)
-                file.save(temp_path)
-            file_paths.append(temp_path)
-
-        # Process PDFs
-        output_path = processor.process_pdfs(file_paths, collection_name)
-
-        # Add to study files and ChromaDB
-        collection_id = f"pdf_{slugify(collection_name)}"
-        append_to_study_files("study_files.json", collection_id, output_path)
-        add_study_files_to_chromadb("study_files.json", "study_files_collection")
-
-        # Cleanup temporary files if they were created by us
-        for path in file_paths:
-            if path.startswith(processor.upload_dir):
-                try:
-                    os.remove(path)
-                except Exception as e:
-                    logger.warning(f"Failed to remove temporary file {path}: {e}")
-
-        return f"Successfully processed PDFs into collection: {collection_id}"
 
-    except Exception as e:
-        logger.error(f"Error in process_pdf_uploads: {str(e)}")
-        return f"Error processing PDF files: {str(e)}"
+def update_interface(study_name: str) -> Tuple[str, gr.update, gr.update, gr.update]:
+    """Update the interface based on the selected study."""
 
+    study_info = get_study_info(study_name)
+    questions = sample_questions.get(study_name, [])[:3]
+    if not questions:
+        questions = sample_questions.get("General", [])[:3]
+    visible_questions = [gr.update(visible=True, value=q) for q in questions]
+    hidden_questions = [gr.update(visible=False) for _ in range(3 - len(questions))]
+    return (study_info, *visible_questions, *hidden_questions)
 
-def chat_response(
-    message: str,
-    history: List[Tuple[str, str]],
-    study_name: str,
-    pdf_processor: PDFProcessor,
-) -> Tuple[List[Tuple[str, str]], str, Any]:
-    """Generate chat response and update history."""
-    if not message.strip():
-        return history, None, None
 
-    rag = get_rag_pipeline(study_name)
-    response, source_info = rag.query(message)
-    history.append((message, response))
-
-    # Generate PDF preview if source information is available
-    preview_image = None
-    if (
-        source_info
-        and source_info.get("source_file")
-        and source_info.get("page_numbers")
-    ):
-        try:
-            # Get the first page number from the source
-            page_num = source_info["page_numbers"][0]
-            preview_image = pdf_processor.render_page(
-                source_info["source_file"], int(page_num)
-            )
-        except Exception as e:
-            logger.error(f"Error generating PDF preview: {str(e)}")
+def set_question(question: str) -> str:
+    return question.lstrip("✨ ")
 
-    return history, preview_image
+def process_multi_input(text, study_name, prompt_type):
+    # Split input based on commas and strip any extra spaces
+    variable_list = [word.strip().upper() for word in text.split(',')]
+    user_message =f"Extract and present in a tabular format the following variables for each {study_name} study: {', '.join(variable_list)}"
+    response = chat_function(user_message, study_name, prompt_type)
+    return response
 
 
 def create_gr_interface() -> gr.Blocks:
-    """Create and configure the Gradio interface for the RAG platform."""
+    """
+    Create and configure the Gradio interface for the RAG platform.
+
+    This function sets up the entire user interface, including:
+    - Chat interface with message input and display
+    - Study selection dropdown
+    - Sample and follow-up question buttons
+    - Prompt type selection
+    - Event handlers for user interactions
+
+    Returns:
+        gr.Blocks: The configured Gradio interface ready for launching.
+    """
+
     with gr.Blocks() as demo:
         gr.Markdown("# ACRES RAG Platform")
+        
+        with gr.Row():
+            with gr.Column(scale=1):
+                gr.Markdown("### Zotero Credentials")
+                zotero_library_id = gr.Textbox(label="Zotero Library ID", type="password", placeholder="Enter Your Zotero Library ID here...")
+                zotero_api_access_key = gr.Textbox(label="Zotero API Access Key", type="password", placeholder="Enter Your Zotero API Access Key...")
+                process_zotero_btn = gr.Button("Process your Zotero Library")
+                zotero_output = gr.Markdown(label="Zotero")
+
+                gr.Markdown("### Study Information")
+                study_dropdown = gr.Dropdown(
+                    choices=list(STUDY_FILES.keys()),
+                    label="Select Study",
+                    value=list(STUDY_FILES.keys())[0],
+                )
+                study_info = gr.Markdown(label="Study Details")
 
-        with gr.Tabs() as tabs:
-            # Tab 1: Original Study Analysis Interface
-            with gr.Tab("Study Analysis"):
+                gr.Markdown("### Settings")
+                prompt_type = gr.Radio(
+                    ["Default", "Highlight", "Evidence-based"],
+                    label="Prompt Type",
+                    value="Default",
+                )
+                # clear = gr.Button("Clear Chat")
+            
+            with gr.Column(scale=3):
+                gr.Markdown("### Study Variables")
                 with gr.Row():
-                    with gr.Column(scale=1):
-                        gr.Markdown("### Zotero Credentials")
-                        zotero_library_id = gr.Textbox(
-                            label="Zotero Library ID",
-                            type="password",
-                            placeholder="Enter Your Zotero Library ID here...",
-                        )
-                        zotero_api_access_key = gr.Textbox(
-                            label="Zotero API Access Key",
-                            type="password",
-                            placeholder="Enter Your Zotero API Access Key...",
-                        )
-                        process_zotero_btn = gr.Button("Process your Zotero Library")
-                        zotero_output = gr.Markdown(label="Zotero")
-
-                        gr.Markdown("### Study Information")
-                        collection = chromadb_client.get_or_create_collection(
-                            "study_files_collection"
-                        )
-                        all_documents = collection.query(
-                            query_texts=[""], n_results=1000
-                        )
-                        study_choices = [
-                            doc_id
-                            for doc_id in all_documents.get("ids")[0]
-                            if all_documents
-                        ]
-
-                        study_dropdown = gr.Dropdown(
-                            choices=study_choices,
-                            label="Select Study",
-                            value=(study_choices[0] if study_choices else None),
-                        )
-                        study_info = gr.Markdown(label="Study Details")
-                        prompt_type = gr.Radio(
-                            ["Default", "Highlight", "Evidence-based"],
-                            label="Prompt Type",
-                            value="Default",
-                        )
-
-                    with gr.Column(scale=3):
-                        gr.Markdown("### Study Variables")
-                        with gr.Row():
-                            study_variables = gr.Textbox(
-                                show_label=False,
-                                placeholder="Type your variables separated by commas e.g (Study ID, Study Title, Authors etc)",
-                                scale=4,
-                                lines=1,
-                                autofocus=True,
-                            )
-                            submit_btn = gr.Button("Submit", scale=1)
-                        answer_output = gr.Markdown(label="Answer")
-                        download_btn = gr.DownloadButton(
-                            "Download as CSV",
-                            variant="primary",
-                            size="sm",
-                            scale=1,
-                            visible=False,
-                        )
-
-            # Tab 2: PDF Chat Interface
-            with gr.Tab("PDF Chat"):
-                pdf_processor = PDFProcessor()
+                    study_variables = gr.Textbox(
+                        show_label=False,
+                        placeholder="Type your variables separated by commas e.g (Study ID, Study Title, Authors etc)",
+                        scale=4,
+                        lines=1,
+                        autofocus=True,
+                    )
+                    submit_btn = gr.Button("Submit", scale=1)
+                answer_output = gr.Markdown(label="Answer")
+
+        def user(
+            user_message: str, history: List[List[str]]
+        ) -> Tuple[str, List[List[str]]]:
+            return "", (
+                history + [[user_message, None]] if user_message.strip() else history
+            )
 
-                with gr.Row():
-                    # Left column: Chat and Input
-                    with gr.Column(scale=7):
-                        chat_history = gr.Chatbot(
-                            value=[], height=600, show_label=False
-                        )
-                        with gr.Row():
-                            query_input = gr.Textbox(
-                                show_label=False,
-                                placeholder="Ask a question about your PDFs...",
-                                scale=8,
-                            )
-                            chat_submit_btn = gr.Button(
-                                "Send", scale=2, variant="primary"
-                            )
-
-                    # Right column: PDF Preview and Upload
-                    with gr.Column(scale=3):
-                        pdf_preview = gr.Image(label="Source Page", height=600)
-                        with gr.Row():
-                            pdf_files = gr.File(
-                                file_count="multiple",
-                                file_types=[".pdf"],
-                                label="Upload PDFs",
-                            )
-                        with gr.Row():
-                            collection_name = gr.Textbox(
-                                label="Collection Name",
-                                placeholder="Name this PDF collection...",
-                            )
-                        with gr.Row():
-                            upload_btn = gr.Button("Process PDFs", variant="primary")
-                        pdf_status = gr.Markdown()
-                        current_collection = gr.State(value=None)
-
-        # Event handlers for Study Analysis tab
-        process_zotero_btn.click(
-            process_zotero_library_items,
-            inputs=[zotero_library_id, zotero_api_access_key],
-            outputs=[zotero_output],
-        )
+        def bot(
+            history: List[List[str]], study_name: str, prompt_type: str
+        ) -> List[List[str]]:
+            """
+            Generate bot response and update the interface.
+
+            This function:
+            1. Processes the latest user message
+            2. Generates a response using the RAG pipeline
+            3. Updates the chat history
+            4. Generates follow-up questions
+            5. Prepares interface updates for follow-up buttons
+
+            Args:
+                history (List[List[str]]): The current chat history.
+                study_name (str): The name of the current study.
+                prompt_type (str): The type of prompt being used.
+
+            Returns:
+                Tuple[List[List[str]], gr.update, gr.update, gr.update]:
+                Updated chat history and interface components for follow-up questions.
+            """
+            if not history:
+                return history, [], [], []
+
+            user_message = history[-1][0]
+            bot_message = chat_function(user_message, history, study_name, prompt_type)
+            history[-1][1] = bot_message
+
+            return history
+
+        # msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then(
+        #     bot,
+        #     [chatbot, study_dropdown, prompt_type],
+        #     [chatbot, *follow_up_btns],
+        # )
+        # send_btn.click(user, [msg, chatbot], [msg, chatbot], queue=False).then(
+        #     bot,
+        #     [chatbot, study_dropdown, prompt_type],
+        #     [chatbot, *follow_up_btns],
+        # )
+        # for btn in follow_up_btns + sample_btns:
+        #     btn.click(set_question, inputs=[btn], outputs=[msg])
+
+        # clear.click(lambda: None, None, chatbot, queue=False)
 
         study_dropdown.change(
-            get_study_info, inputs=[study_dropdown], outputs=[study_info]
+            fn=get_study_info,
+            inputs=study_dropdown,
+            outputs=[study_info],
         )
 
-        submit_btn.click(
-            process_multi_input,
-            inputs=[study_variables, study_dropdown, prompt_type],
-            outputs=[answer_output, download_btn],
-        )
-
-        download_btn.click(
-            fn=download_as_csv, inputs=[answer_output], outputs=[download_btn]
-        ).then(fn=cleanup_temp_files, inputs=None, outputs=None)
-
-        # Event handlers for PDF Chat tab
-
-        def handle_pdf_upload(files, name):
-            if not name:
-                return "Please provide a collection name", None
-            if not files:
-                return "Please select PDF files", None
-
-            try:
-                result = process_pdf_uploads(files, name)
-                collection_id = f"pdf_{slugify(name)}"
-                return result, collection_id
-            except Exception as e:
-                logger.error(f"Error in handle_pdf_upload: {str(e)}")
-                return f"Error: {str(e)}", None
-
-        upload_btn.click(
-            handle_pdf_upload,
-            inputs=[pdf_files, collection_name],
-            outputs=[pdf_status, current_collection],
-        )
-
-        def add_message(history, message):
-            """Add user message to chat history."""
-            if not message.strip():
-                raise gr.Error("Please enter a message")
-            history = history + [(message, None)]
-            return history, "", None
-
-        def generate_chat_response(history, collection_id, pdf_processor):
-            """Generate response for the last message in history."""
-            if not collection_id:
-                raise gr.Error("Please upload PDFs first")
-            if len(history) == 0:
-                return history, None
-
-            last_message = history[-1][0]
-            try:
-                # Get response and source info
-                rag = get_rag_pipeline(collection_id)
-                response, source_info = rag.query(last_message)
-
-                # Generate preview if source information is available
-                preview_image = None
-                if (
-                    source_info
-                    and source_info.get("source_file")
-                    and source_info.get("page_number") is not None
-                ):
-                    try:
-                        page_num = source_info["page_number"]
-                        logger.info(f"Attempting to render page {page_num}")
-                        preview_image = pdf_processor.render_page(
-                            source_info["source_file"], page_num
-                        )
-                        if preview_image:
-                            logger.info(
-                                f"Successfully generated preview for page {page_num}"
-                            )
-                        else:
-                            logger.warning(
-                                f"Failed to generate preview for page {page_num}"
-                            )
-                    except Exception as e:
-                        logger.error(f"Error generating PDF preview: {str(e)}")
-                        preview_image = None
-
-                # Update history with response
-                history[-1] = (last_message, response)
-                return history, preview_image
-
-            except Exception as e:
-                logger.error(f"Error in generate_chat_response: {str(e)}")
-                history[-1] = (last_message, f"Error: {str(e)}")
-                return history, None
-
-        # Update PDF event handlers
-        upload_btn.click(  # Change from pdf_files.upload to upload_btn.click
-            handle_pdf_upload,
-            inputs=[pdf_files, collection_name],
-            outputs=[pdf_status, current_collection],
-        )
-
-        # Fixed chat event handling
-        chat_submit_btn.click(
-            add_message,
-            inputs=[chat_history, query_input],
-            outputs=[chat_history, query_input, pdf_preview],
-        ).success(
-            lambda h, c: generate_chat_response(h, c, pdf_processor),
-            inputs=[chat_history, current_collection],
-            outputs=[chat_history, pdf_preview],
-        )
+        process_zotero_btn.click(process_zotero_library_items, inputs=[zotero_library_id, zotero_api_access_key], outputs=[zotero_output], queue=False)
+        submit_btn.click(process_multi_input, inputs=[study_variables, study_dropdown, prompt_type], outputs=[answer_output], queue=False)
 
     return demo
 

config.py

@@ -1,5 +1,3 @@
-# config.py
-
 import os
 
 from dotenv import load_dotenv

data/exportedris-file-1-of-1-1_zotero_items.json

@@ -1,813 +0,0 @@
-[
-  {
-    "key": "B4DGMAWK",
-    "title": "Afghanistan Safety Nets Evaluation",
-    "abstract": "The primary objective of this evaluation is to estimate the causal impact of an unconditional cash transfer on the ability of the poorest families in select districts in Afghanistan to smooth consumption during seasonal shocks (e.g., winter-related, agricultural, and water shocks). Secondarily, the study will assess intervention effects on other economic outcomes, ownership and preservation of assets, child wellbeing (including nutrition, education, and refrainment from labor), migration, psychological well-being, and confidence in institutions. The target population consists of the poorest quintile of families with children under the age of five. Unconditional cash transfers will be provided to poor families in three installments: before, at the beginning, and at the end of the lean season. The annual benefit will amount to approximately 8,500 AFN (125 USD) for families with one more child under five; each payment installment is of equal amount.",
-    "full_text": "",
-    "authors": [
-      "Matthew Morton",
-      "Lucian Bucur Pop"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://ridie.3ieimpact.org/index.php?r=search/detailView&id=404"
-  },
-  {
-    "key": "P2VD3QWB",
-    "title": "Financial Education and Financial Access for Transnational Households: Field Experimental Evidence from the Philippines",
-    "abstract": "We implemented a randomized controlled trial among transnational households in the Philippines estimating impacts of a financial education treatment, a financial access treatment, and the combination of the two on financial behaviors. We test whether there are complementarities between both interventions and provide insight into the nature of constraints operating in financial services markets. We find no evidence of complementarities between the financial education and access treatments. In addition, while we find no evidence of constraints in access to formal credit and savings products, our results suggest that access constraints exist in the formal insurance market. Impacts on other financial behaviors are suggestive of the importance of information constraints in financial decision-making. These results provide guidance to designers of financial interventions in similar populations.",
-    "full_text": "",
-    "authors": [
-      "Abarcar P",
-      "Barua R",
-      "Yang D"
-    ],
-    "doi": "10.1086/703045",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": ""
-  },
-  {
-    "key": "QD6ZPX6I",
-    "title": "Labor Supply Responses to Large Social Transfers: Longitudinal Evidence from South Africa",
-    "abstract": "We quantify the labor supply responses of prime-aged adults to the presence of pensioners in their households, using longitudinal data collected in South Africa. We compare households and individuals before and after pension receipt and pension loss, which allows us to control for a host of unobservable household and individual characteristics that may determine labor market behavior. We,find large cash transfers to the elderly lead to increased employment among prime-aged adults, which occurs primarily through labor migration. The pension's impact is attributable to the increase in household resources it represents, which can be used to stake migrants until they become self-sufficient, and to the presence of pensioners who can care for small children, which allows prime-aged adults to look for work elsewhere. (JEL H23, H55, I38, J22, O15)",
-    "full_text": "",
-    "authors": [
-      "Ardington C",
-      "Case A",
-      "Hosegood V"
-    ],
-    "doi": "10.1257/app.1.1.22",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": ""
-  },
-  {
-    "key": "VUJDLBR5",
-    "title": "Aid programs' unintended effects: The case of Progresa and migration",
-    "abstract": "This paper analyzes the effect of aid on international and domestic migration and explores the causal effect of income on migration. The theoretical model predicts that the effect of aid \u2026",
-    "full_text": "",
-    "authors": [
-      "Angelucci M"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://papers.ssrn.com/sol3/papers.cfm?abstract_id=868646"
-  },
-  {
-    "key": "29J6T354",
-    "title": "Information, Intermediaries, and International Migration",
-    "abstract": "Job seekers face substantial information frictions, especially in international labor markets where intermediaries match prospective migrants with overseas employers. We conducted a randomized trial in Indonesia to explore how information about intermediary quality shapes migration outcomes. Holding access to information about the return to choosing a high-quality intermediary constant, intermediary-specific quality disclosure reduces the migration rate, cutting use of low-quality providers. Workers who do migrate receive better pre-departure preparation and have improved experiences abroad, despite no change in occupation or destination. These results are not driven by changes in beliefs about average provider quality or the return to migration. Nor does selection explain improved outcomes for those who migrate with quality disclosure. Together, our findings are consistent with an increase in the option value of search: with better ability to differentiate offer quality, workers search longer, select higher-quality intermediaries, and ultimately have better migration experiences.",
-    "full_text": "",
-    "authors": [
-      "Bazzi Samuel",
-      "Cameron Lisa",
-      "Schaner Simone",
-      "Witoelar Firman"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": ""
-  },
-  {
-    "key": "YIL3BQE5",
-    "title": "An Adaptive Targeted Field Experiment: Job Search Assistance for Refugees in Jordan",
-    "abstract": "We introduce an adaptive targeted treatment assignment methodology for field experiments. Our Tempered Thompson Algorithm balances the goals of maximizing the precision of treatment effect estimates and maximizing the welfare of experimental participants. A hierarchical Bayesian model allows us to adaptively target treatments. We implement our methodology in Jordan, testing policies to help Syrian refugees and local jobseekers to find work. The immediate employment impacts of a small cash grant, information and psychological support are small, but targeting raises employment by 1 percentage-point (20%). After four months, cash has a sizable effect on employment and earnings of Syrians.",
-    "full_text": "",
-    "authors": [
-      "Caria Stefano",
-      "Gordon Grant",
-      "Kasy Maximilian",
-      "Quinn Simon",
-      "Shami Soha",
-      "Teytelboym Alexander"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": ""
-  },
-  {
-    "key": "JINCDGIG",
-    "title": "Social Protection and Labor Market Outcomes of Youth in South Africa",
-    "abstract": "An Apartheid-driven spatial mismatch between workers and jobs leads to high job search costs for people living in rural areas of South Africa--costs that many young people cannot pay. In this article, the authors examine whether the arrival of a social grant--specifically a generous state-funded old-age pension given to men and women above prime age--enhances the ability of young men in rural areas to seek better work opportunities elsewhere. Based on eight waves of socioeconomic data on household living arrangements and household members' characteristics and employment status, collected between 2001 and 2011 at a demographic surveillance site in KwaZulu-Natal, the authors find that young men are significantly more likely to become labor migrants when someone in their household becomes age-eligible for the old-age pension. But this effect applies only to those who have completed high school (matric), who are on average 8 percentage points more likely to migrate for work when their households become pension eligible, compared with other potential labor migrants. The authors also find that, upon pension loss, it is the youngest migrants who are the most likely to return to their sending households, perhaps because they are the least likely to be self-sufficient at the time the pension is lost. The evidence is consistent with binding credit constraints limiting young men from poorer households from seeking more lucrative work elsewhere.",
-    "full_text": "",
-    "authors": [
-      "Ardington Cally",
-      "Barnighausen Till",
-      "Case Anne",
-      "Menendez Alicia"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": ""
-  },
-  {
-    "key": "F3ZV6MYR",
-    "title": "Unilateral Facilitation Does Not Raise International Labor Migration From The Philippines",
-    "abstract": "Significant income gains from migrating from poorer to richer countries have motivated unilateral (source-country) policies facilitating labor emigration. However, their effectiveness is unknown. We conducted a large-scale randomized experiment in the Philippines testing the impact of unilaterally facilitating international labor migration. Our most intensive treatment doubled the rate of job offers but had no identifiable effect on international labor migration. Even the highest overseas job-search rate we induced (22%), falls far short of the share initially expressing interest in migrating (34%). We conclude that unilateral migration facilitation will at most induce a trickle, not a flood, of additional emigration.",
-    "full_text": "",
-    "authors": [
-      "Beam Emily A",
-      "Mckenzie David",
-      "Yang Dean"
-    ],
-    "doi": "10.1086/683999",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.journals.uchicago.edu/doi/10.1086/683999"
-  },
-  {
-    "key": "7CUVDKRR",
-    "title": "Long-term impacts of the oportunidades conditional cash transfer program on rural youth in Mexico",
-    "abstract": "This paper studies the long-term effects of partici pation in the Mexican Oportunidades program on a\nvariety of outcomes and behaviors of rural youth in\nMexico. It analyzes data from a social experiment,\nwhich randomly phased-in the program in rural Mex ican villages. In 1997, 320 villages (the treatment\ngroup) were randomly selected for early incorpora tion into the program and 186 villages (the control\ngroup) were designated as a control group to be in corporated eighteen months later. This paper ex amines whether differential exposure to the program\nsignificantly impacted educational attainment, labor\nmarket outcomes, marriage, migration and cognitive\nachievement of youth. The results show positive im pacts of longer exposure on grades of schooling at tained, but no effects on achievement tests. With\nrespect to work, we find an overall reduction in work\nfor male youth.",
-    "full_text": "",
-    "authors": [
-      "Behrman J R",
-      "Parker S W",
-      "Todd P E"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": ""
-  },
-  {
-    "key": "PKYZCRJU",
-    "title": "Climate and Resilience Impact Evaluation Window: Experimental Evidence from Several Countries",
-    "abstract": "The concept of resilience has gained attention because it recognises the importance of addressing shorter-term humanitarian needs while simultaneously supporting communities to face future crises induced by climate change, conflict, and other factors. Many institutions, including the World Food Programme (WFP), have increasingly used the concept as a basis for their programming. WFP's integrated packages of interventions aim to improve food security and nutrition by smoothing food consumption in the short-term, while supporting livelihoods and addressing barriers to development (e.g., better climate information, access to markets, education, WASH, etc.) in the long-term. While all programme activities are potentially important for building resilience, livelihood activities are clearly connected to both immediate and future wellbeing. These activities include cash or in-kind transfers to the household and support for creating assets that could benefit the household or the community in the future. Therefore, livelihood activities have the potential to support households in improving and maintaining their wellbeing when facing future shocks and stressors. This pre-analysis plan describes policy experiments to estimate the impacts of experimentally varying WFP's activities on resilience as measured by community and household wellbeing. This approach follows others in conceptualizing resilience through changes in wellbeing (Knippenberg et al, 2019, Phadera et al. 2019, Jones and Tanner, 2017; Barrett et al, 2020). We design and run these experiments in the context of livelihoods programs implemented by the World Food Programme (WFP) across 6 countries. Beyond testing the overall impact of livelihood activities on wellbeing, a key ambition of this paper is to investigate whether activities themselves can be timed to accommodate households? vulnerability to seasonal fluctuations and shocks that are often connected to weather patterns and agricultural cycle. We identify two such mechanisms: adjusting the timing of cash transfers and labor requirements; and/or allowing for re-targeting participants over time to account for changes in vulnerability status.",
-    "full_text": "",
-    "authors": [],
-    "doi": "10.1257/rct.6851-3.1",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.socialscienceregistry.org/trials/6851"
-  },
-  {
-    "key": "IDE53VC5",
-    "title": "Migration policy: did an emergency provision displace standard rules? Evidence from Italy",
-    "abstract": "In 2011, to manage the exceptional flow of people escaping North Africa, the Italian government released the North Africa Emergency (Emergenza Nord Africa, ENA) provision, temporarily relaxing migration policies for some categories of asylum seekers. Using data from an important charity, we perform baseline difference-in-differences regressions to investigate the impact of this emergency rule on the probability of migrants regularizing their legal status. We exploit the timing of the enactment of the ENA accessibility criteriasuch that potential beneficiaries learned of its existence only after the realization of the state of entitlementto identify the effects of the emergency policy provision on treated and control groups of immigrants. The results show an increased number of successful applications in favor of eligible individuals, although a dramatic boost in the denial rate for other migrants is also observed. This suggests either that some migrants suffered displacement due to the emergency rule, and/or that improper submissions of ENA-oriented applications have occurred. We extend our analysis to the use of multilevel models to shed light on these possible (non-mutually excludable) explanations. Results seem to support the presence of some rule-displacement effects, although the existence of a set of wrongful submissions cannot be excluded. We discuss these possibilities from a policy perspective.",
-    "full_text": "",
-    "authors": [
-      "dalla Pellegrina L",
-      "Saraceno M",
-      "Suardi M"
-    ],
-    "doi": "10.1007/s40888-018-0128-0",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": ""
-  },
-  {
-    "key": "44ICCPGX",
-    "title": "Does the Role Model Encourage Female Labor Force Participation? Field Experiment in Bangladesh",
-    "abstract": "Enhancing female labor force participation is considered key to economic growth, poverty alleviation, and women?s empowerment. Despite its importance both at the national and household/individual levels, the female labor force participation rate remains low in South Asian countries. It has been a great interest for researchers and policy makers how to enhance female labor force participation in these countries. We investigate whether providing unmarried young women and their parents with information about working conditions through the role model working women encourages unmarried young women?s labor force participation in rural Bangladesh, and if so, how. For this objective, we conduct the randomized controlled trial.",
-    "full_text": "",
-    "authors": [],
-    "doi": "10.1257/rct.4940-2.0",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.socialscienceregistry.org/trials/4940"
-  },
-  {
-    "key": "AEMAYDTW",
-    "title": "Managing the impact of climate on migration: Evidence from Mexico",
-    "abstract": "Although there is a growing literature on the impact of climate and weather-related events on migration, little is known about the mitigating effect of policies directed toward the agricultural sector, or aimed at insuring against environmental disasters. This paper uses state-level data on migration flows between Mexico and the USA from 1999 to 2012 to investigate the mitigating impact of an agricultural cash transfer program (PROCAMPO) and a disaster fund (Fonden) on the migration response to weather shocks. We find that Fonden decreases migration in response to heavy rainfall, hurricanes and droughts. Increases in PROCAMPO amounts paid to small producers play a more ambiguous role in the migration response to shocks. Changes in the distribution of PROCAMPO payments favoring more vulnerable producers in the non-irrigated ejido sector, however, seem to mitigate the impact of droughts on migration.",
-    "full_text": "",
-    "authors": [
-      "Chort I",
-      "Rupelle M de La"
-    ],
-    "doi": "10.1007/s00148-022-00894-1",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://link.springer.com/article/10.1007/s00148-022-00894-1"
-  },
-  {
-    "key": "74WHPTAN",
-    "title": "Impact assessment of the Migrant Resource Centres in the Silk Routes Region",
-    "abstract": "Executive summary Several Migrant Resource Centres (MRCs) have been set up in recent years jointly by national administrations of origin and destination countries and the \u2026",
-    "full_text": "",
-    "authors": [
-      "Dennison J"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.budapestprocess.org/wp-content/uploads/2022/12/ICMPD_MRC_impact_assessment_2022.pdf"
-  },
-  {
-    "key": "EYAF27EQ",
-    "title": "Employment and Irregular Migration: Evidence from Two Randomized Controlled Trials in Egypt",
-    "abstract": "Addressing the root causes of irregular migration has become a key policy priority in Europe. The EU Emergency Trust Fund for Africa (EUTF) was launched in 2015 with a budget of 5 billion euros to support projects aiming at deterring irregular migration flows from 26 origin countries. One of these projects is implemented by the Egyptian Micro, Small and Medium Enterprises Development Agency (MSMEDA) and targets unemployed youth in areas with the highest outflows of irregular migrants. We implement two randomized evaluations to assess whether (i) cash-for-work opportunities, and (ii) training and employment support have the intended effects on the direct beneficiaries and their relatives (household decision-maker, other household members, children, and friends). We assess impacts on people?s preferences and attitudes towards migration, irregular migration, as well as changes in their situation, aspirations, and expectations.",
-    "full_text": "",
-    "authors": [],
-    "doi": "10.1257/rct.10604-1.0",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.socialscienceregistry.org/trials/10604"
-  },
-  {
-    "key": "BBGCWFN6",
-    "title": "Enhancing Female Entrepreneurship through Cash Grants: Experimental Evidence from Rural Tunisia",
-    "abstract": "This research is a product of the World Bank's Middle East and North Africa Gender Innovation Lab (MNAGIL), which conducts rigorous impact evaluations and inferential \u2026",
-    "full_text": "",
-    "authors": [
-      "Zineb SB"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://ericmvukiyehe.com/wp-content/uploads/2021/08/3-Enhancing-Female-Entrepreneurship-through-Cash-Grants-Experimental-Evidence-from-Rural-Tunisia-1.pdf"
-  },
-  {
-    "key": "LU7SCHNY",
-    "title": "Estimating the Impacts of Volunteer vs. Paraprofessional Community Worker Interventions among Conditional Cash Transfer Recipients in the Dominican Republic",
-    "abstract": "This project seeks to estimate the impacts of providing social intermediation services to poor recipients of conditional cash transfers (CCT) in the Dominican Republic (DR). Employing a randomized control trial design, the project will randomly assign either i) referred volunteer, ii) publicly-recruited volunteer, or iii) university-recruited paid paraprofessional \"community workers\" to provide monthly visits to newly enrolled and existing CCT beneficiaries. Households in one arm of the study will only receive cash transfers and will not receive home visits by community workers and an additional arm will include households that are pure controls. Key outcomes of interest include beneficiaries' compliance with program conditionalities, household consumption, primary and secondary school attendance, use of health and other local public services, participation in vocational training, labor market participation, and knowledge of the wellness curriculum administered during the monthly household visits. Data on these key outcomes will be collected both administratively by the DR government and through conducting midline and endline household surveys. In addition to estimating impacts on beneficiary households, the project is designed to estimate the impacts of the community worker employment experience on community worker candidates themselves in terms of their own income, political and community engagement, leadership activities, and subjective wellbeing. Specifically, volunteer and paid employment offers will be randomly offered among those community worker candidates screened into the selection process.",
-    "full_text": "",
-    "authors": [],
-    "doi": "10.1257/rct.1778-1.0",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.socialscienceregistry.org/trials/1778"
-  },
-  {
-    "key": "CDAYBE26",
-    "title": "Evaluation of a Program for the Professionalization of Artisans (ProfArts) in Ghana",
-    "abstract": "Training and professionalization interventions are an important vehicle for economic support within the development assistance landscape. Yet rigorous quantitative impact evaluations of such programs remain scarce, especially in developing countries. In order to help fill this research gap, we will conduct a rigorous impact evaluation of a program for the Professionalization of Artisans (ProfArts) in Ghana. The program will deliver top-up training, licensing, certification, and related benefits to up to 10,000 artisans drawn from the Ghanaian construction sector, with beneficiaries to be randomly selected from up to 20,000 baseline respondents. In a first step, we examine the effects of randomly assigned recruitment content on application rates, the composition of the applicant pool, and downstream program outcomes. In a second step, we use a randomized controlled trial to estimate effects of the program on four groups of outcomes: (i) employment, measured e.g. in terms of job retention, acquisition and lengths of employment spells, (ii) job quality and quality of life, including e.g. earnings and workplace conditions, (iii) mobility, e.g. migration intentions and behaviors, and (iv) firm-level outcomes, e.g. firm performance and employment.",
-    "full_text": "",
-    "authors": [],
-    "doi": "10.1257/rct.6842-1.0",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.socialscienceregistry.org/trials/6842"
-  },
-  {
-    "key": "QMSMF872",
-    "title": "Evaluation of the Network for Enterprise Development Learning through Sewing for Girls (N4G) training program for young women in Ghana",
-    "abstract": "Skills trainings are often a central component of contemporary development assistance strategies, but much of the evidence base for their efficacy comes from high-income countries. RCTs in developing countries are still far less numerous. We contribute to this literature with a rigorous impact evaluation of a program for the empowerment and education of young women in Ghana, called Network for Enterprise Development Learning through Sewing for Girls (N4G). The program will deliver fashion-industry related training and empowerment programs to up to 1,000 underprivileged Ghanaian young women from both urban and rural areas. Beneficiaries are randomly selected from approximately 2,000 baseline respondents. We use a randomized controlled trial to estimate effects of the N4G program on four groups of outcomes: (i) employment, measured for example in terms of employment status, job acquisition, and lengths of employment spells, (ii) job quality, including earnings, having a contract or an oral agreement, job satisfaction, and workplace conditions among others, (iii) quality of life, covering indicators of the current living situation, household assets, and financial dependency, and (iv) mobility, e.g. migration intentions and behaviors.",
-    "full_text": "",
-    "authors": [],
-    "doi": "10.1257/rct.7967-1.0",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.socialscienceregistry.org/trials/7967"
-  },
-  {
-    "key": "2WUKVSZA",
-    "title": "The Impact of Tourist Visas on Intercontinental South-South Migration: Ecuador\u2019s Policy of \u201cOpen Doors\u201d as a Quasi-Experiment",
-    "abstract": "Through the implementation of universal visa freedom from 2008 to 2010, Ecuador became one of the most accessible countries in the world. This article employs mixed methods to study the impact of the de facto opening of Ecuador\u2019s borders on intercontinental south-south migration. First, we use a difference-in-difference design to show that Ecuador\u2019s policy of universal visa freedom led to a significant increase of immigration from previously restricted nationalities in Africa, Asia, and the Caribbean. Complementary descriptive statistics and qualitative findings confirm the decisive impact visa freedom had on intercontinental south-south migration and suggest three main motives: taking advantage of Ecuador\u2019s open doors as an exit option from origin countries, settlement in Ecuador based on relatively improved opportunities, and transmigration to third countries. Our findings imply that travel visa policies of southern countries significantly impact which new south-south flows emerge. \u00a9 The Author(s) 2018.",
-    "full_text": "",
-    "authors": [
-      "Freier L F",
-      "Holloway K"
-    ],
-    "doi": "10.1177/0197918318801068",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.scopus.com/inward/record.uri?eid=2-s2.0-85059900710&doi=10.1177%2f0197918318801068&partnerID=40&md5=60533a0c145229926d2e61b312680c19"
-  },
-  {
-    "key": "YK492G5P",
-    "title": "The REFANI Pakistan study\u2014a cluster randomised controlled trial of the effectiveness and cost-effectiveness of cash-based transfer programmes on child nutrition status: study protocol",
-    "abstract": "Cash-based transfer programmes are an emerging strategy in the prevention of wasting in children, especially targeted at vulnerable households during periods of food insecurity or during emergencies. However, the evidence surrounding the use of either cash or voucher transfer programmes in the humanitarian context and on nutritional outcomes is elusive. More evidence is needed not only to inform the global community of practice on best practices in humanitarian settings, but also to help strengthen national mitigation responses. Methods/Design The Research for Food Assistance on Nutrition Impact Pakistan study (REFANI-P) sets out to evaluate the impact of three cash-based interventions on nutritional outcomes in children aged less than five years from poor and very poor households in Dadu District. This four-arm parallel cluster randomised controlled trial is set among Action Against Hunger (ACF) programme villages in Dadu District, Sindh Province. Mothers are the target recipients of either seasonal unconditional cash transfers or fresh food vouchers. A comparison group receives \u2018standard care\u2019 provided by the ACF programme to which all groups have the same access. The primary outcomes are prevalence of wasting and mean weight-for-height Z-score (WHZ) in children. Impact will be assessed at 6 months and at 1 year from baseline. Using a theory-based approach we will determine \u2018how\u2019 the different interventions work by looking at the processes involved and the impact pathways following the theory of change developed for this context. Quantitative and qualitative data are collected on morbidity, health seeking, hygiene and nutrition behaviours, dietary diversity, haemoglobin concentration, women\u2019s empowerment, household food security and expenditures and social capital. The direct and indirect costs of each intervention borne by the implementing organisation and their partners as well as by beneficiaries and their communities are also assessed. Discussion The results of this trial will provide robust evidence to help increase knowledge about the predictability of how different modalities of cash-based transfer work best to reduce the risk of child wasting during a season where food insecurity is at its highest. Evidence on costing and cost-effectiveness will further aid decisions on choice of modality in terms of effectiveness and sustainability.",
-    "full_text": "",
-    "authors": [
-      "Fenn et al"
-    ],
-    "doi": "10.1186/s12889-015-2380-3",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://bmcpublichealth.biomedcentral.com/articles/10.1186/s12889-015-2380-3"
-  },
-  {
-    "key": "4WXF37WF",
-    "title": "How Overseas Opportunities Shape Political Preferences: A Field Experiment on International Migration",
-    "abstract": "This paper demonstrates that access to overseas employment reduces support for taxation and redistribution by bolstering individuals\u2019 economic prospects. We present results from the \ufb01rst randomized controlled trial to result in international migration. Individuals who received the opportunity to migrate from India to the Middle East for work reported signi\ufb01cantly higher wages, greater economic con\ufb01dence, and more \ufb01scally conservative attitudes. Moreover, the program had lasting effects even for those who decided not to migrate, which we link to improved exit options. Our results speak to longstanding debates about the impact of globalization on economic development and welfare state politics.",
-    "full_text": "",
-    "authors": [
-      "Gaikwad N",
-      "Hanson K",
-      "Toth A"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://papers.ssrn.com/sol3/papers.cfm?abstract_id=3816464"
-  },
-  {
-    "key": "68ATB2KG",
-    "title": "How Migrant Resource Centres affect migration decisions: Quasi-experimental evidence from Afghanistan, Bangladesh, Iraq and Pakistan",
-    "abstract": "Several Migrant Resource Centres (MRCs) have been set up in South Asia jointly by national governments of origin and destination countries. Their objectives include encouragement of potential migrants to seek regular rather than irregular routes and to ensure the safety of those migrating, regardless of status. Of theoretical note, their activities utilise innovative, highly personalised counselling. This article provides quasi-experimental evidence of the effect of four activities - telephone and online counselling, and college outreach and pre-departure sessions - on 2215 randomly allocated users of the MRCs' services across six MRCs in four countries: Afghanistan, Bangladesh, Iraq and Pakistan. The effects are a large reduction in self-reported likelihood of migrating irregularly, and a strong increase in awareness of safe options and who to contact for assistance whilst migrating. The effects are consistent across activity type and MRC location, although magnitudes vary. These findings have implications for our understanding of how the decision to migrate is made, what interventions are effective and why.",
-    "full_text": "",
-    "authors": [
-      "Dennison J"
-    ],
-    "doi": "10.1111/imig.13082",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": ""
-  },
-  {
-    "key": "9YRUKMA5",
-    "title": "The Apprenticeship-to-Work Transition : Experimental Evidence from Ghana",
-    "abstract": "This paper examines the effects of a government-sponsored apprenticeship training program designed to address high levels of youth unemployment in Ghana. The study exploits randomized access to the program to examine the short-run effects of apprenticeship training on labor market outcomes. The results show that apprenticeships shift youth out of wage work and into self-employment. However, the loss of wage income is not offset by increases in self-employment profits in the short run. In addition, the study uses the randomized match between apprentices and training providers to examine the causal effect of characteristics of trainers on outcomes for apprentices. Participants who trained with the most experienced trainers or the most profitable ones had higher earnings. These increases more than offset the program's negative treatment effect on earnings. This suggests that training programs can be made more effective through better recruitment of trainers.",
-    "full_text": "",
-    "authors": [
-      "Hardy Morgan L",
-      "Mbiti Isaac Mulangu",
-      "Mccasland Jamie Lee",
-      "Salcher Isabelle"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.proquest.com/working-papers/apprenticeship-work-transition-experimental/docview/2223061072/se-2 http://UnivofPretoria.on.worldcat.org/atoztitles/link?sid=ProQ:&issn=&volume=&issue=&title=IDEAS+Working+Paper+Series+from+RePEc&spage=&date=2019-01-01&atitle=The+Apprenticeship-to-Work+Transition+%3A+Experimental+Evidence+from+Ghana&au=Hardy%2C+Morgan+L%3BMbiti%2C+Isaac+Mulangu%3BMccasland%2C+Jamie+Lee%3BSalcher%2C+Isabelle&id=doi: https://ideas.repec.org/p/wbk/wbrwps/8851.html"
-  },
-  {
-    "key": "2YVH2MHU",
-    "title": "Get Rich or Die Tryin\u2019: Perceived Earnings, Perceived Mortality Rates, and Migration Decisions of Potential Work Migrants from Nepal",
-    "abstract": "This article reports on a randomized field experiment in which potential work migrants from Nepal to Malaysia and the Persian Gulf countries are provided with information on wages and mortality incidences at their intended destinations. It is found that, particularly for the group of potential migrants without prior foreign migration experience, the information changes their expectations of earnings and mortality risks abroad, which further changes their actual migration decisions. Using the exogenous variation in expectations, it is estimated that the elasticity of migration with respect to mortality rate expectation is 0.8, and the elasticity of migration with respect to earnings expectation is 1.1.",
-    "full_text": "",
-    "authors": [],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": ""
-  },
-  {
-    "key": "736V6NV8",
-    "title": "Harnessing the Development Benefits of International Migration: A Randomized Evaluation of Enhanced Pre-Departure Orientation Seminars for Migrants from the Philippines",
-    "abstract": "Pre-departure orientation seminars (PDOS) for migrants have the potential to become a key policy tool for increasing the benefits of migration. PDOS build on the fact that many migrants face important knowledge gaps with respect to various aspects of their destination country upon arrival. These knowledge gaps are particularly large for individuals who move from a developing to a developed country and have to navigate a completely different system. Many migrants may hence not be able to make optimal decisions, or only after costly learning. The principal idea of PDOS is to reduce these knowledge gaps early on and provide migrants with the right information to succeed abroad. There is currently no rigorous evidence on the effects of PDOS and on what kind of training modules matter. Using a randomized control trial, this project evaluates the effects of PDOS on migration outcomes of permanent migrants from the Philippines to the US. Together with the Commission on Filipinos Overseas (CFO), the key government agency tasked to manage permanent migration from the Philippines, we have developed new PDOS modules. The new PDOS aims to foster settlement and labor market integration and increase migrants? wellbeing more generally. It also aims to strengthen migrants? engagement in diaspora activities that contribute to development in the Philippines. We identify the effects of the new PDOS by randomly assigning migrants to different types of PDOS and tracking the impact on 1,273 migrants and their family members remaining in the Philippines over a period of two years after departure.",
-    "full_text": "",
-    "authors": [],
-    "doi": "10.1257/rct.1389-2.1",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.socialscienceregistry.org/trials/1389"
-  },
-  {
-    "key": "CNXHNZ6K",
-    "title": "Gender differences in the effects of vocational training: Constraints on women and drop-out behavior",
-    "abstract": "We provide experimental evidence on the effects of vocational and entrepreneurial training for Malawian youth, in an environment where access to schooling and formal sector employment is extremely low. We track a large fraction of program drop-outs \u2013 a common phenomenon in the training evaluation literature \u2013 and this allows us to examine the determinants and consequences of drop-out and how it mediates the effects of such programs. We find that women make decisions in a more constrained environment, and their participation affected by family obligations. Participation is more expensive for them, resulting in worse training experience. The training results in skills development, continued investment in human capital, and improved well-being, with more positive effects for men, but no improvements in labor market outcomes in the short run.",
-    "full_text": "",
-    "authors": [],
-    "doi": "10.1596/1813-9450-6545",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": ""
-  },
-  {
-    "key": "P4QQT3NG",
-    "title": "Impacts of Targeted Covid-19 Cash Transfers in Togo",
-    "abstract": "In response to COVID-19, a third of social protection measures have taken the form of cash transfers reaching more than 1.1 billion people --- a 240% increase in coverage from pre-COVID levels. In the aftermath of the COVID-19 pandemic, direct cash transfers are an effective tool to protect vulnerable households. We conduct a randomized controlled trial (RCT) of a targeted cash transfer program implemented in rural Togo between November 2020 and May 2021. In collaboration with GiveDirectly, the government of Togo secured sufficient funding to provide benefits to roughly 57,000 of the approximately 580,000 citizens living in the poorest 100 cantons. Using mobile phone and satellite data, we identified the poorest cantons and poorest people living in them. We randomized the beneficiaries among the poorest phone owners. After registration, every month and for five months, eligible women receive a cash transfer of 8,620 F CFA ($15.5 US) and eligible men, a transfer of 7,450 F CFA ($13.5 US). We conduct a telephone survey at the end of the intervention to measure a wide range of outcomes, including consumption, food security, labor supply, access to health care, education, psychological well-being, financial inclusion and the perception of poverty. We also have access to administrative data of mobile phone companies in Togo, which will allow us to exploit phone usage behaviors and build other types of outcomes, such as adoption and use of the mobile money services, migration or predicted poverty.",
-    "full_text": "",
-    "authors": [],
-    "doi": "10.1257/rct.7590-1.1",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.socialscienceregistry.org/trials/7590"
-  },
-  {
-    "key": "VASWRVLM",
-    "title": "Information campaigns and migration perceptions: Evidence from Senegal",
-    "abstract": "The research studies the effect of information campaigns on irregular immigration on the intentions to migrate irregularly among high school students in Dakar. We analyze which actors and information content are effective the most in shaping students' intention to migrate and migration perceptions.",
-    "full_text": "",
-    "authors": [],
-    "doi": "10.1257/rct.8829-1.2",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.socialscienceregistry.org/trials/8829"
-  },
-  {
-    "key": "M9UMATUQ",
-    "title": "Information and Irregular Migration: Evidence from a Field Experiment in Nigeria",
-    "abstract": "Policy projections and recent research suggest that large numbers of irregular migrants from sub-Saharan Africa will continue to attempt to make their way to Europe over the next few decades. In response, European countries have made and continue to make significant investments in information campaigns designed to discourage irregular African migration. These campaigns are frequently accompanied by evaluations of some sort but, to our knowledge, none have involved a well-powered, randomized controlled trial with a representative sample and actual migration as an outcome. This pre-analysis plan describes the design of field experiment that addresses the following core questions: Are beliefs about migration-related risks, interest in attempting irregular migration, and actual migration decisions responsive to information campaigns highlighting the risks of the migration journey? The project takes place in Edo and Delta states, in the South-South region of Nigeria, a major Sub-Saharan African source of irregular migrants to Europe.",
-    "full_text": "",
-    "authors": [],
-    "doi": "10.1257/rct.8718-1.0",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.socialscienceregistry.org/trials/8718"
-  },
-  {
-    "key": "YFJ6ZQ94",
-    "title": "Informing Risky Migration: Evidence from a field experiment in Guinea",
-    "abstract": "Migrants from Western Africa go through risky migration routes to reach Europe. In addition, African migration to Europe often results in low economic outcomes. Potential migrants might be poorly informed about benefits and costs of migration. We then propose to use a Randomized Control Trial (RCT) to answer the following questions: (i) Are migrants are about their earning opportunities in Europe and risks connected to the journey? (ii) Can an intervention providing information about earnings and risks changes potential migrants? beliefs and so influences their migration choices?",
-    "full_text": "",
-    "authors": [],
-    "doi": "10.1257/rct.4062-1.1",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.socialscienceregistry.org/trials/4062"
-  },
-  {
-    "key": "ED6H9RXQ",
-    "title": "Micro-credit programs and off-farm migration in China",
-    "abstract": "This paper seeks to evaluate effects of micro-credit projects on the poor. We utilize data that we collected in Sichuan Province in 1999 to investigate whether micro-credit projects have targeted the poor and whether participation in the micro-credit project increases the likelihood of migration and switching to off-farm jobs. We find that, although the micro-credit programs did not help increase assets of the participants, it did help to move one or more of their members into an off-farm job. Our findings indicate that there is a great deal of benefit in supporting microcredit programs. \u00a9 2004 Blackwell Publishing Ltd.",
-    "full_text": "",
-    "authors": [
-      "Li H",
-      "Rozelle S",
-      "Zhang L"
-    ],
-    "doi": "10.1111/j.1468-0106.2004.00245.x",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.scopus.com/inward/record.uri?eid=2-s2.0-9744253083&doi=10.1111%2fj.1468-0106.2004.00245.x&partnerID=40&md5=f2a50ff821db79c4dfabfa1decc3b723"
-  },
-  {
-    "key": "I4MY83RJ",
-    "title": "Experimental long-term effects of early-childhood and school-age exposure to a conditional cash transfer program",
-    "abstract": "Numerous evaluations of conditional cash transfer (CCT) programs show positive short-term impacts, but there is only limited evidence on whether these benefits translate into sustained longer-term gains. This paper uses the municipal-level randomized assignment of a CCT program implemented for five years in Honduras to estimate long-term effects 13 years after the program began. We estimate intent-to-treat effects using individual-level data from the population census, which allows assignment of individuals to their municipality of birth, thereby circumventing migration selection concerns. For the non-indigenous, we find positive and robust impacts on educational outcomes for cohorts of a very wide age range. These include increases of more than 50 percent for secondary school completion rates and the probability of reaching university studies for those exposed at school-going ages. They also include substantive gains for grades attained and current enrollment for others exposed during early childhood, raising the possibility of further gains going forward. Educational gains are, however, more limited for the indigenous. Finally, exposure to the CCT increased the probability of international migration for young men, from 3 to 7 percentage points, also stronger for the non-indigenous. Both early childhood exposure to the nutrition and health components of the CCT as well as exposure during school-going ages to the educational components led to sustained increases in human capital. \u00a9 2019 The Authors",
-    "full_text": "",
-    "authors": [
-      "Molina Mill\u00e1n",
-      "T",
-      "Macours K",
-      "Maluccio J A",
-      "Tejerina L"
-    ],
-    "doi": "10.1016/j.jdeveco.2019.102385",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.scopus.com/inward/record.uri?eid=2-s2.0-85072577537&doi=10.1016%2fj.jdeveco.2019.102385&partnerID=40&md5=cbacf9b13921db5e8346649371e83718"
-  },
-  {
-    "key": "N6R57IVT",
-    "title": "Bilateral labor agreements and the migration of Filipinos: An instrumental variable approach",
-    "abstract": "Bilateral labor agreements (BLAs) are preferred policy models for regulating migration by many governments around the world. The Philippines has been a leader in both agreement conclusion and exporting labor. A recent Congressional evocation is pushing bureaucrats and academics alike to investigate this policy strategy for outcomes and effectiveness. The following analysis answers the question \"Do BLAs affect the migration outflows of Overseas Filipino Workers (OFWs)?\"using a plausibly exogenous variation to isolate a causal effect. I test for effects of BLAs using two instrumental variables (IVs), such as Bilateral Investment Treaties (BITs) and Formal Alliances, and an original dataset of land-based and sea-based Filipino BLAs and migrant stock in 213 unique areas from 1960 to 2018. I do not find any empirical evidence that these treaties drive migration. However, BLAs have statistically significant effects on gross domestic product (GDP) per capita and exports, suggesting other important channels through which these agreements affect economic outcomes. These null results are critically important for policymakers and diplomats because the resources spent on negotiation are wasted if the primary goal is to increase migration. \u00a9 2021 Brianna O'Steen, published by Sciendo.",
-    "full_text": "",
-    "authors": [
-      "O'Steen B"
-    ],
-    "doi": "10.2478/izajodm-2021-0011",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.scopus.com/inward/record.uri?eid=2-s2.0-85117392815&doi=10.2478%2fizajodm-2021-0011&partnerID=40&md5=4d168fa69f58fc0804c9b0a8a8fd3172"
-  },
-  {
-    "key": "64F7ET9X",
-    "title": "Children on the Move : Progressive Redistribution of Humanitarian Cash Transfers among Refugees",
-    "abstract": "This paper evaluates the impact of the Emergency Social Safety Net (ESSN) in Turkey, the largest cash transfer program for international refugees in the world. The paper provides prima facie evidence that the program quickly caused substantial changes in household size and composition, with a net movement of primarily school-age children from larger ineligible households to smaller eligible ones. A sharp decline in inequality is observed in the entire study population: the Gini index declined by four percentage points (or 15 percent) within six months of program rollout, and the poverty headcount at the $3.20/day international poverty line declined by more than 50 percent after one year. ESSN caused a moderate increase in the diversity and frequency of food consumption among eligible households, and although there was no statistically significant effect on overall school enrollment, there were meaningful gains among the most vulnerable beneficiary households. To strike the right balance between transfer size and coverage, key parameters in the design of any cash transfer program, policy makers should consider the possibility that refugee populations may respond to their eligibility status by altering their household structure and living arrangements.",
-    "full_text": "",
-    "authors": [
-      "Ozler Berk",
-      "Celik Cigdem",
-      "Cunningham Scott",
-      "Cuevas Pablo Facundo",
-      "Parisotto Luca"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://search.ebscohost.com/login.aspx?direct=true&db=edsrep&AN=edsrep.p.wbk.wbrwps.9471&site=eds-live"
-  },
-  {
-    "key": "82H99VKZ",
-    "title": "Do higher salaries lower physician migration?",
-    "abstract": "It is believed that low wages are an important reason why doctors and nurses in developing countries migrate, and this has led to a call for higher wages for health professionals in developing countries. In this paper, we provide some of the first estimates of the impact of raising health workers' salaries on migration. Using aggregate panel data on the stock of foreign doctors in 16 Organization for Economic Cooperation and Development countries, we explore the effect of a wage increase programme in Ghana on physician migration. We find evidence that 6 years after the implementation of this programme, the foreign stock of Ghanaian doctors abroad had fallen by approximately 10% relative to the estimated counterfactual. This result should be interpreted with caution, however, given the sensitivity of the results to changes in model specification. \u00a9 2013 The Author.",
-    "full_text": "",
-    "authors": [
-      "Okeke E N"
-    ],
-    "doi": "10.1093/heapol/czt046",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.scopus.com/inward/record.uri?eid=2-s2.0-84906064721&doi=10.1093%2fheapol%2fczt046&partnerID=40&md5=094f43a6ca58fdbf70d99377b4c2379c"
-  },
-  {
-    "key": "924DRNUJ",
-    "title": "Aid and Migration: An Analysis of the Impact of Progresa on the Timing and Size of Labour Migration",
-    "abstract": "This paper models the short and medium-run impact of aid on migration, considering alternatively the effect of nconditional and conditional cash transfers to financially constrained households. Data from the evaluation of a Mexican development program, Progresa, are used to estimate the effect of the potential grant size on migration. The empirical analysis is consistent with model prediction. It shows that the program is associated with an increase in international migration, which is also a positive function of size of potential transfer. The grant may loosen financial constraints. At the same time, fine-tuned conditional grants targeting prospective migrants (in the form of secondary school subsidies) reduce the short-term migration probability. As regards medium-term migration, secondary school beneficiaries are not more likely to migrate than the control group after they complete the subsidised education cycle.",
-    "full_text": "",
-    "authors": [
-      "Manuela Angelucci"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://docs.iza.org/dp1187.pdf"
-  },
-  {
-    "key": "UK8I5GM4",
-    "title": "Medical worker migration and origin-country human capital: Evidence from us visa policy",
-    "abstract": "We exploit changes in U.S. visa policies for nurses to measure the origin-country human capital response to international migration opportunities. Combining data on all migrant departures and postsecondary institutions in the Philippines, we show that nursing enrollment and graduation increased substantially in response to greater U.S. demand for nurses. The supply of nursing programs expanded. Nurse quality, measured by licensure exam pass rates, declined. Despite this, for each nurse migrant, 9 additional nurses were licensed. New nurses switched from other degree types, but graduated at higher rates than they would have otherwise, increasing the human capital stock in the Philippines.",
-    "full_text": "",
-    "authors": [],
-    "doi": "10.1162/rest_a_01131",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": ""
-  },
-  {
-    "key": "GIQRAEI7",
-    "title": "Returns to International Migration: Evidence from a Bangladesh-Malaysia Visa Lottery",
-    "abstract": "We follow 3,512 (of 1.4 million) applicants to a government lottery that randomly allocated visas to Bangladeshis for low-skilled, temporary labor contracts in Malaysia. Most lottery winners migrate, and their remittance substantially raises their family's standard of living in Bangladesh. The migrant's absence pauses demographic changes (marriage, childbirth, household formation), and shifts decision-making power towards females. Migration removes enterprising individuals, lowering household entrepreneurship, but does not crowd out other family members' labor supply. One group of applicants were offered deferred migration that never materialized. Improved migration prospects induce pre-migration investments in skills that generate no returns in the domestic market.",
-    "full_text": "",
-    "authors": [
-      "Mobarak Ahmed Mushfiq",
-      "Sharif Iffath",
-      "Shrestha Maheshwor"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": ""
-  },
-  {
-    "key": "DEFXYBQK",
-    "title": "Role models and migration intentions",
-    "abstract": "Role models\u2014those individuals who resemble us but have achieved more than us\u2014 are thought to impact our aspirations. In this paper, we study the impact of role models on intentions to migrate. Specifically, we implement a randomized controlled trial to show documentaries in rural villages of Mali (Kayes region). These documentaries focus on economic opportunities and show either negative or positive portraits of migrants, or portraits of local people who have successfully set up flourishing businesses without ever considering migration. This paper adds to the larger debate about the efficiency of information provision. We find very few significant impacts, none of which hold when attrition is controlled for using nonparametric Lee bounds. We also implement a treatment heterogeneity analysis using a causal forest algorithm, which aside from confirming our average treatment effects suggests the presence of heterogeneity. It appears that individuals with living conditions that could facilitate migration are less likely to be significantly impacted. The high aspirations to improve living conditions, coupled with a strong feeling of lack of control over the future may help explaining the fact that confrontations with real life experiences do not significantly modify average aspirations to migrate.",
-    "full_text": "",
-    "authors": [
-      "Mespl\u00e9-Somps S",
-      "Nilsson B",
-      "d'Aiglepierre R"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.cairn-int.info/journal-afd-research-papers-2021-200-page-1.htm"
-  },
-  {
-    "key": "RQXKP6NP",
-    "title": "Broken Promises: Evaluating an Incomplete Cash Transfer Program",
-    "abstract": "Interventions in highly insecure and fragile contexts are always confronted with the latent risk of not being able to implement the program as intended. Despite its high policy relevance, little is known about the impacts of program disruption or cancellation on beneficiaries. This study uses the unplanned cancellation of the South Sudan Youth Business Start-Up Grant Program to assess the socioeconomic, behavioral, and psychological consequences of a program that fails to be implemented as intended. Originally planned as a randomized trial, the Youth Startup Business Grant Program consisted of an unconditional cash grant combined with a business and life skills training targeting the youth in South Sudan. Due to the intensification of violence in the country, the disbursement of the grant was terminated in late 2016 before most of the intended beneficiaries had accessed the grant. The study uses survey data from face-to-face interviews and experimental data from lotteries, trust games, and a list experiment to assess the consequences of the cancellation in a comprehensive form. The empirical analysis employs instrumental variable regressions to control for individual characteristics that might have made it more likely to access the grant before disbursement was frozen. The results show that participants who received the originally planned treatment displayed significant improvements in their consumption, savings, and psychological well-being. However, participants who vainly expected to receive the cash grant showed reduced levels of consumption and women among this subgroup also experienced strong reductions in their trust level. In addition, the study finds some evidence that these women were less likely to migrate.",
-    "full_text": "",
-    "authors": [
-      "Muller Angelika",
-      "Pape Utz Johann",
-      "Ralston Laura R"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": ""
-  },
-  {
-    "key": "G79BE4TK",
-    "title": "The Benefits and Costs of Guest Worker Programs: Experimental Evidence from The India-UAE Migration Corridor",
-    "abstract": "We estimate the returns to temporary migration programs using a randomized control trial with several thousand job seekers in India applying to guest worker jobs in the United Arab \u2026",
-    "full_text": "",
-    "authors": [
-      "Naidu S",
-      "Nyarko Y",
-      "Wang SY"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://econ.cms.arts.ubc.ca/wp-content/uploads/sites/38/2022/09/20221123_Suresh-Naidu.pdf"
-  },
-  {
-    "key": "5J2Z3L7J",
-    "title": "Do social protection programs foster short-term and long-term migration adaptation strategies?",
-    "abstract": "Abstract We examine how migration is influenced by temperature and precipitation variability, and the extent to which the receipt of a cash transfer affects the use of migration as an adaptation strategy. Climate data is merged with georeferenced panel data (2010\u20132014) on individual migration collected from the Zambian Child Grant Program (CGP) sites. We use the person-year dataset to identify the direct and heterogeneous causal effects of the CGP on mobility. Having access to cash transfers doubles the rate of male, short-distance moves during cool periods, irrespective of wealth. Receipt of cash transfers (among wealthier households) during extreme heat causes an additional retention of males. Cash transfers positively spur long-distance migration under normal climate conditions in the long term. They also facilitate short-distance responses to climate, but not long-distance responses that might be demanded by future climate change.",
-    "full_text": "",
-    "authors": [
-      "Mueller Valerie",
-      "Gray Clark",
-      "Handa Sudhanshu",
-      "Seidenfeld David"
-    ],
-    "doi": "10.1017/S1355770X19000214",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.cambridge.org/core/product/identifier/S1355770X19000214/type/journal_article https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7062362/pdf/nihms-1558213.pdf"
-  },
-  {
-    "key": "EX4XE7QK",
-    "title": "Labor Productivity, Remittance Use, and the Impact of the Poverty Alleviation Fund (PAF) Program in Nepal",
-    "abstract": "This dissertation presents three studies related to labor productivity, remittances use, and the effect of an anti-poverty program on migration and remittances. Labor is the biggest endowment available to the poor. Understanding labor issues is important in addressing the problems of poverty, inequality, migration, and economic development. In this dissertation, I estimate the labor productivity of agricultural household because most of the agricultural households in developing countries work in their own farms, it is not possible to observe wages. The first chapter estimates the shadow wage (marginal productivity of labor) of the agricultural household in the context of Nepal. How different is marginal productivity of labor for women compared to men in agricultural households? In developing countries, where most of the families work on their farms, wage or labor-related income cannot be observed directly. This paper contributes to the literature on gender wage difference in labor and development economics by developing a new approach to estimate the shadow wage of agricultural households in Nepal. Using a general functional form, we first derive the shadow wage from a theoretical model. Then, a model with ward-level fixed effects is used to estimate the shadow wage by gender for Nepalese agricultural households. We find that the productivity of women is not that different than that of men. Despite the vast difference in observed market wages for women, the distribution of shadow wages of women is not that different from that of men, calling for policies to increase the market wages for women. The second chapter of this dissertation, attempts to understand the use of remittances among the households of Nepal. Remittances are transfers made by migrant workers to their family and relatives in their country of origin. In Nepal, remittances account for 25-30% of the GDP, and the trend of youths seeking work in other countries--mostly in Southeast Asia and the Middle East--has been increasing. Understanding the expenditure pattern of remittances-receiving households compared to non-recipients provides an understanding of the effect of remittances. In this chapter I employ nationally representative data from Nepal to investigate the effect of remittances on household expenditure patterns, and I compare the prevalence of poverty between remittance recipients and non-recipients. The findings that emerge are as follows: households receiving international and both domestic and international remittances have increased expenditure shares on education, suggesting investments in human capital in the household. In contrast I find a decrease in education expenditures for households receiving domestic remittances. Food expenditures share decreases for households receiving all types of remittances. Households receiving remittances increase the expenditure shares on durables and other consumption expenditures. Households receiving remittances have decreased shares in health expenditures. With regards to poverty, the paper shows that receiving remittances reduces the likelihood of being poor. In the third chapter of the dissertation, I evaluate the effects of the Poverty Alleviation Fund program (PAF) on remittances and migration using the data from a quasi-experiment. The PAF is a social fund program that has been providing services to marginalized communities in Nepal through various income-generating activities since 2006. Unlike previous research that has used conditional cash transfer programs (CCTs) to study the role of a development program on migration and remittances, I employ the data from the community-driven anti-poverty program that provides income-generating activities to participants. Using a panel dataset collected by Center for Economic Development and Administration (CEDA) of the Tribhuwan University and the PAF, and taking advantage of a quasi-randomized phase-in experimental design, I estimate the causal effects of a development program on remittances, migration, and welfare measures. I show that policy makers shou d be aware that community-driven development programs have unintended consequences for migration and remittances, which are distinct from the primary goals of the program: alleviating poverty and improving food security. The program results in a decrease of approximately Rs.6000 (approximately six percent of total household consumption) in remittances received, crowding out private transfers in the presence of public transfers. The paper shows an increase in domestic migration, but no change in international migration due to the program.",
-    "full_text": "",
-    "authors": [
-      "Nepal Atul"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": ""
-  },
-  {
-    "key": "QJH23MCD",
-    "title": "Safe Migration Awareness Campaign In Rural Communities Of Nigeria, The Procedure And Impacts",
-    "abstract": "International migration is undergoing unprecedented changes. The traditional determinants of migration such as poverty, food insecurity and climate change are giving way to new motivations. These new issues, that include but are not limited to ambitions to live a foreign lifestyle, incomplete and asymmetric information are capable of underestimating the risk in irregular migration. As the information about foreign lifestyle flow freely and new opportunities open, it becomes very difficult to manage irregular migration through border control. Within the transitional mode of international migration, the use of awareness campaigns, especially in rural areas of home countries that target the most vulnerable groups, \u2013 school-age youths, could become a veritable means of deterring irregular migration. In 2018, Ricosmigration \u2013 Rural Information Campaign on Safe Migration - received funding from the German Foreign Office to investigate why young people from Nigeria migrate irregularly and implement a safe migration awareness campaign for youth in rural secondary schools. This report is from the result of the project which cut across interviews of Nigerian migrants living in Italy, potential migrants in Nigeria, and the result of the awareness campaigns conducted in 10 secondary schools in Edo, Nigeria. In the report, we explain the profile of a potential migrant from Nigeria. We also explore how reduced capabilities to lead the desired life and how the increasing use of social media internet has greatly raised the likelihood of migration in Nigeria. We equally show in the detail how we implemented a randomized experiment to test the efficiency of the migration awareness campaign. The project provides a new dimension to the discourse of the root cause of migration by introducing the role of limited opportunities and freedom (capabilities), low life satisfaction, and incomplete information through social media. Additionally, it shows the power of light interventions such as awareness campaigns in reducing irregular migration. In particular, the result showed that about 77.2 per cent of Nigerian youths that responded to the interview have the intention to migrate abroad, and 37.2 per cent would migrate if they win a cash lottery that is enough to cover the migration cost. Our randomized experiment showed that migration awareness campaign could reduce the risk of being a victim of human trafficking by more than 50 per cent. It could also reduce the desire to engage in irregular migration by more than 30 per cent and increased the decision to take necessary steps to avoid human traffickers and follow proper procedures for safe migration by more than 50 per cent. The awareness campaign had a wide coverage reaching about 7000 students in rural communities in Edo State, Nigeria. The success factors of our awareness campaign include the utilization of appropriate channels, delivery of an accurate message that was tailored to the group we spoke to, and the use of respected external facilitators.",
-    "full_text": "",
-    "authors": [
-      "Obi Chinedu"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://search.ebscohost.com/login.aspx?direct=true&db=edsrep&AN=edsrep.p.osf.socarx.v3kn2&site=eds-live"
-  },
-  {
-    "key": "PM9LIPT3",
-    "title": "Labour migration and households: A reconsideration of the effects of the social pension on labour supply in South Africa",
-    "abstract": "This paper re-examines the effect of the South African social pension on the labour supply of working-age adults using data from 1993. We take account of the fact that households may include non-resident members, and therefore the pension may play a role in facilitating migration to work or look for work. We find that rural African women are significantly more likely to be migrant workers when they are members of a household in receipt of a pension, and that it is female pension income that drives this result. We explore a number of possible reasons why pension income might have this effect.",
-    "full_text": "",
-    "authors": [
-      "Posel D",
-      "Fairburn J A",
-      "Lund F"
-    ],
-    "doi": "10.1016/j.econmod.2005.10.010",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": ""
-  },
-  {
-    "key": "X835X4JX",
-    "title": "Raising Awareness About the Risk of Irregular Migration: Quasi-Experimental Evidence from Guinea",
-    "abstract": "In response to mounting evidence of harm inflicted on irregular migrants along their journeys from West Africa to Europe, international organizations, civil society organizations, and governments have scaled up campaigns as a tool for raising awareness about the risks of irregular migration. Campaigns aim to counter misinformation by smugglers and facilitate safe migration decisions. Despite the growing number of interventions, there is limited empirical evidence on the impact and effectiveness of such campaigns. Based on a difference-in-difference design, this study investigates the effect of a mobile cinema and community discussion intervention on the perceptions, knowledge, and intentions of potential irregular migrants in Northern Guinea in 2019. The results show that potential migrants who participated in events were significantly more likely to show awareness gains and less likely to report high intentions to migrate irregularly. While the relative importance of risk perceptions and their impact on migration flows remain unclear, the findings provide evidence supporting the assumption that risk awareness can be a relevant factor in the decision-making process of potential irregular migrants. While campaigns may be an effective tool in certain contexts, effect sizes highlight the need for policymakers to keep realistic expectations.",
-    "full_text": "",
-    "authors": [
-      "Tjaden J",
-      "Gninafon H"
-    ],
-    "doi": "10.1111/padr.12468",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": ""
-  },
-  {
-    "key": "EZTFJFFS",
-    "title": "Household Structure and Short-Run Economic Change in Nicaragua",
-    "abstract": "During the economic crises Nicaragua suffered between 2000 and 2002, a conditional cash transfer program targeting poor households began operating. Using panel data on 1,397 households from the program's experimentally designed evaluation, we examined the impact of the program on household structure. Our findings suggest that the program enabled households to avoid reagglomeration during the economic crises, with households in control communities growing more than treated households. These changes were driven primarily by shifts in residence of relatively young men and women with close kinship ties to the household head. In contrast, households that received transfers continued to send off young adult members, suggesting that the program provided resources to overcome the short-term economic pressures on household structure.",
-    "full_text": "",
-    "authors": [
-      "Winters P",
-      "Stecklov G",
-      "Todd J"
-    ],
-    "doi": "10.1111/j.1741-3737.2009.00628.x",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": ""
-  },
-  {
-    "key": "G9HHCW5Z",
-    "title": "Empowering Indonesian Migrant Workers to Access Quality Overseas Placement Services",
-    "abstract": "Nearly 700,000 Indonesians migrate abroad for work each year. The vast majority do so through recruiters and placement agencies that facilitate temporary employment in countries across Asia and the Middle East. These agencies support migrants starting with pre-departure paperwork and training through repatriation, and are therefore a crucial determinant of a worker\u2019s migration experience. Potential migrants ostensibly have a great deal of choice between agencies \u2013 there are over 1,000 formally registered firms in Indonesia \u2013 and in theory, competition between these firms should drive out poorly performing agencies. Yet anecdotal evidence suggests that agency quality is highly variable, and that many agencies engage in exploitative practices. This could be due to market power at the local level, or informational failures, both of which would hamper competitive pressures.\nOur findings from preliminary research suggest that despite the fact that nearly three-quarters of female migrants believe that there is no relationship between the quality of the agencies and the experience with the employer; there is indeed a robust correlation between the two variables. However, migrants attribute the employer quality to \"nasib\", or fate.\nIn 2015, we ran information campaigns designed to transmit information on agency quality to potential migrants. Surveys will be conducted to evaluate the impact of the information campaigns on migrants\u2019 outcomes.",
-    "full_text": "",
-    "authors": [
-      "Schaner Simone",
-      "Cameron Lisa",
-      "Bazzi Samuel",
-      "Kartaadipoetra Firman Witoelar"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.socialscienceregistry.org/trials/630"
-  },
-  {
-    "key": "SNG5JMCQ",
-    "title": "The impact of providing vocational training to young men on labour outcomes and attitudes towards migration in northern Guinea-Bissau",
-    "abstract": "As in most of Sub-Saharan Africa, the lack of quality employment among the rural youth is threatening economic development and inclusive growth, with employment search arguably working as an important driver for both internal and international migration. Active labour market policies, such as vocational trainings, are frequently chosen by governments, international institutions and NGOs as a potential solution to these labour market frictions. Nevertheless, the causal evidence determining the impact of these programs on the youth of rural areas is scarce in the sub-region, and inexistent in Guinea-Bissau. In this project, we run a randomized controlled trial within the GOT project implemented by the NGO ENGIM. This project provides vocational training courses on professional skills thought to be particularly relevant in the local context, and facilitates traineeships in local businesses for young men in two northern regions of the country, aiming to prevent their engagement in irregular migration practices. Assessing the impact of the GOT project on the employment rates, income and attitudes towards migrations of its trainees should contribute to a better understanding of the impact of active labour market policies in rural areas of low income countries, facilitating a better design, implementation and evaluation of such programmes in Guinea-Bissau and similar contexts.",
-    "full_text": "",
-    "authors": [],
-    "doi": "10.1257/rct.6890-1.2",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.socialscienceregistry.org/trials/6890"
-  },
-  {
-    "key": "QYAQM7H9",
-    "title": "Weather Insurance and Investment Choice",
-    "abstract": "Exposure to the risk of extreme weather conditions has been shown to constrain investment\nby subsistence farmers in developing countries and may lead to ine\u00a2cient production choices.\nThis paper evaluates whether insuring farmers against such risks alters resource allocation decisions. In particular I consider the e\u00a7ects of a Mexican government disaster relief program with\ninsurance-like features. The results, based on a regression discontinuity design, indicate that\ninsurance against losses arising from natural disasters changes how rural households invest in\ntheir farms. Insured farmers utilize more expensive capital inputs and adopt di\u00a7erent technologies. Additionally, the insurance changes labor supply patterns. Notably, members of insured\nhouseholds are approximately 10% more likely to migrate internationally. Additional results,\nthat the program matters most when the returns to migration are more unpredictable, are\nconsistent with a model where insurance obviates the need for precautionary savings, allowing\nhouseholds to \u00d6nance international migration.",
-    "full_text": "",
-    "authors": [
-      "Shapiro J"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": ""
-  },
-  {
-    "key": "8EFH4AP4",
-    "title": "Can Public Transfers Reduce Mexican Migration? A study based on randomized experimental data",
-    "abstract": "Prior research on Mexican migration has shown that social networks and economic incentives play an important role in determining migration outcomes. This study utilizes experimental data on PROGRESA, Mexico's primary poverty reduction program, to evaluate the effects of public cash transfers on migration. Our study complements a growing body of literature aimed at overcoming longstanding hurdles towards the establishment of causal validity in empirical studies of migration. We find that public cash transfers reduce US migration but have little effect on domestic migration. Furthermore, we find that the provision of cash transfers appears to reduce migration partly by reducing the relative deprivation levels of poor households. Finally, we find that the effect of public cash transfers on US migration depend on the size of existing US migration networks. Surprisingly, we see that transfers have larger (more negative) effects on US migration in communities with large existing networks. The results suggest that public transfers may be helpful in managing rural out-migration, particularly to the US. Interestingly, such programs may be most effective if they are targeted towards communities with strong existing migration patterns.",
-    "full_text": "",
-    "authors": [
-      "Stecklov Guy",
-      "Stampini Marco",
-      "Davis Benjamin"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": ""
-  },
-  {
-    "key": "5M6JE7W5",
-    "title": "Do Conditional Cash Transfers Influence Migration? A Study Using Experimental Data From The Mexican PROGRESA Program",
-    "abstract": "Prior research on Mexican migration has shown that social networks and economic incentives play an important role in determining migration outcomes. We use experimental data from PROGRESA, Mexico's primary poverty-reduction program, to evaluate the effects of conditional cash transfers on migration both domestically and to the United States. Our study complements a growing body of literature aimed at overcoming longstanding hurdles to the establishment of causal validity in empirical studies of migration. Analysis based on the data collected before and after the program 's onset shows that conditional transfers reduce U.S. migration but not domestic migration. The data also enable us to explore the role of existing family and community migration networks. The results show that migration networks strongly influence migration, but that the effect of conditional transfers on migration is apparently not mediated by existing migration network structures. Our results suggest that conditional transfers may be helpful in managing rural out-migration, particularly to the United States.",
-    "full_text": "",
-    "authors": [
-      "Stecklov Guy",
-      "Winters Paul",
-      "Stampini Marco",
-      "Davis Benjamin"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.jstor.org/stable/4147339?seq=1#page_scan_tab_contents"
-  },
-  {
-    "key": "DG5T267N",
-    "title": "Cash transfers and migration: theory and evidence from a randomized controlled trial",
-    "abstract": "Will the fast expansion of cash-based programming in developing countries increase international migration? Theoretically, cash transfers may favor international migration by relaxing liquidity, credit, and risk constraints. But transfers, especially those conditional upon staying at home, may also increase the opportunity cost of migrating abroad. This paper evaluates the impact of a cash-for-work program on migration. Randomly selected households in Comoros were offered up to US$320 in cash in exchange for their participation in public works projects. We find that the program increased migration to Mayotte - the neighboring and richer French Island - by 38 percent, from 7.8% to 10.8%. The increase in migration is explained by the alleviation of liquidity and risk constraints, and by the fact that the program did not increase the opportunity cost of migration for likely migrants.",
-    "full_text": "",
-    "authors": [
-      "Sterck O",
-      "Gazeaud J",
-      "Mvukiyehe E"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.csae.ox.ac.uk/materials/papers/csae-wps-2019-16.pdf"
-  },
-  {
-    "key": "CAJ5PJQW",
-    "title": "The impact of Indian SHGs: a long-run field experiment in Jharkhand",
-    "abstract": "This field experiment randomized access to Self-Help Groups in villages spread over the entire state of Jharkhand and surveyed a sample of 1,080 households three times between 2004 and 2009, in order to evaluate the changes in their living standards. We study impacts on children education and labor, household consumption and risk-coping behavior (including migration), and local credit markets.",
-    "full_text": "",
-    "authors": [],
-    "doi": "10.1257/rct.5570-1.1",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.socialscienceregistry.org/trials/5570"
-  },
-  {
-    "key": "MHIY5NJJ",
-    "title": "The value of forecasts: Experimental evidence from developing-country agriculture",
-    "abstract": "Climate risk is a key driver of low agricultural productivity in poor countries. We use a cluster-randomized trial to evaluate a novel risk-mitigation approach: long-range forecasts that provide information about the onset of the Indian summer monsoon well in advance of its arrival. In contrast to traditional approaches that allow farmers to cope with risk ex post, this new ex ante technology provides accurate information at least one month in advance of the monsoon's arrival, enabling farmers to alter cropping choices and other up front input decisions. Moreover, forecasts have the potential to be disseminated cheaply, even at scale. We assign 250 villages to one of three groups: a control group; a group that is given an opportunity to purchase the forecast; and a group that is offered insurance. This design allows us to investigate farmers' willingness-to-pay for forecasts; measure how forecasts affect farmer beliefs, up-front investments, and welfare; and study how these effects compare to the canonical ex post loss mitigation tool: weather-based index insurance.",
-    "full_text": "",
-    "authors": [],
-    "doi": "10.1257/rct.8846-1.0",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.socialscienceregistry.org/trials/8846"
-  },
-  {
-    "key": "YYL2HIQB",
-    "title": "Investments in Human Capital: Long-term Effects of Progresa-Oportunidades on Poverty and Migration in Rural Mexico.",
-    "abstract": "This paper evaluates the effects on poverty reduction and migration of a conditional cash transfer program in Mexico named Oportunidades (previously Progresa). This program, the first in Latin America and the most imitated of its kind, was intended to increase human capital, which would eventually translate into poverty reduction. Linear and non-linear panel models are used to explore whether there are any such effects, and the implications for the effectiveness and evaluation strategies of the program. No significant effects of Progresa-Oportunidades in reducing income poverty or affecting international migration were found at the rural level. However, there is a weak effect in the case of domestic migration. As data continues to become available through Mexico's official Secretariat of Social Development, there will be more opportunities for further exploration of the relationship between this program, poverty alleviation, and migration outcomes. Limitations and further recommendations for this study are also discussed.",
-    "full_text": "",
-    "authors": [
-      "TIRADO J ALEJANDRO"
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://search.ebscohost.com/login.aspx?direct=true&profile=ehost&scope=site&authtype=crawler&jrnl=20672640&AN=113660670&h=tw80U6FJzYcJYu3fTX%2BMPrdXbXrYCOOX2ZDxlHcnJdmg5T13YEukzyFOx75iYT%2F%2BuqTBgG5p2dwmyl3bwBylKg%3D%3D&crl=c"
-  },
-  {
-    "key": "X2YYD8XC",
-    "title": "The effect of peer-to-peer risk information on potential migrants ' Evidence from a randomized controlled trial in Senegal",
-    "abstract": "In response to mounting evidence on the dangers of irregular migration from Africa to Europe, the number of information campaigns which aim to raise awareness about the potential risks has rapidly increased. Governments, international organizations and civil society organizations implement a variety of campaigns to counter the spread of misinformation accelerated by smuggling and trafficking networks. The evidence on the effects of such information interventions on potential migrants remains limited and largely anecdotal. More generally, the role of risk perceptions in the decision-making process of potential irregular migrants is rarely explicitly tested, despite the fact that the concept of risk pervades conventional migration models, particularly in the field of economics. We address this gap by assessing the effects of a peer-to-peer information intervention on the perceptions, knowledge and intentions of potential migrants in Dakar, Senegal, using a randomized controlled trial design. The results show that--three months after the intervention--peer-to-peer information events increase potential migrants' subjective information levels, raise risk awareness, and reduce intentions to migrate irregularly. We find no substantial effects on factual migration knowledge. We discuss how the effects may be driven by the trust and identification-enhancing nature of peer-to-peer communication.",
-    "full_text": "",
-    "authors": [
-      "Tjaden Jasper",
-      "Dunsch Felipe Alexander"
-    ],
-    "doi": "10.1016/j.worlddev.2021.105488",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://linkinghub.elsevier.com/retrieve/pii/S0305750X21001005"
-  },
-  {
-    "key": "FC8EU2Q3",
-    "title": "Universal Basic Income in Kenya",
-    "abstract": "A universal basic income (UBI) is a recurring, unconditional cash transfer sized to meet basic needs and paid to all members of a society. Proponents argue that a UBI has the potential to eliminate extreme poverty and to counteract the harmful effects of rising inequality in wealthier countries. Interest has surged, with UBI actively debated in countries ranging from Switzerland and Finland to Namibia and India. Yet, while cash transfers in general have a strong evidence base, a full basic income has never been implemented or rigorously evaluated. This study proposes to fill these evidence gaps with an ambitious long-term randomized control trial in Kenya. The core objective is to test the hypothesis that a universal basic income is an effective way to eliminate extreme global poverty. This is a central policy question currently being debated in a number of emerging markets, and is also highly relevant for foreign aid policy given that the global poverty gap has fallen to $80B a year as of 2015, less than half of official development assistance flows. Positive results could shift public spending away from in-kind and means-tested programs, while negative ones could have the opposite effect. As a secondary goal, the study also aims to inform debate on the role of UBI in wealthy countries. While the populations are obviously different from those we will work with in East Africa, many of the core questions about human behavior are the same ? does economic security motivate people to work more or less? To socialize more or less? etc. At a minimum, positive results from a low-cost emerging-market evaluation could be motivator for testing more expensive guarantees in richer countries. Finally, it is worth emphasizing that in the process of producing this evidence the project will also directly benefit some of the poorest people on the planet by delivering cash transfers to them. The broader evidence base on cash transfers suggests that these recipients will reap life-changing benefits from participating in the project.",
-    "full_text": "",
-    "authors": [],
-    "doi": "10.1257/rct.1952-2.1",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.socialscienceregistry.org/trials/1952"
-  },
-  {
-    "key": "UX36F5C2",
-    "title": "Liquidity Constraints and Migration: Evidence from Indonesia",
-    "abstract": "Although liquidity constraints have been seen as both a factor limiting individuals from migrating and a motivation for households to send a migrant, the consequences of relaxing liquidity constraints on migration behavior have not been adequately explored due to data limitations. In this study, we take advantage of an unusual policy, Bantuan Langsung Tunai -- a national-level unconditional cash transfer program targeted toward the poorest households in Indonesia -- to empirically assess the impact of increased liquidity on the migration behavior of poor Indonesian households.With a highly mobile population and a long history of circular migration, Indonesia is an ideal space to study migration. Using panel data from the Indonesian Family Life Survey, the results demonstrate that a positive liquidity shock increases the probability of migration among low-asset households, among households with a migration history, and, most significantly, among lowasset households with a migration history. [ABSTRACT FROM AUTHOR] Copyright of International Migration Review is the property of Sage Publications Inc. and its content may not be copied or emailed to multiple sites or posted to a listserv without the copyright holder's express written permission. However, users may print, download, or email articles for individual use. This abstract may be abridged. No warranty is given about the accuracy of the copy. Users should refer to the original published version of the material for the full abstract. (Copyright applies to all Abstracts.)",
-    "full_text": "",
-    "authors": [
-      "Tiwari Smriti",
-      "Winters Paul C"
-    ],
-    "doi": "10.1177/0197918318768555",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "http://journals.sagepub.com/doi/10.1177/0197918318768555"
-  }
-]

data/kayongo-papers_zotero_items.json

@@ -1,222 +0,0 @@
-[
-  {
-    "key": "UNWR6MI4",
-    "title": "Le\u00e7ons du Rapid Response Service en Ouganda",
-    "abstract": "R\u00e9sum\u00e9 Le Service de r\u00e9ponse rapide (RRS)\u2013en anglais Rapid Reponse Service\u2013est un service d'application de connaissances. Il r\u00e9pond au besoin de donn\u00e9es des d\u00e9cideurs en \u2026",
-    "full_text": "",
-    "authors": [
-      "I Kawooya",
-      "I Ddumba",
-      "E Kayongo",
-      "..."
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.cres-sn.org/wp-content/uploads/2022/03/Ebook-Donnees-probantes.pdf#page=241"
-  },
-  {
-    "key": "QEZLUVIL",
-    "title": "Brain Magnetic Resonance Imaging and Angiography Findings in Ugandan Children with Sickle Cell Anemia; A Cross Sectional Study",
-    "abstract": "Abstract Sickle Cell Anemia (SCA) is a leading cause of childhood stroke in sub-Saharan Africa and sickle cell brain vasculopathy manifests either as overt stroke or clinically\" silent \u2026",
-    "full_text": "",
-    "authors": [
-      "R Idro",
-      "NS Green",
-      "D Munube",
-      "LR Buluma",
-      "B Kebirungi",
-      "..."
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://ashpublications.org/blood/article/132/Supplement%201/2376/264336"
-  },
-  {
-    "key": "BN43BLJ5",
-    "title": "Accuracy of the chest x-ray in screening for tuberculosis in Uganda: A cross-sectional study.",
-    "abstract": "Abstract ABSTRACT BACKGROUND: The WHO END TB strategy requires\u2265 90% case detection to combat tuberculosis (TB). Increased TB case detection requires a more \u2026",
-    "full_text": "",
-    "authors": [
-      "J Nalunjogi",
-      "F Mugabe",
-      "I Najjingo",
-      "P Lusiba",
-      "F Olweny",
-      "..."
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.researchsquare.com/article/rs-37900/latest.pdf"
-  },
-  {
-    "key": "PF63FYYY",
-    "title": "Rapidly responding to policy queries with evidence: Learning from rapid response services in Uganda",
-    "abstract": "Summary The Rapid Response Service (RRS) is a knowledge translation service in Uganda that responds to a decision maker's needs for evidence with synthesised relevant evidence \u2026",
-    "full_text": "",
-    "authors": [
-      "I Kawooya",
-      "I Ddumba",
-      "E Kayongo",
-      "..."
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://library.oapen.org/bitstream/handle/20.500.12657/39511/9780367440121_text.pdf?sequence=1#page=159"
-  },
-  {
-    "key": "VXFKNQIN",
-    "title": "Pre-diagnostic drop out of presumptive TB patients and its associated factors at Bugembe Health Centre IV in Jinja, Uganda",
-    "abstract": "Background: Drop out of presumptive TB individuals before making a final diagnosis poses a danger to the individual and their community. We aimed to determine the proportion of \u2026",
-    "full_text": "",
-    "authors": [
-      "G Ekuka",
-      "I Kawooya",
-      "E Kayongo",
-      "R Ssenyonga",
-      "..."
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.ajol.info/index.php/ahs/article/view/197831"
-  },
-  {
-    "key": "KQWSK82P",
-    "title": "Point\u2010of\u2010care diagnostic tests for sickle cell disease",
-    "abstract": "Objectives This is a protocol for a Cochrane Review (diagnostic). The objectives are as follows: To determine the accuracy of point\u2010of\u2010care tests (POCT) to screen sickle cell \u2026",
-    "full_text": "",
-    "authors": [
-      "I Kawooya",
-      "E Kayongo",
-      "D Munube",
-      "..."
-    ],
-    "doi": "10.1002/14651858.CD014584",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.cochranelibrary.com/cdsr/doi/10.1002/14651858.CD014584/abstract"
-  },
-  {
-    "key": "8PMU8JR9",
-    "title": "of nodding syndrome",
-    "abstract": "Aims. Nodding syndrome is a poorly understood acquired disorder affecting children in sub-Saharan Africa. The aetiology and pathogenesis are unknown, and no specific treatment is \u2026",
-    "full_text": "",
-    "authors": [
-      "E Kayongo",
-      "N Gumisiriza",
-      "A Lanyero",
-      "..."
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://pdfs.semanticscholar.org/a0a3/7f6c382fe555492d8640855e31bf233881c0.pdf"
-  },
-  {
-    "key": "AN26LWTD",
-    "title": "Accuracy and incremental yield of the chest X-ray in screening for tuberculosis in Uganda: a cross-sectional study",
-    "abstract": "The WHO END TB strategy requires\u2265 90% case detection to combat tuberculosis (TB). Increased TB case detection requires a more sensitive and specific screening tool \u2026",
-    "full_text": "",
-    "authors": [
-      "J Nalunjogi",
-      "F Mugabe",
-      "I Najjingo",
-      "P Lusiba",
-      "..."
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.hindawi.com/journals/trt/2021/6622809/"
-  },
-  {
-    "key": "MAYC8H5C",
-    "title": "Vulnerability of Populations to Malaria after Indoor Residual Spraying is Withdrawn from Areas where its Use has Previously Been Sustained. Protocol for a Systematic \u2026",
-    "abstract": "Background: With its proven effectiveness, indoor residual spraying (IRS) as a malaria vector control strategy forms one of the reliable vector control strategies, especially when at \u2026",
-    "full_text": "",
-    "authors": [
-      "P Orishaba",
-      "E Kayongo",
-      "P Lusiba",
-      "C Nakalema",
-      "..."
-    ],
-    "doi": "10.1101/2022.05.24.22275507.abstract",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.medrxiv.org/content/10.1101/2022.05.24.22275507.abstract"
-  },
-  {
-    "key": "S2FGVY8R",
-    "title": "Asymptomatic malaria parasitaemia and seizure control in children with nodding syndrome; a cross-sectional study",
-    "abstract": "Objective Plasmodium falciparum is epileptogenic and in malaria endemic areas, is a leading cause of acute seizures. In these areas, asymptomatic infections are common but \u2026",
-    "full_text": "",
-    "authors": [
-      "R Ogwang",
-      "R Anguzu",
-      "P Akun",
-      "A Ningwa",
-      "E Kayongo",
-      "..."
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://bmjopen.bmj.com/content/8/10/e023624.abstract"
-  },
-  {
-    "key": "6KALDEWN",
-    "title": "Adherence to the MDR-TB intensive phase treatment protocol amongst individuals followed up at central and peripheral health care facilities in Uganda-a descriptive \u2026",
-    "abstract": "Background: Following initiation of MDR-TB treatment, patients have a choice to receive follow up DOT supervision at either the central initiating facility or at a peripheral facility \u2026",
-    "full_text": "",
-    "authors": [
-      "J Mukasa",
-      "E Kayongo",
-      "I Kawooya",
-      "D Lukoye",
-      "..."
-    ],
-    "doi": "",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://www.ajol.info/index.php/ahs/article/view/197828"
-  },
-  {
-    "key": "76FY87Z6",
-    "title": "The natural history of nodding syndrome",
-    "abstract": "Aims. Nodding syndrome is a poorly understood acquired disorder affecting children in sub\u2010Saharan Africa. The aetiology and pathogenesis are unknown, and no specific treatment is \u2026",
-    "full_text": "",
-    "authors": [
-      "R Idro",
-      "R Ogwang",
-      "E Kayongo",
-      "N Gumisiriza",
-      "..."
-    ],
-    "doi": "10.1684/epd.2018.1012",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://onlinelibrary.wiley.com/doi/abs/10.1684/epd.2018.1012"
-  },
-  {
-    "key": "TQQ2BYRK",
-    "title": "Paper 1: Demand-driven rapid reviews for health policy and systems decision-making: lessons from Lebanon, Ethiopia, and South Africa on researchers and \u2026",
-    "abstract": "Rapid reviews have emerged as an approach to provide contextualized evidence in a timely and efficient manner. Three rapid review centers were established in Ethiopia, Lebanon \u2026",
-    "full_text": "",
-    "authors": [
-      "RM Mijumbi-Deve",
-      "I Kawooya",
-      "E Kayongo",
-      "R Izizinga",
-      "..."
-    ],
-    "doi": "10.1186/s13643-022-02021-3",
-    "year": null,
-    "item_type": "journalArticle",
-    "url": "https://link.springer.com/article/10.1186/s13643-022-02021-3"
-  }
-]

interface.py

@@ -1,47 +0,0 @@
-"""
-Gradio interface module for ACRES RAG Platform.
-Defines the UI components and layout.
-"""
-
-# interface.py
-import gradio as gr
-
-
-def create_chat_interface() -> gr.Blocks:
-    """Create the chat interface component."""
-    with gr.Blocks() as chat_interface:
-        with gr.Row():
-            with gr.Column(scale=7):
-                chat_history = gr.Chatbot(
-                    value=[], elem_id="chatbot", height=600, show_label=False
-                )
-            with gr.Column(scale=3):
-                pdf_preview = gr.Image(label="Source Page", height=600)
-
-        with gr.Row():
-            with gr.Column(scale=8):
-                query_input = gr.Textbox(
-                    show_label=False,
-                    placeholder="Ask a question about your documents...",
-                    container=False,
-                )
-            with gr.Column(scale=2):
-                submit_btn = gr.Button("Send", variant="primary")
-
-        with gr.Row():
-            pdf_files = gr.File(
-                file_count="multiple", file_types=[".pdf"], label="Upload PDF Files"
-            )
-            collection_name = gr.Textbox(
-                label="Collection Name", placeholder="Name this collection of PDFs..."
-            )
-
-    return (
-        chat_interface,
-        chat_history,
-        pdf_preview,
-        query_input,
-        submit_btn,
-        pdf_files,
-        collection_name,
-    )

rag/rag_pipeline.py

@@ -1,159 +1,70 @@
-# rag/rag_pipeline.py
 import json
-import logging
-from typing import Dict, Any, List
-
+from typing import Dict, Any
 from llama_index.core import Document, VectorStoreIndex
 from llama_index.core.node_parser import SentenceWindowNodeParser, SentenceSplitter
 from llama_index.core import PromptTemplate
+from typing import List
 from llama_index.embeddings.openai import OpenAIEmbedding
 from llama_index.llms.openai import OpenAI
-from llama_index.vector_stores.chroma import ChromaVectorStore
-import chromadb
-from typing import Dict, Any, List, Tuple, Optional
-import re
-import logging
-
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
 
 
 class RAGPipeline:
-    def __init__(
-        self,
-        study_json,
-        collection_name="study_files_rag_collection",
-        use_semantic_splitter=False,
-    ):
+    def __init__(self, study_json, use_semantic_splitter=False):
         self.study_json = study_json
-        self.collection_name = collection_name
         self.use_semantic_splitter = use_semantic_splitter
         self.documents = None
-        self.client = chromadb.Client()
-        self.collection = self.client.get_or_create_collection(self.collection_name)
-        self.embedding_model = OpenAIEmbedding(model_name="text-embedding-ada-002")
-        self.is_pdf = self._check_if_pdf_collection()
+        self.index = None
         self.load_documents()
         self.build_index()
 
-    def _check_if_pdf_collection(self) -> bool:
-        """Check if this is a PDF collection based on the JSON structure."""
-        try:
-            with open(self.study_json, "r") as f:
-                data = json.load(f)
-                # Check first document for PDF-specific fields
-                if data and isinstance(data, list) and len(data) > 0:
-                    return "pages" in data[0] and "source_file" in data[0]
-            return False
-        except Exception as e:
-            logger.error(f"Error checking collection type: {str(e)}")
-            return False
-
-    def extract_page_number_from_query(self, query: str) -> int:
-        """Extract page number from query text."""
-        # Look for patterns like "page 3", "p3", "p. 3", etc.
-        patterns = [
-            r"page\s*(\d+)",
-            r"p\.\s*(\d+)",
-            r"p\s*(\d+)",
-            r"pg\.\s*(\d+)",
-            r"pg\s*(\d+)",
-        ]
-
-        for pattern in patterns:
-            match = re.search(pattern, query.lower())
-            if match:
-                return int(match.group(1))
-        return None
-
     def load_documents(self):
         if self.documents is None:
             with open(self.study_json, "r") as f:
                 self.data = json.load(f)
 
             self.documents = []
-            if self.is_pdf:
-                # Handle PDF documents
-                for index, doc_data in enumerate(self.data):
-                    pages = doc_data.get("pages", {})
-                    for page_num, page_content in pages.items():
-                        if isinstance(page_content, dict):
-                            content = page_content.get("text", "")
-                        else:
-                            content = page_content
 
-                        doc_content = (
-                            f"Title: {doc_data['title']}\n"
-                            f"Page {page_num} Content:\n{content}\n"
-                            f"Authors: {', '.join(doc_data['authors'])}\n"
-                        )
-
-                        metadata = {
-                            "title": doc_data.get("title"),
-                            "authors": ", ".join(doc_data.get("authors", [])),
-                            "year": doc_data.get("date"),
-                            "source_file": doc_data.get("source_file"),
-                            "page_number": int(page_num),
-                            "total_pages": doc_data.get("page_count"),
-                        }
-
-                        self.documents.append(
-                            Document(
-                                text=doc_content,
-                                id_=f"doc_{index}_page_{page_num}",
-                                metadata=metadata,
-                            )
-                        )
-            else:
-                # Handle Zotero documents
-                for index, doc_data in enumerate(self.data):
-                    doc_content = (
-                        f"Title: {doc_data.get('title', '')}\n"
-                        f"Abstract: {doc_data.get('abstract', '')}\n"
-                        f"Authors: {', '.join(doc_data.get('authors', []))}\n"
-                    )
+            for index, doc_data in enumerate(self.data):
+                doc_content = (
+                    f"Title: {doc_data['title']}\n"
+                    f"Abstract: {doc_data['abstract']}\n"
+                    f"Authors: {', '.join(doc_data['authors'])}\n"
+                    # f"full_text: {doc_data['full_text']}"
+                )
 
-                    metadata = {
-                        "title": doc_data.get("title"),
-                        "authors": ", ".join(doc_data.get("authors", [])),
-                        "year": doc_data.get("date"),
-                        "doi": doc_data.get("doi"),
-                    }
+                metadata = {
+                    "title": doc_data.get("title"),
+                    "authors": doc_data.get("authors", []),
+                    "year": doc_data.get("date"),
+                    "doi": doc_data.get("doi"),
+                }
 
-                    self.documents.append(
-                        Document(
-                            text=doc_content, id_=f"doc_{index}", metadata=metadata
-                        )
-                    )
+                self.documents.append(
+                    Document(text=doc_content, id_=f"doc_{index}", metadata=metadata)
+                )
 
     def build_index(self):
-        sentence_splitter = SentenceSplitter(chunk_size=2048, chunk_overlap=20)
+        if self.index is None:
+            sentence_splitter = SentenceSplitter(chunk_size=2048, chunk_overlap=20)
 
-        def _split(text: str) -> List[str]:
-            return sentence_splitter.split_text(text)
+            def _split(text: str) -> List[str]:
+                return sentence_splitter.split_text(text)
 
-        node_parser = SentenceWindowNodeParser.from_defaults(
-            sentence_splitter=_split,
-            window_size=5,
-            window_metadata_key="window",
-            original_text_metadata_key="original_text",
-        )
-
-        # Parse documents into nodes for embedding
-        nodes = node_parser.get_nodes_from_documents(self.documents)
-
-        # Initialize ChromaVectorStore with the existing collection
-        vector_store = ChromaVectorStore(chroma_collection=self.collection)
+            node_parser = SentenceWindowNodeParser.from_defaults(
+                sentence_splitter=_split,
+                window_size=5,
+                window_metadata_key="window",
+                original_text_metadata_key="original_text",
+            )
 
-        # Create the VectorStoreIndex using the ChromaVectorStore
-        self.index = VectorStoreIndex(
-            nodes, vector_store=vector_store, embed_model=self.embedding_model
-        )
+            nodes = node_parser.get_nodes_from_documents(self.documents)
+            self.index = VectorStoreIndex(
+                nodes, embed_model=OpenAIEmbedding(model_name="text-embedding-3-large")
+            )
 
     def query(
         self, context: str, prompt_template: PromptTemplate = None
-    ) -> Tuple[str, Optional[Dict[str, Any]]]:
+    ) -> Dict[str, Any]:
         if prompt_template is None:
             prompt_template = PromptTemplate(
                 "Context information is below.\n"
@@ -161,48 +72,22 @@ class RAGPipeline:
                 "{context_str}\n"
                 "---------------------\n"
                 "Given this information, please answer the question: {query_str}\n"
-                "Provide a detailed answer using the content from the context above. "
-                "If the question asks about specific page content, make sure to include that information. "
+                "Provide an answer to the question using evidence from the context above. "
                 "Cite sources using square brackets for EVERY piece of information, e.g. [1], [2], etc. "
-                "If you're unsure about something, say so rather than making assumptions."
+                "Even if there's only one source, still include the citation. "
+                "If you're unsure about a source, use [?]. "
+                "Ensure that EVERY statement from the context is properly cited."
             )
 
-        # Extract page number for PDF documents
-        requested_page = (
-            self.extract_page_number_from_query(context) if self.is_pdf else None
-        )
-
-
         # This is a hack to index all the documents in the store :)
         n_documents = len(self.index.docstore.docs)
-        print(f"n_documents: {n_documents}")
         query_engine = self.index.as_query_engine(
             text_qa_template=prompt_template,
-            similarity_top_k=n_documents if n_documents <= 17 else 15,
+            similarity_top_k=n_documents,
             response_mode="tree_summarize",
             llm=OpenAI(model="gpt-4o-mini"),
         )
 
         response = query_engine.query(context)
 
-        # Handle source information based on document type
-        source_info = None
-        if hasattr(response, "source_nodes") and response.source_nodes:
-            source_node = response.source_nodes[0]
-            metadata = source_node.metadata
-
-            if self.is_pdf:
-                page_number = (
-                    requested_page
-                    if requested_page is not None
-                    else metadata.get("page_number", 0)
-                )
-                source_info = {
-                    "source_file": metadata.get("source_file"),
-                    "page_number": page_number,
-                    "title": metadata.get("title"),
-                    "authors": metadata.get("authors"),
-                    "content": source_node.text,
-                }
-
-        return response.response, source_info
+        return response

rag/rag_pipeline_backup.py

@@ -1,94 +0,0 @@
-import json
-from typing import Dict, Any
-from llama_index.core import Document, VectorStoreIndex
-from llama_index.core.node_parser import SentenceWindowNodeParser, SentenceSplitter
-from llama_index.core import PromptTemplate
-from typing import List
-from llama_index.embeddings.openai import OpenAIEmbedding
-from llama_index.llms.openai import OpenAI
-
-
-class RAGPipeline:
-    def __init__(self, study_json, use_semantic_splitter=False):
-        self.study_json = study_json
-        self.use_semantic_splitter = use_semantic_splitter
-        self.documents = None
-        self.index = None
-        self.load_documents()
-        self.build_index()
-
-    def load_documents(self):
-        if self.documents is None:
-            with open(self.study_json, "r") as f:
-                self.data = json.load(f)
-
-            self.documents = []
-
-            for index, doc_data in enumerate(self.data):
-                doc_content = (
-                    f"Title: {doc_data['title']}\n"
-                    f"Abstract: {doc_data['abstract']}\n"
-                    f"Authors: {', '.join(doc_data['authors'])}\n"
-                    # f"full_text: {doc_data['full_text']}"
-                )
-
-                metadata = {
-                    "title": doc_data.get("title"),
-                    "authors": doc_data.get("authors", []),
-                    "year": doc_data.get("date"),
-                    "doi": doc_data.get("doi"),
-                }
-
-                self.documents.append(
-                    Document(text=doc_content, id_=f"doc_{index}", metadata=metadata)
-                )
-
-    def build_index(self):
-        if self.index is None:
-            sentence_splitter = SentenceSplitter(chunk_size=2048, chunk_overlap=20)
-
-            def _split(text: str) -> List[str]:
-                return sentence_splitter.split_text(text)
-
-            node_parser = SentenceWindowNodeParser.from_defaults(
-                sentence_splitter=_split,
-                window_size=5,
-                window_metadata_key="window",
-                original_text_metadata_key="original_text",
-            )
-
-            nodes = node_parser.get_nodes_from_documents(self.documents)
-            self.index = VectorStoreIndex(
-                nodes, embed_model=OpenAIEmbedding(model_name="text-embedding-3-large")
-            )
-
-    def query(
-        self, context: str, prompt_template: PromptTemplate = None
-    ) -> Dict[str, Any]:
-        if prompt_template is None:
-            prompt_template = PromptTemplate(
-                "Context information is below.\n"
-                "---------------------\n"
-                "{context_str}\n"
-                "---------------------\n"
-                "Given this information, please answer the question: {query_str}\n"
-                "Provide an answer to the question using evidence from the context above. "
-                "Cite sources using square brackets for EVERY piece of information, e.g. [1], [2], etc. "
-                "Even if there's only one source, still include the citation. "
-                "If you're unsure about a source, use [?]. "
-                "Ensure that EVERY statement from the context is properly cited."
-            )
-
-        # This is a hack to index all the documents in the store :)
-        n_documents = len(self.index.docstore.docs)
-        print(f"n_documents: {n_documents}")
-        query_engine = self.index.as_query_engine(
-            text_qa_template=prompt_template,
-            similarity_top_k=n_documents if n_documents <= 17 else 15,
-            response_mode="tree_summarize",
-            llm=OpenAI(model="gpt-4o-mini"),
-        )
-
-        response = query_engine.query(context)
-
-        return response

requirements.txt

@@ -2,13 +2,10 @@ chromadb==0.5.5
 fastapi==0.112.2
 gradio
 llama-index
-llama-index-vector-stores-chroma
 nest-asyncio==1.6.0
 openai
 pandas
 pydantic
 python-dotenv
 pyzotero
-python-slugify
-PyMuPDF==1.23.8  
-Pillow==10.2.0 
+python-slugify

study_files.json

@@ -2,7 +2,5 @@
     "Vaccine coverage": "data/vaccine_coverage_zotero_items.json",
     "Ebola Virus": "data/ebola_virus_zotero_items.json",
     "GeneXpert": "data/gene_xpert_zotero_items.json",
-    "Zotero Collection Pastan": "data/zotero-collection-pastan_zotero_items.json",
-    "EBSCOhost": "data/ebscohost_zotero_items.json",
-    "ExportedRis_file_1_of_1 (1)": "data/exportedris-file-1-of-1-1_zotero_items.json"
+    "Zotero Collection Pastan": "data/zotero-collection-pastan_zotero_items.json"
 }

study_files_backup.json

@@ -1,13 +0,0 @@
-{
-    "Vaccine coverage": "data/vaccine_coverage_zotero_items.json",
-    "Ebola Virus": "data/ebola_virus_zotero_items.json",
-    "GeneXpert": "data/gene_xpert_zotero_items.json",
-    "Zotero Collection Pastan": "data/zotero-collection-pastan_zotero_items.json",
-    "EBSCOhost": "data/ebscohost_zotero_items.json",
-    "ref BMGF": "data/ref-bmgf_zotero_items.json",
-    "scholar (29)": "data/scholar-29_zotero_items.json",
-    "iom": "data/iom_zotero_items.json",
-    "ExportedRis_file_1_of_1 (1)": "data/exportedris-file-1-of-1-1_zotero_items.json",
-    "wb_1813-9450-6689": "data/wb-1813-9450-6689_zotero_items.json",
-    "kayongo papers": "data/kayongo-papers_zotero_items.json"
-}

utils/helpers.py

@@ -1,5 +1,3 @@
-# utils/helpers.py
-
 from typing import Dict, Any
 from llama_index.core import Response
 from typing import List
@@ -10,13 +8,6 @@ from utils.prompts import (
     StudyCharacteristics,
 )
 import json
-import json
-import chromadb
-from chromadb.api.types import Document
-
-# Initialize ChromaDB client
-chromadb_client = chromadb.Client()
-
 
 def read_study_files(file_path):
     """
@@ -31,7 +22,7 @@ def read_study_files(file_path):
     Raises:
         FileNotFoundError: If the file is not found at the provided path.
         json.JSONDecodeError: If the file contents are not valid JSON.
-
+        
     Example:
         Given a JSON file 'study_files.json' with content like:
         {
@@ -48,15 +39,13 @@ def read_study_files(file_path):
         }
     """
     try:
-        with open(file_path, "r") as file:
+        with open(file_path, 'r') as file:
             data = json.load(file)
         return data
     except FileNotFoundError as e:
         raise FileNotFoundError(f"The file at path {file_path} was not found.") from e
     except json.JSONDecodeError as e:
-        raise ValueError(
-            f"The file at path {file_path} does not contain valid JSON."
-        ) from e
+        raise ValueError(f"The file at path {file_path} does not contain valid JSON.") from e
 
 
 def append_to_study_files(file_path, new_key, new_value):
@@ -90,22 +79,20 @@ def append_to_study_files(file_path, new_key, new_value):
     """
     try:
         # Read the existing data from the file
-        with open(file_path, "r") as file:
+        with open(file_path, 'r') as file:
             data = json.load(file)
-
+        
         # Append the new key-value pair to the dictionary
         data[new_key] = new_value
 
         # Write the updated data back to the file
-        with open(file_path, "w") as file:
+        with open(file_path, 'w') as file:
             json.dump(data, file, indent=4)  # indent for pretty printing
 
     except FileNotFoundError as e:
         raise FileNotFoundError(f"The file at path {file_path} was not found.") from e
     except json.JSONDecodeError as e:
-        raise ValueError(
-            f"The file at path {file_path} does not contain valid JSON."
-        ) from e
+        raise ValueError(f"The file at path {file_path} does not contain valid JSON.") from e
     except IOError as e:
         raise IOError(f"Failed to write to the file at {file_path}.") from e
 
@@ -178,43 +165,3 @@ def generate_follow_up_questions(
         if cleaned_q:
             cleaned_questions.append(f"✨ {cleaned_q}")
     return cleaned_questions[:3]
-
-
-def add_study_files_to_chromadb(file_path: str, collection_name: str):
-    """
-    Reads the study files data from a JSON file and adds it to the specified ChromaDB collection.
-
-    :param file_path: Path to the JSON file containing study files data.
-    :param collection_name: Name of the ChromaDB collection to store the data.
-    """
-    # Load study files data from JSON file
-    try:
-        with open(file_path, "r") as f:
-            study_files_data = json.load(f)
-    except FileNotFoundError:
-        print(f"File '{file_path}' not found.")
-        return
-
-    # Get or create the collection in ChromaDB
-    collection = chromadb_client.get_or_create_collection(collection_name)
-
-    # Prepare lists for ids, texts, and metadata to batch insert
-    ids = []
-    documents = []
-    metadatas = []
-
-    # Populate lists with data from the JSON file
-    for name, file_path in study_files_data.items():
-        ids.append(name)  # Document ID
-        documents.append("")  # Optional text, can be left empty if not used
-        metadatas.append({"file_path": file_path})  # Metadata with file path
-
-    # Add documents to the collection in batch
-    collection.add(ids=ids, documents=documents, metadatas=metadatas)
-
-    print("All study files have been successfully added to ChromaDB.")
-
-
-if __name__ == "__main__":
-    # Usage example
-    add_study_files_to_chromadb("study_files.json", "study_files_collection")

utils/pdf_processor.py

@@ -1,193 +0,0 @@
-"""
-PDF processing module for ACRES RAG Platform.
-Handles PDF file processing, text extraction, and page rendering.
-"""
-
-# utils/pdf_processor.py
-import os
-import fitz
-import logging
-from typing import Dict, List, Optional
-import datetime
-from slugify import slugify
-import json
-from PIL import Image
-import re
-
-
-logger = logging.getLogger(__name__)
-
-
-class PDFProcessor:
-    def __init__(self, upload_dir: str = "data/uploads"):
-        """Initialize PDFProcessor with upload directory."""
-        self.upload_dir = upload_dir
-        os.makedirs(upload_dir, exist_ok=True)
-        self.current_page = 0
-
-    def render_page(self, file_path: str, page_num: int) -> Optional[Image.Image]:
-        """Render a specific page from a PDF as an image."""
-        try:
-            logger.info(f"Attempting to render page {page_num} from {file_path}")
-            doc = fitz.open(file_path)
-
-            # Ensure page number is valid
-            if page_num < 0 or page_num >= len(doc):
-                logger.error(
-                    f"Invalid page number {page_num} for document with {len(doc)} pages"
-                )
-                return None
-
-            page = doc[page_num]
-            # Increase resolution for better quality
-            pix = page.get_pixmap(matrix=fitz.Matrix(300 / 72, 300 / 72))
-            image = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
-            doc.close()
-            logger.info(f"Successfully rendered page {page_num}")
-            return image
-        except Exception as e:
-            logger.error(f"Error rendering page {page_num} from {file_path}: {str(e)}")
-            return None
-
-    def is_references_page(self, text: str) -> bool:
-        """
-        Check if the page appears to be a references/bibliography page.
-        """
-        # Common section headers for references
-        ref_headers = [
-            r"^references\s*$",
-            r"^bibliography\s*$",
-            r"^works cited\s*$",
-            r"^citations\s*$",
-            r"^cited literature\s*$",
-        ]
-
-        # Check first few lines of the page
-        first_lines = text.lower().split("\n")[:3]
-        first_block = " ".join(first_lines)
-
-        # Check for reference headers
-        for header in ref_headers:
-            if re.search(header, first_block, re.IGNORECASE):
-                return True
-
-        # Check for reference-like patterns (e.g., [1] Author, et al.)
-        ref_patterns = [
-            r"^\[\d+\]",  # [1] style
-            r"^\d+\.",  # 1. style
-            r"^[A-Z][a-z]+,\s+[A-Z]\.",  # Author, I. style
-        ]
-
-        ref_pattern_count = 0
-        lines = text.split("\n")[:10]  # Check first 10 lines
-        for line in lines:
-            line = line.strip()
-            if any(re.match(pattern, line) for pattern in ref_patterns):
-                ref_pattern_count += 1
-
-        # If multiple reference-like patterns are found, likely a references page
-        return ref_pattern_count >= 3
-
-    def detect_references_start(self, doc: fitz.Document) -> Optional[int]:
-        """
-        Detect the page where references section starts.
-        Returns the page number or None if not found.
-        """
-        for page_num in range(len(doc)):
-            page = doc[page_num]
-            text = page.get_text()
-            if self.is_references_page(text):
-                logger.info(f"Detected references section starting at page {page_num}")
-                return page_num
-        return None
-
-    def process_pdfs(self, file_paths: List[str], collection_name: str) -> str:
-        """Process multiple PDF files and store their content."""
-        processed_docs = []
-
-        for file_path in file_paths:
-            try:
-                doc_data = self.extract_text_from_pdf(file_path)
-                processed_docs.append(doc_data)
-                logger.info(
-                    f"Successfully processed {file_path} ({doc_data['content_pages']} content pages)"
-                )
-            except Exception as e:
-                logger.error(f"Error processing {file_path}: {str(e)}")
-                continue
-
-        if not processed_docs:
-            raise ValueError("No documents were successfully processed")
-
-        # Save to JSON file
-        timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
-        output_filename = f"{slugify(collection_name)}_{timestamp}_documents.json"
-        output_path = os.path.join("data", output_filename)
-
-        # Ensure the data directory exists
-        os.makedirs("data", exist_ok=True)
-
-        with open(output_path, "w", encoding="utf-8") as f:
-            json.dump(processed_docs, f, indent=2, ensure_ascii=False)
-
-        logger.info(f"Saved processed documents to {output_path}")
-        return output_path
-
-    def extract_text_from_pdf(self, file_path: str) -> Dict:
-        """Extract text and metadata from a PDF file."""
-        try:
-            doc = fitz.open(file_path)
-
-            # Find references section start
-            refs_start = self.detect_references_start(doc)
-
-            # Extract text from all pages with page tracking
-            text = ""
-            pages = {}
-            for page_num in range(len(doc)):
-                # Skip if this is after references section starts
-                if refs_start is not None and page_num >= refs_start:
-                    logger.info(
-                        f"Skipping page {page_num} as it appears to be part of references"
-                    )
-                    continue
-
-                page_text = doc[page_num].get_text()
-
-                # Extra check to catch references if they weren't caught by the initial scan
-                if page_num > 0 and self.is_references_page(page_text):
-                    logger.info(
-                        f"Detected references content on page {page_num}, skipping"
-                    )
-                    continue
-
-                pages[str(page_num)] = page_text
-                text += page_text + "\n"
-
-            # Extract metadata
-            metadata = doc.metadata
-            if not metadata.get("title"):
-                metadata["title"] = os.path.basename(file_path)
-
-            # Create structured document
-            document = {
-                "title": metadata.get("title", ""),
-                "authors": (
-                    metadata.get("author", "").split(";")
-                    if metadata.get("author")
-                    else []
-                ),
-                "date": metadata.get("creationDate", ""),
-                "abstract": text[:500] + "..." if len(text) > 500 else text,
-                "full_text": text,
-                "source_file": file_path,
-                "pages": pages,
-                "page_count": len(doc),
-                "content_pages": len(pages),  # Number of pages excluding references
-            }
-
-            doc.close()
-            return document
-        except Exception as e:
-            logger.error(f"Error processing PDF {file_path}: {str(e)}")
-            raise

utils/prompts.py

@@ -1,5 +1,3 @@
-# utils/prompts.py
-
 from llama_index.core import PromptTemplate
 from typing import Optional, List
 from pydantic import BaseModel, Field

utils/zotero_manager.py

@@ -1,5 +1,3 @@
-# utils/zotero_manager.py
-
 import json
 import os
 from typing import Any, Dict, List, Optional
@@ -643,4 +641,4 @@ if __name__ == "__main__":
     ## Save to disc
     zotero_manager.write_zotero_items_to_json_file(
         ebora_virus_zotero_items_json, "zotero_data/ebora_virus_zotero_items.json"
-    )
+    )