backend/main/prepare_data.py

from pathlib import Path

from backend.classes.data_preparer import DataPreparerConfig, DataPreparer
from backend.utils.utils import initialize_logger, read_config, set_env_variables
from dotenv import load_dotenv

load_dotenv()


def run(config: dict):
    """
    Run the RAG application.
    :param config: Configuration dictionary
    """
    logger.info("Prepare data")
    data_preparer_config = DataPreparerConfig(input_data_path=config["data"]["input_data_path"],
                                              output_data_path=config["data"]["output_data_path"],
                                              output_file=config["data"]["output_file"],
                                              pdf_extractor=config["pdf_extractor"],
                                              vector_database=config["vector_database"],
                                              embedding_model=config["embedding_model"]) 
    data_preparer = DataPreparer(data_preparer_config)
    data_preparer.prepare_data()
    logger.info("Data prepared")



if __name__ == '__main__':
    logger = initialize_logger()

    # get current file path using Path
    config = read_config(str(Path(Path(__file__).parent, "../conf/config.yaml")))

    # check if environment variables are set
    env_variables = set_env_variables(config["env_variables"])

    app_config = config[env_variables["APP_ENV"]]
    app_config["env_vars"] = env_variables

    run(app_config)