utils/gdown_down.py

import os
import boto3
import gdown
import tempfile
import shutil
from dotenv import load_dotenv
from utils.utils import upload_file

load_dotenv()
# Environment variables
R2_ACCESS_KEY = os.getenv('R2_ACCESS_KEY')
R2_SECRET_KEY = os.getenv('R2_SECRET_KEY')
R2_BUCKET_NAME = os.getenv('R2_BUCKET_NAME')
R2_ENDPOINT_URL = os.getenv('R2_ENDPOINT_URL')


def download_from_google_folder(url):
    # Create a temporary directory
    with tempfile.TemporaryDirectory() as download_dir:
        print(f'Downloading folder to temporary directory: {download_dir}')
        # Download the entire folder
        gdown.download_folder(url, output=download_dir, quiet=False)

        res = []
        # Upload files to R2
        for root, _, files in os.walk(download_dir):
            for file_name in files:
                file_path = os.path.join(root, file_name)
                object_name = os.path.relpath(file_path, download_dir)
                print(f'Uploading file: {file_path}, object name: {object_name}')
                upload_file(file_path, R2_BUCKET_NAME, object_name, R2_ENDPOINT_URL, R2_ACCESS_KEY, R2_SECRET_KEY)
                res.append(f'https://pub-08a118f4cb7c4b208b55e6877b0bacca.r2.dev/warden-ai/{object_name}')
        print(res)
        return res