File size: 1,273 Bytes
9c4b01e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 |
import os
import boto3
import gdown
import tempfile
import shutil
from dotenv import load_dotenv
from utils.utils import upload_file
load_dotenv()
# Environment variables
R2_ACCESS_KEY = os.getenv('R2_ACCESS_KEY')
R2_SECRET_KEY = os.getenv('R2_SECRET_KEY')
R2_BUCKET_NAME = os.getenv('R2_BUCKET_NAME')
R2_ENDPOINT_URL = os.getenv('R2_ENDPOINT_URL')
def download_from_google_folder(url):
# Create a temporary directory
with tempfile.TemporaryDirectory() as download_dir:
print(f'Downloading folder to temporary directory: {download_dir}')
# Download the entire folder
gdown.download_folder(url, output=download_dir, quiet=False)
res = []
# Upload files to R2
for root, _, files in os.walk(download_dir):
for file_name in files:
file_path = os.path.join(root, file_name)
object_name = os.path.relpath(file_path, download_dir)
print(f'Uploading file: {file_path}, object name: {object_name}')
upload_file(file_path, R2_BUCKET_NAME, object_name, R2_ENDPOINT_URL, R2_ACCESS_KEY, R2_SECRET_KEY)
res.append(f'https://pub-08a118f4cb7c4b208b55e6877b0bacca.r2.dev/warden-ai/{object_name}')
print(res)
return res
|