File size: 1,987 Bytes
d1eeaf0 5b260bd d1eeaf0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 |
import os
import requests
from tqdm import tqdm
# Define constants for your folder and URLs
LOCAL_SAVE_PATH = "data" # Path where files will be saved
GITHUB_API_URL = "https://api.github.com/repos/chen700564/RGB/contents/data"
RAW_BASE_URL = "https://raw.githubusercontent.com/chen700564/RGB/master/data/"
# Ensure the directory exists before downloading
os.makedirs(LOCAL_SAVE_PATH, exist_ok=True)
def get_file_list():
"""Fetch the list of files from the GitHub repository."""
response = requests.get(GITHUB_API_URL)
if response.status_code == 200:
# Filter files starting with "en" and return their names
return [file["name"] for file in response.json() if file["name"].startswith("en")]
else:
print("Failed to fetch file list:", response.text)
return []
def file_exists_in_local_directory(file_name):
"""Check if a file with the given name exists in the local directory."""
local_file_path = os.path.join(LOCAL_SAVE_PATH, file_name)
return os.path.exists(local_file_path)
def download_file(file_name):
"""Download a file if it doesn't exist locally."""
if file_exists_in_local_directory(file_name):
print(f"File already exists: {file_name}. Skipping download.")
return
file_url = RAW_BASE_URL + file_name
local_file_path = os.path.join(LOCAL_SAVE_PATH, file_name)
response = requests.get(file_url, stream=True)
if response.status_code == 200:
total_size = int(response.headers.get("content-length", 0))
with open(local_file_path, "wb") as file, tqdm(
total=total_size, unit="B", unit_scale=True, desc=file_name
) as progress:
for chunk in response.iter_content(chunk_size=1024):
file.write(chunk)
progress.update(len(chunk))
print(f"Downloaded: {file_name}")
else:
print(f"Failed to download {file_name}: {response.status_code}")
|