|
import hashlib
|
|
import os
|
|
import json
|
|
|
|
sha1s = {}
|
|
|
|
|
|
for root, dirs, files in os.walk("."):
|
|
if ".git" in dirs:
|
|
dirs.remove(".git")
|
|
for name in files:
|
|
file_path = os.path.join(root, name)
|
|
with open(file_path, "rb") as f:
|
|
data = f.read()
|
|
sha1 = hashlib.sha1(data).hexdigest()
|
|
sha1s[file_path] = sha1
|
|
|
|
|
|
with open("dl.py", "w") as f:
|
|
f.write("#!/usr/bin/env python\n\nimport os\nimport hashlib\n")
|
|
for file_path, sha1 in sha1s.items():
|
|
f.write(f"print('Downloading {file_path}')\n")
|
|
f.write(f"if not os.path.exists('{file_path}') or '{sha1}' != hashlib.sha1(open('{file_path}', 'rb').read()).hexdigest():\n")
|
|
f.write(f" os.system('curl -L \"https://huggingface.co/AI-Porn/AI-PORN/resolve/main/{file_path}\" --create-dirs -o \"./{file_path}\"')\n")
|
|
|
|
file_list = {}
|
|
for root, dirs, files in os.walk("."):
|
|
if ".git" in dirs:
|
|
dirs.remove(".git")
|
|
folder_name = os.path.basename(root)
|
|
file_list[folder_name] = files
|
|
file_list[folder_name].extend(dirs)
|
|
|
|
|
|
|
|
|
