Upload upload to hub.py
Browse files- upload to hub.py +184 -0
upload to hub.py
ADDED
@@ -0,0 +1,184 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#import required functions
|
2 |
+
import os
|
3 |
+
from huggingface_hub import login, get_token, whoami, HfApi, create_repo, repo_exists, list_repo_refs, create_branch
|
4 |
+
|
5 |
+
#define clear screen function
|
6 |
+
oname = os.name
|
7 |
+
if oname == 'nt':
|
8 |
+
osclear = 'cls'
|
9 |
+
elif oname == 'posix':
|
10 |
+
osclear = 'clear'
|
11 |
+
else:
|
12 |
+
osclear = ''
|
13 |
+
def clear_screen():
|
14 |
+
os.system(osclear)
|
15 |
+
|
16 |
+
#clear before starting
|
17 |
+
clear_screen()
|
18 |
+
|
19 |
+
#get token
|
20 |
+
if os.environ.get('KAGGLE_KERNEL_RUN_TYPE', None) is not None: #check if user in kaggle
|
21 |
+
from kaggle_secrets import UserSecretsClient
|
22 |
+
from kaggle_web_client import BackendError
|
23 |
+
try:
|
24 |
+
login(UserSecretsClient().get_secret("HF_TOKEN")) #login if token secret found
|
25 |
+
except BackendError:
|
26 |
+
print('''
|
27 |
+
When using Kaggle, make sure to use the secret key HF_TOKEN with a 'WRITE' token.
|
28 |
+
This will prevent the need to login every time you run the script.
|
29 |
+
Set your secrets with the secrets add-on on the top of the screen.
|
30 |
+
''')
|
31 |
+
if get_token() is not None:
|
32 |
+
#if the token is found then log in:
|
33 |
+
login(get_token())
|
34 |
+
tfound = "Where are my doritos?" #doesn't matter what this is, only false is used
|
35 |
+
else:
|
36 |
+
#if the token is not found then prompt user to provide it:
|
37 |
+
login(input("API token not detected. Enter your HuggingFace (WRITE) token: "))
|
38 |
+
tfound = "false"
|
39 |
+
|
40 |
+
#if the token is read only then prompt user to provide a write token:
|
41 |
+
while True:
|
42 |
+
if whoami().get('auth', {}).get('accessToken', {}).get('role', None) != 'write':
|
43 |
+
clear_screen()
|
44 |
+
if os.environ.get('HF_TOKEN', None) is not None: #if environ finds HF_TOKEN as read-only then display following text and exit:
|
45 |
+
print('''
|
46 |
+
You have the environment variable HF_TOKEN set.
|
47 |
+
You cannot log in.
|
48 |
+
Either set the environment variable to a 'WRITE' token or remove it.
|
49 |
+
''')
|
50 |
+
input("Press enter to continue.")
|
51 |
+
exit()
|
52 |
+
if os.environ.get('COLAB_BACKEND_VERSION', None) is not None: #read-only colab secret key found
|
53 |
+
print('''
|
54 |
+
Your Colab secret key is read-only
|
55 |
+
Please switch your key to 'write' or disable notebook access on the left.
|
56 |
+
For now, you are stuck in a loop
|
57 |
+
''')
|
58 |
+
elif os.environ.get('KAGGLE_KERNEL_RUN_TYPE', None) is not None: #read-only kaggle secret key found
|
59 |
+
print('''
|
60 |
+
Your Kaggle secret key is read-only
|
61 |
+
Please switch your key to 'write' or unattach from notebook in add-ons at the top.
|
62 |
+
Having a read-only key attched will require login every time.
|
63 |
+
''')
|
64 |
+
print("You do not have write access to this repository. Please use a valid token with (WRITE) access.")
|
65 |
+
login(input("Enter your HuggingFace (WRITE) token: "))
|
66 |
+
continue
|
67 |
+
break
|
68 |
+
clear_screen()
|
69 |
+
|
70 |
+
#store actions into variables
|
71 |
+
#upload directory
|
72 |
+
up_dir = input("Enter the directory you want to upload: ")
|
73 |
+
clear_screen()
|
74 |
+
|
75 |
+
#name of affected repository
|
76 |
+
repo = input("Repository name (User/Repo): ")
|
77 |
+
clear_screen()
|
78 |
+
|
79 |
+
#type of huggingface repository (restricted)
|
80 |
+
while True:
|
81 |
+
r_type = input("Repo type (model) (dataset) (space): ").lower()
|
82 |
+
|
83 |
+
if r_type not in ['model', 'dataset', 'space', 'm', 'd', 's']:
|
84 |
+
clear_screen()
|
85 |
+
print("Please choose one of the following three options.")
|
86 |
+
continue
|
87 |
+
if r_type == 'm':
|
88 |
+
r_type = 'model'
|
89 |
+
elif r_type == 'd':
|
90 |
+
r_type = 'dataset'
|
91 |
+
elif r_type == 's':
|
92 |
+
r_type = 'space'
|
93 |
+
break
|
94 |
+
clear_screen()
|
95 |
+
|
96 |
+
#if new, ask to create private
|
97 |
+
priv = "nothing yet"
|
98 |
+
new_r = False
|
99 |
+
if repo_exists(repo, repo_type=r_type) == False:
|
100 |
+
new_r = True
|
101 |
+
while True:
|
102 |
+
priv = input(f"No existing repo found, create private repository? (y/N): ").lower()
|
103 |
+
if priv == '':
|
104 |
+
priv = 'n'
|
105 |
+
elif priv == 'yes':
|
106 |
+
priv = 'y'
|
107 |
+
elif priv == 'no':
|
108 |
+
priv = 'n'
|
109 |
+
break
|
110 |
+
else:
|
111 |
+
if priv not in ['y', 'n']:
|
112 |
+
clear_screen()
|
113 |
+
print("Please choose one of the following two options.")
|
114 |
+
continue
|
115 |
+
break
|
116 |
+
clear_screen()
|
117 |
+
|
118 |
+
#ask for optional commit message
|
119 |
+
c_mes = input("Commit message (optional): ")
|
120 |
+
if c_mes == "":
|
121 |
+
c_mes = "Uploaded files with huggingface_hub"
|
122 |
+
clear_screen()
|
123 |
+
|
124 |
+
#ask for optional branch name
|
125 |
+
rev = input("Branch name (empty for main): ")
|
126 |
+
if rev == "":
|
127 |
+
rev = "main"
|
128 |
+
clear_screen()
|
129 |
+
|
130 |
+
#if branch doesn't exist, ask to create it
|
131 |
+
if not any(branch.name == rev for branch in list_repo_refs(repo, repo_type=r_type).branches):
|
132 |
+
while True:
|
133 |
+
c_rev = input(f"No existing branch '{rev}' found, create new branch? (Y/n): ").lower()
|
134 |
+
if c_rev == '':
|
135 |
+
c_rev = 'y'
|
136 |
+
elif c_rev == 'yes':
|
137 |
+
c_rev = 'y'
|
138 |
+
elif c_rev == 'no':
|
139 |
+
c_rev = 'n'
|
140 |
+
break
|
141 |
+
else:
|
142 |
+
if c_rev not in ['y', 'n']:
|
143 |
+
clear_screen()
|
144 |
+
print("Please choose one of the following two options.")
|
145 |
+
continue
|
146 |
+
break
|
147 |
+
if c_rev == 'n':
|
148 |
+
exit()
|
149 |
+
else:
|
150 |
+
create_branch(repo, branch=rev, repo_type=r_type)
|
151 |
+
clear_screen()
|
152 |
+
|
153 |
+
#ask for optional path in repo
|
154 |
+
r_path = input("Path in repo (empty for root): ")
|
155 |
+
if r_path == "":
|
156 |
+
r_path = "."
|
157 |
+
|
158 |
+
#upload the folder, create the repo if new
|
159 |
+
if priv == "y":
|
160 |
+
create_repo(repo, repo_type=r_type, private=True) #if private chosen, create private repo first
|
161 |
+
else:
|
162 |
+
if new_r == True:
|
163 |
+
create_repo(repo, repo_type=r_type, private=False) #if public chosen, create public repo first
|
164 |
+
HfApi().upload_folder(folder_path=up_dir, repo_id=repo, repo_type=r_type, commit_message=c_mes, revision=rev, path_in_repo=r_path)
|
165 |
+
|
166 |
+
#Show user new repo
|
167 |
+
if new_r == True:
|
168 |
+
if r_type == 'model':
|
169 |
+
print(f"Repository created at https://huggingface.co/{repo}")
|
170 |
+
elif r_type == 'dataset':
|
171 |
+
print(f"Repository created at https://huggingface.co/datasets/{repo}")
|
172 |
+
elif r_type == 'space':
|
173 |
+
print(f"Repository created at https://huggingface.co/spaces/{repo}")
|
174 |
+
|
175 |
+
#if token wasn't found, (new log in) show user's name.
|
176 |
+
if tfound == 'false':
|
177 |
+
print(f'''
|
178 |
+
You are now logged in as {whoami().get('fullname', None)}.
|
179 |
+
|
180 |
+
To logout, use the hf command line interface 'huggingface-cli logout'
|
181 |
+
To view your active account, use 'huggingface-cli whoami'
|
182 |
+
''')
|
183 |
+
|
184 |
+
input("Press enter to continue.")
|