Anthonyg5005 commited on
Commit
e34be54
·
verified ·
1 Parent(s): 4cd25d5

Upload upload to hub.py

Browse files
Files changed (1) hide show
  1. upload to hub.py +184 -0
upload to hub.py ADDED
@@ -0,0 +1,184 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #import required functions
2
+ import os
3
+ from huggingface_hub import login, get_token, whoami, HfApi, create_repo, repo_exists, list_repo_refs, create_branch
4
+
5
+ #define clear screen function
6
+ oname = os.name
7
+ if oname == 'nt':
8
+ osclear = 'cls'
9
+ elif oname == 'posix':
10
+ osclear = 'clear'
11
+ else:
12
+ osclear = ''
13
+ def clear_screen():
14
+ os.system(osclear)
15
+
16
+ #clear before starting
17
+ clear_screen()
18
+
19
+ #get token
20
+ if os.environ.get('KAGGLE_KERNEL_RUN_TYPE', None) is not None: #check if user in kaggle
21
+ from kaggle_secrets import UserSecretsClient
22
+ from kaggle_web_client import BackendError
23
+ try:
24
+ login(UserSecretsClient().get_secret("HF_TOKEN")) #login if token secret found
25
+ except BackendError:
26
+ print('''
27
+ When using Kaggle, make sure to use the secret key HF_TOKEN with a 'WRITE' token.
28
+ This will prevent the need to login every time you run the script.
29
+ Set your secrets with the secrets add-on on the top of the screen.
30
+ ''')
31
+ if get_token() is not None:
32
+ #if the token is found then log in:
33
+ login(get_token())
34
+ tfound = "Where are my doritos?" #doesn't matter what this is, only false is used
35
+ else:
36
+ #if the token is not found then prompt user to provide it:
37
+ login(input("API token not detected. Enter your HuggingFace (WRITE) token: "))
38
+ tfound = "false"
39
+
40
+ #if the token is read only then prompt user to provide a write token:
41
+ while True:
42
+ if whoami().get('auth', {}).get('accessToken', {}).get('role', None) != 'write':
43
+ clear_screen()
44
+ if os.environ.get('HF_TOKEN', None) is not None: #if environ finds HF_TOKEN as read-only then display following text and exit:
45
+ print('''
46
+ You have the environment variable HF_TOKEN set.
47
+ You cannot log in.
48
+ Either set the environment variable to a 'WRITE' token or remove it.
49
+ ''')
50
+ input("Press enter to continue.")
51
+ exit()
52
+ if os.environ.get('COLAB_BACKEND_VERSION', None) is not None: #read-only colab secret key found
53
+ print('''
54
+ Your Colab secret key is read-only
55
+ Please switch your key to 'write' or disable notebook access on the left.
56
+ For now, you are stuck in a loop
57
+ ''')
58
+ elif os.environ.get('KAGGLE_KERNEL_RUN_TYPE', None) is not None: #read-only kaggle secret key found
59
+ print('''
60
+ Your Kaggle secret key is read-only
61
+ Please switch your key to 'write' or unattach from notebook in add-ons at the top.
62
+ Having a read-only key attched will require login every time.
63
+ ''')
64
+ print("You do not have write access to this repository. Please use a valid token with (WRITE) access.")
65
+ login(input("Enter your HuggingFace (WRITE) token: "))
66
+ continue
67
+ break
68
+ clear_screen()
69
+
70
+ #store actions into variables
71
+ #upload directory
72
+ up_dir = input("Enter the directory you want to upload: ")
73
+ clear_screen()
74
+
75
+ #name of affected repository
76
+ repo = input("Repository name (User/Repo): ")
77
+ clear_screen()
78
+
79
+ #type of huggingface repository (restricted)
80
+ while True:
81
+ r_type = input("Repo type (model) (dataset) (space): ").lower()
82
+
83
+ if r_type not in ['model', 'dataset', 'space', 'm', 'd', 's']:
84
+ clear_screen()
85
+ print("Please choose one of the following three options.")
86
+ continue
87
+ if r_type == 'm':
88
+ r_type = 'model'
89
+ elif r_type == 'd':
90
+ r_type = 'dataset'
91
+ elif r_type == 's':
92
+ r_type = 'space'
93
+ break
94
+ clear_screen()
95
+
96
+ #if new, ask to create private
97
+ priv = "nothing yet"
98
+ new_r = False
99
+ if repo_exists(repo, repo_type=r_type) == False:
100
+ new_r = True
101
+ while True:
102
+ priv = input(f"No existing repo found, create private repository? (y/N): ").lower()
103
+ if priv == '':
104
+ priv = 'n'
105
+ elif priv == 'yes':
106
+ priv = 'y'
107
+ elif priv == 'no':
108
+ priv = 'n'
109
+ break
110
+ else:
111
+ if priv not in ['y', 'n']:
112
+ clear_screen()
113
+ print("Please choose one of the following two options.")
114
+ continue
115
+ break
116
+ clear_screen()
117
+
118
+ #ask for optional commit message
119
+ c_mes = input("Commit message (optional): ")
120
+ if c_mes == "":
121
+ c_mes = "Uploaded files with huggingface_hub"
122
+ clear_screen()
123
+
124
+ #ask for optional branch name
125
+ rev = input("Branch name (empty for main): ")
126
+ if rev == "":
127
+ rev = "main"
128
+ clear_screen()
129
+
130
+ #if branch doesn't exist, ask to create it
131
+ if not any(branch.name == rev for branch in list_repo_refs(repo, repo_type=r_type).branches):
132
+ while True:
133
+ c_rev = input(f"No existing branch '{rev}' found, create new branch? (Y/n): ").lower()
134
+ if c_rev == '':
135
+ c_rev = 'y'
136
+ elif c_rev == 'yes':
137
+ c_rev = 'y'
138
+ elif c_rev == 'no':
139
+ c_rev = 'n'
140
+ break
141
+ else:
142
+ if c_rev not in ['y', 'n']:
143
+ clear_screen()
144
+ print("Please choose one of the following two options.")
145
+ continue
146
+ break
147
+ if c_rev == 'n':
148
+ exit()
149
+ else:
150
+ create_branch(repo, branch=rev, repo_type=r_type)
151
+ clear_screen()
152
+
153
+ #ask for optional path in repo
154
+ r_path = input("Path in repo (empty for root): ")
155
+ if r_path == "":
156
+ r_path = "."
157
+
158
+ #upload the folder, create the repo if new
159
+ if priv == "y":
160
+ create_repo(repo, repo_type=r_type, private=True) #if private chosen, create private repo first
161
+ else:
162
+ if new_r == True:
163
+ create_repo(repo, repo_type=r_type, private=False) #if public chosen, create public repo first
164
+ HfApi().upload_folder(folder_path=up_dir, repo_id=repo, repo_type=r_type, commit_message=c_mes, revision=rev, path_in_repo=r_path)
165
+
166
+ #Show user new repo
167
+ if new_r == True:
168
+ if r_type == 'model':
169
+ print(f"Repository created at https://huggingface.co/{repo}")
170
+ elif r_type == 'dataset':
171
+ print(f"Repository created at https://huggingface.co/datasets/{repo}")
172
+ elif r_type == 'space':
173
+ print(f"Repository created at https://huggingface.co/spaces/{repo}")
174
+
175
+ #if token wasn't found, (new log in) show user's name.
176
+ if tfound == 'false':
177
+ print(f'''
178
+ You are now logged in as {whoami().get('fullname', None)}.
179
+
180
+ To logout, use the hf command line interface 'huggingface-cli logout'
181
+ To view your active account, use 'huggingface-cli whoami'
182
+ ''')
183
+
184
+ input("Press enter to continue.")