Update status_checker.py
Browse files- status_checker.py +5 -13
status_checker.py
CHANGED
@@ -10,33 +10,25 @@ import subprocess
|
|
10 |
|
11 |
HfFolder().save_token(os.getenv("HF_TOKEN"))
|
12 |
|
|
|
13 |
output_dataset_id = "nateraw/asdf123"
|
14 |
# Where user will write outputs from their script
|
15 |
outputs_dir = Path('outputs')
|
16 |
|
|
|
17 |
process_pid = os.getenv('USER_SCRIPT_PID', None)
|
18 |
-
print(f"USER SCRIPT PID? - {process_pid} - type {type(process_pid)}")
|
19 |
-
|
20 |
-
|
21 |
def process_is_complete():
|
22 |
-
# Attempt at solving the todo shown below
|
23 |
p = subprocess.Popen(['ps', '-p', process_pid], stdout=subprocess.PIPE)
|
24 |
out = p.communicate()[0].decode('utf-8').strip().split('\n')
|
25 |
return len(out) == 1
|
26 |
|
27 |
-
|
28 |
-
# A file that marks that the script has finished running
|
29 |
-
# TODO - maybe check process like this instead of with done.txt: https://stackoverflow.com/a/2944076
|
30 |
-
output_file = Path('done.txt')
|
31 |
-
|
32 |
def status_checker():
|
33 |
print("Waiting to find output_file to check if script is done running")
|
34 |
while True:
|
35 |
-
|
36 |
-
|
37 |
-
print("Found the output file - Uploading the outputs from the script")
|
38 |
upload_folder(repo_id=output_dataset_id, folder_path=str(outputs_dir), path_in_repo='.', repo_type='dataset')
|
39 |
-
print("Finished uploading outputs
|
40 |
return
|
41 |
print("Didn't find it...sleeping for 5 seconds.")
|
42 |
time.sleep(5)
|
|
|
10 |
|
11 |
HfFolder().save_token(os.getenv("HF_TOKEN"))
|
12 |
|
13 |
+
# Where outputs from user script will be pushed to for safe keeping
|
14 |
output_dataset_id = "nateraw/asdf123"
|
15 |
# Where user will write outputs from their script
|
16 |
outputs_dir = Path('outputs')
|
17 |
|
18 |
+
# Watch python script's process to see when it's done running
|
19 |
process_pid = os.getenv('USER_SCRIPT_PID', None)
|
|
|
|
|
|
|
20 |
def process_is_complete():
|
|
|
21 |
p = subprocess.Popen(['ps', '-p', process_pid], stdout=subprocess.PIPE)
|
22 |
out = p.communicate()[0].decode('utf-8').strip().split('\n')
|
23 |
return len(out) == 1
|
24 |
|
|
|
|
|
|
|
|
|
|
|
25 |
def status_checker():
|
26 |
print("Waiting to find output_file to check if script is done running")
|
27 |
while True:
|
28 |
+
if process_is_complete():
|
29 |
+
print("Found it! Uploading assets to output dataset repo")
|
|
|
30 |
upload_folder(repo_id=output_dataset_id, folder_path=str(outputs_dir), path_in_repo='.', repo_type='dataset')
|
31 |
+
print("Finished uploading outputs to dataset repo. Done!")
|
32 |
return
|
33 |
print("Didn't find it...sleeping for 5 seconds.")
|
34 |
time.sleep(5)
|