Spaces:
Running
Running
Backoff if no scheduler.
Browse files
main.py
CHANGED
@@ -42,16 +42,18 @@ LOCAL_STORAGE_PATH.mkdir(exist_ok=True, parents=True)
|
|
42 |
EVENTS_FILE_PATH = LOCAL_STORAGE_PATH / FILE_EVENTS
|
43 |
|
44 |
|
45 |
-
scheduler =
|
46 |
-
|
47 |
-
|
48 |
-
|
49 |
-
|
50 |
-
|
51 |
-
|
52 |
-
|
53 |
-
|
54 |
-
|
|
|
|
|
55 |
|
56 |
|
57 |
|
@@ -104,6 +106,7 @@ else:
|
|
104 |
|
105 |
|
106 |
def untyped_save_to_storage(dc, filename):
|
|
|
107 |
with scheduler.lock:
|
108 |
js_str = json.dumps(dataclasses.asdict(dc)) + "\n"
|
109 |
with open(filename, "a") as f:
|
|
|
42 |
EVENTS_FILE_PATH = LOCAL_STORAGE_PATH / FILE_EVENTS
|
43 |
|
44 |
|
45 |
+
scheduler = None
|
46 |
+
|
47 |
+
# scheduler = CommitScheduler(
|
48 |
+
# repo_id="ml-visoft/c-reviewer",
|
49 |
+
# repo_type="dataset",
|
50 |
+
# folder_path=LOCAL_STORAGE_PATH,
|
51 |
+
# every=100,
|
52 |
+
# path_in_repo="raw_data",
|
53 |
+
# token=HF_DATASET_AUTH_TOKEN,
|
54 |
+
# allow_patterns=".jslines",
|
55 |
+
# squash_history=False
|
56 |
+
# )
|
57 |
|
58 |
|
59 |
|
|
|
106 |
|
107 |
|
108 |
def untyped_save_to_storage(dc, filename):
|
109 |
+
if scheduler is None: return
|
110 |
with scheduler.lock:
|
111 |
js_str = json.dumps(dataclasses.asdict(dc)) + "\n"
|
112 |
with open(filename, "a") as f:
|