ml-visoft commited on
Commit
622f41a
1 Parent(s): ca54f98

Backoff if no scheduler.

Browse files
Files changed (1) hide show
  1. main.py +13 -10
main.py CHANGED
@@ -42,16 +42,18 @@ LOCAL_STORAGE_PATH.mkdir(exist_ok=True, parents=True)
42
  EVENTS_FILE_PATH = LOCAL_STORAGE_PATH / FILE_EVENTS
43
 
44
 
45
- scheduler = CommitScheduler(
46
- repo_id="ml-visoft/c-reviewer",
47
- repo_type="dataset",
48
- folder_path=LOCAL_STORAGE_PATH,
49
- every=100,
50
- path_in_repo="raw_data",
51
- token=HF_DATASET_AUTH_TOKEN,
52
- allow_patterns=".jslines",
53
- squash_history=False
54
- )
 
 
55
 
56
 
57
 
@@ -104,6 +106,7 @@ else:
104
 
105
 
106
  def untyped_save_to_storage(dc, filename):
 
107
  with scheduler.lock:
108
  js_str = json.dumps(dataclasses.asdict(dc)) + "\n"
109
  with open(filename, "a") as f:
 
42
  EVENTS_FILE_PATH = LOCAL_STORAGE_PATH / FILE_EVENTS
43
 
44
 
45
+ scheduler = None
46
+
47
+ # scheduler = CommitScheduler(
48
+ # repo_id="ml-visoft/c-reviewer",
49
+ # repo_type="dataset",
50
+ # folder_path=LOCAL_STORAGE_PATH,
51
+ # every=100,
52
+ # path_in_repo="raw_data",
53
+ # token=HF_DATASET_AUTH_TOKEN,
54
+ # allow_patterns=".jslines",
55
+ # squash_history=False
56
+ # )
57
 
58
 
59
 
 
106
 
107
 
108
  def untyped_save_to_storage(dc, filename):
109
+ if scheduler is None: return
110
  with scheduler.lock:
111
  js_str = json.dumps(dataclasses.asdict(dc)) + "\n"
112
  with open(filename, "a") as f: