Yurii Paniv commited on
Commit
ae58991
1 Parent(s): a1d768d

Fix data logging

Browse files
Files changed (2) hide show
  1. app.py +13 -2
  2. data_logger.py +2 -2
app.py CHANGED
@@ -8,13 +8,24 @@ from os import getenv
8
  from data_logger import log_data
9
  from threading import Thread
10
  from queue import Queue
 
11
 
12
 
13
  def check_thread(logging_queue: Queue):
14
  logging_callback = log_data(hf_token=getenv("HF_API_TOKEN"), dataset_name="uk-tts-output", private=True)
15
  while True:
16
- item = logging_queue.get()
17
- logging_callback(item)
 
 
 
 
 
 
 
 
 
 
18
 
19
  if getenv("HF_API_TOKEN") is not None:
20
  log_queue = Queue()
8
  from data_logger import log_data
9
  from threading import Thread
10
  from queue import Queue
11
+ from time import sleep
12
 
13
 
14
  def check_thread(logging_queue: Queue):
15
  logging_callback = log_data(hf_token=getenv("HF_API_TOKEN"), dataset_name="uk-tts-output", private=True)
16
  while True:
17
+ sleep(60)
18
+ batch = []
19
+ while not logging_queue.empty():
20
+ batch.append(logging_queue.get())
21
+
22
+ if len(batch) > 0:
23
+ try:
24
+ logging_callback(batch)
25
+ except:
26
+ print("Error happened while pushing data to HF. Puttting items back in queue...")
27
+ for item in batch:
28
+ logging_queue.put(item)
29
 
30
  if getenv("HF_API_TOKEN") is not None:
31
  log_queue = Queue()
data_logger.py CHANGED
@@ -27,8 +27,8 @@ def log_data(hf_token: str, dataset_name: str, private=True):
27
  with open(log_file, "a", newline="", encoding="utf-8") as csvfile:
28
  writer = csv.writer(csvfile)
29
 
30
- csv_data = data
31
- writer.writerow(utils.sanitize_list_for_csv(csv_data))
32
 
33
  with open(log_file, "r", encoding="utf-8") as csvfile:
34
  line_count = len([None for row in csv.reader(csvfile)]) - 1
27
  with open(log_file, "a", newline="", encoding="utf-8") as csvfile:
28
  writer = csv.writer(csvfile)
29
 
30
+ for row in data:
31
+ writer.writerow(utils.sanitize_list_for_csv(row))
32
 
33
  with open(log_file, "r", encoding="utf-8") as csvfile:
34
  line_count = len([None for row in csv.reader(csvfile)]) - 1