zetavg commited on
Commit
28664d2
1 Parent(s): 6947876

show training speed

Browse files
llama_lora/ui/finetune/style.css CHANGED
@@ -315,6 +315,7 @@
315
  padding: var(--size-1) var(--size-2);
316
  font-size: var(--text-sm);
317
  font-family: var(--font-mono);
 
318
  }
319
  #finetune_training_status .progress-block .status {
320
  white-space: pre-wrap;
@@ -328,6 +329,7 @@
328
  z-index: var(--layer-2);
329
  width: var(--size-full);
330
  padding: 8px 0;
 
331
  }
332
  #finetune_training_status .progress-block .progress-level-inner {
333
  margin: var(--size-2) auto;
 
315
  padding: var(--size-1) var(--size-2);
316
  font-size: var(--text-sm);
317
  font-family: var(--font-mono);
318
+ text-align: right;
319
  }
320
  #finetune_training_status .progress-block .status {
321
  white-space: pre-wrap;
 
329
  z-index: var(--layer-2);
330
  width: var(--size-full);
331
  padding: 8px 0;
332
+ text-align: center;
333
  }
334
  #finetune_training_status .progress-block .progress-level-inner {
335
  margin: var(--size-2) auto;
llama_lora/ui/finetune/training.py CHANGED
@@ -398,10 +398,16 @@ def render_training_status():
398
  if time_remaining >= 0:
399
  meta_info.append(
400
  f"{format_time(time_elapsed)}<{format_time(time_remaining)}")
401
- meta_info.append(f"ETA: {format_timestamp(Global.training_eta)}")
402
  else:
403
  meta_info.append(format_time(time_elapsed))
404
 
 
 
 
 
 
 
 
405
  params_info_html = ""
406
  if Global.training_params_info_text:
407
  params_info_html = f"""
 
398
  if time_remaining >= 0:
399
  meta_info.append(
400
  f"{format_time(time_elapsed)}<{format_time(time_remaining)}")
 
401
  else:
402
  meta_info.append(format_time(time_elapsed))
403
 
404
+ current_speed = Global.training_eta_predictor.get_current_speed()
405
+ if current_speed is not None:
406
+ meta_info.append(f"{current_speed:.2f}it/s")
407
+
408
+ if time_remaining >= 0:
409
+ meta_info.append(f"ETA: {format_timestamp(Global.training_eta)}")
410
+
411
  params_info_html = ""
412
  if Global.training_params_info_text:
413
  params_info_html = f"""
llama_lora/utils/eta_predictor.py CHANGED
@@ -52,3 +52,18 @@ class ETAPredictor:
52
  print("Error predicting ETA:", e)
53
  traceback.print_exc()
54
  return None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
52
  print("Error predicting ETA:", e)
53
  traceback.print_exc()
54
  return None
55
+
56
+ def get_current_speed(self):
57
+ if len(self.data) < 5:
58
+ return None
59
+
60
+ last = self.data[-1]
61
+ sample = self.data[-5]
62
+ if len(self.data) > 100:
63
+ sample = self.data[-2]
64
+
65
+ steps_completed = last[0] - sample[0]
66
+ time_elapsed = last[1] - sample[1]
67
+ steps_per_second = steps_completed / time_elapsed
68
+
69
+ return steps_per_second