Upload src
Browse files
src/training/mindi_trainer.py
CHANGED
|
@@ -528,10 +528,7 @@ class MINDITrainer:
|
|
| 528 |
|
| 529 |
self.model.train()
|
| 530 |
phase_steps = phase.end_step - phase.start_step
|
| 531 |
-
step_in_phase =
|
| 532 |
-
if step_in_phase > 0:
|
| 533 |
-
print(f" [{phase.name}] Resuming from step {step_in_phase}/{phase_steps}")
|
| 534 |
-
self._resume_step_offset = 0 # Clear after use
|
| 535 |
accum_loss = 0.0
|
| 536 |
accum_count = 0
|
| 537 |
phase_start_time = time.time()
|
|
@@ -682,15 +679,6 @@ class MINDITrainer:
|
|
| 682 |
phase_summaries = []
|
| 683 |
|
| 684 |
for phase in self.config.phases:
|
| 685 |
-
# Skip completed phases on resume
|
| 686 |
-
if self.global_step >= phase.end_step:
|
| 687 |
-
print(f" Skipping {phase.name} (already completed, global_step={self.global_step})")
|
| 688 |
-
continue
|
| 689 |
-
# Resume mid-phase: calculate how many steps are already done
|
| 690 |
-
if self.global_step > phase.start_step:
|
| 691 |
-
done_in_phase = self.global_step - phase.start_step
|
| 692 |
-
self._resume_step_offset = done_in_phase
|
| 693 |
-
print(f" Resuming {phase.name} at step {done_in_phase}/{phase.end_step - phase.start_step}")
|
| 694 |
summary = self.train_phase(phase)
|
| 695 |
phase_summaries.append(summary)
|
| 696 |
|
|
|
|
| 528 |
|
| 529 |
self.model.train()
|
| 530 |
phase_steps = phase.end_step - phase.start_step
|
| 531 |
+
step_in_phase = 0
|
|
|
|
|
|
|
|
|
|
| 532 |
accum_loss = 0.0
|
| 533 |
accum_count = 0
|
| 534 |
phase_start_time = time.time()
|
|
|
|
| 679 |
phase_summaries = []
|
| 680 |
|
| 681 |
for phase in self.config.phases:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 682 |
summary = self.train_phase(phase)
|
| 683 |
phase_summaries.append(summary)
|
| 684 |
|