diff --git "a/trainer_0_log.txt" "b/trainer_0_log.txt" new file mode 100644--- /dev/null +++ "b/trainer_0_log.txt" @@ -0,0 +1,9712 @@ + > Training Environment: + | > Backend: Torch + | > Mixed precision: False + | > Precision: float32 + | > Current device: 0 + | > Num. of GPUs: 1 + | > Num. of CPUs: 96 + | > Num. of Torch Threads: 1 + | > Torch seed: 1 + | > Torch CUDNN: True + | > Torch CUDNN deterministic: False + | > Torch CUDNN benchmark: False + | > Torch TF32 MatMul: False + + > Model has 521411048 parameters + + > EPOCH: 0/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.032263645758995645 (+0) + | > avg_loss_text_ce: 0.274087864619035 (+0) + | > avg_loss_mel_ce: 4.5301331006563625 (+0) + | > avg_loss: 4.804220933180589 (+0) + + + > EPOCH: 1/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 14:58:40)  + + --> TIME: 2024-04-09 14:58:41 -- STEP: 0/1354 -- GLOBAL_STEP: 0 + | > loss_text_ce: 0.2870144844055176 (0.2870144844055176) + | > loss_mel_ce: 4.802734375 (4.802734375) + | > loss: 0.060592249035835266 (0.060592249035835266) + | > current_lr: 5e-06 + | > step_time: 0.1898 (0.18976783752441406) + | > loader_time: 1.1502 (1.1501641273498535) + + + --> TIME: 2024-04-09 14:58:53 -- STEP: 50/1354 -- GLOBAL_STEP: 50 + | > loss_text_ce: 0.23488140106201172 (0.2510687032341958) + | > loss_mel_ce: 4.15988302230835 (4.349425196647645) + | > loss: 0.05231862515211105 (0.0547677855193615) + | > current_lr: 5e-06 + | > step_time: 0.2066 (0.1438596343994141) + | > loader_time: 0.0193 (0.01607170581817627) + + + --> TIME: 2024-04-09 14:59:04 -- STEP: 100/1354 -- GLOBAL_STEP: 100 + | > loss_text_ce: 0.253946453332901 (0.25185048997402176) + | > loss_mel_ce: 4.452804088592529 (4.355699384212497) + | > loss: 0.05603274330496788 (0.05485178511589765) + | > current_lr: 5e-06 + | > step_time: 0.1066 (0.14201484441757206) + | > loader_time: 0.0054 (0.015420072078704833) + + + --> TIME: 2024-04-09 14:59:14 -- STEP: 150/1354 -- GLOBAL_STEP: 150 + | > loss_text_ce: 0.261427104473114 (0.2522550893823306) + | > loss_mel_ce: 4.822605609893799 (4.357179082234702) + | > loss: 0.06052419915795326 (0.05487421721220016) + | > current_lr: 5e-06 + | > step_time: 0.097 (0.14092166741689052) + | > loader_time: 0.0184 (0.01455831527709961) + + + --> TIME: 2024-04-09 14:59:25 -- STEP: 200/1354 -- GLOBAL_STEP: 200 + | > loss_text_ce: 0.258465975522995 (0.2510707839578388) + | > loss_mel_ce: 4.812194347381592 (4.321441469192506) + | > loss: 0.06036500260233879 (0.05443467071279883) + | > current_lr: 5e-06 + | > step_time: 0.1105 (0.1416985404491426) + | > loader_time: 0.0184 (0.013388535976409911) + + + --> TIME: 2024-04-09 14:59:36 -- STEP: 250/1354 -- GLOBAL_STEP: 250 + | > loss_text_ce: 0.27347084879875183 (0.250440772771835) + | > loss_mel_ce: 4.958843231201172 (4.313262273788455) + | > loss: 0.06228945404291153 (0.05432979914546013) + | > current_lr: 5e-06 + | > step_time: 0.1342 (0.14108433628082284) + | > loader_time: 0.0075 (0.01251917839050293) + + + --> TIME: 2024-04-09 14:59:47 -- STEP: 300/1354 -- GLOBAL_STEP: 300 + | > loss_text_ce: 0.24012669920921326 (0.24984896714488647) + | > loss_mel_ce: 4.150601387023926 (4.2779683415095064) + | > loss: 0.05227057263255119 (0.0539025880023837) + | > current_lr: 5e-06 + | > step_time: 0.1428 (0.1414461429913839) + | > loader_time: 0.0061 (0.012139840920766204) + + + --> TIME: 2024-04-09 14:59:57 -- STEP: 350/1354 -- GLOBAL_STEP: 350 + | > loss_text_ce: 0.23827174305915833 (0.2502408760786056) + | > loss_mel_ce: 3.9545321464538574 (4.2540052441188285) + | > loss: 0.04991433396935463 (0.05362197857882294) + | > current_lr: 5e-06 + | > step_time: 0.1439 (0.1406935998371669) + | > loader_time: 0.0053 (0.011750063896179204) + + + --> TIME: 2024-04-09 15:00:07 -- STEP: 400/1354 -- GLOBAL_STEP: 400 + | > loss_text_ce: 0.2687925696372986 (0.25067472770810134) + | > loss_mel_ce: 4.188967704772949 (4.228503044843671) + | > loss: 0.053068578243255615 (0.05332354582846164) + | > current_lr: 5e-06 + | > step_time: 0.0874 (0.13881822526454918) + | > loader_time: 0.0054 (0.011420943737030034) + + + --> TIME: 2024-04-09 15:00:18 -- STEP: 450/1354 -- GLOBAL_STEP: 450 + | > loss_text_ce: 0.25258103013038635 (0.2492480552527642) + | > loss_mel_ce: 3.959266424179077 (4.2030617639753505) + | > loss: 0.05014104023575783 (0.05300368928247027) + | > current_lr: 5e-06 + | > step_time: 0.1397 (0.1399121628867253) + | > loader_time: 0.0062 (0.011094852023654518) + + + --> TIME: 2024-04-09 15:00:29 -- STEP: 500/1354 -- GLOBAL_STEP: 500 + | > loss_text_ce: 0.2479471117258072 (0.2488583788871767) + | > loss_mel_ce: 4.1386213302612305 (4.1765152316093435) + | > loss: 0.05222105607390404 (0.05268302009254693) + | > current_lr: 5e-06 + | > step_time: 0.112 (0.14011309099197364) + | > loader_time: 0.0062 (0.010829375267028813) + + + --> TIME: 2024-04-09 15:00:39 -- STEP: 550/1354 -- GLOBAL_STEP: 550 + | > loss_text_ce: 0.22135059535503387 (0.24826846011660336) + | > loss_mel_ce: 3.7596049308776855 (4.150939217914231) + | > loss: 0.047392331063747406 (0.05237152090804142) + | > current_lr: 5e-06 + | > step_time: 0.1422 (0.14028825586492352) + | > loader_time: 0.0066 (0.010595115314830436) + + + --> TIME: 2024-04-09 15:00:50 -- STEP: 600/1354 -- GLOBAL_STEP: 600 + | > loss_text_ce: 0.23912587761878967 (0.2478309545914334) + | > loss_mel_ce: 4.005533695220947 (4.12982649048169) + | > loss: 0.05053165927529335 (0.05211497050399579) + | > current_lr: 5e-06 + | > step_time: 0.1184 (0.14085040966669712) + | > loader_time: 0.0075 (0.010366886059443163) + + + --> TIME: 2024-04-09 15:01:01 -- STEP: 650/1354 -- GLOBAL_STEP: 650 + | > loss_text_ce: 0.24068942666053772 (0.24705999101583792) + | > loss_mel_ce: 4.7948503494262695 (4.1163351242358805) + | > loss: 0.059946902096271515 (0.05194518083563217) + | > current_lr: 5e-06 + | > step_time: 0.1729 (0.14080448920910163) + | > loader_time: 0.0061 (0.010084572205176737) + + + --> TIME: 2024-04-09 15:01:11 -- STEP: 700/1354 -- GLOBAL_STEP: 700 + | > loss_text_ce: 0.25430774688720703 (0.2467543599435263) + | > loss_mel_ce: 3.847133159637451 (4.102159860474718) + | > loss: 0.0488266795873642 (0.051772789231368466) + | > current_lr: 5e-06 + | > step_time: 0.1077 (0.14032638345445902) + | > loader_time: 0.0056 (0.009916747297559479) + + + --> TIME: 2024-04-09 15:01:21 -- STEP: 750/1354 -- GLOBAL_STEP: 750 + | > loss_text_ce: 0.21410375833511353 (0.24615456801652924) + | > loss_mel_ce: 3.945967197418213 (4.077056989987685) + | > loss: 0.04952465370297432 (0.0514668051401774) + | > current_lr: 5e-06 + | > step_time: 0.1737 (0.1402958949406942) + | > loader_time: 0.0058 (0.009721302032470718) + + + --> TIME: 2024-04-09 15:01:32 -- STEP: 800/1354 -- GLOBAL_STEP: 800 + | > loss_text_ce: 0.26632827520370483 (0.24585112337023035) + | > loss_mel_ce: 3.128547191619873 (4.057131978869433) + | > loss: 0.04041518643498421 (0.05122599020600318) + | > current_lr: 5e-06 + | > step_time: 0.0969 (0.14003757596015934) + | > loader_time: 0.0059 (0.009582012295722968) + + + --> TIME: 2024-04-09 15:01:42 -- STEP: 850/1354 -- GLOBAL_STEP: 850 + | > loss_text_ce: 0.22246886789798737 (0.24540382516734754) + | > loss_mel_ce: 3.353721857070923 (4.0353948724971005) + | > loss: 0.04257369786500931 (0.050961890141753584) + | > current_lr: 5e-06 + | > step_time: 0.1132 (0.1401443876939662) + | > loader_time: 0.0056 (0.009433408064000752) + + + --> TIME: 2024-04-09 15:01:53 -- STEP: 900/1354 -- GLOBAL_STEP: 900 + | > loss_text_ce: 0.23920434713363647 (0.24495639171865263) + | > loss_mel_ce: 3.315906047821045 (4.020798238913206) + | > loss: 0.04232274368405342 (0.05078279412041107) + | > current_lr: 5e-06 + | > step_time: 0.1535 (0.14007283926010133) + | > loader_time: 0.0072 (0.009300098154279925) + + + --> TIME: 2024-04-09 15:02:04 -- STEP: 950/1354 -- GLOBAL_STEP: 950 + | > loss_text_ce: 0.21295422315597534 (0.24414859712123882) + | > loss_mel_ce: 3.576324224472046 (4.003031149914378) + | > loss: 0.04511046037077904 (0.05056166455933921) + | > current_lr: 5e-06 + | > step_time: 0.1997 (0.14080397405122455) + | > loader_time: 0.0076 (0.009143879037154351) + + + --> TIME: 2024-04-09 15:02:14 -- STEP: 1000/1354 -- GLOBAL_STEP: 1000 + | > loss_text_ce: 0.24294307827949524 (0.2437474672794343) + | > loss_mel_ce: 3.7764806747436523 (3.9880431926250335) + | > loss: 0.04785028472542763 (0.05037846114113926) + | > current_lr: 5e-06 + | > step_time: 0.1097 (0.14055177330970764) + | > loader_time: 0.0095 (0.00907677483558655) + + + --> TIME: 2024-04-09 15:02:26 -- STEP: 1050/1354 -- GLOBAL_STEP: 1050 + | > loss_text_ce: 0.23952563107013702 (0.24296463211377475) + | > loss_mel_ce: 3.655878782272339 (3.9727317299161515) + | > loss: 0.04637386277318001 (0.05018686236724967) + | > current_lr: 5e-06 + | > step_time: 0.1113 (0.1410540887287685) + | > loader_time: 0.0072 (0.009035401798429947) + + + --> TIME: 2024-04-09 15:02:37 -- STEP: 1100/1354 -- GLOBAL_STEP: 1100 + | > loss_text_ce: 0.20046782493591309 (0.24254687162962837) + | > loss_mel_ce: 3.4632465839385986 (3.9564652328057606) + | > loss: 0.04361565038561821 (0.049988240257582875) + | > current_lr: 5e-06 + | > step_time: 0.151 (0.14098814964294432) + | > loader_time: 0.0059 (0.008911066922274509) + + + --> TIME: 2024-04-09 15:02:47 -- STEP: 1150/1354 -- GLOBAL_STEP: 1150 + | > loss_text_ce: 0.22239971160888672 (0.24213305666394866) + | > loss_mel_ce: 3.296398401260376 (3.9424634877495035) + | > loss: 0.041890453547239304 (0.04981662645612075) + | > current_lr: 5e-06 + | > step_time: 0.1073 (0.14082717356474495) + | > loader_time: 0.0051 (0.008850984158723256) + + + --> TIME: 2024-04-09 15:02:57 -- STEP: 1200/1354 -- GLOBAL_STEP: 1200 + | > loss_text_ce: 0.20214754343032837 (0.24161285111059755) + | > loss_mel_ce: 3.441718101501465 (3.9287540497382376) + | > loss: 0.04337935149669647 (0.049647225936253876) + | > current_lr: 5e-06 + | > step_time: 0.2064 (0.1407860332727432) + | > loader_time: 0.006 (0.00878010372320811) + + + --> TIME: 2024-04-09 15:03:08 -- STEP: 1250/1354 -- GLOBAL_STEP: 1250 + | > loss_text_ce: 0.20767681300640106 (0.24096125882864008) + | > loss_mel_ce: 3.406735420227051 (3.916024679756154) + | > loss: 0.04302871972322464 (0.0494879287660122) + | > current_lr: 5e-06 + | > step_time: 0.1376 (0.14101110858917237) + | > loader_time: 0.0057 (0.008700541687011721) + + + --> TIME: 2024-04-09 15:03:19 -- STEP: 1300/1354 -- GLOBAL_STEP: 1300 + | > loss_text_ce: 0.20557208359241486 (0.24055314705922062) + | > loss_mel_ce: 3.3979437351226807 (3.904593590956458) + | > loss: 0.0428989976644516 (0.04934698589719261) + | > current_lr: 5e-06 + | > step_time: 0.1781 (0.1408986746347868) + | > loader_time: 0.0061 (0.008635476185725289) + + + --> TIME: 2024-04-09 15:03:29 -- STEP: 1350/1354 -- GLOBAL_STEP: 1350 + | > loss_text_ce: 0.244922935962677 (0.24012530353334224) + | > loss_mel_ce: 3.848465919494629 (3.8963121223449613) + | > loss: 0.04873082414269447 (0.04924330360911513) + | > current_lr: 5e-06 + | > step_time: 0.103 (0.14093566435354724) + | > loader_time: 0.012 (0.008576358512595854) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.02307018866905799 (-0.009193457089937653) + | > avg_loss_text_ce: 0.24631039683635422 (-0.02777746778268078) + | > avg_loss_mel_ce: 3.604927209707407 (-0.9252058909489556) + | > avg_loss: 3.8512375904963565 (-0.9529833426842322) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_1354.pth + + > EPOCH: 2/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 15:04:20)  + + --> TIME: 2024-04-09 15:04:32 -- STEP: 46/1354 -- GLOBAL_STEP: 1400 + | > loss_text_ce: 0.2141932249069214 (0.2229069587977036) + | > loss_mel_ce: 3.4242348670959473 (3.516486002051312) + | > loss: 0.0433146208524704 (0.04451658374265484) + | > current_lr: 5e-06 + | > step_time: 0.2172 (0.14850883898527728) + | > loader_time: 0.0062 (0.008544652358345364) + + + --> TIME: 2024-04-09 15:04:43 -- STEP: 96/1354 -- GLOBAL_STEP: 1450 + | > loss_text_ce: 0.2013598382472992 (0.22208298897991577) + | > loss_mel_ce: 3.296741485595703 (3.5521236285567284) + | > loss: 0.04164406284689903 (0.044931032112799606) + | > current_lr: 5e-06 + | > step_time: 0.1621 (0.14780055483182272) + | > loader_time: 0.0068 (0.007901176810264586) + + + --> TIME: 2024-04-09 15:04:53 -- STEP: 146/1354 -- GLOBAL_STEP: 1500 + | > loss_text_ce: 0.2039547711610794 (0.22335080975947313) + | > loss_mel_ce: 4.039611339569092 (3.55701647392691) + | > loss: 0.050518643110990524 (0.04500437333975754) + | > current_lr: 5e-06 + | > step_time: 0.1891 (0.1464666438429323) + | > loader_time: 0.0066 (0.007648097325677738) + + + --> TIME: 2024-04-09 15:05:05 -- STEP: 196/1354 -- GLOBAL_STEP: 1550 + | > loss_text_ce: 0.24447396397590637 (0.22131489194473442) + | > loss_mel_ce: 3.599929094314575 (3.548887932787136) + | > loss: 0.04576670378446579 (0.04488336785259296) + | > current_lr: 5e-06 + | > step_time: 0.0997 (0.14743772453191323) + | > loader_time: 0.0063 (0.007412189123581865) + + + --> TIME: 2024-04-09 15:05:15 -- STEP: 246/1354 -- GLOBAL_STEP: 1600 + | > loss_text_ce: 0.20537732541561127 (0.22132882845353305) + | > loss_mel_ce: 2.930259943008423 (3.549160295385655) + | > loss: 0.037329014390707016 (0.044886776208271834) + | > current_lr: 5e-06 + | > step_time: 0.1587 (0.1468520833224785) + | > loader_time: 0.0073 (0.0073544019606055265) + + + --> TIME: 2024-04-09 15:05:25 -- STEP: 296/1354 -- GLOBAL_STEP: 1650 + | > loss_text_ce: 0.22503726184368134 (0.2215298834080632) + | > loss_mel_ce: 3.2202067375183105 (3.548842294796093) + | > loss: 0.041014812886714935 (0.044885383956637735) + | > current_lr: 5e-06 + | > step_time: 0.1185 (0.14484477929166847) + | > loader_time: 0.0059 (0.0072419015136924916) + + + --> TIME: 2024-04-09 15:05:36 -- STEP: 346/1354 -- GLOBAL_STEP: 1700 + | > loss_text_ce: 0.18121838569641113 (0.22161796645966572) + | > loss_mel_ce: 3.465691089630127 (3.5432086066703574) + | > loss: 0.04341559112071991 (0.04481936490561577) + | > current_lr: 5e-06 + | > step_time: 0.2233 (0.143508357119698) + | > loader_time: 0.0064 (0.0071403842440919355) + + + --> TIME: 2024-04-09 15:05:46 -- STEP: 396/1354 -- GLOBAL_STEP: 1750 + | > loss_text_ce: 0.21359491348266602 (0.22122322354051804) + | > loss_mel_ce: 3.2182607650756836 (3.5404039741766575) + | > loss: 0.04085542634129524 (0.044781277050273585) + | > current_lr: 5e-06 + | > step_time: 0.0974 (0.1429592407110965) + | > loader_time: 0.0062 (0.00716662105887827) + + + --> TIME: 2024-04-09 15:05:57 -- STEP: 446/1354 -- GLOBAL_STEP: 1800 + | > loss_text_ce: 0.20048415660858154 (0.22093015237640376) + | > loss_mel_ce: 3.5302610397338867 (3.5380876267437444) + | > loss: 0.04441363736987114 (0.044750212495436585) + | > current_lr: 5e-06 + | > step_time: 0.152 (0.1430622595842643) + | > loader_time: 0.0104 (0.007195319295464073) + + + --> TIME: 2024-04-09 15:06:07 -- STEP: 496/1354 -- GLOBAL_STEP: 1850 + | > loss_text_ce: 0.18827302753925323 (0.22087672022321533) + | > loss_mel_ce: 3.5109825134277344 (3.5354449998947883) + | > loss: 0.04403875768184662 (0.04471811656689932) + | > current_lr: 5e-06 + | > step_time: 0.1903 (0.1424745297239672) + | > loader_time: 0.0058 (0.007149110397984903) + + + --> TIME: 2024-04-09 15:06:18 -- STEP: 546/1354 -- GLOBAL_STEP: 1900 + | > loss_text_ce: 0.2793266773223877 (0.22038022234981317) + | > loss_mel_ce: 3.2917721271514893 (3.529745521999541) + | > loss: 0.042513079941272736 (0.044644354927889165) + | > current_lr: 5e-06 + | > step_time: 0.0751 (0.1421343856678777) + | > loader_time: 0.0127 (0.0071639835616171136) + + + --> TIME: 2024-04-09 15:06:28 -- STEP: 596/1354 -- GLOBAL_STEP: 1950 + | > loss_text_ce: 0.20374928414821625 (0.21975516785411225) + | > loss_mel_ce: 3.3391971588134766 (3.525514038217148) + | > loss: 0.04217793419957161 (0.04458653899972275) + | > current_lr: 5e-06 + | > step_time: 0.1416 (0.14188447494634837) + | > loader_time: 0.0074 (0.007121517354209951) + + + --> TIME: 2024-04-09 15:06:39 -- STEP: 646/1354 -- GLOBAL_STEP: 2000 + | > loss_text_ce: 0.24362486600875854 (0.2190400305962415) + | > loss_mel_ce: 3.6009390354156494 (3.5183686265266347) + | > loss: 0.0457686185836792 (0.04449296103150304) + | > current_lr: 5e-06 + | > step_time: 0.1038 (0.14223880000158712) + | > loader_time: 0.0058 (0.007091049070328751) + + + --> TIME: 2024-04-09 15:07:08 -- STEP: 696/1354 -- GLOBAL_STEP: 2050 + | > loss_text_ce: 0.20261086523532867 (0.21920458754074984) + | > loss_mel_ce: 3.384472370147705 (3.519914068024734) + | > loss: 0.04270337522029877 (0.0445133181524346) + | > current_lr: 5e-06 + | > step_time: 0.1754 (0.14185387238688843) + | > loader_time: 0.0076 (0.0071274148321699825) + + + --> TIME: 2024-04-09 15:07:19 -- STEP: 746/1354 -- GLOBAL_STEP: 2100 + | > loss_text_ce: 0.2641388773918152 (0.21854748906462826) + | > loss_mel_ce: 3.420597553253174 (3.5148130335692747) + | > loss: 0.04386591166257858 (0.044444768926772994) + | > current_lr: 5e-06 + | > step_time: 0.0694 (0.1425308379027541) + | > loader_time: 0.0132 (0.007136447180372141) + + + --> TIME: 2024-04-09 15:07:30 -- STEP: 796/1354 -- GLOBAL_STEP: 2150 + | > loss_text_ce: 0.1868325024843216 (0.21801601818308758) + | > loss_mel_ce: 3.3106815814971924 (3.5093201524648237) + | > loss: 0.04163707047700882 (0.0443730504537498) + | > current_lr: 5e-06 + | > step_time: 0.1864 (0.1425713417518077) + | > loader_time: 0.0072 (0.007173292301408011) + + + --> TIME: 2024-04-09 15:07:41 -- STEP: 846/1354 -- GLOBAL_STEP: 2200 + | > loss_text_ce: 0.1895640641450882 (0.217945791340607) + | > loss_mel_ce: 3.3338253498077393 (3.509263935100384) + | > loss: 0.04194511100649834 (0.04437154515186247) + | > current_lr: 5e-06 + | > step_time: 0.1993 (0.14260020887316227) + | > loader_time: 0.0069 (0.007159743748658093) + + + --> TIME: 2024-04-09 15:07:52 -- STEP: 896/1354 -- GLOBAL_STEP: 2250 + | > loss_text_ce: 0.1818501204252243 (0.21753665370800132) + | > loss_mel_ce: 3.252758026123047 (3.5054921231099536) + | > loss: 0.04088819399476051 (0.044321771964730215) + | > current_lr: 5e-06 + | > step_time: 0.1893 (0.14275175865207368) + | > loader_time: 0.0068 (0.007154127316815513) + + + --> TIME: 2024-04-09 15:08:03 -- STEP: 946/1354 -- GLOBAL_STEP: 2300 + | > loss_text_ce: 0.2163362056016922 (0.21719718006209138) + | > loss_mel_ce: 3.7480456829071045 (3.5033255664289125) + | > loss: 0.04719502478837967 (0.04429193828688117) + | > current_lr: 5e-06 + | > step_time: 0.1347 (0.14321388628719695) + | > loader_time: 0.0072 (0.007148242904066284) + + + --> TIME: 2024-04-09 15:08:14 -- STEP: 996/1354 -- GLOBAL_STEP: 2350 + | > loss_text_ce: 0.2312449812889099 (0.2168140164221147) + | > loss_mel_ce: 4.054014205932617 (3.501602793793123) + | > loss: 0.05101499333977699 (0.04426686760846037) + | > current_lr: 5e-06 + | > step_time: 0.0975 (0.1431417522660218) + | > loader_time: 0.0097 (0.007195012636452794) + + + --> TIME: 2024-04-09 15:08:25 -- STEP: 1046/1354 -- GLOBAL_STEP: 2400 + | > loss_text_ce: 0.19947703182697296 (0.21629049985119309) + | > loss_mel_ce: 3.566240072250366 (3.4962409511350994) + | > loss: 0.04482996463775635 (0.044196803817244196) + | > current_lr: 5e-06 + | > step_time: 0.1755 (0.14316809656059282) + | > loader_time: 0.0079 (0.007223414417434605) + + + --> TIME: 2024-04-09 15:08:35 -- STEP: 1096/1354 -- GLOBAL_STEP: 2450 + | > loss_text_ce: 0.25773316621780396 (0.21594579490649438) + | > loss_mel_ce: 3.9379611015319824 (3.492292523819163) + | > loss: 0.04994874447584152 (0.044145695100382125) + | > current_lr: 5e-06 + | > step_time: 0.0685 (0.14306889354747546) + | > loader_time: 0.0137 (0.007213200095796238) + + + --> TIME: 2024-04-09 15:08:46 -- STEP: 1146/1354 -- GLOBAL_STEP: 2500 + | > loss_text_ce: 0.23674972355365753 (0.21545174447445767) + | > loss_mel_ce: 3.769501209259033 (3.4906436471206765) + | > loss: 0.047693464905023575 (0.044120184056473916) + | > current_lr: 5e-06 + | > step_time: 0.0892 (0.14318167310318575) + | > loader_time: 0.0162 (0.007188504487016022) + + + --> TIME: 2024-04-09 15:08:56 -- STEP: 1196/1354 -- GLOBAL_STEP: 2550 + | > loss_text_ce: 0.17882002890110016 (0.2152370416923711) + | > loss_mel_ce: 3.1909282207489014 (3.487397611141204) + | > loss: 0.04011605307459831 (0.04407898477748989) + | > current_lr: 5e-06 + | > step_time: 0.1985 (0.14303583165873623) + | > loader_time: 0.0067 (0.007174519990200183) + + + --> TIME: 2024-04-09 15:09:07 -- STEP: 1246/1354 -- GLOBAL_STEP: 2600 + | > loss_text_ce: 0.17585690319538116 (0.21480734587720285) + | > loss_mel_ce: 3.4767978191375732 (3.484831912176949) + | > loss: 0.04348398372530937 (0.04404332532725312) + | > current_lr: 5e-06 + | > step_time: 0.173 (0.14297011690767358) + | > loader_time: 0.0067 (0.007153185183125552) + + + --> TIME: 2024-04-09 15:09:18 -- STEP: 1296/1354 -- GLOBAL_STEP: 2650 + | > loss_text_ce: 0.18431010842323303 (0.21445867960789702) + | > loss_mel_ce: 3.2931642532348633 (3.483062796754601) + | > loss: 0.04139850661158562 (0.04401811363497458) + | > current_lr: 5e-06 + | > step_time: 0.2063 (0.14298423480840347) + | > loader_time: 0.0066 (0.00713790014938072) + + + --> TIME: 2024-04-09 15:09:29 -- STEP: 1346/1354 -- GLOBAL_STEP: 2700 + | > loss_text_ce: 0.20610804855823517 (0.21409987354925106) + | > loss_mel_ce: 3.93856143951416 (3.4808052236222577) + | > loss: 0.04934130609035492 (0.043986966241189904) + | > current_lr: 5e-06 + | > step_time: 0.1501 (0.1431308003449685) + | > loader_time: 0.0068 (0.007129613038866517) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.023867185299213115 (+0.0007969966301551237) + | > avg_loss_text_ce: 0.22347504244400904 (-0.022835354392345186) + | > avg_loss_mel_ce: 3.398358601790208 (-0.2065686079171991) + | > avg_loss: 3.6218336545504055 (-0.229403935945951) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_2708.pth + + > EPOCH: 3/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 15:10:22)  + + --> TIME: 2024-04-09 15:10:33 -- STEP: 42/1354 -- GLOBAL_STEP: 2750 + | > loss_text_ce: 0.20979592204093933 (0.20194307608263834) + | > loss_mel_ce: 3.9560813903808594 (3.4476829256330217) + | > loss: 0.04959378018975258 (0.043447929478826974) + | > current_lr: 5e-06 + | > step_time: 0.1427 (0.14811389786856516) + | > loader_time: 0.0097 (0.00790303094046456) + + + --> TIME: 2024-04-09 15:10:44 -- STEP: 92/1354 -- GLOBAL_STEP: 2800 + | > loss_text_ce: 0.22515588998794556 (0.20260817282225774) + | > loss_mel_ce: 3.2467916011810303 (3.430300085440926) + | > loss: 0.041332706809043884 (0.04324890875621981) + | > current_lr: 5e-06 + | > step_time: 0.0988 (0.1457209146541097) + | > loader_time: 0.0065 (0.007390600183735722) + + + --> TIME: 2024-04-09 15:10:54 -- STEP: 142/1354 -- GLOBAL_STEP: 2850 + | > loss_text_ce: 0.23326332867145538 (0.2031344236202643) + | > loss_mel_ce: 3.417572021484375 (3.4111437730386225) + | > loss: 0.043462324887514114 (0.04302712230825088) + | > current_lr: 5e-06 + | > step_time: 0.099 (0.14390860644864356) + | > loader_time: 0.0063 (0.007257466584863797) + + + --> TIME: 2024-04-09 15:11:04 -- STEP: 192/1354 -- GLOBAL_STEP: 2900 + | > loss_text_ce: 0.20420703291893005 (0.20310222962871194) + | > loss_mel_ce: 3.5009965896606445 (3.3794958516955376) + | > loss: 0.04410956799983978 (0.042649978034508707) + | > current_lr: 5e-06 + | > step_time: 0.1452 (0.14237080514430991) + | > loader_time: 0.0072 (0.007156853874524434) + + + --> TIME: 2024-04-09 15:11:16 -- STEP: 242/1354 -- GLOBAL_STEP: 2950 + | > loss_text_ce: 0.18115182220935822 (0.20173831031588482) + | > loss_mel_ce: 3.2506635189056396 (3.3693735865522023) + | > loss: 0.04085494577884674 (0.04251323774272251) + | > current_lr: 5e-06 + | > step_time: 0.1731 (0.1445235270113984) + | > loader_time: 0.0063 (0.007109294252947342) + + + --> TIME: 2024-04-09 15:11:27 -- STEP: 292/1354 -- GLOBAL_STEP: 3000 + | > loss_text_ce: 0.2096872627735138 (0.20106296261695975) + | > loss_mel_ce: 3.3545567989349365 (3.3690418772501483) + | > loss: 0.04243147745728493 (0.042501248916840706) + | > current_lr: 5e-06 + | > step_time: 0.1063 (0.14478178301902658) + | > loader_time: 0.0076 (0.007077169744935754) + + + --> TIME: 2024-04-09 15:11:56 -- STEP: 342/1354 -- GLOBAL_STEP: 3050 + | > loss_text_ce: 0.17720937728881836 (0.2005116432818056) + | > loss_mel_ce: 3.381467342376709 (3.3699256613937743) + | > loss: 0.04236520081758499 (0.04250520681254345) + | > current_lr: 5e-06 + | > step_time: 0.1992 (0.1459869390342667) + | > loader_time: 0.0079 (0.007230632486399154) + + + --> TIME: 2024-04-09 15:12:07 -- STEP: 392/1354 -- GLOBAL_STEP: 3100 + | > loss_text_ce: 0.17111153900623322 (0.20024582983127662) + | > loss_mel_ce: 3.1576974391937256 (3.3648933968981924) + | > loss: 0.03962868079543114 (0.04244213444846015) + | > current_lr: 5e-06 + | > step_time: 0.176 (0.14513907566362497) + | > loader_time: 0.0089 (0.0072428681412521675) + + + --> TIME: 2024-04-09 15:12:17 -- STEP: 442/1354 -- GLOBAL_STEP: 3150 + | > loss_text_ce: 0.17279422283172607 (0.20022888323165713) + | > loss_mel_ce: 3.1029233932495117 (3.361058323092051) + | > loss: 0.03899664059281349 (0.04239627703754609) + | > current_lr: 5e-06 + | > step_time: 0.1541 (0.14362408294936663) + | > loader_time: 0.0063 (0.007282025673810174) + + + --> TIME: 2024-04-09 15:12:27 -- STEP: 492/1354 -- GLOBAL_STEP: 3200 + | > loss_text_ce: 0.18599887192249298 (0.2002235482620999) + | > loss_mel_ce: 3.410306453704834 (3.3618160936890584) + | > loss: 0.04281315952539444 (0.04240523461191876) + | > current_lr: 5e-06 + | > step_time: 0.1607 (0.14285366079671608) + | > loader_time: 0.0068 (0.007293333852194189) + + + --> TIME: 2024-04-09 15:12:38 -- STEP: 542/1354 -- GLOBAL_STEP: 3250 + | > loss_text_ce: 0.17681239545345306 (0.20005352226020665) + | > loss_mel_ce: 3.005662202835083 (3.3611412593799326) + | > loss: 0.0378866046667099 (0.04239517673712593) + | > current_lr: 5e-06 + | > step_time: 0.1856 (0.1428569031817448) + | > loader_time: 0.0066 (0.007295600602547621) + + + --> TIME: 2024-04-09 15:12:49 -- STEP: 592/1354 -- GLOBAL_STEP: 3300 + | > loss_text_ce: 0.2039758265018463 (0.19938960799796349) + | > loss_mel_ce: 3.3578741550445557 (3.3564437132429443) + | > loss: 0.042402978986501694 (0.04233134982830565) + | > current_lr: 5e-06 + | > step_time: 0.1501 (0.1432827235879126) + | > loader_time: 0.0067 (0.0072917595908448506) + + + --> TIME: 2024-04-09 15:12:59 -- STEP: 642/1354 -- GLOBAL_STEP: 3350 + | > loss_text_ce: 0.18603666126728058 (0.19939996297485715) + | > loss_mel_ce: 3.3102662563323975 (3.348893632398588) + | > loss: 0.04162265360355377 (0.04224159115942835) + | > current_lr: 5e-06 + | > step_time: 0.1162 (0.14265569756706942) + | > loader_time: 0.0061 (0.007277321964038124) + + + --> TIME: 2024-04-09 15:13:09 -- STEP: 692/1354 -- GLOBAL_STEP: 3400 + | > loss_text_ce: 0.21833489835262299 (0.19954090132620278) + | > loss_mel_ce: 4.009819030761719 (3.3475069372640185) + | > loss: 0.05033516511321068 (0.04222676070740795) + | > current_lr: 5e-06 + | > step_time: 0.0932 (0.14210557765354317) + | > loader_time: 0.0073 (0.007275981021065243) + + + --> TIME: 2024-04-09 15:13:20 -- STEP: 742/1354 -- GLOBAL_STEP: 3450 + | > loss_text_ce: 0.17541198432445526 (0.1987458405428818) + | > loss_mel_ce: 3.4075815677642822 (3.3460997715793215) + | > loss: 0.04265468567609787 (0.042200543724422106) + | > current_lr: 5e-06 + | > step_time: 0.212 (0.14249814038649392) + | > loader_time: 0.0076 (0.0072514448525770655) + + + --> TIME: 2024-04-09 15:13:31 -- STEP: 792/1354 -- GLOBAL_STEP: 3500 + | > loss_text_ce: 0.20107266306877136 (0.19844384030485981) + | > loss_mel_ce: 3.2473249435424805 (3.339962119706954) + | > loss: 0.041052352637052536 (0.04212388117807073) + | > current_lr: 5e-06 + | > step_time: 0.1095 (0.14233044331723974) + | > loader_time: 0.0062 (0.007219522288351348) + + + --> TIME: 2024-04-09 15:13:41 -- STEP: 842/1354 -- GLOBAL_STEP: 3550 + | > loss_text_ce: 0.23677386343479156 (0.19802369097703412) + | > loss_mel_ce: 2.7576568126678467 (3.3364748377131743) + | > loss: 0.035647984594106674 (0.042077364134569) + | > current_lr: 5e-06 + | > step_time: 0.1106 (0.14204935025149432) + | > loader_time: 0.0061 (0.007213860679409) + + + --> TIME: 2024-04-09 15:13:51 -- STEP: 892/1354 -- GLOBAL_STEP: 3600 + | > loss_text_ce: 0.16506269574165344 (0.19824869740303308) + | > loss_mel_ce: 3.1609883308410645 (3.3349398828942687) + | > loss: 0.039595846086740494 (0.04206176950792554) + | > current_lr: 5e-06 + | > step_time: 0.1357 (0.14133149706194736) + | > loader_time: 0.0068 (0.007212943827624813) + + + --> TIME: 2024-04-09 15:14:02 -- STEP: 942/1354 -- GLOBAL_STEP: 3650 + | > loss_text_ce: 0.19621030986309052 (0.19799495937654663) + | > loss_mel_ce: 2.9736056327819824 (3.3311247744631114) + | > loss: 0.03773590549826622 (0.042013330871512114) + | > current_lr: 5e-06 + | > step_time: 0.1199 (0.1414185025889401) + | > loader_time: 0.008 (0.00720803980614729) + + + --> TIME: 2024-04-09 15:14:12 -- STEP: 992/1354 -- GLOBAL_STEP: 3700 + | > loss_text_ce: 0.1849312037229538 (0.19771614986201436) + | > loss_mel_ce: 3.3865957260131836 (3.3299952339260814) + | > loss: 0.04251817986369133 (0.04199656479871796) + | > current_lr: 5e-06 + | > step_time: 0.12 (0.14119190242021298) + | > loader_time: 0.0062 (0.007182568792373904) + + + --> TIME: 2024-04-09 15:14:23 -- STEP: 1042/1354 -- GLOBAL_STEP: 3750 + | > loss_text_ce: 0.2466966211795807 (0.19738259438189332) + | > loss_mel_ce: 3.094860315322876 (3.329750199693178) + | > loss: 0.03978044167160988 (0.041989676840245786) + | > current_lr: 5e-06 + | > step_time: 0.0879 (0.1410938078062089) + | > loader_time: 0.0059 (0.007168990865549977) + + + --> TIME: 2024-04-09 15:14:33 -- STEP: 1092/1354 -- GLOBAL_STEP: 3800 + | > loss_text_ce: 0.18417926132678986 (0.19693628431130666) + | > loss_mel_ce: 3.876464366912842 (3.3270043414154333) + | > loss: 0.04834099858999252 (0.04195167483154666) + | > current_lr: 5e-06 + | > step_time: 0.1356 (0.14118074803125277) + | > loader_time: 0.0062 (0.007154488738203222) + + + --> TIME: 2024-04-09 15:14:44 -- STEP: 1142/1354 -- GLOBAL_STEP: 3850 + | > loss_text_ce: 0.2652641832828522 (0.19671073704696987) + | > loss_mel_ce: 3.1283605098724365 (3.324519961811406) + | > loss: 0.040400296449661255 (0.041919413800701966) + | > current_lr: 5e-06 + | > step_time: 0.0603 (0.14127044001310363) + | > loader_time: 0.0097 (0.007153817525887028) + + + --> TIME: 2024-04-09 15:14:54 -- STEP: 1192/1354 -- GLOBAL_STEP: 3900 + | > loss_text_ce: 0.16218766570091248 (0.19650019377770034) + | > loss_mel_ce: 3.3598427772521973 (3.322706660968342) + | > loss: 0.04192893207073212 (0.041895320413996684) + | > current_lr: 5e-06 + | > step_time: 0.2137 (0.14119688516495194) + | > loader_time: 0.0076 (0.0071507340309603885) + + + --> TIME: 2024-04-09 15:15:06 -- STEP: 1242/1354 -- GLOBAL_STEP: 3950 + | > loss_text_ce: 0.15831664204597473 (0.1960256306901453) + | > loss_mel_ce: 3.1018567085266113 (3.3223086466919742) + | > loss: 0.038811586797237396 (0.041884932595030656) + | > current_lr: 5e-06 + | > step_time: 0.2087 (0.14175751992469804) + | > loader_time: 0.0078 (0.007149206651389693) + + + --> TIME: 2024-04-09 15:15:17 -- STEP: 1292/1354 -- GLOBAL_STEP: 4000 + | > loss_text_ce: 0.18440812826156616 (0.1955937532125609) + | > loss_mel_ce: 2.9640748500823975 (3.320270514340589) + | > loss: 0.037481941282749176 (0.041855527725266746) + | > current_lr: 5e-06 + | > step_time: 0.1139 (0.14200803720545085) + | > loader_time: 0.0074 (0.007135860691129607) + + + --> TIME: 2024-04-09 15:15:40 -- STEP: 1342/1354 -- GLOBAL_STEP: 4050 + | > loss_text_ce: 0.17835023999214172 (0.19519814067199556) + | > loss_mel_ce: 3.2695953845977783 (3.317379389008179) + | > loss: 0.04104697331786156 (0.041816399883806386) + | > current_lr: 5e-06 + | > step_time: 0.211 (0.14195744433452745) + | > loader_time: 0.0084 (0.007158480529102944) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.023623576531043418 (-0.00024360876816969732) + | > avg_loss_text_ce: 0.2019951309149082 (-0.021479911529100837) + | > avg_loss_mel_ce: 3.3025841346153846 (-0.0957744671748233) + | > avg_loss: 3.5045792689690223 (-0.11725438558138324) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_4062.pth + + > EPOCH: 4/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 15:16:33)  + + --> TIME: 2024-04-09 15:16:42 -- STEP: 38/1354 -- GLOBAL_STEP: 4100 + | > loss_text_ce: 0.16431784629821777 (0.18187163299635836) + | > loss_mel_ce: 3.2505741119384766 (3.242660014252914) + | > loss: 0.040653474628925323 (0.040768234451350416) + | > current_lr: 5e-06 + | > step_time: 0.1425 (0.1431613533120407) + | > loader_time: 0.0076 (0.007278586688794587) + + + --> TIME: 2024-04-09 15:16:54 -- STEP: 88/1354 -- GLOBAL_STEP: 4150 + | > loss_text_ce: 0.18328611552715302 (0.18320408582010053) + | > loss_mel_ce: 2.88511061668396 (3.2985170158472927) + | > loss: 0.036528535187244415 (0.04144906121390789) + | > current_lr: 5e-06 + | > step_time: 0.1714 (0.14774503220211377) + | > loader_time: 0.0068 (0.007245990363034335) + + + --> TIME: 2024-04-09 15:17:04 -- STEP: 138/1354 -- GLOBAL_STEP: 4200 + | > loss_text_ce: 0.16725514829158783 (0.184844057413115) + | > loss_mel_ce: 3.4307024478912354 (3.302784569021585) + | > loss: 0.04283282905817032 (0.04151938906938271) + | > current_lr: 5e-06 + | > step_time: 0.1746 (0.1442738304967465) + | > loader_time: 0.0064 (0.007261334985926532) + + + --> TIME: 2024-04-09 15:17:14 -- STEP: 188/1354 -- GLOBAL_STEP: 4250 + | > loss_text_ce: 0.1625777631998062 (0.1844521348780774) + | > loss_mel_ce: 2.9404704570770264 (3.3047638078953363) + | > loss: 0.0369410514831543 (0.041538285685980586) + | > current_lr: 5e-06 + | > step_time: 0.1299 (0.142135574462566) + | > loader_time: 0.0068 (0.007247439090241777) + + + --> TIME: 2024-04-09 15:17:25 -- STEP: 238/1354 -- GLOBAL_STEP: 4300 + | > loss_text_ce: 0.1615418940782547 (0.18238247142118566) + | > loss_mel_ce: 2.9899473190307617 (3.280949022589612) + | > loss: 0.0375177301466465 (0.041230137514717456) + | > current_lr: 5e-06 + | > step_time: 0.1858 (0.1431795078165391) + | > loader_time: 0.0075 (0.007239778502648618) + + + --> TIME: 2024-04-09 15:17:36 -- STEP: 288/1354 -- GLOBAL_STEP: 4350 + | > loss_text_ce: 0.17877362668514252 (0.18274556095194486) + | > loss_mel_ce: 3.1894967555999756 (3.285292595624924) + | > loss: 0.04009845852851868 (0.041286169253807105) + | > current_lr: 5e-06 + | > step_time: 0.1192 (0.1424457480510076) + | > loader_time: 0.0064 (0.007192720969518025) + + + --> TIME: 2024-04-09 15:17:46 -- STEP: 338/1354 -- GLOBAL_STEP: 4400 + | > loss_text_ce: 0.20051918923854828 (0.18247097161923637) + | > loss_mel_ce: 3.339158058166504 (3.282028879639665) + | > loss: 0.04213901609182358 (0.04124404658624051) + | > current_lr: 5e-06 + | > step_time: 0.1054 (0.14209122559022622) + | > loader_time: 0.0076 (0.007157478812178211) + + + --> TIME: 2024-04-09 15:17:56 -- STEP: 388/1354 -- GLOBAL_STEP: 4450 + | > loss_text_ce: 0.22334979474544525 (0.1829404415389926) + | > loss_mel_ce: 3.3648674488067627 (3.27258879437889) + | > loss: 0.042716871947050095 (0.041137253529555384) + | > current_lr: 5e-06 + | > step_time: 0.0642 (0.1414052125104924) + | > loader_time: 0.013 (0.007130936863496131) + + + --> TIME: 2024-04-09 15:18:07 -- STEP: 438/1354 -- GLOBAL_STEP: 4500 + | > loss_text_ce: 0.20734411478042603 (0.18302889862289168) + | > loss_mel_ce: 3.7379491329193115 (3.267836661099299) + | > loss: 0.046967778354883194 (0.041081733593385496) + | > current_lr: 5e-06 + | > step_time: 0.0748 (0.14048951129390772) + | > loader_time: 0.0063 (0.007105213322051584) + + + --> TIME: 2024-04-09 15:18:17 -- STEP: 488/1354 -- GLOBAL_STEP: 4550 + | > loss_text_ce: 0.1552542895078659 (0.18269558760841362) + | > loss_mel_ce: 2.7927610874176025 (3.2630366253071146) + | > loss: 0.035095423460006714 (0.041020622316625746) + | > current_lr: 5e-06 + | > step_time: 0.1413 (0.14094552397727972) + | > loader_time: 0.0065 (0.007091249598831427) + + + --> TIME: 2024-04-09 15:18:29 -- STEP: 538/1354 -- GLOBAL_STEP: 4600 + | > loss_text_ce: 0.18752242624759674 (0.18218844359027409) + | > loss_mel_ce: 3.0474460124969482 (3.264830687675334) + | > loss: 0.03851152956485748 (0.04103594280253111) + | > current_lr: 5e-06 + | > step_time: 0.1082 (0.14190059523600188) + | > loader_time: 0.0068 (0.007132745143649304) + + + --> TIME: 2024-04-09 15:18:40 -- STEP: 588/1354 -- GLOBAL_STEP: 4650 + | > loss_text_ce: 0.16654282808303833 (0.18123378645197866) + | > loss_mel_ce: 3.4671590328216553 (3.262258936758755) + | > loss: 0.04325835406780243 (0.04099396174000637) + | > current_lr: 5e-06 + | > step_time: 0.1217 (0.1430174757023247) + | > loader_time: 0.007 (0.007124320990374299) + + + --> TIME: 2024-04-09 15:18:50 -- STEP: 638/1354 -- GLOBAL_STEP: 4700 + | > loss_text_ce: 0.17123249173164368 (0.18093610059785245) + | > loss_mel_ce: 2.8750903606414795 (3.26149231176765) + | > loss: 0.03626574948430061 (0.04098129136411928) + | > current_lr: 5e-06 + | > step_time: 0.158 (0.14217126481585363) + | > loader_time: 0.0079 (0.007147174270175467) + + + --> TIME: 2024-04-09 15:19:00 -- STEP: 688/1354 -- GLOBAL_STEP: 4750 + | > loss_text_ce: 0.1571628898382187 (0.18072137812715625) + | > loss_mel_ce: 2.952810525894165 (3.259214390155881) + | > loss: 0.03702349215745926 (0.04095161700794511) + | > current_lr: 5e-06 + | > step_time: 0.1525 (0.14156951876573773) + | > loader_time: 0.0064 (0.007124370960302131) + + + --> TIME: 2024-04-09 15:19:12 -- STEP: 738/1354 -- GLOBAL_STEP: 4800 + | > loss_text_ce: 0.169524684548378 (0.17998220264750123) + | > loss_mel_ce: 3.6741342544555664 (3.2583256976068182) + | > loss: 0.04575784504413605 (0.04093223764260768) + | > current_lr: 5e-06 + | > step_time: 0.1153 (0.1425095644746691) + | > loader_time: 0.0062 (0.007122092453767936) + + + --> TIME: 2024-04-09 15:19:22 -- STEP: 788/1354 -- GLOBAL_STEP: 4850 + | > loss_text_ce: 0.18925008177757263 (0.17995023856005693) + | > loss_mel_ce: 3.0198049545288086 (3.2572029804820337) + | > loss: 0.03820303454995155 (0.040918491430920974) + | > current_lr: 5e-06 + | > step_time: 0.0996 (0.1419613334128092) + | > loader_time: 0.007 (0.007111983855968804) + + + --> TIME: 2024-04-09 15:19:32 -- STEP: 838/1354 -- GLOBAL_STEP: 4900 + | > loss_text_ce: 0.20287317037582397 (0.1799089651830441) + | > loss_mel_ce: 3.2793197631835938 (3.253786458889454) + | > loss: 0.041454680263996124 (0.04087732720737808) + | > current_lr: 5e-06 + | > step_time: 0.1205 (0.1417623035663065) + | > loader_time: 0.0066 (0.007105614518777852) + + + --> TIME: 2024-04-09 15:19:43 -- STEP: 888/1354 -- GLOBAL_STEP: 4950 + | > loss_text_ce: 0.16376051306724548 (0.17966061078750337) + | > loss_mel_ce: 3.105944871902466 (3.254476919635996) + | > loss: 0.038925062865018845 (0.04088259038204946) + | > current_lr: 5e-06 + | > step_time: 0.1494 (0.14158088317862486) + | > loader_time: 0.0066 (0.007100426667445415) + + + --> TIME: 2024-04-09 15:19:54 -- STEP: 938/1354 -- GLOBAL_STEP: 5000 + | > loss_text_ce: 0.1743054986000061 (0.1793538663846089) + | > loss_mel_ce: 3.318225383758545 (3.2558725975723917) + | > loss: 0.04157774895429611 (0.040895553889559254) + | > current_lr: 5e-06 + | > step_time: 0.1541 (0.1419105738210778) + | > loader_time: 0.0066 (0.007101132925639529) + + + --> TIME: 2024-04-09 15:20:23 -- STEP: 988/1354 -- GLOBAL_STEP: 5050 + | > loss_text_ce: 0.18861423432826996 (0.17919145506403225) + | > loss_mel_ce: 3.8508386611938477 (3.25431538689957) + | > loss: 0.04808872565627098 (0.04087508220210488) + | > current_lr: 5e-06 + | > step_time: 0.0769 (0.14170940129863088) + | > loader_time: 0.0069 (0.0071490865487318775) + + + --> TIME: 2024-04-09 15:20:35 -- STEP: 1038/1354 -- GLOBAL_STEP: 5100 + | > loss_text_ce: 0.1536422073841095 (0.1786980598102645) + | > loss_mel_ce: 3.548220634460449 (3.2507644515046725) + | > loss: 0.0440697968006134 (0.040826935403637556) + | > current_lr: 5e-06 + | > step_time: 0.2287 (0.14200683825277852) + | > loader_time: 0.0087 (0.0071879567897848076) + + + --> TIME: 2024-04-09 15:20:45 -- STEP: 1088/1354 -- GLOBAL_STEP: 5150 + | > loss_text_ce: 0.17638741433620453 (0.1784270743784659) + | > loss_mel_ce: 3.3702445030212402 (3.249863751232624) + | > loss: 0.042221806943416595 (0.040812986765933354) + | > current_lr: 5e-06 + | > step_time: 0.1973 (0.14212232455611212) + | > loader_time: 0.0079 (0.007235585985814822) + + + --> TIME: 2024-04-09 15:20:56 -- STEP: 1138/1354 -- GLOBAL_STEP: 5200 + | > loss_text_ce: 0.1812993437051773 (0.17825272143777512) + | > loss_mel_ce: 3.4694650173187256 (3.2475402780492613) + | > loss: 0.04346148297190666 (0.040783250740416624) + | > current_lr: 5e-06 + | > step_time: 0.1201 (0.14176999810290855) + | > loader_time: 0.0075 (0.007253212962293038) + + + --> TIME: 2024-04-09 15:21:06 -- STEP: 1188/1354 -- GLOBAL_STEP: 5250 + | > loss_text_ce: 0.17012837529182434 (0.17807669008169508) + | > loss_mel_ce: 3.0302796363830566 (3.2486455241839116) + | > loss: 0.038100097328424454 (0.0407943128285183) + | > current_lr: 5e-06 + | > step_time: 0.134 (0.14154093574594542) + | > loader_time: 0.0065 (0.00730225333461055) + + + --> TIME: 2024-04-09 15:21:16 -- STEP: 1238/1354 -- GLOBAL_STEP: 5300 + | > loss_text_ce: 0.1919185072183609 (0.177739391034177) + | > loss_mel_ce: 3.9197022914886475 (3.2466920589206705) + | > loss: 0.04894786700606346 (0.04076704181285394) + | > current_lr: 5e-06 + | > step_time: 0.2119 (0.14147125326566407) + | > loader_time: 0.0064 (0.007307154296480589) + + + --> TIME: 2024-04-09 15:21:28 -- STEP: 1288/1354 -- GLOBAL_STEP: 5350 + | > loss_text_ce: 0.17539949715137482 (0.1773617395101496) + | > loss_mel_ce: 3.19155216217041 (3.2429136567234296) + | > loss: 0.04008276015520096 (0.04071756498888127) + | > current_lr: 5e-06 + | > step_time: 0.109 (0.14175950407241425) + | > loader_time: 0.0068 (0.007301856271968866) + + + --> TIME: 2024-04-09 15:21:38 -- STEP: 1338/1354 -- GLOBAL_STEP: 5400 + | > loss_text_ce: 0.1447492092847824 (0.17711685924207496) + | > loss_mel_ce: 3.460317373275757 (3.2384773353824743) + | > loss: 0.042917460203170776 (0.04066183639659643) + | > current_lr: 5e-06 + | > step_time: 0.2069 (0.14169741532193156) + | > loader_time: 0.0071 (0.007292751774125035) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.02788996696472168 (+0.004266390433678262) + | > avg_loss_text_ce: 0.1831018065030758 (-0.01889332441183239) + | > avg_loss_mel_ce: 3.2283046428973856 (-0.07427949171799897) + | > avg_loss: 3.4114064620091367 (-0.09317280695988561) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_5416.pth + + > EPOCH: 5/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 15:22:33)  + + --> TIME: 2024-04-09 15:22:41 -- STEP: 34/1354 -- GLOBAL_STEP: 5450 + | > loss_text_ce: 0.1559102088212967 (0.16318772689384572) + | > loss_mel_ce: 3.262624740600586 (3.2110824514837826) + | > loss: 0.04069684445858002 (0.04016988373854581) + | > current_lr: 5e-06 + | > step_time: 0.1592 (0.14186925747815302) + | > loader_time: 0.0083 (0.009264048408059512) + + + --> TIME: 2024-04-09 15:22:51 -- STEP: 84/1354 -- GLOBAL_STEP: 5500 + | > loss_text_ce: 0.14062979817390442 (0.16786823315279822) + | > loss_mel_ce: 3.1427974700927734 (3.2567833889098394) + | > loss: 0.0390884205698967 (0.040769662929787516) + | > current_lr: 5e-06 + | > step_time: 0.1225 (0.13580551601591562) + | > loader_time: 0.01 (0.008579120749518984) + + + --> TIME: 2024-04-09 15:23:02 -- STEP: 134/1354 -- GLOBAL_STEP: 5550 + | > loss_text_ce: 0.15009471774101257 (0.16574197329247176) + | > loss_mel_ce: 3.320854425430298 (3.230756442938278) + | > loss: 0.04132082313299179 (0.04043450562366799) + | > current_lr: 5e-06 + | > step_time: 0.1441 (0.13948128116664602) + | > loader_time: 0.0081 (0.008281204237866761) + + + --> TIME: 2024-04-09 15:23:13 -- STEP: 184/1354 -- GLOBAL_STEP: 5600 + | > loss_text_ce: 0.19404032826423645 (0.1658739173865837) + | > loss_mel_ce: 3.1243064403533936 (3.2165315539940544) + | > loss: 0.03950412943959236 (0.04026673258403723) + | > current_lr: 5e-06 + | > step_time: 0.1117 (0.14109028940615462) + | > loader_time: 0.0066 (0.008201030285462095) + + + --> TIME: 2024-04-09 15:23:24 -- STEP: 234/1354 -- GLOBAL_STEP: 5650 + | > loss_text_ce: 0.1311895102262497 (0.16437330838834122) + | > loss_mel_ce: 3.1983776092529297 (3.21290483739641) + | > loss: 0.03963770717382431 (0.04020569299976542) + | > current_lr: 5e-06 + | > step_time: 0.1676 (0.14329134602831994) + | > loader_time: 0.0082 (0.008119040065341527) + + + --> TIME: 2024-04-09 15:23:35 -- STEP: 284/1354 -- GLOBAL_STEP: 5700 + | > loss_text_ce: 0.17632801830768585 (0.164416714846878) + | > loss_mel_ce: 3.0226588249206543 (3.207100471980135) + | > loss: 0.03808317705988884 (0.04013711017307262) + | > current_lr: 5e-06 + | > step_time: 0.1396 (0.14317577573615073) + | > loader_time: 0.0072 (0.008108187729204203) + + + --> TIME: 2024-04-09 15:23:45 -- STEP: 334/1354 -- GLOBAL_STEP: 5750 + | > loss_text_ce: 0.14798012375831604 (0.16400277424982915) + | > loss_mel_ce: 3.239051342010498 (3.2141866355599045) + | > loss: 0.04032180458307266 (0.04021654141966457) + | > current_lr: 5e-06 + | > step_time: 0.1793 (0.1424191105151605) + | > loader_time: 0.0095 (0.008053633981122214) + + + --> TIME: 2024-04-09 15:23:56 -- STEP: 384/1354 -- GLOBAL_STEP: 5800 + | > loss_text_ce: 0.13556231558322906 (0.16371460301646337) + | > loss_mel_ce: 2.815244436264038 (3.2098861957589784) + | > loss: 0.035128653049468994 (0.04016191507495635) + | > current_lr: 5e-06 + | > step_time: 0.2089 (0.14240013435482982) + | > loader_time: 0.0087 (0.008014020820458717) + + + --> TIME: 2024-04-09 15:24:07 -- STEP: 434/1354 -- GLOBAL_STEP: 5850 + | > loss_text_ce: 0.1423025280237198 (0.1636148868343248) + | > loss_mel_ce: 3.096696376800537 (3.2082667790250294) + | > loss: 0.03855951130390167 (0.04014144921975757) + | > current_lr: 5e-06 + | > step_time: 0.2002 (0.14176360912586694) + | > loader_time: 0.0073 (0.007990942572668384) + + + --> TIME: 2024-04-09 15:24:17 -- STEP: 484/1354 -- GLOBAL_STEP: 5900 + | > loss_text_ce: 0.15639038383960724 (0.1630888463051851) + | > loss_mel_ce: 3.387932062149048 (3.1966213923840483) + | > loss: 0.04219431430101395 (0.039996551242002826) + | > current_lr: 5e-06 + | > step_time: 0.1119 (0.14206325613762733) + | > loader_time: 0.0067 (0.007922877950116605) + + + --> TIME: 2024-04-09 15:24:28 -- STEP: 534/1354 -- GLOBAL_STEP: 5950 + | > loss_text_ce: 0.1449303925037384 (0.16280197728885692) + | > loss_mel_ce: 3.076014995574951 (3.1967437753963113) + | > loss: 0.03834458813071251 (0.03999459307487319) + | > current_lr: 5e-06 + | > step_time: 0.1184 (0.14232307292995391) + | > loader_time: 0.0069 (0.00791662953766096) + + + --> TIME: 2024-04-09 15:24:40 -- STEP: 584/1354 -- GLOBAL_STEP: 6000 + | > loss_text_ce: 0.17790007591247559 (0.1621357728663372) + | > loss_mel_ce: 3.9545576572418213 (3.200289983047199) + | > loss: 0.04919592663645744 (0.0400288788321084) + | > current_lr: 5e-06 + | > step_time: 0.1613 (0.1430752714202829) + | > loader_time: 0.0073 (0.00789971955835001) + + + --> TIME: 2024-04-09 15:25:10 -- STEP: 634/1354 -- GLOBAL_STEP: 6050 + | > loss_text_ce: 0.2035301923751831 (0.16196199102262585) + | > loss_mel_ce: 3.0565357208251953 (3.196243055235322) + | > loss: 0.038810309022665024 (0.03997863229211388) + | > current_lr: 5e-06 + | > step_time: 0.0917 (0.14312013729889705) + | > loader_time: 0.0079 (0.007932397469361105) + + + --> TIME: 2024-04-09 15:25:20 -- STEP: 684/1354 -- GLOBAL_STEP: 6100 + | > loss_text_ce: 0.13544662296772003 (0.16196445091382447) + | > loss_mel_ce: 2.8558473587036133 (3.195550342400866) + | > loss: 0.03561064228415489 (0.03997041499623432) + | > current_lr: 5e-06 + | > step_time: 0.2071 (0.14300390607432337) + | > loader_time: 0.0085 (0.007953072151942542) + + + --> TIME: 2024-04-09 15:25:31 -- STEP: 734/1354 -- GLOBAL_STEP: 6150 + | > loss_text_ce: 0.16727209091186523 (0.1617486327074528) + | > loss_mel_ce: 3.3941538333892822 (3.191554565195814) + | > loss: 0.04239792749285698 (0.03992027694953234) + | > current_lr: 5e-06 + | > step_time: 0.1505 (0.14285916853340858) + | > loader_time: 0.0071 (0.007955806456729556) + + + --> TIME: 2024-04-09 15:25:41 -- STEP: 784/1354 -- GLOBAL_STEP: 6200 + | > loss_text_ce: 0.15981876850128174 (0.16169029322205739) + | > loss_mel_ce: 3.4093220233917236 (3.1854320889224788) + | > loss: 0.04248977452516556 (0.039846695812267026) + | > current_lr: 5e-06 + | > step_time: 0.1581 (0.14241544811093088) + | > loader_time: 0.007 (0.007957581354647258) + + + --> TIME: 2024-04-09 15:25:52 -- STEP: 834/1354 -- GLOBAL_STEP: 6250 + | > loss_text_ce: 0.13612967729568481 (0.1612099648343858) + | > loss_mel_ce: 3.054006338119507 (3.1824815559158486) + | > loss: 0.03797781094908714 (0.03980585222067739) + | > current_lr: 5e-06 + | > step_time: 0.1171 (0.1429355113054637) + | > loader_time: 0.0074 (0.007942115946067597) + + + --> TIME: 2024-04-09 15:26:03 -- STEP: 884/1354 -- GLOBAL_STEP: 6300 + | > loss_text_ce: 0.17604494094848633 (0.16097032668161704) + | > loss_mel_ce: 3.2814884185791016 (3.1823498471829668) + | > loss: 0.04116111248731613 (0.03980143142366844) + | > current_lr: 5e-06 + | > step_time: 0.0878 (0.14309114190787758) + | > loader_time: 0.0063 (0.007922137754535237) + + + --> TIME: 2024-04-09 15:26:14 -- STEP: 934/1354 -- GLOBAL_STEP: 6350 + | > loss_text_ce: 0.15135465562343597 (0.1608880623871942) + | > loss_mel_ce: 3.018235683441162 (3.1822140908598624) + | > loss: 0.03773321956396103 (0.039798835946578064) + | > current_lr: 5e-06 + | > step_time: 0.1396 (0.14312076313337377) + | > loader_time: 0.0081 (0.007911836564923764) + + + --> TIME: 2024-04-09 15:26:25 -- STEP: 984/1354 -- GLOBAL_STEP: 6400 + | > loss_text_ce: 0.18301869928836823 (0.1606309437533703) + | > loss_mel_ce: 3.3848602771759033 (3.1793477981071145) + | > loss: 0.04247475042939186 (0.039761652488534076) + | > current_lr: 5e-06 + | > step_time: 0.1045 (0.14327575835755207) + | > loader_time: 0.0091 (0.007912798625666923) + + + --> TIME: 2024-04-09 15:26:36 -- STEP: 1034/1354 -- GLOBAL_STEP: 6450 + | > loss_text_ce: 0.16710464656352997 (0.1603501757797811) + | > loss_mel_ce: 4.198423385620117 (3.181556604817018) + | > loss: 0.051970575004816055 (0.039784605328418786) + | > current_lr: 5e-06 + | > step_time: 0.1766 (0.1432817648642299) + | > loader_time: 0.0068 (0.00791842840626346) + + + --> TIME: 2024-04-09 15:26:47 -- STEP: 1084/1354 -- GLOBAL_STEP: 6500 + | > loss_text_ce: 0.13834600150585175 (0.1600724890608188) + | > loss_mel_ce: 2.988954782485962 (3.1789327782898353) + | > loss: 0.03722977265715599 (0.039750063489448983) + | > current_lr: 5e-06 + | > step_time: 0.1534 (0.14349865803419443) + | > loader_time: 0.0068 (0.007896926130315916) + + + --> TIME: 2024-04-09 15:26:57 -- STEP: 1134/1354 -- GLOBAL_STEP: 6550 + | > loss_text_ce: 0.18335726857185364 (0.16001550826314678) + | > loss_mel_ce: 3.3088297843933105 (3.1783213966527715) + | > loss: 0.0415736548602581 (0.03974210678203923) + | > current_lr: 5e-06 + | > step_time: 0.112 (0.14319323526278158) + | > loader_time: 0.0064 (0.007865886629363637) + + + --> TIME: 2024-04-09 15:27:08 -- STEP: 1184/1354 -- GLOBAL_STEP: 6600 + | > loss_text_ce: 0.12591412663459778 (0.15954409947467818) + | > loss_mel_ce: 3.102123498916626 (3.173860766597694) + | > loss: 0.03842901811003685 (0.039683392033849324) + | > current_lr: 5e-06 + | > step_time: 0.2069 (0.14322521879866304) + | > loader_time: 0.007 (0.007856159999563886) + + + --> TIME: 2024-04-09 15:27:18 -- STEP: 1234/1354 -- GLOBAL_STEP: 6650 + | > loss_text_ce: 0.15928305685520172 (0.15937032702724604) + | > loss_mel_ce: 3.7004215717315674 (3.172189331711593) + | > loss: 0.04594886675477028 (0.039661425280549285) + | > current_lr: 5e-06 + | > step_time: 0.1403 (0.142862214454762) + | > loader_time: 0.0078 (0.007835820969345118) + + + --> TIME: 2024-04-09 15:27:28 -- STEP: 1284/1354 -- GLOBAL_STEP: 6700 + | > loss_text_ce: 0.17328433692455292 (0.15900573006184118) + | > loss_mel_ce: 3.0830681324005127 (3.170747327470331) + | > loss: 0.038766101002693176 (0.03963991811185659) + | > current_lr: 5e-06 + | > step_time: 0.0821 (0.1424492090661947) + | > loader_time: 0.0076 (0.00782248498494751) + + + --> TIME: 2024-04-09 15:27:39 -- STEP: 1334/1354 -- GLOBAL_STEP: 6750 + | > loss_text_ce: 0.16300074756145477 (0.1588034394128748) + | > loss_mel_ce: 3.4853451251983643 (3.17047898504866) + | > loss: 0.04343269020318985 (0.039634315340292436) + | > current_lr: 5e-06 + | > step_time: 0.1529 (0.1421768942336806) + | > loader_time: 0.0078 (0.007806427296491217) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03104288761432354 (+0.003152920649601862) + | > avg_loss_text_ce: 0.16434556245803833 (-0.01875624404503748) + | > avg_loss_mel_ce: 3.1750533764178934 (-0.05325126647949219) + | > avg_loss: 3.3393989159510684 (-0.07200754605806825) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_6770.pth + + > EPOCH: 6/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 15:28:35)  + + --> TIME: 2024-04-09 15:28:42 -- STEP: 30/1354 -- GLOBAL_STEP: 6800 + | > loss_text_ce: 0.18123826384544373 (0.14498729358116785) + | > loss_mel_ce: 3.1210055351257324 (3.190083948771159) + | > loss: 0.03931242600083351 (0.03970323031147321) + | > current_lr: 5e-06 + | > step_time: 0.0703 (0.14937466780344646) + | > loader_time: 0.0067 (0.009447471300760905) + + + --> TIME: 2024-04-09 15:28:54 -- STEP: 80/1354 -- GLOBAL_STEP: 6850 + | > loss_text_ce: 0.18105754256248474 (0.14590988755226136) + | > loss_mel_ce: 3.540283203125 (3.175337168574333) + | > loss: 0.04430167376995087 (0.039538656128570444) + | > current_lr: 5e-06 + | > step_time: 0.1085 (0.14916016459465026) + | > loader_time: 0.0091 (0.008530697226524355) + + + --> TIME: 2024-04-09 15:29:04 -- STEP: 130/1354 -- GLOBAL_STEP: 6900 + | > loss_text_ce: 0.15508298575878143 (0.14653273282142787) + | > loss_mel_ce: 3.2169735431671143 (3.1866083842057447) + | > loss: 0.04014353081583977 (0.0396802521382387) + | > current_lr: 5e-06 + | > step_time: 0.1087 (0.14505472550025353) + | > loader_time: 0.0096 (0.00842858277834379) + + + --> TIME: 2024-04-09 15:29:15 -- STEP: 180/1354 -- GLOBAL_STEP: 6950 + | > loss_text_ce: 0.14026419818401337 (0.14606825734178233) + | > loss_mel_ce: 3.7178590297698975 (3.1716211901770706) + | > loss: 0.04593003913760185 (0.03949630374295844) + | > current_lr: 5e-06 + | > step_time: 0.1393 (0.1465363634957208) + | > loader_time: 0.009 (0.008315241336822512) + + + --> TIME: 2024-04-09 15:29:25 -- STEP: 230/1354 -- GLOBAL_STEP: 7000 + | > loss_text_ce: 0.1459205597639084 (0.14654879356208064) + | > loss_mel_ce: 3.2525887489318848 (3.158571719086689) + | > loss: 0.04045844450592995 (0.03934667355016522) + | > current_lr: 5e-06 + | > step_time: 0.1523 (0.14446162140887722) + | > loader_time: 0.0077 (0.00822102505227794) + + + --> TIME: 2024-04-09 15:29:55 -- STEP: 280/1354 -- GLOBAL_STEP: 7050 + | > loss_text_ce: 0.13644933700561523 (0.1456971812993289) + | > loss_mel_ce: 3.1017355918884277 (3.148873330014092) + | > loss: 0.03854982182383537 (0.0392210782705141) + | > current_lr: 5e-06 + | > step_time: 0.1423 (0.14367904407637466) + | > loader_time: 0.007 (0.008378009285245626) + + + --> TIME: 2024-04-09 15:30:05 -- STEP: 330/1354 -- GLOBAL_STEP: 7100 + | > loss_text_ce: 0.1460275799036026 (0.1453131768739584) + | > loss_mel_ce: 3.1079986095428467 (3.1465356465541947) + | > loss: 0.03873840719461441 (0.03918867725878952) + | > current_lr: 5e-06 + | > step_time: 0.1726 (0.1431854934403391) + | > loader_time: 0.0069 (0.008264194112835509) + + + --> TIME: 2024-04-09 15:30:15 -- STEP: 380/1354 -- GLOBAL_STEP: 7150 + | > loss_text_ce: 0.14300492405891418 (0.1455053488086712) + | > loss_mel_ce: 3.1826837062835693 (3.146396272433431) + | > loss: 0.039591532200574875 (0.039189305798591735) + | > current_lr: 5e-06 + | > step_time: 0.1163 (0.14163320378253344) + | > loader_time: 0.0071 (0.008350935107783274) + + + --> TIME: 2024-04-09 15:30:26 -- STEP: 430/1354 -- GLOBAL_STEP: 7200 + | > loss_text_ce: 0.18373900651931763 (0.14585551752947093) + | > loss_mel_ce: 3.1831111907958984 (3.1463738458101136) + | > loss: 0.04008154943585396 (0.0391932074673647) + | > current_lr: 5e-06 + | > step_time: 0.0907 (0.14166856311088405) + | > loader_time: 0.0079 (0.008338655427444813) + + + --> TIME: 2024-04-09 15:30:36 -- STEP: 480/1354 -- GLOBAL_STEP: 7250 + | > loss_text_ce: 0.18390999734401703 (0.14559011201684668) + | > loss_mel_ce: 2.881366014480591 (3.1368499000867205) + | > loss: 0.036491379141807556 (0.03907666755840181) + | > current_lr: 5e-06 + | > step_time: 0.1107 (0.1408620794614158) + | > loader_time: 0.0078 (0.008318782846132921) + + + --> TIME: 2024-04-09 15:30:47 -- STEP: 530/1354 -- GLOBAL_STEP: 7300 + | > loss_text_ce: 0.11697044968605042 (0.14528269884440131) + | > loss_mel_ce: 3.494777202606201 (3.1403571934070222) + | > loss: 0.04299699887633324 (0.03911476137362561) + | > current_lr: 5e-06 + | > step_time: 0.212 (0.14067582499306172) + | > loader_time: 0.009 (0.008263843464401536) + + + --> TIME: 2024-04-09 15:30:57 -- STEP: 580/1354 -- GLOBAL_STEP: 7350 + | > loss_text_ce: 0.14355063438415527 (0.14525731055644042) + | > loss_mel_ce: 2.9124159812927246 (3.133634215387804) + | > loss: 0.03638055548071861 (0.039034423688105474) + | > current_lr: 5e-06 + | > step_time: 0.0888 (0.14012685397575664) + | > loader_time: 0.007 (0.008200513905492327) + + + --> TIME: 2024-04-09 15:31:07 -- STEP: 630/1354 -- GLOBAL_STEP: 7400 + | > loss_text_ce: 0.1399754285812378 (0.14513709813592915) + | > loss_mel_ce: 2.9891610145568848 (3.132921069008963) + | > loss: 0.03725162521004677 (0.03902450276627429) + | > current_lr: 5e-06 + | > step_time: 0.1127 (0.13966586589813254) + | > loader_time: 0.0071 (0.008147173836117702) + + + --> TIME: 2024-04-09 15:31:18 -- STEP: 680/1354 -- GLOBAL_STEP: 7450 + | > loss_text_ce: 0.13269750773906708 (0.14478080596993942) + | > loss_mel_ce: 3.729621648788452 (3.1332052125650285) + | > loss: 0.04597999155521393 (0.03902364384711666) + | > current_lr: 5e-06 + | > step_time: 0.1415 (0.14003707521102024) + | > loader_time: 0.0072 (0.008104766116422767) + + + --> TIME: 2024-04-09 15:31:28 -- STEP: 730/1354 -- GLOBAL_STEP: 7500 + | > loss_text_ce: 0.12844978272914886 (0.1442774949927035) + | > loss_mel_ce: 3.164337158203125 (3.13132432944154) + | > loss: 0.039199844002723694 (0.038995260597295966) + | > current_lr: 5e-06 + | > step_time: 0.1775 (0.13983278633797014) + | > loader_time: 0.0079 (0.008088620068275775) + + + --> TIME: 2024-04-09 15:31:39 -- STEP: 780/1354 -- GLOBAL_STEP: 7550 + | > loss_text_ce: 0.1426256150007248 (0.14386667690406982) + | > loss_mel_ce: 3.049799919128418 (3.1268063150919394) + | > loss: 0.03800506517291069 (0.03893658400346074) + | > current_lr: 5e-06 + | > step_time: 0.1155 (0.14006429635561427) + | > loader_time: 0.0067 (0.008055421633598143) + + + --> TIME: 2024-04-09 15:31:50 -- STEP: 830/1354 -- GLOBAL_STEP: 7600 + | > loss_text_ce: 0.11698043346405029 (0.14325564949806888) + | > loss_mel_ce: 2.9137136936187744 (3.127792764571775) + | > loss: 0.03607968986034393 (0.038941053305584304) + | > current_lr: 5e-06 + | > step_time: 0.1437 (0.1403781908104218) + | > loader_time: 0.0091 (0.00801565388599076) + + + --> TIME: 2024-04-09 15:32:01 -- STEP: 880/1354 -- GLOBAL_STEP: 7650 + | > loss_text_ce: 0.11961613595485687 (0.14284317707642888) + | > loss_mel_ce: 3.1666159629821777 (3.1270222604274744) + | > loss: 0.03912181034684181 (0.03892697023481812) + | > current_lr: 5e-06 + | > step_time: 0.2139 (0.14059211367910548) + | > loader_time: 0.0072 (0.007989304715936849) + + + --> TIME: 2024-04-09 15:32:11 -- STEP: 930/1354 -- GLOBAL_STEP: 7700 + | > loss_text_ce: 0.12773770093917847 (0.14237299163335088) + | > loss_mel_ce: 2.7561426162719727 (3.123171064674213) + | > loss: 0.03433191031217575 (0.03887552521041328) + | > current_lr: 5e-06 + | > step_time: 0.1619 (0.14037812448316997) + | > loader_time: 0.0081 (0.007967784327845437) + + + --> TIME: 2024-04-09 15:32:21 -- STEP: 980/1354 -- GLOBAL_STEP: 7750 + | > loss_text_ce: 0.12240156531333923 (0.1422339962681335) + | > loss_mel_ce: 3.2872836589813232 (3.124614633102806) + | > loss: 0.040591489523649216 (0.03889105584358382) + | > current_lr: 5e-06 + | > step_time: 0.1537 (0.14035434430959265) + | > loader_time: 0.0077 (0.007983432010728502) + + + --> TIME: 2024-04-09 15:32:32 -- STEP: 1030/1354 -- GLOBAL_STEP: 7800 + | > loss_text_ce: 0.1769365668296814 (0.14197993054985977) + | > loss_mel_ce: 3.1302061080932617 (3.1237400008636764) + | > loss: 0.03937074914574623 (0.03887761896385733) + | > current_lr: 5e-06 + | > step_time: 0.0976 (0.14041323198855493) + | > loader_time: 0.0071 (0.007980145759952892) + + + --> TIME: 2024-04-09 15:32:43 -- STEP: 1080/1354 -- GLOBAL_STEP: 7850 + | > loss_text_ce: 0.11847954243421555 (0.14147734318480426) + | > loss_mel_ce: 2.915168523788452 (3.1217732694413916) + | > loss: 0.036114856600761414 (0.03884822230579127) + | > current_lr: 5e-06 + | > step_time: 0.163 (0.1407219871326728) + | > loader_time: 0.0071 (0.007961276946244426) + + + --> TIME: 2024-04-09 15:32:54 -- STEP: 1130/1354 -- GLOBAL_STEP: 7900 + | > loss_text_ce: 0.17870046198368073 (0.14115048086484955) + | > loss_mel_ce: 2.822421073913574 (3.118697591165526) + | > loss: 0.03572763875126839 (0.03880771587446202) + | > current_lr: 5e-06 + | > step_time: 0.0782 (0.14082370985925724) + | > loader_time: 0.0078 (0.007967868526424988) + + + --> TIME: 2024-04-09 15:33:04 -- STEP: 1180/1354 -- GLOBAL_STEP: 7950 + | > loss_text_ce: 0.15842194855213165 (0.14095029797468128) + | > loss_mel_ce: 3.4938607215881348 (3.1198197395114575) + | > loss: 0.0434795580804348 (0.03881869166142359) + | > current_lr: 5e-06 + | > step_time: 0.1066 (0.14078064853862174) + | > loader_time: 0.0092 (0.007974893157765017) + + + --> TIME: 2024-04-09 15:33:16 -- STEP: 1230/1354 -- GLOBAL_STEP: 8000 + | > loss_text_ce: 0.1291884183883667 (0.14049113658143234) + | > loss_mel_ce: 3.2665114402770996 (3.1199899072569557) + | > loss: 0.040424998849630356 (0.038815251256271166) + | > current_lr: 5e-06 + | > step_time: 0.167 (0.1413417820038833) + | > loader_time: 0.0093 (0.007965947360527225) + + + --> TIME: 2024-04-09 15:33:48 -- STEP: 1280/1354 -- GLOBAL_STEP: 8050 + | > loss_text_ce: 0.1457480788230896 (0.1399873658781869) + | > loss_mel_ce: 2.9292893409729004 (3.1174252999946472) + | > loss: 0.036607589572668076 (0.03877872293523983) + | > current_lr: 5e-06 + | > step_time: 0.0908 (0.14177639093250022) + | > loader_time: 0.0145 (0.008015202917158595) + + + --> TIME: 2024-04-09 15:33:59 -- STEP: 1330/1354 -- GLOBAL_STEP: 8100 + | > loss_text_ce: 0.10367762297391891 (0.13960397934779167) + | > loss_mel_ce: 3.0021934509277344 (3.1186535924897165) + | > loss: 0.03697465360164642 (0.03878878132909315) + | > current_lr: 5e-06 + | > step_time: 0.1603 (0.14174893529791566) + | > loader_time: 0.0072 (0.008027774947030206) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.02471149884737455 (-0.006331388766948991) + | > avg_loss_text_ce: 0.14501618020809615 (-0.019329382249942184) + | > avg_loss_mel_ce: 3.131878687785222 (-0.043174688632671376) + | > avg_loss: 3.2768948811751146 (-0.06250403477595379) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_8124.pth + + > EPOCH: 7/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 15:34:54)  + + --> TIME: 2024-04-09 15:35:01 -- STEP: 26/1354 -- GLOBAL_STEP: 8150 + | > loss_text_ce: 0.10935340821743011 (0.1274021935577576) + | > loss_mel_ce: 2.7982938289642334 (3.012981818272517) + | > loss: 0.03461484611034393 (0.03738552460876794) + | > current_lr: 5e-06 + | > step_time: 0.2137 (0.15076802327082708) + | > loader_time: 0.0081 (0.008881128751314603) + + + --> TIME: 2024-04-09 15:35:11 -- STEP: 76/1354 -- GLOBAL_STEP: 8200 + | > loss_text_ce: 0.1236281767487526 (0.12979679250795592) + | > loss_mel_ce: 3.5642037391662598 (3.0385313724216663) + | > loss: 0.04390276223421097 (0.03771819299282996) + | > current_lr: 5e-06 + | > step_time: 0.1736 (0.13813991609372586) + | > loader_time: 0.0095 (0.008470469399502403) + + + --> TIME: 2024-04-09 15:35:22 -- STEP: 126/1354 -- GLOBAL_STEP: 8250 + | > loss_text_ce: 0.10613599419593811 (0.12919740875562027) + | > loss_mel_ce: 3.2061336040496826 (3.0545922006879533) + | > loss: 0.03943178430199623 (0.037902257819142596) + | > current_lr: 5e-06 + | > step_time: 0.1831 (0.13908585858723468) + | > loader_time: 0.0071 (0.008351895544264056) + + + --> TIME: 2024-04-09 15:35:32 -- STEP: 176/1354 -- GLOBAL_STEP: 8300 + | > loss_text_ce: 0.1341789960861206 (0.13001825292171398) + | > loss_mel_ce: 2.786381244659424 (3.062807614153083) + | > loss: 0.03476857766509056 (0.03800983238033949) + | > current_lr: 5e-06 + | > step_time: 0.1113 (0.13925922323356976) + | > loader_time: 0.0077 (0.008247039534828889) + + + --> TIME: 2024-04-09 15:35:43 -- STEP: 226/1354 -- GLOBAL_STEP: 8350 + | > loss_text_ce: 0.13772521913051605 (0.129593646895569) + | > loss_mel_ce: 2.658233880996704 (3.055144873340574) + | > loss: 0.0332852266728878 (0.03791355447992023) + | > current_lr: 5e-06 + | > step_time: 0.1116 (0.13907476231060203) + | > loader_time: 0.0072 (0.00818883739741503) + + + --> TIME: 2024-04-09 15:35:54 -- STEP: 276/1354 -- GLOBAL_STEP: 8400 + | > loss_text_ce: 0.14847570657730103 (0.12810017492460168) + | > loss_mel_ce: 3.0112040042877197 (3.0427433511485233) + | > loss: 0.03761523589491844 (0.0377481379006328) + | > current_lr: 5e-06 + | > step_time: 0.0987 (0.14073835939600854) + | > loader_time: 0.0072 (0.008221653060636663) + + + --> TIME: 2024-04-09 15:36:05 -- STEP: 326/1354 -- GLOBAL_STEP: 8450 + | > loss_text_ce: 0.11930271983146667 (0.12756813790626317) + | > loss_mel_ce: 2.5487687587738037 (3.03693524606389) + | > loss: 0.031762756407260895 (0.03767266000501034) + | > current_lr: 5e-06 + | > step_time: 0.1582 (0.14176956495624382) + | > loader_time: 0.0072 (0.008204389934890843) + + + --> TIME: 2024-04-09 15:36:15 -- STEP: 376/1354 -- GLOBAL_STEP: 8500 + | > loss_text_ce: 0.11704868823289871 (0.12742802306534123) + | > loss_mel_ce: 3.2173404693603516 (3.0423215786193305) + | > loss: 0.03969511017203331 (0.03773511496392337) + | > current_lr: 5e-06 + | > step_time: 0.1407 (0.1412488938646114) + | > loader_time: 0.0071 (0.008159882210670643) + + + --> TIME: 2024-04-09 15:36:26 -- STEP: 426/1354 -- GLOBAL_STEP: 8550 + | > loss_text_ce: 0.13174237310886383 (0.12732381149776664) + | > loss_mel_ce: 3.015255928039551 (3.0492852316216146) + | > loss: 0.03746426850557327 (0.03781677497454654) + | > current_lr: 5e-06 + | > step_time: 0.0993 (0.1411804677175246) + | > loader_time: 0.0105 (0.008148977454279517) + + + --> TIME: 2024-04-09 15:36:36 -- STEP: 476/1354 -- GLOBAL_STEP: 8600 + | > loss_text_ce: 0.13923580944538116 (0.12724426105430636) + | > loss_mel_ce: 3.4332809448242188 (3.048068145243062) + | > loss: 0.042529962956905365 (0.037801338821461736) + | > current_lr: 5e-06 + | > step_time: 0.105 (0.14089864241976716) + | > loader_time: 0.0072 (0.008093284458673302) + + + --> TIME: 2024-04-09 15:36:46 -- STEP: 526/1354 -- GLOBAL_STEP: 8650 + | > loss_text_ce: 0.11961117386817932 (0.12755631455733732) + | > loss_mel_ce: 3.0887973308563232 (3.0483769373295444) + | > loss: 0.03819534182548523 (0.03780872985975599) + | > current_lr: 5e-06 + | > step_time: 0.1127 (0.1404400465606285) + | > loader_time: 0.0076 (0.008074917720751165) + + + --> TIME: 2024-04-09 15:36:57 -- STEP: 576/1354 -- GLOBAL_STEP: 8700 + | > loss_text_ce: 0.09444227814674377 (0.1272911657708592) + | > loss_mel_ce: 3.2271246910095215 (3.051525062157051) + | > loss: 0.03954246640205383 (0.037843050997859486) + | > current_lr: 5e-06 + | > step_time: 0.2241 (0.14008272190888732) + | > loader_time: 0.0077 (0.008037221514516408) + + + --> TIME: 2024-04-09 15:37:07 -- STEP: 626/1354 -- GLOBAL_STEP: 8750 + | > loss_text_ce: 0.09627939760684967 (0.1272266214243329) + | > loss_mel_ce: 2.843775749206543 (3.057985157250599) + | > loss: 0.03500065580010414 (0.03791918852530157) + | > current_lr: 5e-06 + | > step_time: 0.2186 (0.1400056627992625) + | > loader_time: 0.0074 (0.008019816761199661) + + + --> TIME: 2024-04-09 15:37:17 -- STEP: 676/1354 -- GLOBAL_STEP: 8800 + | > loss_text_ce: 0.10692661255598068 (0.12754378458980967) + | > loss_mel_ce: 2.901343584060669 (3.059439723308274) + | > loss: 0.03581274300813675 (0.037940280539529234) + | > current_lr: 5e-06 + | > step_time: 0.1746 (0.13939269160377915) + | > loader_time: 0.0093 (0.007998004467529664) + + + --> TIME: 2024-04-09 15:37:27 -- STEP: 726/1354 -- GLOBAL_STEP: 8850 + | > loss_text_ce: 0.1117510125041008 (0.12771082487850147) + | > loss_mel_ce: 2.857597827911377 (3.059964774068726) + | > loss: 0.035349391400814056 (0.03794851971497715) + | > current_lr: 5e-06 + | > step_time: 0.1075 (0.1388882257393241) + | > loader_time: 0.0069 (0.007982989973273161) + + + --> TIME: 2024-04-09 15:37:38 -- STEP: 776/1354 -- GLOBAL_STEP: 8900 + | > loss_text_ce: 0.09344322234392166 (0.12712134785719748) + | > loss_mel_ce: 3.0434529781341553 (3.058673323736981) + | > loss: 0.03734400123357773 (0.03792612773567897) + | > current_lr: 5e-06 + | > step_time: 0.1624 (0.13963326719618355) + | > loader_time: 0.0074 (0.007971349450730799) + + + --> TIME: 2024-04-09 15:37:50 -- STEP: 826/1354 -- GLOBAL_STEP: 8950 + | > loss_text_ce: 0.11804404109716415 (0.12669037222032392) + | > loss_mel_ce: 2.788139820098877 (3.057772742922604) + | > loss: 0.034597430378198624 (0.037910275869342064) + | > current_lr: 5e-06 + | > step_time: 0.173 (0.14027394018796702) + | > loader_time: 0.0072 (0.00795647364840381) + + + --> TIME: 2024-04-09 15:38:01 -- STEP: 876/1354 -- GLOBAL_STEP: 9000 + | > loss_text_ce: 0.09752028435468674 (0.12619412495631321) + | > loss_mel_ce: 2.9242098331451416 (3.057812005417535) + | > loss: 0.03597297891974449 (0.037904835578275294) + | > current_lr: 5e-06 + | > step_time: 0.1879 (0.1411170322601111) + | > loader_time: 0.0071 (0.00794718390730418) + + + --> TIME: 2024-04-09 15:38:32 -- STEP: 926/1354 -- GLOBAL_STEP: 9050 + | > loss_text_ce: 0.11895988136529922 (0.12583518889984086) + | > loss_mel_ce: 2.663908004760742 (3.0519645592563895) + | > loss: 0.03312937915325165 (0.037830950060686305) + | > current_lr: 5e-06 + | > step_time: 0.1354 (0.1417971738728799) + | > loader_time: 0.0092 (0.007991786909412359) + + + --> TIME: 2024-04-09 15:38:42 -- STEP: 976/1354 -- GLOBAL_STEP: 9100 + | > loss_text_ce: 0.15027955174446106 (0.12552095527501117) + | > loss_mel_ce: 3.11513352394104 (3.052346712497417) + | > loss: 0.03887396678328514 (0.03783175862989706) + | > current_lr: 5e-06 + | > step_time: 0.0802 (0.14191753576036376) + | > loader_time: 0.0071 (0.007990413757621266) + + + --> TIME: 2024-04-09 15:38:53 -- STEP: 1026/1354 -- GLOBAL_STEP: 9150 + | > loss_text_ce: 0.12678766250610352 (0.12524003658233077) + | > loss_mel_ce: 2.705348014831543 (3.0495029316078646) + | > loss: 0.03371590003371239 (0.03779455982729586) + | > current_lr: 5e-06 + | > step_time: 0.0918 (0.14207483894876122) + | > loader_time: 0.0068 (0.007982038382665917) + + + --> TIME: 2024-04-09 15:39:04 -- STEP: 1076/1354 -- GLOBAL_STEP: 9200 + | > loss_text_ce: 0.10170300304889679 (0.12498052867621051) + | > loss_mel_ce: 2.6060452461242676 (3.0472547163750625) + | > loss: 0.032235097140073776 (0.0377647059806882) + | > current_lr: 5e-06 + | > step_time: 0.1343 (0.14196071394314128) + | > loader_time: 0.0067 (0.007973360084689683) + + + --> TIME: 2024-04-09 15:39:15 -- STEP: 1126/1354 -- GLOBAL_STEP: 9250 + | > loss_text_ce: 0.10652583837509155 (0.12449025187383307) + | > loss_mel_ce: 2.6916372776031494 (3.046684464171769) + | > loss: 0.03331146761775017 (0.03775208064091935) + | > current_lr: 5e-06 + | > step_time: 0.1716 (0.1425232927498557) + | > loader_time: 0.0075 (0.007978137814130485) + + + --> TIME: 2024-04-09 15:39:26 -- STEP: 1176/1354 -- GLOBAL_STEP: 9300 + | > loss_text_ce: 0.11588484793901443 (0.12401870633063657) + | > loss_mel_ce: 3.0998647212982178 (3.046912451179662) + | > loss: 0.03828273341059685 (0.03774918114836169) + | > current_lr: 5e-06 + | > step_time: 0.0974 (0.14279608600804608) + | > loader_time: 0.0079 (0.00798492893880726) + + + --> TIME: 2024-04-09 15:39:38 -- STEP: 1226/1354 -- GLOBAL_STEP: 9350 + | > loss_text_ce: 0.11637205630540848 (0.1236060193236946) + | > loss_mel_ce: 2.7704739570617676 (3.0455414455438743) + | > loss: 0.034367214888334274 (0.03772794670572727) + | > current_lr: 5e-06 + | > step_time: 0.1192 (0.14309484301344627) + | > loader_time: 0.0069 (0.007976751148603467) + + + --> TIME: 2024-04-09 15:39:48 -- STEP: 1276/1354 -- GLOBAL_STEP: 9400 + | > loss_text_ce: 0.10172579437494278 (0.12329683898361296) + | > loss_mel_ce: 2.8249828815460205 (3.0447266836151474) + | > loss: 0.03484177216887474 (0.03771456645445484) + | > current_lr: 5e-06 + | > step_time: 0.1527 (0.1431261712107167) + | > loader_time: 0.0071 (0.00796148545316019) + + + --> TIME: 2024-04-09 15:39:59 -- STEP: 1326/1354 -- GLOBAL_STEP: 9450 + | > loss_text_ce: 0.10999385267496109 (0.12305367355927428) + | > loss_mel_ce: 3.1058573722839355 (3.042884952522155) + | > loss: 0.03828394412994385 (0.03768974625495671) + | > current_lr: 5e-06 + | > step_time: 0.1183 (0.143180283665837) + | > loader_time: 0.0073 (0.007963206793208279) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.024514308342566855 (-0.0001971905048076955) + | > avg_loss_text_ce: 0.12979680930192655 (-0.015219370906169594) + | > avg_loss_mel_ce: 3.09517187338609 (-0.03670681439913226) + | > avg_loss: 3.2249686534588156 (-0.05192622771629907) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_9478.pth + + > EPOCH: 8/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 15:40:57)  + + --> TIME: 2024-04-09 15:41:02 -- STEP: 22/1354 -- GLOBAL_STEP: 9500 + | > loss_text_ce: 0.09014296531677246 (0.11808441308411685) + | > loss_mel_ce: 2.7028708457946777 (2.9662234132940117) + | > loss: 0.03325016424059868 (0.036717951467091385) + | > current_lr: 5e-06 + | > step_time: 0.2124 (0.13618695735931394) + | > loader_time: 0.0078 (0.009239868684248491) + + + --> TIME: 2024-04-09 15:41:13 -- STEP: 72/1354 -- GLOBAL_STEP: 9550 + | > loss_text_ce: 0.099349744617939 (0.11561066088163191) + | > loss_mel_ce: 3.29196834564209 (2.982229358620114) + | > loss: 0.040372833609580994 (0.03687904860513905) + | > current_lr: 5e-06 + | > step_time: 0.2077 (0.14116868045594957) + | > loader_time: 0.0082 (0.008550776375664605) + + + --> TIME: 2024-04-09 15:41:24 -- STEP: 122/1354 -- GLOBAL_STEP: 9600 + | > loss_text_ce: 0.10334166139364243 (0.11519857117387115) + | > loss_mel_ce: 2.755037307739258 (3.00017294727388) + | > loss: 0.034028321504592896 (0.037087756835046344) + | > current_lr: 5e-06 + | > step_time: 0.2101 (0.14358565455577418) + | > loader_time: 0.0102 (0.008585281059390205) + + + --> TIME: 2024-04-09 15:41:36 -- STEP: 172/1354 -- GLOBAL_STEP: 9650 + | > loss_text_ce: 0.09419073164463043 (0.11414805284246456) + | > loss_mel_ce: 2.832397937774658 (3.009875953197478) + | > loss: 0.03484034165740013 (0.03719076258671836) + | > current_lr: 5e-06 + | > step_time: 0.1607 (0.14632197452145948) + | > loader_time: 0.0099 (0.008600991825724751) + + + --> TIME: 2024-04-09 15:41:46 -- STEP: 222/1354 -- GLOBAL_STEP: 9700 + | > loss_text_ce: 0.10385910421609879 (0.11410859710461385) + | > loss_mel_ce: 2.8318824768066406 (3.0135221921645834) + | > loss: 0.03494930639863014 (0.037233700455577494) + | > current_lr: 5e-06 + | > step_time: 0.1424 (0.14504307669562247) + | > loader_time: 0.009 (0.008547108452599327) + + + --> TIME: 2024-04-09 15:41:57 -- STEP: 272/1354 -- GLOBAL_STEP: 9750 + | > loss_text_ce: 0.14630943536758423 (0.1135181559194975) + | > loss_mel_ce: 3.2476203441619873 (3.000447843881214) + | > loss: 0.04040392488241196 (0.03707102441694585) + | > current_lr: 5e-06 + | > step_time: 0.1158 (0.14514150426668265) + | > loader_time: 0.0069 (0.008402753402205074) + + + --> TIME: 2024-04-09 15:42:08 -- STEP: 322/1354 -- GLOBAL_STEP: 9800 + | > loss_text_ce: 0.10100550204515457 (0.11357792878743284) + | > loss_mel_ce: 2.837860584259033 (3.015707010808198) + | > loss: 0.034986503422260284 (0.03725339273881654) + | > current_lr: 5e-06 + | > step_time: 0.1353 (0.14449487413678858) + | > loader_time: 0.0071 (0.008311407166238155) + + + --> TIME: 2024-04-09 15:42:18 -- STEP: 372/1354 -- GLOBAL_STEP: 9850 + | > loss_text_ce: 0.13829874992370605 (0.11386427301312646) + | > loss_mel_ce: 3.0123777389526367 (3.010290074092085) + | > loss: 0.037508055567741394 (0.037192314267358764) + | > current_lr: 5e-06 + | > step_time: 0.0674 (0.14361334039318954) + | > loader_time: 0.007 (0.008276133768020133) + + + --> TIME: 2024-04-09 15:42:29 -- STEP: 422/1354 -- GLOBAL_STEP: 9900 + | > loss_text_ce: 0.12227155268192291 (0.11418882211873317) + | > loss_mel_ce: 2.886164426803589 (3.0055998382975133) + | > loss: 0.035814713686704636 (0.0371403418243002) + | > current_lr: 5e-06 + | > step_time: 0.0942 (0.1436171723768044) + | > loader_time: 0.009 (0.008249705436670385) + + + --> TIME: 2024-04-09 15:42:39 -- STEP: 472/1354 -- GLOBAL_STEP: 9950 + | > loss_text_ce: 0.14904965460300446 (0.11465802834511309) + | > loss_mel_ce: 3.28926157951355 (3.003699739605693) + | > loss: 0.04093227908015251 (0.03712330738445586) + | > current_lr: 5e-06 + | > step_time: 0.0796 (0.14243647759243594) + | > loader_time: 0.0069 (0.008209012827630775) + + + --> TIME: 2024-04-09 15:42:50 -- STEP: 522/1354 -- GLOBAL_STEP: 10000 + | > loss_text_ce: 0.09940872341394424 (0.1143332138172283) + | > loss_mel_ce: 3.1260030269622803 (2.998038872905161) + | > loss: 0.038397759199142456 (0.037052049283723264) + | > current_lr: 5e-06 + | > step_time: 0.1049 (0.14211307220532063) + | > loader_time: 0.007 (0.008228697082548767) + + + > CHECKPOINT : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/checkpoint_10000.pth + + --> TIME: 2024-04-09 15:43:36 -- STEP: 572/1354 -- GLOBAL_STEP: 10050 + | > loss_text_ce: 0.12570598721504211 (0.1144709676712543) + | > loss_mel_ce: 3.2488083839416504 (2.9951917708336886) + | > loss: 0.04017278924584389 (0.03701979513573168) + | > current_lr: 5e-06 + | > step_time: 0.1963 (0.14195529814366695) + | > loader_time: 0.009 (0.008313201107345266) + + + --> TIME: 2024-04-09 15:43:47 -- STEP: 622/1354 -- GLOBAL_STEP: 10100 + | > loss_text_ce: 0.10631317645311356 (0.11400146801563704) + | > loss_mel_ce: 2.905747652053833 (2.991360657851412) + | > loss: 0.03585786744952202 (0.03696859734869558) + | > current_lr: 5e-06 + | > step_time: 0.1333 (0.1420855361165726) + | > loader_time: 0.0086 (0.008351334421581019) + + + --> TIME: 2024-04-09 15:43:58 -- STEP: 672/1354 -- GLOBAL_STEP: 10150 + | > loss_text_ce: 0.12659788131713867 (0.11396448635717943) + | > loss_mel_ce: 3.792799234390259 (2.9958516841843013) + | > loss: 0.04665949195623398 (0.037021621690309106) + | > current_lr: 5e-06 + | > step_time: 0.1603 (0.14234231483368667) + | > loader_time: 0.0077 (0.00832107343844005) + + + --> TIME: 2024-04-09 15:44:09 -- STEP: 722/1354 -- GLOBAL_STEP: 10200 + | > loss_text_ce: 0.11694282293319702 (0.11347931290143415) + | > loss_mel_ce: 3.273956537246704 (2.994115006230213) + | > loss: 0.04036784917116165 (0.036995171097176864) + | > current_lr: 5e-06 + | > step_time: 0.1306 (0.1426995847363884) + | > loader_time: 0.0085 (0.008301862389096925) + + + --> TIME: 2024-04-09 15:44:20 -- STEP: 772/1354 -- GLOBAL_STEP: 10250 + | > loss_text_ce: 0.10903584212064743 (0.11346673645040532) + | > loss_mel_ce: 2.726167917251587 (2.9978741518574044) + | > loss: 0.033752426505088806 (0.037039773121302445) + | > current_lr: 5e-06 + | > step_time: 0.0991 (0.1425877587165241) + | > loader_time: 0.0075 (0.008275349831951712) + + + --> TIME: 2024-04-09 15:44:31 -- STEP: 822/1354 -- GLOBAL_STEP: 10300 + | > loss_text_ce: 0.12088827788829803 (0.1132658158685024) + | > loss_mel_ce: 3.0115456581115723 (2.9995410398845244) + | > loss: 0.03729087859392166 (0.03705722511657615) + | > current_lr: 5e-06 + | > step_time: 0.0913 (0.14286586402976612) + | > loader_time: 0.0078 (0.008288630023779943) + + + --> TIME: 2024-04-09 15:44:42 -- STEP: 872/1354 -- GLOBAL_STEP: 10350 + | > loss_text_ce: 0.09020840376615524 (0.11281779823705144) + | > loss_mel_ce: 2.729570150375366 (2.9967516357198756) + | > loss: 0.033568792045116425 (0.037018684377688765) + | > current_lr: 5e-06 + | > step_time: 0.1496 (0.1430131158697498) + | > loader_time: 0.0079 (0.00832324399860626) + + + --> TIME: 2024-04-09 15:44:52 -- STEP: 922/1354 -- GLOBAL_STEP: 10400 + | > loss_text_ce: 0.10571166127920151 (0.11260881287868008) + | > loss_mel_ce: 3.109067440032959 (2.9953568710939478) + | > loss: 0.03827118128538132 (0.03699959212320166) + | > current_lr: 5e-06 + | > step_time: 0.1307 (0.1430797473468907) + | > loader_time: 0.0095 (0.008311520171010306) + + + --> TIME: 2024-04-09 15:45:03 -- STEP: 972/1354 -- GLOBAL_STEP: 10450 + | > loss_text_ce: 0.09351556748151779 (0.11231250276045544) + | > loss_mel_ce: 2.749891519546509 (2.9957862540527618) + | > loss: 0.033850084990262985 (0.03700117633290928) + | > current_lr: 5e-06 + | > step_time: 0.1427 (0.14325784631226798) + | > loader_time: 0.0083 (0.008293372613412358) + + + --> TIME: 2024-04-09 15:45:14 -- STEP: 1022/1354 -- GLOBAL_STEP: 10500 + | > loss_text_ce: 0.07934033125638962 (0.11229030162445953) + | > loss_mel_ce: 2.954395055770874 (2.997556800011785) + | > loss: 0.036115895956754684 (0.03702198996512976) + | > current_lr: 5e-06 + | > step_time: 0.1899 (0.14292107099656276) + | > loader_time: 0.0079 (0.008280598953978651) + + + --> TIME: 2024-04-09 15:45:25 -- STEP: 1072/1354 -- GLOBAL_STEP: 10550 + | > loss_text_ce: 0.08696676045656204 (0.11185953253880142) + | > loss_mel_ce: 3.291694164276123 (2.9975612619474745) + | > loss: 0.040222153067588806 (0.03701691488464323) + | > current_lr: 5e-06 + | > step_time: 0.1717 (0.14333137291580905) + | > loader_time: 0.0076 (0.008260473164159857) + + + --> TIME: 2024-04-09 15:45:35 -- STEP: 1122/1354 -- GLOBAL_STEP: 10600 + | > loss_text_ce: 0.10486464202404022 (0.11197305571639811) + | > loss_mel_ce: 2.992382049560547 (2.9983188105139487) + | > loss: 0.03687198460102081 (0.03702728479350263) + | > current_lr: 5e-06 + | > step_time: 0.1623 (0.14302070468079822) + | > loader_time: 0.0074 (0.00824088931296173) + + + --> TIME: 2024-04-09 15:45:46 -- STEP: 1172/1354 -- GLOBAL_STEP: 10650 + | > loss_text_ce: 0.0939573422074318 (0.11173748539652638) + | > loss_mel_ce: 2.9926486015319824 (2.9990226327355813) + | > loss: 0.036745309829711914 (0.03703285921714668) + | > current_lr: 5e-06 + | > step_time: 0.1533 (0.14287783729338407) + | > loader_time: 0.007 (0.008217435037723575) + + + --> TIME: 2024-04-09 15:45:56 -- STEP: 1222/1354 -- GLOBAL_STEP: 10700 + | > loss_text_ce: 0.08532080799341202 (0.11172020998699575) + | > loss_mel_ce: 2.6858980655670166 (2.9980913029950127) + | > loss: 0.032990701496601105 (0.037021566296694446) + | > current_lr: 5e-06 + | > step_time: 0.1989 (0.14280151501419888) + | > loader_time: 0.009 (0.008190450223495056) + + + --> TIME: 2024-04-09 15:46:07 -- STEP: 1272/1354 -- GLOBAL_STEP: 10750 + | > loss_text_ce: 0.0966683179140091 (0.11151222055836481) + | > loss_mel_ce: 2.978166341781616 (2.9951391298815913) + | > loss: 0.03660517558455467 (0.036983945311791035) + | > current_lr: 5e-06 + | > step_time: 0.2013 (0.14296740368477198) + | > loader_time: 0.0089 (0.008174114444720668) + + + --> TIME: 2024-04-09 15:46:19 -- STEP: 1322/1354 -- GLOBAL_STEP: 10800 + | > loss_text_ce: 0.0968487411737442 (0.11130026286628532) + | > loss_mel_ce: 2.895503520965576 (2.9949639139305297) + | > loss: 0.03562324121594429 (0.036979336094297224) + | > current_lr: 5e-06 + | > step_time: 0.1877 (0.143241112243029) + | > loader_time: 0.0091 (0.00815633529974725) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.026413862521831807 (+0.0018995541792649513) + | > avg_loss_text_ce: 0.1195776158800492 (-0.010219193421877354) + | > avg_loss_mel_ce: 3.0646284176753116 (-0.03054345571077821) + | > avg_loss: 3.184206008911133 (-0.04076264454768275) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_10832.pth + + > EPOCH: 9/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 15:47:14)  + + --> TIME: 2024-04-09 15:47:20 -- STEP: 18/1354 -- GLOBAL_STEP: 10850 + | > loss_text_ce: 0.07499896734952927 (0.10034403825799625) + | > loss_mel_ce: 2.6737067699432373 (3.0484794510735407) + | > loss: 0.03272268548607826 (0.037485994605554476) + | > current_lr: 5e-06 + | > step_time: 0.1352 (0.15165317058563232) + | > loader_time: 0.0079 (0.009946703910827637) + + + --> TIME: 2024-04-09 15:47:31 -- STEP: 68/1354 -- GLOBAL_STEP: 10900 + | > loss_text_ce: 0.11856712400913239 (0.10047327847603489) + | > loss_mel_ce: 3.5161757469177246 (2.9726680517196655) + | > loss: 0.04327075183391571 (0.036585016340455585) + | > current_lr: 5e-06 + | > step_time: 0.1864 (0.15135780152152567) + | > loader_time: 0.0089 (0.009044503464418296) + + + --> TIME: 2024-04-09 15:47:41 -- STEP: 118/1354 -- GLOBAL_STEP: 10950 + | > loss_text_ce: 0.0873958021402359 (0.10364173744189539) + | > loss_mel_ce: 2.7022957801818848 (2.9581693431078375) + | > loss: 0.03321061655879021 (0.03645013258555681) + | > current_lr: 5e-06 + | > step_time: 0.1285 (0.1435242911516609) + | > loader_time: 0.0084 (0.008765170129679017) + + + --> TIME: 2024-04-09 15:47:51 -- STEP: 168/1354 -- GLOBAL_STEP: 11000 + | > loss_text_ce: 0.12147922068834305 (0.10508365241721981) + | > loss_mel_ce: 2.65152645111084 (2.9361470795813065) + | > loss: 0.03301197290420532 (0.03620512841180675) + | > current_lr: 5e-06 + | > step_time: 0.0748 (0.1404112251031966) + | > loader_time: 0.0071 (0.00863921358471825) + + + --> TIME: 2024-04-09 15:48:23 -- STEP: 218/1354 -- GLOBAL_STEP: 11050 + | > loss_text_ce: 0.11108418554067612 (0.10593070882722871) + | > loss_mel_ce: 2.6227948665618896 (2.9380428812919406) + | > loss: 0.03254618123173714 (0.03623778149143821) + | > current_lr: 5e-06 + | > step_time: 0.0979 (0.13996097582195882) + | > loader_time: 0.0088 (0.008687115590506736) + + + --> TIME: 2024-04-09 15:48:33 -- STEP: 268/1354 -- GLOBAL_STEP: 11100 + | > loss_text_ce: 0.08363410085439682 (0.10650444478352568) + | > loss_mel_ce: 2.8845937252044678 (2.954007764360798) + | > loss: 0.035336047410964966 (0.0364346698399133) + | > current_lr: 5e-06 + | > step_time: 0.1993 (0.14006700355615195) + | > loader_time: 0.0079 (0.00863823606007135) + + + --> TIME: 2024-04-09 15:48:43 -- STEP: 318/1354 -- GLOBAL_STEP: 11150 + | > loss_text_ce: 0.10278700292110443 (0.10673389710345353) + | > loss_mel_ce: 3.0142135620117188 (2.945869831918921) + | > loss: 0.03710715100169182 (0.03634052129122638) + | > current_lr: 5e-06 + | > step_time: 0.1354 (0.13834854866723595) + | > loader_time: 0.0109 (0.00865741045969837) + + + --> TIME: 2024-04-09 15:48:54 -- STEP: 368/1354 -- GLOBAL_STEP: 11200 + | > loss_text_ce: 0.10280545800924301 (0.10681412387234357) + | > loss_mel_ce: 2.6925299167633057 (2.952369281131289) + | > loss: 0.03327780216932297 (0.036418850751310294) + | > current_lr: 5e-06 + | > step_time: 0.1118 (0.1389531298823979) + | > loader_time: 0.0087 (0.008683792922807784) + + + --> TIME: 2024-04-09 15:49:06 -- STEP: 418/1354 -- GLOBAL_STEP: 11250 + | > loss_text_ce: 0.10284101963043213 (0.10599992702309591) + | > loss_mel_ce: 2.9945812225341797 (2.954578600431744) + | > loss: 0.03687407448887825 (0.03643545934386805) + | > current_lr: 5e-06 + | > step_time: 0.1583 (0.14107342313921634) + | > loader_time: 0.0094 (0.008740078319202779) + + + --> TIME: 2024-04-09 15:49:16 -- STEP: 468/1354 -- GLOBAL_STEP: 11300 + | > loss_text_ce: 0.1004859134554863 (0.10582046234645902) + | > loss_mel_ce: 3.058800220489502 (2.9592479380786934) + | > loss: 0.03761054947972298 (0.0364889102096423) + | > current_lr: 5e-06 + | > step_time: 0.164 (0.1411932869854137) + | > loader_time: 0.0096 (0.008716777858571117) + + + --> TIME: 2024-04-09 15:49:26 -- STEP: 518/1354 -- GLOBAL_STEP: 11350 + | > loss_text_ce: 0.10635283589363098 (0.10621984730183386) + | > loss_mel_ce: 3.0227198600769043 (2.95660919342262) + | > loss: 0.03725086525082588 (0.0364622511580932) + | > current_lr: 5e-06 + | > step_time: 0.112 (0.140263031808566) + | > loader_time: 0.0097 (0.008651060487312706) + + + --> TIME: 2024-04-09 15:49:37 -- STEP: 568/1354 -- GLOBAL_STEP: 11400 + | > loss_text_ce: 0.09996868669986725 (0.10622122033800874) + | > loss_mel_ce: 3.3469064235687256 (2.964678938539935) + | > loss: 0.04103422909975052 (0.03655833589621416) + | > current_lr: 5e-06 + | > step_time: 0.1748 (0.14019464816845645) + | > loader_time: 0.0083 (0.008620380515783614) + + + --> TIME: 2024-04-09 15:49:47 -- STEP: 618/1354 -- GLOBAL_STEP: 11450 + | > loss_text_ce: 0.13755163550376892 (0.1059444849183451) + | > loss_mel_ce: 2.666226863861084 (2.9679499096854975) + | > loss: 0.03337831422686577 (0.036593981581502) + | > current_lr: 5e-06 + | > step_time: 0.0796 (0.1401555337566389) + | > loader_time: 0.0067 (0.00855433477938755) + + + --> TIME: 2024-04-09 15:49:58 -- STEP: 668/1354 -- GLOBAL_STEP: 11500 + | > loss_text_ce: 0.10320166498422623 (0.10598771059941385) + | > loss_mel_ce: 2.6966958045959473 (2.9664550223750252) + | > loss: 0.033332113176584244 (0.03657669987540672) + | > current_lr: 5e-06 + | > step_time: 0.1122 (0.13994534179835996) + | > loader_time: 0.0076 (0.008515578544068491) + + + --> TIME: 2024-04-09 15:50:08 -- STEP: 718/1354 -- GLOBAL_STEP: 11550 + | > loss_text_ce: 0.12358416616916656 (0.10601891269051261) + | > loss_mel_ce: 2.8844094276428223 (2.963251967948126) + | > loss: 0.03580944985151291 (0.03653893975701841) + | > current_lr: 5e-06 + | > step_time: 0.1513 (0.1396805376727602) + | > loader_time: 0.0091 (0.008480245356440222) + + + --> TIME: 2024-04-09 15:50:19 -- STEP: 768/1354 -- GLOBAL_STEP: 11600 + | > loss_text_ce: 0.15236562490463257 (0.10569755620478338) + | > loss_mel_ce: 3.468536376953125 (2.963601590755084) + | > loss: 0.04310597851872444 (0.0365392762541888) + | > current_lr: 5e-06 + | > step_time: 0.0639 (0.14009567908942702) + | > loader_time: 0.0076 (0.00847259039680164) + + + --> TIME: 2024-04-09 15:50:30 -- STEP: 818/1354 -- GLOBAL_STEP: 11650 + | > loss_text_ce: 0.10234098881483078 (0.10529825640591838) + | > loss_mel_ce: 3.3658437728881836 (2.9646507437479808) + | > loss: 0.04128791391849518 (0.036547012583860464) + | > current_lr: 5e-06 + | > step_time: 0.1278 (0.14043241695553293) + | > loader_time: 0.0076 (0.008454925273624793) + + + --> TIME: 2024-04-09 15:50:40 -- STEP: 868/1354 -- GLOBAL_STEP: 11700 + | > loss_text_ce: 0.12334053963422775 (0.10526729980269058) + | > loss_mel_ce: 3.073753595352173 (2.961419093169374) + | > loss: 0.03806064650416374 (0.036508172031595044) + | > current_lr: 5e-06 + | > step_time: 0.1121 (0.1400751034784977) + | > loader_time: 0.0078 (0.008434031416194239) + + + --> TIME: 2024-04-09 15:50:51 -- STEP: 918/1354 -- GLOBAL_STEP: 11750 + | > loss_text_ce: 0.0860222578048706 (0.1050521520520347) + | > loss_mel_ce: 2.995797634124756 (2.9576905155493534) + | > loss: 0.03668833523988724 (0.03646122292292862) + | > current_lr: 5e-06 + | > step_time: 0.1548 (0.1398224293016921) + | > loader_time: 0.0089 (0.008430050349183597) + + + --> TIME: 2024-04-09 15:51:02 -- STEP: 968/1354 -- GLOBAL_STEP: 11800 + | > loss_text_ce: 0.10828904062509537 (0.10488947784943765) + | > loss_mel_ce: 3.474036455154419 (2.9596651595486088) + | > loss: 0.042646732181310654 (0.036482793983256795) + | > current_lr: 5e-06 + | > step_time: 0.1525 (0.1402563718724843) + | > loader_time: 0.0077 (0.008406922344333873) + + + --> TIME: 2024-04-09 15:51:12 -- STEP: 1018/1354 -- GLOBAL_STEP: 11850 + | > loss_text_ce: 0.0768800899386406 (0.10477775775906142) + | > loss_mel_ce: 2.4080374240875244 (2.957376314521069) + | > loss: 0.029582351446151733 (0.03645421582327203) + | > current_lr: 5e-06 + | > step_time: 0.1388 (0.14004696727033225) + | > loader_time: 0.0073 (0.008376791107162749) + + + --> TIME: 2024-04-09 15:51:23 -- STEP: 1068/1354 -- GLOBAL_STEP: 11900 + | > loss_text_ce: 0.13242433965206146 (0.10460404858467504) + | > loss_mel_ce: 2.8232059478759766 (2.9575319522329035) + | > loss: 0.035186074674129486 (0.036454000688168474) + | > current_lr: 5e-06 + | > step_time: 0.0907 (0.14030437165878248) + | > loader_time: 0.0086 (0.008367792050936698) + + + --> TIME: 2024-04-09 15:51:33 -- STEP: 1118/1354 -- GLOBAL_STEP: 11950 + | > loss_text_ce: 0.1471414864063263 (0.10440820814751763) + | > loss_mel_ce: 2.993089199066162 (2.957097739567697) + | > loss: 0.03738369792699814 (0.03644650004310986) + | > current_lr: 5e-06 + | > step_time: 0.079 (0.14023046118202284) + | > loader_time: 0.0068 (0.008346925054765137) + + + --> TIME: 2024-04-09 15:51:44 -- STEP: 1168/1354 -- GLOBAL_STEP: 12000 + | > loss_text_ce: 0.07272279262542725 (0.1041026294665181) + | > loss_mel_ce: 2.6804919242858887 (2.956448153069575) + | > loss: 0.0327763669192791 (0.0364351290236674) + | > current_lr: 5e-06 + | > step_time: 0.1862 (0.14026163089765267) + | > loader_time: 0.0077 (0.008324657605118947) + + + --> TIME: 2024-04-09 15:52:28 -- STEP: 1218/1354 -- GLOBAL_STEP: 12050 + | > loss_text_ce: 0.08765903115272522 (0.10413544584671254) + | > loss_mel_ce: 2.9396724700927734 (2.956355814080324) + | > loss: 0.03603966161608696 (0.036434420419639776) + | > current_lr: 5e-06 + | > step_time: 0.1754 (0.13962227977163888) + | > loader_time: 0.0084 (0.008350637550228726) + + + --> TIME: 2024-04-09 15:52:39 -- STEP: 1268/1354 -- GLOBAL_STEP: 12100 + | > loss_text_ce: 0.08068889379501343 (0.1038823806525028) + | > loss_mel_ce: 2.725476026535034 (2.953453164183378) + | > loss: 0.03340672701597214 (0.03639685238009371) + | > current_lr: 5e-06 + | > step_time: 0.1738 (0.13987113511900812) + | > loader_time: 0.0086 (0.008358404087343433) + + + --> TIME: 2024-04-09 15:52:50 -- STEP: 1318/1354 -- GLOBAL_STEP: 12150 + | > loss_text_ce: 0.09583240002393723 (0.10354143396381715) + | > loss_mel_ce: 3.236839771270752 (2.953348145680291) + | > loss: 0.03967466950416565 (0.036391543273917186) + | > current_lr: 5e-06 + | > step_time: 0.1784 (0.1401945804789869) + | > loader_time: 0.0079 (0.008363504872879245) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03132831133328951 (+0.004914448811457705) + | > avg_loss_text_ce: 0.11042956205514762 (-0.00914805382490158) + | > avg_loss_mel_ce: 3.036125843341534 (-0.028502574333777453) + | > avg_loss: 3.146555387056791 (-0.03765062185434198) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_12186.pth + + > EPOCH: 10/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 15:53:49)  + + --> TIME: 2024-04-09 15:53:53 -- STEP: 14/1354 -- GLOBAL_STEP: 12200 + | > loss_text_ce: 0.09141772985458374 (0.09981766875301089) + | > loss_mel_ce: 2.7699005603790283 (2.9803957087653026) + | > loss: 0.03406331315636635 (0.036669207869895866) + | > current_lr: 5e-06 + | > step_time: 0.151 (0.13814796720232284) + | > loader_time: 0.0072 (0.010017667497907366) + + + --> TIME: 2024-04-09 15:54:04 -- STEP: 64/1354 -- GLOBAL_STEP: 12250 + | > loss_text_ce: 0.10460957139730453 (0.09830513934139162) + | > loss_mel_ce: 2.9257094860076904 (2.8995131254196167) + | > loss: 0.036075226962566376 (0.03568831353913994) + | > current_lr: 5e-06 + | > step_time: 0.139 (0.1393335498869419) + | > loader_time: 0.0102 (0.009009897708892822) + + + --> TIME: 2024-04-09 15:54:14 -- STEP: 114/1354 -- GLOBAL_STEP: 12300 + | > loss_text_ce: 0.12002619355916977 (0.09970494379338465) + | > loss_mel_ce: 2.8528809547424316 (2.912280270927831) + | > loss: 0.03539175167679787 (0.035856967644864006) + | > current_lr: 5e-06 + | > step_time: 0.091 (0.13689559593535303) + | > loader_time: 0.0101 (0.008933230450278834) + + + --> TIME: 2024-04-09 15:54:25 -- STEP: 164/1354 -- GLOBAL_STEP: 12350 + | > loss_text_ce: 0.11081711202859879 (0.09822823606976651) + | > loss_mel_ce: 3.4806153774261475 (2.9349509477615365) + | > loss: 0.04275514930486679 (0.036109276820065046) + | > current_lr: 5e-06 + | > step_time: 0.1483 (0.13643822146625054) + | > loader_time: 0.0081 (0.00881782682930551) + + + --> TIME: 2024-04-09 15:54:35 -- STEP: 214/1354 -- GLOBAL_STEP: 12400 + | > loss_text_ce: 0.12381802499294281 (0.09775191966757597) + | > loss_mel_ce: 2.835789442062378 (2.9226372442512867) + | > loss: 0.03523342311382294 (0.03595701467559157) + | > current_lr: 5e-06 + | > step_time: 0.0977 (0.13740597818499403) + | > loader_time: 0.0082 (0.008808368834379682) + + + --> TIME: 2024-04-09 15:54:46 -- STEP: 264/1354 -- GLOBAL_STEP: 12450 + | > loss_text_ce: 0.09951850771903992 (0.09711274764303006) + | > loss_mel_ce: 3.5501248836517334 (2.9281169112884635) + | > loss: 0.04344813525676727 (0.03601463958903243) + | > current_lr: 5e-06 + | > step_time: 0.207 (0.1394759028246908) + | > loader_time: 0.0083 (0.008770605831435236) + + + --> TIME: 2024-04-09 15:54:57 -- STEP: 314/1354 -- GLOBAL_STEP: 12500 + | > loss_text_ce: 0.12496932595968246 (0.0972039275773012) + | > loss_mel_ce: 2.5277645587921143 (2.923503076194957) + | > loss: 0.03158016502857208 (0.03596079843652665) + | > current_lr: 5e-06 + | > step_time: 0.0679 (0.1399483931292394) + | > loader_time: 0.0068 (0.008722254425097426) + + + --> TIME: 2024-04-09 15:55:08 -- STEP: 364/1354 -- GLOBAL_STEP: 12550 + | > loss_text_ce: 0.09072478860616684 (0.09744249904466855) + | > loss_mel_ce: 2.773820161819458 (2.9246981530399108) + | > loss: 0.03410172462463379 (0.035977865697222396) + | > current_lr: 5e-06 + | > step_time: 0.2105 (0.14048713773161495) + | > loader_time: 0.0077 (0.008695627961839947) + + + --> TIME: 2024-04-09 15:55:18 -- STEP: 414/1354 -- GLOBAL_STEP: 12600 + | > loss_text_ce: 0.12090383470058441 (0.09744543577701861) + | > loss_mel_ce: 2.9296257495880127 (2.921093337201841) + | > loss: 0.03631582856178284 (0.035934986141712734) + | > current_lr: 5e-06 + | > step_time: 0.1047 (0.13997964755348524) + | > loader_time: 0.0076 (0.008664385132167647) + + + --> TIME: 2024-04-09 15:55:29 -- STEP: 464/1354 -- GLOBAL_STEP: 12650 + | > loss_text_ce: 0.06703458726406097 (0.09698910483737187) + | > loss_mel_ce: 2.7583744525909424 (2.9203979701831417) + | > loss: 0.03363582119345665 (0.035921275447090624) + | > current_lr: 5e-06 + | > step_time: 0.1155 (0.14090417325496654) + | > loader_time: 0.0075 (0.00864793154700048) + + + --> TIME: 2024-04-09 15:55:39 -- STEP: 514/1354 -- GLOBAL_STEP: 12700 + | > loss_text_ce: 0.08228988200426102 (0.09692889916554731) + | > loss_mel_ce: 3.0478572845458984 (2.9095624304930983) + | > loss: 0.0372636578977108 (0.035791564178345024) + | > current_lr: 5e-06 + | > step_time: 0.2297 (0.13997718842576898) + | > loader_time: 0.0099 (0.008579339498675741) + + + --> TIME: 2024-04-09 15:55:50 -- STEP: 564/1354 -- GLOBAL_STEP: 12750 + | > loss_text_ce: 0.09115983545780182 (0.09720705738225092) + | > loss_mel_ce: 2.6570262908935547 (2.9138463570716526) + | > loss: 0.032716501504182816 (0.035845874683231305) + | > current_lr: 5e-06 + | > step_time: 0.1585 (0.1397113335048052) + | > loader_time: 0.0079 (0.008519344296015741) + + + --> TIME: 2024-04-09 15:56:01 -- STEP: 614/1354 -- GLOBAL_STEP: 12800 + | > loss_text_ce: 0.09452369809150696 (0.09681533030097762) + | > loss_mel_ce: 2.891450881958008 (2.9140979614630576) + | > loss: 0.03554731607437134 (0.03584420654000497) + | > current_lr: 5e-06 + | > step_time: 0.1165 (0.1401531781746432) + | > loader_time: 0.0075 (0.008474280857496226) + + + --> TIME: 2024-04-09 15:56:11 -- STEP: 664/1354 -- GLOBAL_STEP: 12850 + | > loss_text_ce: 0.10348011553287506 (0.09673644344782434) + | > loss_mel_ce: 2.895413398742676 (2.9121898707137057) + | > loss: 0.03570111468434334 (0.03582055203571732) + | > current_lr: 5e-06 + | > step_time: 0.1064 (0.13976335381887056) + | > loader_time: 0.0072 (0.00843494951006877) + + + --> TIME: 2024-04-09 15:56:22 -- STEP: 714/1354 -- GLOBAL_STEP: 12900 + | > loss_text_ce: 0.09311802685260773 (0.09645066822485096) + | > loss_mel_ce: 3.3163111209869385 (2.9131531625234794) + | > loss: 0.040588442236185074 (0.03582861773534058) + | > current_lr: 5e-06 + | > step_time: 0.1888 (0.14021409626434522) + | > loader_time: 0.0074 (0.008412740143741203) + + + --> TIME: 2024-04-09 15:56:33 -- STEP: 764/1354 -- GLOBAL_STEP: 12950 + | > loss_text_ce: 0.07677999138832092 (0.09616284158217345) + | > loss_mel_ce: 2.830279588699341 (2.9122050182981654) + | > loss: 0.03460785374045372 (0.03581390380420527) + | > current_lr: 5e-06 + | > step_time: 0.1741 (0.1409445660900699) + | > loader_time: 0.0101 (0.008425242613747469) + + + --> TIME: 2024-04-09 15:56:44 -- STEP: 814/1354 -- GLOBAL_STEP: 13000 + | > loss_text_ce: 0.09114708006381989 (0.09575036742909827) + | > loss_mel_ce: 2.9182193279266357 (2.9179718749236137) + | > loss: 0.03582579270005226 (0.03587764645045672) + | > current_lr: 5e-06 + | > step_time: 0.1189 (0.14166933927840664) + | > loader_time: 0.0094 (0.008405224697009925) + + + --> TIME: 2024-04-09 15:57:17 -- STEP: 864/1354 -- GLOBAL_STEP: 13050 + | > loss_text_ce: 0.11060551553964615 (0.09569471511834611) + | > loss_mel_ce: 3.4265904426574707 (2.9160890841373703) + | > loss: 0.042109474539756775 (0.035854569741921444) + | > current_lr: 5e-06 + | > step_time: 0.1097 (0.1418694815150017) + | > loader_time: 0.0088 (0.00846839171868783) + + + --> TIME: 2024-04-09 15:57:28 -- STEP: 914/1354 -- GLOBAL_STEP: 13100 + | > loss_text_ce: 0.1063561886548996 (0.0954359481579887) + | > loss_mel_ce: 2.8704042434692383 (2.9155310864521597) + | > loss: 0.03543762490153313 (0.03584484634533753) + | > current_lr: 5e-06 + | > step_time: 0.1059 (0.14228590468609129) + | > loader_time: 0.01 (0.008541780697177741) + + + --> TIME: 2024-04-09 15:57:39 -- STEP: 964/1354 -- GLOBAL_STEP: 13150 + | > loss_text_ce: 0.08394763618707657 (0.0952999516625993) + | > loss_mel_ce: 3.0250391960144043 (2.9160189628601083) + | > loss: 0.03701175004243851 (0.03584903538520776) + | > current_lr: 5e-06 + | > step_time: 0.206 (0.14228179791161658) + | > loader_time: 0.0087 (0.008575722637018224) + + + --> TIME: 2024-04-09 15:57:50 -- STEP: 1014/1354 -- GLOBAL_STEP: 13200 + | > loss_text_ce: 0.07742676138877869 (0.0951018933375794) + | > loss_mel_ce: 2.415369987487793 (2.9144448241773686) + | > loss: 0.029676150530576706 (0.03582793778576383) + | > current_lr: 5e-06 + | > step_time: 0.1583 (0.14252992188907027) + | > loader_time: 0.0078 (0.008558760498167256) + + + --> TIME: 2024-04-09 15:58:00 -- STEP: 1064/1354 -- GLOBAL_STEP: 13250 + | > loss_text_ce: 0.10720629245042801 (0.09491702253909262) + | > loss_mel_ce: 2.4882917404174805 (2.9132152309543224) + | > loss: 0.030898787081241608 (0.0358110989331919) + | > current_lr: 5e-06 + | > step_time: 0.0779 (0.14252202694577407) + | > loader_time: 0.0074 (0.008543430862570164) + + + --> TIME: 2024-04-09 15:58:11 -- STEP: 1114/1354 -- GLOBAL_STEP: 13300 + | > loss_text_ce: 0.100398488342762 (0.09492632849693727) + | > loss_mel_ce: 3.2608697414398193 (2.914709674177738) + | > loss: 0.040015097707509995 (0.03582900070670583) + | > current_lr: 5e-06 + | > step_time: 0.1961 (0.14231743054929308) + | > loader_time: 0.0087 (0.008525660050835595) + + + --> TIME: 2024-04-09 15:58:21 -- STEP: 1164/1354 -- GLOBAL_STEP: 13350 + | > loss_text_ce: 0.12448946386575699 (0.09476306936239565) + | > loss_mel_ce: 3.2643840312957764 (2.911302023122404) + | > loss: 0.04034373536705971 (0.035786489872869975) + | > current_lr: 5e-06 + | > step_time: 0.1132 (0.14221634119237025) + | > loader_time: 0.0089 (0.008519653192500474) + + + --> TIME: 2024-04-09 15:58:32 -- STEP: 1214/1354 -- GLOBAL_STEP: 13400 + | > loss_text_ce: 0.08607779443264008 (0.09450709323599765) + | > loss_mel_ce: 2.690063953399658 (2.910552581609575) + | > loss: 0.03304930403828621 (0.03577452060909208) + | > current_lr: 5e-06 + | > step_time: 0.1746 (0.14196217354675517) + | > loader_time: 0.0073 (0.008499621952188853) + + + --> TIME: 2024-04-09 15:58:42 -- STEP: 1264/1354 -- GLOBAL_STEP: 13450 + | > loss_text_ce: 0.09291369467973709 (0.09461498832554094) + | > loss_mel_ce: 2.547234535217285 (2.9082918289718767) + | > loss: 0.03143033757805824 (0.03574889134674607) + | > current_lr: 5e-06 + | > step_time: 0.0976 (0.14143478832667417) + | > loader_time: 0.0073 (0.008483048481277265) + + + --> TIME: 2024-04-09 15:58:52 -- STEP: 1314/1354 -- GLOBAL_STEP: 13500 + | > loss_text_ce: 0.06583373993635178 (0.09443563851705063) + | > loss_mel_ce: 2.746884822845459 (2.9066244748447976) + | > loss: 0.033484745770692825 (0.035726906782756154) + | > current_lr: 5e-06 + | > step_time: 0.172 (0.14141403122761137) + | > loader_time: 0.0073 (0.008469587773129998) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.02358546623816857 (-0.007742845095120941) + | > avg_loss_text_ce: 0.10246772662951396 (-0.007961835425633654) + | > avg_loss_mel_ce: 3.0147402103130636 (-0.02138563302847052) + | > avg_loss: 3.117207948978131 (-0.029347438078660026) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_13540.pth + + > EPOCH: 11/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 15:59:50)  + + --> TIME: 2024-04-09 15:59:53 -- STEP: 10/1354 -- GLOBAL_STEP: 13550 + | > loss_text_ce: 0.13038335740566254 (0.08812480121850967) + | > loss_mel_ce: 2.629363536834717 (2.8038535118103027) + | > loss: 0.03285412862896919 (0.03442831300199032) + | > current_lr: 5e-06 + | > step_time: 0.075 (0.1381300687789917) + | > loader_time: 0.0113 (0.01196749210357666) + + + --> TIME: 2024-04-09 16:00:04 -- STEP: 60/1354 -- GLOBAL_STEP: 13600 + | > loss_text_ce: 0.08520021289587021 (0.09065705413619676) + | > loss_mel_ce: 2.6999104022979736 (2.9081339836120605) + | > loss: 0.03315608203411102 (0.03569989362731577) + | > current_lr: 5e-06 + | > step_time: 0.1151 (0.14173992474873864) + | > loader_time: 0.0125 (0.010201092561086018) + + + --> TIME: 2024-04-09 16:00:14 -- STEP: 110/1354 -- GLOBAL_STEP: 13650 + | > loss_text_ce: 0.0937197357416153 (0.09147515205497088) + | > loss_mel_ce: 3.009881019592285 (2.8812989820133557) + | > loss: 0.03694763034582138 (0.03539016875015066) + | > current_lr: 5e-06 + | > step_time: 0.2097 (0.1400018475272438) + | > loader_time: 0.0086 (0.009609729593450375) + + + --> TIME: 2024-04-09 16:00:26 -- STEP: 160/1354 -- GLOBAL_STEP: 13700 + | > loss_text_ce: 0.1092926561832428 (0.08998023930471385) + | > loss_mel_ce: 3.0821001529693604 (2.8748504772782324) + | > loss: 0.037992771714925766 (0.035295604390557865) + | > current_lr: 5e-06 + | > step_time: 0.1356 (0.1432789757847786) + | > loader_time: 0.0086 (0.009453977644443518) + + + --> TIME: 2024-04-09 16:00:36 -- STEP: 210/1354 -- GLOBAL_STEP: 13750 + | > loss_text_ce: 0.10590679943561554 (0.09081734988306246) + | > loss_mel_ce: 2.22811222076416 (2.878678072066534) + | > loss: 0.02778594009578228 (0.03535113660175178) + | > current_lr: 5e-06 + | > step_time: 0.0784 (0.14271392935798285) + | > loader_time: 0.0075 (0.009244478316534136) + + + --> TIME: 2024-04-09 16:00:47 -- STEP: 260/1354 -- GLOBAL_STEP: 13800 + | > loss_text_ce: 0.06668253242969513 (0.09005580531576503) + | > loss_mel_ce: 2.896799325942993 (2.8845461753698496) + | > loss: 0.035279545933008194 (0.03541192895947741) + | > current_lr: 5e-06 + | > step_time: 0.1785 (0.14232441370303822) + | > loader_time: 0.0087 (0.009116304837740388) + + + --> TIME: 2024-04-09 16:00:58 -- STEP: 310/1354 -- GLOBAL_STEP: 13850 + | > loss_text_ce: 0.1160765141248703 (0.0898462114314879) + | > loss_mel_ce: 3.061285972595215 (2.8871382851754466) + | > loss: 0.03782574459910393 (0.0354402921853527) + | > current_lr: 5e-06 + | > step_time: 0.1326 (0.14321068179222848) + | > loader_time: 0.0095 (0.009019571735012916) + + + --> TIME: 2024-04-09 16:01:09 -- STEP: 360/1354 -- GLOBAL_STEP: 13900 + | > loss_text_ce: 0.1017039567232132 (0.08960386943072085) + | > loss_mel_ce: 2.99568510055542 (2.878648985756767) + | > loss: 0.03687367960810661 (0.035336344125163226) + | > current_lr: 5e-06 + | > step_time: 0.1505 (0.1427516096168095) + | > loader_time: 0.0094 (0.008977989355723066) + + + --> TIME: 2024-04-09 16:01:19 -- STEP: 410/1354 -- GLOBAL_STEP: 13950 + | > loss_text_ce: 0.11930177360773087 (0.08996047431194204) + | > loss_mel_ce: 3.5331902503967285 (2.8779462186301616) + | > loss: 0.04348205029964447 (0.03533222312789135) + | > current_lr: 5e-06 + | > step_time: 0.1395 (0.14194679667310017) + | > loader_time: 0.0077 (0.008886543715872415) + + + --> TIME: 2024-04-09 16:01:30 -- STEP: 460/1354 -- GLOBAL_STEP: 14000 + | > loss_text_ce: 0.114637590944767 (0.08998272487002873) + | > loss_mel_ce: 2.837416410446167 (2.873643148982006) + | > loss: 0.03514350205659866 (0.035281261008070865) + | > current_lr: 5e-06 + | > step_time: 0.0801 (0.1422546505928039) + | > loader_time: 0.0074 (0.008856137420820153) + + + --> TIME: 2024-04-09 16:02:04 -- STEP: 510/1354 -- GLOBAL_STEP: 14050 + | > loss_text_ce: 0.06417246907949448 (0.0891894898359098) + | > loss_mel_ce: 2.756819725036621 (2.873942119000004) + | > loss: 0.03358324244618416 (0.03527537689489477) + | > current_lr: 5e-06 + | > step_time: 0.1617 (0.14387152241725543) + | > loader_time: 0.0097 (0.008996923297059303) + + + --> TIME: 2024-04-09 16:02:15 -- STEP: 560/1354 -- GLOBAL_STEP: 14100 + | > loss_text_ce: 0.09975866973400116 (0.08910144816951028) + | > loss_mel_ce: 2.6584599018096924 (2.873416989190236) + | > loss: 0.032835934311151505 (0.03526807726427381) + | > current_lr: 5e-06 + | > step_time: 0.1145 (0.1437182869229998) + | > loader_time: 0.013 (0.009098173890795026) + + + --> TIME: 2024-04-09 16:02:26 -- STEP: 610/1354 -- GLOBAL_STEP: 14150 + | > loss_text_ce: 0.07312479615211487 (0.08871211397354718) + | > loss_mel_ce: 2.9903066158294678 (2.872412592074909) + | > loss: 0.03646942228078842 (0.03525148526322647) + | > current_lr: 5e-06 + | > step_time: 0.1531 (0.14428320595475494) + | > loader_time: 0.0099 (0.009204373203340126) + + + --> TIME: 2024-04-09 16:02:37 -- STEP: 660/1354 -- GLOBAL_STEP: 14200 + | > loss_text_ce: 0.07696526497602463 (0.08850509018609014) + | > loss_mel_ce: 2.9473516941070557 (2.869553810899908) + | > loss: 0.036003775894641876 (0.03521498758063624) + | > current_lr: 5e-06 + | > step_time: 0.1795 (0.14474263660835499) + | > loader_time: 0.0105 (0.009248324596520627) + + + --> TIME: 2024-04-09 16:02:48 -- STEP: 710/1354 -- GLOBAL_STEP: 14250 + | > loss_text_ce: 0.11395742744207382 (0.08860137670392716) + | > loss_mel_ce: 3.289813280105591 (2.872604793226215) + | > loss: 0.040521081537008286 (0.035252455065787676) + | > current_lr: 5e-06 + | > step_time: 0.1124 (0.14480407271586673) + | > loader_time: 0.01 (0.00933833760274967) + + + --> TIME: 2024-04-09 16:03:00 -- STEP: 760/1354 -- GLOBAL_STEP: 14300 + | > loss_text_ce: 0.10516109317541122 (0.08818960965758088) + | > loss_mel_ce: 3.4897546768188477 (2.868782744595878) + | > loss: 0.04279661923646927 (0.03520205250794166) + | > current_lr: 5e-06 + | > step_time: 0.1189 (0.14504252138890725) + | > loader_time: 0.0118 (0.009392760615599785) + + + --> TIME: 2024-04-09 16:03:11 -- STEP: 810/1354 -- GLOBAL_STEP: 14350 + | > loss_text_ce: 0.08077313005924225 (0.08775810375809666) + | > loss_mel_ce: 2.3994033336639404 (2.8658804069330652) + | > loss: 0.029525911435484886 (0.035162363884349644) + | > current_lr: 5e-06 + | > step_time: 0.114 (0.14549757286354356) + | > loader_time: 0.0074 (0.009455041532163267) + + + --> TIME: 2024-04-09 16:03:22 -- STEP: 860/1354 -- GLOBAL_STEP: 14400 + | > loss_text_ce: 0.07296469807624817 (0.08767836968912632) + | > loss_mel_ce: 2.960981607437134 (2.8659949873768995) + | > loss: 0.036118410527706146 (0.03516277871419523) + | > current_lr: 5e-06 + | > step_time: 0.1996 (0.14528015796528324) + | > loader_time: 0.0096 (0.009530993395073474) + + + --> TIME: 2024-04-09 16:03:33 -- STEP: 910/1354 -- GLOBAL_STEP: 14450 + | > loss_text_ce: 0.0691862404346466 (0.08758410918679863) + | > loss_mel_ce: 2.2887463569641113 (2.863437635033994) + | > loss: 0.02807062678039074 (0.035131211877687925) + | > current_lr: 5e-06 + | > step_time: 0.1661 (0.145117894109789) + | > loader_time: 0.0102 (0.009566076771243591) + + + --> TIME: 2024-04-09 16:03:43 -- STEP: 960/1354 -- GLOBAL_STEP: 14500 + | > loss_text_ce: 0.09008218348026276 (0.08743006171425802) + | > loss_mel_ce: 2.367980480194092 (2.861186116685469) + | > loss: 0.02926265075802803 (0.03510257419159944) + | > current_lr: 5e-06 + | > step_time: 0.1091 (0.14480107302467046) + | > loader_time: 0.0097 (0.009579319258530939) + + + --> TIME: 2024-04-09 16:03:55 -- STEP: 1010/1354 -- GLOBAL_STEP: 14550 + | > loss_text_ce: 0.057960230857133865 (0.08728284760203099) + | > loss_mel_ce: 2.6932637691497803 (2.861741587195065) + | > loss: 0.032752666622400284 (0.03510743438054132) + | > current_lr: 5e-06 + | > step_time: 0.1819 (0.14509254021219706) + | > loader_time: 0.0115 (0.009624081082863386) + + + --> TIME: 2024-04-09 16:04:06 -- STEP: 1060/1354 -- GLOBAL_STEP: 14600 + | > loss_text_ce: 0.06952103227376938 (0.08741708703971694) + | > loss_mel_ce: 2.917314052581787 (2.862778231332886) + | > loss: 0.03555756062269211 (0.03512137348254051) + | > current_lr: 5e-06 + | > step_time: 0.1866 (0.14505292874462217) + | > loader_time: 0.0077 (0.009616607540058643) + + + --> TIME: 2024-04-09 16:04:16 -- STEP: 1110/1354 -- GLOBAL_STEP: 14650 + | > loss_text_ce: 0.07913616299629211 (0.08724771540377053) + | > loss_mel_ce: 2.496279239654541 (2.8582185937477655) + | > loss: 0.030659707263112068 (0.035065075750085106) + | > current_lr: 5e-06 + | > step_time: 0.1499 (0.14487954324430197) + | > loader_time: 0.0079 (0.009588826024854509) + + + --> TIME: 2024-04-09 16:04:27 -- STEP: 1160/1354 -- GLOBAL_STEP: 14700 + | > loss_text_ce: 0.07003699988126755 (0.08709312629211569) + | > loss_mel_ce: 3.176328659057617 (2.8615746959530064) + | > loss: 0.038647208362817764 (0.035103188999446776) + | > current_lr: 5e-06 + | > step_time: 0.2097 (0.14499845319780819) + | > loader_time: 0.0093 (0.00955981941058718) + + + --> TIME: 2024-04-09 16:04:38 -- STEP: 1210/1354 -- GLOBAL_STEP: 14750 + | > loss_text_ce: 0.04927647486329079 (0.08706366435185936) + | > loss_mel_ce: 2.7868123054504395 (2.861489749053292) + | > loss: 0.033762965351343155 (0.03510182699542644) + | > current_lr: 5e-06 + | > step_time: 0.2256 (0.1447309499929759) + | > loader_time: 0.0107 (0.00959585895223066) + + + --> TIME: 2024-04-09 16:04:49 -- STEP: 1260/1354 -- GLOBAL_STEP: 14800 + | > loss_text_ce: 0.0961637794971466 (0.08693749094529757) + | > loss_mel_ce: 3.556992530822754 (2.860963211835375) + | > loss: 0.043489955365657806 (0.035094056625865254) + | > current_lr: 5e-06 + | > step_time: 0.128 (0.14475782144637328) + | > loader_time: 0.0109 (0.009622023976038377) + + + --> TIME: 2024-04-09 16:05:00 -- STEP: 1310/1354 -- GLOBAL_STEP: 14850 + | > loss_text_ce: 0.09188026934862137 (0.08692879494251184) + | > loss_mel_ce: 3.481241226196289 (2.861786336116208) + | > loss: 0.04253716021776199 (0.035103752198166964) + | > current_lr: 5e-06 + | > step_time: 0.1633 (0.1446251346864772) + | > loader_time: 0.0101 (0.009642478891911398) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.02727337983938364 (+0.0036879136012150683) + | > avg_loss_text_ce: 0.09553876461891028 (-0.0069289620106036814) + | > avg_loss_mel_ce: 2.9934546030484714 (-0.021285607264592166) + | > avg_loss: 3.0889933659480167 (-0.02821458303011415) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_14894.pth + + > EPOCH: 12/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 16:05:59)  + + --> TIME: 2024-04-09 16:06:01 -- STEP: 6/1354 -- GLOBAL_STEP: 14900 + | > loss_text_ce: 0.10116943717002869 (0.08244042098522186) + | > loss_mel_ce: 3.3310441970825195 (2.94068972269694) + | > loss: 0.040859684348106384 (0.035989644937217236) + | > current_lr: 5e-06 + | > step_time: 0.1296 (0.13362014293670654) + | > loader_time: 0.0102 (0.01176603635152181) + + + --> TIME: 2024-04-09 16:06:12 -- STEP: 56/1354 -- GLOBAL_STEP: 14950 + | > loss_text_ce: 0.06598459184169769 (0.0866929441690445) + | > loss_mel_ce: 2.537519693374634 (2.90517064503261) + | > loss: 0.030994098633527756 (0.03561742434145085) + | > current_lr: 5e-06 + | > step_time: 0.1742 (0.14006779023579188) + | > loader_time: 0.0111 (0.010354770081383842) + + + --> TIME: 2024-04-09 16:06:23 -- STEP: 106/1354 -- GLOBAL_STEP: 15000 + | > loss_text_ce: 0.07665649801492691 (0.08445187060619304) + | > loss_mel_ce: 2.5541915893554688 (2.853434225298323) + | > loss: 0.031319621950387955 (0.034974835127732654) + | > current_lr: 5e-06 + | > step_time: 0.1986 (0.14090888230305806) + | > loader_time: 0.0087 (0.010113997279473073) + + + --> TIME: 2024-04-09 16:06:55 -- STEP: 156/1354 -- GLOBAL_STEP: 15050 + | > loss_text_ce: 0.08305645734071732 (0.08375214381764337) + | > loss_mel_ce: 2.860825777053833 (2.855924832515226) + | > loss: 0.03504621610045433 (0.03499615510018208) + | > current_lr: 5e-06 + | > step_time: 0.1634 (0.14235559946451437) + | > loader_time: 0.0097 (0.010348639427087244) + + + --> TIME: 2024-04-09 16:07:06 -- STEP: 206/1354 -- GLOBAL_STEP: 15100 + | > loss_text_ce: 0.06639671325683594 (0.08329089002195497) + | > loss_mel_ce: 2.795585870742798 (2.85624191136036) + | > loss: 0.03407122194766998 (0.03499443876982024) + | > current_lr: 5e-06 + | > step_time: 0.2072 (0.14336897794482778) + | > loader_time: 0.0097 (0.010392013105373937) + + + --> TIME: 2024-04-09 16:07:17 -- STEP: 256/1354 -- GLOBAL_STEP: 15150 + | > loss_text_ce: 0.08488033711910248 (0.08299459454428874) + | > loss_mel_ce: 3.4139347076416016 (2.8729447405785313) + | > loss: 0.041652560234069824 (0.03518975466431583) + | > current_lr: 5e-06 + | > step_time: 0.2112 (0.1441665263846516) + | > loader_time: 0.0115 (0.010520175099372864) + + + --> TIME: 2024-04-09 16:07:29 -- STEP: 306/1354 -- GLOBAL_STEP: 15200 + | > loss_text_ce: 0.07821942865848541 (0.08224912512487449) + | > loss_mel_ce: 2.6963517665863037 (2.854439540626176) + | > loss: 0.03303061053156853 (0.0349605800218734) + | > current_lr: 5e-06 + | > step_time: 0.1927 (0.14535243682611998) + | > loader_time: 0.012 (0.010484246646656712) + + + --> TIME: 2024-04-09 16:07:40 -- STEP: 356/1354 -- GLOBAL_STEP: 15250 + | > loss_text_ce: 0.07759764045476913 (0.08222067839560215) + | > loss_mel_ce: 2.7701594829559326 (2.846967758087628) + | > loss: 0.033901870250701904 (0.03487129154709282) + | > current_lr: 5e-06 + | > step_time: 0.161 (0.14513888452829934) + | > loader_time: 0.0116 (0.010555030254835505) + + + --> TIME: 2024-04-09 16:07:51 -- STEP: 406/1354 -- GLOBAL_STEP: 15300 + | > loss_text_ce: 0.10496082156896591 (0.08196319056115127) + | > loss_mel_ce: 2.653003215789795 (2.8395727391313437) + | > loss: 0.032832905650138855 (0.03478019026663208) + | > current_lr: 5e-06 + | > step_time: 0.1037 (0.1455072364196401) + | > loader_time: 0.0097 (0.010479217092391902) + + + --> TIME: 2024-04-09 16:08:02 -- STEP: 456/1354 -- GLOBAL_STEP: 15350 + | > loss_text_ce: 0.0666385367512703 (0.08203251791399038) + | > loss_mel_ce: 2.636721611022949 (2.84216385870649) + | > loss: 0.032182857394218445 (0.03481186227872966) + | > current_lr: 5e-06 + | > step_time: 0.1867 (0.1456665924766607) + | > loader_time: 0.0111 (0.010415689464201015) + + + --> TIME: 2024-04-09 16:08:13 -- STEP: 506/1354 -- GLOBAL_STEP: 15400 + | > loss_text_ce: 0.09457636624574661 (0.08223601178129906) + | > loss_mel_ce: 3.3649346828460693 (2.849485904802917) + | > loss: 0.04118465632200241 (0.034901452059360585) + | > current_lr: 5e-06 + | > step_time: 0.1233 (0.14534851873345048) + | > loader_time: 0.0104 (0.010401624935888969) + + + --> TIME: 2024-04-09 16:08:24 -- STEP: 556/1354 -- GLOBAL_STEP: 15450 + | > loss_text_ce: 0.1041955053806305 (0.08200181997213649) + | > loss_mel_ce: 3.4720869064331055 (2.850615311440802) + | > loss: 0.042574793100357056 (0.03491210938505023) + | > current_lr: 5e-06 + | > step_time: 0.1454 (0.14536619186401362) + | > loader_time: 0.0091 (0.010381123144849604) + + + --> TIME: 2024-04-09 16:08:35 -- STEP: 606/1354 -- GLOBAL_STEP: 15500 + | > loss_text_ce: 0.0866142138838768 (0.08187872699458805) + | > loss_mel_ce: 3.1247715950012207 (2.8467972613010444) + | > loss: 0.038230784237384796 (0.03486519102498939) + | > current_lr: 5e-06 + | > step_time: 0.1579 (0.14505005354928488) + | > loader_time: 0.0097 (0.010302109686848358) + + + --> TIME: 2024-04-09 16:08:45 -- STEP: 656/1354 -- GLOBAL_STEP: 15550 + | > loss_text_ce: 0.06724443286657333 (0.08192315751618547) + | > loss_mel_ce: 2.6355929374694824 (2.8423910417207825) + | > loss: 0.03217663615942001 (0.03481326496978178) + | > current_lr: 5e-06 + | > step_time: 0.1738 (0.14468099576670937) + | > loader_time: 0.01 (0.010294834288155166) + + + --> TIME: 2024-04-09 16:08:56 -- STEP: 706/1354 -- GLOBAL_STEP: 15600 + | > loss_text_ce: 0.05359979346394539 (0.08192769960413235) + | > loss_mel_ce: 2.6912336349487305 (2.8426017963852472) + | > loss: 0.032676588743925095 (0.03481582799404016) + | > current_lr: 5e-06 + | > step_time: 0.1417 (0.14433517537103488) + | > loader_time: 0.0084 (0.010252012071798612) + + + --> TIME: 2024-04-09 16:09:07 -- STEP: 756/1354 -- GLOBAL_STEP: 15650 + | > loss_text_ce: 0.08238884061574936 (0.08183933329822683) + | > loss_mel_ce: 2.6937670707702637 (2.8425860244130314) + | > loss: 0.03304947540163994 (0.034814588242658866) + | > current_lr: 5e-06 + | > step_time: 0.0746 (0.14473453148332205) + | > loader_time: 0.0084 (0.01017243647701525) + + + --> TIME: 2024-04-09 16:09:18 -- STEP: 806/1354 -- GLOBAL_STEP: 15700 + | > loss_text_ce: 0.09059624373912811 (0.08167840068970363) + | > loss_mel_ce: 2.8605635166168213 (2.8381996281981157) + | > loss: 0.03513285517692566 (0.034760453354650486) + | > current_lr: 5e-06 + | > step_time: 0.1521 (0.14448088096921524) + | > loader_time: 0.01 (0.010062540139513033) + + + --> TIME: 2024-04-09 16:09:28 -- STEP: 856/1354 -- GLOBAL_STEP: 15750 + | > loss_text_ce: 0.10853945463895798 (0.08189401520137611) + | > loss_mel_ce: 2.911738634109497 (2.8416297792274245) + | > loss: 0.03595568984746933 (0.03480385532428589) + | > current_lr: 5e-06 + | > step_time: 0.0765 (0.1436216129320802) + | > loader_time: 0.0077 (0.010005926417413153) + + + --> TIME: 2024-04-09 16:09:38 -- STEP: 906/1354 -- GLOBAL_STEP: 15800 + | > loss_text_ce: 0.08833106607198715 (0.08173979715203604) + | > loss_mel_ce: 3.0707738399505615 (2.8369498618654316) + | > loss: 0.03760839253664017 (0.0347463060992772) + | > current_lr: 5e-06 + | > step_time: 0.1208 (0.14301707202498726) + | > loader_time: 0.0081 (0.009933137735783653) + + + --> TIME: 2024-04-09 16:09:49 -- STEP: 956/1354 -- GLOBAL_STEP: 15850 + | > loss_text_ce: 0.08676517009735107 (0.08163521007856811) + | > loss_mel_ce: 3.026406764984131 (2.8380433415768023) + | > loss: 0.03706157207489014 (0.03475807863969559) + | > current_lr: 5e-06 + | > step_time: 0.1657 (0.1431760633340937) + | > loader_time: 0.0081 (0.009862640412781521) + + + --> TIME: 2024-04-09 16:10:00 -- STEP: 1006/1354 -- GLOBAL_STEP: 15900 + | > loss_text_ce: 0.09477713704109192 (0.08141267048213517) + | > loss_mel_ce: 2.4440791606903076 (2.839175954016728) + | > loss: 0.030224479734897614 (0.03476891283753142) + | > current_lr: 5e-06 + | > step_time: 0.0972 (0.14350701089407744) + | > loader_time: 0.0077 (0.009798255639806177) + + + --> TIME: 2024-04-09 16:10:11 -- STEP: 1056/1354 -- GLOBAL_STEP: 15950 + | > loss_text_ce: 0.08493169397115707 (0.08134519755064196) + | > loss_mel_ce: 2.428905487060547 (2.8367813737554974) + | > loss: 0.029926633462309837 (0.034739602671117725) + | > current_lr: 5e-06 + | > step_time: 0.0793 (0.1433433438792371) + | > loader_time: 0.0078 (0.00974171992504235) + + + --> TIME: 2024-04-09 16:10:21 -- STEP: 1106/1354 -- GLOBAL_STEP: 16000 + | > loss_text_ce: 0.05810185894370079 (0.08105498457674414) + | > loss_mel_ce: 2.1356592178344727 (2.8315090038030526) + | > loss: 0.026116203516721725 (0.03467338144186395) + | > current_lr: 5e-06 + | > step_time: 0.1181 (0.14307212937373984) + | > loader_time: 0.0092 (0.00968948475491073) + + + --> TIME: 2024-04-09 16:11:05 -- STEP: 1156/1354 -- GLOBAL_STEP: 16050 + | > loss_text_ce: 0.06356924027204514 (0.08093541763258848) + | > loss_mel_ce: 2.260960102081299 (2.8298037431644314) + | > loss: 0.027672968804836273 (0.03465165729895491) + | > current_lr: 5e-06 + | > step_time: 0.1391 (0.1432674403421604) + | > loader_time: 0.0096 (0.009725618114933415) + + + --> TIME: 2024-04-09 16:11:15 -- STEP: 1206/1354 -- GLOBAL_STEP: 16100 + | > loss_text_ce: 0.0947369933128357 (0.08080950845805174) + | > loss_mel_ce: 2.9801626205444336 (2.8279607808016607) + | > loss: 0.03660595044493675 (0.034628218359577995) + | > current_lr: 5e-06 + | > step_time: 0.0884 (0.14309904211591512) + | > loader_time: 0.0077 (0.009710899435267904) + + + --> TIME: 2024-04-09 16:11:26 -- STEP: 1256/1354 -- GLOBAL_STEP: 16150 + | > loss_text_ce: 0.04314717650413513 (0.08070184595016822) + | > loss_mel_ce: 2.5489892959594727 (2.827411150286909) + | > loss: 0.03085876628756523 (0.03462039343983078) + | > current_lr: 5e-06 + | > step_time: 0.2083 (0.14298396372491373) + | > loader_time: 0.0085 (0.009699489090852666) + + + --> TIME: 2024-04-09 16:11:37 -- STEP: 1306/1354 -- GLOBAL_STEP: 16200 + | > loss_text_ce: 0.07836326956748962 (0.08069991705451408) + | > loss_mel_ce: 2.4870219230651855 (2.8272453303724103) + | > loss: 0.03054029867053032 (0.03461839642894853) + | > current_lr: 5e-06 + | > step_time: 0.1075 (0.14303611485917916) + | > loader_time: 0.0085 (0.009665424389276889) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.024924681736872747 (-0.002348698102510892) + | > avg_loss_text_ce: 0.089288901251096 (-0.006249863367814287) + | > avg_loss_mel_ce: 2.976671915787917 (-0.01678268726055432) + | > avg_loss: 3.0659608290745664 (-0.023032536873450304) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_16248.pth + + > EPOCH: 13/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 16:12:35)  + + --> TIME: 2024-04-09 16:12:37 -- STEP: 2/1354 -- GLOBAL_STEP: 16250 + | > loss_text_ce: 0.07728711515665054 (0.06974997371435165) + | > loss_mel_ce: 2.5254178047180176 (2.590299367904663) + | > loss: 0.030984584242105484 (0.031667256727814674) + | > current_lr: 5e-06 + | > step_time: 0.1413 (0.1484682559967041) + | > loader_time: 0.0097 (0.013599872589111328) + + + --> TIME: 2024-04-09 16:12:47 -- STEP: 52/1354 -- GLOBAL_STEP: 16300 + | > loss_text_ce: 0.08240846544504166 (0.07573412552189368) + | > loss_mel_ce: 3.0173449516296387 (2.8092907621310306) + | > loss: 0.03690182790160179 (0.0343455351316012) + | > current_lr: 5e-06 + | > step_time: 0.0803 (0.14103917892162612) + | > loader_time: 0.0085 (0.009653806686401367) + + + --> TIME: 2024-04-09 16:12:58 -- STEP: 102/1354 -- GLOBAL_STEP: 16350 + | > loss_text_ce: 0.06708170473575592 (0.07561669319721996) + | > loss_mel_ce: 2.2864184379577637 (2.82314565368727) + | > loss: 0.028017859905958176 (0.03450907607946327) + | > current_lr: 5e-06 + | > step_time: 0.0991 (0.14466611778034888) + | > loader_time: 0.0089 (0.009449049538257074) + + + --> TIME: 2024-04-09 16:13:09 -- STEP: 152/1354 -- GLOBAL_STEP: 16400 + | > loss_text_ce: 0.06924685835838318 (0.07579741771578001) + | > loss_mel_ce: 2.6435577869415283 (2.803803280780189) + | > loss: 0.03229529410600662 (0.03428096123235792) + | > current_lr: 5e-06 + | > step_time: 0.1894 (0.14371508516763393) + | > loader_time: 0.0088 (0.009331913370835151) + + + --> TIME: 2024-04-09 16:13:20 -- STEP: 202/1354 -- GLOBAL_STEP: 16450 + | > loss_text_ce: 0.08940731734037399 (0.07590833505486498) + | > loss_mel_ce: 3.153956651687622 (2.8073209524154663) + | > loss: 0.03861147537827492 (0.034324158766851) + | > current_lr: 5e-06 + | > step_time: 0.0779 (0.14287765545420128) + | > loader_time: 0.0077 (0.009289348479544757) + + + --> TIME: 2024-04-09 16:13:30 -- STEP: 252/1354 -- GLOBAL_STEP: 16500 + | > loss_text_ce: 0.07265733927488327 (0.07604011155605787) + | > loss_mel_ce: 3.2902910709381104 (2.8145729123599947) + | > loss: 0.040035102516412735 (0.0344120603764341) + | > current_lr: 5e-06 + | > step_time: 0.1531 (0.14246682609830583) + | > loader_time: 0.0095 (0.009228859628949839) + + + --> TIME: 2024-04-09 16:13:41 -- STEP: 302/1354 -- GLOBAL_STEP: 16550 + | > loss_text_ce: 0.08234371244907379 (0.07627492156249796) + | > loss_mel_ce: 2.845034599304199 (2.8116764524914566) + | > loss: 0.03484974429011345 (0.03438037412475474) + | > current_lr: 5e-06 + | > step_time: 0.113 (0.14268511020584615) + | > loader_time: 0.01 (0.009141923576001292) + + + --> TIME: 2024-04-09 16:13:52 -- STEP: 352/1354 -- GLOBAL_STEP: 16600 + | > loss_text_ce: 0.08415623754262924 (0.07689658569341352) + | > loss_mel_ce: 2.597578525543213 (2.8185987607999277) + | > loss: 0.03192541375756264 (0.03447018329858441) + | > current_lr: 5e-06 + | > step_time: 0.1102 (0.14264865896918555) + | > loader_time: 0.008 (0.00909462435679002) + + + --> TIME: 2024-04-09 16:14:03 -- STEP: 402/1354 -- GLOBAL_STEP: 16650 + | > loss_text_ce: 0.06426676362752914 (0.07620053179562093) + | > loss_mel_ce: 2.4333159923553467 (2.807099991176852) + | > loss: 0.02973312698304653 (0.03432500687200774) + | > current_lr: 5e-06 + | > step_time: 0.1408 (0.14369338068796048) + | > loader_time: 0.0081 (0.009063379681525534) + + + --> TIME: 2024-04-09 16:14:14 -- STEP: 452/1354 -- GLOBAL_STEP: 16700 + | > loss_text_ce: 0.05289856344461441 (0.07583243863871403) + | > loss_mel_ce: 2.6214632987976074 (2.8017655357850337) + | > loss: 0.031837642192840576 (0.03425711937726723) + | > current_lr: 5e-06 + | > step_time: 0.1931 (0.14408133198729664) + | > loader_time: 0.0096 (0.009053364791701317) + + + --> TIME: 2024-04-09 16:14:25 -- STEP: 502/1354 -- GLOBAL_STEP: 16750 + | > loss_text_ce: 0.06042071804404259 (0.0761338944038547) + | > loss_mel_ce: 2.4021806716918945 (2.8038093587791777) + | > loss: 0.02931668423116207 (0.0342850393886525) + | > current_lr: 5e-06 + | > step_time: 0.2164 (0.1438242159041751) + | > loader_time: 0.0087 (0.009090312923568171) + + + --> TIME: 2024-04-09 16:14:35 -- STEP: 552/1354 -- GLOBAL_STEP: 16800 + | > loss_text_ce: 0.05363117903470993 (0.07614131274538627) + | > loss_mel_ce: 2.604334831237793 (2.8012284822222115) + | > loss: 0.031642451882362366 (0.03425440295815353) + | > current_lr: 5e-06 + | > step_time: 0.1924 (0.14327079878337148) + | > loader_time: 0.0095 (0.0090669540391452) + + + --> TIME: 2024-04-09 16:14:46 -- STEP: 602/1354 -- GLOBAL_STEP: 16850 + | > loss_text_ce: 0.06414534896612167 (0.07627364899034915) + | > loss_mel_ce: 2.684521198272705 (2.8049677527228094) + | > loss: 0.032722219824790955 (0.03430049352199328) + | > current_lr: 5e-06 + | > step_time: 0.1974 (0.14304513828302942) + | > loader_time: 0.0078 (0.009065538545779604) + + + --> TIME: 2024-04-09 16:14:57 -- STEP: 652/1354 -- GLOBAL_STEP: 16900 + | > loss_text_ce: 0.07245364785194397 (0.07627322503645548) + | > loss_mel_ce: 3.1481399536132812 (2.805732772759866) + | > loss: 0.03834040090441704 (0.03430959584899927) + | > current_lr: 5e-06 + | > step_time: 0.1641 (0.14275772695892427) + | > loader_time: 0.0085 (0.009033502976587213) + + + --> TIME: 2024-04-09 16:15:08 -- STEP: 702/1354 -- GLOBAL_STEP: 16950 + | > loss_text_ce: 0.04646485298871994 (0.07606764038509964) + | > loss_mel_ce: 2.648942708969116 (2.8048932898757806) + | > loss: 0.03208818659186363 (0.034297154553001016) + | > current_lr: 5e-06 + | > step_time: 0.1905 (0.1437375783241034) + | > loader_time: 0.0091 (0.009008202457699677) + + + --> TIME: 2024-04-09 16:15:19 -- STEP: 752/1354 -- GLOBAL_STEP: 17000 + | > loss_text_ce: 0.099216029047966 (0.07615591996309772) + | > loss_mel_ce: 2.8531408309936523 (2.8004605044075803) + | > loss: 0.035147104412317276 (0.03424543423449338) + | > current_lr: 5e-06 + | > step_time: 0.1056 (0.1434272048955269) + | > loader_time: 0.0105 (0.008983353351024865) + + + --> TIME: 2024-04-09 16:15:54 -- STEP: 802/1354 -- GLOBAL_STEP: 17050 + | > loss_text_ce: 0.0702020451426506 (0.07631897085585505) + | > loss_mel_ce: 3.907479763031006 (2.8000696918316335) + | > loss: 0.0473533570766449 (0.03424272279573269) + | > current_lr: 5e-06 + | > step_time: 0.131 (0.14289020243428313) + | > loader_time: 0.0095 (0.009066440518062907) + + + --> TIME: 2024-04-09 16:16:05 -- STEP: 852/1354 -- GLOBAL_STEP: 17100 + | > loss_text_ce: 0.05063287541270256 (0.075942819736933) + | > loss_mel_ce: 2.7520804405212402 (2.7971736027601195) + | > loss: 0.0333656370639801 (0.03420376756879991) + | > current_lr: 5e-06 + | > step_time: 0.1614 (0.14327235093139162) + | > loader_time: 0.0119 (0.00910146079712631) + + + --> TIME: 2024-04-09 16:16:16 -- STEP: 902/1354 -- GLOBAL_STEP: 17150 + | > loss_text_ce: 0.06847784668207169 (0.07586486127946176) + | > loss_mel_ce: 2.4843454360961914 (2.795487016637681) + | > loss: 0.030390754342079163 (0.03418276107925793) + | > current_lr: 5e-06 + | > step_time: 0.1013 (0.14328980789480625) + | > loader_time: 0.0097 (0.009090710373516888) + + + --> TIME: 2024-04-09 16:16:26 -- STEP: 952/1354 -- GLOBAL_STEP: 17200 + | > loss_text_ce: 0.10222139209508896 (0.07585508720714511) + | > loss_mel_ce: 2.966402769088745 (2.7939261091606973) + | > loss: 0.0365312434732914 (0.03416406249768338) + | > current_lr: 5e-06 + | > step_time: 0.0695 (0.14295269912030528) + | > loader_time: 0.0087 (0.009084554529991472) + + + --> TIME: 2024-04-09 16:16:37 -- STEP: 1002/1354 -- GLOBAL_STEP: 17250 + | > loss_text_ce: 0.07958744466304779 (0.07580536466366998) + | > loss_mel_ce: 2.6344096660614014 (2.7931688737250626) + | > loss: 0.03230949118733406 (0.03415445586157114) + | > current_lr: 5e-06 + | > step_time: 0.1107 (0.14304138372044375) + | > loader_time: 0.0079 (0.009097742225357636) + + + --> TIME: 2024-04-09 16:16:48 -- STEP: 1052/1354 -- GLOBAL_STEP: 17300 + | > loss_text_ce: 0.09596797078847885 (0.07566000542075807) + | > loss_mel_ce: 3.0424270629882812 (2.7913427181796897) + | > loss: 0.03736184537410736 (0.03413098544491443) + | > current_lr: 5e-06 + | > step_time: 0.0797 (0.14309055655628114) + | > loader_time: 0.0083 (0.009108500788873595) + + + --> TIME: 2024-04-09 16:16:59 -- STEP: 1102/1354 -- GLOBAL_STEP: 17350 + | > loss_text_ce: 0.08747299015522003 (0.07554495016183918) + | > loss_mel_ce: 2.9968738555908203 (2.786012447356314) + | > loss: 0.03671841323375702 (0.034066160121584386) + | > current_lr: 5e-06 + | > step_time: 0.11 (0.14309989692078753) + | > loader_time: 0.0077 (0.00909217862165559) + + + --> TIME: 2024-04-09 16:17:10 -- STEP: 1152/1354 -- GLOBAL_STEP: 17400 + | > loss_text_ce: 0.06549733877182007 (0.0754870534421773) + | > loss_mel_ce: 2.5139522552490234 (2.784762458772296) + | > loss: 0.03070773370563984 (0.03405059004823362) + | > current_lr: 5e-06 + | > step_time: 0.2179 (0.14314859877857897) + | > loader_time: 0.0085 (0.009079096424910761) + + + --> TIME: 2024-04-09 16:17:20 -- STEP: 1202/1354 -- GLOBAL_STEP: 17450 + | > loss_text_ce: 0.08611688762903214 (0.07552798818827881) + | > loss_mel_ce: 2.6640870571136475 (2.7821732090039197) + | > loss: 0.0327405221760273 (0.03402025296279965) + | > current_lr: 5e-06 + | > step_time: 0.1158 (0.14282243719910223) + | > loader_time: 0.0077 (0.009069718259344875) + + + --> TIME: 2024-04-09 16:17:31 -- STEP: 1252/1354 -- GLOBAL_STEP: 17500 + | > loss_text_ce: 0.0626782700419426 (0.07546555554976304) + | > loss_mel_ce: 2.900805950164795 (2.782369636403869) + | > loss: 0.03527957573533058 (0.034021848141730454) + | > current_lr: 5e-06 + | > step_time: 0.2144 (0.14272991670206325) + | > loader_time: 0.0101 (0.00906104877733955) + + + --> TIME: 2024-04-09 16:17:42 -- STEP: 1302/1354 -- GLOBAL_STEP: 17550 + | > loss_text_ce: 0.0836539939045906 (0.07551248400499297) + | > loss_mel_ce: 2.785276412963867 (2.781073430807728) + | > loss: 0.03415393456816673 (0.03400697579575891) + | > current_lr: 5e-06 + | > step_time: 0.1503 (0.14285960761449668) + | > loader_time: 0.0108 (0.009070552439184228) + + + --> TIME: 2024-04-09 16:17:52 -- STEP: 1352/1354 -- GLOBAL_STEP: 17600 + | > loss_text_ce: 0.07018766552209854 (0.07542247493536437) + | > loss_mel_ce: 2.6651971340179443 (2.7771324644956406) + | > loss: 0.03256410360336304 (0.033958987976264544) + | > current_lr: 5e-06 + | > step_time: 0.1607 (0.14265437115578944) + | > loader_time: 0.0091 (0.009060072475636496) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.02678819803091196 (+0.0018635162940392122) + | > avg_loss_text_ce: 0.0835118480026722 (-0.0057770532484238) + | > avg_loss_mel_ce: 2.9588875403771033 (-0.01778437541081379) + | > avg_loss: 3.042399369753324 (-0.023561459321242317) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_17602.pth + + > EPOCH: 14/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 16:18:41)  + + --> TIME: 2024-04-09 16:18:53 -- STEP: 48/1354 -- GLOBAL_STEP: 17650 + | > loss_text_ce: 0.056868184357881546 (0.07548541971482337) + | > loss_mel_ce: 2.4975943565368652 (2.8162819693485894) + | > loss: 0.030410267412662506 (0.03442580264527351) + | > current_lr: 5e-06 + | > step_time: 0.1979 (0.1460370570421219) + | > loader_time: 0.0089 (0.010122915108998615) + + + --> TIME: 2024-04-09 16:19:04 -- STEP: 98/1354 -- GLOBAL_STEP: 17700 + | > loss_text_ce: 0.057936105877161026 (0.07318897716396926) + | > loss_mel_ce: 2.541731357574463 (2.76170099998007) + | > loss: 0.030948422849178314 (0.033748690780176185) + | > current_lr: 5e-06 + | > step_time: 0.1501 (0.14324822231214876) + | > loader_time: 0.0082 (0.009828866744528009) + + + --> TIME: 2024-04-09 16:19:14 -- STEP: 148/1354 -- GLOBAL_STEP: 17750 + | > loss_text_ce: 0.05474520102143288 (0.07284759732617721) + | > loss_mel_ce: 2.741556406021118 (2.7647942562361023) + | > loss: 0.03328930586576462 (0.033781451223468456) + | > current_lr: 5e-06 + | > step_time: 0.1836 (0.14099180537301145) + | > loader_time: 0.0089 (0.009665100961118129) + + + --> TIME: 2024-04-09 16:19:26 -- STEP: 198/1354 -- GLOBAL_STEP: 17800 + | > loss_text_ce: 0.059995654970407486 (0.07136927434064523) + | > loss_mel_ce: 2.9116666316986084 (2.758251444257872) + | > loss: 0.035376932471990585 (0.03368596143719524) + | > current_lr: 5e-06 + | > step_time: 0.2074 (0.14553300419239093) + | > loader_time: 0.0101 (0.009683735442884043) + + + --> TIME: 2024-04-09 16:19:37 -- STEP: 248/1354 -- GLOBAL_STEP: 17850 + | > loss_text_ce: 0.11393295228481293 (0.07144046238353176) + | > loss_mel_ce: 3.199378490447998 (2.75150859067517) + | > loss: 0.03944418206810951 (0.033606536799080426) + | > current_lr: 5e-06 + | > step_time: 0.1382 (0.14531033654366776) + | > loader_time: 0.0082 (0.009608516770024457) + + + --> TIME: 2024-04-09 16:19:47 -- STEP: 298/1354 -- GLOBAL_STEP: 17900 + | > loss_text_ce: 0.06412173807621002 (0.07184101247397445) + | > loss_mel_ce: 2.825291395187378 (2.757822856007007) + | > loss: 0.034397777169942856 (0.03368647512198496) + | > current_lr: 5e-06 + | > step_time: 0.1892 (0.14288526893462117) + | > loader_time: 0.0101 (0.009528232100825986) + + + --> TIME: 2024-04-09 16:19:58 -- STEP: 348/1354 -- GLOBAL_STEP: 17950 + | > loss_text_ce: 0.0705140233039856 (0.07175922342415511) + | > loss_mel_ce: 2.8384110927581787 (2.7600709768547405) + | > loss: 0.03463006019592285 (0.03371226489972111) + | > current_lr: 5e-06 + | > step_time: 0.1161 (0.1433984226193921) + | > loader_time: 0.0086 (0.009495062389593018) + + + --> TIME: 2024-04-09 16:20:08 -- STEP: 398/1354 -- GLOBAL_STEP: 18000 + | > loss_text_ce: 0.06303073465824127 (0.07188354835063965) + | > loss_mel_ce: 2.7172024250030518 (2.7535572099925285) + | > loss: 0.0330980159342289 (0.03363620012942423) + | > current_lr: 5e-06 + | > step_time: 0.135 (0.1425533797872726) + | > loader_time: 0.0087 (0.009513025307775144) + + + --> TIME: 2024-04-09 16:20:42 -- STEP: 448/1354 -- GLOBAL_STEP: 18050 + | > loss_text_ce: 0.061536695808172226 (0.07161179928308616) + | > loss_mel_ce: 2.2443277835845947 (2.756246288440057) + | > loss: 0.027450770139694214 (0.03366497783073491) + | > current_lr: 5e-06 + | > step_time: 0.1142 (0.1431825134371009) + | > loader_time: 0.0099 (0.009601477533578873) + + + --> TIME: 2024-04-09 16:20:52 -- STEP: 498/1354 -- GLOBAL_STEP: 18100 + | > loss_text_ce: 0.07387610524892807 (0.07188616698256216) + | > loss_mel_ce: 3.0744717121124268 (2.7530370950698853) + | > loss: 0.03748033195734024 (0.03363003943444612) + | > current_lr: 5e-06 + | > step_time: 0.1871 (0.14287177913160215) + | > loader_time: 0.0091 (0.009573452922713804) + + + --> TIME: 2024-04-09 16:21:04 -- STEP: 548/1354 -- GLOBAL_STEP: 18150 + | > loss_text_ce: 0.059106454253196716 (0.07166448439451029) + | > loss_mel_ce: 2.2672619819641113 (2.7540932023612257) + | > loss: 0.027694862335920334 (0.03363997307028211) + | > current_lr: 5e-06 + | > step_time: 0.1194 (0.1434146148444963) + | > loader_time: 0.0084 (0.009552691104638316) + + + --> TIME: 2024-04-09 16:21:14 -- STEP: 598/1354 -- GLOBAL_STEP: 18200 + | > loss_text_ce: 0.074929378926754 (0.07213852652364147) + | > loss_mel_ce: 3.2246973514556885 (2.7560625076293945) + | > loss: 0.039281271398067474 (0.03366906054778937) + | > current_lr: 5e-06 + | > step_time: 0.1744 (0.14239859700601645) + | > loader_time: 0.0108 (0.009526030275733973) + + + --> TIME: 2024-04-09 16:21:25 -- STEP: 648/1354 -- GLOBAL_STEP: 18250 + | > loss_text_ce: 0.061032652854919434 (0.07187420660399907) + | > loss_mel_ce: 3.0592973232269287 (2.7630656501393265) + | > loss: 0.037146784365177155 (0.033749284625145363) + | > current_lr: 5e-06 + | > step_time: 0.1117 (0.14286602389665312) + | > loader_time: 0.0165 (0.009548215218532226) + + + --> TIME: 2024-04-09 16:21:36 -- STEP: 698/1354 -- GLOBAL_STEP: 18300 + | > loss_text_ce: 0.09681449830532074 (0.07179268162943056) + | > loss_mel_ce: 3.5454044342041016 (2.764316558837891) + | > loss: 0.043359749019145966 (0.03376320587807362) + | > current_lr: 5e-06 + | > step_time: 0.1678 (0.14298969148564836) + | > loader_time: 0.0109 (0.009536323369744849) + + + --> TIME: 2024-04-09 16:21:46 -- STEP: 748/1354 -- GLOBAL_STEP: 18350 + | > loss_text_ce: 0.05773135647177696 (0.07156177008036466) + | > loss_mel_ce: 2.629671812057495 (2.7634033484254927) + | > loss: 0.031992897391319275 (0.03374958537857635) + | > current_lr: 5e-06 + | > step_time: 0.1596 (0.14274519905049549) + | > loader_time: 0.0086 (0.009525480436131278) + + + --> TIME: 2024-04-09 16:21:58 -- STEP: 798/1354 -- GLOBAL_STEP: 18400 + | > loss_text_ce: 0.09795122593641281 (0.07166068259635967) + | > loss_mel_ce: 3.4551570415496826 (2.764652867514388) + | > loss: 0.04229890927672386 (0.033765638139761724) + | > current_lr: 5e-06 + | > step_time: 0.1409 (0.14316643180703775) + | > loader_time: 0.0084 (0.009505441613065862) + + + --> TIME: 2024-04-09 16:22:08 -- STEP: 848/1354 -- GLOBAL_STEP: 18450 + | > loss_text_ce: 0.05666341260075569 (0.07171437971926528) + | > loss_mel_ce: 2.7156107425689697 (2.763614736778556) + | > loss: 0.03300326690077782 (0.033753918672753645) + | > current_lr: 5e-06 + | > step_time: 0.206 (0.1422781845871009) + | > loader_time: 0.0081 (0.009470148468917267) + + + --> TIME: 2024-04-09 16:22:19 -- STEP: 898/1354 -- GLOBAL_STEP: 18500 + | > loss_text_ce: 0.07694082707166672 (0.0717877013098557) + | > loss_mel_ce: 3.294198751449585 (2.7687730243583033) + | > loss: 0.040132615715265274 (0.03381619974348552) + | > current_lr: 5e-06 + | > step_time: 0.1731 (0.1424723972456494) + | > loader_time: 0.0092 (0.009454790627239537) + + + --> TIME: 2024-04-09 16:22:29 -- STEP: 948/1354 -- GLOBAL_STEP: 18550 + | > loss_text_ce: 0.07739455252885818 (0.0717246246127263) + | > loss_mel_ce: 2.854302406311035 (2.769527819966465) + | > loss: 0.034901153296232224 (0.033824434474440696) + | > current_lr: 5e-06 + | > step_time: 0.1151 (0.14218527807968087) + | > loader_time: 0.0083 (0.00942465323436109) + + + --> TIME: 2024-04-09 16:22:40 -- STEP: 998/1354 -- GLOBAL_STEP: 18600 + | > loss_text_ce: 0.07906726002693176 (0.07161403041639051) + | > loss_mel_ce: 2.888336420059204 (2.7692792509982964) + | > loss: 0.03532623499631882 (0.033820158714253104) + | > current_lr: 5e-06 + | > step_time: 0.1135 (0.142522266728128) + | > loader_time: 0.0101 (0.009402354876837412) + + + --> TIME: 2024-04-09 16:22:51 -- STEP: 1048/1354 -- GLOBAL_STEP: 18650 + | > loss_text_ce: 0.06344173103570938 (0.07142423897028999) + | > loss_mel_ce: 2.7227377891540527 (2.7683862457976085) + | > loss: 0.03316880390048027 (0.03380726826490962) + | > current_lr: 5e-06 + | > step_time: 0.1436 (0.14260931301662952) + | > loader_time: 0.0097 (0.00937488220120204) + + + --> TIME: 2024-04-09 16:23:02 -- STEP: 1098/1354 -- GLOBAL_STEP: 18700 + | > loss_text_ce: 0.06854676455259323 (0.07134931271926306) + | > loss_mel_ce: 2.5881423950195312 (2.7685300375594477) + | > loss: 0.03162725269794464 (0.033808088087848284) + | > current_lr: 5e-06 + | > step_time: 0.1525 (0.1425814910881723) + | > loader_time: 0.0084 (0.009344850298702606) + + + --> TIME: 2024-04-09 16:23:12 -- STEP: 1148/1354 -- GLOBAL_STEP: 18750 + | > loss_text_ce: 0.06436115503311157 (0.07124560507734792) + | > loss_mel_ce: 3.4643678665161133 (2.769345868859143) + | > loss: 0.04200867936015129 (0.03381656576434752) + | > current_lr: 5e-06 + | > step_time: 0.1914 (0.14249112464825028) + | > loader_time: 0.0093 (0.009353243100103183) + + + --> TIME: 2024-04-09 16:23:24 -- STEP: 1198/1354 -- GLOBAL_STEP: 18800 + | > loss_text_ce: 0.05152246356010437 (0.07101039149688067) + | > loss_mel_ce: 2.4437475204467773 (2.76739855889287) + | > loss: 0.02970559522509575 (0.0337905833553616) + | > current_lr: 5e-06 + | > step_time: 0.1416 (0.14312795904124523) + | > loader_time: 0.0094 (0.009343466098002082) + + + --> TIME: 2024-04-09 16:23:35 -- STEP: 1248/1354 -- GLOBAL_STEP: 18850 + | > loss_text_ce: 0.06896983087062836 (0.07101236214228455) + | > loss_mel_ce: 3.099186658859253 (2.767918636114934) + | > loss: 0.03771614655852318 (0.033796798219928155) + | > current_lr: 5e-06 + | > step_time: 0.1853 (0.14338502956506533) + | > loader_time: 0.0095 (0.009326572601611797) + + + --> TIME: 2024-04-09 16:23:46 -- STEP: 1298/1354 -- GLOBAL_STEP: 18900 + | > loss_text_ce: 0.08936235308647156 (0.07100683797397672) + | > loss_mel_ce: 3.1509811878204346 (2.7654296618029583) + | > loss: 0.03857551887631416 (0.03376710182271081) + | > current_lr: 5e-06 + | > step_time: 0.0782 (0.1434227674510703) + | > loader_time: 0.0105 (0.009314404430301232) + + + --> TIME: 2024-04-09 16:23:57 -- STEP: 1348/1354 -- GLOBAL_STEP: 18950 + | > loss_text_ce: 0.09365186840295792 (0.07094138847161029) + | > loss_mel_ce: 3.0171148777008057 (2.76322354088905) + | > loss: 0.03703293576836586 (0.03374005931371528) + | > current_lr: 5e-06 + | > step_time: 0.1087 (0.14331295069318145) + | > loader_time: 0.0076 (0.009297550962657527) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.024673828711876504 (-0.0021143693190354557) + | > avg_loss_text_ce: 0.07838467537210538 (-0.00512717263056682) + | > avg_loss_mel_ce: 2.941446451040415 (-0.01744108933668853) + | > avg_loss: 3.0198311438927283 (-0.022568225860595703) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_18956.pth + + > EPOCH: 15/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 16:24:46)  + + --> TIME: 2024-04-09 16:24:57 -- STEP: 44/1354 -- GLOBAL_STEP: 19000 + | > loss_text_ce: 0.06712259352207184 (0.06933777330612594) + | > loss_mel_ce: 3.2612314224243164 (2.766090528531508) + | > loss: 0.03962326422333717 (0.033755099443210805) + | > current_lr: 5e-06 + | > step_time: 0.1601 (0.1444391283121976) + | > loader_time: 0.0093 (0.009766448627818714) + + + --> TIME: 2024-04-09 16:25:30 -- STEP: 94/1354 -- GLOBAL_STEP: 19050 + | > loss_text_ce: 0.07665429264307022 (0.06711034088375722) + | > loss_mel_ce: 2.25970721244812 (2.700756946776776) + | > loss: 0.027813827618956566 (0.032950801655967206) + | > current_lr: 5e-06 + | > step_time: 0.1518 (0.14386970692492546) + | > loader_time: 0.0102 (0.010303444050728004) + + + --> TIME: 2024-04-09 16:25:41 -- STEP: 144/1354 -- GLOBAL_STEP: 19100 + | > loss_text_ce: 0.0706777572631836 (0.0674926029880428) + | > loss_mel_ce: 3.117225408554077 (2.708605212469896) + | > loss: 0.03795122727751732 (0.033048784179199105) + | > current_lr: 5e-06 + | > step_time: 0.1103 (0.1431350741121504) + | > loader_time: 0.0094 (0.009971564014752703) + + + --> TIME: 2024-04-09 16:25:52 -- STEP: 194/1354 -- GLOBAL_STEP: 19150 + | > loss_text_ce: 0.055162668228149414 (0.06763846573141434) + | > loss_mel_ce: 2.1887834072113037 (2.701225120382211) + | > loss: 0.026713645085692406 (0.032962662388676275) + | > current_lr: 5e-06 + | > step_time: 0.1075 (0.14252499575467462) + | > loader_time: 0.0084 (0.00973170934264193) + + + --> TIME: 2024-04-09 16:26:03 -- STEP: 244/1354 -- GLOBAL_STEP: 19200 + | > loss_text_ce: 0.08990788459777832 (0.06785561384053013) + | > loss_mel_ce: 2.5187129974365234 (2.7074322988752457) + | > loss: 0.031055010855197906 (0.03303914246164632) + | > current_lr: 5e-06 + | > step_time: 0.1042 (0.14315078981587148) + | > loader_time: 0.0089 (0.00969011959482412) + + + --> TIME: 2024-04-09 16:26:14 -- STEP: 294/1354 -- GLOBAL_STEP: 19250 + | > loss_text_ce: 0.07234995067119598 (0.06790867340149115) + | > loss_mel_ce: 2.440847635269165 (2.709007702311691) + | > loss: 0.02991902083158493 (0.03305852892776941) + | > current_lr: 5e-06 + | > step_time: 0.1621 (0.14349970120151026) + | > loader_time: 0.0088 (0.009694739263884875) + + + --> TIME: 2024-04-09 16:26:24 -- STEP: 344/1354 -- GLOBAL_STEP: 19300 + | > loss_text_ce: 0.07218187302350998 (0.06796348208648167) + | > loss_mel_ce: 2.702453851699829 (2.711537874368733) + | > loss: 0.03303137794137001 (0.033089302474806154) + | > current_lr: 5e-06 + | > step_time: 0.1123 (0.143061742533085) + | > loader_time: 0.0105 (0.00965197696242222) + + + --> TIME: 2024-04-09 16:26:35 -- STEP: 394/1354 -- GLOBAL_STEP: 19350 + | > loss_text_ce: 0.08729357272386551 (0.0678044400617555) + | > loss_mel_ce: 3.194352149963379 (2.718147904437203) + | > loss: 0.039067212492227554 (0.033166099963756995) + | > current_lr: 5e-06 + | > step_time: 0.175 (0.14281175463332743) + | > loader_time: 0.0109 (0.009594685535140457) + + + --> TIME: 2024-04-09 16:26:46 -- STEP: 444/1354 -- GLOBAL_STEP: 19400 + | > loss_text_ce: 0.05545460060238838 (0.06771592706911729) + | > loss_mel_ce: 2.5755577087402344 (2.717941364994992) + | > loss: 0.03132157400250435 (0.03316258746492972) + | > current_lr: 5e-06 + | > step_time: 0.1404 (0.14232572403040022) + | > loader_time: 0.0079 (0.009519993721901841) + + + --> TIME: 2024-04-09 16:26:57 -- STEP: 494/1354 -- GLOBAL_STEP: 19450 + | > loss_text_ce: 0.09807871282100677 (0.06771764365073873) + | > loss_mel_ce: 2.6367053985595703 (2.718286165582987) + | > loss: 0.03255695477128029 (0.03316671266277069) + | > current_lr: 5e-06 + | > step_time: 0.0804 (0.143171537260295) + | > loader_time: 0.008 (0.00947808061051466) + + + --> TIME: 2024-04-09 16:27:08 -- STEP: 544/1354 -- GLOBAL_STEP: 19500 + | > loss_text_ce: 0.052939265966415405 (0.0676220497047966) + | > loss_mel_ce: 2.5324196815490723 (2.7243214694454374) + | > loss: 0.030778082087635994 (0.033237423473859544) + | > current_lr: 5e-06 + | > step_time: 0.1184 (0.14352132862105096) + | > loader_time: 0.0082 (0.009443643338540027) + + + --> TIME: 2024-04-09 16:27:19 -- STEP: 594/1354 -- GLOBAL_STEP: 19550 + | > loss_text_ce: 0.05086233466863632 (0.06729638081056492) + | > loss_mel_ce: 2.4118826389312744 (2.7192265925182633) + | > loss: 0.029318392276763916 (0.03317289316543828) + | > current_lr: 5e-06 + | > step_time: 0.1134 (0.14333165815783677) + | > loader_time: 0.0082 (0.009416849524886533) + + + --> TIME: 2024-04-09 16:27:30 -- STEP: 644/1354 -- GLOBAL_STEP: 19600 + | > loss_text_ce: 0.05460995435714722 (0.06698276409705788) + | > loss_mel_ce: 2.8024115562438965 (2.7186587676868674) + | > loss: 0.03401216119527817 (0.03316239980541127) + | > current_lr: 5e-06 + | > step_time: 0.1292 (0.1443984897240348) + | > loader_time: 0.0085 (0.009398667708687178) + + + --> TIME: 2024-04-09 16:27:41 -- STEP: 694/1354 -- GLOBAL_STEP: 19650 + | > loss_text_ce: 0.08310806006193161 (0.06705693210816005) + | > loss_mel_ce: 2.6112868785858154 (2.7187279272148186) + | > loss: 0.03207612782716751 (0.03316410606249267) + | > current_lr: 5e-06 + | > step_time: 0.0797 (0.1440340826765604) + | > loader_time: 0.0093 (0.00941000754276684) + + + --> TIME: 2024-04-09 16:27:51 -- STEP: 744/1354 -- GLOBAL_STEP: 19700 + | > loss_text_ce: 0.09557817876338959 (0.06725422343519587) + | > loss_mel_ce: 2.969433546066284 (2.722706760289849) + | > loss: 0.036488234996795654 (0.033213821843126295) + | > current_lr: 5e-06 + | > step_time: 0.1036 (0.143531261913238) + | > loader_time: 0.0088 (0.009390520472680384) + + + --> TIME: 2024-04-09 16:28:02 -- STEP: 794/1354 -- GLOBAL_STEP: 19750 + | > loss_text_ce: 0.06886398792266846 (0.06721563012308683) + | > loss_mel_ce: 2.0307369232177734 (2.7184034305195364) + | > loss: 0.02499524876475334 (0.03316213227855394) + | > current_lr: 5e-06 + | > step_time: 0.1112 (0.14295655894339548) + | > loader_time: 0.0094 (0.009365158357307949) + + + --> TIME: 2024-04-09 16:28:13 -- STEP: 844/1354 -- GLOBAL_STEP: 19800 + | > loss_text_ce: 0.05066017061471939 (0.06710863167693691) + | > loss_mel_ce: 2.3754868507385254 (2.720209462241539) + | > loss: 0.028882702812552452 (0.03318235885916874) + | > current_lr: 5e-06 + | > step_time: 0.2185 (0.14313860046920068) + | > loader_time: 0.0101 (0.009341374675244514) + + + --> TIME: 2024-04-09 16:28:23 -- STEP: 894/1354 -- GLOBAL_STEP: 19850 + | > loss_text_ce: 0.07579682022333145 (0.06706243142552297) + | > loss_mel_ce: 2.1485679149627686 (2.717415968177036) + | > loss: 0.026480533182621002 (0.03314855298073235) + | > current_lr: 5e-06 + | > step_time: 0.1005 (0.1427113287134193) + | > loader_time: 0.0084 (0.009321224236114976) + + + --> TIME: 2024-04-09 16:28:34 -- STEP: 944/1354 -- GLOBAL_STEP: 19900 + | > loss_text_ce: 0.06487604230642319 (0.06686648555589306) + | > loss_mel_ce: 2.2866177558898926 (2.7162675982562163) + | > loss: 0.027993975207209587 (0.03313254921152499) + | > current_lr: 5e-06 + | > step_time: 0.1214 (0.14273913356207196) + | > loader_time: 0.0094 (0.009344555816407927) + + + --> TIME: 2024-04-09 16:28:44 -- STEP: 994/1354 -- GLOBAL_STEP: 19950 + | > loss_text_ce: 0.03969703987240791 (0.06684890957230656) + | > loss_mel_ce: 2.6152920722961426 (2.7160580635550517) + | > loss: 0.0316070131957531 (0.03312984549663442) + | > current_lr: 5e-06 + | > step_time: 0.1464 (0.142519937194809) + | > loader_time: 0.0109 (0.009378946523071531) + + + --> TIME: 2024-04-09 16:28:55 -- STEP: 1044/1354 -- GLOBAL_STEP: 20000 + | > loss_text_ce: 0.08106723427772522 (0.06673383216063185) + | > loss_mel_ce: 2.252318859100342 (2.71574393291583) + | > loss: 0.02777840755879879 (0.03312473588041481) + | > current_lr: 5e-06 + | > step_time: 0.0893 (0.14253905632486757) + | > loader_time: 0.01 (0.00936751667110399) + + + > CHECKPOINT : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/checkpoint_20000.pth + + --> TIME: 2024-04-09 16:29:44 -- STEP: 1094/1354 -- GLOBAL_STEP: 20050 + | > loss_text_ce: 0.0571085549890995 (0.0666122498872675) + | > loss_mel_ce: 2.4970591068267822 (2.7151415576882503) + | > loss: 0.030406760051846504 (0.03311611733601947) + | > current_lr: 5e-06 + | > step_time: 0.2093 (0.14275402647921562) + | > loader_time: 0.0097 (0.009403209799604396) + + + --> TIME: 2024-04-09 16:29:54 -- STEP: 1144/1354 -- GLOBAL_STEP: 20100 + | > loss_text_ce: 0.08225557953119278 (0.0665873065493168) + | > loss_mel_ce: 3.080063581466675 (2.718200937434509) + | > loss: 0.03764665871858597 (0.03315224157242513) + | > current_lr: 5e-06 + | > step_time: 0.0936 (0.14229781915257875) + | > loader_time: 0.0078 (0.009420438037885647) + + + --> TIME: 2024-04-09 16:30:05 -- STEP: 1194/1354 -- GLOBAL_STEP: 20150 + | > loss_text_ce: 0.05481044948101044 (0.0665936067737827) + | > loss_mel_ce: 2.7621827125549316 (2.718074264039146) + | > loss: 0.033535633236169815 (0.0331508085690127) + | > current_lr: 5e-06 + | > step_time: 0.1433 (0.14218315666844117) + | > loader_time: 0.0109 (0.00939471517975006) + + + --> TIME: 2024-04-09 16:30:16 -- STEP: 1244/1354 -- GLOBAL_STEP: 20200 + | > loss_text_ce: 0.08817271888256073 (0.06665704922205587) + | > loss_mel_ce: 2.924278736114502 (2.719281312162085) + | > loss: 0.0358625166118145 (0.03316593344452109) + | > current_lr: 5e-06 + | > step_time: 0.0924 (0.1422712889131627) + | > loader_time: 0.0102 (0.009388115436701136) + + + --> TIME: 2024-04-09 16:30:26 -- STEP: 1294/1354 -- GLOBAL_STEP: 20250 + | > loss_text_ce: 0.0944456160068512 (0.06664118538661029) + | > loss_mel_ce: 2.9037325382232666 (2.716217806571416) + | > loss: 0.035692598670721054 (0.033129274285178235) + | > current_lr: 5e-06 + | > step_time: 0.0908 (0.14210346437864) + | > loader_time: 0.0083 (0.009366625611896297) + + + --> TIME: 2024-04-09 16:30:37 -- STEP: 1344/1354 -- GLOBAL_STEP: 20300 + | > loss_text_ce: 0.07801337540149689 (0.06669414974471913) + | > loss_mel_ce: 2.2852001190185547 (2.7188657435278065) + | > loss: 0.028133494779467583 (0.03316142786726625) + | > current_lr: 5e-06 + | > step_time: 0.0623 (0.14184856397055454) + | > loader_time: 0.0075 (0.00935734453655423) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.021719895876370944 (-0.00295393283550556) + | > avg_loss_text_ce: 0.07386586557214077 (-0.004518809799964607) + | > avg_loss_mel_ce: 2.928696742424598 (-0.01274970861581659) + | > avg_loss: 3.0025625962477465 (-0.01726854764498187) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_20310.pth + + > EPOCH: 16/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 16:31:32)  + + --> TIME: 2024-04-09 16:31:42 -- STEP: 40/1354 -- GLOBAL_STEP: 20350 + | > loss_text_ce: 0.07344086468219757 (0.0647455275990069) + | > loss_mel_ce: 2.1074273586273193 (2.6791758835315704) + | > loss: 0.025962715968489647 (0.0326657311990857) + | > current_lr: 5e-06 + | > step_time: 0.1112 (0.14057224392890924) + | > loader_time: 0.0143 (0.010422348976135254) + + + --> TIME: 2024-04-09 16:31:53 -- STEP: 90/1354 -- GLOBAL_STEP: 20400 + | > loss_text_ce: 0.05518202483654022 (0.06531855753726425) + | > loss_mel_ce: 2.564884662628174 (2.6881193028555974) + | > loss: 0.031191270798444748 (0.03277902249246834) + | > current_lr: 5e-06 + | > step_time: 0.1447 (0.14286515182918966) + | > loader_time: 0.0102 (0.01017720699310303) + + + --> TIME: 2024-04-09 16:32:03 -- STEP: 140/1354 -- GLOBAL_STEP: 20450 + | > loss_text_ce: 0.04435909166932106 (0.06398706508002108) + | > loss_mel_ce: 2.295484781265259 (2.6633166764463696) + | > loss: 0.02785528637468815 (0.03246790209252918) + | > current_lr: 5e-06 + | > step_time: 0.1755 (0.14260548182896204) + | > loader_time: 0.0097 (0.009960453850882399) + + + --> TIME: 2024-04-09 16:32:14 -- STEP: 190/1354 -- GLOBAL_STEP: 20500 + | > loss_text_ce: 0.09229741990566254 (0.06463840509715832) + | > loss_mel_ce: 2.4810001850128174 (2.6617386748916236) + | > loss: 0.030634494498372078 (0.03245687048490107) + | > current_lr: 5e-06 + | > step_time: 0.0909 (0.14204509885687575) + | > loader_time: 0.0086 (0.00985001639315957) + + + --> TIME: 2024-04-09 16:32:25 -- STEP: 240/1354 -- GLOBAL_STEP: 20550 + | > loss_text_ce: 0.061204131692647934 (0.0645691669701288) + | > loss_mel_ce: 2.5627999305725098 (2.6669196859002122) + | > loss: 0.031238144263625145 (0.03251772489553938) + | > current_lr: 5e-06 + | > step_time: 0.1337 (0.14259772102038068) + | > loader_time: 0.0089 (0.009764409065246588) + + + --> TIME: 2024-04-09 16:32:36 -- STEP: 290/1354 -- GLOBAL_STEP: 20600 + | > loss_text_ce: 0.07414374500513077 (0.06440169542770963) + | > loss_mel_ce: 2.7739598751068115 (2.6737355248681443) + | > loss: 0.0339059941470623 (0.03259687218172796) + | > current_lr: 5e-06 + | > step_time: 0.0748 (0.14205352109054045) + | > loader_time: 0.0197 (0.00981579320184116) + + + --> TIME: 2024-04-09 16:32:47 -- STEP: 340/1354 -- GLOBAL_STEP: 20650 + | > loss_text_ce: 0.07651372998952866 (0.06413227409562648) + | > loss_mel_ce: 2.1081840991973877 (2.6683372799087994) + | > loss: 0.026008307933807373 (0.03252939989461618) + | > current_lr: 5e-06 + | > step_time: 0.0882 (0.14275445938110348) + | > loader_time: 0.0091 (0.009719938390395226) + + + --> TIME: 2024-04-09 16:32:57 -- STEP: 390/1354 -- GLOBAL_STEP: 20700 + | > loss_text_ce: 0.08527661114931107 (0.06433953874959404) + | > loss_mel_ce: 3.19884991645813 (2.675479512336929) + | > loss: 0.03909674659371376 (0.03261689396145252) + | > current_lr: 5e-06 + | > step_time: 0.0839 (0.1416805022802108) + | > loader_time: 0.0113 (0.00966978439917932) + + + --> TIME: 2024-04-09 16:33:08 -- STEP: 440/1354 -- GLOBAL_STEP: 20750 + | > loss_text_ce: 0.06345728784799576 (0.0642917671156201) + | > loss_mel_ce: 2.4173789024353027 (2.6808604741638375) + | > loss: 0.029533764347434044 (0.03268038431944493) + | > current_lr: 5e-06 + | > step_time: 0.1342 (0.14190439256754783) + | > loader_time: 0.011 (0.009627091884613045) + + + --> TIME: 2024-04-09 16:33:18 -- STEP: 490/1354 -- GLOBAL_STEP: 20800 + | > loss_text_ce: 0.07819785177707672 (0.06413825981471008) + | > loss_mel_ce: 2.673508405685425 (2.6729205357785144) + | > loss: 0.03275841102004051 (0.03258403381522821) + | > current_lr: 5e-06 + | > step_time: 0.0652 (0.14140182076668253) + | > loader_time: 0.0083 (0.00963145032220958) + + + --> TIME: 2024-04-09 16:33:30 -- STEP: 540/1354 -- GLOBAL_STEP: 20850 + | > loss_text_ce: 0.05581602081656456 (0.06370965570763308) + | > loss_mel_ce: 2.5383176803588867 (2.6662767249125037) + | > loss: 0.030882542952895164 (0.03249983841001436) + | > current_lr: 5e-06 + | > step_time: 0.0925 (0.1425780305156001) + | > loader_time: 0.0084 (0.00960788903412996) + + + --> TIME: 2024-04-09 16:33:40 -- STEP: 590/1354 -- GLOBAL_STEP: 20900 + | > loss_text_ce: 0.07907067239284515 (0.06361377349091793) + | > loss_mel_ce: 2.1122119426727295 (2.667367272659886) + | > loss: 0.026086697354912758 (0.03251167967662973) + | > current_lr: 5e-06 + | > step_time: 0.0882 (0.14243077868122167) + | > loader_time: 0.0089 (0.009583188719668633) + + + --> TIME: 2024-04-09 16:33:51 -- STEP: 640/1354 -- GLOBAL_STEP: 20950 + | > loss_text_ce: 0.06615136563777924 (0.06360345606226475) + | > loss_mel_ce: 2.7851319313049316 (2.6740900207310934) + | > loss: 0.03394385054707527 (0.032591589569347004) + | > current_lr: 5e-06 + | > step_time: 0.2068 (0.14255184493958964) + | > loader_time: 0.009 (0.009586556255817412) + + + --> TIME: 2024-04-09 16:34:02 -- STEP: 690/1354 -- GLOBAL_STEP: 21000 + | > loss_text_ce: 0.05740988999605179 (0.06379433601446773) + | > loss_mel_ce: 2.3267264366149902 (2.6813552994659005) + | > loss: 0.028382577002048492 (0.03268035336843006) + | > current_lr: 5e-06 + | > step_time: 0.1859 (0.14306478362152547) + | > loader_time: 0.0095 (0.009569420676300492) + + + --> TIME: 2024-04-09 16:34:38 -- STEP: 740/1354 -- GLOBAL_STEP: 21050 + | > loss_text_ce: 0.06839043647050858 (0.06375694764727671) + | > loss_mel_ce: 2.3798158168792725 (2.6763795072967964) + | > loss: 0.029145311564207077 (0.032620672632411585) + | > current_lr: 5e-06 + | > step_time: 0.0913 (0.14282396999565328) + | > loader_time: 0.0091 (0.00964214350726153) + + + --> TIME: 2024-04-09 16:34:48 -- STEP: 790/1354 -- GLOBAL_STEP: 21100 + | > loss_text_ce: 0.044902775436639786 (0.06382002028389062) + | > loss_mel_ce: 2.815138101577759 (2.6767153854611565) + | > loss: 0.03404810652136803 (0.03262542204672027) + | > current_lr: 5e-06 + | > step_time: 0.1608 (0.14242631966554656) + | > loader_time: 0.0092 (0.009639909599400765) + + + --> TIME: 2024-04-09 16:35:00 -- STEP: 840/1354 -- GLOBAL_STEP: 21150 + | > loss_text_ce: 0.04130963236093521 (0.0636540765741041) + | > loss_mel_ce: 2.9323394298553467 (2.6737951573871444) + | > loss: 0.03540058434009552 (0.03258868190903399) + | > current_lr: 5e-06 + | > step_time: 0.1653 (0.14293698129199792) + | > loader_time: 0.0101 (0.009645934899648025) + + + --> TIME: 2024-04-09 16:35:10 -- STEP: 890/1354 -- GLOBAL_STEP: 21200 + | > loss_text_ce: 0.07110992819070816 (0.06382147529989138) + | > loss_mel_ce: 2.161630868911743 (2.6735965217097433) + | > loss: 0.02658024989068508 (0.03258831005208619) + | > current_lr: 5e-06 + | > step_time: 0.0908 (0.14297492986314742) + | > loader_time: 0.0084 (0.009628543157256045) + + + --> TIME: 2024-04-09 16:35:21 -- STEP: 940/1354 -- GLOBAL_STEP: 21250 + | > loss_text_ce: 0.06648927181959152 (0.06371652499792411) + | > loss_mel_ce: 2.4859142303466797 (2.672627434958804) + | > loss: 0.030385755002498627 (0.03257552389095126) + | > current_lr: 5e-06 + | > step_time: 0.1384 (0.14303565304330046) + | > loader_time: 0.0086 (0.009596948674384579) + + + --> TIME: 2024-04-09 16:35:32 -- STEP: 990/1354 -- GLOBAL_STEP: 21300 + | > loss_text_ce: 0.08129804581403732 (0.0637184922668066) + | > loss_mel_ce: 2.9069879055023193 (2.671951293343246) + | > loss: 0.035574834793806076 (0.03256749801074323) + | > current_lr: 5e-06 + | > step_time: 0.081 (0.14261443253719436) + | > loader_time: 0.01 (0.009582394060462407) + + + --> TIME: 2024-04-09 16:35:42 -- STEP: 1040/1354 -- GLOBAL_STEP: 21350 + | > loss_text_ce: 0.07369920611381531 (0.06374574424340741) + | > loss_mel_ce: 2.5152013301849365 (2.671735003705209) + | > loss: 0.030820244923233986 (0.03256524756658257) + | > current_lr: 5e-06 + | > step_time: 0.149 (0.14252270093330963) + | > loader_time: 0.009 (0.009566761897160452) + + + --> TIME: 2024-04-09 16:35:53 -- STEP: 1090/1354 -- GLOBAL_STEP: 21400 + | > loss_text_ce: 0.05118275433778763 (0.06367981433526632) + | > loss_mel_ce: 2.6040308475494385 (2.6723378658294683) + | > loss: 0.03160968795418739 (0.03257163962468922) + | > current_lr: 5e-06 + | > step_time: 0.2066 (0.14270706395490443) + | > loader_time: 0.0086 (0.00955900161638172) + + + --> TIME: 2024-04-09 16:36:03 -- STEP: 1140/1354 -- GLOBAL_STEP: 21450 + | > loss_text_ce: 0.05598962679505348 (0.06361329649624063) + | > loss_mel_ce: 2.962087392807007 (2.6746035134583197) + | > loss: 0.03592948988080025 (0.032597819754951934) + | > current_lr: 5e-06 + | > step_time: 0.1372 (0.1421481209888792) + | > loader_time: 0.0085 (0.009548485069944147) + + + --> TIME: 2024-04-09 16:36:14 -- STEP: 1190/1354 -- GLOBAL_STEP: 21500 + | > loss_text_ce: 0.06727710366249084 (0.06358999818125179) + | > loss_mel_ce: 2.6733105182647705 (2.67457914232206) + | > loss: 0.03262604400515556 (0.03259725226324149) + | > current_lr: 5e-06 + | > step_time: 0.1338 (0.14208647904275837) + | > loader_time: 0.0084 (0.009545093424179973) + + + --> TIME: 2024-04-09 16:36:25 -- STEP: 1240/1354 -- GLOBAL_STEP: 21550 + | > loss_text_ce: 0.052063003182411194 (0.06343173877666543) + | > loss_mel_ce: 2.3975532054901123 (2.672145058455007) + | > loss: 0.029162097722291946 (0.0325663910382578) + | > current_lr: 5e-06 + | > step_time: 0.2068 (0.1423590673554327) + | > loader_time: 0.0089 (0.009545124153937061) + + + --> TIME: 2024-04-09 16:36:36 -- STEP: 1290/1354 -- GLOBAL_STEP: 21600 + | > loss_text_ce: 0.09181265532970428 (0.06337470863332109) + | > loss_mel_ce: 3.0119054317474365 (2.6701511197311927) + | > loss: 0.036949023604393005 (0.03254197474800113) + | > current_lr: 5e-06 + | > step_time: 0.0687 (0.14218199641205534) + | > loader_time: 0.0084 (0.009549974471099622) + + + --> TIME: 2024-04-09 16:36:47 -- STEP: 1340/1354 -- GLOBAL_STEP: 21650 + | > loss_text_ce: 0.06474736332893372 (0.06329870767780188) + | > loss_mel_ce: 3.0295140743255615 (2.6724479802508827) + | > loss: 0.036836445331573486 (0.03256841356129348) + | > current_lr: 5e-06 + | > step_time: 0.1425 (0.14250487907608936) + | > loader_time: 0.0091 (0.009548213588657671) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.023067676104032077 (+0.0013477802276611328) + | > avg_loss_text_ce: 0.0696140441756982 (-0.004251821396442562) + | > avg_loss_mel_ce: 2.9121813774108887 (-0.016515365013709538) + | > avg_loss: 2.9817954210134654 (-0.020767175234281066) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_21664.pth + + > EPOCH: 17/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 16:37:40)  + + --> TIME: 2024-04-09 16:37:48 -- STEP: 36/1354 -- GLOBAL_STEP: 21700 + | > loss_text_ce: 0.07917489111423492 (0.06212476051102082) + | > loss_mel_ce: 2.445812702178955 (2.6653913458188376) + | > loss: 0.03005937859416008 (0.032470430506186344) + | > current_lr: 5e-06 + | > step_time: 0.0778 (0.13711553149753147) + | > loader_time: 0.0108 (0.009898742039998373) + + + --> TIME: 2024-04-09 16:37:59 -- STEP: 86/1354 -- GLOBAL_STEP: 21750 + | > loss_text_ce: 0.0718480795621872 (0.0642818575508373) + | > loss_mel_ce: 3.025949478149414 (2.650584464849428) + | > loss: 0.036878544837236404 (0.032319837834599394) + | > current_lr: 5e-06 + | > step_time: 0.101 (0.13669590617335123) + | > loader_time: 0.0093 (0.009786198305529218) + + + --> TIME: 2024-04-09 16:38:09 -- STEP: 136/1354 -- GLOBAL_STEP: 21800 + | > loss_text_ce: 0.052386317402124405 (0.06163283037569593) + | > loss_mel_ce: 2.4979333877563477 (2.6321236657745697) + | > loss: 0.03036094829440117 (0.0320685304536977) + | > current_lr: 5e-06 + | > step_time: 0.1492 (0.13812816143035883) + | > loader_time: 0.0096 (0.009829091675141281) + + + --> TIME: 2024-04-09 16:38:20 -- STEP: 186/1354 -- GLOBAL_STEP: 21850 + | > loss_text_ce: 0.07020960748195648 (0.061441034419081546) + | > loss_mel_ce: 2.8172388076782227 (2.635677548505926) + | > loss: 0.034374386072158813 (0.03210855517736687) + | > current_lr: 5e-06 + | > step_time: 0.0972 (0.1383550987448743) + | > loader_time: 0.0087 (0.009827634339691494) + + + --> TIME: 2024-04-09 16:38:31 -- STEP: 236/1354 -- GLOBAL_STEP: 21900 + | > loss_text_ce: 0.046491239219903946 (0.06168706374148191) + | > loss_mel_ce: 2.9903132915496826 (2.656423532356649) + | > loss: 0.036152433604002 (0.03235846008095196) + | > current_lr: 5e-06 + | > step_time: 0.2289 (0.13910014649568983) + | > loader_time: 0.0104 (0.009878814220428467) + + + --> TIME: 2024-04-09 16:38:42 -- STEP: 286/1354 -- GLOBAL_STEP: 21950 + | > loss_text_ce: 0.05961558595299721 (0.06182907968499027) + | > loss_mel_ce: 2.3159091472625732 (2.663997508845962) + | > loss: 0.02828005701303482 (0.032450317164192666) + | > current_lr: 5e-06 + | > step_time: 0.0995 (0.14091183755781272) + | > loader_time: 0.0084 (0.009919648403887982) + + + --> TIME: 2024-04-09 16:38:52 -- STEP: 336/1354 -- GLOBAL_STEP: 22000 + | > loss_text_ce: 0.06563877314329147 (0.061822632915296015) + | > loss_mel_ce: 2.3597702980041504 (2.65675062544289) + | > loss: 0.02887391857802868 (0.032363967997731544) + | > current_lr: 5e-06 + | > step_time: 0.0836 (0.13994121054808303) + | > loader_time: 0.0108 (0.009844252041407995) + + + --> TIME: 2024-04-09 16:39:26 -- STEP: 386/1354 -- GLOBAL_STEP: 22050 + | > loss_text_ce: 0.06408100575208664 (0.06141434243151561) + | > loss_mel_ce: 2.479428291320801 (2.642815102567327) + | > loss: 0.030279872938990593 (0.03219320828220541) + | > current_lr: 5e-06 + | > step_time: 0.102 (0.14045171787084082) + | > loader_time: 0.0113 (0.009980794061650896) + + + --> TIME: 2024-04-09 16:39:37 -- STEP: 436/1354 -- GLOBAL_STEP: 22100 + | > loss_text_ce: 0.048445358872413635 (0.06092956368256053) + | > loss_mel_ce: 2.2380974292755127 (2.634152751450145) + | > loss: 0.027220748364925385 (0.03208431385782194) + | > current_lr: 5e-06 + | > step_time: 0.1004 (0.14057551447404648) + | > loader_time: 0.0104 (0.009929741741320417) + + + --> TIME: 2024-04-09 16:39:47 -- STEP: 486/1354 -- GLOBAL_STEP: 22150 + | > loss_text_ce: 0.058880340307950974 (0.061125907157796894) + | > loss_mel_ce: 2.6472489833831787 (2.642473338072192) + | > loss: 0.03221582621335983 (0.0321857059037182) + | > current_lr: 5e-06 + | > step_time: 0.1919 (0.1399449428903714) + | > loader_time: 0.0092 (0.009888317359327787) + + + --> TIME: 2024-04-09 16:39:58 -- STEP: 536/1354 -- GLOBAL_STEP: 22200 + | > loss_text_ce: 0.0718388631939888 (0.06103416391189641) + | > loss_mel_ce: 2.9472312927246094 (2.6464571025389345) + | > loss: 0.03594131022691727 (0.032232039479483145) + | > current_lr: 5e-06 + | > step_time: 0.1104 (0.14000650484170496) + | > loader_time: 0.0096 (0.009907198041232669) + + + --> TIME: 2024-04-09 16:40:09 -- STEP: 586/1354 -- GLOBAL_STEP: 22250 + | > loss_text_ce: 0.056541889905929565 (0.06103086730929365) + | > loss_mel_ce: 2.744609832763672 (2.6447988058926715) + | > loss: 0.03334704414010048 (0.032212258596291066) + | > current_lr: 5e-06 + | > step_time: 0.1489 (0.14051235739281576) + | > loader_time: 0.0097 (0.009944633412279783) + + + --> TIME: 2024-04-09 16:40:20 -- STEP: 636/1354 -- GLOBAL_STEP: 22300 + | > loss_text_ce: 0.04995550215244293 (0.061142007205283865) + | > loss_mel_ce: 2.5228569507598877 (2.644873677747056) + | > loss: 0.03062872216105461 (0.03221447300751626) + | > current_lr: 5e-06 + | > step_time: 0.1915 (0.1403236021785617) + | > loader_time: 0.0101 (0.009926309375642994) + + + --> TIME: 2024-04-09 16:40:30 -- STEP: 686/1354 -- GLOBAL_STEP: 22350 + | > loss_text_ce: 0.051107991486787796 (0.06111241803224858) + | > loss_mel_ce: 2.751246213912964 (2.6447710860574913) + | > loss: 0.03336136043071747 (0.03221289944120901) + | > current_lr: 5e-06 + | > step_time: 0.2285 (0.14059520740898296) + | > loader_time: 0.0103 (0.009906638815521162) + + + --> TIME: 2024-04-09 16:40:41 -- STEP: 736/1354 -- GLOBAL_STEP: 22400 + | > loss_text_ce: 0.04482828080654144 (0.06094386795049776) + | > loss_mel_ce: 2.3074424266815186 (2.6469014214108824) + | > loss: 0.02800322137773037 (0.03223625402239597) + | > current_lr: 5e-06 + | > step_time: 0.1499 (0.14080165099838512) + | > loader_time: 0.0108 (0.00988089163666187) + + + --> TIME: 2024-04-09 16:40:52 -- STEP: 786/1354 -- GLOBAL_STEP: 22450 + | > loss_text_ce: 0.060733385384082794 (0.06089449938832197) + | > loss_mel_ce: 2.639798641204834 (2.6477591375661564) + | > loss: 0.032149191945791245 (0.03224587720598429) + | > current_lr: 5e-06 + | > step_time: 0.1198 (0.14060120485514485) + | > loader_time: 0.011 (0.009833961952733639) + + + --> TIME: 2024-04-09 16:41:03 -- STEP: 836/1354 -- GLOBAL_STEP: 22500 + | > loss_text_ce: 0.060091324150562286 (0.060902748666881945) + | > loss_mel_ce: 3.1350724697113037 (2.6511416328295567) + | > loss: 0.03803766518831253 (0.032286243199072265) + | > current_lr: 5e-06 + | > step_time: 0.1351 (0.14112126199822683) + | > loader_time: 0.0105 (0.009807516513258653) + + + --> TIME: 2024-04-09 16:41:14 -- STEP: 886/1354 -- GLOBAL_STEP: 22550 + | > loss_text_ce: 0.07803048193454742 (0.060758266753557034) + | > loss_mel_ce: 3.631925106048584 (2.65217865414867) + | > loss: 0.044166140258312225 (0.03229686867605486) + | > current_lr: 5e-06 + | > step_time: 0.0888 (0.14129545780121608) + | > loader_time: 0.0083 (0.009783283851485764) + + + --> TIME: 2024-04-09 16:41:24 -- STEP: 936/1354 -- GLOBAL_STEP: 22600 + | > loss_text_ce: 0.0708106979727745 (0.06083657548754898) + | > loss_mel_ce: 3.0270824432373047 (2.6506129850943885) + | > loss: 0.036879681050777435 (0.03227916201297994) + | > current_lr: 5e-06 + | > step_time: 0.186 (0.14103916185533893) + | > loader_time: 0.0098 (0.009768324784743486) + + + --> TIME: 2024-04-09 16:41:35 -- STEP: 986/1354 -- GLOBAL_STEP: 22650 + | > loss_text_ce: 0.06215890124440193 (0.06073453081261677) + | > loss_mel_ce: 2.4492955207824707 (2.6508463895586876) + | > loss: 0.0298982672393322 (0.032280725829348635) + | > current_lr: 5e-06 + | > step_time: 0.0914 (0.1410452603810215) + | > loader_time: 0.0088 (0.009748994941401922) + + + --> TIME: 2024-04-09 16:41:46 -- STEP: 1036/1354 -- GLOBAL_STEP: 22700 + | > loss_text_ce: 0.08851955085992813 (0.06060727113591773) + | > loss_mel_ce: 2.5521609783172607 (2.648310452239394) + | > loss: 0.031436674296855927 (0.032249021102965604) + | > current_lr: 5e-06 + | > step_time: 0.0875 (0.14123508345666533) + | > loader_time: 0.0087 (0.009736137500600931) + + + --> TIME: 2024-04-09 16:41:57 -- STEP: 1086/1354 -- GLOBAL_STEP: 22750 + | > loss_text_ce: 0.07158584892749786 (0.06060286847665275) + | > loss_mel_ce: 2.646456718444824 (2.6489636618129455) + | > loss: 0.032357651740312576 (0.03225674500781532) + | > current_lr: 5e-06 + | > step_time: 0.081 (0.14118749682635445) + | > loader_time: 0.0102 (0.009722334464830154) + + + --> TIME: 2024-04-09 16:42:07 -- STEP: 1136/1354 -- GLOBAL_STEP: 22800 + | > loss_text_ce: 0.049962081015110016 (0.06065205362638537) + | > loss_mel_ce: 2.7780961990356445 (2.6515252545056205) + | > loss: 0.03366735950112343 (0.032287825705667225) + | > current_lr: 5e-06 + | > step_time: 0.2282 (0.14112879272917614) + | > loader_time: 0.009 (0.009729550338127252) + + + --> TIME: 2024-04-09 16:42:18 -- STEP: 1186/1354 -- GLOBAL_STEP: 22850 + | > loss_text_ce: 0.06926856935024261 (0.06074542430359839) + | > loss_mel_ce: 2.6132822036743164 (2.6528509243141736) + | > loss: 0.031935129314661026 (0.032304719046487) + | > current_lr: 5e-06 + | > step_time: 0.1225 (0.1408102705345989) + | > loader_time: 0.0106 (0.009705865966127868) + + + --> TIME: 2024-04-09 16:42:29 -- STEP: 1236/1354 -- GLOBAL_STEP: 22900 + | > loss_text_ce: 0.054750531911849976 (0.06068961291866298) + | > loss_mel_ce: 2.689851999282837 (2.651092026901091) + | > loss: 0.03267383947968483 (0.03228311538455171) + | > current_lr: 5e-06 + | > step_time: 0.2154 (0.1411984726834835) + | > loader_time: 0.0136 (0.009691250169933044) + + + --> TIME: 2024-04-09 16:42:39 -- STEP: 1286/1354 -- GLOBAL_STEP: 22950 + | > loss_text_ce: 0.05745912715792656 (0.06065419609448402) + | > loss_mel_ce: 2.9573583602905273 (2.651188144509566) + | > loss: 0.03589068353176117 (0.03228383801130123) + | > current_lr: 5e-06 + | > step_time: 0.1415 (0.14111345478016313) + | > loader_time: 0.0104 (0.009689696839957004) + + + --> TIME: 2024-04-09 16:42:50 -- STEP: 1336/1354 -- GLOBAL_STEP: 23000 + | > loss_text_ce: 0.04897547513246536 (0.060510187429336906) + | > loss_mel_ce: 2.685215711593628 (2.6499067321865843) + | > loss: 0.0325498953461647 (0.032266868713373205) + | > current_lr: 5e-06 + | > step_time: 0.1179 (0.141161649348493) + | > loader_time: 0.0086 (0.009671518010293662) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.02366177852337177 (+0.000594102419339694) + | > avg_loss_text_ce: 0.0656982958316803 (-0.003915748344017908) + | > avg_loss_mel_ce: 2.8996125734769382 (-0.012568803933950434) + | > avg_loss: 2.965310885356023 (-0.016484535657442567) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_23018.pth + + > EPOCH: 18/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 16:44:12)  + + --> TIME: 2024-04-09 16:44:20 -- STEP: 32/1354 -- GLOBAL_STEP: 23050 + | > loss_text_ce: 0.061351094394922256 (0.05992035544477403) + | > loss_mel_ce: 2.8764636516571045 (2.7580177038908005) + | > loss: 0.03497398644685745 (0.033546882332302616) + | > current_lr: 5e-06 + | > step_time: 0.1842 (0.13906583189964294) + | > loader_time: 0.0111 (0.010463304817676544) + + + --> TIME: 2024-04-09 16:44:30 -- STEP: 82/1354 -- GLOBAL_STEP: 23100 + | > loss_text_ce: 0.0674123615026474 (0.060389611461177106) + | > loss_mel_ce: 2.46872878074646 (2.6456680966586603) + | > loss: 0.03019215725362301 (0.0322149733717485) + | > current_lr: 5e-06 + | > step_time: 0.1042 (0.1370402574539184) + | > loader_time: 0.0106 (0.009969676413187164) + + + --> TIME: 2024-04-09 16:44:40 -- STEP: 132/1354 -- GLOBAL_STEP: 23150 + | > loss_text_ce: 0.049883708357810974 (0.06031392046222181) + | > loss_mel_ce: 2.6412415504455566 (2.635208969766444) + | > loss: 0.03203720599412918 (0.03208955895212112) + | > current_lr: 5e-06 + | > step_time: 0.214 (0.13601996501286828) + | > loader_time: 0.0101 (0.010071761680371834) + + + --> TIME: 2024-04-09 16:44:51 -- STEP: 182/1354 -- GLOBAL_STEP: 23200 + | > loss_text_ce: 0.06643195450305939 (0.06046785659842439) + | > loss_mel_ce: 2.3182547092437744 (2.626990292098497) + | > loss: 0.028389127925038338 (0.03199355007446077) + | > current_lr: 5e-06 + | > step_time: 0.0903 (0.13651695225265006) + | > loader_time: 0.009 (0.0100899161873283) + + + --> TIME: 2024-04-09 16:45:02 -- STEP: 232/1354 -- GLOBAL_STEP: 23250 + | > loss_text_ce: 0.04660951346158981 (0.059952554373263287) + | > loss_mel_ce: 2.215357542037964 (2.627247066847209) + | > loss: 0.02692817896604538 (0.031990472339736) + | > current_lr: 5e-06 + | > step_time: 0.1531 (0.13900065113758228) + | > loader_time: 0.0089 (0.010064518657223931) + + + --> TIME: 2024-04-09 16:45:13 -- STEP: 282/1354 -- GLOBAL_STEP: 23300 + | > loss_text_ce: 0.04748751223087311 (0.0599304006058485) + | > loss_mel_ce: 2.4624011516571045 (2.6150590675942436) + | > loss: 0.029879627749323845 (0.03184511340124177) + | > current_lr: 5e-06 + | > step_time: 0.118 (0.1390280579844266) + | > loader_time: 0.0093 (0.00995736426495491) + + + --> TIME: 2024-04-09 16:45:23 -- STEP: 332/1354 -- GLOBAL_STEP: 23350 + | > loss_text_ce: 0.056703418493270874 (0.05951615277094295) + | > loss_mel_ce: 2.4601895809173584 (2.615789528711733) + | > loss: 0.029963010922074318 (0.0318488778310817) + | > current_lr: 5e-06 + | > step_time: 0.1206 (0.13923957189881667) + | > loader_time: 0.0092 (0.009869344981319934) + + + --> TIME: 2024-04-09 16:45:35 -- STEP: 382/1354 -- GLOBAL_STEP: 23400 + | > loss_text_ce: 0.055087268352508545 (0.0591304313235414) + | > loss_mel_ce: 2.5272271633148193 (2.6160370067776175) + | > loss: 0.030741840600967407 (0.0318472320506476) + | > current_lr: 5e-06 + | > step_time: 0.1377 (0.14121400186528732) + | > loader_time: 0.0087 (0.009836319853498048) + + + --> TIME: 2024-04-09 16:45:45 -- STEP: 432/1354 -- GLOBAL_STEP: 23450 + | > loss_text_ce: 0.04525153711438179 (0.059168864489981425) + | > loss_mel_ce: 2.635551929473877 (2.6113076866776845) + | > loss: 0.031914327293634415 (0.03179138812184746) + | > current_lr: 5e-06 + | > step_time: 0.1423 (0.140395635256061) + | > loader_time: 0.0108 (0.009786791823528427) + + + --> TIME: 2024-04-09 16:45:56 -- STEP: 482/1354 -- GLOBAL_STEP: 23500 + | > loss_text_ce: 0.04271857813000679 (0.0589210794179766) + | > loss_mel_ce: 2.4458706378936768 (2.6042368550518233) + | > loss: 0.029626064002513885 (0.03170426170963838) + | > current_lr: 5e-06 + | > step_time: 0.1438 (0.14070232824665888) + | > loader_time: 0.0112 (0.009742885704357097) + + + --> TIME: 2024-04-09 16:46:06 -- STEP: 532/1354 -- GLOBAL_STEP: 23550 + | > loss_text_ce: 0.047092508524656296 (0.0589487523862854) + | > loss_mel_ce: 2.140465259552002 (2.601669596102004) + | > loss: 0.026042353361845016 (0.03167402851780441) + | > current_lr: 5e-06 + | > step_time: 0.1412 (0.14007246763186357) + | > loader_time: 0.009 (0.009719869247952794) + + + --> TIME: 2024-04-09 16:46:17 -- STEP: 582/1354 -- GLOBAL_STEP: 23600 + | > loss_text_ce: 0.03655370697379112 (0.05877258675332946) + | > loss_mel_ce: 2.0414421558380127 (2.6031697240891725) + | > loss: 0.024738045409321785 (0.03168978996229236) + | > current_lr: 5e-06 + | > step_time: 0.1919 (0.14065672523786937) + | > loader_time: 0.0094 (0.009685735931920837) + + + --> TIME: 2024-04-09 16:46:28 -- STEP: 632/1354 -- GLOBAL_STEP: 23650 + | > loss_text_ce: 0.06113525107502937 (0.058759108792896135) + | > loss_mel_ce: 2.1148014068603516 (2.6065606431493267) + | > loss: 0.025904009118676186 (0.03172999759251846) + | > current_lr: 5e-06 + | > step_time: 0.0746 (0.14058026746858535) + | > loader_time: 0.0082 (0.00970260176477552) + + + --> TIME: 2024-04-09 16:46:39 -- STEP: 682/1354 -- GLOBAL_STEP: 23700 + | > loss_text_ce: 0.05480119213461876 (0.05891250786740235) + | > loss_mel_ce: 2.5720391273498535 (2.6073093515686967) + | > loss: 0.031271908432245255 (0.03174073697556681) + | > current_lr: 5e-06 + | > step_time: 0.1114 (0.14078748086330833) + | > loader_time: 0.0086 (0.009692238572755509) + + + --> TIME: 2024-04-09 16:46:49 -- STEP: 732/1354 -- GLOBAL_STEP: 23750 + | > loss_text_ce: 0.06591577082872391 (0.05874521302089828) + | > loss_mel_ce: 2.2857964038848877 (2.605992807228056) + | > loss: 0.027996575459837914 (0.031723072240892554) + | > current_lr: 5e-06 + | > step_time: 0.078 (0.14069371275563058) + | > loader_time: 0.0082 (0.009696175491874974) + + + --> TIME: 2024-04-09 16:47:00 -- STEP: 782/1354 -- GLOBAL_STEP: 23800 + | > loss_text_ce: 0.04316532984375954 (0.05855761754714772) + | > loss_mel_ce: 2.367065906524658 (2.6025727414109214) + | > loss: 0.02869323082268238 (0.03168012389241512) + | > current_lr: 5e-06 + | > step_time: 0.1999 (0.14106913448294708) + | > loader_time: 0.0092 (0.009685900815002748) + + + --> TIME: 2024-04-09 16:47:12 -- STEP: 832/1354 -- GLOBAL_STEP: 23850 + | > loss_text_ce: 0.051707617938518524 (0.058455925896011576) + | > loss_mel_ce: 2.4886837005615234 (2.6030198484659195) + | > loss: 0.03024275228381157 (0.03168423598309842) + | > current_lr: 5e-06 + | > step_time: 0.1856 (0.14152731030033172) + | > loader_time: 0.0098 (0.009686717333701939) + + + --> TIME: 2024-04-09 16:47:22 -- STEP: 882/1354 -- GLOBAL_STEP: 23900 + | > loss_text_ce: 0.09571889787912369 (0.05842736476183343) + | > loss_mel_ce: 3.0816476345062256 (2.6037058138252656) + | > loss: 0.03782579302787781 (0.031692062225826406) + | > current_lr: 5e-06 + | > step_time: 0.0935 (0.14158563035415675) + | > loader_time: 0.0112 (0.009679619417168925) + + + --> TIME: 2024-04-09 16:47:33 -- STEP: 932/1354 -- GLOBAL_STEP: 23950 + | > loss_text_ce: 0.07379757612943649 (0.05827221755226155) + | > loss_mel_ce: 2.690600872039795 (2.6028915419854832) + | > loss: 0.03290950506925583 (0.03168052152126971) + | > current_lr: 5e-06 + | > step_time: 0.1049 (0.1412634082106561) + | > loader_time: 0.0095 (0.009665550847933526) + + + --> TIME: 2024-04-09 16:47:43 -- STEP: 982/1354 -- GLOBAL_STEP: 24000 + | > loss_text_ce: 0.05007033795118332 (0.05820795182373883) + | > loss_mel_ce: 2.732863664627075 (2.6009905895487595) + | > loss: 0.03313016518950462 (0.031657126075882225) + | > current_lr: 5e-06 + | > step_time: 0.1604 (0.14104301089424703) + | > loader_time: 0.0094 (0.009666534402462712) + + + --> TIME: 2024-04-09 16:48:18 -- STEP: 1032/1354 -- GLOBAL_STEP: 24050 + | > loss_text_ce: 0.06461576372385025 (0.05811700888714472) + | > loss_mel_ce: 3.1659023761749268 (2.598062810047652) + | > loss: 0.03845854848623276 (0.03162118889817379) + | > current_lr: 5e-06 + | > step_time: 0.0891 (0.14102098946423489) + | > loader_time: 0.011 (0.00976142264151758) + + + --> TIME: 2024-04-09 16:48:29 -- STEP: 1082/1354 -- GLOBAL_STEP: 24100 + | > loss_text_ce: 0.04391733184456825 (0.0581327444613806) + | > loss_mel_ce: 2.401737689971924 (2.602082596688967) + | > loss: 0.029114942997694016 (0.03166923083213475) + | > current_lr: 5e-06 + | > step_time: 0.1427 (0.14103752583981444) + | > loader_time: 0.0098 (0.00977814924694914) + + + --> TIME: 2024-04-09 16:48:40 -- STEP: 1132/1354 -- GLOBAL_STEP: 24150 + | > loss_text_ce: 0.05443662032485008 (0.05803397104025622) + | > loss_mel_ce: 2.3436174392700195 (2.601166782025313) + | > loss: 0.028548263013362885 (0.03165715239948657) + | > current_lr: 5e-06 + | > step_time: 0.1668 (0.140948525376539) + | > loader_time: 0.009 (0.009769420109873523) + + + --> TIME: 2024-04-09 16:48:51 -- STEP: 1182/1354 -- GLOBAL_STEP: 24200 + | > loss_text_ce: 0.05795348063111305 (0.057831218882184926) + | > loss_mel_ce: 3.192800998687744 (2.5987328910182166) + | > loss: 0.03869945928454399 (0.031625763784192656) + | > current_lr: 5e-06 + | > step_time: 0.1711 (0.14114829166689913) + | > loader_time: 0.0107 (0.009770789323926166) + + + --> TIME: 2024-04-09 16:49:02 -- STEP: 1232/1354 -- GLOBAL_STEP: 24250 + | > loss_text_ce: 0.053732093423604965 (0.05765063028068993) + | > loss_mel_ce: 2.720280885696411 (2.5966014164415276) + | > loss: 0.03302396461367607 (0.031598239236015374) + | > current_lr: 5e-06 + | > step_time: 0.1415 (0.14176807736421573) + | > loader_time: 0.0087 (0.009780865404512961) + + + --> TIME: 2024-04-09 16:49:14 -- STEP: 1282/1354 -- GLOBAL_STEP: 24300 + | > loss_text_ce: 0.04788254201412201 (0.05761990768547268) + | > loss_mel_ce: 2.449293851852417 (2.5973779991524837) + | > loss: 0.029728291556239128 (0.031607118523832196) + | > current_lr: 5e-06 + | > step_time: 0.153 (0.1421555037059577) + | > loader_time: 0.0088 (0.009785112278323817) + + + --> TIME: 2024-04-09 16:49:24 -- STEP: 1332/1354 -- GLOBAL_STEP: 24350 + | > loss_text_ce: 0.06749563664197922 (0.05760399447110767) + | > loss_mel_ce: 2.6249020099639893 (2.5970148624958567) + | > loss: 0.03205235302448273 (0.031602606019312814) + | > current_lr: 5e-06 + | > step_time: 0.082 (0.14218980628806904) + | > loader_time: 0.0101 (0.009776830673217746) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0303041751568134 (+0.006642396633441631) + | > avg_loss_text_ce: 0.06225843240435307 (-0.0034398634273272305) + | > avg_loss_mel_ce: 2.890443655160757 (-0.009168918316181163) + | > avg_loss: 2.9527020821204553 (-0.012608803235567567) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_24372.pth + + > EPOCH: 19/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 16:50:26)  + + --> TIME: 2024-04-09 16:50:33 -- STEP: 28/1354 -- GLOBAL_STEP: 24400 + | > loss_text_ce: 0.04227643087506294 (0.054555420364652364) + | > loss_mel_ce: 2.157681941986084 (2.5783159349645888) + | > loss: 0.026189981028437614 (0.031343707615243535) + | > current_lr: 5e-06 + | > step_time: 0.1934 (0.14162382057734899) + | > loader_time: 0.0114 (0.010879593236105782) + + + --> TIME: 2024-04-09 16:50:45 -- STEP: 78/1354 -- GLOBAL_STEP: 24450 + | > loss_text_ce: 0.05965717136859894 (0.05502634314008248) + | > loss_mel_ce: 3.187486410140991 (2.575565888331487) + | > loss: 0.03865646943449974 (0.03131657511664507) + | > current_lr: 5e-06 + | > step_time: 0.1017 (0.14621277038867658) + | > loader_time: 0.0108 (0.010815620422363283) + + + --> TIME: 2024-04-09 16:50:56 -- STEP: 128/1354 -- GLOBAL_STEP: 24500 + | > loss_text_ce: 0.04138806834816933 (0.05467880386277102) + | > loss_mel_ce: 2.2905516624450684 (2.570969657972455) + | > loss: 0.02776118740439415 (0.031257720591383986) + | > current_lr: 5e-06 + | > step_time: 0.161 (0.14685654453933245) + | > loader_time: 0.0116 (0.010643979534506798) + + + --> TIME: 2024-04-09 16:51:06 -- STEP: 178/1354 -- GLOBAL_STEP: 24550 + | > loss_text_ce: 0.045697350054979324 (0.05473797186539414) + | > loss_mel_ce: 2.4751217365264893 (2.552514450603657) + | > loss: 0.03000975213944912 (0.031038720066544047) + | > current_lr: 5e-06 + | > step_time: 0.1736 (0.1445365528042397) + | > loader_time: 0.0093 (0.010522444596451321) + + + --> TIME: 2024-04-09 16:51:17 -- STEP: 228/1354 -- GLOBAL_STEP: 24600 + | > loss_text_ce: 0.04021269083023071 (0.05471866233110951) + | > loss_mel_ce: 2.2658426761627197 (2.552362901599783) + | > loss: 0.027453040704131126 (0.031036685996999342) + | > current_lr: 5e-06 + | > step_time: 0.2081 (0.14464096646559862) + | > loader_time: 0.0096 (0.01037037581728216) + + + --> TIME: 2024-04-09 16:51:28 -- STEP: 278/1354 -- GLOBAL_STEP: 24650 + | > loss_text_ce: 0.04535973072052002 (0.054858503876615776) + | > loss_mel_ce: 2.5976169109344482 (2.5634388653494464) + | > loss: 0.03146401047706604 (0.03117020751798539) + | > current_lr: 5e-06 + | > step_time: 0.1993 (0.14313962819764936) + | > loader_time: 0.0095 (0.010290901437937787) + + + --> TIME: 2024-04-09 16:51:38 -- STEP: 328/1354 -- GLOBAL_STEP: 24700 + | > loss_text_ce: 0.047370899468660355 (0.05539803944028369) + | > loss_mel_ce: 2.675847053527832 (2.579652695757586) + | > loss: 0.03241926059126854 (0.031369652354908066) + | > current_lr: 5e-06 + | > step_time: 0.211 (0.14312465670632143) + | > loader_time: 0.0089 (0.010242473061491803) + + + --> TIME: 2024-04-09 16:51:49 -- STEP: 378/1354 -- GLOBAL_STEP: 24750 + | > loss_text_ce: 0.06504334509372711 (0.05553563544280315) + | > loss_mel_ce: 2.880204916000366 (2.5824027074077134) + | > loss: 0.035062480717897415 (0.03140402858258871) + | > current_lr: 5e-06 + | > step_time: 0.1133 (0.1427178698242026) + | > loader_time: 0.0105 (0.0103088516406912) + + + --> TIME: 2024-04-09 16:51:59 -- STEP: 428/1354 -- GLOBAL_STEP: 24800 + | > loss_text_ce: 0.05673997849225998 (0.055817420141788844) + | > loss_mel_ce: 2.8839986324310303 (2.5869277520714506) + | > loss: 0.03500879555940628 (0.03146125273517918) + | > current_lr: 5e-06 + | > step_time: 0.1195 (0.1415608023928706) + | > loader_time: 0.01 (0.010293552251619714) + + + --> TIME: 2024-04-09 16:52:10 -- STEP: 478/1354 -- GLOBAL_STEP: 24850 + | > loss_text_ce: 0.04785158112645149 (0.05558551494889429) + | > loss_mel_ce: 2.3161418437957764 (2.5781098843618406) + | > loss: 0.0281427800655365 (0.031353517310795186) + | > current_lr: 5e-06 + | > step_time: 0.1324 (0.14149015318898484) + | > loader_time: 0.0117 (0.010225846178860845) + + + --> TIME: 2024-04-09 16:52:22 -- STEP: 528/1354 -- GLOBAL_STEP: 24900 + | > loss_text_ce: 0.052046820521354675 (0.055252258531805695) + | > loss_mel_ce: 2.960437297821045 (2.5766531147740097) + | > loss: 0.03586290776729584 (0.031332207437971576) + | > current_lr: 5e-06 + | > step_time: 0.1525 (0.14256534838315219) + | > loader_time: 0.0113 (0.010181767922459227) + + + --> TIME: 2024-04-09 16:52:32 -- STEP: 578/1354 -- GLOBAL_STEP: 24950 + | > loss_text_ce: 0.04418780282139778 (0.055135111372827665) + | > loss_mel_ce: 2.6522347927093506 (2.5764298783454107) + | > loss: 0.032100267708301544 (0.03132815526920828) + | > current_lr: 5e-06 + | > step_time: 0.1778 (0.14253283866961525) + | > loader_time: 0.0093 (0.010138861860783454) + + + --> TIME: 2024-04-09 16:52:44 -- STEP: 628/1354 -- GLOBAL_STEP: 25000 + | > loss_text_ce: 0.04776109755039215 (0.054947717471202466) + | > loss_mel_ce: 2.4796268939971924 (2.573313060839464) + | > loss: 0.030087953433394432 (0.03128881941721509) + | > current_lr: 5e-06 + | > step_time: 0.1528 (0.1432506958390497) + | > loader_time: 0.0089 (0.010128416073550079) + + + --> TIME: 2024-04-09 16:53:19 -- STEP: 678/1354 -- GLOBAL_STEP: 25050 + | > loss_text_ce: 0.05524643138051033 (0.054703823711096715) + | > loss_mel_ce: 2.55804443359375 (2.565994465421434) + | > loss: 0.031110605224967003 (0.031198789780426523) + | > current_lr: 5e-06 + | > step_time: 0.1452 (0.1436235834363639) + | > loader_time: 0.0115 (0.010238980121668805) + + + --> TIME: 2024-04-09 16:53:30 -- STEP: 728/1354 -- GLOBAL_STEP: 25100 + | > loss_text_ce: 0.04286177456378937 (0.05447305476927495) + | > loss_mel_ce: 2.2632975578308105 (2.565857541921374) + | > loss: 0.027454277500510216 (0.031194412471869822) + | > current_lr: 5e-06 + | > step_time: 0.1124 (0.14384057823118274) + | > loader_time: 0.0084 (0.010223776757062123) + + + --> TIME: 2024-04-09 16:53:41 -- STEP: 778/1354 -- GLOBAL_STEP: 25150 + | > loss_text_ce: 0.059788063168525696 (0.05450940850384744) + | > loss_mel_ce: 2.5010151863098145 (2.5629784660964505) + | > loss: 0.03048575296998024 (0.031160570544628837) + | > current_lr: 5e-06 + | > step_time: 0.0642 (0.14311022133324633) + | > loader_time: 0.012 (0.010240450616360926) + + + --> TIME: 2024-04-09 16:53:51 -- STEP: 828/1354 -- GLOBAL_STEP: 25200 + | > loss_text_ce: 0.052259936928749084 (0.054529302353084376) + | > loss_mel_ce: 2.1218223571777344 (2.557543740036406) + | > loss: 0.0258819330483675 (0.031096108275767115) + | > current_lr: 5e-06 + | > step_time: 0.1465 (0.14281420748014956) + | > loader_time: 0.0121 (0.010266119155330926) + + + --> TIME: 2024-04-09 16:54:02 -- STEP: 878/1354 -- GLOBAL_STEP: 25250 + | > loss_text_ce: 0.04634153097867966 (0.054596429750119635) + | > loss_mel_ce: 2.422475814819336 (2.558102542148363) + | > loss: 0.02939068153500557 (0.031103559829091024) + | > current_lr: 5e-06 + | > step_time: 0.1352 (0.14303759018761147) + | > loader_time: 0.0107 (0.010281306736018491) + + + --> TIME: 2024-04-09 16:54:13 -- STEP: 928/1354 -- GLOBAL_STEP: 25300 + | > loss_text_ce: 0.04444967955350876 (0.05454704474159998) + | > loss_mel_ce: 2.6436948776245117 (2.5545043150155697) + | > loss: 0.03200171887874603 (0.031060135875154164) + | > current_lr: 5e-06 + | > step_time: 0.1588 (0.14316411676078) + | > loader_time: 0.009 (0.010289045500344237) + + + --> TIME: 2024-04-09 16:54:24 -- STEP: 978/1354 -- GLOBAL_STEP: 25350 + | > loss_text_ce: 0.03823542222380638 (0.054449118422401466) + | > loss_mel_ce: 2.4389541149139404 (2.551977450740361) + | > loss: 0.029490351676940918 (0.031028888359711584) + | > current_lr: 5e-06 + | > step_time: 0.2 (0.14328906531226654) + | > loader_time: 0.0099 (0.010267579726158712) + + + --> TIME: 2024-04-09 16:54:35 -- STEP: 1028/1354 -- GLOBAL_STEP: 25400 + | > loss_text_ce: 0.057789769023656845 (0.054341648150858714) + | > loss_mel_ce: 2.549528121948242 (2.5545934799348324) + | > loss: 0.03103950060904026 (0.03105875215554748) + | > current_lr: 5e-06 + | > step_time: 0.0812 (0.14349760109348514) + | > loader_time: 0.0099 (0.010258193145930064) + + + --> TIME: 2024-04-09 16:54:46 -- STEP: 1078/1354 -- GLOBAL_STEP: 25450 + | > loss_text_ce: 0.055263761430978775 (0.054339260567871216) + | > loss_mel_ce: 2.759484052658081 (2.556861011128257) + | > loss: 0.03350890427827835 (0.031085718144940867) + | > current_lr: 5e-06 + | > step_time: 0.2041 (0.14335112757497015) + | > loader_time: 0.012 (0.010253643502994455) + + + --> TIME: 2024-04-09 16:54:57 -- STEP: 1128/1354 -- GLOBAL_STEP: 25500 + | > loss_text_ce: 0.045178722590208054 (0.05435615869780592) + | > loss_mel_ce: 2.2799935340881348 (2.5565703810318134) + | > loss: 0.027680622413754463 (0.031082459419952534) + | > current_lr: 5e-06 + | > step_time: 0.1068 (0.14322469716376446) + | > loader_time: 0.0102 (0.010248074506191491) + + + --> TIME: 2024-04-09 16:55:08 -- STEP: 1178/1354 -- GLOBAL_STEP: 25550 + | > loss_text_ce: 0.05306432023644447 (0.05449418272310789) + | > loss_mel_ce: 2.820387125015259 (2.5629946499405167) + | > loss: 0.03420775756239891 (0.03116058196290594) + | > current_lr: 5e-06 + | > step_time: 0.2009 (0.14312041352276889) + | > loader_time: 0.0097 (0.010230976373513799) + + + --> TIME: 2024-04-09 16:55:18 -- STEP: 1228/1354 -- GLOBAL_STEP: 25600 + | > loss_text_ce: 0.05775328353047371 (0.05453078749707933) + | > loss_mel_ce: 2.5780484676361084 (2.5640303782803238) + | > loss: 0.03137859329581261 (0.031173347824644478) + | > current_lr: 5e-06 + | > step_time: 0.1076 (0.14265820917943398) + | > loader_time: 0.0087 (0.010207585480780084) + + + --> TIME: 2024-04-09 16:55:28 -- STEP: 1278/1354 -- GLOBAL_STEP: 25650 + | > loss_text_ce: 0.04454071819782257 (0.05447510301961865) + | > loss_mel_ce: 2.7126753330230713 (2.5591767240018353) + | > loss: 0.03282399848103523 (0.031114903309255697) + | > current_lr: 5e-06 + | > step_time: 0.2087 (0.14258641629524998) + | > loader_time: 0.0097 (0.010209567483415439) + + + --> TIME: 2024-04-09 16:55:39 -- STEP: 1328/1354 -- GLOBAL_STEP: 25700 + | > loss_text_ce: 0.055380433797836304 (0.05450799237061516) + | > loss_mel_ce: 2.5428409576416016 (2.560999547232347) + | > loss: 0.030931206420063972 (0.031136995126740133) + | > current_lr: 5e-06 + | > step_time: 0.1094 (0.1424788084015789) + | > loader_time: 0.0083 (0.010183593594884303) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.035313111085158125 (+0.005008935928344723) + | > avg_loss_text_ce: 0.05921256456237573 (-0.003045867841977336) + | > avg_loss_mel_ce: 2.881040096282959 (-0.00940355887779809) + | > avg_loss: 2.940252634195181 (-0.01244944792527436) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_25726.pth + + > EPOCH: 20/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 16:56:37)  + + --> TIME: 2024-04-09 16:56:43 -- STEP: 24/1354 -- GLOBAL_STEP: 25750 + | > loss_text_ce: 0.049638647586107254 (0.05028238566592336) + | > loss_mel_ce: 2.5480566024780273 (2.548516780138016) + | > loss: 0.030924946069717407 (0.030938086022312444) + | > current_lr: 5e-06 + | > step_time: 0.162 (0.14765595396359762) + | > loader_time: 0.0118 (0.010706682999928793) + + + --> TIME: 2024-04-09 16:56:54 -- STEP: 74/1354 -- GLOBAL_STEP: 25800 + | > loss_text_ce: 0.03995085135102272 (0.05202923035500823) + | > loss_mel_ce: 2.4929208755493164 (2.5681261355812484) + | > loss: 0.03015323542058468 (0.03119232631414324) + | > current_lr: 5e-06 + | > step_time: 0.2169 (0.1476682746732557) + | > loader_time: 0.0095 (0.010740276929494499) + + + --> TIME: 2024-04-09 16:57:05 -- STEP: 124/1354 -- GLOBAL_STEP: 25850 + | > loss_text_ce: 0.07178358733654022 (0.053293899452734377) + | > loss_mel_ce: 3.268458366394043 (2.565290146297025) + | > loss: 0.03976478427648544 (0.031173620117648015) + | > current_lr: 5e-06 + | > step_time: 0.1209 (0.14337182429529) + | > loader_time: 0.0097 (0.01038364056617983) + + + --> TIME: 2024-04-09 16:57:16 -- STEP: 174/1354 -- GLOBAL_STEP: 25900 + | > loss_text_ce: 0.04963180422782898 (0.052747939895281835) + | > loss_mel_ce: 2.2793445587158203 (2.562892448628085) + | > loss: 0.027725910767912865 (0.03113857662069731) + | > current_lr: 5e-06 + | > step_time: 0.2097 (0.14562621061829312) + | > loader_time: 0.0098 (0.010220259085468864) + + + --> TIME: 2024-04-09 16:57:27 -- STEP: 224/1354 -- GLOBAL_STEP: 25950 + | > loss_text_ce: 0.06878499686717987 (0.05309119922042425) + | > loss_mel_ce: 3.7136807441711426 (2.5648697758359567) + | > loss: 0.045029353350400925 (0.0311662026132191) + | > current_lr: 5e-06 + | > step_time: 0.1508 (0.1440788901277951) + | > loader_time: 0.0111 (0.010160573891230993) + + + --> TIME: 2024-04-09 16:57:37 -- STEP: 274/1354 -- GLOBAL_STEP: 26000 + | > loss_text_ce: 0.06059752777218819 (0.05362992060717439) + | > loss_mel_ce: 2.3498282432556152 (2.576587922816729) + | > loss: 0.028695544227957726 (0.0313121177133744) + | > current_lr: 5e-06 + | > step_time: 0.0919 (0.1424739891595214) + | > loader_time: 0.0094 (0.010101193059099851) + + + --> TIME: 2024-04-09 16:58:11 -- STEP: 324/1354 -- GLOBAL_STEP: 26050 + | > loss_text_ce: 0.04461634159088135 (0.05356377872557922) + | > loss_mel_ce: 2.3336946964263916 (2.5711378885640035) + | > loss: 0.028313228860497475 (0.03124644898200109) + | > current_lr: 5e-06 + | > step_time: 0.1391 (0.14380759074364186) + | > loader_time: 0.0095 (0.01011334598800282) + + + --> TIME: 2024-04-09 16:58:21 -- STEP: 374/1354 -- GLOBAL_STEP: 26100 + | > loss_text_ce: 0.04926765337586403 (0.0535435341398187) + | > loss_mel_ce: 2.173691511154175 (2.563598995540232) + | > loss: 0.02646380104124546 (0.0311564592654533) + | > current_lr: 5e-06 + | > step_time: 0.0995 (0.14183074714028268) + | > loader_time: 0.0093 (0.01006654430838192) + + + --> TIME: 2024-04-09 16:58:31 -- STEP: 424/1354 -- GLOBAL_STEP: 26150 + | > loss_text_ce: 0.061341773718595505 (0.05352264688402978) + | > loss_mel_ce: 2.817234516143799 (2.565034309929273) + | > loss: 0.03426876664161682 (0.031173297709275807) + | > current_lr: 5e-06 + | > step_time: 0.0884 (0.14090905043314098) + | > loader_time: 0.0099 (0.010039111915624357) + + + --> TIME: 2024-04-09 16:58:42 -- STEP: 474/1354 -- GLOBAL_STEP: 26200 + | > loss_text_ce: 0.05065886676311493 (0.053252606943469775) + | > loss_mel_ce: 2.2534172534942627 (2.5585375677181204) + | > loss: 0.027429478242993355 (0.031092740732877815) + | > current_lr: 5e-06 + | > step_time: 0.112 (0.1417548928079727) + | > loader_time: 0.0101 (0.010035765321948853) + + + --> TIME: 2024-04-09 16:58:53 -- STEP: 524/1354 -- GLOBAL_STEP: 26250 + | > loss_text_ce: 0.06086019426584244 (0.05313160956147633) + | > loss_mel_ce: 3.062276840209961 (2.558941449827821) + | > loss: 0.037180203944444656 (0.031096108389293192) + | > current_lr: 5e-06 + | > step_time: 0.1009 (0.14159876108169567) + | > loader_time: 0.0095 (0.010054616982700252) + + + --> TIME: 2024-04-09 16:59:04 -- STEP: 574/1354 -- GLOBAL_STEP: 26300 + | > loss_text_ce: 0.06994682550430298 (0.05302656442900463) + | > loss_mel_ce: 3.0189437866210938 (2.555561786536973) + | > loss: 0.036772508174180984 (0.031054623778714742) + | > current_lr: 5e-06 + | > step_time: 0.0799 (0.1419110335539443) + | > loader_time: 0.0105 (0.010021949478973481) + + + --> TIME: 2024-04-09 16:59:15 -- STEP: 624/1354 -- GLOBAL_STEP: 26350 + | > loss_text_ce: 0.04036739841103554 (0.05289544395875568) + | > loss_mel_ce: 2.2159814834594727 (2.5500540804022402) + | > loss: 0.026861296966671944 (0.030987494899771918) + | > current_lr: 5e-06 + | > step_time: 0.1917 (0.14212054892992362) + | > loader_time: 0.0112 (0.009979990430367288) + + + --> TIME: 2024-04-09 16:59:26 -- STEP: 674/1354 -- GLOBAL_STEP: 26400 + | > loss_text_ce: 0.049009498208761215 (0.05300421291790925) + | > loss_mel_ce: 2.260495901107788 (2.551499110654836) + | > loss: 0.027494113892316818 (0.03100599251820178) + | > current_lr: 5e-06 + | > step_time: 0.0788 (0.1419050856230875) + | > loader_time: 0.0089 (0.009942715882547519) + + + --> TIME: 2024-04-09 16:59:36 -- STEP: 724/1354 -- GLOBAL_STEP: 26450 + | > loss_text_ce: 0.03708628565073013 (0.05296684823446593) + | > loss_mel_ce: 2.640584707260132 (2.5493238334497677) + | > loss: 0.03187703713774681 (0.030979651540970606) + | > current_lr: 5e-06 + | > step_time: 0.1635 (0.1419454770193575) + | > loader_time: 0.0096 (0.009925491572743606) + + + --> TIME: 2024-04-09 16:59:47 -- STEP: 774/1354 -- GLOBAL_STEP: 26500 + | > loss_text_ce: 0.043464407324790955 (0.05295285454515647) + | > loss_mel_ce: 2.290121078491211 (2.5542000470543393) + | > loss: 0.02778078056871891 (0.031037535104486678) + | > current_lr: 5e-06 + | > step_time: 0.1414 (0.14207288522744999) + | > loader_time: 0.0091 (0.009904506594635707) + + + --> TIME: 2024-04-09 16:59:58 -- STEP: 824/1354 -- GLOBAL_STEP: 26550 + | > loss_text_ce: 0.06795264035463333 (0.05301094958016161) + | > loss_mel_ce: 3.304486036300659 (2.557827240055047) + | > loss: 0.04014807939529419 (0.031081407575063336) + | > current_lr: 5e-06 + | > step_time: 0.2142 (0.14209298832902653) + | > loader_time: 0.0098 (0.0098870480523526) + + + --> TIME: 2024-04-09 17:00:09 -- STEP: 874/1354 -- GLOBAL_STEP: 26600 + | > loss_text_ce: 0.06849192082881927 (0.052864406338852386) + | > loss_mel_ce: 2.42480206489563 (2.5553556381974123) + | > loss: 0.02968207187950611 (0.031050239186691336) + | > current_lr: 5e-06 + | > step_time: 0.0919 (0.14221430506804464) + | > loader_time: 0.009 (0.009884314351551038) + + + --> TIME: 2024-04-09 17:00:20 -- STEP: 924/1354 -- GLOBAL_STEP: 26650 + | > loss_text_ce: 0.05140931159257889 (0.05290828129978149) + | > loss_mel_ce: 2.3634350299835205 (2.555420697380454) + | > loss: 0.028748147189617157 (0.031051536026164825) + | > current_lr: 5e-06 + | > step_time: 0.102 (0.1425043032282877) + | > loader_time: 0.0094 (0.009870117102866557) + + + --> TIME: 2024-04-09 17:00:31 -- STEP: 974/1354 -- GLOBAL_STEP: 26700 + | > loss_text_ce: 0.040740277618169785 (0.05292503460782754) + | > loss_mel_ce: 2.3829104900360107 (2.5571365413969303) + | > loss: 0.02885298617184162 (0.031072162187350358) + | > current_lr: 5e-06 + | > step_time: 0.1775 (0.14232527499815786) + | > loader_time: 0.0137 (0.009886770033004096) + + + --> TIME: 2024-04-09 17:00:42 -- STEP: 1024/1354 -- GLOBAL_STEP: 26750 + | > loss_text_ce: 0.04406832531094551 (0.0528145694770501) + | > loss_mel_ce: 2.1286983489990234 (2.5548771634930745) + | > loss: 0.025866270065307617 (0.031043949769809842) + | > current_lr: 5e-06 + | > step_time: 0.1623 (0.1426379366312178) + | > loader_time: 0.0116 (0.00989051931537687) + + + --> TIME: 2024-04-09 17:00:53 -- STEP: 1074/1354 -- GLOBAL_STEP: 26800 + | > loss_text_ce: 0.04760916903614998 (0.05270880242043089) + | > loss_mel_ce: 2.7976009845733643 (2.5543317911345196) + | > loss: 0.03387155011296272 (0.03103619811144177) + | > current_lr: 5e-06 + | > step_time: 0.143 (0.1429269222795854) + | > loader_time: 0.0107 (0.009891144612203978) + + + --> TIME: 2024-04-09 17:01:04 -- STEP: 1124/1354 -- GLOBAL_STEP: 26850 + | > loss_text_ce: 0.04360170662403107 (0.05268683355281731) + | > loss_mel_ce: 2.4683706760406494 (2.55167549827345) + | > loss: 0.029904434457421303 (0.031004314034103392) + | > current_lr: 5e-06 + | > step_time: 0.1912 (0.14279428028975524) + | > loader_time: 0.0098 (0.009874868435367565) + + + --> TIME: 2024-04-09 17:01:15 -- STEP: 1174/1354 -- GLOBAL_STEP: 26900 + | > loss_text_ce: 0.04686160385608673 (0.0526566577034493) + | > loss_mel_ce: 2.503141164779663 (2.551350415666831) + | > loss: 0.030357176437973976 (0.031000084783152075) + | > current_lr: 5e-06 + | > step_time: 0.1794 (0.1429442932902244) + | > loader_time: 0.0112 (0.009884993506046526) + + + --> TIME: 2024-04-09 17:01:26 -- STEP: 1224/1354 -- GLOBAL_STEP: 26950 + | > loss_text_ce: 0.04452463611960411 (0.05257110229352475) + | > loss_mel_ce: 2.7820229530334473 (2.5507234200543043) + | > loss: 0.03364937752485275 (0.030991602020971235) + | > current_lr: 5e-06 + | > step_time: 0.1733 (0.1432891252383687) + | > loader_time: 0.009 (0.009887502473943367) + + + --> TIME: 2024-04-09 17:01:37 -- STEP: 1274/1354 -- GLOBAL_STEP: 27000 + | > loss_text_ce: 0.052061375230550766 (0.05253726590038794) + | > loss_mel_ce: 2.732901096343994 (2.5508277640799326) + | > loss: 0.03315431624650955 (0.03099244139106064) + | > current_lr: 5e-06 + | > step_time: 0.1513 (0.1430898264775657) + | > loader_time: 0.0092 (0.009887176368449876) + + + --> TIME: 2024-04-09 17:02:10 -- STEP: 1324/1354 -- GLOBAL_STEP: 27050 + | > loss_text_ce: 0.07375571876764297 (0.052526713274363014) + | > loss_mel_ce: 3.060767889022827 (2.551045250856624) + | > loss: 0.03731575980782509 (0.030994904891579352) + | > current_lr: 5e-06 + | > step_time: 0.0977 (0.1429534954246797) + | > loader_time: 0.0106 (0.009939898176855904) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.028811087975135215 (-0.00650202311002291) + | > avg_loss_text_ce: 0.05655559610861998 (-0.002656968453755751) + | > avg_loss_mel_ce: 2.8732045797201304 (-0.007835516562828548) + | > avg_loss: 2.9297601809868445 (-0.01049245320833636) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_27080.pth + + > EPOCH: 21/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 17:03:10)  + + --> TIME: 2024-04-09 17:03:15 -- STEP: 20/1354 -- GLOBAL_STEP: 27100 + | > loss_text_ce: 0.05497290566563606 (0.05256882645189762) + | > loss_mel_ce: 2.697458505630493 (2.5398702204227446) + | > loss: 0.03276704251766205 (0.030862370505928993) + | > current_lr: 5e-06 + | > step_time: 0.1196 (0.13276757001876832) + | > loader_time: 0.0103 (0.012067127227783202) + + + --> TIME: 2024-04-09 17:03:25 -- STEP: 70/1354 -- GLOBAL_STEP: 27150 + | > loss_text_ce: 0.04102584719657898 (0.05277894256370408) + | > loss_mel_ce: 2.3998937606811523 (2.57323717560087) + | > loss: 0.02905856817960739 (0.031262097427887575) + | > current_lr: 5e-06 + | > step_time: 0.141 (0.13278866154806956) + | > loader_time: 0.0091 (0.01090254443032401) + + + --> TIME: 2024-04-09 17:03:36 -- STEP: 120/1354 -- GLOBAL_STEP: 27200 + | > loss_text_ce: 0.0758633017539978 (0.05176289348552624) + | > loss_mel_ce: 2.74680757522583 (2.5359853138526285) + | > loss: 0.03360322490334511 (0.030806527007371187) + | > current_lr: 5e-06 + | > step_time: 0.0881 (0.13809703191121409) + | > loader_time: 0.01 (0.010650151968002319) + + + --> TIME: 2024-04-09 17:03:47 -- STEP: 170/1354 -- GLOBAL_STEP: 27250 + | > loss_text_ce: 0.04661722108721733 (0.051963543365983404) + | > loss_mel_ce: 2.6756253242492676 (2.546303406182458) + | > loss: 0.032407648861408234 (0.030931750049485878) + | > current_lr: 5e-06 + | > step_time: 0.12 (0.1399918331819421) + | > loader_time: 0.0112 (0.01055105994729435) + + + --> TIME: 2024-04-09 17:03:58 -- STEP: 220/1354 -- GLOBAL_STEP: 27300 + | > loss_text_ce: 0.061824627220630646 (0.05233703164214438) + | > loss_mel_ce: 3.1375443935394287 (2.567863827943802) + | > loss: 0.03808772563934326 (0.0311928679658608) + | > current_lr: 5e-06 + | > step_time: 0.0808 (0.1403063210574062) + | > loader_time: 0.0115 (0.010399350253018465) + + + --> TIME: 2024-04-09 17:04:08 -- STEP: 270/1354 -- GLOBAL_STEP: 27350 + | > loss_text_ce: 0.05622312054038048 (0.052365587389579527) + | > loss_mel_ce: 2.885335683822632 (2.558416576738711) + | > loss: 0.03501855954527855 (0.031080740635041838) + | > current_lr: 5e-06 + | > step_time: 0.1882 (0.138328719139099) + | > loader_time: 0.0093 (0.01031384997897678) + + + --> TIME: 2024-04-09 17:04:19 -- STEP: 320/1354 -- GLOBAL_STEP: 27400 + | > loss_text_ce: 0.06533639132976532 (0.05210346133681013) + | > loss_mel_ce: 3.1716933250427246 (2.5594541341066352) + | > loss: 0.03853606805205345 (0.03108997196541168) + | > current_lr: 5e-06 + | > step_time: 0.0925 (0.13957449644803993) + | > loader_time: 0.0096 (0.010314068198204041) + + + --> TIME: 2024-04-09 17:04:30 -- STEP: 370/1354 -- GLOBAL_STEP: 27450 + | > loss_text_ce: 0.040573600679636 (0.051806701286821744) + | > loss_mel_ce: 2.5346577167510986 (2.5548699291976713) + | > loss: 0.030657516792416573 (0.03103186526914706) + | > current_lr: 5e-06 + | > step_time: 0.2137 (0.14095753399101457) + | > loader_time: 0.0094 (0.01031022136275833) + + + --> TIME: 2024-04-09 17:04:41 -- STEP: 420/1354 -- GLOBAL_STEP: 27500 + | > loss_text_ce: 0.03573840484023094 (0.05169087691853443) + | > loss_mel_ce: 2.3176722526550293 (2.5482085980120157) + | > loss: 0.028016794472932816 (0.030951184845928636) + | > current_lr: 5e-06 + | > step_time: 0.1195 (0.14142315841856445) + | > loader_time: 0.0088 (0.010278278873080299) + + + --> TIME: 2024-04-09 17:04:52 -- STEP: 470/1354 -- GLOBAL_STEP: 27550 + | > loss_text_ce: 0.035973209887742996 (0.05145050165976615) + | > loss_mel_ce: 2.515995979309082 (2.5398743776564903) + | > loss: 0.03038058802485466 (0.030849106340332236) + | > current_lr: 5e-06 + | > step_time: 0.1848 (0.14110504972173804) + | > loader_time: 0.0091 (0.01024837443169127) + + + --> TIME: 2024-04-09 17:05:03 -- STEP: 520/1354 -- GLOBAL_STEP: 27600 + | > loss_text_ce: 0.03493235632777214 (0.05139604474489505) + | > loss_mel_ce: 1.893621563911438 (2.5336431943453275) + | > loss: 0.022958975285291672 (0.03077427728029971) + | > current_lr: 5e-06 + | > step_time: 0.1389 (0.14174171640322752) + | > loader_time: 0.0095 (0.01020427208680373) + + + --> TIME: 2024-04-09 17:05:14 -- STEP: 570/1354 -- GLOBAL_STEP: 27650 + | > loss_text_ce: 0.037516336888074875 (0.051378588182361504) + | > loss_mel_ce: 2.235837697982788 (2.5377629029123407) + | > loss: 0.02706373855471611 (0.0308231136255097) + | > current_lr: 5e-06 + | > step_time: 0.1615 (0.1417539358139038) + | > loader_time: 0.012 (0.010173227912501286) + + + --> TIME: 2024-04-09 17:05:25 -- STEP: 620/1354 -- GLOBAL_STEP: 27700 + | > loss_text_ce: 0.05661612004041672 (0.05125260815144544) + | > loss_mel_ce: 2.3647029399871826 (2.536467751187663) + | > loss: 0.028825227171182632 (0.03080619541147063) + | > current_lr: 5e-06 + | > step_time: 0.1196 (0.1426201916510059) + | > loader_time: 0.011 (0.010163599060427759) + + + --> TIME: 2024-04-09 17:05:35 -- STEP: 670/1354 -- GLOBAL_STEP: 27750 + | > loss_text_ce: 0.056601960211992264 (0.051215364617198236) + | > loss_mel_ce: 3.4794867038726807 (2.5311643483033843) + | > loss: 0.04209629446268082 (0.03074261628980957) + | > current_lr: 5e-06 + | > step_time: 0.1176 (0.141949379266198) + | > loader_time: 0.0087 (0.010125369812125588) + + + --> TIME: 2024-04-09 17:05:47 -- STEP: 720/1354 -- GLOBAL_STEP: 27800 + | > loss_text_ce: 0.047711875289678574 (0.0511201507049716) + | > loss_mel_ce: 2.913557291030884 (2.5284806441929604) + | > loss: 0.03525320440530777 (0.030709533934067522) + | > current_lr: 5e-06 + | > step_time: 0.1978 (0.1425544927517573) + | > loader_time: 0.0092 (0.010100333889325458) + + + --> TIME: 2024-04-09 17:05:57 -- STEP: 770/1354 -- GLOBAL_STEP: 27850 + | > loss_text_ce: 0.05437289923429489 (0.05123115890211871) + | > loss_mel_ce: 2.4150166511535645 (2.531152083657008) + | > loss: 0.029397493228316307 (0.03074265829493086) + | > current_lr: 5e-06 + | > step_time: 0.1 (0.14184863536388836) + | > loader_time: 0.0087 (0.010073790612158833) + + + --> TIME: 2024-04-09 17:06:08 -- STEP: 820/1354 -- GLOBAL_STEP: 27900 + | > loss_text_ce: 0.04687372222542763 (0.05121012161735716) + | > loss_mel_ce: 3.0755372047424316 (2.5370522155994357) + | > loss: 0.037171561270952225 (0.030812647491239194) + | > current_lr: 5e-06 + | > step_time: 0.1402 (0.14207985953586852) + | > loader_time: 0.0099 (0.010056992856467635) + + + --> TIME: 2024-04-09 17:06:19 -- STEP: 870/1354 -- GLOBAL_STEP: 27950 + | > loss_text_ce: 0.046701449900865555 (0.051086430403607316) + | > loss_mel_ce: 2.0054025650024414 (2.5339151751035964) + | > loss: 0.0244298093020916 (0.030773829268398643) + | > current_lr: 5e-06 + | > step_time: 0.1194 (0.1425890799226433) + | > loader_time: 0.0095 (0.010037542485642701) + + + --> TIME: 2024-04-09 17:06:30 -- STEP: 920/1354 -- GLOBAL_STEP: 28000 + | > loss_text_ce: 0.048876214772462845 (0.051100838854504005) + | > loss_mel_ce: 2.314159393310547 (2.5313136756420165) + | > loss: 0.028131376951932907 (0.030743030557894836) + | > current_lr: 5e-06 + | > step_time: 0.1738 (0.14240505980408708) + | > loader_time: 0.0098 (0.010023535852846887) + + + --> TIME: 2024-04-09 17:07:05 -- STEP: 970/1354 -- GLOBAL_STEP: 28050 + | > loss_text_ce: 0.050339873880147934 (0.05108873981828848) + | > loss_mel_ce: 2.7719430923461914 (2.5292656457301295) + | > loss: 0.033598609268665314 (0.030718505223156865) + | > current_lr: 5e-06 + | > step_time: 0.1158 (0.14267254318158648) + | > loader_time: 0.0093 (0.010062888971309067) + + + --> TIME: 2024-04-09 17:07:16 -- STEP: 1020/1354 -- GLOBAL_STEP: 28100 + | > loss_text_ce: 0.05210743844509125 (0.051043288824239766) + | > loss_mel_ce: 2.6120102405548096 (2.529268379772413) + | > loss: 0.03171568736433983 (0.030717996690495342) + | > current_lr: 5e-06 + | > step_time: 0.2031 (0.14310022592544552) + | > loader_time: 0.0104 (0.010079063387478094) + + + --> TIME: 2024-04-09 17:07:27 -- STEP: 1070/1354 -- GLOBAL_STEP: 28150 + | > loss_text_ce: 0.03742188215255737 (0.05096939088271877) + | > loss_mel_ce: 2.365220546722412 (2.5295046970108985) + | > loss: 0.02860288694500923 (0.030719930239926987) + | > current_lr: 5e-06 + | > step_time: 0.1563 (0.14315950491718044) + | > loader_time: 0.0114 (0.010137772782940728) + + + --> TIME: 2024-04-09 17:07:39 -- STEP: 1120/1354 -- GLOBAL_STEP: 28200 + | > loss_text_ce: 0.05722901225090027 (0.050849697145167724) + | > loss_mel_ce: 3.3618037700653076 (2.5286228513079054) + | > loss: 0.04070277139544487 (0.030708007160241583) + | > current_lr: 5e-06 + | > step_time: 0.2182 (0.14364091392074307) + | > loader_time: 0.0188 (0.010172503760882785) + + + --> TIME: 2024-04-09 17:07:49 -- STEP: 1170/1354 -- GLOBAL_STEP: 28250 + | > loss_text_ce: 0.06285523623228073 (0.05091178932219233) + | > loss_mel_ce: 2.7956197261810303 (2.5275380319000327) + | > loss: 0.03402946516871452 (0.030695831820241406) + | > current_lr: 5e-06 + | > step_time: 0.0876 (0.1431278291930499) + | > loader_time: 0.0087 (0.010161874233148037) + + + --> TIME: 2024-04-09 17:07:59 -- STEP: 1220/1354 -- GLOBAL_STEP: 28300 + | > loss_text_ce: 0.050371959805488586 (0.050876775620596795) + | > loss_mel_ce: 3.1828067302703857 (2.5238139955723877) + | > loss: 0.03849022090435028 (0.030651081228231798) + | > current_lr: 5e-06 + | > step_time: 0.0885 (0.14299873993044984) + | > loader_time: 0.0091 (0.010149296385342954) + + + --> TIME: 2024-04-09 17:08:10 -- STEP: 1270/1354 -- GLOBAL_STEP: 28350 + | > loss_text_ce: 0.054240882396698 (0.05076463370488618) + | > loss_mel_ce: 2.751080274581909 (2.521460847685657) + | > loss: 0.033396683633327484 (0.030621732533799382) + | > current_lr: 5e-06 + | > step_time: 0.0811 (0.14313817193189007) + | > loader_time: 0.009 (0.010135378612308047) + + + --> TIME: 2024-04-09 17:08:21 -- STEP: 1320/1354 -- GLOBAL_STEP: 28400 + | > loss_text_ce: 0.06274570524692535 (0.050788979654228586) + | > loss_mel_ce: 3.000922441482544 (2.5231749022548864) + | > loss: 0.03647224232554436 (0.03064242777640395) + | > current_lr: 5e-06 + | > step_time: 0.1066 (0.14286966721216843) + | > loader_time: 0.0095 (0.010117650032043441) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.029329593365009014 (+0.0005185053898737989) + | > avg_loss_text_ce: 0.05453396617219998 (-0.002021629936420001) + | > avg_loss_mel_ce: 2.869131161616399 (-0.00407341810373163) + | > avg_loss: 2.923665138391348 (-0.006095042595496647) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_28434.pth + + > EPOCH: 22/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 17:09:22)  + + --> TIME: 2024-04-09 17:09:27 -- STEP: 16/1354 -- GLOBAL_STEP: 28450 + | > loss_text_ce: 0.04155205935239792 (0.05137653835117817) + | > loss_mel_ce: 2.3599915504455566 (2.424931950867176) + | > loss: 0.028589805588126183 (0.02947986323852092) + | > current_lr: 5e-06 + | > step_time: 0.1594 (0.14852406084537506) + | > loader_time: 0.0098 (0.012247413396835327) + + + --> TIME: 2024-04-09 17:09:38 -- STEP: 66/1354 -- GLOBAL_STEP: 28500 + | > loss_text_ce: 0.045016322284936905 (0.04883358819466649) + | > loss_mel_ce: 2.300650119781494 (2.5035505023869593) + | > loss: 0.02792460098862648 (0.030385525502038723) + | > current_lr: 5e-06 + | > step_time: 0.2129 (0.14887806863495798) + | > loader_time: 0.0101 (0.011351065202192827) + + + --> TIME: 2024-04-09 17:09:49 -- STEP: 116/1354 -- GLOBAL_STEP: 28550 + | > loss_text_ce: 0.04818502441048622 (0.04926742600469754) + | > loss_mel_ce: 2.281191110610962 (2.507408488413383) + | > loss: 0.02773066982626915 (0.030436618590791678) + | > current_lr: 5e-06 + | > step_time: 0.1371 (0.14797871893849865) + | > loader_time: 0.0091 (0.010797060769179771) + + + --> TIME: 2024-04-09 17:10:00 -- STEP: 166/1354 -- GLOBAL_STEP: 28600 + | > loss_text_ce: 0.04906228929758072 (0.05031506328966962) + | > loss_mel_ce: 2.7205164432525635 (2.5521859713347554) + | > loss: 0.032971177250146866 (0.030982155846544058) + | > current_lr: 5e-06 + | > step_time: 0.1501 (0.1444398179111711) + | > loader_time: 0.0096 (0.010658898985529517) + + + --> TIME: 2024-04-09 17:10:10 -- STEP: 216/1354 -- GLOBAL_STEP: 28650 + | > loss_text_ce: 0.06361638754606247 (0.05039620213210583) + | > loss_mel_ce: 3.4278666973114014 (2.582107769118415) + | > loss: 0.04156527295708656 (0.03133933362551033) + | > current_lr: 5e-06 + | > step_time: 0.1451 (0.1432437234454684) + | > loader_time: 0.0099 (0.010603539369724413) + + + --> TIME: 2024-04-09 17:10:21 -- STEP: 266/1354 -- GLOBAL_STEP: 28700 + | > loss_text_ce: 0.04877587780356407 (0.050682864999188525) + | > loss_mel_ce: 2.1298611164093018 (2.581093288005743) + | > loss: 0.025936154648661613 (0.03133066909569769) + | > current_lr: 5e-06 + | > step_time: 0.1652 (0.14266591771204662) + | > loader_time: 0.0094 (0.010543770359871082) + + + --> TIME: 2024-04-09 17:10:31 -- STEP: 316/1354 -- GLOBAL_STEP: 28750 + | > loss_text_ce: 0.06424036622047424 (0.05050957465803701) + | > loss_mel_ce: 2.7122175693511963 (2.5607676283468166) + | > loss: 0.03305307403206825 (0.031086633983952335) + | > current_lr: 5e-06 + | > step_time: 0.0772 (0.14232113542436023) + | > loader_time: 0.0093 (0.010516060304038132) + + + --> TIME: 2024-04-09 17:10:43 -- STEP: 366/1354 -- GLOBAL_STEP: 28800 + | > loss_text_ce: 0.044683489948511124 (0.050125814775829454) + | > loss_mel_ce: 2.8233401775360107 (2.549007356818255) + | > loss: 0.03414313867688179 (0.030942062165364215) + | > current_lr: 5e-06 + | > step_time: 0.1921 (0.14289840257884362) + | > loader_time: 0.0116 (0.01054917463187964) + + + --> TIME: 2024-04-09 17:10:53 -- STEP: 416/1354 -- GLOBAL_STEP: 28850 + | > loss_text_ce: 0.033805523067712784 (0.05030378218865597) + | > loss_mel_ce: 2.5184786319732666 (2.5544262537016302) + | > loss: 0.030384337529540062 (0.031008691500084333) + | > current_lr: 5e-06 + | > step_time: 0.1923 (0.1428350107027933) + | > loader_time: 0.0117 (0.010557159208334417) + + + --> TIME: 2024-04-09 17:11:04 -- STEP: 466/1354 -- GLOBAL_STEP: 28900 + | > loss_text_ce: 0.05104072764515877 (0.050094999996505594) + | > loss_mel_ce: 2.539067029953003 (2.5473978816183838) + | > loss: 0.030834615230560303 (0.030922534899554283) + | > current_lr: 5e-06 + | > step_time: 0.1289 (0.14294719849533266) + | > loader_time: 0.0122 (0.010522855709550727) + + + --> TIME: 2024-04-09 17:11:15 -- STEP: 516/1354 -- GLOBAL_STEP: 28950 + | > loss_text_ce: 0.040564488619565964 (0.05000218123832878) + | > loss_mel_ce: 3.2056994438171387 (2.538866262565286) + | > loss: 0.03864600136876106 (0.030819863005173066) + | > current_lr: 5e-06 + | > step_time: 0.1915 (0.1432115255400191) + | > loader_time: 0.0096 (0.010483241820520215) + + + --> TIME: 2024-04-09 17:11:26 -- STEP: 566/1354 -- GLOBAL_STEP: 29000 + | > loss_text_ce: 0.06033160537481308 (0.04986490255925977) + | > loss_mel_ce: 2.852710008621216 (2.5273291746634885) + | > loss: 0.03467906638979912 (0.030680882458183453) + | > current_lr: 5e-06 + | > step_time: 0.1333 (0.1430311236701668) + | > loader_time: 0.0107 (0.010467555834632042) + + + --> TIME: 2024-04-09 17:12:01 -- STEP: 616/1354 -- GLOBAL_STEP: 29050 + | > loss_text_ce: 0.06520570069551468 (0.04974973497476867) + | > loss_mel_ce: 3.2927253246307373 (2.521837519360826) + | > loss: 0.0399753712117672 (0.03061413456528605) + | > current_lr: 5e-06 + | > step_time: 0.1307 (0.14304720233013093) + | > loader_time: 0.0095 (0.010564278472553602) + + + --> TIME: 2024-04-09 17:12:11 -- STEP: 666/1354 -- GLOBAL_STEP: 29100 + | > loss_text_ce: 0.03741397708654404 (0.049599091606298544) + | > loss_mel_ce: 2.7335128784179688 (2.5171715652978466) + | > loss: 0.03298722580075264 (0.030556794135711034) + | > current_lr: 5e-06 + | > step_time: 0.1134 (0.14304810708707516) + | > loader_time: 0.0096 (0.010543528619829243) + + + --> TIME: 2024-04-09 17:12:22 -- STEP: 716/1354 -- GLOBAL_STEP: 29150 + | > loss_text_ce: 0.046929288655519485 (0.04950372412717542) + | > loss_mel_ce: 2.6337976455688477 (2.509961993667669) + | > loss: 0.03191341832280159 (0.03046983056935851) + | > current_lr: 5e-06 + | > step_time: 0.2151 (0.14319617555128122) + | > loader_time: 0.0098 (0.010520338013185467) + + + --> TIME: 2024-04-09 17:12:33 -- STEP: 766/1354 -- GLOBAL_STEP: 29200 + | > loss_text_ce: 0.04303174465894699 (0.04954936748307305) + | > loss_mel_ce: 1.9764480590820312 (2.5132737503661833) + | > loss: 0.024041425436735153 (0.030509799609432488) + | > current_lr: 5e-06 + | > step_time: 0.1049 (0.14292632599723545) + | > loader_time: 0.0092 (0.01051606802031514) + + + --> TIME: 2024-04-09 17:12:44 -- STEP: 816/1354 -- GLOBAL_STEP: 29250 + | > loss_text_ce: 0.0570584237575531 (0.049464693206234596) + | > loss_mel_ce: 2.6116464138031006 (2.5086275397562536) + | > loss: 0.03177029639482498 (0.03045347953607858) + | > current_lr: 5e-06 + | > step_time: 0.1511 (0.14281028713665747) + | > loader_time: 0.0103 (0.010521284797612353) + + + --> TIME: 2024-04-09 17:12:54 -- STEP: 866/1354 -- GLOBAL_STEP: 29300 + | > loss_text_ce: 0.06337586790323257 (0.04948791288114723) + | > loss_mel_ce: 1.9736220836639404 (2.5055631827261946) + | > loss: 0.024249976500868797 (0.03041727552685121) + | > current_lr: 5e-06 + | > step_time: 0.0875 (0.14255943254290218) + | > loader_time: 0.0094 (0.010525613403760804) + + + --> TIME: 2024-04-09 17:13:06 -- STEP: 916/1354 -- GLOBAL_STEP: 29350 + | > loss_text_ce: 0.04048249125480652 (0.0493503990481449) + | > loss_mel_ce: 2.451526403427124 (2.4991074204444903) + | > loss: 0.029666773974895477 (0.03033878413480442) + | > current_lr: 5e-06 + | > step_time: 0.1517 (0.1433673787325229) + | > loader_time: 0.0097 (0.010527159709597255) + + + --> TIME: 2024-04-09 17:13:17 -- STEP: 966/1354 -- GLOBAL_STEP: 29400 + | > loss_text_ce: 0.0373324379324913 (0.049365388908217164) + | > loss_mel_ce: 2.1622872352600098 (2.4940798929265835) + | > loss: 0.02618595026433468 (0.030279111065063046) + | > current_lr: 5e-06 + | > step_time: 0.1528 (0.14304747813483437) + | > loader_time: 0.012 (0.010518632320143417) + + + --> TIME: 2024-04-09 17:13:27 -- STEP: 1016/1354 -- GLOBAL_STEP: 29450 + | > loss_text_ce: 0.05131224915385246 (0.04935352902448788) + | > loss_mel_ce: 3.1450235843658447 (2.4897295276480413) + | > loss: 0.03805161640048027 (0.03022717981503939) + | > current_lr: 5e-06 + | > step_time: 0.1644 (0.14276812747707507) + | > loader_time: 0.012 (0.010492283058917425) + + + --> TIME: 2024-04-09 17:13:38 -- STEP: 1066/1354 -- GLOBAL_STEP: 29500 + | > loss_text_ce: 0.04649174213409424 (0.04939413591031175) + | > loss_mel_ce: 2.000645637512207 (2.4915931289683604) + | > loss: 0.02437068149447441 (0.030249848958335497) + | > current_lr: 5e-06 + | > step_time: 0.0701 (0.1424524500043485) + | > loader_time: 0.0093 (0.010474577406333929) + + + --> TIME: 2024-04-09 17:13:49 -- STEP: 1116/1354 -- GLOBAL_STEP: 29550 + | > loss_text_ce: 0.05433270335197449 (0.04935820370338404) + | > loss_mel_ce: 1.7680981159210205 (2.4889115875980785) + | > loss: 0.021695606410503387 (0.030217498083562194) + | > current_lr: 5e-06 + | > step_time: 0.0918 (0.14274355972112285) + | > loader_time: 0.0093 (0.010464317268795428) + + + --> TIME: 2024-04-09 17:14:00 -- STEP: 1166/1354 -- GLOBAL_STEP: 29600 + | > loss_text_ce: 0.04045595973730087 (0.04931455264171495) + | > loss_mel_ce: 2.031860113143921 (2.485533650804755) + | > loss: 0.0246704313904047 (0.03017676489085828) + | > current_lr: 5e-06 + | > step_time: 0.1602 (0.14270172315437213) + | > loader_time: 0.01 (0.010453140960549402) + + + --> TIME: 2024-04-09 17:14:10 -- STEP: 1216/1354 -- GLOBAL_STEP: 29650 + | > loss_text_ce: 0.0661776214838028 (0.0494033447941643) + | > loss_mel_ce: 2.621962308883667 (2.4875674202646096) + | > loss: 0.032001666724681854 (0.030202033480715106) + | > current_lr: 5e-06 + | > step_time: 0.1505 (0.14255909700142713) + | > loader_time: 0.0097 (0.010438368312622369) + + + --> TIME: 2024-04-09 17:14:21 -- STEP: 1266/1354 -- GLOBAL_STEP: 29700 + | > loss_text_ce: 0.037638913840055466 (0.04935932672460452) + | > loss_mel_ce: 1.6115907430648804 (2.484569461518933) + | > loss: 0.019633686169981956 (0.030165819467397003) + | > current_lr: 5e-06 + | > step_time: 0.1334 (0.14260121092412145) + | > loader_time: 0.01 (0.010423233347104814) + + + --> TIME: 2024-04-09 17:14:32 -- STEP: 1316/1354 -- GLOBAL_STEP: 29750 + | > loss_text_ce: 0.04183771088719368 (0.04933268772891089) + | > loss_mel_ce: 1.8575091361999512 (2.4825796670645577) + | > loss: 0.02261127158999443 (0.0301418142933908) + | > current_lr: 5e-06 + | > step_time: 0.0814 (0.14236797895112668) + | > loader_time: 0.0089 (0.010414103968889876) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.02705097198486328 (-0.0022786213801457324) + | > avg_loss_text_ce: 0.0530245562012379 (-0.0015094099709620779) + | > avg_loss_mel_ce: 2.8616642218369703 (-0.007466939779428472) + | > avg_loss: 2.9146887889275184 (-0.008976349463829525) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_29788.pth + + > EPOCH: 23/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 17:15:31)  + + --> TIME: 2024-04-09 17:15:34 -- STEP: 12/1354 -- GLOBAL_STEP: 29800 + | > loss_text_ce: 0.07331594079732895 (0.04820427795251211) + | > loss_mel_ce: 2.4561655521392822 (2.4911958376566568) + | > loss: 0.030112873762845993 (0.03023095444465677) + | > current_lr: 5e-06 + | > step_time: 0.1131 (0.1440647840499878) + | > loader_time: 0.0127 (0.011847078800201416) + + + --> TIME: 2024-04-09 17:15:45 -- STEP: 62/1354 -- GLOBAL_STEP: 29850 + | > loss_text_ce: 0.04390731081366539 (0.0476952314737343) + | > loss_mel_ce: 2.7386560440063477 (2.46974709726149) + | > loss: 0.033125754445791245 (0.029969551961027806) + | > current_lr: 5e-06 + | > step_time: 0.207 (0.14356854654127557) + | > loader_time: 0.0096 (0.010773551079534715) + + + --> TIME: 2024-04-09 17:15:56 -- STEP: 112/1354 -- GLOBAL_STEP: 29900 + | > loss_text_ce: 0.04575784131884575 (0.048325534045164084) + | > loss_mel_ce: 2.415867328643799 (2.4719108140894344) + | > loss: 0.02930506132543087 (0.03000281408562192) + | > current_lr: 5e-06 + | > step_time: 0.1733 (0.13988877194268365) + | > loader_time: 0.0099 (0.010688702975000653) + + + --> TIME: 2024-04-09 17:16:06 -- STEP: 162/1354 -- GLOBAL_STEP: 29950 + | > loss_text_ce: 0.04720240458846092 (0.048074258031484525) + | > loss_mel_ce: 1.775134801864624 (2.4288372684408115) + | > loss: 0.021694490686058998 (0.02948704242338369) + | > current_lr: 5e-06 + | > step_time: 0.1589 (0.13920360288502265) + | > loader_time: 0.0097 (0.010753037017068742) + + + --> TIME: 2024-04-09 17:16:17 -- STEP: 212/1354 -- GLOBAL_STEP: 30000 + | > loss_text_ce: 0.05879974365234375 (0.0482709541178818) + | > loss_mel_ce: 2.0399506092071533 (2.442469321331889) + | > loss: 0.024985123425722122 (0.029651670453121077) + | > current_lr: 5e-06 + | > step_time: 0.0796 (0.13874788891594364) + | > loader_time: 0.0097 (0.010682339938181748) + + + > CHECKPOINT : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/checkpoint_30000.pth + + --> TIME: 2024-04-09 17:17:07 -- STEP: 262/1354 -- GLOBAL_STEP: 30050 + | > loss_text_ce: 0.05175181105732918 (0.04818560474530886) + | > loss_mel_ce: 2.317930221557617 (2.427039192378067) + | > loss: 0.028210502117872238 (0.0294669623493807) + | > current_lr: 5e-06 + | > step_time: 0.1542 (0.14048036455198099) + | > loader_time: 0.0137 (0.010775989248552395) + + + --> TIME: 2024-04-09 17:17:17 -- STEP: 312/1354 -- GLOBAL_STEP: 30100 + | > loss_text_ce: 0.05699543654918671 (0.04819011744350578) + | > loss_mel_ce: 3.0814225673675537 (2.430616768888939) + | > loss: 0.03736212104558945 (0.029509606317449838) + | > current_lr: 5e-06 + | > step_time: 0.1143 (0.13943165999192456) + | > loader_time: 0.0107 (0.010707441812906513) + + + --> TIME: 2024-04-09 17:17:28 -- STEP: 362/1354 -- GLOBAL_STEP: 30150 + | > loss_text_ce: 0.045182134956121445 (0.0484691689315751) + | > loss_mel_ce: 2.8007524013519287 (2.4384162175721213) + | > loss: 0.03388017416000366 (0.029605778947843043) + | > current_lr: 5e-06 + | > step_time: 0.1007 (0.13955631769823104) + | > loader_time: 0.0092 (0.010706802099449205) + + + --> TIME: 2024-04-09 17:17:39 -- STEP: 412/1354 -- GLOBAL_STEP: 30200 + | > loss_text_ce: 0.03223872557282448 (0.048138276158461286) + | > loss_mel_ce: 2.2159230709075928 (2.4260372325633313) + | > loss: 0.026763832196593285 (0.029454470901451642) + | > current_lr: 5e-06 + | > step_time: 0.1599 (0.1400336686847279) + | > loader_time: 0.0094 (0.010683649951971857) + + + --> TIME: 2024-04-09 17:17:50 -- STEP: 462/1354 -- GLOBAL_STEP: 30250 + | > loss_text_ce: 0.05765852704644203 (0.048226307642150225) + | > loss_mel_ce: 2.752317190170288 (2.436781189142366) + | > loss: 0.03345208987593651 (0.029583423104121057) + | > current_lr: 5e-06 + | > step_time: 0.082 (0.140371102791328) + | > loader_time: 0.0112 (0.01070590917166178) + + + --> TIME: 2024-04-09 17:18:00 -- STEP: 512/1354 -- GLOBAL_STEP: 30300 + | > loss_text_ce: 0.0596223920583725 (0.048329524266591754) + | > loss_mel_ce: 2.937191963195801 (2.444039933616294) + | > loss: 0.03567636013031006 (0.029671065487491433) + | > current_lr: 5e-06 + | > step_time: 0.1007 (0.14062068937346345) + | > loader_time: 0.0092 (0.010659588500857358) + + + --> TIME: 2024-04-09 17:18:11 -- STEP: 562/1354 -- GLOBAL_STEP: 30350 + | > loss_text_ce: 0.05543510615825653 (0.04849353855561956) + | > loss_mel_ce: 2.9550182819366455 (2.4504301357099583) + | > loss: 0.03583873063325882 (0.02974909186045046) + | > current_lr: 5e-06 + | > step_time: 0.1114 (0.14008754216055008) + | > loader_time: 0.0115 (0.010588787628662546) + + + --> TIME: 2024-04-09 17:18:21 -- STEP: 612/1354 -- GLOBAL_STEP: 30400 + | > loss_text_ce: 0.03165581449866295 (0.048516233689060405) + | > loss_mel_ce: 2.3626108169555664 (2.4576572803110848) + | > loss: 0.02850317396223545 (0.029835399465477155) + | > current_lr: 5e-06 + | > step_time: 0.2071 (0.13995482563193334) + | > loader_time: 0.0097 (0.010537787590151528) + + + --> TIME: 2024-04-09 17:18:32 -- STEP: 662/1354 -- GLOBAL_STEP: 30450 + | > loss_text_ce: 0.057900819927453995 (0.04847701888227573) + | > loss_mel_ce: 3.0419600009918213 (2.461953167288683) + | > loss: 0.036903105676174164 (0.0298860741362786) + | > current_lr: 5e-06 + | > step_time: 0.1493 (0.14005692127608096) + | > loader_time: 0.0103 (0.010533870527031211) + + + --> TIME: 2024-04-09 17:18:43 -- STEP: 712/1354 -- GLOBAL_STEP: 30500 + | > loss_text_ce: 0.05444399267435074 (0.04851292504855758) + | > loss_mel_ce: 2.2186641693115234 (2.4677512699298636) + | > loss: 0.02706081233918667 (0.029955526635971633) + | > current_lr: 5e-06 + | > step_time: 0.1158 (0.13998461908169005) + | > loader_time: 0.0094 (0.010504219639167369) + + + --> TIME: 2024-04-09 17:18:54 -- STEP: 762/1354 -- GLOBAL_STEP: 30550 + | > loss_text_ce: 0.04189595207571983 (0.048488866088036806) + | > loss_mel_ce: 1.8164241313934326 (2.4643975468132435) + | > loss: 0.022122858092188835 (0.029915314937729066) + | > current_lr: 5e-06 + | > step_time: 0.1598 (0.14085384647990148) + | > loader_time: 0.0103 (0.010518719205080377) + + + --> TIME: 2024-04-09 17:19:05 -- STEP: 812/1354 -- GLOBAL_STEP: 30600 + | > loss_text_ce: 0.04528134688735008 (0.04850298018089218) + | > loss_mel_ce: 2.1492362022399902 (2.46031621729799) + | > loss: 0.026125209406018257 (0.029866895707223215) + | > current_lr: 5e-06 + | > step_time: 0.163 (0.14102978952999765) + | > loader_time: 0.0102 (0.010542027468751812) + + + --> TIME: 2024-04-09 17:19:16 -- STEP: 862/1354 -- GLOBAL_STEP: 30650 + | > loss_text_ce: 0.041489098221063614 (0.048441354667210404) + | > loss_mel_ce: 1.7783513069152832 (2.4639431507847425) + | > loss: 0.02166476659476757 (0.02990933985010723) + | > current_lr: 5e-06 + | > step_time: 0.1515 (0.14102621891658976) + | > loader_time: 0.0105 (0.010541979785308492) + + + --> TIME: 2024-04-09 17:19:27 -- STEP: 912/1354 -- GLOBAL_STEP: 30700 + | > loss_text_ce: 0.041760385036468506 (0.048400690378784625) + | > loss_mel_ce: 2.1116397380828857 (2.462994761895715) + | > loss: 0.025635717436671257 (0.029897565401992517) + | > current_lr: 5e-06 + | > step_time: 0.13 (0.14127957115047837) + | > loader_time: 0.0101 (0.010562209967981313) + + + --> TIME: 2024-04-09 17:19:38 -- STEP: 962/1354 -- GLOBAL_STEP: 30750 + | > loss_text_ce: 0.07025017589330673 (0.04836672961541497) + | > loss_mel_ce: 3.406278610229492 (2.4627550200959996) + | > loss: 0.041387248784303665 (0.029894307044696783) + | > current_lr: 5e-06 + | > step_time: 0.1051 (0.14154846479887756) + | > loader_time: 0.009 (0.010577123031298988) + + + --> TIME: 2024-04-09 17:19:50 -- STEP: 1012/1354 -- GLOBAL_STEP: 30800 + | > loss_text_ce: 0.034156035631895065 (0.04815449807866526) + | > loss_mel_ce: 2.4145870208740234 (2.4551798746048696) + | > loss: 0.029151704162359238 (0.029801600172806517) + | > current_lr: 5e-06 + | > step_time: 0.1186 (0.14222433821485903) + | > loader_time: 0.0091 (0.010583967088239473) + + + --> TIME: 2024-04-09 17:20:00 -- STEP: 1062/1354 -- GLOBAL_STEP: 30850 + | > loss_text_ce: 0.05368449538946152 (0.04826649308569463) + | > loss_mel_ce: 2.5796091556549072 (2.458413630443329) + | > loss: 0.031348735094070435 (0.029841430551896888) + | > current_lr: 5e-06 + | > step_time: 0.1317 (0.14220918010632633) + | > loader_time: 0.0115 (0.010578616639764078) + + + --> TIME: 2024-04-09 17:20:11 -- STEP: 1112/1354 -- GLOBAL_STEP: 30900 + | > loss_text_ce: 0.044379301369190216 (0.04818275957803825) + | > loss_mel_ce: 2.2097127437591553 (2.4540551258720087) + | > loss: 0.026834428310394287 (0.02978854676835882) + | > current_lr: 5e-06 + | > step_time: 0.1775 (0.1422734376337887) + | > loader_time: 0.0114 (0.010583422166838066) + + + --> TIME: 2024-04-09 17:20:21 -- STEP: 1162/1354 -- GLOBAL_STEP: 30950 + | > loss_text_ce: 0.05667150393128395 (0.048186210944299646) + | > loss_mel_ce: 3.317723035812378 (2.4529365286769655) + | > loss: 0.040171366184949875 (0.029775271225855292) + | > current_lr: 5e-06 + | > step_time: 0.1576 (0.14183149957410626) + | > loader_time: 0.0093 (0.010572275893823624) + + + --> TIME: 2024-04-09 17:20:33 -- STEP: 1212/1354 -- GLOBAL_STEP: 31000 + | > loss_text_ce: 0.0574369803071022 (0.04812340745118389) + | > loss_mel_ce: 2.7293496131896973 (2.4530025568142206) + | > loss: 0.033176030963659286 (0.029775309619238667) + | > current_lr: 5e-06 + | > step_time: 0.1121 (0.14197891852249775) + | > loader_time: 0.0098 (0.010577685565444894) + + + --> TIME: 2024-04-09 17:21:06 -- STEP: 1262/1354 -- GLOBAL_STEP: 31050 + | > loss_text_ce: 0.04844392091035843 (0.04803874089180386) + | > loss_mel_ce: 2.190246105194092 (2.4532441251251487) + | > loss: 0.02665107324719429 (0.029777177492903453) + | > current_lr: 5e-06 + | > step_time: 0.0648 (0.14206708025441087) + | > loader_time: 0.0106 (0.010604519851612022) + + + --> TIME: 2024-04-09 17:21:16 -- STEP: 1312/1354 -- GLOBAL_STEP: 31100 + | > loss_text_ce: 0.04932029917836189 (0.04798842247904893) + | > loss_mel_ce: 1.5500319004058838 (2.4534011094308505) + | > loss: 0.019039908424019814 (0.02977844732728355) + | > current_lr: 5e-06 + | > step_time: 0.0933 (0.14185564637911005) + | > loader_time: 0.0096 (0.010608865720469773) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03855285277733436 (+0.01150188079247108) + | > avg_loss_text_ce: 0.052023981912777975 (-0.0010005742884599267) + | > avg_loss_mel_ce: 2.8602935717656064 (-0.0013706500713639436) + | > avg_loss: 2.912317551099337 (-0.002371237828181272) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_31142.pth + + > EPOCH: 24/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 17:22:18)  + + --> TIME: 2024-04-09 17:22:22 -- STEP: 8/1354 -- GLOBAL_STEP: 31150 + | > loss_text_ce: 0.04605792462825775 (0.045989866368472576) + | > loss_mel_ce: 2.1225180625915527 (2.157777726650238) + | > loss: 0.025816380977630615 (0.026235328521579504) + | > current_lr: 5e-06 + | > step_time: 0.1662 (0.15192055702209473) + | > loader_time: 0.0119 (0.013260632753372192) + + + --> TIME: 2024-04-09 17:22:32 -- STEP: 58/1354 -- GLOBAL_STEP: 31200 + | > loss_text_ce: 0.06320113688707352 (0.04737963039299537) + | > loss_mel_ce: 3.6164753437042236 (2.4066760211155334) + | > loss: 0.043805673718452454 (0.02921494854421451) + | > current_lr: 5e-06 + | > step_time: 0.1383 (0.14341099919943973) + | > loader_time: 0.0099 (0.011915219241175157) + + + --> TIME: 2024-04-09 17:22:43 -- STEP: 108/1354 -- GLOBAL_STEP: 31250 + | > loss_text_ce: 0.04754576086997986 (0.04744850347439448) + | > loss_mel_ce: 2.2967917919158936 (2.4339696003331084) + | > loss: 0.027908779680728912 (0.029540692249106035) + | > current_lr: 5e-06 + | > step_time: 0.1339 (0.14325633313920763) + | > loader_time: 0.0099 (0.011450897764276578) + + + --> TIME: 2024-04-09 17:22:54 -- STEP: 158/1354 -- GLOBAL_STEP: 31300 + | > loss_text_ce: 0.03787640854716301 (0.04739149780118767) + | > loss_mel_ce: 2.1630330085754395 (2.4099868324738516) + | > loss: 0.02620130218565464 (0.029254504503045654) + | > current_lr: 5e-06 + | > step_time: 0.16 (0.1405272438556333) + | > loader_time: 0.0093 (0.01119044762623461) + + + --> TIME: 2024-04-09 17:23:05 -- STEP: 208/1354 -- GLOBAL_STEP: 31350 + | > loss_text_ce: 0.06968043744564056 (0.04778856808283868) + | > loss_mel_ce: 3.135624885559082 (2.402466036379337) + | > loss: 0.03815839812159538 (0.029169698245823383) + | > current_lr: 5e-06 + | > step_time: 0.114 (0.1419359101698949) + | > loader_time: 0.0111 (0.011029597658377428) + + + --> TIME: 2024-04-09 17:23:15 -- STEP: 258/1354 -- GLOBAL_STEP: 31400 + | > loss_text_ce: 0.03785693645477295 (0.0474483804964973) + | > loss_mel_ce: 2.0565571784973145 (2.4046233726102244) + | > loss: 0.024933503940701485 (0.029191330963269223) + | > current_lr: 5e-06 + | > step_time: 0.1765 (0.14183497244073437) + | > loader_time: 0.0122 (0.01079710324605306) + + + --> TIME: 2024-04-09 17:23:26 -- STEP: 308/1354 -- GLOBAL_STEP: 31450 + | > loss_text_ce: 0.04184730350971222 (0.04732375067066063) + | > loss_mel_ce: 2.2782516479492188 (2.402583457432783) + | > loss: 0.02762022614479065 (0.02916556252650433) + | > current_lr: 5e-06 + | > step_time: 0.2165 (0.1416625844967829) + | > loader_time: 0.0098 (0.010728137833731513) + + + --> TIME: 2024-04-09 17:23:37 -- STEP: 358/1354 -- GLOBAL_STEP: 31500 + | > loss_text_ce: 0.03768708556890488 (0.04716893765222092) + | > loss_mel_ce: 2.0131545066833496 (2.4029096568096953) + | > loss: 0.024414781481027603 (0.02916760283766845) + | > current_lr: 5e-06 + | > step_time: 0.1754 (0.14218001925079501) + | > loader_time: 0.0097 (0.010679231675643495) + + + --> TIME: 2024-04-09 17:23:48 -- STEP: 408/1354 -- GLOBAL_STEP: 31550 + | > loss_text_ce: 0.03371177986264229 (0.04715907654049348) + | > loss_mel_ce: 1.5500588417053223 (2.4034341167585502) + | > loss: 0.01885441318154335 (0.02917372902362224) + | > current_lr: 5e-06 + | > step_time: 0.165 (0.14152559579587437) + | > loader_time: 0.0098 (0.010647309177062095) + + + --> TIME: 2024-04-09 17:23:58 -- STEP: 458/1354 -- GLOBAL_STEP: 31600 + | > loss_text_ce: 0.062138933688402176 (0.047073455173603555) + | > loss_mel_ce: 2.769996166229248 (2.4119811823274353) + | > loss: 0.03371589630842209 (0.02927446051390931) + | > current_lr: 5e-06 + | > step_time: 0.1137 (0.1409697917871599) + | > loader_time: 0.0089 (0.0105928453295512) + + + --> TIME: 2024-04-09 17:24:09 -- STEP: 508/1354 -- GLOBAL_STEP: 31650 + | > loss_text_ce: 0.06507308781147003 (0.04707714840109774) + | > loss_mel_ce: 2.8080053329467773 (2.406321188596288) + | > loss: 0.034203313291072845 (0.029207123568149532) + | > current_lr: 5e-06 + | > step_time: 0.0896 (0.1413472745362228) + | > loader_time: 0.0099 (0.010536426164972504) + + + --> TIME: 2024-04-09 17:24:20 -- STEP: 558/1354 -- GLOBAL_STEP: 31700 + | > loss_text_ce: 0.04295273870229721 (0.04694210819726454) + | > loss_mel_ce: 2.4959676265716553 (2.4081495021833725) + | > loss: 0.030225243419408798 (0.02922728159348063) + | > current_lr: 5e-06 + | > step_time: 0.1766 (0.14137476447662564) + | > loader_time: 0.0126 (0.01052495869257117) + + + --> TIME: 2024-04-09 17:24:31 -- STEP: 608/1354 -- GLOBAL_STEP: 31750 + | > loss_text_ce: 0.045519281178712845 (0.046717010606938066) + | > loss_mel_ce: 2.381061553955078 (2.402999438541498) + | > loss: 0.02888786792755127 (0.029163291604307136) + | > current_lr: 5e-06 + | > step_time: 0.1095 (0.14145502134373306) + | > loader_time: 0.0128 (0.010508591015087929) + + + --> TIME: 2024-04-09 17:24:42 -- STEP: 658/1354 -- GLOBAL_STEP: 31800 + | > loss_text_ce: 0.0365527980029583 (0.046680677096192405) + | > loss_mel_ce: 1.9545319080352783 (2.406131510676583) + | > loss: 0.02370338886976242 (0.02920014561241702) + | > current_lr: 5e-06 + | > step_time: 0.1903 (0.1423597882946211) + | > loader_time: 0.0121 (0.010511908487708374) + + + --> TIME: 2024-04-09 17:24:53 -- STEP: 708/1354 -- GLOBAL_STEP: 31850 + | > loss_text_ce: 0.05315380543470383 (0.04672307157257606) + | > loss_mel_ce: 2.264129877090454 (2.4093760204853987) + | > loss: 0.027586709707975388 (0.029239275442741516) + | > current_lr: 5e-06 + | > step_time: 0.1157 (0.1425386626841658) + | > loader_time: 0.0105 (0.010486863427242997) + + + --> TIME: 2024-04-09 17:25:04 -- STEP: 758/1354 -- GLOBAL_STEP: 31900 + | > loss_text_ce: 0.035403311252593994 (0.04668463402154974) + | > loss_mel_ce: 2.5510294437408447 (2.409384877511886) + | > loss: 0.030790865421295166 (0.02923892330781294) + | > current_lr: 5e-06 + | > step_time: 0.154 (0.1427124153655562) + | > loader_time: 0.0101 (0.010473740446850623) + + + --> TIME: 2024-04-09 17:25:15 -- STEP: 808/1354 -- GLOBAL_STEP: 31950 + | > loss_text_ce: 0.04727959632873535 (0.04683089978753195) + | > loss_mel_ce: 2.439608335494995 (2.4059448587422367) + | > loss: 0.029605809599161148 (0.02919971195768823) + | > current_lr: 5e-06 + | > step_time: 0.1118 (0.1421725599482507) + | > loader_time: 0.0093 (0.01044906572540208) + + + --> TIME: 2024-04-09 17:25:26 -- STEP: 858/1354 -- GLOBAL_STEP: 32000 + | > loss_text_ce: 0.03618890047073364 (0.0467415267844752) + | > loss_mel_ce: 2.6763837337493896 (2.4072903355796345) + | > loss: 0.032292529940605164 (0.02921466556505793) + | > current_lr: 5e-06 + | > step_time: 0.1907 (0.1423826462183242) + | > loader_time: 0.0096 (0.010445996320053141) + + + --> TIME: 2024-04-09 17:25:59 -- STEP: 908/1354 -- GLOBAL_STEP: 32050 + | > loss_text_ce: 0.057418595999479294 (0.04677841009465479) + | > loss_mel_ce: 2.344989061355591 (2.405852100134951) + | > loss: 0.028600091114640236 (0.02919798280165781) + | > current_lr: 5e-06 + | > step_time: 0.1776 (0.14213952330240576) + | > loader_time: 0.0115 (0.010541617870330814) + + + --> TIME: 2024-04-09 17:26:11 -- STEP: 958/1354 -- GLOBAL_STEP: 32100 + | > loss_text_ce: 0.05797034502029419 (0.0467378449332677) + | > loss_mel_ce: 1.915350317955017 (2.406730511243263) + | > loss: 0.023491913452744484 (0.029207957158009046) + | > current_lr: 5e-06 + | > step_time: 0.0692 (0.14237241655401478) + | > loader_time: 0.0104 (0.01056627415913879) + + + --> TIME: 2024-04-09 17:26:21 -- STEP: 1008/1354 -- GLOBAL_STEP: 32150 + | > loss_text_ce: 0.062168724834918976 (0.04675718793083753) + | > loss_mel_ce: 2.175448417663574 (2.4072866966090545) + | > loss: 0.026638301089406013 (0.029214808691499963) + | > current_lr: 5e-06 + | > step_time: 0.1331 (0.14193642612487534) + | > loader_time: 0.0122 (0.010569309194882715) + + + --> TIME: 2024-04-09 17:26:32 -- STEP: 1058/1354 -- GLOBAL_STEP: 32200 + | > loss_text_ce: 0.039888978004455566 (0.04678265096193678) + | > loss_mel_ce: 2.016988515853882 (2.410385373747421) + | > loss: 0.024486638605594635 (0.02925200084671519) + | > current_lr: 5e-06 + | > step_time: 0.1022 (0.14183346445483822) + | > loader_time: 0.0124 (0.010573211599613188) + + + --> TIME: 2024-04-09 17:26:43 -- STEP: 1108/1354 -- GLOBAL_STEP: 32250 + | > loss_text_ce: 0.04800283908843994 (0.04679702329936874) + | > loss_mel_ce: 2.8044791221618652 (2.4150403436770977) + | > loss: 0.03395811840891838 (0.029307588254057867) + | > current_lr: 5e-06 + | > step_time: 0.1117 (0.1420021887720706) + | > loader_time: 0.012 (0.010573986420132195) + + + --> TIME: 2024-04-09 17:26:54 -- STEP: 1158/1354 -- GLOBAL_STEP: 32300 + | > loss_text_ce: 0.05139237269759178 (0.04678214180230169) + | > loss_mel_ce: 1.7510278224945068 (2.4131206171088504) + | > loss: 0.021457383409142494 (0.029284557203773494) + | > current_lr: 5e-06 + | > step_time: 0.0749 (0.14229929282463477) + | > loader_time: 0.0099 (0.010581614645857476) + + + --> TIME: 2024-04-09 17:27:05 -- STEP: 1208/1354 -- GLOBAL_STEP: 32350 + | > loss_text_ce: 0.05064160376787186 (0.04674761560252121) + | > loss_mel_ce: 2.1435136795043945 (2.4127358835659307) + | > loss: 0.026120895519852638 (0.029279566013766144) + | > current_lr: 5e-06 + | > step_time: 0.133 (0.14214220327257318) + | > loader_time: 0.0111 (0.010585096695565233) + + + --> TIME: 2024-04-09 17:27:15 -- STEP: 1258/1354 -- GLOBAL_STEP: 32400 + | > loss_text_ce: 0.05400055646896362 (0.04672448611440687) + | > loss_mel_ce: 2.548288583755493 (2.4143783321251764) + | > loss: 0.030979633331298828 (0.029298843613075054) + | > current_lr: 5e-06 + | > step_time: 0.1056 (0.14166796643328403) + | > loader_time: 0.0108 (0.010586535418924356) + + + --> TIME: 2024-04-09 17:27:26 -- STEP: 1308/1354 -- GLOBAL_STEP: 32450 + | > loss_text_ce: 0.05088008940219879 (0.0466721149343735) + | > loss_mel_ce: 2.5919125080108643 (2.4106485618546123) + | > loss: 0.031461820006370544 (0.02925381812605927) + | > current_lr: 5e-06 + | > step_time: 0.1598 (0.14195477798444425) + | > loader_time: 0.0113 (0.0105962363222688) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03133713282071627 (-0.007215719956618093) + | > avg_loss_text_ce: 0.05112612906556863 (-0.0008978528472093419) + | > avg_loss_mel_ce: 2.8569507965674767 (-0.003342775198129644) + | > avg_loss: 2.9080769282120924 (-0.004240622887244694) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_32496.pth + + > EPOCH: 25/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 17:28:29)  + + --> TIME: 2024-04-09 17:28:31 -- STEP: 4/1354 -- GLOBAL_STEP: 32500 + | > loss_text_ce: 0.031126342713832855 (0.045954261906445026) + | > loss_mel_ce: 1.982979416847229 (2.122877150774002) + | > loss: 0.023977451026439667 (0.025819422211498022) + | > current_lr: 5e-06 + | > step_time: 0.2196 (0.1566786766052246) + | > loader_time: 0.0157 (0.01350712776184082) + + + --> TIME: 2024-04-09 17:28:42 -- STEP: 54/1354 -- GLOBAL_STEP: 32550 + | > loss_text_ce: 0.04457036033272743 (0.04665970181425413) + | > loss_mel_ce: 2.392516613006592 (2.4083198728384794) + | > loss: 0.02901294082403183 (0.029225947611309862) + | > current_lr: 5e-06 + | > step_time: 0.0915 (0.14563822304760973) + | > loader_time: 0.0101 (0.011903709835476346) + + + --> TIME: 2024-04-09 17:28:53 -- STEP: 104/1354 -- GLOBAL_STEP: 32600 + | > loss_text_ce: 0.041557226330041885 (0.046544709887642116) + | > loss_mel_ce: 1.8607007265090942 (2.408748518962127) + | > loss: 0.02264592796564102 (0.0292296817418761) + | > current_lr: 5e-06 + | > step_time: 0.1137 (0.1452302405467401) + | > loader_time: 0.0106 (0.011501977076897254) + + + --> TIME: 2024-04-09 17:29:04 -- STEP: 154/1354 -- GLOBAL_STEP: 32650 + | > loss_text_ce: 0.05783570557832718 (0.046721480714229775) + | > loss_mel_ce: 3.2259654998779297 (2.427284236078139) + | > loss: 0.03909287229180336 (0.029452449460695317) + | > current_lr: 5e-06 + | > step_time: 0.1169 (0.14571045435868313) + | > loader_time: 0.0105 (0.011668281121687451) + + + --> TIME: 2024-04-09 17:29:15 -- STEP: 204/1354 -- GLOBAL_STEP: 32700 + | > loss_text_ce: 0.05017302557826042 (0.04631441135835997) + | > loss_mel_ce: 3.2766692638397217 (2.436418684089886) + | > loss: 0.039605267345905304 (0.029556346828957983) + | > current_lr: 5e-06 + | > step_time: 0.1797 (0.14536645599440037) + | > loader_time: 0.0134 (0.011457260917214784) + + + --> TIME: 2024-04-09 17:29:25 -- STEP: 254/1354 -- GLOBAL_STEP: 32750 + | > loss_text_ce: 0.04022242873907089 (0.046193673891404004) + | > loss_mel_ce: 2.4344558715820312 (2.418072100699418) + | > loss: 0.02946045622229576 (0.029336497796274077) + | > current_lr: 5e-06 + | > step_time: 0.1034 (0.14359986500477237) + | > loader_time: 0.0121 (0.01136009054859792) + + + --> TIME: 2024-04-09 17:29:37 -- STEP: 304/1354 -- GLOBAL_STEP: 32800 + | > loss_text_ce: 0.033929672092199326 (0.046296423687109414) + | > loss_mel_ce: 1.9181228876113892 (2.4187761913788948) + | > loss: 0.02323872223496437 (0.029346103038589814) + | > current_lr: 5e-06 + | > step_time: 0.2137 (0.14409534084169487) + | > loader_time: 0.0101 (0.011272758245468141) + + + --> TIME: 2024-04-09 17:29:47 -- STEP: 354/1354 -- GLOBAL_STEP: 32850 + | > loss_text_ce: 0.03832114487886429 (0.04639817626441769) + | > loss_mel_ce: 1.6280535459518433 (2.4116046240100757) + | > loss: 0.019837794825434685 (0.029261938559343923) + | > current_lr: 5e-06 + | > step_time: 0.1293 (0.1436178677499631) + | > loader_time: 0.0096 (0.011197961656387246) + + + --> TIME: 2024-04-09 17:29:58 -- STEP: 404/1354 -- GLOBAL_STEP: 32900 + | > loss_text_ce: 0.04169414937496185 (0.04634681598993367) + | > loss_mel_ce: 2.229551076889038 (2.4045875568201054) + | > loss: 0.027038633823394775 (0.029177790650599958) + | > current_lr: 5e-06 + | > step_time: 0.1131 (0.1432732514815756) + | > loader_time: 0.0106 (0.011145020475481998) + + + --> TIME: 2024-04-09 17:30:09 -- STEP: 454/1354 -- GLOBAL_STEP: 32950 + | > loss_text_ce: 0.03606606647372246 (0.04639286820321355) + | > loss_mel_ce: 2.450784683227539 (2.398879737580925) + | > loss: 0.02960536628961563 (0.02911038866619444) + | > current_lr: 5e-06 + | > step_time: 0.1915 (0.14360983361231588) + | > loader_time: 0.0102 (0.011108293407288944) + + + --> TIME: 2024-04-09 17:30:20 -- STEP: 504/1354 -- GLOBAL_STEP: 33000 + | > loss_text_ce: 0.06636795401573181 (0.046385174609779825) + | > loss_mel_ce: 3.059204578399658 (2.402998469415165) + | > loss: 0.037209197878837585 (0.02915932960985672) + | > current_lr: 5e-06 + | > step_time: 0.0997 (0.14383046683811) + | > loader_time: 0.0101 (0.011087443147386826) + + + --> TIME: 2024-04-09 17:30:54 -- STEP: 554/1354 -- GLOBAL_STEP: 33050 + | > loss_text_ce: 0.04580460488796234 (0.04649835355606748) + | > loss_mel_ce: 2.3917839527130127 (2.397737421929191) + | > loss: 0.029018910601735115 (0.029098045457463834) + | > current_lr: 5e-06 + | > step_time: 0.1602 (0.14337159408128644) + | > loader_time: 0.0117 (0.011200230044148031) + + + --> TIME: 2024-04-09 17:31:06 -- STEP: 604/1354 -- GLOBAL_STEP: 33100 + | > loss_text_ce: 0.05469344183802605 (0.04650309051861922) + | > loss_mel_ce: 2.0739595890045166 (2.399500206803643) + | > loss: 0.025341108441352844 (0.029119087389318753) + | > current_lr: 5e-06 + | > step_time: 0.1114 (0.1437704736823278) + | > loader_time: 0.0106 (0.011216282055077968) + + + --> TIME: 2024-04-09 17:31:17 -- STEP: 654/1354 -- GLOBAL_STEP: 33150 + | > loss_text_ce: 0.04530623182654381 (0.04645790236600492) + | > loss_mel_ce: 2.9586658477783203 (2.401206660890433) + | > loss: 0.035761572420597076 (0.029138864376881252) + | > current_lr: 5e-06 + | > step_time: 0.1632 (0.14386379791691403) + | > loader_time: 0.01 (0.01121607322576214) + + + --> TIME: 2024-04-09 17:31:28 -- STEP: 704/1354 -- GLOBAL_STEP: 33200 + | > loss_text_ce: 0.03742314875125885 (0.04625738616662352) + | > loss_mel_ce: 2.2714736461639404 (2.400105858560314) + | > loss: 0.02748686634004116 (0.029123372501503167) + | > current_lr: 5e-06 + | > step_time: 0.1912 (0.14412459663369442) + | > loader_time: 0.0122 (0.011221124705943199) + + + --> TIME: 2024-04-09 17:31:39 -- STEP: 754/1354 -- GLOBAL_STEP: 33250 + | > loss_text_ce: 0.03752024099230766 (0.04625384443605055) + | > loss_mel_ce: 1.9819940328598022 (2.400638691942637) + | > loss: 0.024041837081313133 (0.029129673586046664) + | > current_lr: 5e-06 + | > step_time: 0.1661 (0.1441442011522046) + | > loader_time: 0.0134 (0.011251021443374597) + + + --> TIME: 2024-04-09 17:31:50 -- STEP: 804/1354 -- GLOBAL_STEP: 33300 + | > loss_text_ce: 0.034183014184236526 (0.046124848166471374) + | > loss_mel_ce: 2.2945334911346436 (2.3941210371937913) + | > loss: 0.027722816914319992 (0.029050546777040803) + | > current_lr: 5e-06 + | > step_time: 0.1609 (0.1444739806118296) + | > loader_time: 0.0126 (0.01126519542428392) + + + --> TIME: 2024-04-09 17:32:02 -- STEP: 854/1354 -- GLOBAL_STEP: 33350 + | > loss_text_ce: 0.05572553351521492 (0.04610762251999044) + | > loss_mel_ce: 3.2182908058166504 (2.3935063553917315) + | > loss: 0.038976386189460754 (0.029043024067769145) + | > current_lr: 5e-06 + | > step_time: 0.1011 (0.14466085729889339) + | > loader_time: 0.0117 (0.01127169980935805) + + + --> TIME: 2024-04-09 17:32:13 -- STEP: 904/1354 -- GLOBAL_STEP: 33400 + | > loss_text_ce: 0.053788311779499054 (0.04614014059420574) + | > loss_mel_ce: 2.287590265274048 (2.393300712504217) + | > loss: 0.027873557060956955 (0.029040963049005487) + | > current_lr: 5e-06 + | > step_time: 0.1183 (0.14511334737845227) + | > loader_time: 0.0116 (0.011262972270492963) + + + --> TIME: 2024-04-09 17:32:23 -- STEP: 954/1354 -- GLOBAL_STEP: 33450 + | > loss_text_ce: 0.04342212155461311 (0.0460715109220778) + | > loss_mel_ce: 2.1303064823150635 (2.3903839375737816) + | > loss: 0.02587772347033024 (0.02900542251851636) + | > current_lr: 5e-06 + | > step_time: 0.1278 (0.14452347115650616) + | > loader_time: 0.0097 (0.011233737633663157) + + + --> TIME: 2024-04-09 17:32:34 -- STEP: 1004/1354 -- GLOBAL_STEP: 33500 + | > loss_text_ce: 0.03434983268380165 (0.04611563268099053) + | > loss_mel_ce: 2.0547091960906982 (2.3935531107078027) + | > loss: 0.024869751185178757 (0.029043676032948187) + | > current_lr: 5e-06 + | > step_time: 0.1544 (0.14439427995111837) + | > loader_time: 0.0104 (0.01122200394056708) + + + --> TIME: 2024-04-09 17:32:45 -- STEP: 1054/1354 -- GLOBAL_STEP: 33550 + | > loss_text_ce: 0.04517161101102829 (0.046119903180698724) + | > loss_mel_ce: 2.7512669563293457 (2.396211847860853) + | > loss: 0.033290933817625046 (0.0290753784955283) + | > current_lr: 5e-06 + | > step_time: 0.1651 (0.14431556442883253) + | > loader_time: 0.011 (0.01121697638473656) + + + --> TIME: 2024-04-09 17:32:56 -- STEP: 1104/1354 -- GLOBAL_STEP: 33600 + | > loss_text_ce: 0.03796014562249184 (0.04608722764701728) + | > loss_mel_ce: 1.4789767265319824 (2.393849172453949) + | > loss: 0.01805877313017845 (0.02904686241484907) + | > current_lr: 5e-06 + | > step_time: 0.0982 (0.14421635218288587) + | > loader_time: 0.0097 (0.01120470593804899) + + + --> TIME: 2024-04-09 17:33:07 -- STEP: 1154/1354 -- GLOBAL_STEP: 33650 + | > loss_text_ce: 0.04114509001374245 (0.04610660344207304) + | > loss_mel_ce: 1.4271794557571411 (2.391646672266196) + | > loss: 0.01748005487024784 (0.029020872835450805) + | > current_lr: 5e-06 + | > step_time: 0.0983 (0.14428557700879435) + | > loader_time: 0.0108 (0.01120629281683646) + + + --> TIME: 2024-04-09 17:33:19 -- STEP: 1204/1354 -- GLOBAL_STEP: 33700 + | > loss_text_ce: 0.04326549544930458 (0.046028861401608764) + | > loss_mel_ce: 1.4447566270828247 (2.390505613678714) + | > loss: 0.017714548856019974 (0.029006363308843076) + | > current_lr: 5e-06 + | > step_time: 0.1005 (0.14456487593064646) + | > loader_time: 0.0131 (0.011240860750508865) + + + --> TIME: 2024-04-09 17:33:30 -- STEP: 1254/1354 -- GLOBAL_STEP: 33750 + | > loss_text_ce: 0.0284095611423254 (0.04598184495694944) + | > loss_mel_ce: 2.414442300796509 (2.3913835347934596) + | > loss: 0.029081569984555244 (0.02901625503270486) + | > current_lr: 5e-06 + | > step_time: 0.1791 (0.14462515003563275) + | > loader_time: 0.0102 (0.011269094270952592) + + + --> TIME: 2024-04-09 17:33:40 -- STEP: 1304/1354 -- GLOBAL_STEP: 33800 + | > loss_text_ce: 0.04764924943447113 (0.04604801826858946) + | > loss_mel_ce: 1.6790114641189575 (2.393557482519032) + | > loss: 0.020555485039949417 (0.029042923153075235) + | > current_lr: 5e-06 + | > step_time: 0.1436 (0.14430988292021227) + | > loader_time: 0.01 (0.011241654676893751) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.02728924384483924 (-0.004047888975877028) + | > avg_loss_text_ce: 0.050489205007369704 (-0.0006369240581989288) + | > avg_loss_mel_ce: 2.855554305590116 (-0.0013964909773607204) + | > avg_loss: 2.906043529510498 (-0.0020333987015943578) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_33850.pth + + > EPOCH: 26/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 17:34:42)  + + --> TIME: 2024-04-09 17:34:43 -- STEP: 0/1354 -- GLOBAL_STEP: 33850 + | > loss_text_ce: 0.03819730877876282 (0.03819730877876282) + | > loss_mel_ce: 2.0851337909698486 (2.0851337909698486) + | > loss: 0.025277750566601753 (0.025277750566601753) + | > current_lr: 5e-06 + | > step_time: 0.1258 (0.12583518028259277) + | > loader_time: 0.8235 (0.8235092163085938) + + + --> TIME: 2024-04-09 17:34:54 -- STEP: 50/1354 -- GLOBAL_STEP: 33900 + | > loss_text_ce: 0.042243842035532 (0.0460629178956151) + | > loss_mel_ce: 1.9243321418762207 (2.4281789803504945) + | > loss: 0.023411618545651436 (0.02945526096969843) + | > current_lr: 5e-06 + | > step_time: 0.0923 (0.143032546043396) + | > loader_time: 0.0104 (0.011704049110412597) + + + --> TIME: 2024-04-09 17:35:05 -- STEP: 100/1354 -- GLOBAL_STEP: 33950 + | > loss_text_ce: 0.03784092143177986 (0.045736413840204486) + | > loss_mel_ce: 2.1411192417144775 (2.4132975256443028) + | > loss: 0.025940001010894775 (0.029274213910102844) + | > current_lr: 5e-06 + | > step_time: 0.1687 (0.14273079395294178) + | > loader_time: 0.0134 (0.011313414573669434) + + + --> TIME: 2024-04-09 17:35:15 -- STEP: 150/1354 -- GLOBAL_STEP: 34000 + | > loss_text_ce: 0.06391921639442444 (0.04545806271334489) + | > loss_mel_ce: 2.679642677307129 (2.3860946098963427) + | > loss: 0.032661452889442444 (0.02894705604761839) + | > current_lr: 5e-06 + | > step_time: 0.0812 (0.13948546727498362) + | > loader_time: 0.012 (0.011264990170796713) + + + --> TIME: 2024-04-09 17:35:50 -- STEP: 200/1354 -- GLOBAL_STEP: 34050 + | > loss_text_ce: 0.047680217772722244 (0.04550826841965318) + | > loss_mel_ce: 1.8752957582473755 (2.378616785407069) + | > loss: 0.022892571985721588 (0.02885863200761378) + | > current_lr: 5e-06 + | > step_time: 0.0827 (0.14133688449859602) + | > loader_time: 0.012 (0.011579655408859253) + + + --> TIME: 2024-04-09 17:36:01 -- STEP: 250/1354 -- GLOBAL_STEP: 34100 + | > loss_text_ce: 0.03174133226275444 (0.04516703936457634) + | > loss_mel_ce: 1.9293767213821411 (2.370807727336885) + | > loss: 0.023346643894910812 (0.028761604771018027) + | > current_lr: 5e-06 + | > step_time: 0.1602 (0.1421113672256468) + | > loader_time: 0.012 (0.011583531379699707) + + + --> TIME: 2024-04-09 17:36:12 -- STEP: 300/1354 -- GLOBAL_STEP: 34150 + | > loss_text_ce: 0.03576542064547539 (0.044567411908259) + | > loss_mel_ce: 1.5494599342346191 (2.3536352109909067) + | > loss: 0.01887173019349575 (0.0285500316383938) + | > current_lr: 5e-06 + | > step_time: 0.1601 (0.14255979617436723) + | > loader_time: 0.011 (0.011622860431671142) + + + --> TIME: 2024-04-09 17:36:22 -- STEP: 350/1354 -- GLOBAL_STEP: 34200 + | > loss_text_ce: 0.04471848905086517 (0.04502219527427639) + | > loss_mel_ce: 2.188424587249756 (2.365475302083152) + | > loss: 0.026585036888718605 (0.02869639921401228) + | > current_lr: 5e-06 + | > step_time: 0.1097 (0.14165504046848834) + | > loader_time: 0.0104 (0.011701727594648093) + + + --> TIME: 2024-04-09 17:36:34 -- STEP: 400/1354 -- GLOBAL_STEP: 34250 + | > loss_text_ce: 0.04473016783595085 (0.04485911321826278) + | > loss_mel_ce: 1.96246337890625 (2.3570190113782883) + | > loss: 0.023895161226391792 (0.028593787644058465) + | > current_lr: 5e-06 + | > step_time: 0.0925 (0.14225488901138303) + | > loader_time: 0.0107 (0.011725794672966007) + + + --> TIME: 2024-04-09 17:36:45 -- STEP: 450/1354 -- GLOBAL_STEP: 34300 + | > loss_text_ce: 0.039448413997888565 (0.044731104444298474) + | > loss_mel_ce: 1.6571667194366455 (2.34789609750112) + | > loss: 0.020197799429297447 (0.02848365762995349) + | > current_lr: 5e-06 + | > step_time: 0.1647 (0.14367836846245646) + | > loader_time: 0.0112 (0.011676393085055884) + + + --> TIME: 2024-04-09 17:36:57 -- STEP: 500/1354 -- GLOBAL_STEP: 34350 + | > loss_text_ce: 0.030209926888346672 (0.04469628547877073) + | > loss_mel_ce: 2.1764612197875977 (2.3496726109981547) + | > loss: 0.02626989595592022 (0.028504392113536597) + | > current_lr: 5e-06 + | > step_time: 0.2292 (0.1443049449920653) + | > loader_time: 0.0107 (0.011635582447052005) + + + --> TIME: 2024-04-09 17:37:07 -- STEP: 550/1354 -- GLOBAL_STEP: 34400 + | > loss_text_ce: 0.047659166157245636 (0.044769975065507665) + | > loss_mel_ce: 2.0930871963500977 (2.347916615659541) + | > loss: 0.025485076010227203 (0.028484364649788894) + | > current_lr: 5e-06 + | > step_time: 0.112 (0.14394134218042542) + | > loader_time: 0.011 (0.011587479331276636) + + + --> TIME: 2024-04-09 17:37:18 -- STEP: 600/1354 -- GLOBAL_STEP: 34450 + | > loss_text_ce: 0.05695544555783272 (0.044819790525361905) + | > loss_mel_ce: 2.601811170578003 (2.342883277535438) + | > loss: 0.03165198117494583 (0.02842503700405358) + | > current_lr: 5e-06 + | > step_time: 0.1643 (0.14392779072125747) + | > loader_time: 0.0109 (0.011575167179107665) + + + --> TIME: 2024-04-09 17:37:29 -- STEP: 650/1354 -- GLOBAL_STEP: 34500 + | > loss_text_ce: 0.054868172854185104 (0.04484334528446202) + | > loss_mel_ce: 2.7598748207092285 (2.347789594393509) + | > loss: 0.033508844673633575 (0.02848372594668314) + | > current_lr: 5e-06 + | > step_time: 0.1924 (0.14401613162114066) + | > loader_time: 0.0119 (0.011573065977830147) + + + --> TIME: 2024-04-09 17:37:40 -- STEP: 700/1354 -- GLOBAL_STEP: 34550 + | > loss_text_ce: 0.058829545974731445 (0.044808315214301925) + | > loss_mel_ce: 2.9858036041259766 (2.35154535634177) + | > loss: 0.03624563291668892 (0.0285280203845884) + | > current_lr: 5e-06 + | > step_time: 0.1171 (0.14394349949700488) + | > loader_time: 0.0114 (0.011558312007359095) + + + --> TIME: 2024-04-09 17:37:51 -- STEP: 750/1354 -- GLOBAL_STEP: 34600 + | > loss_text_ce: 0.040910594165325165 (0.04474504767855014) + | > loss_mel_ce: 1.8094254732131958 (2.349374608834582) + | > loss: 0.022027811035513878 (0.028501424970726164) + | > current_lr: 5e-06 + | > step_time: 0.1203 (0.14349447472890206) + | > loader_time: 0.0131 (0.011536665280659993) + + + --> TIME: 2024-04-09 17:38:01 -- STEP: 800/1354 -- GLOBAL_STEP: 34650 + | > loss_text_ce: 0.053856417536735535 (0.0447334955586121) + | > loss_mel_ce: 2.4735186100006104 (2.3429385687410806) + | > loss: 0.030087798833847046 (0.028424667927902183) + | > current_lr: 5e-06 + | > step_time: 0.0706 (0.14306127995252602) + | > loader_time: 0.0099 (0.011516009867191314) + + + --> TIME: 2024-04-09 17:38:12 -- STEP: 850/1354 -- GLOBAL_STEP: 34700 + | > loss_text_ce: 0.041702207177877426 (0.044827069911886716) + | > loss_mel_ce: 2.6823806762695312 (2.3485683749703785) + | > loss: 0.03242956101894379 (0.028492803418022735) + | > current_lr: 5e-06 + | > step_time: 0.1102 (0.14274373531341555) + | > loader_time: 0.0109 (0.011481088750502643) + + + --> TIME: 2024-04-09 17:38:22 -- STEP: 900/1354 -- GLOBAL_STEP: 34750 + | > loss_text_ce: 0.05676889792084694 (0.044748266126132724) + | > loss_mel_ce: 2.5628647804260254 (2.3409001635180564) + | > loss: 0.031186114996671677 (0.028400577033559474) + | > current_lr: 5e-06 + | > step_time: 0.1083 (0.14238142861260308) + | > loader_time: 0.0109 (0.01144893831676907) + + + --> TIME: 2024-04-09 17:38:33 -- STEP: 950/1354 -- GLOBAL_STEP: 34800 + | > loss_text_ce: 0.056987445801496506 (0.04481568089245186) + | > loss_mel_ce: 2.533721923828125 (2.3395124880891083) + | > loss: 0.030841778963804245 (0.028384859657993436) + | > current_lr: 5e-06 + | > step_time: 0.0937 (0.14236681285657382) + | > loader_time: 0.0091 (0.011437022058587326) + + + --> TIME: 2024-04-09 17:38:44 -- STEP: 1000/1354 -- GLOBAL_STEP: 34850 + | > loss_text_ce: 0.04661836102604866 (0.04468734640814368) + | > loss_mel_ce: 2.435519218444824 (2.3377490785121906) + | > loss: 0.029549257829785347 (0.028362338908016673) + | > current_lr: 5e-06 + | > step_time: 0.1437 (0.1425396602153778) + | > loader_time: 0.01 (0.011393435955047607) + + + --> TIME: 2024-04-09 17:38:55 -- STEP: 1050/1354 -- GLOBAL_STEP: 34900 + | > loss_text_ce: 0.05205066502094269 (0.04467924598604444) + | > loss_mel_ce: 2.380667209625244 (2.330392155420211) + | > loss: 0.028960926458239555 (0.028274660055481245) + | > current_lr: 5e-06 + | > step_time: 0.0819 (0.14225727944147018) + | > loader_time: 0.0115 (0.011349330629621233) + + + --> TIME: 2024-04-09 17:39:05 -- STEP: 1100/1354 -- GLOBAL_STEP: 34950 + | > loss_text_ce: 0.03795507177710533 (0.04472760564054958) + | > loss_mel_ce: 2.0745902061462402 (2.3342062843929616) + | > loss: 0.025149349123239517 (0.028320642071352757) + | > current_lr: 5e-06 + | > step_time: 0.1506 (0.14206233739852892) + | > loader_time: 0.0114 (0.01129599744623358) + + + --> TIME: 2024-04-09 17:39:17 -- STEP: 1150/1354 -- GLOBAL_STEP: 35000 + | > loss_text_ce: 0.04155927896499634 (0.04466480867532287) + | > loss_mel_ce: 2.1907565593719482 (2.3332416480520464) + | > loss: 0.02657518908381462 (0.02830841072551582) + | > current_lr: 5e-06 + | > step_time: 0.1456 (0.14245472431182857) + | > loader_time: 0.0109 (0.0112614455430404) + + + --> TIME: 2024-04-09 17:39:52 -- STEP: 1200/1354 -- GLOBAL_STEP: 35050 + | > loss_text_ce: 0.04436236247420311 (0.0447478448211526) + | > loss_mel_ce: 2.2959485054016113 (2.3341224890947307) + | > loss: 0.027860842645168304 (0.028319885454451046) + | > current_lr: 5e-06 + | > step_time: 0.1606 (0.14261178235212954) + | > loader_time: 0.0119 (0.011302557190259296) + + + --> TIME: 2024-04-09 17:40:03 -- STEP: 1250/1354 -- GLOBAL_STEP: 35100 + | > loss_text_ce: 0.0502825491130352 (0.044691136901080626) + | > loss_mel_ce: 3.1807503700256348 (2.3345957973480194) + | > loss: 0.03846467658877373 (0.0283248449832201) + | > current_lr: 5e-06 + | > step_time: 0.1654 (0.14271320152282718) + | > loader_time: 0.0117 (0.01129081192016601) + + + --> TIME: 2024-04-09 17:40:13 -- STEP: 1300/1354 -- GLOBAL_STEP: 35150 + | > loss_text_ce: 0.05408244952559471 (0.04463979705595057) + | > loss_mel_ce: 3.230008840560913 (2.3346271649690737) + | > loss: 0.039096325635910034 (0.028324607223845444) + | > current_lr: 5e-06 + | > step_time: 0.1131 (0.1424725006176874) + | > loader_time: 0.0096 (0.011264916566702027) + + + --> TIME: 2024-04-09 17:40:24 -- STEP: 1350/1354 -- GLOBAL_STEP: 35200 + | > loss_text_ce: 0.05608384311199188 (0.04455581024013187) + | > loss_mel_ce: 2.2429513931274414 (2.3328565271695405) + | > loss: 0.02736946940422058 (0.028302528365618654) + | > current_lr: 5e-06 + | > step_time: 0.1099 (0.142627101474338) + | > loader_time: 0.0095 (0.011231033890335638) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.03740396866431603 (+0.010114724819476787) + | > avg_loss_text_ce: 0.04981045797467232 (-0.0006787470326973868) + | > avg_loss_mel_ce: 2.857993511053232 (+0.0024392054631161564) + | > avg_loss: 2.907804012298584 (+0.0017604827880859375) + + + > EPOCH: 27/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 17:40:41)  + + --> TIME: 2024-04-09 17:40:52 -- STEP: 46/1354 -- GLOBAL_STEP: 35250 + | > loss_text_ce: 0.04684658721089363 (0.043268833025966) + | > loss_mel_ce: 3.045283794403076 (2.3085531022237697) + | > loss: 0.03681107610464096 (0.027997880607195522) + | > current_lr: 5e-06 + | > step_time: 0.1489 (0.14393812677134638) + | > loader_time: 0.0105 (0.011221128961314325) + + + --> TIME: 2024-04-09 17:41:03 -- STEP: 96/1354 -- GLOBAL_STEP: 35300 + | > loss_text_ce: 0.03254351764917374 (0.044522464682813755) + | > loss_mel_ce: 1.3442423343658447 (2.333420347422361) + | > loss: 0.016390308737754822 (0.028308843495324254) + | > current_lr: 5e-06 + | > step_time: 0.1339 (0.14205157508452732) + | > loader_time: 0.011 (0.011112968126932778) + + + --> TIME: 2024-04-09 17:41:14 -- STEP: 146/1354 -- GLOBAL_STEP: 35350 + | > loss_text_ce: 0.061397287994623184 (0.044020259995268644) + | > loss_mel_ce: 3.1728506088256836 (2.355155229568481) + | > loss: 0.038502953946590424 (0.028561613509712154) + | > current_lr: 5e-06 + | > step_time: 0.1342 (0.1438383435549801) + | > loader_time: 0.0102 (0.011010831349516567) + + + --> TIME: 2024-04-09 17:41:25 -- STEP: 196/1354 -- GLOBAL_STEP: 35400 + | > loss_text_ce: 0.031632132828235626 (0.04396253006950933) + | > loss_mel_ce: 1.9916943311691284 (2.355062271259267) + | > loss: 0.024087218567728996 (0.028559819633635332) + | > current_lr: 5e-06 + | > step_time: 0.2004 (0.14636336297405014) + | > loader_time: 0.0102 (0.010967995439256938) + + + --> TIME: 2024-04-09 17:41:36 -- STEP: 246/1354 -- GLOBAL_STEP: 35450 + | > loss_text_ce: 0.05044588819146156 (0.04445726356310088) + | > loss_mel_ce: 1.989201545715332 (2.350379598334551) + | > loss: 0.024281516671180725 (0.028509963137226375) + | > current_lr: 5e-06 + | > step_time: 0.105 (0.1464372262722109) + | > loader_time: 0.0097 (0.010917879701629885) + + + --> TIME: 2024-04-09 17:41:47 -- STEP: 296/1354 -- GLOBAL_STEP: 35500 + | > loss_text_ce: 0.03726432844996452 (0.04449437293727454) + | > loss_mel_ce: 1.800328254699707 (2.3523154955457986) + | > loss: 0.021876102313399315 (0.028533451334648834) + | > current_lr: 5e-06 + | > step_time: 0.1128 (0.14532889627121592) + | > loader_time: 0.0122 (0.01087568579493342) + + + --> TIME: 2024-04-09 17:41:58 -- STEP: 346/1354 -- GLOBAL_STEP: 35550 + | > loss_text_ce: 0.03402730077505112 (0.044381751867301894) + | > loss_mel_ce: 2.2640926837921143 (2.343227876404116) + | > loss: 0.02735857293009758 (0.028423924671992052) + | > current_lr: 5e-06 + | > step_time: 0.1424 (0.14441652173940855) + | > loader_time: 0.0102 (0.010828738267711125) + + + --> TIME: 2024-04-09 17:42:09 -- STEP: 396/1354 -- GLOBAL_STEP: 35600 + | > loss_text_ce: 0.026903750374913216 (0.044155430862435494) + | > loss_mel_ce: 2.022047996520996 (2.3313111039725203) + | > loss: 0.02439228445291519 (0.028279364033780913) + | > current_lr: 5e-06 + | > step_time: 0.155 (0.14500383656434362) + | > loader_time: 0.0103 (0.010813266339928209) + + + --> TIME: 2024-04-09 17:42:20 -- STEP: 446/1354 -- GLOBAL_STEP: 35650 + | > loss_text_ce: 0.04938291758298874 (0.04416883144644612) + | > loss_mel_ce: 2.168112277984619 (2.341823799995027) + | > loss: 0.026398753747344017 (0.028404674726651116) + | > current_lr: 5e-06 + | > step_time: 0.11 (0.14471976051416116) + | > loader_time: 0.0101 (0.01078718018638713) + + + --> TIME: 2024-04-09 17:42:30 -- STEP: 496/1354 -- GLOBAL_STEP: 35700 + | > loss_text_ce: 0.03923442214727402 (0.04423048063707088) + | > loss_mel_ce: 2.4634792804718018 (2.3398692850624343) + | > loss: 0.02979421056807041 (0.028382140617336953) + | > current_lr: 5e-06 + | > step_time: 0.1639 (0.14386217872942664) + | > loader_time: 0.0109 (0.010779161607065504) + + + --> TIME: 2024-04-09 17:42:41 -- STEP: 546/1354 -- GLOBAL_STEP: 35750 + | > loss_text_ce: 0.029757224023342133 (0.044198886030814157) + | > loss_mel_ce: 1.9658706188201904 (2.3326184334772395) + | > loss: 0.02375747449696064 (0.02829544486361982) + | > current_lr: 5e-06 + | > step_time: 0.1507 (0.1433623081598528) + | > loader_time: 0.0101 (0.010767213154188441) + + + --> TIME: 2024-04-09 17:42:52 -- STEP: 596/1354 -- GLOBAL_STEP: 35800 + | > loss_text_ce: 0.03770175948739052 (0.04403425974696374) + | > loss_mel_ce: 2.519523859024048 (2.3220695407198577) + | > loss: 0.030443163588643074 (0.028167902967023768) + | > current_lr: 5e-06 + | > step_time: 0.2145 (0.14364377804250544) + | > loader_time: 0.0109 (0.010777499051702097) + + + --> TIME: 2024-04-09 17:43:03 -- STEP: 646/1354 -- GLOBAL_STEP: 35850 + | > loss_text_ce: 0.03976356238126755 (0.0440471217428383) + | > loss_mel_ce: 1.9526225328445435 (2.316785612342529) + | > loss: 0.023718882352113724 (0.028105152182673152) + | > current_lr: 5e-06 + | > step_time: 0.1303 (0.14350440590743324) + | > loader_time: 0.0098 (0.010784618256630903) + + + --> TIME: 2024-04-09 17:43:14 -- STEP: 696/1354 -- GLOBAL_STEP: 35900 + | > loss_text_ce: 0.05134193226695061 (0.043833019144982964) + | > loss_mel_ce: 2.219606399536133 (2.304137464398627) + | > loss: 0.027035100385546684 (0.02795203014319741) + | > current_lr: 5e-06 + | > step_time: 0.0882 (0.14407596711454732) + | > loader_time: 0.01 (0.010776399880990205) + + + --> TIME: 2024-04-09 17:43:26 -- STEP: 746/1354 -- GLOBAL_STEP: 35950 + | > loss_text_ce: 0.03394312784075737 (0.04392986852256207) + | > loss_mel_ce: 2.230440378189087 (2.3081553712607086) + | > loss: 0.026956947520375252 (0.02800101533172597) + | > current_lr: 5e-06 + | > step_time: 0.2078 (0.14445287740262527) + | > loader_time: 0.0117 (0.010772718181558963) + + + --> TIME: 2024-04-09 17:43:36 -- STEP: 796/1354 -- GLOBAL_STEP: 36000 + | > loss_text_ce: 0.0694335326552391 (0.04409608521952701) + | > loss_mel_ce: 2.8638556003570557 (2.318378426621309) + | > loss: 0.0349201075732708 (0.02812469714606677) + | > current_lr: 5e-06 + | > step_time: 0.0941 (0.14394191910873105) + | > loader_time: 0.0216 (0.01077823063836025) + + + --> TIME: 2024-04-09 17:44:12 -- STEP: 846/1354 -- GLOBAL_STEP: 36050 + | > loss_text_ce: 0.03311868757009506 (0.044001480645938264) + | > loss_mel_ce: 2.0483806133270264 (2.3113243512228037) + | > loss: 0.024779753759503365 (0.028039593815398414) + | > current_lr: 5e-06 + | > step_time: 0.1761 (0.14450291285278097) + | > loader_time: 0.0116 (0.010902810885832939) + + + --> TIME: 2024-04-09 17:44:23 -- STEP: 896/1354 -- GLOBAL_STEP: 36100 + | > loss_text_ce: 0.057509005069732666 (0.044010894381374656) + | > loss_mel_ce: 2.459822177886963 (2.308514274922866) + | > loss: 0.029968228191137314 (0.028006252585328184) + | > current_lr: 5e-06 + | > step_time: 0.081 (0.14442405743258327) + | > loader_time: 0.0112 (0.010921298659273548) + + + --> TIME: 2024-04-09 17:44:34 -- STEP: 946/1354 -- GLOBAL_STEP: 36150 + | > loss_text_ce: 0.04661109670996666 (0.04390671416668194) + | > loss_mel_ce: 2.158980131149292 (2.304031018474894) + | > loss: 0.026257038116455078 (0.027951640250204607) + | > current_lr: 5e-06 + | > step_time: 0.1451 (0.1441556603409522) + | > loader_time: 0.0121 (0.010895806437566963) + + + --> TIME: 2024-04-09 17:44:45 -- STEP: 996/1354 -- GLOBAL_STEP: 36200 + | > loss_text_ce: 0.04029848799109459 (0.04387842013826033) + | > loss_mel_ce: 2.0718350410461426 (2.305493468261628) + | > loss: 0.02514444664120674 (0.027968713527463048) + | > current_lr: 5e-06 + | > step_time: 0.1163 (0.14440841105089597) + | > loader_time: 0.0092 (0.010879064420140883) + + + --> TIME: 2024-04-09 17:44:56 -- STEP: 1046/1354 -- GLOBAL_STEP: 36250 + | > loss_text_ce: 0.025995943695306778 (0.043886021880101746) + | > loss_mel_ce: 1.687581181526184 (2.302971178441386) + | > loss: 0.020399728789925575 (0.027938776759940057) + | > current_lr: 5e-06 + | > step_time: 0.1909 (0.14463848183305486) + | > loader_time: 0.0116 (0.010887799025949505) + + + --> TIME: 2024-04-09 17:45:07 -- STEP: 1096/1354 -- GLOBAL_STEP: 36300 + | > loss_text_ce: 0.051851872354745865 (0.04383298546334143) + | > loss_mel_ce: 2.3301217555999756 (2.2980955053855046) + | > loss: 0.028356831520795822 (0.027880101649933597) + | > current_lr: 5e-06 + | > step_time: 0.092 (0.14427424369067163) + | > loader_time: 0.0102 (0.010904990191007175) + + + --> TIME: 2024-04-09 17:45:17 -- STEP: 1146/1354 -- GLOBAL_STEP: 36350 + | > loss_text_ce: 0.052829593420028687 (0.043730073314270604) + | > loss_mel_ce: 2.4670588970184326 (2.2987396919914582) + | > loss: 0.029998671263456345 (0.027886545384728574) + | > current_lr: 5e-06 + | > step_time: 0.1517 (0.144164655013026) + | > loader_time: 0.0114 (0.010914931330589298) + + + --> TIME: 2024-04-09 17:45:28 -- STEP: 1196/1354 -- GLOBAL_STEP: 36400 + | > loss_text_ce: 0.04231734201312065 (0.04380725908021511) + | > loss_mel_ce: 2.462240219116211 (2.2994437049264507) + | > loss: 0.0298161618411541 (0.02789584537002454) + | > current_lr: 5e-06 + | > step_time: 0.2138 (0.14383895819801143) + | > loader_time: 0.0102 (0.010911420436208458) + + + --> TIME: 2024-04-09 17:45:38 -- STEP: 1246/1354 -- GLOBAL_STEP: 36450 + | > loss_text_ce: 0.04256408289074898 (0.04389798743007509) + | > loss_mel_ce: 2.2819204330444336 (2.2991499696076407) + | > loss: 0.027672436088323593 (0.027893428617481435) + | > current_lr: 5e-06 + | > step_time: 0.1384 (0.14368445762279122) + | > loader_time: 0.0098 (0.0109106239116785) + + + --> TIME: 2024-04-09 17:45:49 -- STEP: 1296/1354 -- GLOBAL_STEP: 36500 + | > loss_text_ce: 0.037052322179079056 (0.04386902890846503) + | > loss_mel_ce: 1.8568429946899414 (2.2982898888948546) + | > loss: 0.022546373307704926 (0.02788284481258741) + | > current_lr: 5e-06 + | > step_time: 0.118 (0.14334979855719895) + | > loader_time: 0.0115 (0.010899054783361927) + + + --> TIME: 2024-04-09 17:46:00 -- STEP: 1346/1354 -- GLOBAL_STEP: 36550 + | > loss_text_ce: 0.03305092453956604 (0.0438222679410567) + | > loss_mel_ce: 2.0823006629943848 (2.2951899208958766) + | > loss: 0.025182759389281273 (0.027845383749757223) + | > current_lr: 5e-06 + | > step_time: 0.2253 (0.14350065681944973) + | > loader_time: 0.0132 (0.010890780373809951) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.025413311444796048 (-0.01199065721951998) + | > avg_loss_text_ce: 0.04929639914861092 (-0.0005140588260613976) + | > avg_loss_mel_ce: 2.8581173236553488 (+0.00012381260211657974) + | > avg_loss: 2.907413739424485 (-0.00039027287409876266) + + + > EPOCH: 28/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 17:46:16)  + + --> TIME: 2024-04-09 17:46:26 -- STEP: 42/1354 -- GLOBAL_STEP: 36600 + | > loss_text_ce: 0.042584680020809174 (0.04410980597493194) + | > loss_mel_ce: 2.027611494064331 (2.3735622394652593) + | > loss: 0.024645192548632622 (0.028781810775399208) + | > current_lr: 5e-06 + | > step_time: 0.1133 (0.14371757280258907) + | > loader_time: 0.0121 (0.011664776574997674) + + + --> TIME: 2024-04-09 17:46:37 -- STEP: 92/1354 -- GLOBAL_STEP: 36650 + | > loss_text_ce: 0.03229815512895584 (0.043193956533365926) + | > loss_mel_ce: 2.433119297027588 (2.3405912013157564) + | > loss: 0.02935020811855793 (0.028378395262457754) + | > current_lr: 5e-06 + | > step_time: 0.171 (0.14000128922255142) + | > loader_time: 0.0108 (0.011417186778524645) + + + --> TIME: 2024-04-09 17:46:48 -- STEP: 142/1354 -- GLOBAL_STEP: 36700 + | > loss_text_ce: 0.04787830263376236 (0.043283387310278254) + | > loss_mel_ce: 2.6561288833618164 (2.335114381682706) + | > loss: 0.032190561294555664 (0.028314259650946503) + | > current_lr: 5e-06 + | > step_time: 0.1524 (0.1408803076811239) + | > loader_time: 0.0104 (0.011140615167752117) + + + --> TIME: 2024-04-09 17:46:58 -- STEP: 192/1354 -- GLOBAL_STEP: 36750 + | > loss_text_ce: 0.03457609936594963 (0.043620936262110895) + | > loss_mel_ce: 1.7106353044509888 (2.319634217148026) + | > loss: 0.020776325836777687 (0.028133990466206644) + | > current_lr: 5e-06 + | > step_time: 0.158 (0.13925336177150402) + | > loader_time: 0.0102 (0.010997095455725988) + + + --> TIME: 2024-04-09 17:47:09 -- STEP: 242/1354 -- GLOBAL_STEP: 36800 + | > loss_text_ce: 0.04007510468363762 (0.043510104414106424) + | > loss_mel_ce: 1.7781881093978882 (2.333949906766907) + | > loss: 0.021645991131663322 (0.028303095955432445) + | > current_lr: 5e-06 + | > step_time: 0.1006 (0.14112822773042782) + | > loader_time: 0.0099 (0.010951647088547383) + + + --> TIME: 2024-04-09 17:47:20 -- STEP: 292/1354 -- GLOBAL_STEP: 36850 + | > loss_text_ce: 0.052661314606666565 (0.043799206072286) + | > loss_mel_ce: 2.909630298614502 (2.334368522036564) + | > loss: 0.03526537865400314 (0.028311521152978483) + | > current_lr: 5e-06 + | > step_time: 0.0733 (0.14060648336802442) + | > loader_time: 0.0126 (0.01093855704346748) + + + --> TIME: 2024-04-09 17:47:31 -- STEP: 342/1354 -- GLOBAL_STEP: 36900 + | > loss_text_ce: 0.05113416537642479 (0.04370068715155477) + | > loss_mel_ce: 2.72780442237854 (2.3306092149332946) + | > loss: 0.03308260068297386 (0.02826559464093189) + | > current_lr: 5e-06 + | > step_time: 0.0898 (0.14066414038340255) + | > loader_time: 0.01 (0.010902777052762207) + + + --> TIME: 2024-04-09 17:47:42 -- STEP: 392/1354 -- GLOBAL_STEP: 36950 + | > loss_text_ce: 0.04149128869175911 (0.043569506355086204) + | > loss_mel_ce: 2.3475539684295654 (2.318742345790473) + | > loss: 0.028441015630960464 (0.028122760737980052) + | > current_lr: 5e-06 + | > step_time: 0.1643 (0.14135391432411817) + | > loader_time: 0.0098 (0.010903791505463256) + + + --> TIME: 2024-04-09 17:47:52 -- STEP: 442/1354 -- GLOBAL_STEP: 37000 + | > loss_text_ce: 0.038987115025520325 (0.04353610330948198) + | > loss_mel_ce: 2.0079774856567383 (2.3127293762038734) + | > loss: 0.02436862699687481 (0.028050780075083895) + | > current_lr: 5e-06 + | > step_time: 0.1638 (0.14127178051892458) + | > loader_time: 0.0102 (0.010889533418336053) + + + --> TIME: 2024-04-09 17:48:27 -- STEP: 492/1354 -- GLOBAL_STEP: 37050 + | > loss_text_ce: 0.04695754498243332 (0.04376004381120452) + | > loss_mel_ce: 2.0269458293914795 (2.310482863488236) + | > loss: 0.0246893260627985 (0.028026701789727906) + | > current_lr: 5e-06 + | > step_time: 0.1057 (0.14130607707713683) + | > loader_time: 0.0117 (0.010999476037374346) + + + --> TIME: 2024-04-09 17:48:38 -- STEP: 542/1354 -- GLOBAL_STEP: 37100 + | > loss_text_ce: 0.054248638451099396 (0.04370794684808306) + | > loss_mel_ce: 2.667419195175171 (2.3084892276468314) + | > loss: 0.03240080550312996 (0.028002347826366594) + | > current_lr: 5e-06 + | > step_time: 0.1119 (0.1414708607311179) + | > loader_time: 0.0096 (0.010982921642570908) + + + --> TIME: 2024-04-09 17:48:50 -- STEP: 592/1354 -- GLOBAL_STEP: 37150 + | > loss_text_ce: 0.03945259004831314 (0.04361431535043931) + | > loss_mel_ce: 1.929253339767456 (2.3061252757906927) + | > loss: 0.023436974734067917 (0.027973090880550444) + | > current_lr: 5e-06 + | > step_time: 0.186 (0.14266005198697793) + | > loader_time: 0.0101 (0.010953332926775949) + + + --> TIME: 2024-04-09 17:49:00 -- STEP: 642/1354 -- GLOBAL_STEP: 37200 + | > loss_text_ce: 0.039824895560741425 (0.04374461172502546) + | > loss_mel_ce: 2.800802230834961 (2.3038214989169) + | > loss: 0.03381698951125145 (0.02794721612062688) + | > current_lr: 5e-06 + | > step_time: 0.1407 (0.14192310374845227) + | > loader_time: 0.0102 (0.010927883264060323) + + + --> TIME: 2024-04-09 17:49:11 -- STEP: 692/1354 -- GLOBAL_STEP: 37250 + | > loss_text_ce: 0.032733965665102005 (0.04360357006591871) + | > loss_mel_ce: 2.054258346557617 (2.302572599466826) + | > loss: 0.024845145642757416 (0.02793066919447499) + | > current_lr: 5e-06 + | > step_time: 0.2135 (0.1422932141089026) + | > loader_time: 0.0118 (0.01090588183761331) + + + --> TIME: 2024-04-09 17:49:23 -- STEP: 742/1354 -- GLOBAL_STEP: 37300 + | > loss_text_ce: 0.03898249566555023 (0.04350679011066571) + | > loss_mel_ce: 2.3512001037597656 (2.2977343159062493) + | > loss: 0.028454553335905075 (0.027871918426500258) + | > current_lr: 5e-06 + | > step_time: 0.2061 (0.1428397710753901) + | > loader_time: 0.0099 (0.010879800647416832) + + + --> TIME: 2024-04-09 17:49:34 -- STEP: 792/1354 -- GLOBAL_STEP: 37350 + | > loss_text_ce: 0.04463445395231247 (0.04348023742616105) + | > loss_mel_ce: 2.2146286964416504 (2.2963241681155515) + | > loss: 0.02689598873257637 (0.027854814849125994) + | > current_lr: 5e-06 + | > step_time: 0.1753 (0.1431160153764666) + | > loader_time: 0.0101 (0.010865339727112728) + + + --> TIME: 2024-04-09 17:49:45 -- STEP: 842/1354 -- GLOBAL_STEP: 37400 + | > loss_text_ce: 0.04603103548288345 (0.04336869393667816) + | > loss_mel_ce: 2.985290288925171 (2.2977654365490836) + | > loss: 0.03608715906739235 (0.027870644920344096) + | > current_lr: 5e-06 + | > step_time: 0.1112 (0.14340023813225033) + | > loader_time: 0.0092 (0.010849127293765673) + + + --> TIME: 2024-04-09 17:49:56 -- STEP: 892/1354 -- GLOBAL_STEP: 37450 + | > loss_text_ce: 0.05544527992606163 (0.043270043661241604) + | > loss_mel_ce: 2.6163947582244873 (2.289602333053344) + | > loss: 0.03180762007832527 (0.02777229069790598) + | > current_lr: 5e-06 + | > step_time: 0.1922 (0.1437604486140435) + | > loader_time: 0.0114 (0.010836178144531923) + + + --> TIME: 2024-04-09 17:50:07 -- STEP: 942/1354 -- GLOBAL_STEP: 37500 + | > loss_text_ce: 0.027818357571959496 (0.04306769379732971) + | > loss_mel_ce: 1.8803640604019165 (2.2827707215858863) + | > loss: 0.022716457024216652 (0.02768855306938797) + | > current_lr: 5e-06 + | > step_time: 0.1873 (0.1439592709207231) + | > loader_time: 0.0117 (0.010826223215479748) + + + --> TIME: 2024-04-09 17:50:19 -- STEP: 992/1354 -- GLOBAL_STEP: 37550 + | > loss_text_ce: 0.04451704025268555 (0.0431162842188872) + | > loss_mel_ce: 1.9979853630065918 (2.28327443132237) + | > loss: 0.024315504357218742 (0.0276951280736079) + | > current_lr: 5e-06 + | > step_time: 0.1011 (0.1442004576806099) + | > loader_time: 0.0116 (0.010810973903825195) + + + --> TIME: 2024-04-09 17:50:29 -- STEP: 1042/1354 -- GLOBAL_STEP: 37600 + | > loss_text_ce: 0.04914252832531929 (0.04310815925611589) + | > loss_mel_ce: 1.6984755992889404 (2.282936298172212) + | > loss: 0.020804977044463158 (0.027691005953478547) + | > current_lr: 5e-06 + | > step_time: 0.1522 (0.1439602315540277) + | > loader_time: 0.0114 (0.010781873439415392) + + + --> TIME: 2024-04-09 17:50:40 -- STEP: 1092/1354 -- GLOBAL_STEP: 37650 + | > loss_text_ce: 0.04707148298621178 (0.0431923587165173) + | > loss_mel_ce: 1.63372004032135 (2.2807501928382727) + | > loss: 0.020009422674775124 (0.027665983253557763) + | > current_lr: 5e-06 + | > step_time: 0.0686 (0.1436549062693949) + | > loader_time: 0.0099 (0.010765605357103717) + + + --> TIME: 2024-04-09 17:50:51 -- STEP: 1142/1354 -- GLOBAL_STEP: 37700 + | > loss_text_ce: 0.0547962561249733 (0.04309983731568805) + | > loss_mel_ce: 2.7690329551696777 (2.274684972865358) + | > loss: 0.033617015928030014 (0.027592676808330135) + | > current_lr: 5e-06 + | > step_time: 0.1519 (0.14354613011021125) + | > loader_time: 0.0101 (0.010755507624504224) + + + --> TIME: 2024-04-09 17:51:02 -- STEP: 1192/1354 -- GLOBAL_STEP: 37750 + | > loss_text_ce: 0.037035271525382996 (0.04302534496649206) + | > loss_mel_ce: 1.8647974729537964 (2.268509257409797) + | > loss: 0.022640865296125412 (0.027518269564525126) + | > current_lr: 5e-06 + | > step_time: 0.1562 (0.14365119362037468) + | > loader_time: 0.0104 (0.010768771571600985) + + + --> TIME: 2024-04-09 17:51:12 -- STEP: 1242/1354 -- GLOBAL_STEP: 37800 + | > loss_text_ce: 0.045179400593042374 (0.043021680884687184) + | > loss_mel_ce: 2.3075671195983887 (2.270420256012304) + | > loss: 0.02800888754427433 (0.027540975924425302) + | > current_lr: 5e-06 + | > step_time: 0.145 (0.1432307649541786) + | > loader_time: 0.0097 (0.010758073049658918) + + + --> TIME: 2024-04-09 17:51:23 -- STEP: 1292/1354 -- GLOBAL_STEP: 37850 + | > loss_text_ce: 0.043354298919439316 (0.04302791917650466) + | > loss_mel_ce: 1.9709659814834595 (2.2704561492514674) + | > loss: 0.023980004712939262 (0.027541477486185552) + | > current_lr: 5e-06 + | > step_time: 0.1122 (0.14307951096779806) + | > loader_time: 0.0098 (0.01076054259350424) + + + --> TIME: 2024-04-09 17:51:33 -- STEP: 1342/1354 -- GLOBAL_STEP: 37900 + | > loss_text_ce: 0.04869145154953003 (0.04295993157996951) + | > loss_mel_ce: 2.510118007659912 (2.268744140952249) + | > loss: 0.03046201914548874 (0.027520287064377386) + | > current_lr: 5e-06 + | > step_time: 0.153 (0.1430266749130455) + | > loader_time: 0.0114 (0.010770700549937203) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.02674277012164776 (+0.0013294586768517122) + | > avg_loss_text_ce: 0.0488099054648326 (-0.0004864936837783232) + | > avg_loss_mel_ce: 2.8565210745884824 (-0.0015962490668663953) + | > avg_loss: 2.9053309697371263 (-0.0020827696873588764) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_37912.pth + + > EPOCH: 29/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 17:52:28)  + + --> TIME: 2024-04-09 17:52:38 -- STEP: 38/1354 -- GLOBAL_STEP: 37950 + | > loss_text_ce: 0.040391020476818085 (0.04136374872177839) + | > loss_mel_ce: 2.524327278137207 (2.2049552233595593) + | > loss: 0.030532360076904297 (0.026741893128737024) + | > current_lr: 5e-06 + | > step_time: 0.1455 (0.1468264429192794) + | > loader_time: 0.0157 (0.012295729235598917) + + + --> TIME: 2024-04-09 17:52:48 -- STEP: 88/1354 -- GLOBAL_STEP: 38000 + | > loss_text_ce: 0.05565667897462845 (0.04265340222892436) + | > loss_mel_ce: 2.6816184520721436 (2.2496437213637623) + | > loss: 0.032586608082056046 (0.027289251945066182) + | > current_lr: 5e-06 + | > step_time: 0.1369 (0.13953534039584076) + | > loader_time: 0.0126 (0.011571120132099499) + + + --> TIME: 2024-04-09 17:53:23 -- STEP: 138/1354 -- GLOBAL_STEP: 38050 + | > loss_text_ce: 0.038243819028139114 (0.042696615708047066) + | > loss_mel_ce: 2.324416399002075 (2.257815694031508) + | > loss: 0.028126906603574753 (0.027387051714881174) + | > current_lr: 5e-06 + | > step_time: 0.2167 (0.14006838591202445) + | > loader_time: 0.0145 (0.012591679890950521) + + + --> TIME: 2024-04-09 17:53:33 -- STEP: 188/1354 -- GLOBAL_STEP: 38100 + | > loss_text_ce: 0.03976792097091675 (0.04300039084231918) + | > loss_mel_ce: 2.2570908069610596 (2.276548013091087) + | > loss: 0.02734355814754963 (0.02761367192749489) + | > current_lr: 5e-06 + | > step_time: 0.1155 (0.13781694782541154) + | > loader_time: 0.0111 (0.01234412446935126) + + + --> TIME: 2024-04-09 17:53:43 -- STEP: 238/1354 -- GLOBAL_STEP: 38150 + | > loss_text_ce: 0.04727553203701973 (0.04305948591751963) + | > loss_mel_ce: 1.6912091970443726 (2.283804151941748) + | > loss: 0.02069624699652195 (0.027700758050000218) + | > current_lr: 5e-06 + | > step_time: 0.1121 (0.13786743368421286) + | > loader_time: 0.0107 (0.012231077466692244) + + + --> TIME: 2024-04-09 17:53:54 -- STEP: 288/1354 -- GLOBAL_STEP: 38200 + | > loss_text_ce: 0.05072663724422455 (0.04288311444300538) + | > loss_mel_ce: 2.0120136737823486 (2.2674012740867004) + | > loss: 0.024556433781981468 (0.027503386007285576) + | > current_lr: 5e-06 + | > step_time: 0.1602 (0.13848502520057898) + | > loader_time: 0.0106 (0.012160443597369723) + + + --> TIME: 2024-04-09 17:54:06 -- STEP: 338/1354 -- GLOBAL_STEP: 38250 + | > loss_text_ce: 0.04182622954249382 (0.04256919854094644) + | > loss_mel_ce: 2.1636319160461426 (2.2591117946706576) + | > loss: 0.026255454868078232 (0.027400964636840587) + | > current_lr: 5e-06 + | > step_time: 0.1334 (0.1402802615476078) + | > loader_time: 0.01 (0.012017320599076316) + + + --> TIME: 2024-04-09 17:54:16 -- STEP: 388/1354 -- GLOBAL_STEP: 38300 + | > loss_text_ce: 0.04296444356441498 (0.04254799205617806) + | > loss_mel_ce: 2.541318893432617 (2.2566195386893986) + | > loss: 0.03076527826488018 (0.027371042463256372) + | > current_lr: 5e-06 + | > step_time: 0.0995 (0.13869786631200728) + | > loader_time: 0.0102 (0.01194697987173022) + + + --> TIME: 2024-04-09 17:54:27 -- STEP: 438/1354 -- GLOBAL_STEP: 38350 + | > loss_text_ce: 0.04995952919125557 (0.04255527324685359) + | > loss_mel_ce: 2.624175548553467 (2.2448515765471004) + | > loss: 0.03183494135737419 (0.027231034335837654) + | > current_lr: 5e-06 + | > step_time: 0.1285 (0.13934825109020202) + | > loader_time: 0.0111 (0.011893875522700622) + + + --> TIME: 2024-04-09 17:54:37 -- STEP: 488/1354 -- GLOBAL_STEP: 38400 + | > loss_text_ce: 0.04303794726729393 (0.04253499630502746) + | > loss_mel_ce: 1.749223232269287 (2.235406933138605) + | > loss: 0.021336441859602928 (0.027118356736208937) + | > current_lr: 5e-06 + | > step_time: 0.1637 (0.13954764604568468) + | > loader_time: 0.0107 (0.011825702229484187) + + + --> TIME: 2024-04-09 17:54:48 -- STEP: 538/1354 -- GLOBAL_STEP: 38450 + | > loss_text_ce: 0.03758852556347847 (0.042733746234562976) + | > loss_mel_ce: 1.9178717136383057 (2.235021181376893) + | > loss: 0.023279288783669472 (0.027116130519879905) + | > current_lr: 5e-06 + | > step_time: 0.165 (0.13936051914683045) + | > loader_time: 0.0118 (0.011771120546475668) + + + --> TIME: 2024-04-09 17:55:00 -- STEP: 588/1354 -- GLOBAL_STEP: 38500 + | > loss_text_ce: 0.03525116667151451 (0.04275994381683618) + | > loss_mel_ce: 1.9207432270050049 (2.24522548498345) + | > loss: 0.023285647854208946 (0.027237922198702993) + | > current_lr: 5e-06 + | > step_time: 0.1294 (0.14063407612495665) + | > loader_time: 0.0119 (0.0117312535947683) + + + --> TIME: 2024-04-09 17:55:11 -- STEP: 638/1354 -- GLOBAL_STEP: 38550 + | > loss_text_ce: 0.028384407982230186 (0.04266187054072988) + | > loss_mel_ce: 2.106564998626709 (2.239455914142363) + | > loss: 0.025416065007448196 (0.027168069290271653) + | > current_lr: 5e-06 + | > step_time: 0.1561 (0.1408644484875716) + | > loader_time: 0.0126 (0.011679651968905186) + + + --> TIME: 2024-04-09 17:55:22 -- STEP: 688/1354 -- GLOBAL_STEP: 38600 + | > loss_text_ce: 0.0330771878361702 (0.04254611078541464) + | > loss_mel_ce: 1.7390270233154297 (2.236727452122195) + | > loss: 0.021096479147672653 (0.027134209515277816) + | > current_lr: 5e-06 + | > step_time: 0.1918 (0.14125719389250097) + | > loader_time: 0.0118 (0.011635206813036019) + + + --> TIME: 2024-04-09 17:55:32 -- STEP: 738/1354 -- GLOBAL_STEP: 38650 + | > loss_text_ce: 0.036389827728271484 (0.04263279465377977) + | > loss_mel_ce: 1.4948827028274536 (2.2347497268905503) + | > loss: 0.01822943612933159 (0.027111697120485993) + | > current_lr: 5e-06 + | > step_time: 0.1388 (0.14097759814120225) + | > loader_time: 0.0105 (0.011597475062217826) + + + --> TIME: 2024-04-09 17:55:43 -- STEP: 788/1354 -- GLOBAL_STEP: 38700 + | > loss_text_ce: 0.04374122992157936 (0.04257339564374256) + | > loss_mel_ce: 2.696702241897583 (2.228263564143086) + | > loss: 0.03262432664632797 (0.027033773783906388) + | > current_lr: 5e-06 + | > step_time: 0.1286 (0.14097716723601816) + | > loader_time: 0.0111 (0.011573498624230408) + + + --> TIME: 2024-04-09 17:55:54 -- STEP: 838/1354 -- GLOBAL_STEP: 38750 + | > loss_text_ce: 0.046196017414331436 (0.04256884938857222) + | > loss_mel_ce: 2.761727809906006 (2.223576276925415) + | > loss: 0.03342766314744949 (0.026977918618680968) + | > current_lr: 5e-06 + | > step_time: 0.1337 (0.14133599350731235) + | > loader_time: 0.0122 (0.011557496817413557) + + + --> TIME: 2024-04-09 17:56:05 -- STEP: 888/1354 -- GLOBAL_STEP: 38800 + | > loss_text_ce: 0.0388466976583004 (0.042622602760775) + | > loss_mel_ce: 1.8497885465621948 (2.2260524158676493) + | > loss: 0.02248375304043293 (0.027008036389602936) + | > current_lr: 5e-06 + | > step_time: 0.1075 (0.1415268848608205) + | > loader_time: 0.0095 (0.01156304843790897) + + + --> TIME: 2024-04-09 17:56:17 -- STEP: 938/1354 -- GLOBAL_STEP: 38850 + | > loss_text_ce: 0.02995891496539116 (0.04250028217112078) + | > loss_mel_ce: 1.8524765968322754 (2.2204835224253303) + | > loss: 0.022409947589039803 (0.026940283847095042) + | > current_lr: 5e-06 + | > step_time: 0.1016 (0.14197661627584424) + | > loader_time: 0.0129 (0.011527830857966253) + + + --> TIME: 2024-04-09 17:56:28 -- STEP: 988/1354 -- GLOBAL_STEP: 38900 + | > loss_text_ce: 0.031672850251197815 (0.042424840777207484) + | > loss_mel_ce: 2.046942949295044 (2.220882780638785) + | > loss: 0.024745427072048187 (0.02694413879874958) + | > current_lr: 5e-06 + | > step_time: 0.1891 (0.14222425318922585) + | > loader_time: 0.01 (0.01147820930249296) + + + --> TIME: 2024-04-09 17:56:38 -- STEP: 1038/1354 -- GLOBAL_STEP: 38950 + | > loss_text_ce: 0.03891759365797043 (0.04236479423875581) + | > loss_mel_ce: 1.8565359115600586 (2.2178895030177874) + | > loss: 0.022564923390746117 (0.02690778969408431) + | > current_lr: 5e-06 + | > step_time: 0.16 (0.14193499134225418) + | > loader_time: 0.0112 (0.01143246946536967) + + + --> TIME: 2024-04-09 17:56:50 -- STEP: 1088/1354 -- GLOBAL_STEP: 39000 + | > loss_text_ce: 0.026552388444542885 (0.04224717127748162) + | > loss_mel_ce: 1.6101019382476807 (2.213469991679579) + | > loss: 0.019483981654047966 (0.026853776197051895) + | > current_lr: 5e-06 + | > step_time: 0.2071 (0.14238560638006992) + | > loader_time: 0.01 (0.011410257395576036) + + + --> TIME: 2024-04-09 17:57:22 -- STEP: 1138/1354 -- GLOBAL_STEP: 39050 + | > loss_text_ce: 0.05025067552924156 (0.04218979109969047) + | > loss_mel_ce: 1.7755787372589111 (2.2103246127783955) + | > loss: 0.021736064925789833 (0.026815648112930555) + | > current_lr: 5e-06 + | > step_time: 0.1121 (0.14229400924722846) + | > loader_time: 0.0114 (0.011459591099583206) + + + --> TIME: 2024-04-09 17:57:33 -- STEP: 1188/1354 -- GLOBAL_STEP: 39100 + | > loss_text_ce: 0.03917219862341881 (0.04211365482033001) + | > loss_mel_ce: 1.3151562213897705 (2.2066344284650072) + | > loss: 0.016122957691550255 (0.02677081097058501) + | > current_lr: 5e-06 + | > step_time: 0.1606 (0.1427611072456797) + | > loader_time: 0.0102 (0.011445202410020418) + + + --> TIME: 2024-04-09 17:57:44 -- STEP: 1238/1354 -- GLOBAL_STEP: 39150 + | > loss_text_ce: 0.03843129426240921 (0.0422090649189539) + | > loss_mel_ce: 1.6755999326705933 (2.2091342262767277) + | > loss: 0.020405134186148643 (0.026801706295496573) + | > current_lr: 5e-06 + | > step_time: 0.1165 (0.14237110622479948) + | > loader_time: 0.0098 (0.011418363196784355) + + + --> TIME: 2024-04-09 17:57:55 -- STEP: 1288/1354 -- GLOBAL_STEP: 39200 + | > loss_text_ce: 0.03307890519499779 (0.04226350679600279) + | > loss_mel_ce: 2.0443222522735596 (2.2117389841294486) + | > loss: 0.024730967357754707 (0.026833363436307087) + | > current_lr: 5e-06 + | > step_time: 0.2156 (0.14237341840074663) + | > loader_time: 0.0105 (0.011395702265804608) + + + --> TIME: 2024-04-09 17:58:05 -- STEP: 1338/1354 -- GLOBAL_STEP: 39250 + | > loss_text_ce: 0.04169275984168053 (0.04225195002406613) + | > loss_mel_ce: 1.818932294845581 (2.210139912696524) + | > loss: 0.022150298580527306 (0.026814189289177257) + | > current_lr: 5e-06 + | > step_time: 0.0882 (0.1421127194602749) + | > loader_time: 0.0103 (0.011368191473865372) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.026154279708862305 (-0.0005884904127854557) + | > avg_loss_text_ce: 0.04825985718231935 (-0.0005500482825132463) + | > avg_loss_mel_ce: 2.856355226956881 (-0.00016584763160132354) + | > avg_loss: 2.9046150904435377 (-0.0007158792935886282) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_39266.pth + + > EPOCH: 30/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 17:59:01)  + + --> TIME: 2024-04-09 17:59:09 -- STEP: 34/1354 -- GLOBAL_STEP: 39300 + | > loss_text_ce: 0.06301799416542053 (0.04313933493240791) + | > loss_mel_ce: 3.145674228668213 (2.3391445559613846) + | > loss: 0.038198716938495636 (0.02836052333826528) + | > current_lr: 5e-06 + | > step_time: 0.1603 (0.1463134849772734) + | > loader_time: 0.0111 (0.01232302188873291) + + + --> TIME: 2024-04-09 17:59:20 -- STEP: 84/1354 -- GLOBAL_STEP: 39350 + | > loss_text_ce: 0.031495191156864166 (0.043393817419807114) + | > loss_mel_ce: 1.5604331493377686 (2.2504399362064547) + | > loss: 0.018951527774333954 (0.027307545311660283) + | > current_lr: 5e-06 + | > step_time: 0.1445 (0.14087790818441479) + | > loader_time: 0.0112 (0.011825572876703171) + + + --> TIME: 2024-04-09 17:59:31 -- STEP: 134/1354 -- GLOBAL_STEP: 39400 + | > loss_text_ce: 0.0463699996471405 (0.04253321560794738) + | > loss_mel_ce: 2.3122055530548096 (2.2484445812097245) + | > loss: 0.02807828038930893 (0.027273545827049373) + | > current_lr: 5e-06 + | > step_time: 0.1581 (0.14214582585576754) + | > loader_time: 0.0103 (0.01158950044148004) + + + --> TIME: 2024-04-09 17:59:42 -- STEP: 184/1354 -- GLOBAL_STEP: 39450 + | > loss_text_ce: 0.03420976921916008 (0.042980106284275) + | > loss_mel_ce: 1.794577956199646 (2.267690872368605) + | > loss: 0.021771281957626343 (0.02750798842727976) + | > current_lr: 5e-06 + | > step_time: 0.1444 (0.1436725064464237) + | > loader_time: 0.0104 (0.01146464114603789) + + + --> TIME: 2024-04-09 17:59:53 -- STEP: 234/1354 -- GLOBAL_STEP: 39500 + | > loss_text_ce: 0.038061149418354034 (0.0423716178132046) + | > loss_mel_ce: 1.6770317554473877 (2.2473360274592014) + | > loss: 0.020417772233486176 (0.027258424872421045) + | > current_lr: 5e-06 + | > step_time: 0.1193 (0.1447725408097617) + | > loader_time: 0.0103 (0.011395697919731466) + + + --> TIME: 2024-04-09 18:00:04 -- STEP: 284/1354 -- GLOBAL_STEP: 39550 + | > loss_text_ce: 0.03747987747192383 (0.04209909365105797) + | > loss_mel_ce: 1.7051761150360107 (2.2396454194062194) + | > loss: 0.020745905116200447 (0.027163625639063157) + | > current_lr: 5e-06 + | > step_time: 0.1746 (0.14439172895861338) + | > loader_time: 0.0126 (0.011311011415132334) + + + --> TIME: 2024-04-09 18:00:15 -- STEP: 334/1354 -- GLOBAL_STEP: 39600 + | > loss_text_ce: 0.030090872198343277 (0.042186212553057136) + | > loss_mel_ce: 1.8978042602539062 (2.239651186380557) + | > loss: 0.02295113354921341 (0.027164731428450977) + | > current_lr: 5e-06 + | > step_time: 0.1628 (0.14385893744622863) + | > loader_time: 0.0132 (0.011223707370415422) + + + --> TIME: 2024-04-09 18:00:26 -- STEP: 384/1354 -- GLOBAL_STEP: 39650 + | > loss_text_ce: 0.04554452374577522 (0.042092137067811564) + | > loss_mel_ce: 2.8533661365509033 (2.236899548831085) + | > loss: 0.03451084345579147 (0.02713085390132618) + | > current_lr: 5e-06 + | > step_time: 0.1015 (0.1445856634527443) + | > loader_time: 0.0121 (0.011218569551904997) + + + --> TIME: 2024-04-09 18:00:37 -- STEP: 434/1354 -- GLOBAL_STEP: 39700 + | > loss_text_ce: 0.03400834649801254 (0.041892551722580114) + | > loss_mel_ce: 1.9214637279510498 (2.2381720213296785) + | > loss: 0.023279430344700813 (0.02714362636440291) + | > current_lr: 5e-06 + | > step_time: 0.1191 (0.14458496065183707) + | > loader_time: 0.0104 (0.011181556134729341) + + + --> TIME: 2024-04-09 18:00:49 -- STEP: 484/1354 -- GLOBAL_STEP: 39750 + | > loss_text_ce: 0.04990120604634285 (0.04185100182629198) + | > loss_mel_ce: 2.1317567825317383 (2.2353196678575404) + | > loss: 0.025972120463848114 (0.027109175129631943) + | > current_lr: 5e-06 + | > step_time: 0.154 (0.14540455952163558) + | > loader_time: 0.0106 (0.011170971984705649) + + + --> TIME: 2024-04-09 18:00:59 -- STEP: 534/1354 -- GLOBAL_STEP: 39800 + | > loss_text_ce: 0.030978690832853317 (0.041786927307785) + | > loss_mel_ce: 2.1032135486602783 (2.2312830186515256) + | > loss: 0.025407051667571068 (0.027060356999185863) + | > current_lr: 5e-06 + | > step_time: 0.1279 (0.1447785887378877) + | > loader_time: 0.0102 (0.01114092069618711) + + + --> TIME: 2024-04-09 18:01:10 -- STEP: 584/1354 -- GLOBAL_STEP: 39850 + | > loss_text_ce: 0.03747909143567085 (0.041906458038632605) + | > loss_mel_ce: 1.7361786365509033 (2.2320669436291474) + | > loss: 0.021114973351359367 (0.027071112425589596) + | > current_lr: 5e-06 + | > step_time: 0.133 (0.14397413885756694) + | > loader_time: 0.0099 (0.011120089521146804) + + + --> TIME: 2024-04-09 18:01:21 -- STEP: 634/1354 -- GLOBAL_STEP: 39900 + | > loss_text_ce: 0.037181396037340164 (0.04187592461834481) + | > loss_mel_ce: 2.0314862728118896 (2.227321060109965) + | > loss: 0.024626996368169785 (0.027014250323552428) + | > current_lr: 5e-06 + | > step_time: 0.0759 (0.14410903589206517) + | > loader_time: 0.0102 (0.011133354545015638) + + + --> TIME: 2024-04-09 18:01:32 -- STEP: 684/1354 -- GLOBAL_STEP: 39950 + | > loss_text_ce: 0.04886389151215553 (0.04191530043564263) + | > loss_mel_ce: 1.9864764213562012 (2.2288599029967657) + | > loss: 0.024230241775512695 (0.027033038657942876) + | > current_lr: 5e-06 + | > step_time: 0.0911 (0.14425777900985787) + | > loader_time: 0.0108 (0.011163388079369972) + + + --> TIME: 2024-04-09 18:01:43 -- STEP: 734/1354 -- GLOBAL_STEP: 40000 + | > loss_text_ce: 0.039835114032030106 (0.04179418359550698) + | > loss_mel_ce: 1.746026873588562 (2.2204954033001885) + | > loss: 0.02126026153564453 (0.02693201940029615) + | > current_lr: 5e-06 + | > step_time: 0.1643 (0.14423546641659013) + | > loader_time: 0.0108 (0.011187157449020676) + + + > CHECKPOINT : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/checkpoint_40000.pth + + --> TIME: 2024-04-09 18:02:33 -- STEP: 784/1354 -- GLOBAL_STEP: 40050 + | > loss_text_ce: 0.04098498821258545 (0.04179220404761029) + | > loss_mel_ce: 1.6882513761520386 (2.2119167763359697) + | > loss: 0.020586147904396057 (0.026829869315332287) + | > current_lr: 5e-06 + | > step_time: 0.1761 (0.14421685313691884) + | > loader_time: 0.0128 (0.011399640416612431) + + + --> TIME: 2024-04-09 18:02:44 -- STEP: 834/1354 -- GLOBAL_STEP: 40100 + | > loss_text_ce: 0.03105328232049942 (0.041729561930937756) + | > loss_mel_ce: 1.9148507118225098 (2.2072565529843904) + | > loss: 0.023165524005889893 (0.026773644721792104) + | > current_lr: 5e-06 + | > step_time: 0.1415 (0.14399494827508347) + | > loader_time: 0.0129 (0.011426371350276956) + + + --> TIME: 2024-04-09 18:02:55 -- STEP: 884/1354 -- GLOBAL_STEP: 40150 + | > loss_text_ce: 0.0508948378264904 (0.041682381581144255) + | > loss_mel_ce: 2.5194249153137207 (2.203973991299108) + | > loss: 0.030599044635891914 (0.026734004936041934) + | > current_lr: 5e-06 + | > step_time: 0.1897 (0.1443799531837393) + | > loader_time: 0.011 (0.011461288141449113) + + + --> TIME: 2024-04-09 18:03:06 -- STEP: 934/1354 -- GLOBAL_STEP: 40200 + | > loss_text_ce: 0.041144873946905136 (0.041630050388805004) + | > loss_mel_ce: 1.9663310050964355 (2.2030282206913157) + | > loss: 0.023898523300886154 (0.026722122762527716) + | > current_lr: 5e-06 + | > step_time: 0.1539 (0.14398333321802098) + | > loader_time: 0.0113 (0.011490410859886023) + + + --> TIME: 2024-04-09 18:03:16 -- STEP: 984/1354 -- GLOBAL_STEP: 40250 + | > loss_text_ce: 0.03392228111624718 (0.041701259767273235) + | > loss_mel_ce: 1.8950203657150269 (2.2029155377692344) + | > loss: 0.02296360395848751 (0.02672162903681003) + | > current_lr: 5e-06 + | > step_time: 0.1409 (0.14344499411621703) + | > loader_time: 0.0117 (0.011512849631348276) + + + --> TIME: 2024-04-09 18:03:27 -- STEP: 1034/1354 -- GLOBAL_STEP: 40300 + | > loss_text_ce: 0.04239063709974289 (0.0417230167647741) + | > loss_mel_ce: 1.7451165914535522 (2.2033584966419766) + | > loss: 0.021279847249388695 (0.026727161368416397) + | > current_lr: 5e-06 + | > step_time: 0.1899 (0.143559036448569) + | > loader_time: 0.0138 (0.011541182114031144) + + + --> TIME: 2024-04-09 18:03:39 -- STEP: 1084/1354 -- GLOBAL_STEP: 40350 + | > loss_text_ce: 0.05505306273698807 (0.04181692101288163) + | > loss_mel_ce: 2.907255172729492 (2.206955637659096) + | > loss: 0.03526557236909866 (0.026771102385849743) + | > current_lr: 5e-06 + | > step_time: 0.1118 (0.14384733068987005) + | > loader_time: 0.0117 (0.011554170138721537) + + + --> TIME: 2024-04-09 18:03:50 -- STEP: 1134/1354 -- GLOBAL_STEP: 40400 + | > loss_text_ce: 0.0447702631354332 (0.04185503223546285) + | > loss_mel_ce: 3.7026379108428955 (2.2075140994482387) + | > loss: 0.04461200162768364 (0.02677820444376433) + | > current_lr: 5e-06 + | > step_time: 0.1661 (0.14378356723348087) + | > loader_time: 0.0129 (0.011577583494640532) + + + --> TIME: 2024-04-09 18:04:01 -- STEP: 1184/1354 -- GLOBAL_STEP: 40450 + | > loss_text_ce: 0.05204419791698456 (0.04182649588187202) + | > loss_mel_ce: 2.355363607406616 (2.205103924749672) + | > loss: 0.02865961752831936 (0.026749172170860445) + | > current_lr: 5e-06 + | > step_time: 0.1877 (0.14395697978702746) + | > loader_time: 0.0115 (0.011610435070218262) + + + --> TIME: 2024-04-09 18:04:12 -- STEP: 1234/1354 -- GLOBAL_STEP: 40500 + | > loss_text_ce: 0.05086551606655121 (0.04183306651887286) + | > loss_mel_ce: 2.9268059730529785 (2.2037848966434685) + | > loss: 0.03544846922159195 (0.026733547669489734) + | > current_lr: 5e-06 + | > step_time: 0.1129 (0.14410015868213225) + | > loader_time: 0.0114 (0.01162978721785584) + + + --> TIME: 2024-04-09 18:04:23 -- STEP: 1284/1354 -- GLOBAL_STEP: 40550 + | > loss_text_ce: 0.05388049781322479 (0.04180875304227586) + | > loss_mel_ce: 2.5419771671295166 (2.2026053524091633) + | > loss: 0.03090306743979454 (0.026719216033207066) + | > current_lr: 5e-06 + | > step_time: 0.135 (0.1441255380802808) + | > loader_time: 0.012 (0.011652358038774534) + + + --> TIME: 2024-04-09 18:04:34 -- STEP: 1334/1354 -- GLOBAL_STEP: 40600 + | > loss_text_ce: 0.04416666924953461 (0.04180084422197213) + | > loss_mel_ce: 1.5153545141220093 (2.2003307901043603) + | > loss: 0.01856572926044464 (0.026692043756018923) + | > current_lr: 5e-06 + | > step_time: 0.1087 (0.14422125330214383) + | > loader_time: 0.0127 (0.011657821006145794) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.031234887930063102 (+0.005080608221200798) + | > avg_loss_text_ce: 0.04781859110181148 (-0.0004412660805078669) + | > avg_loss_mel_ce: 2.856790689321665 (+0.00043546236478375633) + | > avg_loss: 2.9046092950380764 (-5.795405461306302e-06) + + > BEST MODEL : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/best_model_40620.pth + + > EPOCH: 31/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 18:05:29)  + + --> TIME: 2024-04-09 18:05:36 -- STEP: 30/1354 -- GLOBAL_STEP: 40650 + | > loss_text_ce: 0.05710335075855255 (0.041650172385076685) + | > loss_mel_ce: 2.7984797954559326 (2.0741418917973835) + | > loss: 0.03399503976106644 (0.025188001369436584) + | > current_lr: 5e-06 + | > step_time: 0.0762 (0.13279569149017328) + | > loader_time: 0.0102 (0.011923623085021973) + + + --> TIME: 2024-04-09 18:05:46 -- STEP: 80/1354 -- GLOBAL_STEP: 40700 + | > loss_text_ce: 0.030809832736849785 (0.04207896240986883) + | > loss_mel_ce: 1.9002293348312378 (2.1225913614034666) + | > loss: 0.022988563403487206 (0.025769885256886484) + | > current_lr: 5e-06 + | > step_time: 0.1408 (0.13566441833972923) + | > loader_time: 0.0104 (0.012227660417556763) + + + --> TIME: 2024-04-09 18:05:58 -- STEP: 130/1354 -- GLOBAL_STEP: 40750 + | > loss_text_ce: 0.05498833954334259 (0.04104573730952465) + | > loss_mel_ce: 2.4315690994262695 (2.1320428197200503) + | > loss: 0.029601875692605972 (0.025870102328749804) + | > current_lr: 5e-06 + | > step_time: 0.162 (0.1404256692299475) + | > loader_time: 0.0101 (0.011817205869234525) + + + --> TIME: 2024-04-09 18:06:09 -- STEP: 180/1354 -- GLOBAL_STEP: 40800 + | > loss_text_ce: 0.04483550786972046 (0.04123679535049531) + | > loss_mel_ce: 2.896670341491699 (2.149046659469607) + | > loss: 0.03501792997121811 (0.026074803486052487) + | > current_lr: 5e-06 + | > step_time: 0.1759 (0.14098831547631152) + | > loader_time: 0.0112 (0.011781031555599631) + + + --> TIME: 2024-04-09 18:06:21 -- STEP: 230/1354 -- GLOBAL_STEP: 40850 + | > loss_text_ce: 0.04243215546011925 (0.04095963573326236) + | > loss_mel_ce: 1.7730602025985718 (2.1489143210908668) + | > loss: 0.02161300554871559 (0.02606992859231389) + | > current_lr: 5e-06 + | > step_time: 0.1353 (0.14499342856199848) + | > loader_time: 0.0107 (0.011762855363928749) + + + --> TIME: 2024-04-09 18:06:32 -- STEP: 280/1354 -- GLOBAL_STEP: 40900 + | > loss_text_ce: 0.029123429208993912 (0.041121730055393914) + | > loss_mel_ce: 1.606937050819397 (2.1496139573199438) + | > loss: 0.019476911053061485 (0.026080187263765504) + | > current_lr: 5e-06 + | > step_time: 0.2009 (0.145855496610914) + | > loader_time: 0.0114 (0.011785581282206942) + + + --> TIME: 2024-04-09 18:06:43 -- STEP: 330/1354 -- GLOBAL_STEP: 40950 + | > loss_text_ce: 0.04676895588636398 (0.041405822472138844) + | > loss_mel_ce: 1.9019649028778076 (2.1672303799426946) + | > loss: 0.023199211806058884 (0.026293288622841686) + | > current_lr: 5e-06 + | > step_time: 0.2076 (0.1460711471962206) + | > loader_time: 0.0111 (0.011753389329621283) + + + --> TIME: 2024-04-09 18:06:54 -- STEP: 380/1354 -- GLOBAL_STEP: 41000 + | > loss_text_ce: 0.026354359462857246 (0.04142796688860184) + | > loss_mel_ce: 1.8158284425735474 (2.1740215938342273) + | > loss: 0.021930746734142303 (0.02637440002964515) + | > current_lr: 5e-06 + | > step_time: 0.1549 (0.14522876614018485) + | > loader_time: 0.0103 (0.011732143477389686) + + + --> TIME: 2024-04-09 18:07:26 -- STEP: 430/1354 -- GLOBAL_STEP: 41050 + | > loss_text_ce: 0.028626710176467896 (0.041429029392121834) + | > loss_mel_ce: 0.7619134187698364 (2.1662804714469037) + | > loss: 0.009411191567778587 (0.02628225645180358) + | > current_lr: 5e-06 + | > step_time: 0.2062 (0.14409821310708684) + | > loader_time: 0.0127 (0.011921861005383866) + + + --> TIME: 2024-04-09 18:07:37 -- STEP: 480/1354 -- GLOBAL_STEP: 41100 + | > loss_text_ce: 0.05048484727740288 (0.04129267807196205) + | > loss_mel_ce: 2.8697288036346436 (2.1730283657709766) + | > loss: 0.034764450043439865 (0.026360965298954393) + | > current_lr: 5e-06 + | > step_time: 0.1133 (0.14413351416587825) + | > loader_time: 0.0119 (0.011974339187145232) + + + --> TIME: 2024-04-09 18:07:47 -- STEP: 530/1354 -- GLOBAL_STEP: 41150 + | > loss_text_ce: 0.055366456508636475 (0.04118850208326892) + | > loss_mel_ce: 2.8041200637817383 (2.171689233015169) + | > loss: 0.03404150903224945 (0.026343783059702166) + | > current_lr: 5e-06 + | > step_time: 0.1175 (0.1437446108404195) + | > loader_time: 0.0118 (0.012028780073489781) + + + --> TIME: 2024-04-09 18:07:58 -- STEP: 580/1354 -- GLOBAL_STEP: 41200 + | > loss_text_ce: 0.05952093005180359 (0.041264543300173394) + | > loss_mel_ce: 2.8705942630767822 (2.1681172276365355) + | > loss: 0.0348823256790638 (0.02630216444457142) + | > current_lr: 5e-06 + | > step_time: 0.1516 (0.143831024498775) + | > loader_time: 0.0136 (0.012084375578781647) + + + --> TIME: 2024-04-09 18:08:10 -- STEP: 630/1354 -- GLOBAL_STEP: 41250 + | > loss_text_ce: 0.04381178319454193 (0.041306568422014744) + | > loss_mel_ce: 2.4939520359039307 (2.1702042924033274) + | > loss: 0.030211474746465683 (0.026327510750187282) + | > current_lr: 5e-06 + | > step_time: 0.1518 (0.1442416895003546) + | > loader_time: 0.0106 (0.012132441808306972) + + + --> TIME: 2024-04-09 18:08:20 -- STEP: 680/1354 -- GLOBAL_STEP: 41300 + | > loss_text_ce: 0.039900269359350204 (0.041378425678019116) + | > loss_mel_ce: 2.850048780441284 (2.1732510589501426) + | > loss: 0.03440415486693382 (0.026364637211458198) + | > current_lr: 5e-06 + | > step_time: 0.12 (0.1433748396003948) + | > loader_time: 0.0125 (0.01205199676401474) + + + --> TIME: 2024-04-09 18:08:31 -- STEP: 730/1354 -- GLOBAL_STEP: 41350 + | > loss_text_ce: 0.04884237051010132 (0.04136554277565788) + | > loss_mel_ce: 2.237642526626587 (2.168625198161763) + | > loss: 0.02722005918622017 (0.0263094140714264) + | > current_lr: 5e-06 + | > step_time: 0.1333 (0.14382489184810693) + | > loader_time: 0.0139 (0.012027096095150454) + + + --> TIME: 2024-04-09 18:08:43 -- STEP: 780/1354 -- GLOBAL_STEP: 41400 + | > loss_text_ce: 0.04813070595264435 (0.04123462939825958) + | > loss_mel_ce: 2.549852132797241 (2.161780392053798) + | > loss: 0.030928367748856544 (0.02622636979660735) + | > current_lr: 5e-06 + | > step_time: 0.1228 (0.14394952884087192) + | > loader_time: 0.013 (0.011976015262114687) + + + --> TIME: 2024-04-09 18:08:53 -- STEP: 830/1354 -- GLOBAL_STEP: 41450 + | > loss_text_ce: 0.04224861413240433 (0.04128041411245083) + | > loss_mel_ce: 2.1053640842437744 (2.163718446766035) + | > loss: 0.025566820055246353 (0.026249986956546813) + | > current_lr: 5e-06 + | > step_time: 0.1616 (0.14377782976770964) + | > loader_time: 0.0131 (0.011919846017676655) + + + --> TIME: 2024-04-09 18:09:04 -- STEP: 880/1354 -- GLOBAL_STEP: 41500 + | > loss_text_ce: 0.04055878147482872 (0.0412443163466047) + | > loss_mel_ce: 1.500133752822876 (2.16012602103027) + | > loss: 0.018341578543186188 (0.02620679025051437) + | > current_lr: 5e-06 + | > step_time: 0.0943 (0.1438275345347144) + | > loader_time: 0.0124 (0.011869121410629959) + + + --> TIME: 2024-04-09 18:09:15 -- STEP: 930/1354 -- GLOBAL_STEP: 41550 + | > loss_text_ce: 0.03431180492043495 (0.04120374423281478) + | > loss_mel_ce: 1.8233716487884521 (2.1592162248267894) + | > loss: 0.022115278989076614 (0.026195476338347432) + | > current_lr: 5e-06 + | > step_time: 0.1525 (0.1436161284805625) + | > loader_time: 0.0102 (0.01182682155280984) + + + --> TIME: 2024-04-09 18:09:26 -- STEP: 980/1354 -- GLOBAL_STEP: 41600 + | > loss_text_ce: 0.04573970288038254 (0.041195779983714514) + | > loss_mel_ce: 2.2551074028015137 (2.1564333974706864) + | > loss: 0.027391036972403526 (0.0261622526283775) + | > current_lr: 5e-06 + | > step_time: 0.1738 (0.143750792376849) + | > loader_time: 0.0106 (0.011783938018643119) + + + --> TIME: 2024-04-09 18:09:37 -- STEP: 1030/1354 -- GLOBAL_STEP: 41650 + | > loss_text_ce: 0.04856168478727341 (0.04125506345508167) + | > loss_mel_ce: 3.1324620246887207 (2.161687366765678) + | > loss: 0.03786933049559593 (0.026225505635431662) + | > current_lr: 5e-06 + | > step_time: 0.1413 (0.14329051114980446) + | > loader_time: 0.0103 (0.0117459417546837) + + + --> TIME: 2024-04-09 18:09:47 -- STEP: 1080/1354 -- GLOBAL_STEP: 41700 + | > loss_text_ce: 0.02814212441444397 (0.04130748940321305) + | > loss_mel_ce: 1.727552890777588 (2.1652421086474694) + | > loss: 0.020901130512356758 (0.026268448099424985) + | > current_lr: 5e-06 + | > step_time: 0.2001 (0.14312213526831724) + | > loader_time: 0.0103 (0.011727961345955168) + + + --> TIME: 2024-04-09 18:09:59 -- STEP: 1130/1354 -- GLOBAL_STEP: 41750 + | > loss_text_ce: 0.04835442453622818 (0.04123269846514523) + | > loss_mel_ce: 3.0744540691375732 (2.162316318581587) + | > loss: 0.03717629238963127 (0.026232726895044328) + | > current_lr: 5e-06 + | > step_time: 0.1204 (0.14345930740896573) + | > loader_time: 0.0133 (0.011708113788503451) + + + --> TIME: 2024-04-09 18:10:10 -- STEP: 1180/1354 -- GLOBAL_STEP: 41800 + | > loss_text_ce: 0.04531683027744293 (0.041252886874885354) + | > loss_mel_ce: 1.5055522918701172 (2.1621000253042904) + | > loss: 0.018462728708982468 (0.0262303923169877) + | > current_lr: 5e-06 + | > step_time: 0.0929 (0.1435899823398913) + | > loader_time: 0.0099 (0.011710788031755858) + + + --> TIME: 2024-04-09 18:10:21 -- STEP: 1230/1354 -- GLOBAL_STEP: 41850 + | > loss_text_ce: 0.03941279649734497 (0.04129727492033223) + | > loss_mel_ce: 1.7110973596572876 (2.1667353809849006) + | > loss: 0.020839408040046692 (0.026286103549920684) + | > current_lr: 5e-06 + | > step_time: 0.1437 (0.14363598901081845) + | > loader_time: 0.0107 (0.011688862777337791) + + + --> TIME: 2024-04-09 18:10:32 -- STEP: 1280/1354 -- GLOBAL_STEP: 41900 + | > loss_text_ce: 0.03368477150797844 (0.04133236465859223) + | > loss_mel_ce: 1.34974205493927 (2.1673058728221766) + | > loss: 0.01646936684846878 (0.026293312841880828) + | > current_lr: 5e-06 + | > step_time: 0.1422 (0.14389138706028443) + | > loader_time: 0.0099 (0.01171798463910818) + + + --> TIME: 2024-04-09 18:10:43 -- STEP: 1330/1354 -- GLOBAL_STEP: 41950 + | > loss_text_ce: 0.031429171562194824 (0.04131558281894685) + | > loss_mel_ce: 1.5107671022415161 (2.1655689906805042) + | > loss: 0.018359480425715446 (0.026272435891757106) + | > current_lr: 5e-06 + | > step_time: 0.1192 (0.14388633097024772) + | > loader_time: 0.0099 (0.011693215370178225) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.026816313083355244 (-0.004418574846707858) + | > avg_loss_text_ce: 0.04740050091193272 (-0.00041809018987876145) + | > avg_loss_mel_ce: 2.860998410444993 (+0.004207721123328234) + | > avg_loss: 2.9083989216731143 (+0.0037896266350379015) + + + > EPOCH: 32/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 18:11:04)  + + --> TIME: 2024-04-09 18:11:11 -- STEP: 26/1354 -- GLOBAL_STEP: 42000 + | > loss_text_ce: 0.039875686168670654 (0.041321028382159196) + | > loss_mel_ce: 2.546319007873535 (2.2205933836790233) + | > loss: 0.030788034200668335 (0.026927553260555633) + | > current_lr: 5e-06 + | > step_time: 0.216 (0.15609688942249006) + | > loader_time: 0.0137 (0.011842562602116512) + + + --> TIME: 2024-04-09 18:11:46 -- STEP: 76/1354 -- GLOBAL_STEP: 42050 + | > loss_text_ce: 0.05845777317881584 (0.04218279955124384) + | > loss_mel_ce: 2.8692095279693604 (2.2147277138735104) + | > loss: 0.03485318273305893 (0.0268679828456554) + | > current_lr: 5e-06 + | > step_time: 0.1173 (0.1428204335664448) + | > loader_time: 0.0139 (0.012834056427604268) + + + --> TIME: 2024-04-09 18:11:57 -- STEP: 126/1354 -- GLOBAL_STEP: 42100 + | > loss_text_ce: 0.03877931833267212 (0.04190014438733222) + | > loss_mel_ce: 1.7494192123413086 (2.170115039462135) + | > loss: 0.021288078278303146 (0.026333514593600754) + | > current_lr: 5e-06 + | > step_time: 0.1692 (0.13962241013844814) + | > loader_time: 0.0132 (0.012535590974111402) + + + --> TIME: 2024-04-09 18:12:08 -- STEP: 176/1354 -- GLOBAL_STEP: 42150 + | > loss_text_ce: 0.04169502109289169 (0.04111807418733159) + | > loss_mel_ce: 1.512648582458496 (2.1363439871506262) + | > loss: 0.01850409060716629 (0.02592216784515503) + | > current_lr: 5e-06 + | > step_time: 0.1117 (0.1396385539661755) + | > loader_time: 0.0108 (0.01243228668516332) + + + --> TIME: 2024-04-09 18:12:18 -- STEP: 226/1354 -- GLOBAL_STEP: 42200 + | > loss_text_ce: 0.04210237041115761 (0.040986119794239) + | > loss_mel_ce: 3.2872302532196045 (2.1325372441680046) + | > loss: 0.039634913206100464 (0.02587527861670319) + | > current_lr: 5e-06 + | > step_time: 0.0676 (0.13943486720059828) + | > loader_time: 0.0129 (0.012338895713333532) + + + --> TIME: 2024-04-09 18:12:30 -- STEP: 276/1354 -- GLOBAL_STEP: 42250 + | > loss_text_ce: 0.04630541801452637 (0.04107154907145795) + | > loss_mel_ce: 2.185709238052368 (2.1427135921042906) + | > loss: 0.02657160349190235 (0.025997442611749622) + | > current_lr: 5e-06 + | > step_time: 0.1446 (0.14088941657024887) + | > loader_time: 0.0119 (0.01227013356443764) + + + --> TIME: 2024-04-09 18:12:40 -- STEP: 326/1354 -- GLOBAL_STEP: 42300 + | > loss_text_ce: 0.039439890533685684 (0.04127291168880427) + | > loss_mel_ce: 2.0684049129486084 (2.153474889284264) + | > loss: 0.02509339153766632 (0.02612795048833259) + | > current_lr: 5e-06 + | > step_time: 0.209 (0.1400890569745397) + | > loader_time: 0.0108 (0.012169398413114015) + + + --> TIME: 2024-04-09 18:12:52 -- STEP: 376/1354 -- GLOBAL_STEP: 42350 + | > loss_text_ce: 0.04798717051744461 (0.04130490883154439) + | > loss_mel_ce: 2.6556923389434814 (2.155345261414) + | > loss: 0.032186660915613174 (0.026150597715155877) + | > current_lr: 5e-06 + | > step_time: 0.2109 (0.14171575168345835) + | > loader_time: 0.0115 (0.012115036553524906) + + + --> TIME: 2024-04-09 18:13:04 -- STEP: 426/1354 -- GLOBAL_STEP: 42400 + | > loss_text_ce: 0.049381326884031296 (0.04101682481456532) + | > loss_mel_ce: 2.667311191558838 (2.145928915397662) + | > loss: 0.03234157711267471 (0.026035068781845305) + | > current_lr: 5e-06 + | > step_time: 0.145 (0.143638403762674) + | > loader_time: 0.0141 (0.012068243653561583) + + + --> TIME: 2024-04-09 18:13:15 -- STEP: 476/1354 -- GLOBAL_STEP: 42450 + | > loss_text_ce: 0.0241318978369236 (0.0408215966631322) + | > loss_mel_ce: 1.5544825792312622 (2.135488363123746) + | > loss: 0.018793029710650444 (0.025908452337372954) + | > current_lr: 5e-06 + | > step_time: 0.2148 (0.1439734026163565) + | > loader_time: 0.0109 (0.012009636193764306) + + + --> TIME: 2024-04-09 18:13:26 -- STEP: 526/1354 -- GLOBAL_STEP: 42500 + | > loss_text_ce: 0.04253815487027168 (0.040756039700325694) + | > loss_mel_ce: 2.816180944442749 (2.137809330519618) + | > loss: 0.034032370895147324 (0.025935302472570664) + | > current_lr: 5e-06 + | > step_time: 0.1095 (0.1448842341455669) + | > loader_time: 0.011 (0.012006682588120375) + + + --> TIME: 2024-04-09 18:13:37 -- STEP: 576/1354 -- GLOBAL_STEP: 42550 + | > loss_text_ce: 0.06566350907087326 (0.04076532050385141) + | > loss_mel_ce: 3.103135347366333 (2.132043426649441) + | > loss: 0.03772379830479622 (0.025866771249436878) + | > current_lr: 5e-06 + | > step_time: 0.1603 (0.14465571857160986) + | > loader_time: 0.0125 (0.01197638528214561) + + + --> TIME: 2024-04-09 18:13:48 -- STEP: 626/1354 -- GLOBAL_STEP: 42600 + | > loss_text_ce: 0.027779443189501762 (0.040809864945971566) + | > loss_mel_ce: 1.853930950164795 (2.1365232025853382) + | > loss: 0.022401314228773117 (0.02592063221497276) + | > current_lr: 5e-06 + | > step_time: 0.2156 (0.1444580132207169) + | > loader_time: 0.0106 (0.0119177495328763) + + + --> TIME: 2024-04-09 18:13:59 -- STEP: 676/1354 -- GLOBAL_STEP: 42650 + | > loss_text_ce: 0.032190270721912384 (0.0407954865691842) + | > loss_mel_ce: 1.9960167407989502 (2.1335306038870625) + | > loss: 0.024145321920514107 (0.025884834860765628) + | > current_lr: 5e-06 + | > step_time: 0.2092 (0.14393168017709032) + | > loader_time: 0.0116 (0.011900730739683785) + + + --> TIME: 2024-04-09 18:14:09 -- STEP: 726/1354 -- GLOBAL_STEP: 42700 + | > loss_text_ce: 0.03561434522271156 (0.04079896307835158) + | > loss_mel_ce: 2.0930118560791016 (2.1356051145506307) + | > loss: 0.025340788066387177 (0.025909572811515038) + | > current_lr: 5e-06 + | > step_time: 0.1558 (0.1434428291215712) + | > loader_time: 0.0114 (0.011866576087047893) + + + --> TIME: 2024-04-09 18:14:20 -- STEP: 776/1354 -- GLOBAL_STEP: 42750 + | > loss_text_ce: 0.037127189338207245 (0.04088002592133188) + | > loss_mel_ce: 2.3150312900543213 (2.1378010704959793) + | > loss: 0.02800188772380352 (0.025936680201431445) + | > current_lr: 5e-06 + | > step_time: 0.1441 (0.14307638480491253) + | > loader_time: 0.0139 (0.011856362070004955) + + + --> TIME: 2024-04-09 18:14:31 -- STEP: 826/1354 -- GLOBAL_STEP: 42800 + | > loss_text_ce: 0.03400725498795509 (0.04089461052767736) + | > loss_mel_ce: 1.8984907865524292 (2.1347787938741476) + | > loss: 0.023005930706858635 (0.025900874345520502) + | > current_lr: 5e-06 + | > step_time: 0.1325 (0.14332327196153544) + | > loader_time: 0.0133 (0.011848058596650269) + + + --> TIME: 2024-04-09 18:14:42 -- STEP: 876/1354 -- GLOBAL_STEP: 42850 + | > loss_text_ce: 0.04998960345983505 (0.040890504229711765) + | > loss_mel_ce: 2.7808117866516113 (2.136041075943814) + | > loss: 0.033700019121170044 (0.02591585262240544) + | > current_lr: 5e-06 + | > step_time: 0.1868 (0.14356509980545742) + | > loader_time: 0.0137 (0.01183001091491141) + + + --> TIME: 2024-04-09 18:14:53 -- STEP: 926/1354 -- GLOBAL_STEP: 42900 + | > loss_text_ce: 0.02482476644217968 (0.04095534473601288) + | > loss_mel_ce: 1.8701409101486206 (2.1410709902740725) + | > loss: 0.022559115663170815 (0.025976504460701284) + | > current_lr: 5e-06 + | > step_time: 0.1919 (0.14312920405591795) + | > loader_time: 0.0112 (0.011802644245567892) + + + --> TIME: 2024-04-09 18:15:04 -- STEP: 976/1354 -- GLOBAL_STEP: 42950 + | > loss_text_ce: 0.0409904420375824 (0.040879232486419885) + | > loss_mel_ce: 2.1319026947021484 (2.1387640490639424) + | > loss: 0.025867775082588196 (0.025948134773853614) + | > current_lr: 5e-06 + | > step_time: 0.1747 (0.1437095508712237) + | > loader_time: 0.0108 (0.011798215694114803) + + + --> TIME: 2024-04-09 18:15:15 -- STEP: 1026/1354 -- GLOBAL_STEP: 43000 + | > loss_text_ce: 0.04083830118179321 (0.04090901684618718) + | > loss_mel_ce: 2.4915201663970947 (2.145434197742806) + | > loss: 0.030147124081850052 (0.026027895882362623) + | > current_lr: 5e-06 + | > step_time: 0.1399 (0.14348179392414948) + | > loader_time: 0.0105 (0.011787204946923202) + + + --> TIME: 2024-04-09 18:15:47 -- STEP: 1076/1354 -- GLOBAL_STEP: 43050 + | > loss_text_ce: 0.03419974818825722 (0.040915341630619606) + | > loss_mel_ce: 1.8065662384033203 (2.14128478608167) + | > loss: 0.021913880482316017 (0.025978573417069467) + | > current_lr: 5e-06 + | > step_time: 0.1425 (0.14332564324694494) + | > loader_time: 0.0124 (0.011844455531095475) + + + --> TIME: 2024-04-09 18:15:58 -- STEP: 1126/1354 -- GLOBAL_STEP: 43100 + | > loss_text_ce: 0.026038266718387604 (0.04098658723885701) + | > loss_mel_ce: 1.4877077341079712 (2.1462792918271343) + | > loss: 0.018020786345005035 (0.026038879982723494) + | > current_lr: 5e-06 + | > step_time: 0.2302 (0.14347899849419068) + | > loader_time: 0.0135 (0.011904062640391693) + + + --> TIME: 2024-04-09 18:16:09 -- STEP: 1176/1354 -- GLOBAL_STEP: 43150 + | > loss_text_ce: 0.03960663080215454 (0.04093982709958188) + | > loss_mel_ce: 1.855471134185791 (2.1450681630648747) + | > loss: 0.022560449317097664 (0.02602390511686832) + | > current_lr: 5e-06 + | > step_time: 0.14 (0.14359622017866913) + | > loader_time: 0.0131 (0.011922048468168089) + + + --> TIME: 2024-04-09 18:16:21 -- STEP: 1226/1354 -- GLOBAL_STEP: 43200 + | > loss_text_ce: 0.0350860059261322 (0.041012649020243916) + | > loss_mel_ce: 1.8359599113464355 (2.151209516879203) + | > loss: 0.022274356335401535 (0.02609788341186394) + | > current_lr: 5e-06 + | > step_time: 0.1591 (0.1439107596193599) + | > loader_time: 0.0127 (0.011918845806775243) + + + --> TIME: 2024-04-09 18:16:32 -- STEP: 1276/1354 -- GLOBAL_STEP: 43250 + | > loss_text_ce: 0.02632892318069935 (0.04094743755643129) + | > loss_mel_ce: 1.836240530014038 (2.146794375020509) + | > loss: 0.0221734456717968 (0.026044545881304202) + | > current_lr: 5e-06 + | > step_time: 0.1458 (0.14418405107569945) + | > loader_time: 0.0125 (0.011911199758045351) + + + --> TIME: 2024-04-09 18:16:42 -- STEP: 1326/1354 -- GLOBAL_STEP: 43300 + | > loss_text_ce: 0.05448700487613678 (0.041080557378809654) + | > loss_mel_ce: 1.9080064296722412 (2.151194984556864) + | > loss: 0.023363016545772552 (0.02609851885499885) + | > current_lr: 5e-06 + | > step_time: 0.0822 (0.14368004568741533) + | > loader_time: 0.0117 (0.011888674121816415) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.02992569483243502 (+0.0031093817490797752) + | > avg_loss_text_ce: 0.046999576286627695 (-0.0004009246253050269) + | > avg_loss_mel_ce: 2.8602512983175425 (-0.0007471121274504888) + | > avg_loss: 2.907250899534959 (-0.0011480221381554578) + + + > EPOCH: 33/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 18:17:05)  + + --> TIME: 2024-04-09 18:17:11 -- STEP: 22/1354 -- GLOBAL_STEP: 43350 + | > loss_text_ce: 0.03387821093201637 (0.04317640792578459) + | > loss_mel_ce: 1.7116676568984985 (2.204222110184756) + | > loss: 0.020780308172106743 (0.02675474485890432) + | > current_lr: 5e-06 + | > step_time: 0.1191 (0.14873820001428778) + | > loader_time: 0.0109 (0.01201296936381947) + + + --> TIME: 2024-04-09 18:17:22 -- STEP: 72/1354 -- GLOBAL_STEP: 43400 + | > loss_text_ce: 0.048380397260189056 (0.041161698884227195) + | > loss_mel_ce: 2.386019229888916 (2.193950103388892) + | > loss: 0.02898094803094864 (0.026608474384475913) + | > current_lr: 5e-06 + | > step_time: 0.0985 (0.14429527852270335) + | > loader_time: 0.0113 (0.012174430820677016) + + + --> TIME: 2024-04-09 18:17:33 -- STEP: 122/1354 -- GLOBAL_STEP: 43450 + | > loss_text_ce: 0.028278114274144173 (0.04117747535165704) + | > loss_mel_ce: 1.5981980562210083 (2.1933450337316187) + | > loss: 0.019362810999155045 (0.026601458892043007) + | > current_lr: 5e-06 + | > step_time: 0.2239 (0.14392701719628007) + | > loader_time: 0.0118 (0.012639934899377042) + + + --> TIME: 2024-04-09 18:17:44 -- STEP: 172/1354 -- GLOBAL_STEP: 43500 + | > loss_text_ce: 0.03533956781029701 (0.040663420450115624) + | > loss_mel_ce: 1.3744041919708252 (2.181562253902127) + | > loss: 0.016782663762569427 (0.026455067980731297) + | > current_lr: 5e-06 + | > step_time: 0.1615 (0.14638716536898946) + | > loader_time: 0.0141 (0.012562157109726307) + + + --> TIME: 2024-04-09 18:17:55 -- STEP: 222/1354 -- GLOBAL_STEP: 43550 + | > loss_text_ce: 0.05037244036793709 (0.04031176797978513) + | > loss_mel_ce: 1.6879215240478516 (2.1594063038224593) + | > loss: 0.020693976432085037 (0.02618712034529528) + | > current_lr: 5e-06 + | > step_time: 0.1389 (0.14538241399301077) + | > loader_time: 0.0107 (0.012353754258370615) + + + --> TIME: 2024-04-09 18:18:07 -- STEP: 272/1354 -- GLOBAL_STEP: 43600 + | > loss_text_ce: 0.03024442307651043 (0.040012261549503936) + | > loss_mel_ce: 1.9885005950927734 (2.1300922347342293) + | > loss: 0.024032678455114365 (0.025834577784681803) + | > current_lr: 5e-06 + | > step_time: 0.2077 (0.14664880756069643) + | > loader_time: 0.0108 (0.01224392477203818) + + + --> TIME: 2024-04-09 18:18:18 -- STEP: 322/1354 -- GLOBAL_STEP: 43650 + | > loss_text_ce: 0.03134213760495186 (0.03981663736728778) + | > loss_mel_ce: 1.606806993484497 (2.118414365356757) + | > loss: 0.01950177550315857 (0.02569322664325373) + | > current_lr: 5e-06 + | > step_time: 0.2185 (0.14700548249001835) + | > loader_time: 0.0104 (0.01214600645977519) + + + --> TIME: 2024-04-09 18:18:30 -- STEP: 372/1354 -- GLOBAL_STEP: 43700 + | > loss_text_ce: 0.026635782793164253 (0.03979752649382873) + | > loss_mel_ce: 1.5461996793746948 (2.104928007369403) + | > loss: 0.01872423104941845 (0.025532447282345053) + | > current_lr: 5e-06 + | > step_time: 0.1336 (0.14741617312995342) + | > loader_time: 0.0116 (0.012126820702706624) + + + --> TIME: 2024-04-09 18:18:41 -- STEP: 422/1354 -- GLOBAL_STEP: 43750 + | > loss_text_ce: 0.040258921682834625 (0.040097988241515406) + | > loss_mel_ce: 1.4531633853912354 (2.120242088044427) + | > loss: 0.01777883619070053 (0.02571833470474345) + | > current_lr: 5e-06 + | > step_time: 0.0765 (0.1471684154176036) + | > loader_time: 0.0104 (0.01211481523739784) + + + --> TIME: 2024-04-09 18:18:52 -- STEP: 472/1354 -- GLOBAL_STEP: 43800 + | > loss_text_ce: 0.03301087021827698 (0.040285109830388785) + | > loss_mel_ce: 1.4030548334121704 (2.1200907341504522) + | > loss: 0.01709602028131485 (0.02571876051291128) + | > current_lr: 5e-06 + | > step_time: 0.1393 (0.1465980440883315) + | > loader_time: 0.0101 (0.012078572632902768) + + + --> TIME: 2024-04-09 18:19:03 -- STEP: 522/1354 -- GLOBAL_STEP: 43850 + | > loss_text_ce: 0.03527342900633812 (0.040191558889074074) + | > loss_mel_ce: 1.9720364809036255 (2.1159321810779006) + | > loss: 0.02389654703438282 (0.025668140229594197) + | > current_lr: 5e-06 + | > step_time: 0.2002 (0.14679812791247038) + | > loader_time: 0.0104 (0.012025771926646039) + + + --> TIME: 2024-04-09 18:19:14 -- STEP: 572/1354 -- GLOBAL_STEP: 43900 + | > loss_text_ce: 0.043161869049072266 (0.04013263761111485) + | > loss_mel_ce: 1.924025058746338 (2.115671098544882) + | > loss: 0.023418892174959183 (0.025664330657141708) + | > current_lr: 5e-06 + | > step_time: 0.1134 (0.14667760909020502) + | > loader_time: 0.0119 (0.01196840301260249) + + + --> TIME: 2024-04-09 18:19:25 -- STEP: 622/1354 -- GLOBAL_STEP: 43950 + | > loss_text_ce: 0.035419464111328125 (0.03993844502632443) + | > loss_mel_ce: 1.9470791816711426 (2.1167964054648905) + | > loss: 0.023601174354553223 (0.02567541534440454) + | > current_lr: 5e-06 + | > step_time: 0.1662 (0.1468711497316025) + | > loader_time: 0.013 (0.011937010709879103) + + + --> TIME: 2024-04-09 18:19:37 -- STEP: 672/1354 -- GLOBAL_STEP: 44000 + | > loss_text_ce: 0.04753461107611656 (0.03992236353778496) + | > loss_mel_ce: 1.9167394638061523 (2.118269062201892) + | > loss: 0.02338421531021595 (0.02569275553936937) + | > current_lr: 5e-06 + | > step_time: 0.0984 (0.14712113071055674) + | > loader_time: 0.0129 (0.011914159925211049) + + + --> TIME: 2024-04-09 18:20:09 -- STEP: 722/1354 -- GLOBAL_STEP: 44050 + | > loss_text_ce: 0.0348644033074379 (0.0400983291082187) + | > loss_mel_ce: 1.7133536338806152 (2.121015644882524) + | > loss: 0.02081212028861046 (0.025727547774829197) + | > current_lr: 5e-06 + | > step_time: 0.159 (0.14707129021430632) + | > loader_time: 0.011 (0.011941115942027765) + + + --> TIME: 2024-04-09 18:20:21 -- STEP: 772/1354 -- GLOBAL_STEP: 44100 + | > loss_text_ce: 0.04340197145938873 (0.040063127059817565) + | > loss_mel_ce: 3.224231243133545 (2.1225156676738375) + | > loss: 0.03890039771795273 (0.025744986119607724) + | > current_lr: 5e-06 + | > step_time: 0.1048 (0.14752485004731447) + | > loader_time: 0.0121 (0.01197730722822674) + + + --> TIME: 2024-04-09 18:20:32 -- STEP: 822/1354 -- GLOBAL_STEP: 44150 + | > loss_text_ce: 0.033764541149139404 (0.03997668069436089) + | > loss_mel_ce: 2.213719129562378 (2.1147259793577406) + | > loss: 0.026755759492516518 (0.02565122262060352) + | > current_lr: 5e-06 + | > step_time: 0.1652 (0.1477023383416692) + | > loader_time: 0.0136 (0.01201244398335181) + + + --> TIME: 2024-04-09 18:20:43 -- STEP: 872/1354 -- GLOBAL_STEP: 44200 + | > loss_text_ce: 0.04586183279752731 (0.03991690367340626) + | > loss_mel_ce: 2.7319297790527344 (2.112556080684203) + | > loss: 0.03306894749403 (0.025624678888862287) + | > current_lr: 5e-06 + | > step_time: 0.0758 (0.1475185579663027) + | > loader_time: 0.0109 (0.012023027610341347) + + + --> TIME: 2024-04-09 18:20:54 -- STEP: 922/1354 -- GLOBAL_STEP: 44250 + | > loss_text_ce: 0.04715612903237343 (0.039866038476906805) + | > loss_mel_ce: 2.3641550540924072 (2.1051820565718127) + | > loss: 0.028706084936857224 (0.02553628734161435) + | > current_lr: 5e-06 + | > step_time: 0.1613 (0.1473882955997992) + | > loader_time: 0.0108 (0.01197727098899395) + + + --> TIME: 2024-04-09 18:21:06 -- STEP: 972/1354 -- GLOBAL_STEP: 44300 + | > loss_text_ce: 0.036249224096536636 (0.040020764979759926) + | > loss_mel_ce: 2.3369710445404053 (2.113346969394527) + | > loss: 0.028252622112631798 (0.02563533067143687) + | > current_lr: 5e-06 + | > step_time: 0.2169 (0.1475092538099724) + | > loader_time: 0.0124 (0.01196791409465021) + + + --> TIME: 2024-04-09 18:21:17 -- STEP: 1022/1354 -- GLOBAL_STEP: 44350 + | > loss_text_ce: 0.03954242169857025 (0.039970240956664384) + | > loss_mel_ce: 2.025935173034668 (2.1098725760752908) + | > loss: 0.024589018896222115 (0.025593367366962237) + | > current_lr: 5e-06 + | > step_time: 0.1531 (0.14791479502638757) + | > loader_time: 0.011 (0.011984990067677958) + + + --> TIME: 2024-04-09 18:21:29 -- STEP: 1072/1354 -- GLOBAL_STEP: 44400 + | > loss_text_ce: 0.031596001237630844 (0.03999434082956507) + | > loss_mel_ce: 1.8334369659423828 (2.110854336686099) + | > loss: 0.02220277301967144 (0.025605341893926596) + | > current_lr: 5e-06 + | > step_time: 0.1648 (0.14817644922591003) + | > loader_time: 0.0118 (0.011967610067396025) + + + --> TIME: 2024-04-09 18:21:41 -- STEP: 1122/1354 -- GLOBAL_STEP: 44450 + | > loss_text_ce: 0.03298940882086754 (0.039955264203077705) + | > loss_mel_ce: 1.660264253616333 (2.106473904348737) + | > loss: 0.02015778236091137 (0.02555272869231202) + | > current_lr: 5e-06 + | > step_time: 0.1306 (0.14839060527543155) + | > loader_time: 0.0103 (0.011940917016775639) + + + --> TIME: 2024-04-09 18:21:52 -- STEP: 1172/1354 -- GLOBAL_STEP: 44500 + | > loss_text_ce: 0.04634346812963486 (0.03993286489772862) + | > loss_mel_ce: 1.6557849645614624 (2.100937721468892) + | > loss: 0.0202634334564209 (0.025486555093511574) + | > current_lr: 5e-06 + | > step_time: 0.1441 (0.14841797897026424) + | > loader_time: 0.0104 (0.011915125944508628) + + + --> TIME: 2024-04-09 18:22:03 -- STEP: 1222/1354 -- GLOBAL_STEP: 44550 + | > loss_text_ce: 0.04629962891340256 (0.039972412555422054) + | > loss_mel_ce: 2.0764386653900146 (2.102882308382453) + | > loss: 0.025270694866776466 (0.025510175748074307) + | > current_lr: 5e-06 + | > step_time: 0.1538 (0.14841251794718308) + | > loader_time: 0.0111 (0.011893501453430875) + + + --> TIME: 2024-04-09 18:22:14 -- STEP: 1272/1354 -- GLOBAL_STEP: 44600 + | > loss_text_ce: 0.03470185771584511 (0.03995529801362196) + | > loss_mel_ce: 1.5000449419021606 (2.1001610553489587) + | > loss: 0.018270794302225113 (0.02547757612151802) + | > current_lr: 5e-06 + | > step_time: 0.1672 (0.14813929196423706) + | > loader_time: 0.0102 (0.011864129289890987) + + + --> TIME: 2024-04-09 18:22:25 -- STEP: 1322/1354 -- GLOBAL_STEP: 44650 + | > loss_text_ce: 0.04056703299283981 (0.03995108247541575) + | > loss_mel_ce: 2.1697540283203125 (2.1001163059211594) + | > loss: 0.02631334587931633 (0.02547699320672499) + | > current_lr: 5e-06 + | > step_time: 0.1737 (0.1478292447896661) + | > loader_time: 0.0111 (0.011840630046538613) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.027399705006526068 (-0.0025259898259089517) + | > avg_loss_text_ce: 0.04662407132295462 (-0.00037550496367307756) + | > avg_loss_mel_ce: 2.86202990091764 (+0.0017786026000976562) + | > avg_loss: 2.9086539561931906 (+0.001403056658231705) + + + > EPOCH: 34/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 18:22:47)  + + --> TIME: 2024-04-09 18:22:52 -- STEP: 18/1354 -- GLOBAL_STEP: 44700 + | > loss_text_ce: 0.028780043125152588 (0.042177483439445496) + | > loss_mel_ce: 1.4536598920822144 (2.1693315307299295) + | > loss: 0.01764809526503086 (0.026327488840454154) + | > current_lr: 5e-06 + | > step_time: 0.1664 (0.1364734040366279) + | > loader_time: 0.0115 (0.01302896605597602) + + + --> TIME: 2024-04-09 18:23:03 -- STEP: 68/1354 -- GLOBAL_STEP: 44750 + | > loss_text_ce: 0.03186851739883423 (0.040182869552689446) + | > loss_mel_ce: 1.5219523906707764 (2.0684786789557514) + | > loss: 0.01849786750972271 (0.025103114353602424) + | > current_lr: 5e-06 + | > step_time: 0.1563 (0.1472216669250937) + | > loader_time: 0.0106 (0.012746295508216411) + + + --> TIME: 2024-04-09 18:23:14 -- STEP: 118/1354 -- GLOBAL_STEP: 44800 + | > loss_text_ce: 0.05060862377285957 (0.04037877882562452) + | > loss_mel_ce: 2.7219841480255127 (2.0777287857007183) + | > loss: 0.03300705924630165 (0.025215566868625455) + | > current_lr: 5e-06 + | > step_time: 0.0943 (0.14642304687176722) + | > loader_time: 0.0108 (0.012273479316194179) + + + --> TIME: 2024-04-09 18:23:26 -- STEP: 168/1354 -- GLOBAL_STEP: 44850 + | > loss_text_ce: 0.05626588314771652 (0.04018565942533316) + | > loss_mel_ce: 2.7911574840545654 (2.0885494365578587) + | > loss: 0.033897899091243744 (0.025342085088292752) + | > current_lr: 5e-06 + | > step_time: 0.1041 (0.14635142825898667) + | > loader_time: 0.0133 (0.012106855710347494) + + + --> TIME: 2024-04-09 18:23:37 -- STEP: 218/1354 -- GLOBAL_STEP: 44900 + | > loss_text_ce: 0.030752049759030342 (0.040616060991626284) + | > loss_mel_ce: 1.6943498849868774 (2.1046991441227947) + | > loss: 0.02053692750632763 (0.025539467260754158) + | > current_lr: 5e-06 + | > step_time: 0.1524 (0.14677273789677045) + | > loader_time: 0.0137 (0.01197629346760041) + + + --> TIME: 2024-04-09 18:23:48 -- STEP: 268/1354 -- GLOBAL_STEP: 44950 + | > loss_text_ce: 0.04127536714076996 (0.040190028951072446) + | > loss_mel_ce: 1.4641120433807373 (2.0728316774119167) + | > loss: 0.01792128011584282 (0.025155020870768752) + | > current_lr: 5e-06 + | > step_time: 0.1192 (0.1468203912920026) + | > loader_time: 0.0136 (0.011878662144959862) + + + --> TIME: 2024-04-09 18:24:00 -- STEP: 318/1354 -- GLOBAL_STEP: 45000 + | > loss_text_ce: 0.037693873047828674 (0.040054940331563274) + | > loss_mel_ce: 1.8151851892471313 (2.0731286762645396) + | > loss: 0.02205808460712433 (0.025156948358148523) + | > current_lr: 5e-06 + | > step_time: 0.2129 (0.14792960019981335) + | > loader_time: 0.0128 (0.01184273365908449) + + + --> TIME: 2024-04-09 18:24:35 -- STEP: 368/1354 -- GLOBAL_STEP: 45050 + | > loss_text_ce: 0.03458563610911369 (0.04031864157877865) + | > loss_mel_ce: 1.871522307395935 (2.074830713479415) + | > loss: 0.022691762074828148 (0.0251803499666731) + | > current_lr: 5e-06 + | > step_time: 0.2066 (0.1466267186662424) + | > loader_time: 0.0126 (0.012022929347079738) + + + --> TIME: 2024-04-09 18:24:46 -- STEP: 418/1354 -- GLOBAL_STEP: 45100 + | > loss_text_ce: 0.04380744323134422 (0.04022948982185157) + | > loss_mel_ce: 2.422243595123291 (2.073693110897209) + | > loss: 0.029357751831412315 (0.025165745736719175) + | > current_lr: 5e-06 + | > step_time: 0.1091 (0.14640941460166818) + | > loader_time: 0.0126 (0.012099826164793173) + + + --> TIME: 2024-04-09 18:24:57 -- STEP: 468/1354 -- GLOBAL_STEP: 45150 + | > loss_text_ce: 0.05777598172426224 (0.04029507416053718) + | > loss_mel_ce: 2.460730791091919 (2.0830182527375003) + | > loss: 0.029982224106788635 (0.025277540099639922) + | > current_lr: 5e-06 + | > step_time: 0.1075 (0.14638502424598757) + | > loader_time: 0.0128 (0.012011125556424137) + + + --> TIME: 2024-04-09 18:25:09 -- STEP: 518/1354 -- GLOBAL_STEP: 45200 + | > loss_text_ce: 0.03773221746087074 (0.0400892974571487) + | > loss_mel_ce: 2.334092140197754 (2.075717658609957) + | > loss: 0.02823600359261036 (0.025188178537794637) + | > current_lr: 5e-06 + | > step_time: 0.1519 (0.14671903549474175) + | > loader_time: 0.0131 (0.011929374864202672) + + + --> TIME: 2024-04-09 18:25:20 -- STEP: 568/1354 -- GLOBAL_STEP: 45250 + | > loss_text_ce: 0.03843224421143532 (0.04014819661076643) + | > loss_mel_ce: 1.6160597801208496 (2.0758155529767697) + | > loss: 0.01969633437693119 (0.0251900451323411) + | > current_lr: 5e-06 + | > step_time: 0.163 (0.14652978450479634) + | > loader_time: 0.0137 (0.01186958668937146) + + + --> TIME: 2024-04-09 18:25:31 -- STEP: 618/1354 -- GLOBAL_STEP: 45300 + | > loss_text_ce: 0.04853121191263199 (0.04021644894889647) + | > loss_mel_ce: 3.087235927581787 (2.083183752487392) + | > loss: 0.037330564111471176 (0.025278574301828054) + | > current_lr: 5e-06 + | > step_time: 0.1 (0.14701125999870426) + | > loader_time: 0.0132 (0.011859015739465614) + + + --> TIME: 2024-04-09 18:25:42 -- STEP: 668/1354 -- GLOBAL_STEP: 45350 + | > loss_text_ce: 0.03503436595201492 (0.040147906216697976) + | > loss_mel_ce: 1.520706295967102 (2.082582131652774) + | > loss: 0.018520724028348923 (0.025270596184226814) + | > current_lr: 5e-06 + | > step_time: 0.169 (0.14681644960791765) + | > loader_time: 0.011 (0.01188272309160519) + + + --> TIME: 2024-04-09 18:25:53 -- STEP: 718/1354 -- GLOBAL_STEP: 45400 + | > loss_text_ce: 0.035213686525821686 (0.04006699128154261) + | > loss_mel_ce: 1.3444030284881592 (2.07431157186503) + | > loss: 0.016424009576439857 (0.025171173841518676) + | > current_lr: 5e-06 + | > step_time: 0.1193 (0.14649011530916006) + | > loader_time: 0.0107 (0.011874874654254549) + + + --> TIME: 2024-04-09 18:26:04 -- STEP: 768/1354 -- GLOBAL_STEP: 45450 + | > loss_text_ce: 0.05102970823645592 (0.040180731448344886) + | > loss_mel_ce: 2.7741875648498535 (2.0763769036469357) + | > loss: 0.03363353759050369 (0.025197115162882238) + | > current_lr: 5e-06 + | > step_time: 0.0825 (0.1461542006582021) + | > loader_time: 0.0105 (0.011859265466531125) + + + --> TIME: 2024-04-09 18:26:15 -- STEP: 818/1354 -- GLOBAL_STEP: 45500 + | > loss_text_ce: 0.028445372357964516 (0.04018230558644006) + | > loss_mel_ce: 1.5138812065124512 (2.0761199890838573) + | > loss: 0.018361030146479607 (0.025194075399176327) + | > current_lr: 5e-06 + | > step_time: 0.2158 (0.14617245587859587) + | > loader_time: 0.0127 (0.011839393590073721) + + + --> TIME: 2024-04-09 18:26:27 -- STEP: 868/1354 -- GLOBAL_STEP: 45550 + | > loss_text_ce: 0.04203050583600998 (0.040073033441544816) + | > loss_mel_ce: 2.1535348892211914 (2.072116056757589) + | > loss: 0.026137685403227806 (0.025145108679584145) + | > current_lr: 5e-06 + | > step_time: 0.1556 (0.14655605604022323) + | > loader_time: 0.0114 (0.011821026900946276) + + + --> TIME: 2024-04-09 18:26:38 -- STEP: 918/1354 -- GLOBAL_STEP: 45600 + | > loss_text_ce: 0.039162132889032364 (0.04004165111211066) + | > loss_mel_ce: 2.3260717391967773 (2.071903663782773) + | > loss: 0.028157547116279602 (0.02514220659432359) + | > current_lr: 5e-06 + | > step_time: 0.2008 (0.14660487325622618) + | > loader_time: 0.0133 (0.011812460967917854) + + + --> TIME: 2024-04-09 18:26:49 -- STEP: 968/1354 -- GLOBAL_STEP: 45650 + | > loss_text_ce: 0.03884468227624893 (0.04011402562191356) + | > loss_mel_ce: 1.672351360321045 (2.072690763503068) + | > loss: 0.02037138119339943 (0.025152438437596934) + | > current_lr: 5e-06 + | > step_time: 0.1876 (0.1465617979853606) + | > loader_time: 0.0138 (0.011802380488923764) + + + --> TIME: 2024-04-09 18:27:00 -- STEP: 1018/1354 -- GLOBAL_STEP: 45700 + | > loss_text_ce: 0.03032337687909603 (0.04015229893793409) + | > loss_mel_ce: 2.0619587898254395 (2.077353707816608) + | > loss: 0.024908119812607765 (0.025208405306844543) + | > current_lr: 5e-06 + | > step_time: 0.1321 (0.14643014460977724) + | > loader_time: 0.0133 (0.011801360164035286) + + + --> TIME: 2024-04-09 18:27:11 -- STEP: 1068/1354 -- GLOBAL_STEP: 45750 + | > loss_text_ce: 0.04669282212853432 (0.04029222388929698) + | > loss_mel_ce: 2.657769203186035 (2.079750406998822) + | > loss: 0.03219597786664963 (0.025238603217464475) + | > current_lr: 5e-06 + | > step_time: 0.0936 (0.14589109536860315) + | > loader_time: 0.0108 (0.011780336331785397) + + + --> TIME: 2024-04-09 18:27:22 -- STEP: 1118/1354 -- GLOBAL_STEP: 45800 + | > loss_text_ce: 0.03170958161354065 (0.04028406572963024) + | > loss_mel_ce: 1.926297903060913 (2.080844865821982) + | > loss: 0.02330961264669895 (0.025251535368893345) + | > current_lr: 5e-06 + | > step_time: 0.1759 (0.14572912486593279) + | > loader_time: 0.0114 (0.011773136945871208) + + + --> TIME: 2024-04-09 18:27:33 -- STEP: 1168/1354 -- GLOBAL_STEP: 45850 + | > loss_text_ce: 0.047905776649713516 (0.0402455462321435) + | > loss_mel_ce: 2.172969102859497 (2.0822852734119106) + | > loss: 0.026438986882567406 (0.025268224521686814) + | > current_lr: 5e-06 + | > step_time: 0.1116 (0.1456739914743867) + | > loader_time: 0.0105 (0.011772710778941849) + + + --> TIME: 2024-04-09 18:27:44 -- STEP: 1218/1354 -- GLOBAL_STEP: 45900 + | > loss_text_ce: 0.03871672973036766 (0.0402366634739688) + | > loss_mel_ce: 2.1562976837158203 (2.0784360954816314) + | > loss: 0.026131125167012215 (0.025222295239164024) + | > current_lr: 5e-06 + | > step_time: 0.1998 (0.14583547694733962) + | > loader_time: 0.0115 (0.011769571523556763) + + + --> TIME: 2024-04-09 18:27:55 -- STEP: 1268/1354 -- GLOBAL_STEP: 45950 + | > loss_text_ce: 0.03468228504061699 (0.04025634054876039) + | > loss_mel_ce: 2.200176954269409 (2.0778519577502443) + | > loss: 0.02660546824336052 (0.025215575464175605) + | > current_lr: 5e-06 + | > step_time: 0.1398 (0.14555685546495933) + | > loader_time: 0.0112 (0.011763116350685388) + + + --> TIME: 2024-04-09 18:28:06 -- STEP: 1318/1354 -- GLOBAL_STEP: 46000 + | > loss_text_ce: 0.029923388734459877 (0.0401960441585466) + | > loss_mel_ce: 1.4757812023162842 (2.0749863384797456) + | > loss: 0.01792505569756031 (0.025180743128903584) + | > current_lr: 5e-06 + | > step_time: 0.1066 (0.14557770025748215) + | > loader_time: 0.0104 (0.011764904978785419) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.025837494776799128 (-0.0015622102297269402) + | > avg_loss_text_ce: 0.04620455835874264 (-0.000419512964211978) + | > avg_loss_mel_ce: 2.8661063267634463 (+0.004076425845806142) + | > avg_loss: 2.9123108753791223 (+0.003656919185931695) + + + > EPOCH: 35/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 18:28:55)  + + --> TIME: 2024-04-09 18:28:59 -- STEP: 14/1354 -- GLOBAL_STEP: 46050 + | > loss_text_ce: 0.037303633987903595 (0.03875049629381725) + | > loss_mel_ce: 1.7454525232315063 (2.1539475406919206) + | > loss: 0.021223288029432297 (0.02610354857253177) + | > current_lr: 5e-06 + | > step_time: 0.1602 (0.16207679680415563) + | > loader_time: 0.0122 (0.013621347291128976) + + + --> TIME: 2024-04-09 18:29:10 -- STEP: 64/1354 -- GLOBAL_STEP: 46100 + | > loss_text_ce: 0.047273505479097366 (0.03908383270027117) + | > loss_mel_ce: 2.4702816009521484 (2.044395824894308) + | > loss: 0.029970893636345863 (0.0248033297539223) + | > current_lr: 5e-06 + | > step_time: 0.1131 (0.14655662328004834) + | > loader_time: 0.0125 (0.01316487044095993) + + + --> TIME: 2024-04-09 18:29:22 -- STEP: 114/1354 -- GLOBAL_STEP: 46150 + | > loss_text_ce: 0.03395428881049156 (0.03914223831931225) + | > loss_mel_ce: 1.407734990119934 (2.0285129892198643) + | > loss: 0.01716296747326851 (0.024614943730595865) + | > current_lr: 5e-06 + | > step_time: 0.1538 (0.14839098118899158) + | > loader_time: 0.0126 (0.012952223158719247) + + + --> TIME: 2024-04-09 18:29:33 -- STEP: 164/1354 -- GLOBAL_STEP: 46200 + | > loss_text_ce: 0.027779851108789444 (0.039013475168314625) + | > loss_mel_ce: 1.844362497329712 (2.028550406054751) + | > loss: 0.022287409752607346 (0.024613856292534167) + | > current_lr: 5e-06 + | > step_time: 0.1925 (0.1488366461381679) + | > loader_time: 0.0109 (0.012871832382388232) + + + --> TIME: 2024-04-09 18:29:43 -- STEP: 214/1354 -- GLOBAL_STEP: 46250 + | > loss_text_ce: 0.03119092620909214 (0.039505509241858354) + | > loss_mel_ce: 1.5314278602600098 (2.040574426405895) + | > loss: 0.018602604046463966 (0.024762856913295304) + | > current_lr: 5e-06 + | > step_time: 0.1613 (0.14597454137891247) + | > loader_time: 0.0119 (0.012733473956027877) + + + --> TIME: 2024-04-09 18:29:54 -- STEP: 264/1354 -- GLOBAL_STEP: 46300 + | > loss_text_ce: 0.03936263918876648 (0.03910203707743096) + | > loss_mel_ce: 2.324676990509033 (2.009991514411838) + | > loss: 0.028143329545855522 (0.02439397137089999) + | > current_lr: 5e-06 + | > step_time: 0.1758 (0.14536440553087174) + | > loader_time: 0.0134 (0.012642163218873922) + + + --> TIME: 2024-04-09 18:30:06 -- STEP: 314/1354 -- GLOBAL_STEP: 46350 + | > loss_text_ce: 0.044884536415338516 (0.038616163015460496) + | > loss_mel_ce: 1.6424121856689453 (1.9863168088493819) + | > loss: 0.0200868658721447 (0.024106345373757516) + | > current_lr: 5e-06 + | > step_time: 0.0783 (0.14675604917441198) + | > loader_time: 0.0131 (0.012637033584011591) + + + --> TIME: 2024-04-09 18:30:17 -- STEP: 364/1354 -- GLOBAL_STEP: 46400 + | > loss_text_ce: 0.037846535444259644 (0.03857254133569994) + | > loss_mel_ce: 2.015488862991333 (1.992661757292327) + | > loss: 0.024444470182061195 (0.02418136119504797) + | > current_lr: 5e-06 + | > step_time: 0.2095 (0.14600759417146109) + | > loader_time: 0.0123 (0.012569961967049067) + + + --> TIME: 2024-04-09 18:30:28 -- STEP: 414/1354 -- GLOBAL_STEP: 46450 + | > loss_text_ce: 0.03180505335330963 (0.038468826764627616) + | > loss_mel_ce: 1.9247040748596191 (1.9961346648453504) + | > loss: 0.023291775956749916 (0.024221470653740824) + | > current_lr: 5e-06 + | > step_time: 0.1614 (0.14563017248531462) + | > loader_time: 0.0106 (0.012471748434978985) + + + --> TIME: 2024-04-09 18:30:38 -- STEP: 464/1354 -- GLOBAL_STEP: 46500 + | > loss_text_ce: 0.0341288223862648 (0.03863770925391719) + | > loss_mel_ce: 1.936710238456726 (1.9888816263912044) + | > loss: 0.023462370038032532 (0.024137135474653595) + | > current_lr: 5e-06 + | > step_time: 0.2307 (0.14453528924234976) + | > loader_time: 0.0124 (0.012289503011210214) + + + --> TIME: 2024-04-09 18:30:49 -- STEP: 514/1354 -- GLOBAL_STEP: 46550 + | > loss_text_ce: 0.03218666836619377 (0.03887172612626146) + | > loss_mel_ce: 1.4019882678985596 (2.007140402548043) + | > loss: 0.017073510214686394 (0.024357287764027426) + | > current_lr: 5e-06 + | > step_time: 0.1522 (0.14357453144014107) + | > loader_time: 0.0106 (0.012196907737375699) + + + --> TIME: 2024-04-09 18:30:59 -- STEP: 564/1354 -- GLOBAL_STEP: 46600 + | > loss_text_ce: 0.03913017734885216 (0.038945363907156624) + | > loss_mel_ce: 2.0450150966644287 (2.0130354328781146) + | > loss: 0.02481125481426716 (0.024428343335959182) + | > current_lr: 5e-06 + | > step_time: 0.1636 (0.14335313883233572) + | > loader_time: 0.0108 (0.012196356945849482) + + + --> TIME: 2024-04-09 18:31:11 -- STEP: 614/1354 -- GLOBAL_STEP: 46650 + | > loss_text_ce: 0.04053029790520668 (0.03899052296537623) + | > loss_mel_ce: 1.7888784408569336 (2.0186932430593516) + | > loss: 0.02177867665886879 (0.024496235821019945) + | > current_lr: 5e-06 + | > step_time: 0.1187 (0.14391449452999738) + | > loader_time: 0.0106 (0.012182044672267056) + + + --> TIME: 2024-04-09 18:31:22 -- STEP: 664/1354 -- GLOBAL_STEP: 46700 + | > loss_text_ce: 0.050438523292541504 (0.03918355897192402) + | > loss_mel_ce: 2.1234042644500732 (2.0205283164080368) + | > loss: 0.025879083201289177 (0.02452037998486355) + | > current_lr: 5e-06 + | > step_time: 0.0812 (0.14365147610744786) + | > loader_time: 0.0099 (0.012174725532531742) + + + --> TIME: 2024-04-09 18:31:33 -- STEP: 714/1354 -- GLOBAL_STEP: 46750 + | > loss_text_ce: 0.05239196494221687 (0.03918179963044924) + | > loss_mel_ce: 2.62121844291687 (2.0186878034547577) + | > loss: 0.03182869777083397 (0.024498448185413992) + | > current_lr: 5e-06 + | > step_time: 0.1113 (0.14385038964888622) + | > loader_time: 0.01 (0.0121553187944642) + + + --> TIME: 2024-04-09 18:31:44 -- STEP: 764/1354 -- GLOBAL_STEP: 46800 + | > loss_text_ce: 0.029632568359375 (0.039271472672887935) + | > loss_mel_ce: 1.7900469303131104 (2.0267702541582224) + | > loss: 0.02166285179555416 (0.024595735367894642) + | > current_lr: 5e-06 + | > step_time: 0.2114 (0.14363288785774672) + | > loader_time: 0.0105 (0.012084502512247778) + + + --> TIME: 2024-04-09 18:31:54 -- STEP: 814/1354 -- GLOBAL_STEP: 46850 + | > loss_text_ce: 0.03150861710309982 (0.039177681305685975) + | > loss_mel_ce: 2.524463176727295 (2.0272038715184455) + | > loss: 0.030428236350417137 (0.024599780917799224) + | > current_lr: 5e-06 + | > step_time: 0.1193 (0.14355721198370183) + | > loader_time: 0.0114 (0.012021210914161928) + + + --> TIME: 2024-04-09 18:32:05 -- STEP: 864/1354 -- GLOBAL_STEP: 46900 + | > loss_text_ce: 0.050059761852025986 (0.03922061036832422) + | > loss_mel_ce: 2.9831669330596924 (2.0286859800970087) + | > loss: 0.036109842360019684 (0.024617936128008836) + | > current_lr: 5e-06 + | > step_time: 0.112 (0.14335873088351) + | > loader_time: 0.0106 (0.011965118348598482) + + + --> TIME: 2024-04-09 18:32:16 -- STEP: 914/1354 -- GLOBAL_STEP: 46950 + | > loss_text_ce: 0.045834191143512726 (0.03932372973600087) + | > loss_mel_ce: 1.1864508390426636 (2.0298364286349666) + | > loss: 0.014670060016214848 (0.024632859558347996) + | > current_lr: 5e-06 + | > step_time: 0.092 (0.14329036111643859) + | > loader_time: 0.0129 (0.011914846933644503) + + + --> TIME: 2024-04-09 18:32:27 -- STEP: 964/1354 -- GLOBAL_STEP: 47000 + | > loss_text_ce: 0.031573761254549026 (0.03936631853703825) + | > loss_mel_ce: 1.6209216117858887 (2.0312114210791585) + | > loss: 0.019672565162181854 (0.024649735518897348) + | > current_lr: 5e-06 + | > step_time: 0.1655 (0.1432876436047535) + | > loader_time: 0.0128 (0.011876731987316086) + + + --> TIME: 2024-04-09 18:33:02 -- STEP: 1014/1354 -- GLOBAL_STEP: 47050 + | > loss_text_ce: 0.054959993809461594 (0.039397943191803404) + | > loss_mel_ce: 2.368281126022339 (2.033607042400085) + | > loss: 0.028848109766840935 (0.02467863130267467) + | > current_lr: 5e-06 + | > step_time: 0.1438 (0.14326989697751685) + | > loader_time: 0.0157 (0.011917734052067445) + + + --> TIME: 2024-04-09 18:33:13 -- STEP: 1064/1354 -- GLOBAL_STEP: 47100 + | > loss_text_ce: 0.03576711192727089 (0.039407693296040686) + | > loss_mel_ce: 1.8879413604736328 (2.0353119908866986) + | > loss: 0.022901291027665138 (0.024699044377977188) + | > current_lr: 5e-06 + | > step_time: 0.1805 (0.1432875535990063) + | > loader_time: 0.0133 (0.011904659351908174) + + + --> TIME: 2024-04-09 18:33:25 -- STEP: 1114/1354 -- GLOBAL_STEP: 47150 + | > loss_text_ce: 0.0385233610868454 (0.03926457023946987) + | > loss_mel_ce: 2.109597682952881 (2.030510740819478) + | > loss: 0.025572871789336205 (0.024640182787686243) + | > current_lr: 5e-06 + | > step_time: 0.2154 (0.1438236925717531) + | > loader_time: 0.0131 (0.011903447351507052) + + + --> TIME: 2024-04-09 18:33:35 -- STEP: 1164/1354 -- GLOBAL_STEP: 47200 + | > loss_text_ce: 0.033934786915779114 (0.039265240406572734) + | > loss_mel_ce: 2.0634777545928955 (2.0248438311913537) + | > loss: 0.024969197809696198 (0.0245727275556619) + | > current_lr: 5e-06 + | > step_time: 0.1468 (0.14336629155575212) + | > loader_time: 0.0128 (0.011887520449268033) + + + --> TIME: 2024-04-09 18:33:46 -- STEP: 1214/1354 -- GLOBAL_STEP: 47250 + | > loss_text_ce: 0.02708982117474079 (0.03925328392111984) + | > loss_mel_ce: 1.2984143495559692 (2.02336478621209) + | > loss: 0.015779811888933182 (0.024554977529870934) + | > current_lr: 5e-06 + | > step_time: 0.1501 (0.14348620205773954) + | > loader_time: 0.0105 (0.011890791393386854) + + + --> TIME: 2024-04-09 18:33:58 -- STEP: 1264/1354 -- GLOBAL_STEP: 47300 + | > loss_text_ce: 0.05874542519450188 (0.039174617450872966) + | > loss_mel_ce: 2.930671215057373 (2.0222902421992757) + | > loss: 0.03558829426765442 (0.024541248837441694) + | > current_lr: 5e-06 + | > step_time: 0.1621 (0.14377878097039257) + | > loader_time: 0.0126 (0.011876206043400327) + + + --> TIME: 2024-04-09 18:34:09 -- STEP: 1314/1354 -- GLOBAL_STEP: 47350 + | > loss_text_ce: 0.028195008635520935 (0.0391516547431798) + | > loss_mel_ce: 1.451560378074646 (2.0224598713298505) + | > loss: 0.017616135999560356 (0.024542994868461055) + | > current_lr: 5e-06 + | > step_time: 0.1779 (0.14406427618575438) + | > loader_time: 0.013 (0.011861160284126532) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.030216198701124925 (+0.004378703924325798) + | > avg_loss_text_ce: 0.04591792879196314 (-0.00028662956677950185) + | > avg_loss_mel_ce: 2.8667632249685435 (+0.0006568982050971783) + | > avg_loss: 2.912681121092576 (+0.0003702457134537873) + + + > EPOCH: 36/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 18:34:34)  + + --> TIME: 2024-04-09 18:34:38 -- STEP: 10/1354 -- GLOBAL_STEP: 47400 + | > loss_text_ce: 0.04821564257144928 (0.036501909233629705) + | > loss_mel_ce: 2.723017930984497 (1.8406696796417237) + | > loss: 0.03299087658524513 (0.022347281128168105) + | > current_lr: 5e-06 + | > step_time: 0.1754 (0.17238011360168456) + | > loader_time: 0.0183 (0.014345002174377442) + + + --> TIME: 2024-04-09 18:34:49 -- STEP: 60/1354 -- GLOBAL_STEP: 47450 + | > loss_text_ce: 0.03866066038608551 (0.04006496910005808) + | > loss_mel_ce: 2.040083885192871 (2.088675734400749) + | > loss: 0.024746960029006004 (0.025342151600246627) + | > current_lr: 5e-06 + | > step_time: 0.1418 (0.14619791507720942) + | > loader_time: 0.0125 (0.013106846809387207) + + + --> TIME: 2024-04-09 18:35:00 -- STEP: 110/1354 -- GLOBAL_STEP: 47500 + | > loss_text_ce: 0.0438234880566597 (0.040146900222382764) + | > loss_mel_ce: 2.2101638317108154 (2.12420784831047) + | > loss: 0.02683318220078945 (0.025766128293153914) + | > current_lr: 5e-06 + | > step_time: 0.0991 (0.14692064415324807) + | > loader_time: 0.0108 (0.012892593036998402) + + + --> TIME: 2024-04-09 18:35:11 -- STEP: 160/1354 -- GLOBAL_STEP: 47550 + | > loss_text_ce: 0.035970356315374374 (0.039771670126356184) + | > loss_mel_ce: 1.978689193725586 (2.084098793566227) + | > loss: 0.023984042927622795 (0.025284172588726513) + | > current_lr: 5e-06 + | > step_time: 0.1524 (0.14655901491641987) + | > loader_time: 0.0112 (0.01266845315694809) + + + --> TIME: 2024-04-09 18:35:22 -- STEP: 210/1354 -- GLOBAL_STEP: 47600 + | > loss_text_ce: 0.03652340918779373 (0.03963296919883715) + | > loss_mel_ce: 1.2864450216293335 (2.0576083620389283) + | > loss: 0.015749625861644745 (0.024967159100231673) + | > current_lr: 5e-06 + | > step_time: 0.1058 (0.1461806887672061) + | > loader_time: 0.0107 (0.01248396805354527) + + + --> TIME: 2024-04-09 18:35:33 -- STEP: 260/1354 -- GLOBAL_STEP: 47650 + | > loss_text_ce: 0.04258878156542778 (0.039424325489940536) + | > loss_mel_ce: 2.4413836002349854 (2.0529881504865775) + | > loss: 0.02957109920680523 (0.024909672753598826) + | > current_lr: 5e-06 + | > step_time: 0.1413 (0.14515410386599026) + | > loader_time: 0.011 (0.012286935402796819) + + + --> TIME: 2024-04-09 18:35:45 -- STEP: 310/1354 -- GLOBAL_STEP: 47700 + | > loss_text_ce: 0.04584355279803276 (0.03939614216045986) + | > loss_mel_ce: 2.3461596965789795 (2.044609130197953) + | > loss: 0.028476230800151825 (0.024809587004804814) + | > current_lr: 5e-06 + | > step_time: 0.1547 (0.14628064478597333) + | > loader_time: 0.0106 (0.012173886452951742) + + + --> TIME: 2024-04-09 18:35:55 -- STEP: 360/1354 -- GLOBAL_STEP: 47750 + | > loss_text_ce: 0.02714662067592144 (0.03947836057179502) + | > loss_mel_ce: 1.1664751768112183 (2.0576055258512467) + | > loss: 0.01420978270471096 (0.02496528480874582) + | > current_lr: 5e-06 + | > step_time: 0.1515 (0.1453684012095133) + | > loader_time: 0.0113 (0.01215097506841024) + + + --> TIME: 2024-04-09 18:36:07 -- STEP: 410/1354 -- GLOBAL_STEP: 47800 + | > loss_text_ce: 0.06464174389839172 (0.039242757888647124) + | > loss_mel_ce: 2.649648666381836 (2.0417245007142766) + | > loss: 0.03231298178434372 (0.02477342019357333) + | > current_lr: 5e-06 + | > step_time: 0.1062 (0.14642929100408786) + | > loader_time: 0.0107 (0.012063943467489105) + + + --> TIME: 2024-04-09 18:36:18 -- STEP: 460/1354 -- GLOBAL_STEP: 47850 + | > loss_text_ce: 0.043540362268686295 (0.039322676254517366) + | > loss_mel_ce: 1.8542401790618896 (2.0413079497606836) + | > loss: 0.02259262651205063 (0.024769412657326982) + | > current_lr: 5e-06 + | > step_time: 0.1525 (0.14574605744817987) + | > loader_time: 0.013 (0.011985737344493038) + + + --> TIME: 2024-04-09 18:36:29 -- STEP: 510/1354 -- GLOBAL_STEP: 47900 + | > loss_text_ce: 0.03135502338409424 (0.03914325066714311) + | > loss_mel_ce: 1.5682373046875 (2.0253930858537244) + | > loss: 0.019042765721678734 (0.02457781397901914) + | > current_lr: 5e-06 + | > step_time: 0.1653 (0.14568506222145233) + | > loader_time: 0.0129 (0.01193214210809446) + + + --> TIME: 2024-04-09 18:36:40 -- STEP: 560/1354 -- GLOBAL_STEP: 47950 + | > loss_text_ce: 0.04328535124659538 (0.03920759948502694) + | > loss_mel_ce: 1.7647950649261475 (2.022121016574756) + | > loss: 0.021524768322706223 (0.02453962681548937) + | > current_lr: 5e-06 + | > step_time: 0.0923 (0.1460745573043823) + | > loader_time: 0.0128 (0.01187694455896105) + + + --> TIME: 2024-04-09 18:36:51 -- STEP: 610/1354 -- GLOBAL_STEP: 48000 + | > loss_text_ce: 0.05310283601284027 (0.039285207286354935) + | > loss_mel_ce: 2.0875351428985596 (2.025082046282095) + | > loss: 0.025483785197138786 (0.02457580107920728) + | > current_lr: 5e-06 + | > step_time: 0.1112 (0.14522219994029062) + | > loader_time: 0.0129 (0.011821775358231344) + + + --> TIME: 2024-04-09 18:37:28 -- STEP: 660/1354 -- GLOBAL_STEP: 48050 + | > loss_text_ce: 0.03924141451716423 (0.03914596987041559) + | > loss_mel_ce: 2.2528412342071533 (2.017864741881687) + | > loss: 0.027286697179079056 (0.024488223204389224) + | > current_lr: 5e-06 + | > step_time: 0.1206 (0.14623267542232163) + | > loader_time: 0.0107 (0.0118369637113629) + + + --> TIME: 2024-04-09 18:37:39 -- STEP: 710/1354 -- GLOBAL_STEP: 48100 + | > loss_text_ce: 0.04353577643632889 (0.03910652048833353) + | > loss_mel_ce: 1.5736509561538696 (2.0167283832187395) + | > loss: 0.019252223894000053 (0.02447422548446438) + | > current_lr: 5e-06 + | > step_time: 0.0923 (0.14633198019484392) + | > loader_time: 0.0104 (0.011856760105616612) + + + --> TIME: 2024-04-09 18:37:51 -- STEP: 760/1354 -- GLOBAL_STEP: 48150 + | > loss_text_ce: 0.041753292083740234 (0.03929407796087235) + | > loss_mel_ce: 2.458728551864624 (2.0236859290223386) + | > loss: 0.02976764179766178 (0.02455928624534098) + | > current_lr: 5e-06 + | > step_time: 0.19 (0.14692360978377486) + | > loader_time: 0.0116 (0.011874114839654223) + + + --> TIME: 2024-04-09 18:38:02 -- STEP: 810/1354 -- GLOBAL_STEP: 48200 + | > loss_text_ce: 0.046288784593343735 (0.039377685337338916) + | > loss_mel_ce: 2.524897813796997 (2.028474591249303) + | > loss: 0.030609363690018654 (0.024617289467772224) + | > current_lr: 5e-06 + | > step_time: 0.1554 (0.14713039192152602) + | > loader_time: 0.0106 (0.01185307797090507) + + + --> TIME: 2024-04-09 18:38:13 -- STEP: 860/1354 -- GLOBAL_STEP: 48250 + | > loss_text_ce: 0.05217256769537926 (0.039322308487757) + | > loss_mel_ce: 2.048470973968506 (2.0306941634000744) + | > loss: 0.025007663294672966 (0.024643053713276297) + | > current_lr: 5e-06 + | > step_time: 0.1365 (0.146927019607189) + | > loader_time: 0.011 (0.011822394992029946) + + + --> TIME: 2024-04-09 18:38:24 -- STEP: 910/1354 -- GLOBAL_STEP: 48300 + | > loss_text_ce: 0.04545293748378754 (0.039354532744203294) + | > loss_mel_ce: 1.5778518915176392 (2.032828417060142) + | > loss: 0.01932505890727043 (0.024668845124952092) + | > current_lr: 5e-06 + | > step_time: 0.0806 (0.14705539981087476) + | > loader_time: 0.0121 (0.011827281543186735) + + + --> TIME: 2024-04-09 18:38:36 -- STEP: 960/1354 -- GLOBAL_STEP: 48350 + | > loss_text_ce: 0.03430319204926491 (0.03925904363277368) + | > loss_mel_ce: 2.0336484909057617 (2.027407713358603) + | > loss: 0.024618472903966904 (0.024603176168360132) + | > current_lr: 5e-06 + | > step_time: 0.1195 (0.14760570252935082) + | > loader_time: 0.0107 (0.011808944245179497) + + + --> TIME: 2024-04-09 18:38:47 -- STEP: 1010/1354 -- GLOBAL_STEP: 48400 + | > loss_text_ce: 0.037947360426187515 (0.03918431322136433) + | > loss_mel_ce: 1.1899590492248535 (2.0232481987169484) + | > loss: 0.014617934823036194 (0.024552768491909356) + | > current_lr: 5e-06 + | > step_time: 0.1749 (0.14736175395474568) + | > loader_time: 0.011 (0.011793089385079867) + + + --> TIME: 2024-04-09 18:38:59 -- STEP: 1060/1354 -- GLOBAL_STEP: 48450 + | > loss_text_ce: 0.051112398505210876 (0.039074659825495954) + | > loss_mel_ce: 2.4083878993988037 (2.0202247653367396) + | > loss: 0.029279766604304314 (0.024515469830904937) + | > current_lr: 5e-06 + | > step_time: 0.107 (0.14778201737493832) + | > loader_time: 0.0108 (0.011803912666608705) + + + --> TIME: 2024-04-09 18:39:10 -- STEP: 1110/1354 -- GLOBAL_STEP: 48500 + | > loss_text_ce: 0.03151633217930794 (0.039091091529206125) + | > loss_mel_ce: 1.7819108963012695 (2.016643039278081) + | > loss: 0.021588420495390892 (0.024473025857026258) + | > current_lr: 5e-06 + | > step_time: 0.1904 (0.14758064145440444) + | > loader_time: 0.0118 (0.01178375557736234) + + + --> TIME: 2024-04-09 18:39:21 -- STEP: 1160/1354 -- GLOBAL_STEP: 48550 + | > loss_text_ce: 0.03517402336001396 (0.039124559805375235) + | > loss_mel_ce: 2.1909141540527344 (2.0166096832217826) + | > loss: 0.02650105208158493 (0.024473027188876846) + | > current_lr: 5e-06 + | > step_time: 0.212 (0.14749979459006207) + | > loader_time: 0.0109 (0.011756303803674113) + + + --> TIME: 2024-04-09 18:39:31 -- STEP: 1210/1354 -- GLOBAL_STEP: 48600 + | > loss_text_ce: 0.03309427201747894 (0.039193278384849074) + | > loss_mel_ce: 1.9830868244171143 (2.0213744505377833) + | > loss: 0.024002155289053917 (0.024530568675807687) + | > current_lr: 5e-06 + | > step_time: 0.1529 (0.14697085884976974) + | > loader_time: 0.0125 (0.011754798298039717) + + + --> TIME: 2024-04-09 18:39:42 -- STEP: 1260/1354 -- GLOBAL_STEP: 48650 + | > loss_text_ce: 0.03617196902632713 (0.039207095562643034) + | > loss_mel_ce: 2.0535802841186523 (2.020864729654221) + | > loss: 0.02487800270318985 (0.024524665061414) + | > current_lr: 5e-06 + | > step_time: 0.0983 (0.14674017694261338) + | > loader_time: 0.0127 (0.011728144070458801) + + + --> TIME: 2024-04-09 18:39:53 -- STEP: 1310/1354 -- GLOBAL_STEP: 48700 + | > loss_text_ce: 0.050967149436473846 (0.03919965831833035) + | > loss_mel_ce: 2.3469324111938477 (2.0211432577999515) + | > loss: 0.028546424582600594 (0.024527892333149687) + | > current_lr: 5e-06 + | > step_time: 0.1436 (0.1465934673338445) + | > loader_time: 0.012 (0.0117077396116184) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.026130199432373047 (-0.004085999268751878) + | > avg_loss_text_ce: 0.045648955954955175 (-0.0002689728370079622) + | > avg_loss_mel_ce: 2.871354891703679 (+0.004591666735135558) + | > avg_loss: 2.917003870010376 (+0.0043227489177999345) + + + > EPOCH: 37/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 18:40:18)  + + --> TIME: 2024-04-09 18:40:21 -- STEP: 6/1354 -- GLOBAL_STEP: 48750 + | > loss_text_ce: 0.02880842797458172 (0.03267235370973746) + | > loss_mel_ce: 1.7982211112976074 (1.7780808011690776) + | > loss: 0.021750353276729584 (0.021556586182365816) + | > current_lr: 5e-06 + | > step_time: 0.1948 (0.16261609395345053) + | > loader_time: 0.0119 (0.01290114720662435) + + + --> TIME: 2024-04-09 18:40:33 -- STEP: 56/1354 -- GLOBAL_STEP: 48800 + | > loss_text_ce: 0.029671665281057358 (0.03771742935558515) + | > loss_mel_ce: 1.448926568031311 (1.9460960647889547) + | > loss: 0.017602359876036644 (0.023616827691772153) + | > current_lr: 5e-06 + | > step_time: 0.217 (0.15515381097793576) + | > loader_time: 0.0118 (0.013450699193137032) + + + --> TIME: 2024-04-09 18:40:44 -- STEP: 106/1354 -- GLOBAL_STEP: 48850 + | > loss_text_ce: 0.04059688374400139 (0.03891095758046744) + | > loss_mel_ce: 2.922711133956909 (1.9785858876300306) + | > loss: 0.03527747839689255 (0.02401781995903771) + | > current_lr: 5e-06 + | > step_time: 0.0819 (0.1483479265896779) + | > loader_time: 0.0108 (0.013126510494160202) + + + --> TIME: 2024-04-09 18:40:54 -- STEP: 156/1354 -- GLOBAL_STEP: 48900 + | > loss_text_ce: 0.04765705391764641 (0.038845494771615065) + | > loss_mel_ce: 2.4827373027801514 (2.0212555947976227) + | > loss: 0.030123742297291756 (0.024525013304530423) + | > current_lr: 5e-06 + | > step_time: 0.1686 (0.14506913606937116) + | > loader_time: 0.0115 (0.01281644289310162) + + + --> TIME: 2024-04-09 18:41:05 -- STEP: 206/1354 -- GLOBAL_STEP: 48950 + | > loss_text_ce: 0.03810872882604599 (0.03904778893711499) + | > loss_mel_ce: 1.8034006357192993 (2.0331457737580076) + | > loss: 0.021922729909420013 (0.024668971316219823) + | > current_lr: 5e-06 + | > step_time: 0.1909 (0.14261048279919672) + | > loader_time: 0.0113 (0.01258135073393294) + + + --> TIME: 2024-04-09 18:41:16 -- STEP: 256/1354 -- GLOBAL_STEP: 49000 + | > loss_text_ce: 0.022250112146139145 (0.038948474473727415) + | > loss_mel_ce: 1.475826621055603 (2.0485229436308146) + | > loss: 0.017834246158599854 (0.024850850542861735) + | > current_lr: 5e-06 + | > step_time: 0.1918 (0.14312492497265333) + | > loader_time: 0.0114 (0.012504004873335361) + + + --> TIME: 2024-04-09 18:41:52 -- STEP: 306/1354 -- GLOBAL_STEP: 49050 + | > loss_text_ce: 0.04899115860462189 (0.03877608968078896) + | > loss_mel_ce: 2.1613945960998535 (2.0353182073122524) + | > loss: 0.02631411701440811 (0.024691599135110574) + | > current_lr: 5e-06 + | > step_time: 0.2024 (0.14317825572942589) + | > loader_time: 0.0128 (0.012844941195319677) + + + --> TIME: 2024-04-09 18:42:03 -- STEP: 356/1354 -- GLOBAL_STEP: 49100 + | > loss_text_ce: 0.04576139897108078 (0.03844105562697467) + | > loss_mel_ce: 2.0920779705047607 (2.0082487140813567) + | > loss: 0.025450468063354492 (0.02436535477931244) + | > current_lr: 5e-06 + | > step_time: 0.1507 (0.1437185795119638) + | > loader_time: 0.0121 (0.012837740142693678) + + + --> TIME: 2024-04-09 18:42:13 -- STEP: 406/1354 -- GLOBAL_STEP: 49150 + | > loss_text_ce: 0.036306869238615036 (0.038713140569150165) + | > loss_mel_ce: 1.8811531066894531 (2.0163429480761734) + | > loss: 0.02282690443098545 (0.024464953821519465) + | > current_lr: 5e-06 + | > step_time: 0.1149 (0.14191218726153432) + | > loader_time: 0.0119 (0.012819195028596322) + + + --> TIME: 2024-04-09 18:42:25 -- STEP: 456/1354 -- GLOBAL_STEP: 49200 + | > loss_text_ce: 0.04943850263953209 (0.03865902942376571) + | > loss_mel_ce: 2.5038321018218994 (2.013163062825538) + | > loss: 0.030396079644560814 (0.024426453859185824) + | > current_lr: 5e-06 + | > step_time: 0.0909 (0.14330501775992543) + | > loader_time: 0.0124 (0.012805531945144919) + + + --> TIME: 2024-04-09 18:42:36 -- STEP: 506/1354 -- GLOBAL_STEP: 49250 + | > loss_text_ce: 0.025830427184700966 (0.0386085699536937) + | > loss_mel_ce: 1.081248164176941 (2.005473704912918) + | > loss: 0.013179507106542587 (0.02433431317124206) + | > current_lr: 5e-06 + | > step_time: 0.2088 (0.14335044994655805) + | > loader_time: 0.0127 (0.012862893903679526) + + + --> TIME: 2024-04-09 18:42:47 -- STEP: 556/1354 -- GLOBAL_STEP: 49300 + | > loss_text_ce: 0.02707604505121708 (0.0386469127248517) + | > loss_mel_ce: 1.424850344657898 (2.0064633074853067) + | > loss: 0.01728483848273754 (0.02434655060663336) + | > current_lr: 5e-06 + | > step_time: 0.1438 (0.14290938317346913) + | > loader_time: 0.0122 (0.012827699990581266) + + + --> TIME: 2024-04-09 18:42:58 -- STEP: 606/1354 -- GLOBAL_STEP: 49350 + | > loss_text_ce: 0.04325764626264572 (0.038840311640879685) + | > loss_mel_ce: 2.7706594467163086 (2.019001109568594) + | > loss: 0.033499013632535934 (0.024498112551966298) + | > current_lr: 5e-06 + | > step_time: 0.2001 (0.14273555050588677) + | > loader_time: 0.0117 (0.01279119729208868) + + + --> TIME: 2024-04-09 18:43:09 -- STEP: 656/1354 -- GLOBAL_STEP: 49400 + | > loss_text_ce: 0.03335866704583168 (0.038815345754279036) + | > loss_mel_ce: 1.47520112991333 (2.014662522550038) + | > loss: 0.017959045246243477 (0.02444616550432016) + | > current_lr: 5e-06 + | > step_time: 0.1408 (0.14269974646044942) + | > loader_time: 0.0113 (0.012753791561940826) + + + --> TIME: 2024-04-09 18:43:20 -- STEP: 706/1354 -- GLOBAL_STEP: 49450 + | > loss_text_ce: 0.04592480510473251 (0.03872121650746707) + | > loss_mel_ce: 2.291069984436035 (2.00458474959598) + | > loss: 0.027821367606520653 (0.024325071446196535) + | > current_lr: 5e-06 + | > step_time: 0.1303 (0.14339815050616825) + | > loader_time: 0.0105 (0.012713857142850973) + + + --> TIME: 2024-04-09 18:43:32 -- STEP: 756/1354 -- GLOBAL_STEP: 49500 + | > loss_text_ce: 0.027915388345718384 (0.03874777364618483) + | > loss_mel_ce: 1.3709062337875366 (2.001595294507094) + | > loss: 0.016652638092637062 (0.024289798855081812) + | > current_lr: 5e-06 + | > step_time: 0.155 (0.14396996094436254) + | > loader_time: 0.0144 (0.012695664451235822) + + + --> TIME: 2024-04-09 18:43:43 -- STEP: 806/1354 -- GLOBAL_STEP: 49550 + | > loss_text_ce: 0.04636574164032936 (0.0387266716404492) + | > loss_mel_ce: 2.9787540435791016 (1.9979273552634298) + | > loss: 0.03601333126425743 (0.024245881695050736) + | > current_lr: 5e-06 + | > step_time: 0.1134 (0.14399746097347) + | > loader_time: 0.0128 (0.012772117891619287) + + + --> TIME: 2024-04-09 18:43:54 -- STEP: 856/1354 -- GLOBAL_STEP: 49600 + | > loss_text_ce: 0.034809742122888565 (0.0386369723421913) + | > loss_mel_ce: 1.2750006914138794 (1.9907374334669572) + | > loss: 0.01559298112988472 (0.02415921953049809) + | > current_lr: 5e-06 + | > step_time: 0.1478 (0.1441718133810526) + | > loader_time: 0.0137 (0.012832242194737234) + + + --> TIME: 2024-04-09 18:44:05 -- STEP: 906/1354 -- GLOBAL_STEP: 49650 + | > loss_text_ce: 0.030989812687039375 (0.038642505734797064) + | > loss_mel_ce: 1.3742831945419312 (1.9903945632065119) + | > loss: 0.016729440540075302 (0.024155203612813213) + | > current_lr: 5e-06 + | > step_time: 0.1686 (0.14417437633404948) + | > loader_time: 0.0184 (0.012776702971384755) + + + --> TIME: 2024-04-09 18:44:16 -- STEP: 956/1354 -- GLOBAL_STEP: 49700 + | > loss_text_ce: 0.033368293195962906 (0.03876692800155243) + | > loss_mel_ce: 2.0298891067504883 (1.9920653313024286) + | > loss: 0.024562589824199677 (0.024176574933681638) + | > current_lr: 5e-06 + | > step_time: 0.1145 (0.14382024514625277) + | > loader_time: 0.0106 (0.012728993114567206) + + + --> TIME: 2024-04-09 18:44:27 -- STEP: 1006/1354 -- GLOBAL_STEP: 49750 + | > loss_text_ce: 0.044261422008275986 (0.03881003950013376) + | > loss_mel_ce: 2.125931739807129 (1.9891263122113052) + | > loss: 0.025835633277893066 (0.024142099856389727) + | > current_lr: 5e-06 + | > step_time: 0.1431 (0.1438113422564438) + | > loader_time: 0.0114 (0.012685434244736055) + + + --> TIME: 2024-04-09 18:44:38 -- STEP: 1056/1354 -- GLOBAL_STEP: 49800 + | > loss_text_ce: 0.03486652672290802 (0.03866488439433106) + | > loss_mel_ce: 1.7199132442474365 (1.9833306550528071) + | > loss: 0.020890235900878906 (0.024071375906439204) + | > current_lr: 5e-06 + | > step_time: 0.118 (0.14376032781420348) + | > loader_time: 0.0117 (0.012651830685861187) + + + --> TIME: 2024-04-09 18:44:49 -- STEP: 1106/1354 -- GLOBAL_STEP: 49850 + | > loss_text_ce: 0.030203169211745262 (0.03865952803256497) + | > loss_mel_ce: 1.6825103759765625 (1.9798822039721464) + | > loss: 0.02038944885134697 (0.02403025914936798) + | > current_lr: 5e-06 + | > step_time: 0.1663 (0.14411746959789765) + | > loader_time: 0.0109 (0.012613713202381652) + + + --> TIME: 2024-04-09 18:45:00 -- STEP: 1156/1354 -- GLOBAL_STEP: 49900 + | > loss_text_ce: 0.04182950034737587 (0.038619870912915265) + | > loss_mel_ce: 2.290877342224121 (1.978361186271721) + | > loss: 0.02777032181620598 (0.024011679695416413) + | > current_lr: 5e-06 + | > step_time: 0.1774 (0.1441100934797506) + | > loader_time: 0.0142 (0.01258659507164081) + + + --> TIME: 2024-04-09 18:45:11 -- STEP: 1206/1354 -- GLOBAL_STEP: 49950 + | > loss_text_ce: 0.0373690128326416 (0.038670601190421476) + | > loss_mel_ce: 1.6493415832519531 (1.985141522749938) + | > loss: 0.02007988840341568 (0.024093001916430015) + | > current_lr: 5e-06 + | > step_time: 0.1227 (0.14399759271251633) + | > loader_time: 0.0115 (0.012535665561112996) + + + --> TIME: 2024-04-09 18:45:22 -- STEP: 1256/1354 -- GLOBAL_STEP: 50000 + | > loss_text_ce: 0.05140397325158119 (0.03868630023785881) + | > loss_mel_ce: 2.708881139755249 (1.987327682459431) + | > loss: 0.03286053612828255 (0.024119214523304823) + | > current_lr: 5e-06 + | > step_time: 0.1351 (0.14418357079196129) + | > loader_time: 0.0131 (0.012522552803063847) + + + > CHECKPOINT : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/checkpoint_50000.pth + + --> TIME: 2024-04-09 18:46:15 -- STEP: 1306/1354 -- GLOBAL_STEP: 50050 + | > loss_text_ce: 0.02833956852555275 (0.03865308799619738) + | > loss_mel_ce: 1.689813256263733 (1.986359748504429) + | > loss: 0.020454201847314835 (0.024107296117488703) + | > current_lr: 5e-06 + | > step_time: 0.2187 (0.14454537328863218) + | > loader_time: 0.0165 (0.012582103289652376) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.027736737177922174 (+0.0016065377455491275) + | > avg_loss_text_ce: 0.045376050644195996 (-0.0002729053107591792) + | > avg_loss_mel_ce: 2.871726311170138 (+0.0003714194664588) + | > avg_loss: 2.9171023735633264 (+9.850355295037971e-05) + + + > EPOCH: 38/60 + --> /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011 + + > TRAINING (2024-04-09 18:46:41)  + + --> TIME: 2024-04-09 18:46:42 -- STEP: 2/1354 -- GLOBAL_STEP: 50100 + | > loss_text_ce: 0.04994580149650574 (0.04185745678842068) + | > loss_mel_ce: 2.634519577026367 (1.9760881662368774) + | > loss: 0.031957920640707016 (0.024023162201046944) + | > current_lr: 5e-06 + | > step_time: 0.1356 (0.12617099285125732) + | > loader_time: 0.0109 (0.010815978050231934) + + + --> TIME: 2024-04-09 18:46:54 -- STEP: 52/1354 -- GLOBAL_STEP: 50150 + | > loss_text_ce: 0.0367312952876091 (0.03776772094603914) + | > loss_mel_ce: 1.0838931798934937 (1.9118826756110558) + | > loss: 0.013340767472982407 (0.02321012422012595) + | > current_lr: 5e-06 + | > step_time: 0.1599 (0.1462066861299368) + | > loader_time: 0.0122 (0.012714230097257175) + + + --> TIME: 2024-04-09 18:47:04 -- STEP: 102/1354 -- GLOBAL_STEP: 50200 + | > loss_text_ce: 0.0429220013320446 (0.03862551988705117) + | > loss_mel_ce: 2.1764330863952637 (2.0192288207072835) + | > loss: 0.026420895010232925 (0.024498266387073423) + | > current_lr: 5e-06 + | > step_time: 0.0827 (0.14363783013586895) + | > loader_time: 0.0111 (0.012438362719965916) + + + --> TIME: 2024-04-09 18:47:16 -- STEP: 152/1354 -- GLOBAL_STEP: 50250 + | > loss_text_ce: 0.040825776755809784 (0.03802632484094878) + | > loss_mel_ce: 2.030543565750122 (1.9703127263407954) + | > loss: 0.024659160524606705 (0.023908798693140082) + | > current_lr: 5e-06 + | > step_time: 0.1875 (0.14830217706529716) + | > loader_time: 0.0113 (0.012312633426565875) + + + --> TIME: 2024-04-09 18:47:28 -- STEP: 202/1354 -- GLOBAL_STEP: 50300 + | > loss_text_ce: 0.03385138139128685 (0.03773539607403893) + | > loss_mel_ce: 1.1654412746429443 (1.948490095905738) + | > loss: 0.014277294278144836 (0.023645541953420876) + | > current_lr: 5e-06 + | > step_time: 0.1429 (0.14932730174300693) + | > loader_time: 0.011 (0.012199380610248832) + + + --> TIME: 2024-04-09 18:47:39 -- STEP: 252/1354 -- GLOBAL_STEP: 50350 + | > loss_text_ce: 0.055949702858924866 (0.0382089649238402) + | > loss_mel_ce: 2.7265255451202393 (1.9550906223437141) + | > loss: 0.03312470391392708 (0.023729757372555987) + | > current_lr: 5e-06 + | > step_time: 0.1155 (0.1474211963396224) + | > loader_time: 0.0115 (0.012141373422410754) + + + --> TIME: 2024-04-09 18:47:49 -- STEP: 302/1354 -- GLOBAL_STEP: 50400 + | > loss_text_ce: 0.046106427907943726 (0.03833871310120388) + | > loss_mel_ce: 3.2123351097106934 (1.9700681999424436) + | > loss: 0.03879097104072571 (0.023909606529856166) + | > current_lr: 5e-06 + | > step_time: 0.0893 (0.14654696698220346) + | > loader_time: 0.0108 (0.012108202012169444) + + > Keyboard interrupt detected. + > Saving model before exiting... + + > CHECKPOINT : /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011/checkpoint_50429.pth + ! Run is kept in /workspace/coqui-TTS/finetuning/bambara/run/training/GPT_XTTS_v2.0_BAM_FT-April-09-2024_02+58PM-58810011