2hx8pk65: saving weights and logs of step 50k

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8387a279a508da48c4b9b41b29ad79013e3f168689d639a6ed6588c9f0a4ed1d
+oid sha256:372221fdf608b28a94812360d641b0852fa54d1adc2d7c546610d3200abb28d8
 size 2353616717

wandb/run-20220828_085247-2hx8pk65/files/output.log

@@ -43966,5 +43966,10170 @@ To disable this warning, you can either:
 	- Avoid using `tokenizers` before the fork if possible
 	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
 huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+Training...:  11% 463/4393 [45:57<182:26:10, 167.12s/it]
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+  return jax.tree_map(4393 [45:57<182:26:10, 167.12s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  return jax.tree_map(lambda x: x[0], tree) 4.64s/it]
+run_flax_speech_recognition_seq2seq.py:336: FutureWarning: jax.tree_map is deprecated, and will be removed in a future release. Use jax.tree_util.tree_map instead.
+  return jax.tree_map(lambda x: x.astype(jnp.float32) if x.dtype == jnp.bfloat16 else x, t)
+Step... (40000/50000 | Eval Loss: 0.9527401328086853 | Eval wer: 0.04187346053453917 | Eval cer: 0.02872188479352137 |):  75% 9/12 [53:25:09<17:31:17, 21025.84s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  49% 2162/4393 [3:00:16<3:33:52,  5.75s/it]
+Step... (40000/50000 | Eval Loss: 0.9527401328086853 | Eval wer: 0.04187346053453917 | Eval cer: 0.02872188479352137 |)
+Step... (40025 | Loss: 0.008939584717154503, Learning Rate: 2.0153533114353195e-05, Gradient Norm: 0.29531925916671753)
+Step... (40050 | Loss: 0.013731228187680244, Learning Rate: 2.0103030692553148e-05, Gradient Norm: 0.28097018599510193)
+Step... (40075 | Loss: 0.005788063630461693, Learning Rate: 2.00525282707531e-05, Gradient Norm: 0.2137639969587326)
+Step... (40100 | Loss: 0.006226960103958845, Learning Rate: 2.000201857299544e-05, Gradient Norm: 0.1285054236650467)
+Step... (40125 | Loss: 0.007349521853029728, Learning Rate: 1.9951516151195392e-05, Gradient Norm: 0.233669713139534)
+Step... (40150 | Loss: 0.010777842253446579, Learning Rate: 1.9901013729395345e-05, Gradient Norm: 0.24161839485168457)
+Step... (40175 | Loss: 0.0042556338012218475, Learning Rate: 1.9850504031637684e-05, Gradient Norm: 0.1810857057571411)
+Step... (40200 | Loss: 0.013205939903855324, Learning Rate: 1.9800001609837636e-05, Gradient Norm: 0.2645622491836548)
+Step... (40225 | Loss: 0.010763286612927914, Learning Rate: 1.9749491912079975e-05, Gradient Norm: 0.2870539128780365)
+Step... (40250 | Loss: 0.012334151193499565, Learning Rate: 1.9698989490279928e-05, Gradient Norm: 0.2389720231294632)
+Step... (40275 | Loss: 0.0026933024637401104, Learning Rate: 1.9648485249490477e-05, Gradient Norm: 0.09100257605314255)
+Step... (40300 | Loss: 0.007666177581995726, Learning Rate: 1.959797737072222e-05, Gradient Norm: 0.14644892513751984)
+Step... (40325 | Loss: 0.006105078849941492, Learning Rate: 1.954747312993277e-05, Gradient Norm: 0.1783716231584549)
+Step... (40350 | Loss: 0.007519055623561144, Learning Rate: 1.949697070813272e-05, Gradient Norm: 0.196736142039299)
+Step... (40375 | Loss: 0.008998542092740536, Learning Rate: 1.944646101037506e-05, Gradient Norm: 0.26219305396080017)
+Step... (40400 | Loss: 0.006001105532050133, Learning Rate: 1.9395958588575013e-05, Gradient Norm: 0.17653429508209229)
+Step... (40425 | Loss: 0.003485818626359105, Learning Rate: 1.9345456166774966e-05, Gradient Norm: 0.1488671600818634)
+Step... (40450 | Loss: 0.011663154698908329, Learning Rate: 1.9294946469017304e-05, Gradient Norm: 0.21771906316280365)
+Step... (40475 | Loss: 0.01133037731051445, Learning Rate: 1.9244444047217257e-05, Gradient Norm: 0.3833330273628235)
+Step... (40500 | Loss: 0.004920892883092165, Learning Rate: 1.919394162541721e-05, Gradient Norm: 0.1484016329050064)
+Step... (40525 | Loss: 0.016179556027054787, Learning Rate: 1.914343192765955e-05, Gradient Norm: 0.27268052101135254)
+Step... (40550 | Loss: 0.007338401861488819, Learning Rate: 1.90929295058595e-05, Gradient Norm: 0.18785510957241058)
+Step... (40575 | Loss: 0.003818551544100046, Learning Rate: 1.904242526507005e-05, Gradient Norm: 0.1895657181739807)
+Step... (40600 | Loss: 0.008862589485943317, Learning Rate: 1.8991917386301793e-05, Gradient Norm: 0.2292691171169281)
+Step... (40625 | Loss: 0.0069699776358902454, Learning Rate: 1.8941413145512342e-05, Gradient Norm: 0.1485799252986908)
+Step... (40650 | Loss: 0.010429407469928265, Learning Rate: 1.8890910723712295e-05, Gradient Norm: 0.23329854011535645)
+Step... (40675 | Loss: 0.005073365289717913, Learning Rate: 1.8840401025954634e-05, Gradient Norm: 0.1328592151403427)
+Step... (40700 | Loss: 0.0052896239794790745, Learning Rate: 1.8789898604154587e-05, Gradient Norm: 0.1811048984527588)
+Step... (40725 | Loss: 0.0049026296474039555, Learning Rate: 1.873939618235454e-05, Gradient Norm: 0.1749919205904007)
+Step... (40750 | Loss: 0.004890045616775751, Learning Rate: 1.8688886484596878e-05, Gradient Norm: 0.15074510872364044)
+Step... (40775 | Loss: 0.003988181706517935, Learning Rate: 1.863838406279683e-05, Gradient Norm: 0.19411082565784454)
+Step... (40800 | Loss: 0.00978496391326189, Learning Rate: 1.8587881640996784e-05, Gradient Norm: 0.20905667543411255)
+Step... (40825 | Loss: 0.006088871043175459, Learning Rate: 1.8537371943239123e-05, Gradient Norm: 0.3005000054836273)
+Step... (40850 | Loss: 0.010514942929148674, Learning Rate: 1.8486869521439075e-05, Gradient Norm: 0.2221343070268631)
+Step... (40875 | Loss: 0.006962936837226152, Learning Rate: 1.8436365280649625e-05, Gradient Norm: 0.19761787354946136)
+Step... (40900 | Loss: 0.011204604059457779, Learning Rate: 1.8385857401881367e-05, Gradient Norm: 0.25883570313453674)
+Step... (40925 | Loss: 0.005226748064160347, Learning Rate: 1.8335353161091916e-05, Gradient Norm: 0.2500029504299164)
+Step... (40950 | Loss: 0.010302098467946053, Learning Rate: 1.828485073929187e-05, Gradient Norm: 0.18769465386867523)
+Step... (40975 | Loss: 0.005125986412167549, Learning Rate: 1.8234341041534208e-05, Gradient Norm: 0.2729404866695404)
+Step... (41000 | Loss: 0.010441966354846954, Learning Rate: 1.818383861973416e-05, Gradient Norm: 0.3461083769798279)
+Step... (41025 | Loss: 0.0025685643777251244, Learning Rate: 1.8133336197934113e-05, Gradient Norm: 0.12441041320562363)
+Step... (41050 | Loss: 0.005768472328782082, Learning Rate: 1.8082826500176452e-05, Gradient Norm: 0.16698002815246582)
+Step... (41075 | Loss: 0.00754266232252121, Learning Rate: 1.8032324078376405e-05, Gradient Norm: 0.2963465750217438)
+Step... (41100 | Loss: 0.010014562867581844, Learning Rate: 1.7981821656576358e-05, Gradient Norm: 0.16458502411842346)
+Step... (41125 | Loss: 0.0019032791024073958, Learning Rate: 1.7931311958818696e-05, Gradient Norm: 0.06771261245012283)
+Step... (41150 | Loss: 0.007408216595649719, Learning Rate: 1.788080953701865e-05, Gradient Norm: 0.1833212673664093)
+Step... (41175 | Loss: 0.006798009388148785, Learning Rate: 1.78303052962292e-05, Gradient Norm: 0.28412139415740967)
+Step... (41200 | Loss: 0.010547134093940258, Learning Rate: 1.777979741746094e-05, Gradient Norm: 0.6392953991889954)
+Step... (41225 | Loss: 0.002890239469707012, Learning Rate: 1.772929317667149e-05, Gradient Norm: 0.14131464064121246)
+Step... (41250 | Loss: 0.012237129732966423, Learning Rate: 1.7678790754871443e-05, Gradient Norm: 0.24268372356891632)
+Step... (41275 | Loss: 0.003835662268102169, Learning Rate: 1.762828105711378e-05, Gradient Norm: 0.20543915033340454)
+Step... (41300 | Loss: 0.009825979359447956, Learning Rate: 1.7577778635313734e-05, Gradient Norm: 0.2134653478860855)
+Step... (41325 | Loss: 0.0023147186730057, Learning Rate: 1.7527270756545477e-05, Gradient Norm: 0.10707230120897293)
+Step... (41350 | Loss: 0.0070909978821873665, Learning Rate: 1.7476766515756026e-05, Gradient Norm: 0.17582404613494873)
+Step... (41375 | Loss: 0.0033097451087087393, Learning Rate: 1.742626409395598e-05, Gradient Norm: 0.13646838068962097)
+Step... (41400 | Loss: 0.008395623415708542, Learning Rate: 1.7375754396198317e-05, Gradient Norm: 0.1624133586883545)
+Step... (41425 | Loss: 0.004059635568410158, Learning Rate: 1.732525197439827e-05, Gradient Norm: 0.21894600987434387)
+Step... (41450 | Loss: 0.00695464015007019, Learning Rate: 1.7274749552598223e-05, Gradient Norm: 0.1792638897895813)
+Step... (41475 | Loss: 0.009820655919611454, Learning Rate: 1.7224239854840562e-05, Gradient Norm: 0.4551573097705841)
+Step... (41500 | Loss: 0.006216324865818024, Learning Rate: 1.7173737433040515e-05, Gradient Norm: 0.17348088324069977)
+Step... (41525 | Loss: 0.008928379975259304, Learning Rate: 1.7123233192251064e-05, Gradient Norm: 0.32115671038627625)
+Step... (41550 | Loss: 0.012027319520711899, Learning Rate: 1.7072725313482806e-05, Gradient Norm: 0.2573080360889435)
+Step... (41575 | Loss: 0.007203887682408094, Learning Rate: 1.7022221072693355e-05, Gradient Norm: 0.24148206412792206)
+Step... (41600 | Loss: 0.01733251102268696, Learning Rate: 1.6971718650893308e-05, Gradient Norm: 0.28181859850883484)
+Step... (41625 | Loss: 0.00894920527935028, Learning Rate: 1.6921208953135647e-05, Gradient Norm: 0.282490611076355)
+Step... (41650 | Loss: 0.009486709721386433, Learning Rate: 1.68707065313356e-05, Gradient Norm: 0.2058715522289276)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  88% 3862/4393 [5:14:58<51:10,  5.78s/it]
+Step... (41700 | Loss: 0.005744441412389278, Learning Rate: 1.676969441177789e-05, Gradient Norm: 0.1374588906764984)
+Step... (41725 | Loss: 0.008426920510828495, Learning Rate: 1.6719191989977844e-05, Gradient Norm: 0.8020673990249634)
+Step... (41750 | Loss: 0.0075722914189100266, Learning Rate: 1.6668689568177797e-05, Gradient Norm: 0.32124248147010803)
+Step... (41775 | Loss: 0.005043766926974058, Learning Rate: 1.6618179870420136e-05, Gradient Norm: 0.20893269777297974)
+Step... (41800 | Loss: 0.010060012340545654, Learning Rate: 1.656767744862009e-05, Gradient Norm: 0.21614883840084076)
+Step... (41825 | Loss: 0.014381218701601028, Learning Rate: 1.6517173207830638e-05, Gradient Norm: 0.35426953434944153)
+Step... (41850 | Loss: 0.01061459444463253, Learning Rate: 1.646666532906238e-05, Gradient Norm: 0.29452937841415405)
+Step... (41875 | Loss: 0.012180848978459835, Learning Rate: 1.641616108827293e-05, Gradient Norm: 0.3592362105846405)
+Step... (41900 | Loss: 0.00836265180259943, Learning Rate: 1.6365658666472882e-05, Gradient Norm: 0.22259344160556793)
+Step... (41925 | Loss: 0.008231023326516151, Learning Rate: 1.631514896871522e-05, Gradient Norm: 0.20443135499954224)
+Step... (41950 | Loss: 0.01289241760969162, Learning Rate: 1.6264646546915174e-05, Gradient Norm: 0.3498441278934479)
+Step... (41975 | Loss: 0.0034512323327362537, Learning Rate: 1.6214144125115126e-05, Gradient Norm: 0.13234050571918488)
+Step... (42000 | Loss: 0.011287801899015903, Learning Rate: 1.6163634427357465e-05, Gradient Norm: 0.4331746995449066)
+Step... (42025 | Loss: 0.003170725191012025, Learning Rate: 1.6113132005557418e-05, Gradient Norm: 0.13511165976524353)
+Step... (42050 | Loss: 0.011994440108537674, Learning Rate: 1.606262958375737e-05, Gradient Norm: 0.2843227684497833)
+Step... (42075 | Loss: 0.003315770300105214, Learning Rate: 1.601211988599971e-05, Gradient Norm: 0.18491697311401367)
+Step... (42100 | Loss: 0.011447119526565075, Learning Rate: 1.5961617464199662e-05, Gradient Norm: 0.23545017838478088)
+Step... (42125 | Loss: 0.005759582854807377, Learning Rate: 1.591111322341021e-05, Gradient Norm: 0.2273390293121338)
+Step... (42150 | Loss: 0.010285439901053905, Learning Rate: 1.5860605344641954e-05, Gradient Norm: 0.2514696419239044)
+Step... (42175 | Loss: 0.004782717674970627, Learning Rate: 1.5810101103852503e-05, Gradient Norm: 0.17328862845897675)
+Step... (42200 | Loss: 0.010704944841563702, Learning Rate: 1.5759598682052456e-05, Gradient Norm: 0.26759225130081177)
+Step... (42225 | Loss: 0.008894861675798893, Learning Rate: 1.5709088984294794e-05, Gradient Norm: 0.21136222779750824)
+Step... (42250 | Loss: 0.014378275722265244, Learning Rate: 1.5658586562494747e-05, Gradient Norm: 0.2818372845649719)
+Step... (42275 | Loss: 0.003270916873589158, Learning Rate: 1.56080841406947e-05, Gradient Norm: 0.1778145730495453)
+Step... (42300 | Loss: 0.01763617806136608, Learning Rate: 1.555757444293704e-05, Gradient Norm: 0.8628730773925781)
+Step... (42325 | Loss: 0.003942512441426516, Learning Rate: 1.550707202113699e-05, Gradient Norm: 0.13781097531318665)
+Step... (42350 | Loss: 0.0093576330691576, Learning Rate: 1.545656232337933e-05, Gradient Norm: 0.1558447927236557)
+Step... (42375 | Loss: 0.005177142098546028, Learning Rate: 1.5406059901579283e-05, Gradient Norm: 0.19731150567531586)
+Step... (42400 | Loss: 0.008837847970426083, Learning Rate: 1.5355557479779236e-05, Gradient Norm: 0.2248029261827469)
+Step... (42425 | Loss: 0.004063450265675783, Learning Rate: 1.5305047782021575e-05, Gradient Norm: 0.15096257627010345)
+Step... (42450 | Loss: 0.0156438909471035, Learning Rate: 1.5254544450726826e-05, Gradient Norm: 0.5371332168579102)
+Step... (42475 | Loss: 0.004050035495311022, Learning Rate: 1.5204042028926779e-05, Gradient Norm: 0.2783503830432892)
+Step... (42500 | Loss: 0.008893079124391079, Learning Rate: 1.5153533240663819e-05, Gradient Norm: 0.16905491054058075)
+Step... (42525 | Loss: 0.0038313837721943855, Learning Rate: 1.510302990936907e-05, Gradient Norm: 0.13977232575416565)
+Step... (42550 | Loss: 0.008676256984472275, Learning Rate: 1.5052526578074321e-05, Gradient Norm: 0.1817447394132614)
+Step... (42575 | Loss: 0.005087659694254398, Learning Rate: 1.5002017789811362e-05, Gradient Norm: 0.44567009806632996)
+Step... (42600 | Loss: 0.013834933750331402, Learning Rate: 1.4951514458516613e-05, Gradient Norm: 0.46296262741088867)
+Step... (42625 | Loss: 0.01469030138105154, Learning Rate: 1.4901012036716565e-05, Gradient Norm: 0.6844345331192017)
+Step... (42650 | Loss: 0.005226873327046633, Learning Rate: 1.4850503248453606e-05, Gradient Norm: 0.18194365501403809)
+Step... (42675 | Loss: 0.010348882526159286, Learning Rate: 1.4799999917158857e-05, Gradient Norm: 4.603023529052734)
+Step... (42700 | Loss: 0.00900035910308361, Learning Rate: 1.4749496585864108e-05, Gradient Norm: 0.22959791123867035)
+Step... (42725 | Loss: 0.004127497784793377, Learning Rate: 1.4698987797601148e-05, Gradient Norm: 0.16160161793231964)
+Step... (42750 | Loss: 0.009940936230123043, Learning Rate: 1.46484844663064e-05, Gradient Norm: 0.2361784130334854)
+Step... (42775 | Loss: 0.009671159088611603, Learning Rate: 1.4597982044506352e-05, Gradient Norm: 0.24877315759658813)
+Step... (42800 | Loss: 0.010518192313611507, Learning Rate: 1.4547473256243393e-05, Gradient Norm: 0.2851322591304779)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Step... (40000/50000 | Eval Loss: 0.9527401328086853 | Eval wer: 0.04187346053453917 | Eval cer: 0.02872188479352137 |):  83% 10/12 [58:33:51<11:44:51, 21145.58s/it]
+Training...:   0% 0/4393 [00:00<?, ?it/s]
+Step... (42875 | Loss: 0.013169726356863976, Learning Rate: 1.4395957805390935e-05, Gradient Norm: 0.2936961054801941)
+Step... (42900 | Loss: 0.007032770197838545, Learning Rate: 1.4345454474096186e-05, Gradient Norm: 0.20587222278118134)
+Step... (42925 | Loss: 0.004653589334338903, Learning Rate: 1.429495205229614e-05, Gradient Norm: 0.18729034066200256)
+Step... (42950 | Loss: 0.005974963773041964, Learning Rate: 1.424444326403318e-05, Gradient Norm: 0.1742350161075592)
+Step... (42975 | Loss: 0.0030157752335071564, Learning Rate: 1.419393993273843e-05, Gradient Norm: 0.09114522486925125)
+Step... (43000 | Loss: 0.017072968184947968, Learning Rate: 1.4143436601443682e-05, Gradient Norm: 0.344232439994812)
+Step... (43025 | Loss: 0.007778532337397337, Learning Rate: 1.4092927813180722e-05, Gradient Norm: 0.24894601106643677)
+Step... (43050 | Loss: 0.005137579515576363, Learning Rate: 1.4042424481885973e-05, Gradient Norm: 0.15516820549964905)
+Step... (43075 | Loss: 0.008852087892591953, Learning Rate: 1.3991922060085926e-05, Gradient Norm: 0.2543483078479767)
+Step... (43100 | Loss: 0.005538458935916424, Learning Rate: 1.3941412362328265e-05, Gradient Norm: 0.20741888880729675)
+Step... (43125 | Loss: 0.009585311636328697, Learning Rate: 1.3890909940528218e-05, Gradient Norm: 0.32343360781669617)
+Step... (43150 | Loss: 0.009101358242332935, Learning Rate: 1.3840406609233469e-05, Gradient Norm: 0.1852715015411377)
+Step... (43175 | Loss: 0.0038933581672608852, Learning Rate: 1.378989782097051e-05, Gradient Norm: 0.200323686003685)
+Step... (43200 | Loss: 0.006398520432412624, Learning Rate: 1.373939448967576e-05, Gradient Norm: 0.3812198340892792)
+Step... (43225 | Loss: 0.00649980129674077, Learning Rate: 1.3688892067875713e-05, Gradient Norm: 0.2510472238063812)
+Step... (43250 | Loss: 0.007147450000047684, Learning Rate: 1.3638382370118052e-05, Gradient Norm: 0.2402760088443756)
+Step... (43275 | Loss: 0.005947265308350325, Learning Rate: 1.3587879948318005e-05, Gradient Norm: 0.21199539303779602)
+Step... (43300 | Loss: 0.01116273459047079, Learning Rate: 1.3537376617023256e-05, Gradient Norm: 0.1889847368001938)
+Step... (43325 | Loss: 0.009462771005928516, Learning Rate: 1.3486867828760296e-05, Gradient Norm: 0.4412449598312378)
+Step... (43350 | Loss: 0.01658274605870247, Learning Rate: 1.3436364497465547e-05, Gradient Norm: 0.24183489382266998)
+Step... (43375 | Loss: 0.005811340641230345, Learning Rate: 1.3385855709202588e-05, Gradient Norm: 0.13360491394996643)
+Step... (43400 | Loss: 0.006220472510904074, Learning Rate: 1.3335352377907839e-05, Gradient Norm: 0.15546762943267822)
+Step... (43425 | Loss: 0.006581796798855066, Learning Rate: 1.3284849956107792e-05, Gradient Norm: 0.34990987181663513)
+Step... (43450 | Loss: 0.011822747066617012, Learning Rate: 1.3234341167844832e-05, Gradient Norm: 0.2574056386947632)
+Step... (43475 | Loss: 0.00934094749391079, Learning Rate: 1.3183837836550083e-05, Gradient Norm: 0.3183192014694214)
+Step... (43500 | Loss: 0.005102855619043112, Learning Rate: 1.3133334505255334e-05, Gradient Norm: 0.14597401022911072)
+Step... (43525 | Loss: 0.008145734667778015, Learning Rate: 1.3082825716992375e-05, Gradient Norm: 0.19706687331199646)
+Step... (43550 | Loss: 0.009935390204191208, Learning Rate: 1.3032322385697626e-05, Gradient Norm: 0.26017606258392334)
+Step... (43575 | Loss: 0.007766792085021734, Learning Rate: 1.2981819963897578e-05, Gradient Norm: 0.2769607603549957)
+Step... (43600 | Loss: 0.010773919522762299, Learning Rate: 1.2931311175634619e-05, Gradient Norm: 0.23994655907154083)
+Step... (43625 | Loss: 0.0029236481059342623, Learning Rate: 1.288080784433987e-05, Gradient Norm: 0.1163998618721962)
+Step... (43650 | Loss: 0.013007597997784615, Learning Rate: 1.2830304513045121e-05, Gradient Norm: 0.2571600377559662)
+Step... (43675 | Loss: 0.006016268394887447, Learning Rate: 1.2779795724782161e-05, Gradient Norm: 0.2098936140537262)
+Step... (43700 | Loss: 0.00377640244551003, Learning Rate: 1.2729292393487412e-05, Gradient Norm: 0.0722104161977768)
+Step... (43725 | Loss: 0.005282220430672169, Learning Rate: 1.2678789971687365e-05, Gradient Norm: 0.19065354764461517)
+Step... (43750 | Loss: 0.005376732908189297, Learning Rate: 1.2628281183424406e-05, Gradient Norm: 0.16388669610023499)
+Step... (43775 | Loss: 0.008202340453863144, Learning Rate: 1.2577777852129657e-05, Gradient Norm: 0.1876869797706604)
+Step... (43800 | Loss: 0.004423276986926794, Learning Rate: 1.2527274520834908e-05, Gradient Norm: 0.14415715634822845)
+Step... (43825 | Loss: 0.0041627101600170135, Learning Rate: 1.2476765732571948e-05, Gradient Norm: 0.18802481889724731)
+Step... (43850 | Loss: 0.012115256860852242, Learning Rate: 1.24262624012772e-05, Gradient Norm: 0.35901159048080444)
+Step... (43875 | Loss: 0.005729792173951864, Learning Rate: 1.2375759979477152e-05, Gradient Norm: 0.3747013211250305)
+Step... (43900 | Loss: 0.005576059687882662, Learning Rate: 1.2325251191214193e-05, Gradient Norm: 0.14393489062786102)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  39% 1719/4393 [2:16:28<4:08:58,  5.59s/it]
+Step... (43950 | Loss: 0.001941631082445383, Learning Rate: 1.2224244528624695e-05, Gradient Norm: 0.07129886001348495)
+Step... (43975 | Loss: 0.0024623877834528685, Learning Rate: 1.2173735740361735e-05, Gradient Norm: 0.16456308960914612)
+Step... (44000 | Loss: 0.005050975829362869, Learning Rate: 1.2123232409066986e-05, Gradient Norm: 0.16223661601543427)
+Step... (44025 | Loss: 0.003012409433722496, Learning Rate: 1.2072729987266939e-05, Gradient Norm: 0.2435944676399231)
+Step... (44050 | Loss: 0.002762197982519865, Learning Rate: 1.202222119900398e-05, Gradient Norm: 0.0841917023062706)
+Step... (44075 | Loss: 0.009030467830598354, Learning Rate: 1.197171786770923e-05, Gradient Norm: 0.832758903503418)
+Step... (44100 | Loss: 0.005016005132347345, Learning Rate: 1.1921214536414482e-05, Gradient Norm: 0.1769050508737564)
+Step... (44125 | Loss: 0.0022536602336913347, Learning Rate: 1.1870705748151522e-05, Gradient Norm: 0.17979957163333893)
+Step... (44150 | Loss: 0.004251351114362478, Learning Rate: 1.1820202416856773e-05, Gradient Norm: 0.2023152858018875)
+Step... (44175 | Loss: 0.001228940673172474, Learning Rate: 1.1769699995056726e-05, Gradient Norm: 0.08239232003688812)
+Step... (44200 | Loss: 0.0023271413519978523, Learning Rate: 1.1719191206793766e-05, Gradient Norm: 0.13417169451713562)
+Step... (44225 | Loss: 0.0019650713074952364, Learning Rate: 1.1668687875499018e-05, Gradient Norm: 0.14780059456825256)
+Step... (44250 | Loss: 0.00524902855977416, Learning Rate: 1.1618184544204269e-05, Gradient Norm: 0.23804089426994324)
+Step... (44275 | Loss: 0.0012860909337177873, Learning Rate: 1.1567675755941309e-05, Gradient Norm: 0.12285351753234863)
+Step... (44300 | Loss: 0.007503165397793055, Learning Rate: 1.151717242464656e-05, Gradient Norm: 0.2330530434846878)
+Step... (44325 | Loss: 0.0006487671053037047, Learning Rate: 1.1466670002846513e-05, Gradient Norm: 0.03001929260790348)
+Step... (44350 | Loss: 0.004298559855669737, Learning Rate: 1.1416161214583553e-05, Gradient Norm: 0.13478074967861176)
+Step... (44375 | Loss: 0.002003872999921441, Learning Rate: 1.1365657883288804e-05, Gradient Norm: 0.19664216041564941)
+Step... (44400 | Loss: 0.0063357907347381115, Learning Rate: 1.1315149095025845e-05, Gradient Norm: 0.33638206124305725)
+Step... (44425 | Loss: 0.0043112244457006454, Learning Rate: 1.1264645763731096e-05, Gradient Norm: 0.4819294214248657)
+Step... (44450 | Loss: 0.005926346872001886, Learning Rate: 1.1214142432436347e-05, Gradient Norm: 0.20366427302360535)
+Step... (44475 | Loss: 0.0005286787636578083, Learning Rate: 1.1163633644173387e-05, Gradient Norm: 0.023813458159565926)
+Step... (44500 | Loss: 0.002619663020595908, Learning Rate: 1.1113130312878639e-05, Gradient Norm: 0.14585494995117188)
+Step... (44525 | Loss: 0.0008802820811979473, Learning Rate: 1.1062627891078591e-05, Gradient Norm: 0.05259334295988083)
+Step... (44550 | Loss: 0.007913529872894287, Learning Rate: 1.1012119102815632e-05, Gradient Norm: 0.2658507525920868)
+Step... (44575 | Loss: 0.0005179516738280654, Learning Rate: 1.0961615771520883e-05, Gradient Norm: 0.0357116162776947)
+Step... (44600 | Loss: 0.006706691812723875, Learning Rate: 1.0911112440226134e-05, Gradient Norm: 0.19824419915676117)
+Step... (44625 | Loss: 0.000933133764192462, Learning Rate: 1.0860603651963174e-05, Gradient Norm: 0.06666836887598038)
+Step... (44650 | Loss: 0.0057255276478827, Learning Rate: 1.0810100320668425e-05, Gradient Norm: 0.14937716722488403)
+Step... (44675 | Loss: 0.0018604991491883993, Learning Rate: 1.0759597898868378e-05, Gradient Norm: 0.1628139764070511)
+Step... (44700 | Loss: 0.00716938404366374, Learning Rate: 1.0709089110605419e-05, Gradient Norm: 0.1885882467031479)
+Step... (44725 | Loss: 0.001772589748725295, Learning Rate: 1.065858577931067e-05, Gradient Norm: 0.1799585074186325)
+Step... (44750 | Loss: 0.0021098533179610968, Learning Rate: 1.060808244801592e-05, Gradient Norm: 0.07331665605306625)
+Step... (44775 | Loss: 0.0028699678368866444, Learning Rate: 1.0557573659752961e-05, Gradient Norm: 0.3392123579978943)
+Step... (44800 | Loss: 0.008694657124578953, Learning Rate: 1.0507070328458212e-05, Gradient Norm: 0.3680751323699951)
+Step... (44825 | Loss: 0.0008265204378403723, Learning Rate: 1.0456567906658165e-05, Gradient Norm: 0.055144477635622025)
+Step... (44850 | Loss: 0.002472145017236471, Learning Rate: 1.0406059118395206e-05, Gradient Norm: 0.12441207468509674)
+Step... (44875 | Loss: 0.0018344148993492126, Learning Rate: 1.0355555787100457e-05, Gradient Norm: 0.10230448096990585)
+Step... (44900 | Loss: 0.005570901557803154, Learning Rate: 1.0305052455805708e-05, Gradient Norm: 0.11906146258115768)
+Step... (44925 | Loss: 0.000813481688965112, Learning Rate: 1.0254543667542748e-05, Gradient Norm: 0.06892543286085129)
+Step... (44950 | Loss: 0.004630820360034704, Learning Rate: 1.0204040336248e-05, Gradient Norm: 0.18374110758304596)
+Step... (44975 | Loss: 0.000776787637732923, Learning Rate: 1.0153537914447952e-05, Gradient Norm: 0.04277883097529411)
+Step... (45000 | Loss: 0.00541204446926713, Learning Rate: 1.0103029126184992e-05, Gradient Norm: 0.1890871673822403)
+Step... (45025 | Loss: 0.002748969243839383, Learning Rate: 1.0052525794890244e-05, Gradient Norm: 0.21585719287395477)
+Step... (45050 | Loss: 0.004655120428651571, Learning Rate: 1.0002022463595495e-05, Gradient Norm: 0.15042269229888916)
+Step... (45075 | Loss: 0.001320728100836277, Learning Rate: 9.951513675332535e-06, Gradient Norm: 0.12991033494472504)
+Step... (45100 | Loss: 0.00846528448164463, Learning Rate: 9.901010344037786e-06, Gradient Norm: 0.2679833173751831)
+Step... (45125 | Loss: 0.0013782067690044641, Learning Rate: 9.850507922237739e-06, Gradient Norm: 0.13327546417713165)
+Step... (45150 | Loss: 0.00527406157925725, Learning Rate: 9.79999913397478e-06, Gradient Norm: 0.14178180694580078)
+Step... (45175 | Loss: 0.0010317324195057154, Learning Rate: 9.74949580268003e-06, Gradient Norm: 0.06164395809173584)
+Step... (45200 | Loss: 0.008524654433131218, Learning Rate: 9.698992471385282e-06, Gradient Norm: 0.26919326186180115)
+Step... (45225 | Loss: 0.0011031378526240587, Learning Rate: 9.648483683122322e-06, Gradient Norm: 0.0625748485326767)
+Step... (45250 | Loss: 0.011103019118309021, Learning Rate: 9.597980351827573e-06, Gradient Norm: 0.256208598613739)
+Step... (45275 | Loss: 0.0013706317404285073, Learning Rate: 9.547477930027526e-06, Gradient Norm: 0.09869988262653351)
+Step... (45300 | Loss: 0.0035194139927625656, Learning Rate: 9.496969141764566e-06, Gradient Norm: 0.1313745528459549)
+Step... (45325 | Loss: 0.0014361287467181683, Learning Rate: 9.446465810469817e-06, Gradient Norm: 0.07393680512905121)
+Step... (45350 | Loss: 0.0033916528336703777, Learning Rate: 9.395962479175068e-06, Gradient Norm: 0.11351648718118668)
+Step... (45375 | Loss: 0.005707416217774153, Learning Rate: 9.345453690912109e-06, Gradient Norm: 0.35709279775619507)
+Step... (45400 | Loss: 0.0034715337678790092, Learning Rate: 9.29495035961736e-06, Gradient Norm: 0.16799037158489227)
+Step... (45425 | Loss: 0.0010701533174142241, Learning Rate: 9.2444415713544e-06, Gradient Norm: 0.09919247031211853)
+Step... (45450 | Loss: 0.006887105293571949, Learning Rate: 9.193939149554353e-06, Gradient Norm: 0.12867707014083862)
+Step... (45475 | Loss: 0.0009366113226860762, Learning Rate: 9.143435818259604e-06, Gradient Norm: 0.06387642025947571)
+Step... (45500 | Loss: 0.0053701805882155895, Learning Rate: 9.092927029996645e-06, Gradient Norm: 0.1894822120666504)
+Step... (45525 | Loss: 0.0031545665115118027, Learning Rate: 9.042423698701896e-06, Gradient Norm: 0.20755034685134888)
+Step... (45550 | Loss: 0.005287761799991131, Learning Rate: 8.991920367407147e-06, Gradient Norm: 0.15759794414043427)
+Step... (45575 | Loss: 0.0004927063127979636, Learning Rate: 8.941411579144187e-06, Gradient Norm: 0.03520292416214943)
+Step... (45600 | Loss: 0.0040636854246258736, Learning Rate: 8.89090915734414e-06, Gradient Norm: 0.12357061356306076)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  78% 3420/4393 [4:31:14<1:34:13,  5.81s/it]
+Step... (45650 | Loss: 0.0038062885869294405, Learning Rate: 8.789897037786432e-06, Gradient Norm: 0.12565107643604279)
+Step... (45675 | Loss: 0.001629532314836979, Learning Rate: 8.739393706491683e-06, Gradient Norm: 0.1391201615333557)
+Step... (45700 | Loss: 0.002758459188044071, Learning Rate: 8.688890375196934e-06, Gradient Norm: 0.14725351333618164)
+Step... (45725 | Loss: 0.0015460265567526221, Learning Rate: 8.638381586933974e-06, Gradient Norm: 0.12334706634283066)
+Step... (45750 | Loss: 0.008942586369812489, Learning Rate: 8.587879165133927e-06, Gradient Norm: 0.19431252777576447)
+Step... (45775 | Loss: 0.0018675904721021652, Learning Rate: 8.537375833839178e-06, Gradient Norm: 0.22979864478111267)
+Step... (45800 | Loss: 0.0032472757156938314, Learning Rate: 8.486867045576219e-06, Gradient Norm: 0.16855953633785248)
+Step... (45825 | Loss: 0.0020079060923308134, Learning Rate: 8.43636371428147e-06, Gradient Norm: 0.1927141398191452)
+Step... (45850 | Loss: 0.002314024604856968, Learning Rate: 8.38586038298672e-06, Gradient Norm: 0.10700368136167526)
+Step... (45875 | Loss: 0.0023375945165753365, Learning Rate: 8.335351594723761e-06, Gradient Norm: 0.25536608695983887)
+Step... (45900 | Loss: 0.005266377702355385, Learning Rate: 8.284849172923714e-06, Gradient Norm: 0.14284679293632507)
+Step... (45925 | Loss: 0.002833940787240863, Learning Rate: 8.234345841628965e-06, Gradient Norm: 0.13957801461219788)
+Step... (45950 | Loss: 0.004558590240776539, Learning Rate: 8.183837053366005e-06, Gradient Norm: 0.1979239284992218)
+Step... (45975 | Loss: 0.0004439667100086808, Learning Rate: 8.133333722071256e-06, Gradient Norm: 0.023552939295768738)
+Step... (46000 | Loss: 0.003100600326433778, Learning Rate: 8.082830390776508e-06, Gradient Norm: 0.09397352486848831)
+Step... (46025 | Loss: 0.00844496488571167, Learning Rate: 8.032321602513548e-06, Gradient Norm: 0.34702354669570923)
+Step... (46050 | Loss: 0.00445242365822196, Learning Rate: 7.981818271218799e-06, Gradient Norm: 0.14293652772903442)
+Step... (46075 | Loss: 0.0017299428582191467, Learning Rate: 7.931315849418752e-06, Gradient Norm: 0.1346822828054428)
+Step... (46100 | Loss: 0.004057885613292456, Learning Rate: 7.880807061155792e-06, Gradient Norm: 0.12305817753076553)
+Step... (46125 | Loss: 0.001030999468639493, Learning Rate: 7.830303729861043e-06, Gradient Norm: 0.05135162174701691)
+Step... (46150 | Loss: 0.002076568314805627, Learning Rate: 7.779800398566294e-06, Gradient Norm: 0.10389827191829681)
+Step... (46175 | Loss: 0.0029374691657721996, Learning Rate: 7.729291610303335e-06, Gradient Norm: 0.2249249517917633)
+Step... (46200 | Loss: 0.003002022160217166, Learning Rate: 7.678788279008586e-06, Gradient Norm: 0.20660805702209473)
+Step... (46225 | Loss: 0.0015353269409388304, Learning Rate: 7.628285857208539e-06, Gradient Norm: 0.13045524060726166)
+Step... (46250 | Loss: 0.010492435656487942, Learning Rate: 7.577776614198228e-06, Gradient Norm: 0.2511439621448517)
+Step... (46275 | Loss: 0.001488243229687214, Learning Rate: 7.52727373765083e-06, Gradient Norm: 0.11494113504886627)
+Step... (46300 | Loss: 0.002643989399075508, Learning Rate: 7.476770861103432e-06, Gradient Norm: 0.12378372997045517)
+Step... (46325 | Loss: 0.0028585607651621103, Learning Rate: 7.426261618093122e-06, Gradient Norm: 0.17171597480773926)
+Step... (46350 | Loss: 0.005086139775812626, Learning Rate: 7.375758741545724e-06, Gradient Norm: 0.11662453413009644)
+Step... (46375 | Loss: 0.001086848322302103, Learning Rate: 7.325255864998326e-06, Gradient Norm: 0.06781523674726486)
+Step... (46400 | Loss: 0.002841377630829811, Learning Rate: 7.274746621988015e-06, Gradient Norm: 0.13784265518188477)
+Step... (46425 | Loss: 0.0030452024657279253, Learning Rate: 7.224243745440617e-06, Gradient Norm: 0.27625468373298645)
+Step... (46450 | Loss: 0.010637513361871243, Learning Rate: 7.173734502430307e-06, Gradient Norm: 0.2823644280433655)
+Step... (46475 | Loss: 0.004557167179882526, Learning Rate: 7.123231625882909e-06, Gradient Norm: 0.536411702632904)
+Step... (46500 | Loss: 0.00306858797557652, Learning Rate: 7.072728749335511e-06, Gradient Norm: 0.09498882293701172)
+Step... (46525 | Loss: 0.0018582177581265569, Learning Rate: 7.0222195063252e-06, Gradient Norm: 0.15445643663406372)
+Step... (46550 | Loss: 0.0055601089261472225, Learning Rate: 6.971716629777802e-06, Gradient Norm: 0.2561977505683899)
+Step... (46575 | Loss: 0.0004325220361351967, Learning Rate: 6.921213753230404e-06, Gradient Norm: 0.02027466520667076)
+Step... (46600 | Loss: 0.004028357565402985, Learning Rate: 6.870704510220094e-06, Gradient Norm: 0.14455808699131012)
+Step... (46625 | Loss: 0.0032141306437551975, Learning Rate: 6.820201633672696e-06, Gradient Norm: 0.19990961253643036)
+Step... (46650 | Loss: 0.003672192106023431, Learning Rate: 6.7696987571252976e-06, Gradient Norm: 0.0882977843284607)
+Step... (46675 | Loss: 0.002671650843694806, Learning Rate: 6.719189514114987e-06, Gradient Norm: 0.29029080271720886)
+Step... (46700 | Loss: 0.0021792224142700434, Learning Rate: 6.668686637567589e-06, Gradient Norm: 0.0928763747215271)
+Step... (46725 | Loss: 0.0018916006665676832, Learning Rate: 6.618183761020191e-06, Gradient Norm: 0.18946678936481476)
+Step... (46750 | Loss: 0.002835277933627367, Learning Rate: 6.5676745180098806e-06, Gradient Norm: 0.1169157475233078)
+Step... (46775 | Loss: 0.0008665637578815222, Learning Rate: 6.5171716414624825e-06, Gradient Norm: 0.06307638436555862)
+Step... (46800 | Loss: 0.005216824356466532, Learning Rate: 6.4666687649150845e-06, Gradient Norm: 0.17191486060619354)
+Step... (46825 | Loss: 0.0019698829855769873, Learning Rate: 6.416159521904774e-06, Gradient Norm: 0.12737050652503967)
+Step... (46850 | Loss: 0.002469806233420968, Learning Rate: 6.365656645357376e-06, Gradient Norm: 0.09056347608566284)
+Step... (46875 | Loss: 0.0011732580605894327, Learning Rate: 6.315153768809978e-06, Gradient Norm: 0.07834524661302567)
+Step... (46900 | Loss: 0.0027421254198998213, Learning Rate: 6.2646445257996675e-06, Gradient Norm: 0.12933117151260376)
+Step... (46925 | Loss: 0.002762808231636882, Learning Rate: 6.214141649252269e-06, Gradient Norm: 0.2802877128124237)
+Step... (46950 | Loss: 0.003183376742526889, Learning Rate: 6.163638772704871e-06, Gradient Norm: 0.09256728738546371)
+Step... (46975 | Loss: 0.0011139512062072754, Learning Rate: 6.113129529694561e-06, Gradient Norm: 0.11546455323696136)
+Step... (47000 | Loss: 0.003426450304687023, Learning Rate: 6.062626653147163e-06, Gradient Norm: 0.13084343075752258)
+Step... (47025 | Loss: 0.0024451303761452436, Learning Rate: 6.012123776599765e-06, Gradient Norm: 0.28388726711273193)
+Step... (47050 | Loss: 0.0022857924923300743, Learning Rate: 5.961614533589454e-06, Gradient Norm: 0.14490585029125214)
+Step... (47075 | Loss: 0.0015037512639537454, Learning Rate: 5.911111657042056e-06, Gradient Norm: 0.20273494720458984)
+Step... (47100 | Loss: 0.004381109494715929, Learning Rate: 5.860608780494658e-06, Gradient Norm: 0.1728397160768509)
+Step... (47125 | Loss: 0.0018952905666083097, Learning Rate: 5.810099537484348e-06, Gradient Norm: 0.1924336552619934)
+Step... (47150 | Loss: 0.003915037028491497, Learning Rate: 5.75959666093695e-06, Gradient Norm: 0.13231444358825684)
+Step... (47175 | Loss: 0.0013379824813455343, Learning Rate: 5.709093784389552e-06, Gradient Norm: 0.09984851628541946)
+Step... (47200 | Loss: 0.0042300745844841, Learning Rate: 5.658584541379241e-06, Gradient Norm: 0.15374016761779785)
+Step... (47225 | Loss: 0.0012262854725122452, Learning Rate: 5.608081664831843e-06, Gradient Norm: 0.10231379419565201)
+Step... (47250 | Loss: 0.0038886629045009613, Learning Rate: 5.557578788284445e-06, Gradient Norm: 0.15827754139900208)
+Step... (47275 | Loss: 0.0015737712383270264, Learning Rate: 5.507069545274135e-06, Gradient Norm: 0.12450241297483444)
+Step... (47300 | Loss: 0.0057818274945020676, Learning Rate: 5.456566668726737e-06, Gradient Norm: 0.14007054269313812)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Step... (40000/50000 | Eval Loss: 0.9527401328086853 | Eval wer: 0.04187346053453917 | Eval cer: 0.02872188479352137 |):  92% 11/12 [64:22:01<5:51:07, 21067.42s/it]
+Step... (47350 | Loss: 0.003852912224829197, Learning Rate: 5.355554549169028e-06, Gradient Norm: 0.1632377952337265)
+Step... (47375 | Loss: 0.0011032108450308442, Learning Rate: 5.30505167262163e-06, Gradient Norm: 0.07574035227298737)
+Step... (47400 | Loss: 0.0037336035165935755, Learning Rate: 5.254548796074232e-06, Gradient Norm: 0.1369144171476364)
+Step... (47425 | Loss: 0.0005997814005240798, Learning Rate: 5.204039553063922e-06, Gradient Norm: 0.06363093107938766)
+Step... (47450 | Loss: 0.0032057648058980703, Learning Rate: 5.153536676516524e-06, Gradient Norm: 0.14400610327720642)
+Step... (47475 | Loss: 0.006660703103989363, Learning Rate: 5.1030337999691255e-06, Gradient Norm: 0.6412332057952881)
+Step... (47500 | Loss: 0.005310621112585068, Learning Rate: 5.052524556958815e-06, Gradient Norm: 0.1291334629058838)
+Step... (47525 | Loss: 0.0011368023697286844, Learning Rate: 5.002021680411417e-06, Gradient Norm: 0.10793142765760422)
+Step... (47550 | Loss: 0.007397759705781937, Learning Rate: 4.9515128921484575e-06, Gradient Norm: 0.17516382038593292)
+Step... (47575 | Loss: 0.007730554323643446, Learning Rate: 4.9010095608537085e-06, Gradient Norm: 0.5665212273597717)
+Step... (47600 | Loss: 0.010844968259334564, Learning Rate: 4.8505066843063105e-06, Gradient Norm: 0.20311814546585083)
+Step... (47625 | Loss: 0.002866756869480014, Learning Rate: 4.799997896043351e-06, Gradient Norm: 0.19541442394256592)
+Step... (47650 | Loss: 0.004438326694071293, Learning Rate: 4.749494564748602e-06, Gradient Norm: 0.16690881550312042)
+Step... (47675 | Loss: 0.0008846035343594849, Learning Rate: 4.698991688201204e-06, Gradient Norm: 0.08671644330024719)
+Step... (47700 | Loss: 0.003090446349233389, Learning Rate: 4.648482899938244e-06, Gradient Norm: 0.30508506298065186)
+Step... (47725 | Loss: 0.0011106578167527914, Learning Rate: 4.5979795686434954e-06, Gradient Norm: 0.1418560892343521)
+Step... (47750 | Loss: 0.004936729092150927, Learning Rate: 4.547476692096097e-06, Gradient Norm: 0.16554218530654907)
+Step... (47775 | Loss: 0.002189367078244686, Learning Rate: 4.496967903833138e-06, Gradient Norm: 0.7181901335716248)
+Step... (47800 | Loss: 0.005074391607195139, Learning Rate: 4.446464572538389e-06, Gradient Norm: 0.15904028713703156)
+Step... (47825 | Loss: 0.0004910172428935766, Learning Rate: 4.395961695990991e-06, Gradient Norm: 0.026046399027109146)
+Step... (47850 | Loss: 0.0034627090208232403, Learning Rate: 4.34545245298068e-06, Gradient Norm: 0.12519370019435883)
+Step... (47875 | Loss: 0.00038485752884298563, Learning Rate: 4.294949576433282e-06, Gradient Norm: 0.021067677065730095)
+Step... (47900 | Loss: 0.0040473295375704765, Learning Rate: 4.244446699885884e-06, Gradient Norm: 0.14678457379341125)
+Step... (47925 | Loss: 0.0014786692336201668, Learning Rate: 4.193937456875574e-06, Gradient Norm: 0.1168798878788948)
+Step... (47950 | Loss: 0.0030647104140371084, Learning Rate: 4.143434580328176e-06, Gradient Norm: 0.1271444410085678)
+Step... (47975 | Loss: 0.0010495752794668078, Learning Rate: 4.092931703780778e-06, Gradient Norm: 0.09029767662286758)
+Step... (48000 | Loss: 0.0059296865947544575, Learning Rate: 4.042422460770467e-06, Gradient Norm: 0.14342369139194489)
+Step... (48025 | Loss: 0.0016038187313824892, Learning Rate: 3.991919584223069e-06, Gradient Norm: 0.10601940006017685)
+Step... (48050 | Loss: 0.0042427415028214455, Learning Rate: 3.941416707675671e-06, Gradient Norm: 0.14081290364265442)
+Step... (48075 | Loss: 0.0007257165852934122, Learning Rate: 3.890907464665361e-06, Gradient Norm: 0.04569496214389801)
+Step... (48100 | Loss: 0.0033516022376716137, Learning Rate: 3.840404588117963e-06, Gradient Norm: 0.13442592322826385)
+Step... (48125 | Loss: 0.002494408516213298, Learning Rate: 3.7899017115705647e-06, Gradient Norm: 0.1595575213432312)
+Step... (48150 | Loss: 0.006877778097987175, Learning Rate: 3.7393926959339296e-06, Gradient Norm: 0.22835040092468262)
+Step... (48175 | Loss: 0.0006659498903900385, Learning Rate: 3.688889592012856e-06, Gradient Norm: 0.059616561979055405)
+Step... (48200 | Loss: 0.005937668960541487, Learning Rate: 3.638386715465458e-06, Gradient Norm: 0.1822139024734497)
+Step... (48225 | Loss: 0.00040371695649810135, Learning Rate: 3.587877699828823e-06, Gradient Norm: 0.022684333845973015)
+Step... (48250 | Loss: 0.007745903916656971, Learning Rate: 3.5373745959077496e-06, Gradient Norm: 0.24524857103824615)
+Step... (48275 | Loss: 0.0005150886136107147, Learning Rate: 3.4868717193603516e-06, Gradient Norm: 0.06518520414829254)
+Step... (48300 | Loss: 0.004047275520861149, Learning Rate: 3.4363627037237165e-06, Gradient Norm: 0.21056640148162842)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Step... (40000/50000 | Eval Loss: 0.9527401328086853 | Eval wer: 0.04187346053453917 | Eval cer: 0.02872188479352137 |): 100% 12/12 [66:36:10<00:00, 19980.90s/it]
+run_flax_speech_recognition_seq2seq.py:1425: FutureWarning: jax.tree_map is deprecated, and will be removed in a future release. Use jax.tree_util.tree_map instead.
+  params = jax.device_get(jax.tree_map(lambda x: x[0], state.params))
+Configuration saved in /home/sanchitgandhi/flax-wav2vec2-2-bart-large-ls-960h-black-box/config.json
+Model weights saved in /home/sanchitgandhi/flax-wav2vec2-2-bart-large-ls-960h-black-box/flax_model.msgpack
+tokenizer config file saved in ./tokenizer_config.json
+Special tokens file saved in ./special_tokens_map.json
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
 To disable this warning, you can either:
 	- Avoid using `tokenizers` before the fork if possible

wandb/run-20220828_085247-2hx8pk65/files/wandb-summary.json

@@ -1 +1 @@
-{"train/decoder_grad_norm": 0.18293321132659912, "train/decoder_param_norm": 1063.0654296875, "train/encoder_grad_norm": 0.148331880569458, "train/encoder_param_norm": 2323.336669921875, "train/grad_norm": 0.23551413416862488, "layer_grad_norm/": {"decoder": {"model": {"decoder": {"embed_positions": {"embedding": 0.008046639151871204}, "embed_tokens": {"embedding": 0.060666970908641815}, "layernorm_embedding": {"bias": 0.003096886444836855, "scale": 0.0024350089952349663}, "layers": {"FlaxBartDecoderLayers": {"encoder_attn": {"k_proj": {"bias": 5.256703389022732e-06, "kernel": 0.011271456256508827}, "out_proj": {"bias": 0.007880721241235733, "kernel": 0.03872065246105194}, "q_proj": {"bias": 0.0004976371419616044, "kernel": 0.01105893962085247}, "v_proj": {"bias": 0.015226359479129314, "kernel": 0.030586158856749535}}, "encoder_attn_layer_norm": {"bias": 0.01159473042935133, "scale": 0.012393548153340816}, "fc1": {"bias": 0.004439335782080889, "kernel": 0.10113218426704407}, "fc2": {"bias": 0.011387723498046398, "kernel": 0.09879624843597412}, "final_layer_norm": {"bias": 0.026725012809038162, "scale": 0.026244478300213814}, "self_attn": {"k_proj": {"bias": 1.8891130366682773e-06, "kernel": 0.010783703066408634}, "out_proj": {"bias": 0.016153542324900627, "kernel": 0.039619628340005875}, "q_proj": {"bias": 0.0008692306582815945, "kernel": 0.01054653525352478}, "v_proj": {"bias": 0.01716933213174343, "kernel": 0.04919762164354324}}, "self_attn_layer_norm": {"bias": 0.007215098012238741, "scale": 0.008393414318561554}}}}}}, "encoder": {"adapter": {"layers": {"0": {"conv": {"bias": 0.022141050547361374, "kernel": 0.049536462873220444}}, "1": {"conv": {"bias": 0.017486272379755974, "kernel": 0.03386249393224716}}, "2": {"conv": {"bias": 0.020221581682562828, "kernel": 0.05212334543466568}}}}, "encoder": {"layer_norm": {"bias": 0.07752241939306259, "scale": 0.03339512273669243}, "layers": {"FlaxWav2Vec2EncoderLayers": {"attention": {"k_proj": {"bias": 1.787670839803468e-06, "kernel": 0.016130739822983742}, "out_proj": {"bias": 0.001517767203040421, "kernel": 0.0303343553096056}, "q_proj": {"bias": 0.002082030288875103, "kernel": 0.01582256518304348}, "v_proj": {"bias": 0.00666964752599597, "kernel": 0.027823949232697487}}, "feed_forward": {"intermediate_dense": {"bias": 0.002947209170088172, "kernel": 0.03786110132932663}, "output_dense": {"bias": 0.0014471356989815831, "kernel": 0.03480111435055733}}, "final_layer_norm": {"bias": 0.014765393920242786, "scale": 0.01634989120066166}, "layer_norm": {"bias": 0.03281298279762268, "scale": 0.024696579203009605}}}, "pos_conv_embed": {"conv": {"bias": 0.0003536914009600878, "weight_g": 0.0017076540971174836, "weight_v": 0.0059150331653654575}}}, "feature_extractor": {"conv_layers": {"0": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "1": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "2": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "3": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "4": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "5": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "6": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}}}, "feature_projection": {"layer_norm": {"bias": 0.00199363986030221, "scale": 0.0019663891289383173}, "projection": {"bias": 0.000548368610907346, "kernel": 0.015020000748336315}}, "masked_spec_embed": 0.0}}, "layer_param_norm/": {"decoder": {"model": {"decoder": {"embed_positions": {"embedding": 58.655296325683594}, "embed_tokens": {"embedding": 628.4320678710938}, "layernorm_embedding": {"bias": 2.423579692840576, "scale": 13.864307403564453}, "layers": {"FlaxBartDecoderLayers": {"encoder_attn": {"k_proj": {"bias": 47.95212173461914, "kernel": 331.52020263671875}, "out_proj": {"bias": 6.132379531860352, "kernel": 227.88941955566406}, "q_proj": {"bias": 20.880966186523438, "kernel": 338.4452819824219}, "v_proj": {"bias": 3.570829391479492, "kernel": 232.0838623046875}}, "encoder_attn_layer_norm": {"bias": 11.024534225463867, "scale": 57.68368148803711}, "fc1": {"bias": 26.054283142089844, "kernel": 346.79296875}, "fc2": {"bias": 7.841187953948975, "kernel": 248.94561767578125}, "final_layer_norm": {"bias": 3.914222002029419, "scale": 63.57343673706055}, "self_attn": {"k_proj": {"bias": 59.552547454833984, "kernel": 280.0233459472656}, "out_proj": {"bias": 3.7681872844696045, "kernel": 132.86544799804688}, "q_proj": {"bias": 32.200714111328125, "kernel": 283.1105651855469}, "v_proj": {"bias": 2.561887741088867, "kernel": 141.2297821044922}}, "self_attn_layer_norm": {"bias": 8.89633846282959, "scale": 84.71704864501953}}}}}}, "encoder": {"adapter": {"layers": {"0": {"conv": {"bias": 1.1991273164749146, "kernel": 62.47041320800781}}, "1": {"conv": {"bias": 1.3806315660476685, "kernel": 60.54581069946289}}, "2": {"conv": {"bias": 1.5864932537078857, "kernel": 60.361602783203125}}}}, "encoder": {"layer_norm": {"bias": 0.2923617959022522, "scale": 4.219600677490234}, "layers": {"FlaxWav2Vec2EncoderLayers": {"attention": {"k_proj": {"bias": 19.385311126708984, "kernel": 553.5303955078125}, "out_proj": {"bias": 16.84536361694336, "kernel": 705.300537109375}, "q_proj": {"bias": 40.93048858642578, "kernel": 546.0474243164062}, "v_proj": {"bias": 15.559945106506348, "kernel": 696.8070068359375}}, "feed_forward": {"intermediate_dense": {"bias": 24.44249725341797, "kernel": 1377.5845947265625}, "output_dense": {"bias": 20.795345306396484, "kernel": 1303.997314453125}}, "final_layer_norm": {"bias": 32.52366256713867, "scale": 142.01145935058594}, "layer_norm": {"bias": 7.272017478942871, "scale": 45.770084381103516}}}, "pos_conv_embed": {"conv": {"bias": 15.211658477783203, "weight_g": 21.0489501953125, "weight_v": 214.11346435546875}}}, "feature_extractor": {"conv_layers": {"0": {"conv": {"bias": 0.5982058644294739, "kernel": 8.08896541595459}, "layer_norm": {"bias": 10.069783210754395, "scale": 10.451257705688477}}, "1": {"conv": {"bias": 4.74075174331665, "kernel": 90.8435287475586}, "layer_norm": {"bias": 6.922820091247559, "scale": 19.5467586517334}}, "2": {"conv": {"bias": 6.7732415199279785, "kernel": 146.13897705078125}, "layer_norm": {"bias": 9.044225692749023, "scale": 19.424888610839844}}, "3": {"conv": {"bias": 5.224758148193359, "kernel": 159.10508728027344}, "layer_norm": {"bias": 8.319666862487793, "scale": 17.64743423461914}}, "4": {"conv": {"bias": 4.434978008270264, "kernel": 157.35813903808594}, "layer_norm": {"bias": 9.193974494934082, "scale": 15.562357902526855}}, "5": {"conv": {"bias": 5.297643661499023, "kernel": 131.1835174560547}, "layer_norm": {"bias": 10.735219955444336, "scale": 13.812533378601074}}, "6": {"conv": {"bias": 5.615579128265381, "kernel": 136.41822814941406}, "layer_norm": {"bias": 12.515308380126953, "scale": 11.152680397033691}}}}, "feature_projection": {"layer_norm": {"bias": 9.234997749328613, "scale": 27.593135833740234}, "projection": {"bias": 4.323590278625488, "kernel": 88.11954498291016}}, "masked_spec_embed": 26.247730255126953}}, "train/learning_rate": 2.0254545233910903e-05, "train/loss": 0.008378183469176292, "train/param_norm": 2554.995361328125, "_timestamp": 1661871981, "_runtime": 195214, "_step": 39975, "eval/loss": 0.9876799583435059, "eval/wer": 0.04759016212639241, "eval/cer": 0.035079873533571844, "eval/step_10k": {"_type": "table-file", "sha256": "8b44e8a00a036a18ffdf81b4d076c8bf849ea6649001c69e94fa439b14f110ee", "size": 26434, "artifact_path": "wandb-client-artifact://18m0dj4hts3yiat04x5pvmncavkjapd5wb8bznb37vw8c0lqna3m2yjd1wtdrfstuoo7ejt2sphvjo0zuw1e5ne5d3qbkd7c1fylclfggig6us5tsmsj2uum5pchx48n:latest/eval/step_10k.table.json", "_latest_artifact_path": "wandb-client-artifact://18m0dj4hts3yiat04x5pvmncavkjapd5wb8bznb37vw8c0lqna3m2yjd1wtdrfstuoo7ejt2sphvjo0zuw1e5ne5d3qbkd7c1fylclfggig6us5tsmsj2uum5pchx48n:latest/eval/step_10k.table.json", "path": "media/table/eval/step_10k_10000_8b44e8a00a036a18ffdf.table.json", "ncols": 7, "nrows": 50}, "eval/step_20k": {"_type": "table-file", "sha256": "a0a50c5d8793ca99e4646f70c3624f8742c5285825bc1c59ab4083ac4de9d6e3", "size": 26657, "artifact_path": "wandb-client-artifact://13ri9hnxp93kf7dsdol2hs1j0v7bpkwwvujpi27awdck0fjm6vfog0dun9k9toif5xrt3cijlotddakikiw0bnbo3go679b4d2spq9c0w865vq0k9auiszkkbvev62fc:latest/eval/step_20k.table.json", "_latest_artifact_path": "wandb-client-artifact://13ri9hnxp93kf7dsdol2hs1j0v7bpkwwvujpi27awdck0fjm6vfog0dun9k9toif5xrt3cijlotddakikiw0bnbo3go679b4d2spq9c0w865vq0k9auiszkkbvev62fc:latest/eval/step_20k.table.json", "path": "media/table/eval/step_20k_20000_a0a50c5d8793ca99e464.table.json", "ncols": 7, "nrows": 50}, "eval/step_30k": {"_type": "table-file", "sha256": "509ad8614e16ae2800f100e82fd40356d3559e0425cf50e8e19c3155dc6c604e", "size": 26610, "artifact_path": "wandb-client-artifact://1v7mr89p2ldfoyyvrj6x6g0s8qpvgnioclm4puk5i0wphipeujkyna0xf7y63nt4a5qivkdzytt4lzhwuraoe53a5683ln1yzntyw9kh7vhcxz7oci70rpw62o1tz6ho:latest/eval/step_30k.table.json", "_latest_artifact_path": "wandb-client-artifact://1v7mr89p2ldfoyyvrj6x6g0s8qpvgnioclm4puk5i0wphipeujkyna0xf7y63nt4a5qivkdzytt4lzhwuraoe53a5683ln1yzntyw9kh7vhcxz7oci70rpw62o1tz6ho:latest/eval/step_30k.table.json", "path": "media/table/eval/step_30k_30000_509ad8614e16ae2800f1.table.json", "ncols": 7, "nrows": 50}}
+{"train/decoder_grad_norm": 0.07715553045272827, "train/decoder_param_norm": 1063.1578369140625, "train/encoder_grad_norm": 0.07033974677324295, "train/encoder_param_norm": 2323.486572265625, "train/grad_norm": 0.1044062077999115, "layer_grad_norm/": {"decoder": {"model": {"decoder": {"embed_positions": {"embedding": 0.005401859525591135}, "embed_tokens": {"embedding": 0.03052273392677307}, "layernorm_embedding": {"bias": 0.0020871469751000404, "scale": 0.0012841655407100916}, "layers": {"FlaxBartDecoderLayers": {"encoder_attn": {"k_proj": {"bias": 1.8833308104149182e-06, "kernel": 0.004420427139848471}, "out_proj": {"bias": 0.00347402086481452, "kernel": 0.015828590840101242}, "q_proj": {"bias": 0.00020214646065142006, "kernel": 0.004523565527051687}, "v_proj": {"bias": 0.006682044826447964, "kernel": 0.012984688393771648}}, "encoder_attn_layer_norm": {"bias": 0.004988634027540684, "scale": 0.005556626245379448}, "fc1": {"bias": 0.0017050342867150903, "kernel": 0.04050837457180023}, "fc2": {"bias": 0.005049724597483873, "kernel": 0.03927548602223396}, "final_layer_norm": {"bias": 0.013566207140684128, "scale": 0.008338828571140766}, "self_attn": {"k_proj": {"bias": 5.705806529476831e-07, "kernel": 0.003607015358284116}, "out_proj": {"bias": 0.008329739794135094, "kernel": 0.015504402108490467}, "q_proj": {"bias": 0.0002979960117954761, "kernel": 0.003639827948063612}, "v_proj": {"bias": 0.00870799645781517, "kernel": 0.022596830502152443}}, "self_attn_layer_norm": {"bias": 0.003212399547919631, "scale": 0.00401513883844018}}}}}}, "encoder": {"adapter": {"layers": {"0": {"conv": {"bias": 0.009786142967641354, "kernel": 0.02071578986942768}}, "1": {"conv": {"bias": 0.007721441797912121, "kernel": 0.01437926385551691}}, "2": {"conv": {"bias": 0.009065583348274231, "kernel": 0.021983565762639046}}}}, "encoder": {"layer_norm": {"bias": 0.03353462368249893, "scale": 0.01526513509452343}, "layers": {"FlaxWav2Vec2EncoderLayers": {"attention": {"k_proj": {"bias": 1.1060902807003004e-06, "kernel": 0.00823200587183237}, "out_proj": {"bias": 0.0008450962486676872, "kernel": 0.01538914442062378}, "q_proj": {"bias": 0.0011472117621451616, "kernel": 0.008041738532483578}, "v_proj": {"bias": 0.0036684211809188128, "kernel": 0.014704424887895584}}, "feed_forward": {"intermediate_dense": {"bias": 0.002029425697401166, "kernel": 0.019275842234492302}, "output_dense": {"bias": 0.0007828964735381305, "kernel": 0.01704106293618679}}, "final_layer_norm": {"bias": 0.010386798530817032, "scale": 0.011868822388350964}, "layer_norm": {"bias": 0.01724540814757347, "scale": 0.015423807315528393}}}, "pos_conv_embed": {"conv": {"bias": 0.0002918594400398433, "weight_g": 0.0027034797240048647, "weight_v": 0.005031736101955175}}}, "feature_extractor": {"conv_layers": {"0": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "1": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "2": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "3": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "4": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "5": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "6": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}}}, "feature_projection": {"layer_norm": {"bias": 0.0014273464912548661, "scale": 0.001675860257819295}, "projection": {"bias": 0.0004162571858614683, "kernel": 0.01199527457356453}}, "masked_spec_embed": 0.0}}, "layer_param_norm/": {"decoder": {"model": {"decoder": {"embed_positions": {"embedding": 58.65697479248047}, "embed_tokens": {"embedding": 628.4302978515625}, "layernorm_embedding": {"bias": 2.422621965408325, "scale": 13.860967636108398}, "layers": {"FlaxBartDecoderLayers": {"encoder_attn": {"k_proj": {"bias": 47.94939041137695, "kernel": 331.5506591796875}, "out_proj": {"bias": 6.131547927856445, "kernel": 227.9162139892578}, "q_proj": {"bias": 20.88374137878418, "kernel": 338.4700622558594}, "v_proj": {"bias": 3.56612491607666, "kernel": 232.11032104492188}}, "encoder_attn_layer_norm": {"bias": 11.035151481628418, "scale": 57.715087890625}, "fc1": {"bias": 26.05365562438965, "kernel": 346.8977355957031}, "fc2": {"bias": 7.839730739593506, "kernel": 249.0161590576172}, "final_layer_norm": {"bias": 3.914802074432373, "scale": 63.62209701538086}, "self_attn": {"k_proj": {"bias": 59.55521011352539, "kernel": 280.0371398925781}, "out_proj": {"bias": 3.7659459114074707, "kernel": 132.872802734375}, "q_proj": {"bias": 32.20103073120117, "kernel": 283.1228942871094}, "v_proj": {"bias": 2.560319423675537, "kernel": 141.23605346679688}}, "self_attn_layer_norm": {"bias": 8.895964622497559, "scale": 84.7233657836914}}}}}}, "encoder": {"adapter": {"layers": {"0": {"conv": {"bias": 1.2016971111297607, "kernel": 62.545352935791016}}, "1": {"conv": {"bias": 1.3880459070205688, "kernel": 60.6671257019043}}, "2": {"conv": {"bias": 1.5934075117111206, "kernel": 60.4710807800293}}}}, "encoder": {"layer_norm": {"bias": 0.2923099100589752, "scale": 4.224204063415527}, "layers": {"FlaxWav2Vec2EncoderLayers": {"attention": {"k_proj": {"bias": 19.386531829833984, "kernel": 553.5894165039062}, "out_proj": {"bias": 16.841169357299805, "kernel": 705.33203125}, "q_proj": {"bias": 40.93061828613281, "kernel": 546.1094970703125}, "v_proj": {"bias": 15.554323196411133, "kernel": 696.839111328125}}, "feed_forward": {"intermediate_dense": {"bias": 24.43609619140625, "kernel": 1377.650146484375}, "output_dense": {"bias": 20.79059600830078, "kernel": 1304.09033203125}}, "final_layer_norm": {"bias": 32.522762298583984, "scale": 142.02593994140625}, "layer_norm": {"bias": 7.270603656768799, "scale": 45.793148040771484}}}, "pos_conv_embed": {"conv": {"bias": 15.207948684692383, "weight_g": 21.04806900024414, "weight_v": 214.14303588867188}}}, "feature_extractor": {"conv_layers": {"0": {"conv": {"bias": 0.5982058644294739, "kernel": 8.08896541595459}, "layer_norm": {"bias": 10.069783210754395, "scale": 10.451257705688477}}, "1": {"conv": {"bias": 4.74075174331665, "kernel": 90.8435287475586}, "layer_norm": {"bias": 6.922820091247559, "scale": 19.5467586517334}}, "2": {"conv": {"bias": 6.7732415199279785, "kernel": 146.13897705078125}, "layer_norm": {"bias": 9.044225692749023, "scale": 19.424888610839844}}, "3": {"conv": {"bias": 5.224758148193359, "kernel": 159.10508728027344}, "layer_norm": {"bias": 8.319666862487793, "scale": 17.64743423461914}}, "4": {"conv": {"bias": 4.434978008270264, "kernel": 157.35813903808594}, "layer_norm": {"bias": 9.193974494934082, "scale": 15.562357902526855}}, "5": {"conv": {"bias": 5.297643661499023, "kernel": 131.1835174560547}, "layer_norm": {"bias": 10.735219955444336, "scale": 13.812533378601074}}, "6": {"conv": {"bias": 5.615579128265381, "kernel": 136.41822814941406}, "layer_norm": {"bias": 12.515308380126953, "scale": 11.152680397033691}}}}, "feature_projection": {"layer_norm": {"bias": 9.220093727111816, "scale": 27.58247947692871}, "projection": {"bias": 4.322972774505615, "kernel": 88.09733581542969}}, "masked_spec_embed": 26.247730255126953}}, "train/learning_rate": 5.252361034990827e-08, "train/loss": 0.0029563589487224817, "train/param_norm": 2555.17041015625, "_timestamp": 1661920077, "_runtime": 243310, "_step": 49975, "eval/loss": 0.9527401328086853, "eval/wer": 0.04187346053453917, "eval/cer": 0.02872188479352137, "eval/step_10k": {"_type": "table-file", "sha256": "8b44e8a00a036a18ffdf81b4d076c8bf849ea6649001c69e94fa439b14f110ee", "size": 26434, "artifact_path": "wandb-client-artifact://18m0dj4hts3yiat04x5pvmncavkjapd5wb8bznb37vw8c0lqna3m2yjd1wtdrfstuoo7ejt2sphvjo0zuw1e5ne5d3qbkd7c1fylclfggig6us5tsmsj2uum5pchx48n:latest/eval/step_10k.table.json", "_latest_artifact_path": "wandb-client-artifact://18m0dj4hts3yiat04x5pvmncavkjapd5wb8bznb37vw8c0lqna3m2yjd1wtdrfstuoo7ejt2sphvjo0zuw1e5ne5d3qbkd7c1fylclfggig6us5tsmsj2uum5pchx48n:latest/eval/step_10k.table.json", "path": "media/table/eval/step_10k_10000_8b44e8a00a036a18ffdf.table.json", "ncols": 7, "nrows": 50}, "eval/step_20k": {"_type": "table-file", "sha256": "a0a50c5d8793ca99e4646f70c3624f8742c5285825bc1c59ab4083ac4de9d6e3", "size": 26657, "artifact_path": "wandb-client-artifact://13ri9hnxp93kf7dsdol2hs1j0v7bpkwwvujpi27awdck0fjm6vfog0dun9k9toif5xrt3cijlotddakikiw0bnbo3go679b4d2spq9c0w865vq0k9auiszkkbvev62fc:latest/eval/step_20k.table.json", "_latest_artifact_path": "wandb-client-artifact://13ri9hnxp93kf7dsdol2hs1j0v7bpkwwvujpi27awdck0fjm6vfog0dun9k9toif5xrt3cijlotddakikiw0bnbo3go679b4d2spq9c0w865vq0k9auiszkkbvev62fc:latest/eval/step_20k.table.json", "path": "media/table/eval/step_20k_20000_a0a50c5d8793ca99e464.table.json", "ncols": 7, "nrows": 50}, "eval/step_30k": {"_type": "table-file", "sha256": "509ad8614e16ae2800f100e82fd40356d3559e0425cf50e8e19c3155dc6c604e", "size": 26610, "artifact_path": "wandb-client-artifact://1v7mr89p2ldfoyyvrj6x6g0s8qpvgnioclm4puk5i0wphipeujkyna0xf7y63nt4a5qivkdzytt4lzhwuraoe53a5683ln1yzntyw9kh7vhcxz7oci70rpw62o1tz6ho:latest/eval/step_30k.table.json", "_latest_artifact_path": "wandb-client-artifact://1v7mr89p2ldfoyyvrj6x6g0s8qpvgnioclm4puk5i0wphipeujkyna0xf7y63nt4a5qivkdzytt4lzhwuraoe53a5683ln1yzntyw9kh7vhcxz7oci70rpw62o1tz6ho:latest/eval/step_30k.table.json", "path": "media/table/eval/step_30k_30000_509ad8614e16ae2800f1.table.json", "ncols": 7, "nrows": 50}, "eval/step_40k": {"_type": "table-file", "sha256": "1c187b6d789cca3b4ec0da0aa76f40085364204904c4e4e7f9a7ae021659a9ab", "size": 26557, "artifact_path": "wandb-client-artifact://tbhmdixx2c1ta253dre6w8pjaxb8fbzac6u2cugh4vsvvgiykum9l788fn7omfbr8f3ir5gq1fo2cb7bwlha8musehb13eb9eb7hppcmvnyc3gg5i8eprg7w7z0jqrgu:latest/eval/step_40k.table.json", "_latest_artifact_path": "wandb-client-artifact://tbhmdixx2c1ta253dre6w8pjaxb8fbzac6u2cugh4vsvvgiykum9l788fn7omfbr8f3ir5gq1fo2cb7bwlha8musehb13eb9eb7hppcmvnyc3gg5i8eprg7w7z0jqrgu:latest/eval/step_40k.table.json", "path": "media/table/eval/step_40k_40000_1c187b6d789cca3b4ec0.table.json", "ncols": 7, "nrows": 50}}

wandb/run-20220828_085247-2hx8pk65/logs/debug-internal.log

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d093f3f44fdf5811d85d2d82d6e9265c3c9bd01c68e29987453a71c23bf7fee1
-size 13709864
+oid sha256:1f80a39e659e1401a6be03c1a16ddc1e34a04ddabd64c6c956b346c7ad49ffd6
+size 17019871

wandb/run-20220828_085247-2hx8pk65/run-2hx8pk65.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:704bf667df7d300df731d8d0bb3301cfbf4b05488ba632c3bc0bcf3bd69bea8c
-size 17153865
+oid sha256:4091a31807c620f1680b2cbc9c2340da78bef2667936aaabb64b632931b7b707
+size 21371807