2hx8pk65: saving weights and logs of step 20k

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d4bbb8026d3a4c9acb651189cbf65ab582eb2284bbcae68d0c6512395b962329
+oid sha256:1b20c6ce9070a647fc4b56ff847349b2a6ad959d336f8591e71fc135e07d67a9
 size 2353616717

wandb/run-20220828_085247-2hx8pk65/files/media/table/eval/step_20k_20000_a0a50c5d8793ca99e464.table.json

@@ -0,0 +1 @@
+{"columns": ["id", "label_str", "beam_1", "beam_2", "beam_3", "beam_4", "beam_5"], "data": [["2277-149896-0000", "he was in a fevered state of mind owing to the blight his wife's action threatened to cast upon his entire future", "he was in a fevered state of mind owing to the blight his wife's action threatened to cast upon his entire future", "he was in a fevered state of mind owing to the blight his wife's action threatened to cast upon his entire future", "he was in a feverred state of mind owing to the blight his wife's action threatened to cast upon his entire future", "he was in a fevered state of mind owing to the blight his wife's action threatened to cast up on his entire future", "he was in a fever'd state of mind owing to the blight his wife's action threatened to cast upon his entire future"], ["2277-149896-0001", "he would have to pay her the money which she would now regularly demand or there would be trouble it did not matter what he did", "he would have to pay her the money which she would now regularly demand or there would be trouble it did not matter what he did", "he would have to pay her the money which she would now regularly demand or there would be trouble it did not matter what he did", "he would have to pay her the money which she could now regularly demand or there would be trouble it did not matter what he did", "he would have to pay her the money which she would now regularly demand or there would be trouble it did not matter what he did a", "he would have to pay her the money which he would now regularly demand or there would be trouble it did not matter what he did"], ["2277-149896-0002", "hurstwood walked the floor mentally arranging the chief points of his situation", "hurstwood walked the floor mentally arranging the chief points of his situation", "hurstwood walked to the floor mentally arranging the chief points of his situation", "hurstwood walked the floor mentally arranging the chief points in his situation", "hurstwood walked the floor mentally arranging the chief points of this situation", "hurst wood walked the floor mentally arranging the chief points of his situation"], ["2277-149896-0003", "he also thought of his managerial position", "he also thought of his managerial position", "he also thought of this managerial position", "he also thought in his managerial position", "he also thought of his managerial position and", "he also thought of his administrative position"], ["2277-149896-0004", "how would the papers talk about it", "how would the papers talk about it", "how would the papers talk about it and", "how would the papers talk about it i", "how would the papers talk about it a", "how would the papers talk about it the"], ["2277-149896-0005", "many little wrinkles gathered between his eyes as he contemplated this and his brow moistened", "many little wrinkles gathered between his eyes as he contemplated this and his brow moistened", "many little wrinkles gathered between his eyes as he contemplated this his brow moistened", "many little wrinkles gathered between his eyes as he contemplated this and his brow moistened", "many little wrinkles gathered between his eyes as he contemplated this this and his brow moistened", "many little wrinkles gathered between his eyes as he considered this and his brow moistened"], ["2277-149896-0006", "he could arrange that satisfactorily for carrie would be glad to wait if necessary", "he could arrange that satisfactorily for carrie would be glad to wait if necessary", "he could arrange that satisfactorily for carry would be glad to wait if necessary", "he could arrange it satisfactorily for carrie would be glad to wait if necessary", "he could arrange the satisfactorily for carrie would be glad to wait if necessary", "he could arrange this satisfactorily for carrie would be glad to wait if necessary"], ["2277-149896-0007", "he would see how things turned out to morrow and then he would talk to her they were going to meet as usual", "he would see how things turned out to morrow and then he would talk to her they were going to meet as usual", "he would see how things turned out tomorrow and then he would talk to her they were going to meet as usual", "he would see how things turned out tonight and then he would talk to her they were going to meet as usual", "he would see how things turned out to morrow then he would talk to her they were going to meet as usual", "he would see how things turned out today and then he would talk to her they were going to meet as usual"], ["2277-149896-0008", "for some reason he felt as if something might come that way and was relieved when all the envelopes had been scanned and nothing suspicious noticed", "for some reason he felt as if something might come that way and was relieved when all the envelopes had been scanned and nothing suspicious noticed", "for some reason he felt as if something might come that way and was relieved when all the envelopes had been scanned and nothing suspicious notice", "for some reason he felt as if something might come that way and was relieved when all the envelopes had been screened and nothing suspicious noticed", "for some reason he felt as if something might come this way and was relieved when all the envelopes had been scanned and nothing suspicious noticed", "for some reason he felt as if something might come that way and was relieved when all of the envelopes had been scanned and nothing suspicious noticed"], ["2277-149896-0009", "while the danger had not lessened it had not as yet materialised and with him no news was good news", "while the danger had not lessened it had not as yet materialized and with him no news was good news", "while the danger had not lessened it had not as yet materialised and with him no news was good news", "while the danger had not lessened it had not as yet materialized and with him no night was good news", "while the danger had not lessened it had not as yet materialized and with him no news was good night", "while the danger had not lessened it had not as yet materialized and with him no noise was good news"], ["2277-149896-0010", "so little did he consider drouet that it never once occurred to him to worry about his finding out", "so little did he consider drue that it never once occurred to him to worry about his finding out", "so little did he consider drouet that it never once occurred to him to worry about his finding out", "so little did he consider druda that it never once occurred to him to worry about his finding out", "so little did he consider drude that it never once occurred to him to worry about his finding out", "so little did he consider drua that it never once occurred to him to worry about his finding out"], ["2277-149896-0011", "he grew restless as he ruminated and then decided that perhaps it was nothing", "he grew restless as he ruminated and then decided that perhaps it was nothing", "he grew restless as he ruminated and then decided that perhaps it was nothing", "he grew restless as he ruminated then decided that perhaps it was nothing", "he grew restless as he ruminated and then decided that perhaps it was nothing i", "he grew restless as he ruminated and then decided that perhaps it was nothing and"], ["2277-149896-0012", "she had not been able to get away this morning", "she had not been able to get away this morning", "she had not been able to get away this morning and", "she had not been able to get away this morning a", "she had not been able to get away this morning i", "she had not been able to get away this morning the"], ["2277-149896-0013", "he would get one to day it would probably be on his desk when he got back he would look for it at once", "he would get one to day it would probably be on his desk when he got back he would look for it at once", "he would get one today it would probably be on his desk when he got back he would look for it at once", "he would get one to day it would probably be on his deck when he got back he would look for it at once", "he would get one to day it would probably be in his desk when he got back he would look for it at once", "he would get one tomorrow it would probably be on his desk when he got back he would look for it at once"], ["2277-149896-0014", "after a time he gave up waiting and drearily headed for the madison car", "after a time he gave up waiting and drearily headed for the madison car", "after a time he gave up waiting and drearily headed for the maddison car", "after a time he gave up waiting and drearily headed for the madison cart", "after a time he gave up waiting and driarily headed for the madison car", "after a time he gave up waiting and drearily headed for the madison park"], ["2277-149896-0015", "he went in and examined his letters but there was nothing from carrie", "he went in and examined his letters but there was nothing from carrie", "he went in and examined his letters but there was nothing from perry", "he went in and examined his letters but there was nothing from kerry", "he went in and examined his letters but there was nothing from carry", "he went in and examined his letters but there was nothing from carey"], ["2277-149896-0016", "fortunately there was nothing from his wife either", "fortunately there was nothing from his wife either", "fortunately there was nothing from his wife either", "fortunately there was nothing from his wife either a", "fortunately there was nothing from this wife either", "fortunately there was nothing from his wife either and"], ["2277-149896-0017", "at one thirty he went to rector's for lunch and when he returned a messenger was waiting for him", "at one thirty he went to rector's for lunch and when he returned a messenger was waiting for him", "at one thirty he went to rector's for lunch and when he returned a messenger was waiting for him", "at one hundred he went to rector's for lunch and when he returned a messenger was waiting for him", "at one thirty he went to rectors for lunch and when he returned a messenger was waiting for him", "at one thirty he went to regters for lunch and when he returned a messenger was waiting for him"], ["2277-149896-0018", "his first impulse was to write but four words in reply go to the devil", "his first impulse was to write but four words in reply go to the devil", "his first impulse was to write but four words in reply go to the devil and", "his first impulse was to write but four words in reply go to the devil i", "his first impulse was to write but four words in reply go to a devil", "his first impulse was to write but four words in reply go to the devil oh"], ["2277-149896-0019", "but he compromised by telling the boy that there would be no reply", "but he compromised by telling the boy that there would be no reply", "but he compromised by telling the boy there would be no reply", "but he compromised by telling the boy that there would be no reply i", "but he comprised by telling the boy that there would be no reply", "but he compromised by telling the boy that there would be no reply and"], ["2277-149896-0020", "then he sat down in his chair and gazed without seeing contemplating the result of his work", "then he sat down in his chair and gazed without seeing contemplating the result of his work", "then he sate down in his chair and gazed without seeing contemplating the result of his work", "then he sat down in his chair and gazed without seeing contemplating the result of this work", "then he sat down in his chair and gazed without seeing contemplating the result of his works", "then he sat down in his chair and gazed without seeing contemplated the result of his work"], ["2277-149896-0021", "what would she do about that the confounded wretch", "what would she do about that the confounded wretch", "what would she do about that that confounded wretch", "what would she do about that the confounded wretch and", "what could she do about that the confounded wretch", "what would she do about that the confounded wretch oh"], ["2277-149896-0022", "later however his old discretion asserted itself", "later however his old discretion asserted itself", "later however his old discretion asserted itself and", "later however his older discretion asserted itself", "later however this old discretion asserted itself", "later however his old discretion asserted itself a"], ["2277-149896-0023", "something had to be done a climax was near and she would not sit idle", "something had to be done a climax was near and she would not sit idle", "something had to be done the climax was near and she would not sit idle", "something had to be done a climax was near and she would not sit idle and", "something had to be done a climax was near she would not sit idle", "some thing had to be done a climax was near and she would not sit idle"], ["2277-149896-0024", "he knew her well enough to know that when she had decided upon a plan she would follow it up", "he knew her well enough to know that when she had decided upon a plan she would follow it up", "he knew her well enough to know that when she decided upon a plan she would follow it up", "he knew her well enough to know that when she had decided upon a plan she would follow it up and", "he knew her well enough to know that when she had decided on a plan she would follow it up", "he knew her well enough to know that when she had decided upon a plan she would follow it up i"], ["2277-149896-0025", "he arose from his chair and went and looked out into the street", "he arose from his chair and went and looked out into the street", "he rose from his chair and went and looked out into the street", "he arose from his chair and went and looked out into the street and", "he arose from his chair and went and looked out into the street a", "he arose from his chair went and looked out into the street"], ["2277-149896-0026", "the long drizzle had begun pedestrians had turned up collars and trousers at the bottom", "the long drizzle had begun pedestrians had turned up collars and trousers at the bottom", "the long drizzle had begun pedestrians had turned up collars and trowsers at the bottom", "the long drizzle had begun petersians had turned up collars and trousers at the bottom", "the long drizzle had begun pedestrians had turned up collars and trousers at the bottom", "the long drizzle had begun pedestrians had turned up collars or trousers at the bottom"], ["2277-149896-0027", "hurstwood almost exclaimed out loud at the insistency of this thing", "hurstwood almost exclaimed out loud at the insistency of this thing", "hurstwood almost exclaimed out loud at the insistence of this thing", "hurstwood almost exclaimed out loud at the insistency of this thing", "hurstwood almost exclaimed out loud at the insstancy of this thing", "hurstwood almost exclaimed out aloud at the insistency of this thing"], ["2277-149896-0028", "he put on his hat and looked around for his umbrella", "he put on his hat and looked around for his umbrella", "he put on his hat and looked round for his umbrella", "he put on his hat and looked around for his umbrella a", "he put on his hat and looked around for his umbrella and", "he put on his hat and looked about for his umbrella"], ["2277-149896-0029", "he would have some arrangement of this thing", "he would have some arrangement of this thing", "he would have some arrangement of the thing", "he would have some arrangements of this thing", "he would have some arrangement of this thing and", "he would have some arrangement of his thing"], ["2277-149896-0030", "he began to wish that he had compromised in some way or other that he had sent the money perhaps he could do it up here", "he began to wish that he had compromised in some way or other that he had sent the money perhaps he could do it up here", "he began to wish that he had compromised some way or other that he had sent the money perhaps he could do it up here", "he began to wish he had compromised in some way or other that he had sent the money perhaps he could do it up here", "he began to wish that he had compromised in some way or other that he had sent the money perhaps he would do it up here", "he began to wish that he had compromised in some way or another that he had sent the money perhaps he could do it up here"], ["2277-149896-0031", "he would go in and see anyhow he would have no row", "he would go in and see anyhow he would have no row", "he would go in and see anyhow he would have no rue", "he would go in and see anyhow he would have no raoul", "he would go in and see anyhow he would have no row he would have no row", "he would go in and see anything he would have no row"], ["2277-149896-0032", "by the time he reached his own street he was keenly alive to the difficulties of his situation and wished over and over that some solution would offer itself that he could see his way out", "by the time he reached his own street he was keenly alive to the difficulties of his situation and wished over and over that some solution would offer itself that he could see his way out", "by the time he reached his own street he was keenly alive to the difficulties of this situation and wished over and over that some solution would offer itself that he could see his way out", "by the time he reached his own street he was keenly alive to the difficulties of his situation and wished over and over that some solution would offer itself' that he could see his way out", "by the time he reached his own streets he was keenly alive to the difficulties of his situation and wished over and over that some solution would offer itself that he could see his way out", "by the time he reached his own street he was keenly alive to the difficulties of his situation and wished over and over that some solution would offer itself he could see his way out"], ["2277-149896-0033", "then he rang the bell no answer", "then he rang the bell no answer", "then he rang the bell no answer i", "then he rang the bell no answer to", "then he rang the bell no answer and", "then he rang the bell no answer he"], ["2277-149896-0034", "he rang again this time harder still no answer", "he rang again this time harder still no answer", "he rang again this time harder still no answer i", "he rang again this time harder still no answer and", "he rang again this time harder still no answer a", "he rang again this time harder still no answer to"], ["2277-149897-0000", "when hurstwood got back to his office again he was in a greater quandary than ever", "when hurstwood got back to his office again he was in a greater quandary than ever", "when hurstwood got back to his office again he was in a greater quondary than ever", "when hurstwood got back to his office again he was in a greater quadrille than ever", "when hurstwood went back to his office again he was in a greater quandary than ever", "when hurstwood got back to his office again he was in a greater quandery than ever"], ["2277-149897-0001", "he could hardly realise how it had all come about", "he could hardly realize how it had all come about", "he could hardly realise how it had all come about", "he could hardly realize how it had all come about and", "he could hardly realise how it had all come about", "he could hardly realise how it had all come about and"], ["2277-149897-0002", "no letter had come no word of any kind and yet here it was late in the evening and she had agreed to meet him that morning", "no letter had come no word of any kind and yet here it was late in the evening and she had agreed to meet him that morning", "no letter had come no word of any kind and yet here it was late in the evening she had agreed to meet him that morning", "no letter had come no word of anything kind and yet here it was late in the evening and she had agreed to meet him that morning", "no letter had come no word of any kind and yet here it was late in evening and she had agreed to meet him that morning", "no letter had come no word of any kind and yet here it was late at the evening and she had agreed to meet him that morning"], ["2277-149897-0003", "he saw that in the excitement of recent events he had not formulated a plan upon that score", "he saw that in the excitement of recent events he had not formulated a plan upon that score", "he saw that in the excitement of recent events he had not formulated a plan upon the score", "he saw that in the excitement of recent events he had not formulated a plan on that score", "he saw that in the excitement of recent events he had not formulated a plan upon his score", "he saw that in the excitement of recent events he had not formulated a plan upon that score a"], ["2277-149897-0004", "he was getting some vague comfort out of a good cigar but it was no panacea for the ill which affected him", "he was getting some vague comfort out of a good cigar but it was no panacea for the ill which affected him", "he was getting some vague comfort out of a good cigar but it was no panatia for the ill which affected him", "he was getting some vague comfort out of a good cigar but it was no pannacia for the ill which affected him", "he was getting some vague comfort out of a good cigar but it was no panamia for the ill which affected him", "he was getting some vague comfort out of a good cigar but it was no panama for the ill which affected him"], ["2277-149897-0005", "it was with great opposition after two or three hours of the most urgent mental affirmation and denial that at last he got an envelope placed in it the requested amount and slowly sealed it up", "it was with great opposition after two or three hours of the most urgent mental affirmation and denial that at last he got an envelope placed in it the requested amount and slowly sealed it up", "it was with great opposition after two or three hours of the most urgent mental affirmation and denial that at last he had an envelope placed in it the requested amount and slowly sealed it up", "it was with great opposition after two or three hours of the most urgent mental affirmation and denial that at last he got an envelope placed in it the required amount and slowly sealed it up", "it was with great opposition after two or three hours of the most urgent mental affirmation and denial that at last he got an envelope placed in the requested amount and slowly sealed it up", "was with great opposition after two or three hours of the most urgent mental affirmation and denial that at last he got an envelope placed in it the requested amount and slowly sealed it up"], ["2277-149897-0006", "then he called harry the boy of all work around the place", "then he called harry the boy of all work around the place", "then he called harry the boy of all work round the place", "then he called henry the boy of all work around the place", "then he called harry the boy of all work around the place and", "then he called harry the boy of all work around the place a"], ["2277-149897-0007", "you take this to this address he said handing him the envelope and give it to missus hurstwood yes sir said the boy", "you take this to this address he said handing him the envelope and give it to missus hurstwood yes sir said the boy", "you take this to this address he said handing him the envelope and give it to the missus hurstwood yes sir said the boy", "you take this to this address he said handing him the umbrella and give it to missus hurstwood yes sir said the boy", "you take this to this address he said handing him the envelope and give it to missus hurstwood yes sir said the boy i", "you take this to this addressed he said handing him the envelope and give it to missus hurstwood yes sir said the boy"], ["2277-149897-0008", "any answer i guess not", "any answer i guess not", "any answer i guess not and", "any answer i guess not i guess not", "any answer i guess not to", "any answer i guess not i'll not"], ["2277-149897-0009", "the boy hastened away and the manager fell to his musings", "the boy hastened away and the manager fell to his musings", "the boy hastened away and the manager fell into his musings", "the boy hastened away and the manager fell in his musings", "the boy hastened away and the manager fell to his musings and", "the boy hastened away and the manager fell to his musings'"], ["2277-149897-0010", "he was beaten for to night and he might just as well make the best of it", "he was beaten for to night and he might just as well make the best of it", "he was beaten for tonight and he might just as well make the best of it", "he was beaten for to night and might just as well make the best of it", "he was beaten for tomorrow and he might just as well make the best of it", "he was beaten for to night and he might just as well make best of it"], ["2277-149897-0011", "she would take the envelope and know that she had triumphed", "she would take the envelope and know that she had triumphed", "she would take the envelope and know what she had triumphed", "she would take the envelope and know whether she had triumphed", "she would take a envelope and know that she had triumphed", "she would take the envelope and know that she had triumphed and"], ["2277-149897-0012", "if he only had that letter back he wouldn't send it", "if he only had that letter back he wouldn't send it", "if he only had the letter back he wouldn't send it", "if he only had that letter back he wouldn't send it i", "if he only had had that letter back he wouldn't send it", "if he only had that letter back he wouldn't send it i don't"], ["2277-149897-0013", "for relief he arose and joined in conversation with a few friends who were drinking", "for relief he arose and joined in the conversation with a few friends who were drinking", "for relief he arose and joined in the conversation with the few friends who were drinking", "for relief he rose and joined in the conversation with a few friends who were drinking", "for relief he rose and joined in the conversation with the few friends who were drinking", "for relief he arose and joined in his conversation with a few friends who were drinking"], ["2277-149897-0014", "all the time his thoughts would run out to his home and see the scene being therein enacted", "all the time his thoughts would run out to his home and see the scene being therein enacted", "all the time his thoughts would run out to his home and see the scene being therein enacted", "all the time his thoughts would run out to his home and see the scene being thereon enacted", "all the time his thoughts would run out to his home and see the scene being there in enacted", "all this time his thoughts would run out to his home and see the scene being therein enacted"]]}

wandb/run-20220828_085247-2hx8pk65/files/output.log

@@ -12998,5 +12998,10310 @@ To disable this warning, you can either:
 	- Avoid using `tokenizers` before the fork if possible
 	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
 huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+Training...:  28% 1214/4393 [1:58:17<384:10:13, 435.05s/it]
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+  return jax.tree_map(/4393 [1:58:17<384:10:13, 435.05s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  return jax.tree_map(lambda x: x[0], tree)7,  4.59s/it]
+run_flax_speech_recognition_seq2seq.py:336: FutureWarning: jax.tree_map is deprecated, and will be removed in a future release. Use jax.tree_util.tree_map instead.
+  return jax.tree_map(lambda x: x.astype(jnp.float32) if x.dtype == jnp.bfloat16 else x, t)
+Step... (10000/50000 | Eval Loss: 0.6138997077941895 | Eval wer: 0.05543913826697548 | Eval cer: 0.039964500651745845 |):  17% 2/12 [13:29:59<57:16:35, 20619.59s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  67% 2964/4393 [4:15:20<2:21:10,  5.93s/it]
+Step... (10000/50000 | Eval Loss: 0.6138997077941895 | Eval wer: 0.05543913826697548 | Eval cer: 0.039964500651745845 |)
+Step... (10025 | Loss: 0.08830500394105911, Learning Rate: 8.075959340203553e-05, Gradient Norm: 0.5834270119667053)
+Step... (10050 | Loss: 0.09379326552152634, Learning Rate: 8.070908370427787e-05, Gradient Norm: 0.5947138667106628)
+Step... (10075 | Loss: 0.09823241084814072, Learning Rate: 8.065858128247783e-05, Gradient Norm: 0.6362175345420837)
+Step... (10100 | Loss: 0.06507033109664917, Learning Rate: 8.060807886067778e-05, Gradient Norm: 0.44929641485214233)
+Step... (10125 | Loss: 0.08986574411392212, Learning Rate: 8.055756916292012e-05, Gradient Norm: 0.5437695384025574)
+Step... (10150 | Loss: 0.07754779607057571, Learning Rate: 8.050706674112007e-05, Gradient Norm: 0.5932506918907166)
+Step... (10175 | Loss: 0.06109367311000824, Learning Rate: 8.045656431932002e-05, Gradient Norm: 0.48796671628952026)
+Step... (10200 | Loss: 0.049698445945978165, Learning Rate: 8.040605462156236e-05, Gradient Norm: 0.8630101680755615)
+Step... (10225 | Loss: 0.10865950584411621, Learning Rate: 8.035555219976231e-05, Gradient Norm: 0.8655366897583008)
+Step... (10250 | Loss: 0.09409575164318085, Learning Rate: 8.030504977796227e-05, Gradient Norm: 0.4608936905860901)
+Step... (10275 | Loss: 0.05792190134525299, Learning Rate: 8.02545400802046e-05, Gradient Norm: 0.48416876792907715)
+Step... (10300 | Loss: 0.07172185927629471, Learning Rate: 8.020403765840456e-05, Gradient Norm: 0.4808516502380371)
+Step... (10325 | Loss: 0.11542685329914093, Learning Rate: 8.015353523660451e-05, Gradient Norm: 0.5415076017379761)
+Step... (10350 | Loss: 0.09445095807313919, Learning Rate: 8.010302553884685e-05, Gradient Norm: 0.7509729266166687)
+Step... (10375 | Loss: 0.0917729064822197, Learning Rate: 8.00525231170468e-05, Gradient Norm: 0.7841768264770508)
+Step... (10400 | Loss: 0.09692507982254028, Learning Rate: 8.000202069524676e-05, Gradient Norm: 0.4939047396183014)
+Step... (10425 | Loss: 0.10967368632555008, Learning Rate: 7.99515109974891e-05, Gradient Norm: 0.6144236922264099)
+Step... (10450 | Loss: 0.08952171355485916, Learning Rate: 7.990100857568905e-05, Gradient Norm: 0.4610723853111267)
+Step... (10475 | Loss: 0.09783611446619034, Learning Rate: 7.9850506153889e-05, Gradient Norm: 1.0754166841506958)
+Step... (10500 | Loss: 0.09750466048717499, Learning Rate: 7.979999645613134e-05, Gradient Norm: 1.0963139533996582)
+Step... (10525 | Loss: 0.12679395079612732, Learning Rate: 7.974949403433129e-05, Gradient Norm: 0.8020402789115906)
+Step... (10550 | Loss: 0.09429805725812912, Learning Rate: 7.969899161253124e-05, Gradient Norm: 1.0350173711776733)
+Step... (10575 | Loss: 0.10793081670999527, Learning Rate: 7.964848191477358e-05, Gradient Norm: 0.6100561618804932)
+Step... (10600 | Loss: 0.08640165627002716, Learning Rate: 7.959797949297354e-05, Gradient Norm: 0.7656195163726807)
+Step... (10625 | Loss: 0.08220628648996353, Learning Rate: 7.954747707117349e-05, Gradient Norm: 0.5055153965950012)
+Step... (10650 | Loss: 0.05531294271349907, Learning Rate: 7.949696737341583e-05, Gradient Norm: 0.3716430068016052)
+Step... (10675 | Loss: 0.10128755122423172, Learning Rate: 7.944646495161578e-05, Gradient Norm: 0.6751567125320435)
+Step... (10700 | Loss: 0.09419593960046768, Learning Rate: 7.939596252981573e-05, Gradient Norm: 0.5146047472953796)
+Step... (10725 | Loss: 0.09003331512212753, Learning Rate: 7.934545283205807e-05, Gradient Norm: 0.6622697710990906)
+Step... (10750 | Loss: 0.08137448877096176, Learning Rate: 7.929495041025802e-05, Gradient Norm: 0.4655037224292755)
+Step... (10775 | Loss: 0.11204398423433304, Learning Rate: 7.924444798845798e-05, Gradient Norm: 0.795628011226654)
+Step... (10800 | Loss: 0.07448437809944153, Learning Rate: 7.919393829070032e-05, Gradient Norm: 0.8185940384864807)
+Step... (10825 | Loss: 0.08662033081054688, Learning Rate: 7.914343586890027e-05, Gradient Norm: 1.1144791841506958)
+Step... (10850 | Loss: 0.08633825182914734, Learning Rate: 7.909292617114261e-05, Gradient Norm: 0.5531063675880432)
+Step... (10875 | Loss: 0.08835204690694809, Learning Rate: 7.904242374934256e-05, Gradient Norm: 0.5773115158081055)
+Step... (10900 | Loss: 0.07958956062793732, Learning Rate: 7.899192132754251e-05, Gradient Norm: 0.456502765417099)
+Step... (10925 | Loss: 0.08420322835445404, Learning Rate: 7.894141162978485e-05, Gradient Norm: 0.6213755011558533)
+Step... (10950 | Loss: 0.09074170887470245, Learning Rate: 7.88909092079848e-05, Gradient Norm: 0.5302470326423645)
+Step... (10975 | Loss: 0.12342730909585953, Learning Rate: 7.884039951022714e-05, Gradient Norm: 0.6465903520584106)
+Step... (11000 | Loss: 0.08631566911935806, Learning Rate: 7.87898970884271e-05, Gradient Norm: 0.8472545146942139)
+Step... (11025 | Loss: 0.09519127011299133, Learning Rate: 7.873938739066944e-05, Gradient Norm: 1.0324699878692627)
+Step... (11050 | Loss: 0.07494750618934631, Learning Rate: 7.868888496886939e-05, Gradient Norm: 0.47489726543426514)
+Step... (11075 | Loss: 0.08446040004491806, Learning Rate: 7.863838254706934e-05, Gradient Norm: 0.4666726291179657)
+Step... (11100 | Loss: 0.11089751124382019, Learning Rate: 7.858787284931168e-05, Gradient Norm: 0.5858631134033203)
+Step... (11125 | Loss: 0.08997974544763565, Learning Rate: 7.853737042751163e-05, Gradient Norm: 0.5574468970298767)
+Step... (11150 | Loss: 0.0976918414235115, Learning Rate: 7.848686800571159e-05, Gradient Norm: 0.7729601263999939)
+Step... (11175 | Loss: 0.11596120148897171, Learning Rate: 7.843635830795392e-05, Gradient Norm: 0.6145332455635071)
+Step... (11200 | Loss: 0.14628462493419647, Learning Rate: 7.838585588615388e-05, Gradient Norm: 0.7486353516578674)
+Step... (11225 | Loss: 0.08686906844377518, Learning Rate: 7.833535346435383e-05, Gradient Norm: 0.6828690767288208)
+Step... (11250 | Loss: 0.08807458728551865, Learning Rate: 7.828484376659617e-05, Gradient Norm: 0.48905521631240845)
+Step... (11275 | Loss: 0.0733414962887764, Learning Rate: 7.823434134479612e-05, Gradient Norm: 0.6179749965667725)
+Step... (11300 | Loss: 0.08363909274339676, Learning Rate: 7.818383892299607e-05, Gradient Norm: 0.5266070365905762)
+Step... (11325 | Loss: 0.09379070997238159, Learning Rate: 7.813332922523841e-05, Gradient Norm: 2.2078020572662354)
+Step... (11350 | Loss: 0.12219930440187454, Learning Rate: 7.808282680343837e-05, Gradient Norm: 0.7328922748565674)
+Step... (11375 | Loss: 0.12322746962308884, Learning Rate: 7.803232438163832e-05, Gradient Norm: 0.7159468531608582)
+Step... (11400 | Loss: 0.07447610795497894, Learning Rate: 7.798181468388066e-05, Gradient Norm: 0.4881691336631775)
+Step... (11425 | Loss: 0.11930616945028305, Learning Rate: 7.793131226208061e-05, Gradient Norm: 0.7435767650604248)
+Step... (11450 | Loss: 0.12145199626684189, Learning Rate: 7.788080984028056e-05, Gradient Norm: 0.5640356540679932)
+Step... (11475 | Loss: 0.062346503138542175, Learning Rate: 7.78303001425229e-05, Gradient Norm: 0.7859042882919312)
+Step... (11500 | Loss: 0.07021258771419525, Learning Rate: 7.777979772072285e-05, Gradient Norm: 0.4334392547607422)
+Step... (11525 | Loss: 0.09780385345220566, Learning Rate: 7.772929529892281e-05, Gradient Norm: 0.5772568583488464)
+Step... (11550 | Loss: 0.07436276227235794, Learning Rate: 7.767878560116515e-05, Gradient Norm: 0.450169175863266)
+Step... (11575 | Loss: 0.08741279691457748, Learning Rate: 7.76282831793651e-05, Gradient Norm: 0.5794440507888794)
+Step... (11600 | Loss: 0.07495800405740738, Learning Rate: 7.757777348160744e-05, Gradient Norm: 0.43371060490608215)
+Step... (11625 | Loss: 0.11196848750114441, Learning Rate: 7.752727105980739e-05, Gradient Norm: 0.6372781991958618)
+Step... (11650 | Loss: 0.08077137917280197, Learning Rate: 7.747676863800734e-05, Gradient Norm: 13.57120132446289)
+Step... (11675 | Loss: 0.12308752536773682, Learning Rate: 7.742625894024968e-05, Gradient Norm: 0.7368566393852234)
+Step... (11700 | Loss: 0.057600561529397964, Learning Rate: 7.737575651844963e-05, Gradient Norm: 0.3913513720035553)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Step... (10000/50000 | Eval Loss: 0.6138997077941895 | Eval wer: 0.05543913826697548 | Eval cer: 0.039964500651745845 |):  25% 3/12 [17:36:17<53:07:59, 21253.24s/it]
+Step... (11750 | Loss: 0.07048530131578445, Learning Rate: 7.727474439889193e-05, Gradient Norm: 0.4489003121852875)
+Step... (11775 | Loss: 0.1157061755657196, Learning Rate: 7.722424197709188e-05, Gradient Norm: 0.7761936187744141)
+Step... (11800 | Loss: 0.07189354300498962, Learning Rate: 7.717373955529183e-05, Gradient Norm: 0.4329056441783905)
+Step... (11825 | Loss: 0.10324515402317047, Learning Rate: 7.712322985753417e-05, Gradient Norm: 0.5789346098899841)
+Step... (11850 | Loss: 0.07645975798368454, Learning Rate: 7.707272743573412e-05, Gradient Norm: 1.8985731601715088)
+Step... (11875 | Loss: 0.10933918505907059, Learning Rate: 7.702221773797646e-05, Gradient Norm: 0.5749704837799072)
+Step... (11900 | Loss: 0.09483492374420166, Learning Rate: 7.697171531617641e-05, Gradient Norm: 0.49331286549568176)
+Step... (11925 | Loss: 0.07966803759336472, Learning Rate: 7.692121289437637e-05, Gradient Norm: 0.5429652333259583)
+Step... (11950 | Loss: 0.08023077994585037, Learning Rate: 7.68707031966187e-05, Gradient Norm: 1.0837703943252563)
+Step... (11975 | Loss: 0.13338258862495422, Learning Rate: 7.682020077481866e-05, Gradient Norm: 0.6416175961494446)
+Step... (12000 | Loss: 0.044519901275634766, Learning Rate: 7.676969835301861e-05, Gradient Norm: 0.41103145480155945)
+Step... (12025 | Loss: 0.11081162840127945, Learning Rate: 7.671918865526095e-05, Gradient Norm: 0.6642196774482727)
+Step... (12050 | Loss: 0.0988469272851944, Learning Rate: 7.66686862334609e-05, Gradient Norm: 0.48495012521743774)
+Step... (12075 | Loss: 0.0654635801911354, Learning Rate: 7.661818381166086e-05, Gradient Norm: 0.4860776960849762)
+Step... (12100 | Loss: 0.08458191901445389, Learning Rate: 7.65676741139032e-05, Gradient Norm: 0.4967586100101471)
+Step... (12125 | Loss: 0.08746153116226196, Learning Rate: 7.651717169210315e-05, Gradient Norm: 0.5080762505531311)
+Step... (12150 | Loss: 0.10310745984315872, Learning Rate: 7.64666692703031e-05, Gradient Norm: 0.5726480484008789)
+Step... (12175 | Loss: 0.10443349927663803, Learning Rate: 7.641615957254544e-05, Gradient Norm: 0.6160019636154175)
+Step... (12200 | Loss: 0.08901195973157883, Learning Rate: 7.636565715074539e-05, Gradient Norm: 0.43671485781669617)
+Step... (12225 | Loss: 0.06880508363246918, Learning Rate: 7.631514745298773e-05, Gradient Norm: 0.7653963565826416)
+Step... (12250 | Loss: 0.07732104510068893, Learning Rate: 7.626464503118768e-05, Gradient Norm: 0.6781657338142395)
+Step... (12275 | Loss: 0.12528778612613678, Learning Rate: 7.621413533343002e-05, Gradient Norm: 0.7398132085800171)
+Step... (12300 | Loss: 0.06724114716053009, Learning Rate: 7.616363291162997e-05, Gradient Norm: 0.6260006427764893)
+Step... (12325 | Loss: 0.06048065423965454, Learning Rate: 7.611312321387231e-05, Gradient Norm: 0.46516552567481995)
+Step... (12350 | Loss: 0.07060635089874268, Learning Rate: 7.606262079207227e-05, Gradient Norm: 0.500216543674469)
+Step... (12375 | Loss: 0.09021947532892227, Learning Rate: 7.601211837027222e-05, Gradient Norm: 0.5638901591300964)
+Step... (12400 | Loss: 0.05326595529913902, Learning Rate: 7.596160867251456e-05, Gradient Norm: 0.3534490168094635)
+Step... (12425 | Loss: 0.07285761088132858, Learning Rate: 7.591110625071451e-05, Gradient Norm: 0.5041208863258362)
+Step... (12450 | Loss: 0.10327259451150894, Learning Rate: 7.586060382891446e-05, Gradient Norm: 0.6328871846199036)
+Step... (12475 | Loss: 0.15358465909957886, Learning Rate: 7.58100941311568e-05, Gradient Norm: 0.8533681035041809)
+Step... (12500 | Loss: 0.08255607634782791, Learning Rate: 7.575959170935676e-05, Gradient Norm: 0.6552234292030334)
+Step... (12525 | Loss: 0.12218253314495087, Learning Rate: 7.570908928755671e-05, Gradient Norm: 0.5760861039161682)
+Step... (12550 | Loss: 0.07060109823942184, Learning Rate: 7.565857958979905e-05, Gradient Norm: 0.41352447867393494)
+Step... (12575 | Loss: 0.06469850987195969, Learning Rate: 7.5608077167999e-05, Gradient Norm: 0.46946582198143005)
+Step... (12600 | Loss: 0.08960758149623871, Learning Rate: 7.555757474619895e-05, Gradient Norm: 0.6394600868225098)
+Step... (12625 | Loss: 0.07029081881046295, Learning Rate: 7.550706504844129e-05, Gradient Norm: 0.4810219407081604)
+Step... (12650 | Loss: 0.07843610644340515, Learning Rate: 7.545656262664124e-05, Gradient Norm: 0.4629886746406555)
+Step... (12675 | Loss: 0.0552670955657959, Learning Rate: 7.54060602048412e-05, Gradient Norm: 0.57724928855896)
+Step... (12700 | Loss: 0.08477842062711716, Learning Rate: 7.535555050708354e-05, Gradient Norm: 0.4033758044242859)
+Step... (12725 | Loss: 0.11607497185468674, Learning Rate: 7.530504808528349e-05, Gradient Norm: 0.5471760630607605)
+Step... (12750 | Loss: 0.10754743963479996, Learning Rate: 7.525454566348344e-05, Gradient Norm: 0.67010098695755)
+Step... (12775 | Loss: 0.07305696606636047, Learning Rate: 7.520403596572578e-05, Gradient Norm: 0.5921106934547424)
+Step... (12800 | Loss: 0.07606924325227737, Learning Rate: 7.515353354392573e-05, Gradient Norm: 0.4610716700553894)
+Step... (12825 | Loss: 0.05552862957119942, Learning Rate: 7.510303112212569e-05, Gradient Norm: 0.9628118872642517)
+Step... (12850 | Loss: 0.07874363660812378, Learning Rate: 7.505252142436802e-05, Gradient Norm: 0.4508274495601654)
+Step... (12875 | Loss: 0.06021196022629738, Learning Rate: 7.500201900256798e-05, Gradient Norm: 0.45127421617507935)
+Step... (12900 | Loss: 0.07773559540510178, Learning Rate: 7.495151658076793e-05, Gradient Norm: 0.7120715379714966)
+Step... (12925 | Loss: 0.1171378344297409, Learning Rate: 7.490100688301027e-05, Gradient Norm: 0.8830618858337402)
+Step... (12950 | Loss: 0.06144741550087929, Learning Rate: 7.485050446121022e-05, Gradient Norm: 0.3950470983982086)
+Step... (12975 | Loss: 0.053323205560445786, Learning Rate: 7.480000203941017e-05, Gradient Norm: 0.5374091267585754)
+Step... (13000 | Loss: 0.06425655633211136, Learning Rate: 7.474949234165251e-05, Gradient Norm: 0.690994918346405)
+Step... (13025 | Loss: 0.09800120443105698, Learning Rate: 7.469898991985247e-05, Gradient Norm: 0.5918130278587341)
+Step... (13050 | Loss: 0.07990693300962448, Learning Rate: 7.464848749805242e-05, Gradient Norm: 0.3634057343006134)
+Step... (13075 | Loss: 0.09834995120763779, Learning Rate: 7.459797780029476e-05, Gradient Norm: 0.535110354423523)
+Step... (13100 | Loss: 0.10143288224935532, Learning Rate: 7.454747537849471e-05, Gradient Norm: 0.4474470913410187)
+Step... (13125 | Loss: 0.0711093470454216, Learning Rate: 7.449697295669466e-05, Gradient Norm: 0.4224787950515747)
+Step... (13150 | Loss: 0.07919856905937195, Learning Rate: 7.4446463258937e-05, Gradient Norm: 0.5301010012626648)
+Step... (13175 | Loss: 0.08455699682235718, Learning Rate: 7.439596083713695e-05, Gradient Norm: 0.7841456532478333)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  40% 1745/4393 [2:16:50<2:06:04,  2.86s/it]
+Step... (13200 | Loss: 0.03330820053815842, Learning Rate: 7.434545841533691e-05, Gradient Norm: 0.2786265015602112)
+Step... (13225 | Loss: 0.07699587941169739, Learning Rate: 7.429494871757925e-05, Gradient Norm: 0.5894073247909546)
+Step... (13250 | Loss: 0.05665012076497078, Learning Rate: 7.42444462957792e-05, Gradient Norm: 0.37814077734947205)
+Step... (13275 | Loss: 0.03803393617272377, Learning Rate: 7.419394387397915e-05, Gradient Norm: 0.5187913179397583)
+Step... (13300 | Loss: 0.05835753306746483, Learning Rate: 7.414343417622149e-05, Gradient Norm: 0.4063417911529541)
+Step... (13325 | Loss: 0.0394291952252388, Learning Rate: 7.409293175442144e-05, Gradient Norm: 0.4015607237815857)
+Step... (13350 | Loss: 0.047713086009025574, Learning Rate: 7.404242205666378e-05, Gradient Norm: 0.4759356379508972)
+Step... (13375 | Loss: 0.05229204520583153, Learning Rate: 7.399191963486373e-05, Gradient Norm: 0.6608598232269287)
+Step... (13400 | Loss: 0.05382212623953819, Learning Rate: 7.394141721306369e-05, Gradient Norm: 0.486329585313797)
+Step... (13425 | Loss: 0.05246791988611221, Learning Rate: 7.389090751530603e-05, Gradient Norm: 0.9899137616157532)
+Step... (13450 | Loss: 0.08576538413763046, Learning Rate: 7.384040509350598e-05, Gradient Norm: 0.48174503445625305)
+Step... (13475 | Loss: 0.04886705428361893, Learning Rate: 7.378989539574832e-05, Gradient Norm: 0.5763976573944092)
+Step... (13500 | Loss: 0.06871508061885834, Learning Rate: 7.373939297394827e-05, Gradient Norm: 0.45707303285598755)
+Step... (13525 | Loss: 0.06088908761739731, Learning Rate: 7.368888327619061e-05, Gradient Norm: 2.149138927459717)
+Step... (13550 | Loss: 0.03509918600320816, Learning Rate: 7.363838085439056e-05, Gradient Norm: 0.34174251556396484)
+Step... (13575 | Loss: 0.05192377790808678, Learning Rate: 7.358787843259051e-05, Gradient Norm: 0.6240000128746033)
+Step... (13600 | Loss: 0.08757586777210236, Learning Rate: 7.353736873483285e-05, Gradient Norm: 0.4689755439758301)
+Step... (13625 | Loss: 0.0971912071108818, Learning Rate: 7.34868663130328e-05, Gradient Norm: 0.7313928008079529)
+Step... (13650 | Loss: 0.05947166308760643, Learning Rate: 7.343636389123276e-05, Gradient Norm: 0.4908653795719147)
+Step... (13675 | Loss: 0.04817585647106171, Learning Rate: 7.33858541934751e-05, Gradient Norm: 0.6736827492713928)
+Step... (13700 | Loss: 0.06351016461849213, Learning Rate: 7.333534449571744e-05, Gradient Norm: 0.5939023494720459)
+Step... (13725 | Loss: 0.07226695865392685, Learning Rate: 7.3284849349875e-05, Gradient Norm: 1.5131006240844727)
+Step... (13750 | Loss: 0.07176526635885239, Learning Rate: 7.323433965211734e-05, Gradient Norm: 0.49994346499443054)
+Step... (13775 | Loss: 0.06307905912399292, Learning Rate: 7.318382995435968e-05, Gradient Norm: 1.0378952026367188)
+Step... (13800 | Loss: 0.07338855415582657, Learning Rate: 7.313333480851725e-05, Gradient Norm: 0.5438399314880371)
+Step... (13825 | Loss: 0.07098106294870377, Learning Rate: 7.308282511075959e-05, Gradient Norm: 0.7232956886291504)
+Step... (13850 | Loss: 0.07737265527248383, Learning Rate: 7.303231541300192e-05, Gradient Norm: 0.45953837037086487)
+Step... (13875 | Loss: 0.052374329417943954, Learning Rate: 7.298182026715949e-05, Gradient Norm: 0.7172098755836487)
+Step... (13900 | Loss: 0.04204665124416351, Learning Rate: 7.293131056940183e-05, Gradient Norm: 1.2757434844970703)
+Step... (13925 | Loss: 0.0753527358174324, Learning Rate: 7.288080087164417e-05, Gradient Norm: 0.6898155808448792)
+Step... (13950 | Loss: 0.061371222138404846, Learning Rate: 7.283030572580174e-05, Gradient Norm: 0.4435370862483978)
+Step... (13975 | Loss: 0.07333303987979889, Learning Rate: 7.277979602804407e-05, Gradient Norm: 0.7842553853988647)
+Step... (14000 | Loss: 0.062362585216760635, Learning Rate: 7.272928633028641e-05, Gradient Norm: 0.7911753058433533)
+Step... (14025 | Loss: 0.052322275936603546, Learning Rate: 7.267879118444398e-05, Gradient Norm: 0.6138515472412109)
+Step... (14050 | Loss: 0.06605450809001923, Learning Rate: 7.262828148668632e-05, Gradient Norm: 0.44087859988212585)
+Step... (14075 | Loss: 0.06452997773885727, Learning Rate: 7.257777178892866e-05, Gradient Norm: 0.606708288192749)
+Step... (14100 | Loss: 0.06375371664762497, Learning Rate: 7.252726936712861e-05, Gradient Norm: 0.6149083971977234)
+Step... (14125 | Loss: 0.0548078753054142, Learning Rate: 7.247676694532856e-05, Gradient Norm: 0.6855599284172058)
+Step... (14150 | Loss: 0.05743681266903877, Learning Rate: 7.24262572475709e-05, Gradient Norm: 0.4224940836429596)
+Step... (14175 | Loss: 0.04027267172932625, Learning Rate: 7.237575482577085e-05, Gradient Norm: 0.5648296475410461)
+Step... (14200 | Loss: 0.039936624467372894, Learning Rate: 7.232525240397081e-05, Gradient Norm: 1.8949949741363525)
+Step... (14225 | Loss: 0.09349177032709122, Learning Rate: 7.227474270621315e-05, Gradient Norm: 0.7991975545883179)
+Step... (14250 | Loss: 0.047797802835702896, Learning Rate: 7.22242402844131e-05, Gradient Norm: 0.3861495554447174)
+Step... (14275 | Loss: 0.07225294411182404, Learning Rate: 7.217373786261305e-05, Gradient Norm: 0.6475846767425537)
+Step... (14300 | Loss: 0.05031309649348259, Learning Rate: 7.212322816485539e-05, Gradient Norm: 0.40261930227279663)
+Step... (14325 | Loss: 0.04167238622903824, Learning Rate: 7.207272574305534e-05, Gradient Norm: 0.6085657477378845)
+Step... (14350 | Loss: 0.06230664998292923, Learning Rate: 7.20222233212553e-05, Gradient Norm: 0.39848047494888306)
+Step... (14375 | Loss: 0.05305255204439163, Learning Rate: 7.197171362349764e-05, Gradient Norm: 0.7188547253608704)
+Step... (14400 | Loss: 0.07000914216041565, Learning Rate: 7.192121120169759e-05, Gradient Norm: 1.0084856748580933)
+Step... (14425 | Loss: 0.08697624504566193, Learning Rate: 7.187070877989754e-05, Gradient Norm: 0.8690377473831177)
+Step... (14450 | Loss: 0.06024784967303276, Learning Rate: 7.182019908213988e-05, Gradient Norm: 0.5216410756111145)
+Step... (14475 | Loss: 0.05281940847635269, Learning Rate: 7.176969666033983e-05, Gradient Norm: 0.5339590907096863)
+Step... (14500 | Loss: 0.052328869700431824, Learning Rate: 7.171919423853979e-05, Gradient Norm: 0.34157058596611023)
+Step... (14525 | Loss: 0.08468654006719589, Learning Rate: 7.166868454078212e-05, Gradient Norm: 0.8269289135932922)
+Step... (14550 | Loss: 0.05382496491074562, Learning Rate: 7.161818211898208e-05, Gradient Norm: 0.6192730069160461)
+Step... (14575 | Loss: 0.07302450388669968, Learning Rate: 7.156767969718203e-05, Gradient Norm: 0.7902158498764038)
+Step... (14600 | Loss: 0.10266885161399841, Learning Rate: 7.151716999942437e-05, Gradient Norm: 0.5735669732093811)
+Step... (14625 | Loss: 0.0325029231607914, Learning Rate: 7.146666757762432e-05, Gradient Norm: 0.43449780344963074)
+Step... (14650 | Loss: 0.07611238211393356, Learning Rate: 7.141616515582427e-05, Gradient Norm: 0.4062472879886627)
+Step... (14675 | Loss: 0.0571739599108696, Learning Rate: 7.136565545806661e-05, Gradient Norm: 0.5664569735527039)
+Step... (14700 | Loss: 0.06980940699577332, Learning Rate: 7.131515303626657e-05, Gradient Norm: 0.4309927821159363)
+Step... (14725 | Loss: 0.06994973868131638, Learning Rate: 7.12646433385089e-05, Gradient Norm: 0.7621283531188965)
+Step... (14750 | Loss: 0.051209256052970886, Learning Rate: 7.121414091670886e-05, Gradient Norm: 0.3452795445919037)
+Step... (14775 | Loss: 0.09154806286096573, Learning Rate: 7.11636312189512e-05, Gradient Norm: 0.899725615978241)
+Step... (14800 | Loss: 0.03553222864866257, Learning Rate: 7.111312879715115e-05, Gradient Norm: 0.3117157220840454)
+Step... (14825 | Loss: 0.06072215735912323, Learning Rate: 7.106261909939349e-05, Gradient Norm: 0.7525957822799683)
+Step... (14850 | Loss: 0.08820261061191559, Learning Rate: 7.101211667759344e-05, Gradient Norm: 0.525641679763794)
+Step... (14875 | Loss: 0.05997798219323158, Learning Rate: 7.096161425579339e-05, Gradient Norm: 0.5963910818099976)
+Step... (14900 | Loss: 0.041395582258701324, Learning Rate: 7.091110455803573e-05, Gradient Norm: 0.37330615520477295)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  80% 3521/4393 [4:36:08<1:23:05,  5.72s/it]
+Step... (14950 | Loss: 0.07344218343496323, Learning Rate: 7.081009971443564e-05, Gradient Norm: 0.568691074848175)
+Step... (14975 | Loss: 0.06512783467769623, Learning Rate: 7.075959001667798e-05, Gradient Norm: 0.6864513754844666)
+Step... (15000 | Loss: 0.04500700533390045, Learning Rate: 7.070908759487793e-05, Gradient Norm: 0.416413277387619)
+Step... (15025 | Loss: 0.047238558530807495, Learning Rate: 7.065858517307788e-05, Gradient Norm: 0.572498619556427)
+Step... (15050 | Loss: 0.07442279160022736, Learning Rate: 7.060807547532022e-05, Gradient Norm: 0.557904839515686)
+Step... (15075 | Loss: 0.041969139128923416, Learning Rate: 7.055757305352017e-05, Gradient Norm: 0.5410632491111755)
+Step... (15100 | Loss: 0.0562279112637043, Learning Rate: 7.050707063172013e-05, Gradient Norm: 0.407163143157959)
+Step... (15125 | Loss: 0.07544904947280884, Learning Rate: 7.045656093396246e-05, Gradient Norm: 0.8896680474281311)
+Step... (15150 | Loss: 0.0593624971807003, Learning Rate: 7.040605851216242e-05, Gradient Norm: 0.6524192094802856)
+Step... (15175 | Loss: 0.07540460675954819, Learning Rate: 7.035555609036237e-05, Gradient Norm: 0.7891389727592468)
+Step... (15200 | Loss: 0.06489015370607376, Learning Rate: 7.030504639260471e-05, Gradient Norm: 0.3985648453235626)
+Step... (15225 | Loss: 0.04916614294052124, Learning Rate: 7.025454397080466e-05, Gradient Norm: 1.7696620225906372)
+Step... (15250 | Loss: 0.06432841718196869, Learning Rate: 7.020404154900461e-05, Gradient Norm: 0.45687729120254517)
+Step... (15275 | Loss: 0.0661030113697052, Learning Rate: 7.015353185124695e-05, Gradient Norm: 0.7411028146743774)
+Step... (15300 | Loss: 0.05507814139127731, Learning Rate: 7.01030294294469e-05, Gradient Norm: 0.4231397807598114)
+Step... (15325 | Loss: 0.058054737746715546, Learning Rate: 7.005252700764686e-05, Gradient Norm: 0.6334179043769836)
+Step... (15350 | Loss: 0.08492830395698547, Learning Rate: 7.00020173098892e-05, Gradient Norm: 0.558548092842102)
+Step... (15375 | Loss: 0.05771920830011368, Learning Rate: 6.995151488808915e-05, Gradient Norm: 0.6630625128746033)
+Step... (15400 | Loss: 0.05129191651940346, Learning Rate: 6.99010124662891e-05, Gradient Norm: 0.43164342641830444)
+Step... (15425 | Loss: 0.058036696165800095, Learning Rate: 6.985050276853144e-05, Gradient Norm: 0.6926518082618713)
+Step... (15450 | Loss: 0.09404358267784119, Learning Rate: 6.98000003467314e-05, Gradient Norm: 1.4935879707336426)
+Step... (15475 | Loss: 0.045152828097343445, Learning Rate: 6.974949792493135e-05, Gradient Norm: 0.6178054213523865)
+Step... (15500 | Loss: 0.04112134501338005, Learning Rate: 6.969898822717369e-05, Gradient Norm: 0.34816980361938477)
+Step... (15525 | Loss: 0.05875265598297119, Learning Rate: 6.964848580537364e-05, Gradient Norm: 0.6528817415237427)
+Step... (15550 | Loss: 0.056780021637678146, Learning Rate: 6.959798338357359e-05, Gradient Norm: 0.5109397172927856)
+Step... (15575 | Loss: 0.07518711686134338, Learning Rate: 6.954747368581593e-05, Gradient Norm: 0.8571561574935913)
+Step... (15600 | Loss: 0.06897410750389099, Learning Rate: 6.949696398805827e-05, Gradient Norm: 0.5484206080436707)
+Step... (15625 | Loss: 0.06446701288223267, Learning Rate: 6.944646884221584e-05, Gradient Norm: 0.6554291844367981)
+Step... (15650 | Loss: 0.0701836496591568, Learning Rate: 6.939595914445817e-05, Gradient Norm: 0.5167053937911987)
+Step... (15675 | Loss: 0.08150122314691544, Learning Rate: 6.934544944670051e-05, Gradient Norm: 0.7669651508331299)
+Step... (15700 | Loss: 0.05067530646920204, Learning Rate: 6.929495430085808e-05, Gradient Norm: 0.36807140707969666)
+Step... (15725 | Loss: 0.06164156273007393, Learning Rate: 6.924444460310042e-05, Gradient Norm: 0.5428230166435242)
+Step... (15750 | Loss: 0.061378516256809235, Learning Rate: 6.919393490534276e-05, Gradient Norm: 0.39144206047058105)
+Step... (15775 | Loss: 0.08585003763437271, Learning Rate: 6.914343975950032e-05, Gradient Norm: 0.8241029977798462)
+Step... (15800 | Loss: 0.061143457889556885, Learning Rate: 6.909293006174266e-05, Gradient Norm: 0.4439859390258789)
+Step... (15825 | Loss: 0.10255448520183563, Learning Rate: 6.9042420363985e-05, Gradient Norm: 0.8529015779495239)
+Step... (15850 | Loss: 0.0470162034034729, Learning Rate: 6.899192521814257e-05, Gradient Norm: 0.35999107360839844)
+Step... (15875 | Loss: 0.06504394859075546, Learning Rate: 6.894141552038491e-05, Gradient Norm: 0.618028998374939)
+Step... (15900 | Loss: 0.05916808173060417, Learning Rate: 6.889090582262725e-05, Gradient Norm: 0.3941057324409485)
+Step... (15925 | Loss: 0.05540382117033005, Learning Rate: 6.88404034008272e-05, Gradient Norm: 0.7572468519210815)
+Step... (15950 | Loss: 0.06875719875097275, Learning Rate: 6.878990097902715e-05, Gradient Norm: 0.44904303550720215)
+Step... (15975 | Loss: 0.0626574382185936, Learning Rate: 6.873939128126949e-05, Gradient Norm: 0.8361836075782776)
+Step... (16000 | Loss: 0.06566054373979568, Learning Rate: 6.868888885946944e-05, Gradient Norm: 0.4539177715778351)
+Step... (16025 | Loss: 0.04651724547147751, Learning Rate: 6.863837916171178e-05, Gradient Norm: 0.5512727499008179)
+Step... (16050 | Loss: 0.07435376197099686, Learning Rate: 6.858787673991174e-05, Gradient Norm: 0.48684749007225037)
+Step... (16075 | Loss: 0.07451683282852173, Learning Rate: 6.853737431811169e-05, Gradient Norm: 0.7406129240989685)
+Step... (16100 | Loss: 0.06162319332361221, Learning Rate: 6.848686462035403e-05, Gradient Norm: 0.5421441197395325)
+Step... (16125 | Loss: 0.05605998635292053, Learning Rate: 6.843636219855398e-05, Gradient Norm: 0.9254444241523743)
+Step... (16150 | Loss: 0.06836410611867905, Learning Rate: 6.838585250079632e-05, Gradient Norm: 0.4331052601337433)
+Step... (16175 | Loss: 0.04999904707074165, Learning Rate: 6.833535007899627e-05, Gradient Norm: 0.6736144423484802)
+Step... (16200 | Loss: 0.08609673380851746, Learning Rate: 6.828484038123861e-05, Gradient Norm: 0.5675368905067444)
+Step... (16225 | Loss: 0.07071548700332642, Learning Rate: 6.823433795943856e-05, Gradient Norm: 0.7580640912055969)
+Step... (16250 | Loss: 0.0654497742652893, Learning Rate: 6.818383553763852e-05, Gradient Norm: 0.39117610454559326)
+Step... (16275 | Loss: 0.0967632606625557, Learning Rate: 6.813332583988085e-05, Gradient Norm: 0.8968567848205566)
+Step... (16300 | Loss: 0.05536797270178795, Learning Rate: 6.80828234180808e-05, Gradient Norm: 0.35227692127227783)
+Step... (16325 | Loss: 0.07816967368125916, Learning Rate: 6.803232099628076e-05, Gradient Norm: 0.7050227522850037)
+Step... (16350 | Loss: 0.04643617197871208, Learning Rate: 6.79818112985231e-05, Gradient Norm: 0.5755912065505981)
+Step... (16375 | Loss: 0.09213122725486755, Learning Rate: 6.793130887672305e-05, Gradient Norm: 0.743951141834259)
+Step... (16400 | Loss: 0.06681258976459503, Learning Rate: 6.7880806454923e-05, Gradient Norm: 0.4667063057422638)
+Step... (16425 | Loss: 0.0660945326089859, Learning Rate: 6.783029675716534e-05, Gradient Norm: 0.7734231352806091)
+Step... (16450 | Loss: 0.05684332177042961, Learning Rate: 6.77797943353653e-05, Gradient Norm: 0.42356690764427185)
+Step... (16475 | Loss: 0.09014812111854553, Learning Rate: 6.772929191356525e-05, Gradient Norm: 0.6372724771499634)
+Step... (16500 | Loss: 0.04981550574302673, Learning Rate: 6.767878221580759e-05, Gradient Norm: 0.355785995721817)
+Step... (16525 | Loss: 0.05794210731983185, Learning Rate: 6.762827979400754e-05, Gradient Norm: 0.5047518014907837)
+Step... (16550 | Loss: 0.07783947139978409, Learning Rate: 6.757777737220749e-05, Gradient Norm: 0.462705135345459)
+Step... (16575 | Loss: 0.10741002857685089, Learning Rate: 6.752726767444983e-05, Gradient Norm: 0.9183163642883301)
+Step... (16600 | Loss: 0.0607137456536293, Learning Rate: 6.747676525264978e-05, Gradient Norm: 1.044562578201294)
+Step... (16625 | Loss: 0.04184560477733612, Learning Rate: 6.742626283084974e-05, Gradient Norm: 0.5442230105400085)
+Step... (16650 | Loss: 0.056308675557374954, Learning Rate: 6.737575313309208e-05, Gradient Norm: 0.5075458884239197)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Step... (10000/50000 | Eval Loss: 0.6138997077941895 | Eval wer: 0.05543913826697548 | Eval cer: 0.039964500651745845 |):  33% 4/12 [23:20:47<46:43:02, 21022.76s/it]
+Training...:   0% 0/4393 [00:00<?, ?it/s]
+Step... (16700 | Loss: 0.05502559617161751, Learning Rate: 6.727474828949198e-05, Gradient Norm: 0.41517171263694763)
+Step... (16725 | Loss: 0.08551020920276642, Learning Rate: 6.722423859173432e-05, Gradient Norm: 0.8342790603637695)
+Step... (16750 | Loss: 0.04508410021662712, Learning Rate: 6.717373616993427e-05, Gradient Norm: 0.3625466823577881)
+Step... (16775 | Loss: 0.08115522563457489, Learning Rate: 6.712323374813423e-05, Gradient Norm: 0.877686083316803)
+Step... (16800 | Loss: 0.08558391034603119, Learning Rate: 6.707272405037656e-05, Gradient Norm: 0.4395720064640045)
+Step... (16825 | Loss: 0.1271662563085556, Learning Rate: 6.702222162857652e-05, Gradient Norm: 2.5135364532470703)
+Step... (16850 | Loss: 0.07428053021430969, Learning Rate: 6.697171920677647e-05, Gradient Norm: 0.7400687336921692)
+Step... (16875 | Loss: 0.03849925845861435, Learning Rate: 6.692120950901881e-05, Gradient Norm: 0.6532626748085022)
+Step... (16900 | Loss: 0.046999331563711166, Learning Rate: 6.687070708721876e-05, Gradient Norm: 0.3451361656188965)
+Step... (16925 | Loss: 0.09734851866960526, Learning Rate: 6.682020466541871e-05, Gradient Norm: 1.0243014097213745)
+Step... (16950 | Loss: 0.06578902155160904, Learning Rate: 6.676969496766105e-05, Gradient Norm: 0.34928444027900696)
+Step... (16975 | Loss: 0.0750298947095871, Learning Rate: 6.6719192545861e-05, Gradient Norm: 0.6673861742019653)
+Step... (17000 | Loss: 0.06695280969142914, Learning Rate: 6.666869012406096e-05, Gradient Norm: 0.5307708978652954)
+Step... (17025 | Loss: 0.07898154109716415, Learning Rate: 6.66181804263033e-05, Gradient Norm: 0.809246838092804)
+Step... (17050 | Loss: 0.0646851509809494, Learning Rate: 6.656767800450325e-05, Gradient Norm: 0.478837788105011)
+Step... (17075 | Loss: 0.04616394266486168, Learning Rate: 6.65171755827032e-05, Gradient Norm: 0.7529241442680359)
+Step... (17100 | Loss: 0.08262103796005249, Learning Rate: 6.646666588494554e-05, Gradient Norm: 0.4371437430381775)
+Step... (17125 | Loss: 0.060473013669252396, Learning Rate: 6.64161634631455e-05, Gradient Norm: 0.7513884902000427)
+Step... (17150 | Loss: 0.043580979108810425, Learning Rate: 6.636566104134545e-05, Gradient Norm: 0.3903401792049408)
+Step... (17175 | Loss: 0.07462380826473236, Learning Rate: 6.631515134358779e-05, Gradient Norm: 0.6930056214332581)
+Step... (17200 | Loss: 0.06158146262168884, Learning Rate: 6.626464892178774e-05, Gradient Norm: 0.5356814861297607)
+Step... (17225 | Loss: 0.08772927522659302, Learning Rate: 6.621413922403008e-05, Gradient Norm: 0.8890146017074585)
+Step... (17250 | Loss: 0.06375957280397415, Learning Rate: 6.616363680223003e-05, Gradient Norm: 0.3810139000415802)
+Step... (17275 | Loss: 0.10558323562145233, Learning Rate: 6.611312710447237e-05, Gradient Norm: 0.8314210176467896)
+Step... (17300 | Loss: 0.049320243299007416, Learning Rate: 6.606262468267232e-05, Gradient Norm: 0.36878520250320435)
+Step... (17325 | Loss: 0.08733182400465012, Learning Rate: 6.601211498491466e-05, Gradient Norm: 0.8897705674171448)
+Step... (17350 | Loss: 0.06899122893810272, Learning Rate: 6.596161256311461e-05, Gradient Norm: 0.40162110328674316)
+Step... (17375 | Loss: 0.03307316452264786, Learning Rate: 6.591111014131457e-05, Gradient Norm: 0.5850245356559753)
+Step... (17400 | Loss: 0.05450913682579994, Learning Rate: 6.58606004435569e-05, Gradient Norm: 0.40691012144088745)
+Step... (17425 | Loss: 0.06244771555066109, Learning Rate: 6.581009802175686e-05, Gradient Norm: 0.6149278879165649)
+Step... (17450 | Loss: 0.08476860821247101, Learning Rate: 6.575959559995681e-05, Gradient Norm: 0.5099309086799622)
+Step... (17475 | Loss: 0.03256306052207947, Learning Rate: 6.570908590219915e-05, Gradient Norm: 0.3710366487503052)
+Step... (17500 | Loss: 0.04107552394270897, Learning Rate: 6.56585834803991e-05, Gradient Norm: 0.32978546619415283)
+Step... (17525 | Loss: 0.12821224331855774, Learning Rate: 6.560808105859905e-05, Gradient Norm: 0.8722378015518188)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  39% 1727/4393 [2:16:55<3:53:42,  5.26s/it]
+Step... (17575 | Loss: 0.05494557321071625, Learning Rate: 6.550706893904135e-05, Gradient Norm: 0.3551705777645111)
+Step... (17600 | Loss: 0.0307022575289011, Learning Rate: 6.54565665172413e-05, Gradient Norm: 0.3670017421245575)
+Step... (17625 | Loss: 0.05794623866677284, Learning Rate: 6.540605681948364e-05, Gradient Norm: 0.5661908388137817)
+Step... (17650 | Loss: 0.06338990479707718, Learning Rate: 6.535555439768359e-05, Gradient Norm: 0.9140436053276062)
+Step... (17675 | Loss: 0.05870993807911873, Learning Rate: 6.530505197588354e-05, Gradient Norm: 0.4363075792789459)
+Step... (17700 | Loss: 0.060519590973854065, Learning Rate: 6.525454227812588e-05, Gradient Norm: 0.5052237510681152)
+Step... (17725 | Loss: 0.04964590072631836, Learning Rate: 6.520403258036822e-05, Gradient Norm: 0.534722089767456)
+Step... (17750 | Loss: 0.04412442818284035, Learning Rate: 6.515353743452579e-05, Gradient Norm: 0.38217034935951233)
+Step... (17775 | Loss: 0.07080918550491333, Learning Rate: 6.510302773676813e-05, Gradient Norm: 0.5178171396255493)
+Step... (17800 | Loss: 0.042954571545124054, Learning Rate: 6.505251803901047e-05, Gradient Norm: 0.5687723159790039)
+Step... (17825 | Loss: 0.1522335410118103, Learning Rate: 6.500202289316803e-05, Gradient Norm: 1.4223401546478271)
+Step... (17850 | Loss: 0.025624962523579597, Learning Rate: 6.495151319541037e-05, Gradient Norm: 0.3468577265739441)
+Step... (17875 | Loss: 0.0511438213288784, Learning Rate: 6.490100349765271e-05, Gradient Norm: 0.5264740586280823)
+Step... (17900 | Loss: 0.03367552533745766, Learning Rate: 6.485050835181028e-05, Gradient Norm: 0.33524781465530396)
+Step... (17925 | Loss: 0.06573169678449631, Learning Rate: 6.479999865405262e-05, Gradient Norm: 1.7670120000839233)
+Step... (17950 | Loss: 0.030942659825086594, Learning Rate: 6.474948895629495e-05, Gradient Norm: 0.4611154794692993)
+Step... (17975 | Loss: 0.050055235624313354, Learning Rate: 6.469899381045252e-05, Gradient Norm: 0.44552916288375854)
+Step... (18000 | Loss: 0.04747422784566879, Learning Rate: 6.464848411269486e-05, Gradient Norm: 0.7189703583717346)
+Step... (18025 | Loss: 0.05041767656803131, Learning Rate: 6.45979744149372e-05, Gradient Norm: 0.4791944622993469)
+Step... (18050 | Loss: 0.038698114454746246, Learning Rate: 6.454747926909477e-05, Gradient Norm: 0.3972455859184265)
+Step... (18075 | Loss: 0.06088162586092949, Learning Rate: 6.44969695713371e-05, Gradient Norm: 0.4667831063270569)
+Step... (18100 | Loss: 0.037376631051301956, Learning Rate: 6.444645987357944e-05, Gradient Norm: 0.4643656611442566)
+Step... (18125 | Loss: 0.061981990933418274, Learning Rate: 6.439596472773701e-05, Gradient Norm: 0.42906495928764343)
+Step... (18150 | Loss: 0.055675018578767776, Learning Rate: 6.434545502997935e-05, Gradient Norm: 0.5672094821929932)
+Step... (18175 | Loss: 0.04679860919713974, Learning Rate: 6.429494533222169e-05, Gradient Norm: 0.3329954743385315)
+Step... (18200 | Loss: 0.03453730419278145, Learning Rate: 6.424444291042164e-05, Gradient Norm: 0.5234297513961792)
+Step... (18225 | Loss: 0.040372561663389206, Learning Rate: 6.419394048862159e-05, Gradient Norm: 0.45973172783851624)
+Step... (18250 | Loss: 0.03428987041115761, Learning Rate: 6.414343079086393e-05, Gradient Norm: 0.3846223056316376)
+Step... (18275 | Loss: 0.04736689478158951, Learning Rate: 6.409292836906388e-05, Gradient Norm: 0.4310668706893921)
+Step... (18300 | Loss: 0.04380634054541588, Learning Rate: 6.404242594726384e-05, Gradient Norm: 0.37499678134918213)
+Step... (18325 | Loss: 0.04469010978937149, Learning Rate: 6.399191624950618e-05, Gradient Norm: 0.3715912103652954)
+Step... (18350 | Loss: 0.041518259793519974, Learning Rate: 6.394141382770613e-05, Gradient Norm: 0.5135173201560974)
+Step... (18375 | Loss: 0.050326090306043625, Learning Rate: 6.389091140590608e-05, Gradient Norm: 0.5566602945327759)
+Step... (18400 | Loss: 0.03716479241847992, Learning Rate: 6.384040170814842e-05, Gradient Norm: 0.3112817704677582)
+Step... (18425 | Loss: 0.047295041382312775, Learning Rate: 6.378989928634837e-05, Gradient Norm: 0.3363286554813385)
+Step... (18450 | Loss: 0.031067634001374245, Learning Rate: 6.373939686454833e-05, Gradient Norm: 0.3189374506473541)
+Step... (18475 | Loss: 0.06171252578496933, Learning Rate: 6.368888716679066e-05, Gradient Norm: 0.733445405960083)
+Step... (18500 | Loss: 0.02312381938099861, Learning Rate: 6.363838474499062e-05, Gradient Norm: 0.2859801948070526)
+Step... (18525 | Loss: 0.05208982527256012, Learning Rate: 6.358787504723296e-05, Gradient Norm: 0.4195255637168884)
+Step... (18550 | Loss: 0.04370296373963356, Learning Rate: 6.353737262543291e-05, Gradient Norm: 0.3430011570453644)
+Step... (18575 | Loss: 0.04457544535398483, Learning Rate: 6.348686292767525e-05, Gradient Norm: 0.4072102904319763)
+Step... (18600 | Loss: 0.053745806217193604, Learning Rate: 6.34363605058752e-05, Gradient Norm: 0.9405742287635803)
+Step... (18625 | Loss: 0.05881518870592117, Learning Rate: 6.338585808407515e-05, Gradient Norm: 0.551983118057251)
+Step... (18650 | Loss: 0.028300169855356216, Learning Rate: 6.333534838631749e-05, Gradient Norm: 0.26978495717048645)
+Step... (18675 | Loss: 0.0804310217499733, Learning Rate: 6.328484596451744e-05, Gradient Norm: 0.4554004371166229)
+Step... (18700 | Loss: 0.04370126873254776, Learning Rate: 6.323433626675978e-05, Gradient Norm: 0.3332192897796631)
+Step... (18725 | Loss: 0.046010829508304596, Learning Rate: 6.318383384495974e-05, Gradient Norm: 0.34220054745674133)
+Step... (18750 | Loss: 0.03679228201508522, Learning Rate: 6.313333142315969e-05, Gradient Norm: 0.41170281171798706)
+Step... (18775 | Loss: 0.07983490824699402, Learning Rate: 6.308282172540203e-05, Gradient Norm: 0.6871702075004578)
+Step... (18800 | Loss: 0.033616289496421814, Learning Rate: 6.303231930360198e-05, Gradient Norm: 0.3212784230709076)
+Step... (18825 | Loss: 0.04366293549537659, Learning Rate: 6.298181688180193e-05, Gradient Norm: 0.512840986251831)
+Step... (18850 | Loss: 0.043552931398153305, Learning Rate: 6.293130718404427e-05, Gradient Norm: 0.43920889496803284)
+Step... (18875 | Loss: 0.06454277783632278, Learning Rate: 6.288080476224422e-05, Gradient Norm: 0.4973304569721222)
+Step... (18900 | Loss: 0.04304179176688194, Learning Rate: 6.283030234044418e-05, Gradient Norm: 0.368775874376297)
+Step... (18925 | Loss: 0.050083015114068985, Learning Rate: 6.277979264268652e-05, Gradient Norm: 0.34253236651420593)
+Step... (18950 | Loss: 0.03213053569197655, Learning Rate: 6.272929022088647e-05, Gradient Norm: 0.3148396611213684)
+Step... (18975 | Loss: 0.04660707339644432, Learning Rate: 6.267878779908642e-05, Gradient Norm: 0.4746305048465729)
+Step... (19000 | Loss: 0.06602873653173447, Learning Rate: 6.262827810132876e-05, Gradient Norm: 0.6007050275802612)
+Step... (19025 | Loss: 0.042278241366147995, Learning Rate: 6.257777567952871e-05, Gradient Norm: 0.467171311378479)
+Step... (19050 | Loss: 0.03172172233462334, Learning Rate: 6.252727325772867e-05, Gradient Norm: 0.30905085802078247)
+Step... (19075 | Loss: 0.06033332645893097, Learning Rate: 6.2476763559971e-05, Gradient Norm: 0.3641526401042938)
+Step... (19100 | Loss: 0.0643489882349968, Learning Rate: 6.242626113817096e-05, Gradient Norm: 0.5142189860343933)
+Step... (19125 | Loss: 0.052221208810806274, Learning Rate: 6.237575871637091e-05, Gradient Norm: 1.0921443700790405)
+Step... (19150 | Loss: 0.03727109357714653, Learning Rate: 6.232524901861325e-05, Gradient Norm: 1.8303654193878174)
+Step... (19175 | Loss: 0.04352663457393646, Learning Rate: 6.22747465968132e-05, Gradient Norm: 0.4804896116256714)
+Step... (19200 | Loss: 0.08508438616991043, Learning Rate: 6.222424417501315e-05, Gradient Norm: 5.189798355102539)
+Step... (19225 | Loss: 0.03526046872138977, Learning Rate: 6.21737344772555e-05, Gradient Norm: 0.3649713695049286)
+Step... (19250 | Loss: 0.052900392562150955, Learning Rate: 6.212323205545545e-05, Gradient Norm: 0.48944270610809326)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  55% 2427/4393 [3:12:16<2:54:00,  5.31s/it]
+Step... (19300 | Loss: 0.04498684033751488, Learning Rate: 6.202221993589774e-05, Gradient Norm: 0.3940765857696533)
+Step... (19325 | Loss: 0.04594220966100693, Learning Rate: 6.197171751409769e-05, Gradient Norm: 0.3983670473098755)
+Step... (19350 | Loss: 0.05320899561047554, Learning Rate: 6.192121509229764e-05, Gradient Norm: 0.43670183420181274)
+Step... (19375 | Loss: 0.0990297943353653, Learning Rate: 6.187070539453998e-05, Gradient Norm: 0.5052773952484131)
+Step... (19400 | Loss: 0.05028213933110237, Learning Rate: 6.182020297273993e-05, Gradient Norm: 0.4826207756996155)
+Step... (19425 | Loss: 0.062484271824359894, Learning Rate: 6.176970055093989e-05, Gradient Norm: 0.4715140461921692)
+Step... (19450 | Loss: 0.03580809384584427, Learning Rate: 6.171919085318223e-05, Gradient Norm: 0.34322232007980347)
+Step... (19475 | Loss: 0.06652875244617462, Learning Rate: 6.166868843138218e-05, Gradient Norm: 3.6110024452209473)
+Step... (19500 | Loss: 0.04379062354564667, Learning Rate: 6.161818600958213e-05, Gradient Norm: 0.582844078540802)
+Step... (19525 | Loss: 0.05458759889006615, Learning Rate: 6.156767631182447e-05, Gradient Norm: 0.347485214471817)
+Step... (19550 | Loss: 0.040893107652664185, Learning Rate: 6.151717389002442e-05, Gradient Norm: 0.35938355326652527)
+Step... (19575 | Loss: 0.04561559855937958, Learning Rate: 6.146667146822438e-05, Gradient Norm: 0.4187242388725281)
+Step... (19600 | Loss: 0.029457340016961098, Learning Rate: 6.141616177046672e-05, Gradient Norm: 0.34712833166122437)
+Step... (19625 | Loss: 0.040251851081848145, Learning Rate: 6.136565934866667e-05, Gradient Norm: 0.2937728762626648)
+Step... (19650 | Loss: 0.04119085520505905, Learning Rate: 6.131515692686662e-05, Gradient Norm: 0.35085585713386536)
+Step... (19675 | Loss: 0.06086919829249382, Learning Rate: 6.126464722910896e-05, Gradient Norm: 0.399850457906723)
+Step... (19700 | Loss: 0.05186273902654648, Learning Rate: 6.121414480730891e-05, Gradient Norm: 0.3567661643028259)
+Step... (19725 | Loss: 0.05835634842514992, Learning Rate: 6.116363510955125e-05, Gradient Norm: 0.7113293409347534)
+Step... (19750 | Loss: 0.031978655606508255, Learning Rate: 6.11131326877512e-05, Gradient Norm: 0.34440669417381287)
+Step... (19775 | Loss: 0.03358345478773117, Learning Rate: 6.106262298999354e-05, Gradient Norm: 0.3868075907230377)
+Step... (19800 | Loss: 0.05269327387213707, Learning Rate: 6.10121242061723e-05, Gradient Norm: 0.44065412878990173)
+Step... (19825 | Loss: 0.044944509863853455, Learning Rate: 6.096161450841464e-05, Gradient Norm: 0.33990970253944397)
+Step... (19850 | Loss: 0.049920689314603806, Learning Rate: 6.091110481065698e-05, Gradient Norm: 0.332441508769989)
+Step... (19875 | Loss: 0.05676373839378357, Learning Rate: 6.086060966481455e-05, Gradient Norm: 0.46614909172058105)
+Step... (19900 | Loss: 0.0489303357899189, Learning Rate: 6.0810099967056885e-05, Gradient Norm: 0.3743494153022766)
+Step... (19925 | Loss: 0.04911625757813454, Learning Rate: 6.0759590269299224e-05, Gradient Norm: 0.3483722507953644)
+Step... (19950 | Loss: 0.05078908056020737, Learning Rate: 6.070909512345679e-05, Gradient Norm: 0.34993499517440796)
+Step... (19975 | Loss: 0.04709053412079811, Learning Rate: 6.065858542569913e-05, Gradient Norm: 0.3195936977863312)
+Training...:  55% 2427/4393 [3:12:22<2:54:00,  5.31s/it]
+/home/sanchitgandhi/hf/lib/python3.8/site-packages/flax/jax_utils.py:312: FutureWarning: jax.tree_map is deprecated, and will be removed in a future release. Use jax.tree_util.tree_map instead.
+  return jax.tree_map(pad, tree)
+/home/sanchitgandhi/hf/lib/python3.8/site-packages/flax/jax_utils.py:321: FutureWarning: jax.tree_map is deprecated, and will be removed in a future release. Use jax.tree_util.tree_map instead.
+  return out if static_return else jax.tree_map(unpad, out)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  device_metrics = jax.tree_map(lambda x: x[0], device_metrics)
+/home/sanchitgandhi/hf/lib/python3.8/site-packages/flax/training/common_utils.py:45: FutureWarning: jax.tree_map is deprecated, and will be removed in a future release. Use jax.tree_util.tree_map instead.
+  return jax.tree_map(stack_args, *forest)
+run_flax_speech_recognition_seq2seq.py:1392: FutureWarning: jax.tree_map is deprecated, and will be removed in a future release. Use jax.tree_util.tree_map instead.
+  eval_metrics = jax.tree_map(jnp.mean, eval_metrics)
+Step... (10000/50000 | Eval Loss: 0.6138997077941895 | Erun_flax_speech_recognition_seq2seq.py:1425: FutureWarning: jax.tree_map is deprecated, and will be removed in a future release. Use jax.tree_util.tree_map instead.
+  params = jax.device_get(jax.tree_map(lambda x: x[0], state.params))
+Configuration saved in /home/sanchitgandhi/flax-wav2vec2-2-bart-large-ls-960h-black-box/config.json
+tcmalloc: large alloc 2586787840 bytes == 0x3b7a32000 @  0x7f7cba873680 0x7f7cba893bdd 0x7f7b690721ff 0x7f7b6908142c 0x7f7b6908241d 0x7f7b6908241d 0x7f7b6908241d 0x7f7b6908241d 0x7f7b6908241d 0x7f7b6908241d 0x7f7b6908241d 0x7f7b6907c164 0x7f7b6907c91e 0x505166 0x56bbfa 0x569dba 0x5f6eb3 0x56cc1f 0x569dba 0x5f6eb3 0x56cc1f 0x5f6cd6 0x56bacd 0x569dba 0x50bca0 0x56cc1f 0x569dba 0x5f6eb3 0x56bacd 0x569dba 0x5f6eb3
+tcmalloc: large alloc 2353618944 bytes == 0x452526000 @  0x7f7cba873680 0x7f7cba894824 0x5fb391 0x7f7b6907c209 0x7f7b6907c91e 0x505166 0x56bbfa 0x569dba 0x5f6eb3 0x56cc1f 0x569dba 0x5f6eb3 0x56cc1f 0x5f6cd6 0x56bacd 0x569dba 0x50bca0 0x56cc1f 0x569dba 0x5f6eb3 0x56bacd 0x569dba 0x5f6eb3 0x56bacd 0x569dba 0x6902a7 0x67f951 0x67f9cf 0x67fa71 0x681b97 0x6b9d32
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+Model weights saved in /home/sanchitgandhi/flax-wav2vec2-2-bart-large-ls-960h-black-box/flax_model.msgpack
+tokenizer config file saved in ./tokenizer_config.json
+Special tokens file saved in ./special_tokens_map.json
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
 To disable this warning, you can either:
 	- Avoid using `tokenizers` before the fork if possible

wandb/run-20220828_085247-2hx8pk65/files/wandb-summary.json

@@ -1 +1 @@
-{"train/decoder_grad_norm": 0.5876523852348328, "train/decoder_param_norm": 1057.45703125, "train/encoder_grad_norm": 0.38440409302711487, "train/encoder_param_norm": 2316.3564453125, "train/grad_norm": 0.7022120952606201, "layer_grad_norm/": {"decoder": {"model": {"decoder": {"embed_positions": {"embedding": 0.10323784500360489}, "embed_tokens": {"embedding": 0.16808316111564636}, "layernorm_embedding": {"bias": 0.03703528642654419, "scale": 0.060806743800640106}, "layers": {"FlaxBartDecoderLayers": {"encoder_attn": {"k_proj": {"bias": 1.75027107616188e-05, "kernel": 0.030463965609669685}, "out_proj": {"bias": 0.024376848712563515, "kernel": 0.08760593086481094}, "q_proj": {"bias": 0.0016024636570364237, "kernel": 0.034829143434762955}, "v_proj": {"bias": 0.04787713289260864, "kernel": 0.07169140875339508}}, "encoder_attn_layer_norm": {"bias": 0.03529948368668556, "scale": 0.0380270853638649}, "fc1": {"bias": 0.013248836621642113, "kernel": 0.33658137917518616}, "fc2": {"bias": 0.030859898775815964, "kernel": 0.2677602767944336}, "final_layer_norm": {"bias": 0.1120176762342453, "scale": 0.05825764685869217}, "self_attn": {"k_proj": {"bias": 6.563532224390656e-06, "kernel": 0.047542572021484375}, "out_proj": {"bias": 0.068998321890831, "kernel": 0.15063460171222687}, "q_proj": {"bias": 0.003958633169531822, "kernel": 0.05425203591585159}, "v_proj": {"bias": 0.07329808175563812, "kernel": 0.198069229722023}}, "self_attn_layer_norm": {"bias": 0.023308640345931053, "scale": 0.030806636437773705}}}}}}, "encoder": {"adapter": {"layers": {"0": {"conv": {"bias": 0.04864540696144104, "kernel": 0.133722722530365}}, "1": {"conv": {"bias": 0.04470941796898842, "kernel": 0.09400613605976105}}, "2": {"conv": {"bias": 0.05692768096923828, "kernel": 0.1417897492647171}}}}, "encoder": {"layer_norm": {"bias": 0.16896693408489227, "scale": 0.08190205693244934}, "layers": {"FlaxWav2Vec2EncoderLayers": {"attention": {"k_proj": {"bias": 5.699832854588749e-06, "kernel": 0.03451818600296974}, "out_proj": {"bias": 0.004949449095875025, "kernel": 0.0711507499217987}, "q_proj": {"bias": 0.006232084706425667, "kernel": 0.03630899265408516}, "v_proj": {"bias": 0.021894006058573723, "kernel": 0.0699479877948761}}, "feed_forward": {"intermediate_dense": {"bias": 0.010628663003444672, "kernel": 0.08824677765369415}, "output_dense": {"bias": 0.0046577295288443565, "kernel": 0.07864432781934738}}, "final_layer_norm": {"bias": 0.053700175136327744, "scale": 0.06233147531747818}, "layer_norm": {"bias": 0.09289932250976562, "scale": 0.07505689561367035}}}, "pos_conv_embed": {"conv": {"bias": 0.001811191556043923, "weight_g": 0.04629991203546524, "weight_v": 0.05902065336704254}}}, "feature_extractor": {"conv_layers": {"0": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "1": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "2": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "3": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "4": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "5": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "6": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}}}, "feature_projection": {"layer_norm": {"bias": 0.01040860079228878, "scale": 0.009696024470031261}, "projection": {"bias": 0.002452271291986108, "kernel": 0.06397733092308044}}, "masked_spec_embed": 0.0}}, "layer_param_norm/": {"decoder": {"model": {"decoder": {"embed_positions": {"embedding": 58.57985305786133}, "embed_tokens": {"embedding": 628.9428100585938}, "layernorm_embedding": {"bias": 2.4099645614624023, "scale": 13.944293022155762}, "layers": {"FlaxBartDecoderLayers": {"encoder_attn": {"k_proj": {"bias": 47.96258544921875, "kernel": 330.1817932128906}, "out_proj": {"bias": 6.197176456451416, "kernel": 226.72259521484375}, "q_proj": {"bias": 20.796918869018555, "kernel": 337.1412658691406}, "v_proj": {"bias": 3.727905035018921, "kernel": 230.9994354248047}}, "encoder_attn_layer_norm": {"bias": 10.427277565002441, "scale": 56.38846206665039}, "fc1": {"bias": 25.47351837158203, "kernel": 339.21954345703125}, "fc2": {"bias": 7.897115707397461, "kernel": 243.82398986816406}, "final_layer_norm": {"bias": 4.000784873962402, "scale": 63.70562744140625}, "self_attn": {"k_proj": {"bias": 59.513954162597656, "kernel": 278.91595458984375}, "out_proj": {"bias": 3.8339650630950928, "kernel": 131.7364501953125}, "q_proj": {"bias": 32.09528732299805, "kernel": 282.0332336425781}, "v_proj": {"bias": 2.626418352127075, "kernel": 140.15884399414062}}, "self_attn_layer_norm": {"bias": 8.851421356201172, "scale": 84.72929382324219}}}}}}, "encoder": {"adapter": {"layers": {"0": {"conv": {"bias": 0.5224539637565613, "kernel": 58.06698226928711}}, "1": {"conv": {"bias": 0.6238547563552856, "kernel": 55.76792907714844}}, "2": {"conv": {"bias": 0.8834269046783447, "kernel": 55.83806610107422}}}}, "encoder": {"layer_norm": {"bias": 0.2885725498199463, "scale": 4.501636505126953}, "layers": {"FlaxWav2Vec2EncoderLayers": {"attention": {"k_proj": {"bias": 19.359642028808594, "kernel": 551.2367553710938}, "out_proj": {"bias": 16.819419860839844, "kernel": 703.838134765625}, "q_proj": {"bias": 40.78517532348633, "kernel": 543.7529907226562}, "v_proj": {"bias": 15.60958194732666, "kernel": 695.4569091796875}}, "feed_forward": {"intermediate_dense": {"bias": 24.515138626098633, "kernel": 1373.99365234375}, "output_dense": {"bias": 20.76974868774414, "kernel": 1299.6435546875}}, "final_layer_norm": {"bias": 32.476783752441406, "scale": 141.65736389160156}, "layer_norm": {"bias": 7.329699516296387, "scale": 45.53441619873047}}}, "pos_conv_embed": {"conv": {"bias": 15.283638954162598, "weight_g": 21.029205322265625, "weight_v": 212.9462127685547}}}, "feature_extractor": {"conv_layers": {"0": {"conv": {"bias": 0.5982058644294739, "kernel": 8.08896541595459}, "layer_norm": {"bias": 10.069783210754395, "scale": 10.451257705688477}}, "1": {"conv": {"bias": 4.74075174331665, "kernel": 90.8435287475586}, "layer_norm": {"bias": 6.922820091247559, "scale": 19.5467586517334}}, "2": {"conv": {"bias": 6.7732415199279785, "kernel": 146.13897705078125}, "layer_norm": {"bias": 9.044225692749023, "scale": 19.424888610839844}}, "3": {"conv": {"bias": 5.224758148193359, "kernel": 159.10508728027344}, "layer_norm": {"bias": 8.319666862487793, "scale": 17.64743423461914}}, "4": {"conv": {"bias": 4.434978008270264, "kernel": 157.35813903808594}, "layer_norm": {"bias": 9.193974494934082, "scale": 15.562357902526855}}, "5": {"conv": {"bias": 5.297643661499023, "kernel": 131.1835174560547}, "layer_norm": {"bias": 10.735219955444336, "scale": 13.812533378601074}}, "6": {"conv": {"bias": 5.615579128265381, "kernel": 136.41822814941406}, "layer_norm": {"bias": 12.515308380126953, "scale": 11.152680397033691}}}}, "feature_projection": {"layer_norm": {"bias": 9.422893524169922, "scale": 27.84585189819336}, "projection": {"bias": 4.289161682128906, "kernel": 88.30554962158203}}, "masked_spec_embed": 26.247730255126953}}, "train/learning_rate": 8.086059824563563e-05, "train/loss": 0.1043805480003357, "train/param_norm": 2546.3154296875, "_timestamp": 1661727380, "_runtime": 50613, "_step": 9975}
+{"train/decoder_grad_norm": 0.24299649894237518, "train/decoder_param_norm": 1060.7144775390625, "train/encoder_grad_norm": 0.20758813619613647, "train/encoder_param_norm": 2320.3076171875, "train/grad_norm": 0.3195936977863312, "layer_grad_norm/": {"decoder": {"model": {"decoder": {"embed_positions": {"embedding": 0.015636462718248367}, "embed_tokens": {"embedding": 0.08405420929193497}, "layernorm_embedding": {"bias": 0.006145514082163572, "scale": 0.004968359600752592}, "layers": {"FlaxBartDecoderLayers": {"encoder_attn": {"k_proj": {"bias": 6.667326488241088e-06, "kernel": 0.014865289442241192}, "out_proj": {"bias": 0.01036052592098713, "kernel": 0.049787431955337524}, "q_proj": {"bias": 0.0007108663558028638, "kernel": 0.015181932598352432}, "v_proj": {"bias": 0.019716547802090645, "kernel": 0.03999572619795799}}, "encoder_attn_layer_norm": {"bias": 0.015086976811289787, "scale": 0.016625721007585526}, "fc1": {"bias": 0.0057020955719053745, "kernel": 0.14046090841293335}, "fc2": {"bias": 0.0147400489076972, "kernel": 0.12780840694904327}, "final_layer_norm": {"bias": 0.032790299504995346, "scale": 0.03732256218791008}, "self_attn": {"k_proj": {"bias": 2.526882781239692e-06, "kernel": 0.01344823744148016}, "out_proj": {"bias": 0.020945662632584572, "kernel": 0.0472058430314064}, "q_proj": {"bias": 0.001085575670003891, "kernel": 0.014002838172018528}, "v_proj": {"bias": 0.020695650950074196, "kernel": 0.06064840778708458}}, "self_attn_layer_norm": {"bias": 0.009373994544148445, "scale": 0.011083677411079407}}}}}}, "encoder": {"adapter": {"layers": {"0": {"conv": {"bias": 0.02808896079659462, "kernel": 0.06710191071033478}}, "1": {"conv": {"bias": 0.02258674055337906, "kernel": 0.046897199004888535}}, "2": {"conv": {"bias": 0.02590387687087059, "kernel": 0.07143399119377136}}}}, "encoder": {"layer_norm": {"bias": 0.09637241810560226, "scale": 0.0566645972430706}, "layers": {"FlaxWav2Vec2EncoderLayers": {"attention": {"k_proj": {"bias": 2.509430260033696e-06, "kernel": 0.019797371700406075}, "out_proj": {"bias": 0.0026633520610630512, "kernel": 0.042062435299158096}, "q_proj": {"bias": 0.003061411203816533, "kernel": 0.01928599737584591}, "v_proj": {"bias": 0.011404206976294518, "kernel": 0.04114246740937233}}, "feed_forward": {"intermediate_dense": {"bias": 0.006081512663513422, "kernel": 0.05225696042180061}, "output_dense": {"bias": 0.002437157789245248, "kernel": 0.045792415738105774}}, "final_layer_norm": {"bias": 0.031238090246915817, "scale": 0.03307477384805679}, "layer_norm": {"bias": 0.04994071274995804, "scale": 0.04129469022154808}}}, "pos_conv_embed": {"conv": {"bias": 0.0008167774649336934, "weight_g": 0.0036110610235482454, "weight_v": 0.012701401486992836}}}, "feature_extractor": {"conv_layers": {"0": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "1": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "2": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "3": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "4": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "5": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "6": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}}}, "feature_projection": {"layer_norm": {"bias": 0.0036688754335045815, "scale": 0.004511996638029814}, "projection": {"bias": 0.0011850270675495267, "kernel": 0.033120427280664444}}, "masked_spec_embed": 0.0}}, "layer_param_norm/": {"decoder": {"model": {"decoder": {"embed_positions": {"embedding": 58.625732421875}, "embed_tokens": {"embedding": 628.6316528320312}, "layernorm_embedding": {"bias": 2.4100914001464844, "scale": 13.898597717285156}, "layers": {"FlaxBartDecoderLayers": {"encoder_attn": {"k_proj": {"bias": 47.98827362060547, "kernel": 330.9457092285156}, "out_proj": {"bias": 6.15840482711792, "kernel": 227.3690948486328}, "q_proj": {"bias": 20.846906661987305, "kernel": 337.8941650390625}, "v_proj": {"bias": 3.635528802871704, "kernel": 231.59678649902344}}, "encoder_attn_layer_norm": {"bias": 10.790207862854004, "scale": 57.176551818847656}, "fc1": {"bias": 25.858234405517578, "kernel": 343.7656555175781}, "fc2": {"bias": 7.86165714263916, "kernel": 246.748291015625}, "final_layer_norm": {"bias": 3.946004867553711, "scale": 63.51469421386719}, "self_attn": {"k_proj": {"bias": 59.53274917602539, "kernel": 279.572998046875}, "out_proj": {"bias": 3.8042359352111816, "kernel": 132.37606811523438}, "q_proj": {"bias": 32.1674919128418, "kernel": 282.6742248535156}, "v_proj": {"bias": 2.5869321823120117, "kernel": 140.7692108154297}}, "self_attn_layer_norm": {"bias": 8.874273300170898, "scale": 84.6916732788086}}}}}}, "encoder": {"adapter": {"layers": {"0": {"conv": {"bias": 0.9373133182525635, "kernel": 60.65754699707031}}, "1": {"conv": {"bias": 1.0780301094055176, "kernel": 58.4447135925293}}, "2": {"conv": {"bias": 1.3075517416000366, "kernel": 58.37184524536133}}}}, "encoder": {"layer_norm": {"bias": 0.29216912388801575, "scale": 4.3043694496154785}, "layers": {"FlaxWav2Vec2EncoderLayers": {"attention": {"k_proj": {"bias": 19.379070281982422, "kernel": 552.49365234375}, "out_proj": {"bias": 16.84808349609375, "kernel": 704.6502075195312}, "q_proj": {"bias": 40.86162567138672, "kernel": 545.0123291015625}, "v_proj": {"bias": 15.593074798583984, "kernel": 696.20166015625}}, "feed_forward": {"intermediate_dense": {"bias": 24.4946231842041, "kernel": 1376.0777587890625}, "output_dense": {"bias": 20.806180953979492, "kernel": 1302.1185302734375}}, "final_layer_norm": {"bias": 32.5095329284668, "scale": 141.84854125976562}, "layer_norm": {"bias": 7.295251846313477, "scale": 45.60984420776367}}}, "pos_conv_embed": {"conv": {"bias": 15.245718002319336, "weight_g": 21.039236068725586, "weight_v": 213.549072265625}}}, "feature_extractor": {"conv_layers": {"0": {"conv": {"bias": 0.5982058644294739, "kernel": 8.08896541595459}, "layer_norm": {"bias": 10.069783210754395, "scale": 10.451257705688477}}, "1": {"conv": {"bias": 4.74075174331665, "kernel": 90.8435287475586}, "layer_norm": {"bias": 6.922820091247559, "scale": 19.5467586517334}}, "2": {"conv": {"bias": 6.7732415199279785, "kernel": 146.13897705078125}, "layer_norm": {"bias": 9.044225692749023, "scale": 19.424888610839844}}, "3": {"conv": {"bias": 5.224758148193359, "kernel": 159.10508728027344}, "layer_norm": {"bias": 8.319666862487793, "scale": 17.64743423461914}}, "4": {"conv": {"bias": 4.434978008270264, "kernel": 157.35813903808594}, "layer_norm": {"bias": 9.193974494934082, "scale": 15.562357902526855}}, "5": {"conv": {"bias": 5.297643661499023, "kernel": 131.1835174560547}, "layer_norm": {"bias": 10.735219955444336, "scale": 13.812533378601074}}, "6": {"conv": {"bias": 5.615579128265381, "kernel": 136.41822814941406}, "layer_norm": {"bias": 12.515308380126953, "scale": 11.152680397033691}}}}, "feature_projection": {"layer_norm": {"bias": 9.315728187561035, "scale": 27.725435256958008}, "projection": {"bias": 4.307735443115234, "kernel": 88.24262237548828}}, "masked_spec_embed": 26.247730255126953}}, "train/learning_rate": 6.065858542569913e-05, "train/loss": 0.04709053412079811, "train/param_norm": 2551.262939453125, "_timestamp": 1661775899, "_runtime": 99132, "_step": 19975, "eval/loss": 0.6138997077941895, "eval/wer": 0.05543913826697548, "eval/cer": 0.039964500651745845, "eval/step_10k": {"_type": "table-file", "sha256": "8b44e8a00a036a18ffdf81b4d076c8bf849ea6649001c69e94fa439b14f110ee", "size": 26434, "artifact_path": "wandb-client-artifact://18m0dj4hts3yiat04x5pvmncavkjapd5wb8bznb37vw8c0lqna3m2yjd1wtdrfstuoo7ejt2sphvjo0zuw1e5ne5d3qbkd7c1fylclfggig6us5tsmsj2uum5pchx48n:latest/eval/step_10k.table.json", "_latest_artifact_path": "wandb-client-artifact://18m0dj4hts3yiat04x5pvmncavkjapd5wb8bznb37vw8c0lqna3m2yjd1wtdrfstuoo7ejt2sphvjo0zuw1e5ne5d3qbkd7c1fylclfggig6us5tsmsj2uum5pchx48n:latest/eval/step_10k.table.json", "path": "media/table/eval/step_10k_10000_8b44e8a00a036a18ffdf.table.json", "ncols": 7, "nrows": 50}}

wandb/run-20220828_085247-2hx8pk65/run-2hx8pk65.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:094e92de49c7288ddfac32754880e9359cb30d1406e2d3bdff46b108a8c651aa
-size 4469804
+oid sha256:b64db33311cf3882e158023908e58b8d0a72a15e4fe5d8fa86d30c5ba22a41e2
+size 8691435