2hx8pk65: saving weights and logs of step 30k

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1b20c6ce9070a647fc4b56ff847349b2a6ad959d336f8591e71fc135e07d67a9
+oid sha256:32e9052e8f2cf429458f533122ce50b5a0fc3cfc6e3096288daefda2766c5f0b
 size 2353616717

wandb/run-20220828_085247-2hx8pk65/files/media/table/eval/step_30k_30000_509ad8614e16ae2800f1.table.json

@@ -0,0 +1 @@
+{"columns": ["id", "label_str", "beam_1", "beam_2", "beam_3", "beam_4", "beam_5"], "data": [["2277-149896-0000", "he was in a fevered state of mind owing to the blight his wife's action threatened to cast upon his entire future", "he was in a fevered state of mind owing to the blight his wife's action threatened to cast upon his entire future", "he was in a fevered state of mind owing to the blight his wife's action threatened to cast up upon his entire future", "he was in a fevered state of mind owing to the blight his wife's action threatened to cast up on his entire future", "he was in a fervored state of mind owing to the blight his wife's action threatened to cast upon his entire future", "he was in a fevered state of mind owing to blight his wife's action threatened to cast upon his entire future"], ["2277-149896-0001", "he would have to pay her the money which she would now regularly demand or there would be trouble it did not matter what he did", "he would have to pay her the money which she would now regularly demand or there would be trouble it did not matter what he did", "he would have to pay her her the money which she would now regularly demand or there would be trouble it did not matter what he did", "he would have to pay her the money which he would now regularly demand or there would be trouble it did not matter what he did", "he would have to pay her her money which she would now regularly demand or there would be trouble it did not matter what he did", "he would have to pay her the money which she would now regularly demand or there would be trouble it did not matter what he did it"], ["2277-149896-0002", "hurstwood walked the floor mentally arranging the chief points of his situation", "hurstwood walked the floor mentally arranging the chief points of his situation", "hurstwood walked to the floor mentally arranging the chief points of his situation", "whistwood walked the floor mentally arranging the chief points of his situation", "hirstwood walked the floor mentally arranging the chief points of his situation", "hurstwood walked the floor mentally arranging the chief points of his situation"], ["2277-149896-0003", "he also thought of his managerial position", "he also thought of his managerial position", "he also thought of this managerial position", "he also thought in his managerial position", "he also thought of his managerial position and", "he also thought of his managemental position"], ["2277-149896-0004", "how would the papers talk about it", "how would the papers talk about it", "how would the papers talk about it you", "how would the papers talk about it and", "how would the papers talk about it i", "oh would the papers talk about it"], ["2277-149896-0005", "many little wrinkles gathered between his eyes as he contemplated this and his brow moistened", "many little wrinkles gathered between his eyes as he contemplated this and his brow moistened", "many little wrinkles gathered between his eyes as he considered this and his brow moistened", "many little wrinkles gathering between his eyes as he contemplated this and his brow moistened", "many little wrinkles gathered between his eyes as he contemplated this and his brow moistened", "many little wrinkles gathered between his eyes as he contemplated this then his brow moistened"], ["2277-149896-0006", "he could arrange that satisfactorily for carrie would be glad to wait if necessary", "he could arrange that satisfactorily for carrie would be glad to wait if necessary", "he could arrange that satisfactorily for carry would be glad to wait if necessary", "he could arrange that satisfactorily for perry would be glad to wait if necessary", "he could arrange this satisfactorily for carrie would be glad to wait if necessary", "he could arrange as satisfactorily for carrie would be glad to wait if necessary"], ["2277-149896-0007", "he would see how things turned out to morrow and then he would talk to her they were going to meet as usual", "he would see how things turned out to morrow and then he would talk to her they were going to meet as usual", "he would see how things turned out tomorrow and then he would talk to her they were going to meet as usual", "he would see how things turned out of morrow and then he would talk to her they were going to meet as usual", "he would see how things turned out today and then he would talk to her they were going to meet as usual", "he would see how things turned out to morrow then he would talk to her they were going to meet as usual"], ["2277-149896-0008", "for some reason he felt as if something might come that way and was relieved when all the envelopes had been scanned and nothing suspicious noticed", "for some reason he felt as if something might come that way and was relieved when all the envelopes had been scanned and nothing suspicious noticed", "for some reason he felt as if something might come that way and was relieved when all of the envelopes had been scanned and nothing suspicious noticed", "for some reason he felt as if something might come that way and was relieved when all the envelops had been scanned and nothing suspicious noticed", "for some reason he felt as if something might come this way and was relieved when all the envelopes had been scanned and nothing suspicious noticed", "for some reason he felt as if something might come that way and was relieved while all the envelopes had been scanned and nothing suspicious noticed"], ["2277-149896-0009", "while the danger had not lessened it had not as yet materialised and with him no news was good news", "while the danger had not lessened it had not as yet materialized and with him no news was good news", "while the danger had not lessened it had not as yet materialised and with him no news was good news", "while the danger had not lessen'd it had not as yet materialized and with him no news was good news", "while this danger had not lessened it had not as yet materialized and with him no news was good news", "while the danger had not lessen it it had not as yet materialized and with him no news was good news"], ["2277-149896-0010", "so little did he consider drouet that it never once occurred to him to worry about his finding out", "so little did he consider drue that it never once occurred to him to worry about his finding out", "so little did he consider druda that it never once occurred to him to worry about his finding out", "so little did he consider drua that it never once occurred to him to worry about his finding out", "so little did he consider drude that it never once occurred to him to worry about his finding out", "so little did he consider drusa that it never once occurred to him to worry about his finding out"], ["2277-149896-0011", "he grew restless as he ruminated and then decided that perhaps it was nothing", "he grew restless as he ruminated and then decided that perhaps it was nothing", "he grew restless as he ruminated and then decided that perhaps it was nothing i", "he grew restless as he ruminated and then deciding that perhaps it was nothing", "he grew restless as he ruminated and then decided that perhaps it was nothing and", "he grew restless as he ruminated and then decided that perhaps it was nothing it"], ["2277-149896-0012", "she had not been able to get away this morning", "she had not been able to get away this morning", "she had not been able to get away this morning she had not been able to get away this morning", "she had not been able to get away this morning i had not been able to get away this morning", "she had not been able to get away this morning she had not been able to go away this morning", "she had not been able to get away this morning and"], ["2277-149896-0013", "he would get one to day it would probably be on his desk when he got back he would look for it at once", "he would get one to day it would probably be on his desk when he got back he would look for it at once", "he would get one today it would probably be on his desk when he got back he would look for it at once", "he would get one to day it would probably be upon his desk when he got back he would look for it at once", "he would get one to day it would probably be in his desk when he got back he would look for it at once", "he would get one to day it would probably be on his desk when he got up he would look for it at once"], ["2277-149896-0014", "after a time he gave up waiting and drearily headed for the madison car", "after a time he gave up waiting and drearily headed for the madison car", "after a time he gave up waiting and drearily heading for the madison car", "after a time he gave up waiting and drearily headed for the maddison car", "after a time he gave up waiting and drearily headed for the hadsan car", "after a time he gave up waiting and dreadily headed for the madison car"], ["2277-149896-0015", "he went in and examined his letters but there was nothing from carrie", "he went in and examined his letters but there was nothing from carrie", "he went in and examined his letters but there was nothing from kerry", "he went in and examined his letters but there was nothing from carry", "he went in and examined his letters but there was nothing from cary", "he went in and examined his letters but there was nothing from perry"], ["2277-149896-0016", "fortunately there was nothing from his wife either", "fortunately there was nothing from his wife either", "fortunately there was nothing from his wife either", "fortunately there was nothing from his wife either", "fortunally there was nothing from his wife either", "fortunate there was nothing from his wife either"], ["2277-149896-0017", "at one thirty he went to rector's for lunch and when he returned a messenger was waiting for him", "at one thirty he went to rector's for lunch and when he returned a messenger was waiting for him", "at one thirty he went into rector's for lunch and when he returned a messenger was waiting for him", "at one thirty he went to rector's for lunch and when he returned a messenger was waiting for him", "i at one thirty he went to rector's for lunch and when he returned a messenger was waiting for him", "at one thirty he went to rectors for lunch and when he returned a messenger was waiting for him"], ["2277-149896-0018", "his first impulse was to write but four words in reply go to the devil", "his first impulse was to write but four words in reply go to the devil", "his first impulse was to write but four words in reply go to that devil", "his first impulse was to write but four words and reply go to the devil", "his first impulse was to write but four words in reply go to the devil i", "his first impulse was to write but four words in reply go to the devil'"], ["2277-149896-0019", "but he compromised by telling the boy that there would be no reply", "but he compromised by telling the boy that there would be no reply", "but he compromised by telling the boy there would be no reply", "but he compromised by telling the boy that there would be no reply i", "but hecompromised by telling the boy that there would be no reply", "but he compromised by telling the boy that there would be no reply to"], ["2277-149896-0020", "then he sat down in his chair and gazed without seeing contemplating the result of his work", "then he sat down in his chair and gazed without seeing contemplating the result of his work", "then he sate down in his chair and gazed without seeing contemplating the result of his work", "then he sat down in his chair and gazed without seeing contemplating the result of his works", "then he set down in his chair and gazed without seeing contemplating the result of his work", "then he sat down in his chair and gazed without seeing contemplating the results of his work"], ["2277-149896-0021", "what would she do about that the confounded wretch", "what would she do about that the confounded wretch", "what would she do about that that confounded wretch", "what would she do about that the con founded wretch", "what would she do about that the confounded wretch and", "what would she do about that the confounded wretch oh"], ["2277-149896-0022", "later however his old discretion asserted itself", "later however his old discretion asserted itself", "later however his old discretion asserted itself and", "later however his old discretion asserted itself", "later however this old discretion asserted itself", "later however his old discretion asserted itself if"], ["2277-149896-0023", "something had to be done a climax was near and she would not sit idle", "something had to be done a climax was near and she would not sit idle", "something had to be done a climax was near she would not sit idle", "something had to be done a climax was nearer and she would not sit idle", "something had to be done a climax was near as she would not sit idle", "something had to be done the climax was near and she would not sit idle"], ["2277-149896-0024", "he knew her well enough to know that when she had decided upon a plan she would follow it up", "he knew her well enough to know that when she had decided upon a plan she would follow it up", "he knew her well enough to know that when she decided upon a plan she would follow it up", "he knew her well enough to know that when she had decided upon the plan she would follow it up", "he knew her well enough to know that when she had decided up on a plan she would follow it up", "he knew her well enough to know that when she had decided upon plan she would follow it up"], ["2277-149896-0025", "he arose from his chair and went and looked out into the street", "he arose from his chair and went and looked out into the street", "he rose from his chair and went and looked out into the street", "he arose from his chair and went and looked out into his street", "he arose from his chair and went and looked out into the street and", "he arose from his chair and went and looked out into the street it"], ["2277-149896-0026", "the long drizzle had begun pedestrians had turned up collars and trousers at the bottom", "the long drizzle had begun pedestrians had turned up collars and trousers at the bottom", "the long drizzle had begun petersians had turned up collars and trousers at the bottom", "the long drizzle had begun pedestrians had turned up collars and trousers on the bottom", "the long drizzle had begun pedestrian had turned up collars and trousers at the bottom", "the long drizzle had begun pedestrians had turned up collars and trousers in the bottom"], ["2277-149896-0027", "hurstwood almost exclaimed out loud at the insistency of this thing", "hurstwood almost exclaimed out loud at the insistence of this thing", "hurstwood almost exclaimed out loud at the insistency of this thing", "hurstwood almost exclaimed out loud at the insistency of this thing", "hurstwood almost exclaimed out loud at the insistence of this thing", " hurstwood almost exclaimed out loud at the insistence of this thing"], ["2277-149896-0028", "he put on his hat and looked around for his umbrella", "he put on his hat and looked around for his umbrella", "he put on his hat and looked round for his umbrella", "he put on his hat and looked around for his umbrella and", "he put on his shirt and looked around for his umbrella", "he put on his hand and looked around for his umbrella"], ["2277-149896-0029", "he would have some arrangement of this thing", "he would have some arrangement of this thing", "he would have some arrangement of the thing", "he would have some arrangement of this thing he would have some arrangement of this thing", "he would have some arrangement of his thing", "he would have some arrangement of this thing he would have some arrangement of the thing"], ["2277-149896-0030", "he began to wish that he had compromised in some way or other that he had sent the money perhaps he could do it up here", "he began to wish that he had compromised in some way or other that he had sent the money perhaps he could do it up here", "he began to wish he had compromised in some way or other that he had sent the money perhaps he could do it up here", "he began to wish that he had compromised in some way or other than he had sent the money perhaps he could do it up here", "he began to wish that he had compromised in some way or other that he had set the money perhaps he could do it up here", "he began to wish that he had compromised in some way or other that he had sent the money perhaps he would do it up here"], ["2277-149896-0031", "he would go in and see anyhow he would have no row", "he would go in and see anyhow he would have no row", "he would go in and see anyhow he would have no row he", "he would go in and see anyhow he would have no rue", "he would go in and see anyhow he would have no rowhe", "he would go in and see anyhow he would have no row and"], ["2277-149896-0032", "by the time he reached his own street he was keenly alive to the difficulties of his situation and wished over and over that some solution would offer itself that he could see his way out", "by the time he reached his own street he was keenly alive to the difficulties of his situation and wished over and over that some solution would offer itself that he could see his way out", "by the time he reached his own street he was keenly alive to the difficulties of this situation and wished over and over that some solution would offer itself that he could see his way out", "by that time he reached his own street he was keenly alive to the difficulties of his situation and wished over and over that some solution would offer itself that he could see his way out", "by the time he reached his own street he was keenly alive to the difficulties of his situation and wished over and over that some solution would offer itself that he can see his way out", ""], ["2277-149896-0033", "then he rang the bell no answer", "then he rang the bell no answer", "then he ranged the bell no answer", "then he rang the bell no answer i", "then he rang the bell no answer to", "then he rang the bell no answer you"], ["2277-149896-0034", "he rang again this time harder still no answer", "he rang again this time harder still no answer", "he ring again this time harder still no answer", "he ringed again this time harder still no answer", "he rang again this time harder still no answer i", "he ranged again this time harder still no answer"], ["2277-149897-0000", "when hurstwood got back to his office again he was in a greater quandary than ever", "when hurstwood got back to his office again he was in a greater quandary than ever", "when hurstwood got back to his office again he was in a greater quondary than ever", "when hurstwood got back to his office again he was in a greater quadrille than ever", "when hurstwood got back to his office again he was in a greater quadry than ever", "when hurstwood went back to his office again he was in a greater quandary than ever"], ["2277-149897-0001", "he could hardly realise how it had all come about", "he could hardly realise how it had all come about", "he could hardly realize how it had all come about", "he could hardly realise how it had all come about", "he could hardly realise how it had all come above", "he could hardly realize how it had all come above"], ["2277-149897-0002", "no letter had come no word of any kind and yet here it was late in the evening and she had agreed to meet him that morning", "no letter had come no word of any kind and yet here it was late in the evening and she had agreed to meet him that morning", "no letter had come no word of an kind and yet here it was late in the evening and she had agreed to meet him that morning", "no letter had come no word of any kind and yet here it was late in the evening she had agreed to meet him that morning", "no letter had come no word of any kind and yet here it was late at the evening and she had agreed to meet him that morning", "no letter had come no word of the kind and yet here it was late in the evening and she had agreed to meet him that morning"], ["2277-149897-0003", "he saw that in the excitement of recent events he had not formulated a plan upon that score", "he saw that in the excitement of recent events he had not formulated a plan upon that score", "he saw that in the excitement of recent events he had not formulated a plan upon the score", "he saw that in the excitement of recent events he had not formulated a plan upon this score", "he saw that in the excitement of recent events he had not formulated a plan upon his score", "he saw that in the excitement of recent events he had not formulated a plan upon that score and"], ["2277-149897-0004", "he was getting some vague comfort out of a good cigar but it was no panacea for the ill which affected him", "he was getting some vague comfort out of a good cigar but it was no panacea for the ill which affected him", "he was getting some vague comfort out of a good cigar but it was no panatia for the ill which affected him", "he was getting some vague comfort out of a good cigar but it was no panatya for the ill which affected him", "he was getting some vague comfort out of a good cigar but it was no panaty for the ill which affected him", "he was getting some vague comfort out of a good cigar but it was no penatia for the ill which affected him"], ["2277-149897-0005", "it was with great opposition after two or three hours of the most urgent mental affirmation and denial that at last he got an envelope placed in it the requested amount and slowly sealed it up", "it was with great opposition after two or three hours of the most urgent mental affirmation and denial that at last he got an envelope placed in it the requested amount and slowly sealed it up", "it was with great opposition after two or three hours of the most urgent mental affirmation and denial that at last he got an envelope placed in it the requested amount and slowly sealed it up", "it was with great opposition after two or three hours of the most urgent personal affirmation and denial that at last he got an envelope placed in it the requested amount and slowly sealed it up", "it was with great opposition after two or three hours of that most urgent mental affirmation and denial that at last he got an envelope placed in it the requested amount and slowly sealed it up", "it was with great opposition after two or three hours of the most urgent mental affirmation and denial that at last he got an envelope placed in it the requested account and slowly sealed it up"], ["2277-149897-0006", "then he called harry the boy of all work around the place", "then he called harry the boy of all work around the place", "then he called harry the boy of all work round the place", "then he called harry that boy of all work around the place", "then he called harry the boy of all work around the place and", "then he called harry the boy of all work around the place i"], ["2277-149897-0007", "you take this to this address he said handing him the envelope and give it to missus hurstwood yes sir said the boy", "you take this to this address he said handing him the envelope and give it to missus hurstwood yes sir said the boy", "you take this to this address he said handing him the envelope and give it to missus hurstwood yes sir said the boy i", "you take this to this address he said handing him the envelope and give it me to missus hurstwood yes sir said the boy", "you take this to this address he said handing him an envelope and give it to missus hurstwood yes sir said the boy", "you take this to this address he said handing him the envelope an give it to missus hurstwood yes sir said the boy"], ["2277-149897-0008", "any answer i guess not", "any answer i guess not", "any answer i guess no", "any answer i guess not i", "any answer i guess not you", "any answer i guess not to"], ["2277-149897-0009", "the boy hastened away and the manager fell to his musings", "the boy hastened away and the manager fell to his musings", "the boy hasted away and the manager fell to his musings", "the boy hastily away and the manager fell to his musings", "the boy hurried away and the manager fell to his musings", "the boy hastened away and the major fell to his musings"], ["2277-149897-0010", "he was beaten for to night and he might just as well make the best of it", "he was beaten for to night and he might just as well make the best of it", "he was beaten for tonight and he might just as well make the best of it", "he was beaten for to night and he might just as well make best of it", "he was beaten for to night and i might just as well make the best of it", "he was beaten for today and he might just as well make the best of it"], ["2277-149897-0011", "she would take the envelope and know that she had triumphed", "she would take the envelope and know that she had triumphed", "she would take the envelope and know what she had triumphed", "she would take a envelope and know that she had triumphed", "she would take an envelope and know that she had triumphed", "she would take the envelope and know that she had triumphed and"], ["2277-149897-0012", "if he only had that letter back he wouldn't send it", "if he only had that letter back he wouldn't send it", "if he only had that letter back he wouldn't send it", "if he only had that letter backhe wouldn't send it", "if he only had that letter round he wouldn't send it", "if he only had that letter back he wouldn't send itif"], ["2277-149897-0013", "for relief he arose and joined in conversation with a few friends who were drinking", "for relief he arose and joined in the conversation with a few friends who were drinking", "for relief he arose and joined in the conversation with the few friends who were drinking", "for relief he arose and rejoined in the conversation with a few friends who were drinking", "for relief he arose and joined in his conversation with a few friends who were drinking", "or relief he arose and joined in the conversation with a few friends who were drinking"], ["2277-149897-0014", "all the time his thoughts would run out to his home and see the scene being therein enacted", "all the time his thoughts would run out to his home and see the scene being therein enacted", "all this time his thoughts would run out to his home and see the scene being therein enacted", "all the time his thoughts would run out to his home and see the scene being therein enacted", "all the time his thoughts would run out to his home and see that scene being therein enacted", "all that time his thoughts would run out to his home and see the scene being therein enacted"]]}

wandb/run-20220828_085247-2hx8pk65/files/output.log

@@ -23303,5 +23303,10315 @@ To disable this warning, you can either:
 	- Avoid using `tokenizers` before the fork if possible
 	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
 huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+Training...:  55% 2428/4393 [3:21:24<91:44:40, 168.08s/it]
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+  return jax.tree_map(/4393 [3:21:24<91:44:40, 168.08s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  return jax.tree_map(lambda x: x[0], tree)7,  4.17s/it]
+run_flax_speech_recognition_seq2seq.py:336: FutureWarning: jax.tree_map is deprecated, and will be removed in a future release. Use jax.tree_util.tree_map instead.
+  return jax.tree_map(lambda x: x.astype(jnp.float32) if x.dtype == jnp.bfloat16 else x, t)
+Step... (20000/50000 | Eval Loss: 1.021510124206543 | Eval wer: 0.05054961214661226 | Eval cer: 0.0362100285658818 |):  33% 4/12 [26:43:52<46:43:02, 21022.76s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  95% 4152/4393 [5:36:46<15:55,  3.96s/it]
+Step... (20000/50000 | Eval Loss: 1.021510124206543 | Eval wer: 0.05054961214661226 | Eval cer: 0.0362100285658818 |)
+Step... (20025 | Loss: 0.04840395227074623, Learning Rate: 6.055757694412023e-05, Gradient Norm: 0.3350682854652405)
+Step... (20050 | Loss: 0.040021102875471115, Learning Rate: 6.0507070884341374e-05, Gradient Norm: 0.3626783788204193)
+Step... (20075 | Loss: 0.06721880286931992, Learning Rate: 6.045656118658371e-05, Gradient Norm: 0.661290168762207)
+Step... (20100 | Loss: 0.039792388677597046, Learning Rate: 6.040606240276247e-05, Gradient Norm: 0.41450798511505127)
+Step... (20125 | Loss: 0.06491632759571075, Learning Rate: 6.035555270500481e-05, Gradient Norm: 0.5559267401695251)
+Step... (20150 | Loss: 0.04650925472378731, Learning Rate: 6.030504664522596e-05, Gradient Norm: 0.35466456413269043)
+Step... (20175 | Loss: 0.04336107149720192, Learning Rate: 6.025454786140472e-05, Gradient Norm: 0.3757917582988739)
+Step... (20200 | Loss: 0.05388178676366806, Learning Rate: 6.0204038163647056e-05, Gradient Norm: 0.39275360107421875)
+Step... (20225 | Loss: 0.033549536019563675, Learning Rate: 6.01535321038682e-05, Gradient Norm: 0.33024901151657104)
+Step... (20250 | Loss: 0.04022737964987755, Learning Rate: 6.010303332004696e-05, Gradient Norm: 0.31290462613105774)
+Step... (20275 | Loss: 0.04750262200832367, Learning Rate: 6.00525236222893e-05, Gradient Norm: 0.303996205329895)
+Step... (20300 | Loss: 0.05061691999435425, Learning Rate: 6.000201392453164e-05, Gradient Norm: 0.5085467100143433)
+Step... (20325 | Loss: 0.03429495915770531, Learning Rate: 5.995151150273159e-05, Gradient Norm: 0.4001585841178894)
+Step... (20350 | Loss: 0.04250844568014145, Learning Rate: 5.9901009080931544e-05, Gradient Norm: 0.3081212639808655)
+Step... (20375 | Loss: 0.03005947545170784, Learning Rate: 5.985049938317388e-05, Gradient Norm: 0.25889503955841064)
+Step... (20400 | Loss: 0.03023005835711956, Learning Rate: 5.9799996961373836e-05, Gradient Norm: 0.323447048664093)
+Step... (20425 | Loss: 0.03575645387172699, Learning Rate: 5.974949453957379e-05, Gradient Norm: 0.28628385066986084)
+Step... (20450 | Loss: 0.03535941615700722, Learning Rate: 5.969898484181613e-05, Gradient Norm: 0.29935961961746216)
+Step... (20475 | Loss: 0.035111647099256516, Learning Rate: 5.964848242001608e-05, Gradient Norm: 0.32121407985687256)
+Step... (20500 | Loss: 0.07113418728113174, Learning Rate: 5.959797999821603e-05, Gradient Norm: 0.7147608399391174)
+Step... (20525 | Loss: 0.04110198840498924, Learning Rate: 5.954747030045837e-05, Gradient Norm: 0.2974580228328705)
+Step... (20550 | Loss: 0.06231129169464111, Learning Rate: 5.9496967878658324e-05, Gradient Norm: 0.40457063913345337)
+Step... (20575 | Loss: 0.03932349756360054, Learning Rate: 5.944646545685828e-05, Gradient Norm: 0.28113406896591187)
+Step... (20600 | Loss: 0.035716500133275986, Learning Rate: 5.9395955759100616e-05, Gradient Norm: 0.30872511863708496)
+Step... (20625 | Loss: 0.0612182579934597, Learning Rate: 5.934545333730057e-05, Gradient Norm: 0.4794360101222992)
+Step... (20650 | Loss: 0.06335697323083878, Learning Rate: 5.929495091550052e-05, Gradient Norm: 0.9689046144485474)
+Step... (20675 | Loss: 0.056457117199897766, Learning Rate: 5.924444121774286e-05, Gradient Norm: 0.4037126302719116)
+Step... (20700 | Loss: 0.04219113290309906, Learning Rate: 5.919393879594281e-05, Gradient Norm: 0.49446389079093933)
+Step... (20725 | Loss: 0.03880039229989052, Learning Rate: 5.914343273616396e-05, Gradient Norm: 0.522761344909668)
+Step... (20750 | Loss: 0.051740579307079315, Learning Rate: 5.9092926676385105e-05, Gradient Norm: 0.4382433295249939)
+Step... (20775 | Loss: 0.04807671159505844, Learning Rate: 5.904242425458506e-05, Gradient Norm: 0.3645383417606354)
+Step... (20800 | Loss: 0.03817301243543625, Learning Rate: 5.89919181948062e-05, Gradient Norm: 0.3344947099685669)
+Step... (20825 | Loss: 0.04550916701555252, Learning Rate: 5.894141213502735e-05, Gradient Norm: 0.30412057042121887)
+Step... (20850 | Loss: 0.02587965875864029, Learning Rate: 5.8890906075248495e-05, Gradient Norm: 0.2640466094017029)
+Step... (20875 | Loss: 0.04702045023441315, Learning Rate: 5.884040365344845e-05, Gradient Norm: 0.405086487531662)
+Step... (20900 | Loss: 0.03893226757645607, Learning Rate: 5.8789893955690786e-05, Gradient Norm: 0.2916216552257538)
+Step... (20925 | Loss: 0.05458155274391174, Learning Rate: 5.873939153389074e-05, Gradient Norm: 0.5137640237808228)
+Step... (20950 | Loss: 0.026842793449759483, Learning Rate: 5.868888911209069e-05, Gradient Norm: 0.29405421018600464)
+Step... (20975 | Loss: 0.04555520787835121, Learning Rate: 5.863837941433303e-05, Gradient Norm: 0.39362606406211853)
+Step... (21000 | Loss: 0.06246699020266533, Learning Rate: 5.8587876992532983e-05, Gradient Norm: 0.6405248641967773)
+Step... (21025 | Loss: 0.07497163116931915, Learning Rate: 5.8537374570732936e-05, Gradient Norm: 0.649531364440918)
+Step... (21050 | Loss: 0.05232103168964386, Learning Rate: 5.8486864872975275e-05, Gradient Norm: 0.40732765197753906)
+Step... (21075 | Loss: 0.055781468749046326, Learning Rate: 5.843636245117523e-05, Gradient Norm: 0.3719920217990875)
+Step... (21100 | Loss: 0.04000868275761604, Learning Rate: 5.838586002937518e-05, Gradient Norm: 0.30638593435287476)
+Step... (21125 | Loss: 0.0311842430382967, Learning Rate: 5.833535033161752e-05, Gradient Norm: 0.31067410111427307)
+Step... (21150 | Loss: 0.06602907925844193, Learning Rate: 5.828484790981747e-05, Gradient Norm: 0.4416637122631073)
+Step... (21175 | Loss: 0.04677121713757515, Learning Rate: 5.8234345488017425e-05, Gradient Norm: 0.5177642107009888)
+Step... (21200 | Loss: 0.045037493109703064, Learning Rate: 5.8183835790259764e-05, Gradient Norm: 0.34402015805244446)
+Step... (21225 | Loss: 0.03223634138703346, Learning Rate: 5.8133333368459716e-05, Gradient Norm: 0.3138290047645569)
+Step... (21250 | Loss: 0.0355977863073349, Learning Rate: 5.808283094665967e-05, Gradient Norm: 0.2971906363964081)
+Step... (21275 | Loss: 0.07619132101535797, Learning Rate: 5.803232124890201e-05, Gradient Norm: 0.5961235165596008)
+Step... (21300 | Loss: 0.031767457723617554, Learning Rate: 5.798181882710196e-05, Gradient Norm: 0.3297065496444702)
+Step... (21325 | Loss: 0.04114881902933121, Learning Rate: 5.7931312767323107e-05, Gradient Norm: 0.29805976152420044)
+Step... (21350 | Loss: 0.06210632622241974, Learning Rate: 5.788080670754425e-05, Gradient Norm: 0.4565739035606384)
+Step... (21375 | Loss: 0.05203666165471077, Learning Rate: 5.78303006477654e-05, Gradient Norm: 0.5167878270149231)
+Step... (21400 | Loss: 0.04761577770113945, Learning Rate: 5.777979822596535e-05, Gradient Norm: 0.7846924662590027)
+Step... (21425 | Loss: 0.05161000043153763, Learning Rate: 5.77292921661865e-05, Gradient Norm: 0.4291480779647827)
+Step... (21450 | Loss: 0.04251234605908394, Learning Rate: 5.767878610640764e-05, Gradient Norm: 0.402444064617157)
+Step... (21475 | Loss: 0.05318749323487282, Learning Rate: 5.7628283684607595e-05, Gradient Norm: 0.4376218020915985)
+Step... (21500 | Loss: 0.04826676845550537, Learning Rate: 5.7577773986849934e-05, Gradient Norm: 0.38663220405578613)
+Step... (21525 | Loss: 0.06910662353038788, Learning Rate: 5.752727156504989e-05, Gradient Norm: 0.44211477041244507)
+Step... (21550 | Loss: 0.06470490247011185, Learning Rate: 5.747676914324984e-05, Gradient Norm: 0.4820682406425476)
+Step... (21575 | Loss: 0.036486972123384476, Learning Rate: 5.742625944549218e-05, Gradient Norm: 0.3340790271759033)
+Step... (21600 | Loss: 0.043834902346134186, Learning Rate: 5.737575702369213e-05, Gradient Norm: 0.40074360370635986)
+Step... (21625 | Loss: 0.029955435544252396, Learning Rate: 5.7325254601892084e-05, Gradient Norm: 0.3754229247570038)
+Step... (21650 | Loss: 0.039664458483457565, Learning Rate: 5.727474490413442e-05, Gradient Norm: 0.4904446005821228)
+Step... (21675 | Loss: 0.04634978622198105, Learning Rate: 5.7224242482334375e-05, Gradient Norm: 0.40155136585235596)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Step... (20000/50000 | Eval Loss: 1.021510124206543 | Eval wer: 0.05054961214661226 | Eval cer: 0.0362100285658818 |):  42% 5/12 [29:16:34<41:06:17, 21139.65s/it]
+Training...:   0% 0/4393 [00:00<?, ?it/s]
+Step... (21725 | Loss: 0.044285327196121216, Learning Rate: 5.712323036277667e-05, Gradient Norm: 0.30181413888931274)
+Step... (21750 | Loss: 0.04501153156161308, Learning Rate: 5.707272794097662e-05, Gradient Norm: 0.3694307208061218)
+Step... (21775 | Loss: 0.0468716062605381, Learning Rate: 5.702222551917657e-05, Gradient Norm: 0.4017857015132904)
+Step... (21800 | Loss: 0.08984293788671494, Learning Rate: 5.697171582141891e-05, Gradient Norm: 0.7361651062965393)
+Step... (21825 | Loss: 0.06769392639398575, Learning Rate: 5.692120612366125e-05, Gradient Norm: 0.44246554374694824)
+Step... (21850 | Loss: 0.04700716957449913, Learning Rate: 5.687071097781882e-05, Gradient Norm: 0.3806375563144684)
+Step... (21875 | Loss: 0.052144188433885574, Learning Rate: 5.6820201280061156e-05, Gradient Norm: 0.5597290992736816)
+Step... (21900 | Loss: 0.04265350103378296, Learning Rate: 5.6769691582303494e-05, Gradient Norm: 0.7775082588195801)
+Step... (21925 | Loss: 0.07321219891309738, Learning Rate: 5.6719192798482254e-05, Gradient Norm: 0.4794605076313019)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  39% 1735/4393 [2:17:11<3:39:45,  4.96s/it]
+Step... (21975 | Loss: 0.028883015736937523, Learning Rate: 5.661817704094574e-05, Gradient Norm: 0.27709388732910156)
+Step... (22000 | Loss: 0.032305553555488586, Learning Rate: 5.65676782571245e-05, Gradient Norm: 0.36947354674339294)
+Step... (22025 | Loss: 0.029987312853336334, Learning Rate: 5.6517172197345644e-05, Gradient Norm: 0.3085069954395294)
+Step... (22050 | Loss: 0.03197702020406723, Learning Rate: 5.646666249958798e-05, Gradient Norm: 0.27674245834350586)
+Step... (22075 | Loss: 0.04232995584607124, Learning Rate: 5.641616371576674e-05, Gradient Norm: 0.34863319993019104)
+Step... (22100 | Loss: 0.01950874924659729, Learning Rate: 5.636565401800908e-05, Gradient Norm: 0.258765310049057)
+Step... (22125 | Loss: 0.0584186315536499, Learning Rate: 5.631514795823023e-05, Gradient Norm: 0.5440660715103149)
+Step... (22150 | Loss: 0.029707320034503937, Learning Rate: 5.626464917440899e-05, Gradient Norm: 0.3775123953819275)
+Step... (22175 | Loss: 0.04275369271636009, Learning Rate: 5.6214139476651326e-05, Gradient Norm: 0.33941468596458435)
+Step... (22200 | Loss: 0.0313958078622818, Learning Rate: 5.6163629778893664e-05, Gradient Norm: 0.38821160793304443)
+Step... (22225 | Loss: 0.03712507337331772, Learning Rate: 5.611313463305123e-05, Gradient Norm: 0.28947848081588745)
+Step... (22250 | Loss: 0.04902244731783867, Learning Rate: 5.606262493529357e-05, Gradient Norm: 0.3721737265586853)
+Step... (22275 | Loss: 0.027602573856711388, Learning Rate: 5.601211523753591e-05, Gradient Norm: 0.2835336923599243)
+Step... (22300 | Loss: 0.04686018079519272, Learning Rate: 5.5961620091693476e-05, Gradient Norm: 0.43980643153190613)
+Step... (22325 | Loss: 0.031802356243133545, Learning Rate: 5.5911110393935814e-05, Gradient Norm: 0.3008425235748291)
+Step... (22350 | Loss: 0.02849770523607731, Learning Rate: 5.586060069617815e-05, Gradient Norm: 0.365633100271225)
+Step... (22375 | Loss: 0.028170589357614517, Learning Rate: 5.5810098274378106e-05, Gradient Norm: 0.31893348693847656)
+Step... (22400 | Loss: 0.027144117280840874, Learning Rate: 5.575959585257806e-05, Gradient Norm: 0.3039880096912384)
+Step... (22425 | Loss: 0.040041469037532806, Learning Rate: 5.57090861548204e-05, Gradient Norm: 0.4088221788406372)
+Step... (22450 | Loss: 0.03567557409405708, Learning Rate: 5.565858373302035e-05, Gradient Norm: 0.35828736424446106)
+Step... (22475 | Loss: 0.020282002165913582, Learning Rate: 5.56080813112203e-05, Gradient Norm: 0.254996120929718)
+Step... (22500 | Loss: 0.04376010596752167, Learning Rate: 5.555757161346264e-05, Gradient Norm: 0.7653269171714783)
+Step... (22525 | Loss: 0.04300732538104057, Learning Rate: 5.5507069191662595e-05, Gradient Norm: 0.3325899541378021)
+Step... (22550 | Loss: 0.04675268009305, Learning Rate: 5.545656676986255e-05, Gradient Norm: 0.4212055802345276)
+Step... (22575 | Loss: 0.03605210781097412, Learning Rate: 5.5406057072104886e-05, Gradient Norm: 0.5294095277786255)
+Step... (22600 | Loss: 0.02685914747416973, Learning Rate: 5.535555465030484e-05, Gradient Norm: 0.378471314907074)
+Step... (22625 | Loss: 0.037933044135570526, Learning Rate: 5.5305048590525985e-05, Gradient Norm: 0.6845491528511047)
+Step... (22650 | Loss: 0.03763023763895035, Learning Rate: 5.525454253074713e-05, Gradient Norm: 0.3493119180202484)
+Step... (22675 | Loss: 0.03691916540265083, Learning Rate: 5.520404010894708e-05, Gradient Norm: 0.3682186007499695)
+Step... (22700 | Loss: 0.02609202265739441, Learning Rate: 5.515353404916823e-05, Gradient Norm: 0.6833606362342834)
+Step... (22725 | Loss: 0.047541745007038116, Learning Rate: 5.5103027989389375e-05, Gradient Norm: 0.4514157474040985)
+Step... (22750 | Loss: 0.033493347465991974, Learning Rate: 5.505252192961052e-05, Gradient Norm: 0.3551611006259918)
+Step... (22775 | Loss: 0.04429641366004944, Learning Rate: 5.5002019507810473e-05, Gradient Norm: 0.4048822522163391)
+Step... (22800 | Loss: 0.04791601374745369, Learning Rate: 5.495150981005281e-05, Gradient Norm: 0.44947221875190735)
+Step... (22825 | Loss: 0.031418006867170334, Learning Rate: 5.4901007388252765e-05, Gradient Norm: 0.44471606612205505)
+Step... (22850 | Loss: 0.03244362398982048, Learning Rate: 5.485050496645272e-05, Gradient Norm: 0.4113829433917999)
+Step... (22875 | Loss: 0.03053554892539978, Learning Rate: 5.4799995268695056e-05, Gradient Norm: 0.3072253167629242)
+Step... (22900 | Loss: 0.04872916638851166, Learning Rate: 5.474949284689501e-05, Gradient Norm: 0.5141371488571167)
+Step... (22925 | Loss: 0.03374813497066498, Learning Rate: 5.469899042509496e-05, Gradient Norm: 0.5266069173812866)
+Step... (22950 | Loss: 0.031113576143980026, Learning Rate: 5.46484807273373e-05, Gradient Norm: 0.3346673548221588)
+Step... (22975 | Loss: 0.03883219510316849, Learning Rate: 5.4597978305537254e-05, Gradient Norm: 0.3280452489852905)
+Step... (23000 | Loss: 0.025303302332758904, Learning Rate: 5.4547475883737206e-05, Gradient Norm: 0.26770514249801636)
+Step... (23025 | Loss: 0.03750527650117874, Learning Rate: 5.4496966185979545e-05, Gradient Norm: 0.29126110672950745)
+Step... (23050 | Loss: 0.04249219223856926, Learning Rate: 5.44464637641795e-05, Gradient Norm: 0.374131441116333)
+Step... (23075 | Loss: 0.04200226441025734, Learning Rate: 5.439596134237945e-05, Gradient Norm: 0.43781113624572754)
+Step... (23100 | Loss: 0.047300681471824646, Learning Rate: 5.434545164462179e-05, Gradient Norm: 0.40795448422431946)
+Step... (23125 | Loss: 0.049821797758340836, Learning Rate: 5.429494922282174e-05, Gradient Norm: 0.375959187746048)
+Step... (23150 | Loss: 0.02656714618206024, Learning Rate: 5.4244446801021695e-05, Gradient Norm: 0.376505583524704)
+Step... (23175 | Loss: 0.03251540660858154, Learning Rate: 5.4193937103264034e-05, Gradient Norm: 0.327536940574646)
+Step... (23200 | Loss: 0.029152745380997658, Learning Rate: 5.414343468146399e-05, Gradient Norm: 0.39311301708221436)
+Step... (23225 | Loss: 0.03586068004369736, Learning Rate: 5.409292862168513e-05, Gradient Norm: 0.36502012610435486)
+Step... (23250 | Loss: 0.03459014371037483, Learning Rate: 5.404242256190628e-05, Gradient Norm: 0.4572829306125641)
+Step... (23275 | Loss: 0.04172315075993538, Learning Rate: 5.399192014010623e-05, Gradient Norm: 0.3692755401134491)
+Step... (23300 | Loss: 0.02289118990302086, Learning Rate: 5.394141408032738e-05, Gradient Norm: 0.36271020770072937)
+Step... (23325 | Loss: 0.02884475328028202, Learning Rate: 5.389090802054852e-05, Gradient Norm: 0.4460407793521881)
+Step... (23350 | Loss: 0.033021628856658936, Learning Rate: 5.384040196076967e-05, Gradient Norm: 0.3724048435688019)
+Step... (23375 | Loss: 0.057164158672094345, Learning Rate: 5.378989953896962e-05, Gradient Norm: 0.3597463369369507)
+Step... (23400 | Loss: 0.020201342180371284, Learning Rate: 5.373938984121196e-05, Gradient Norm: 0.28142717480659485)
+Step... (23425 | Loss: 0.022390734404325485, Learning Rate: 5.368888741941191e-05, Gradient Norm: 0.3492378890514374)
+Step... (23450 | Loss: 0.03480976074934006, Learning Rate: 5.3638384997611865e-05, Gradient Norm: 0.37160617113113403)
+Step... (23475 | Loss: 0.027854222804307938, Learning Rate: 5.3587875299854204e-05, Gradient Norm: 0.2876638174057007)
+Step... (23500 | Loss: 0.04249017685651779, Learning Rate: 5.353737287805416e-05, Gradient Norm: 0.41824018955230713)
+Step... (23525 | Loss: 0.0388215109705925, Learning Rate: 5.348687045625411e-05, Gradient Norm: 0.32904794812202454)
+Step... (23550 | Loss: 0.024838045239448547, Learning Rate: 5.343636075849645e-05, Gradient Norm: 0.25835245847702026)
+Step... (23575 | Loss: 0.043581828474998474, Learning Rate: 5.33858583366964e-05, Gradient Norm: 0.3848859369754791)
+Step... (23600 | Loss: 0.03622807189822197, Learning Rate: 5.3335355914896354e-05, Gradient Norm: 0.36198726296424866)
+Step... (23625 | Loss: 0.03490854427218437, Learning Rate: 5.328484621713869e-05, Gradient Norm: 0.33471915125846863)
+Step... (23650 | Loss: 0.04838581383228302, Learning Rate: 5.3234343795338646e-05, Gradient Norm: 7.672711372375488)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  79% 3459/4393 [4:33:00<1:30:43,  5.83s/it]
+Step... (23700 | Loss: 0.04192772135138512, Learning Rate: 5.313333167578094e-05, Gradient Norm: 0.35688653588294983)
+Step... (23725 | Loss: 0.03448999300599098, Learning Rate: 5.308282925398089e-05, Gradient Norm: 0.3494829535484314)
+Step... (23750 | Loss: 0.02678181789815426, Learning Rate: 5.303232683218084e-05, Gradient Norm: 0.29735514521598816)
+Step... (23775 | Loss: 0.035396356135606766, Learning Rate: 5.298181713442318e-05, Gradient Norm: 0.27143338322639465)
+Step... (23800 | Loss: 0.023322325199842453, Learning Rate: 5.2931314712623134e-05, Gradient Norm: 0.2984119653701782)
+Step... (23825 | Loss: 0.037303145974874496, Learning Rate: 5.288080865284428e-05, Gradient Norm: 0.3529098331928253)
+Step... (23850 | Loss: 0.026544779539108276, Learning Rate: 5.2830302593065426e-05, Gradient Norm: 0.3024252653121948)
+Step... (23875 | Loss: 0.026226522400975227, Learning Rate: 5.277980017126538e-05, Gradient Norm: 0.242228701710701)
+Step... (23900 | Loss: 0.025965316221117973, Learning Rate: 5.2729294111486524e-05, Gradient Norm: 0.28718680143356323)
+Step... (23925 | Loss: 0.03736674785614014, Learning Rate: 5.267878805170767e-05, Gradient Norm: 0.45029348134994507)
+Step... (23950 | Loss: 0.05321226641535759, Learning Rate: 5.262827835395001e-05, Gradient Norm: 1.0017284154891968)
+Step... (23975 | Loss: 0.029298288747668266, Learning Rate: 5.257777957012877e-05, Gradient Norm: 0.552095890045166)
+Step... (24000 | Loss: 0.045580193400382996, Learning Rate: 5.252726987237111e-05, Gradient Norm: 0.6694679856300354)
+Step... (24025 | Loss: 0.038904935121536255, Learning Rate: 5.247676381259225e-05, Gradient Norm: 0.6009917259216309)
+Step... (24050 | Loss: 0.06289307773113251, Learning Rate: 5.242626502877101e-05, Gradient Norm: 0.5505169034004211)
+Step... (24075 | Loss: 0.03959391266107559, Learning Rate: 5.237575533101335e-05, Gradient Norm: 0.3400160074234009)
+Step... (24100 | Loss: 0.022501932457089424, Learning Rate: 5.232524563325569e-05, Gradient Norm: 0.3429368734359741)
+Step... (24125 | Loss: 0.03071441687643528, Learning Rate: 5.227475048741326e-05, Gradient Norm: 0.679693877696991)
+Step... (24150 | Loss: 0.043432459235191345, Learning Rate: 5.2224240789655596e-05, Gradient Norm: 0.3760949671268463)
+Step... (24175 | Loss: 0.03176066651940346, Learning Rate: 5.2173731091897935e-05, Gradient Norm: 0.3627036213874817)
+Step... (24200 | Loss: 0.029321245849132538, Learning Rate: 5.21232359460555e-05, Gradient Norm: 0.2902766466140747)
+Step... (24225 | Loss: 0.031011415645480156, Learning Rate: 5.207272624829784e-05, Gradient Norm: 0.3715132474899292)
+Step... (24250 | Loss: 0.04117705672979355, Learning Rate: 5.202221655054018e-05, Gradient Norm: 0.3848026990890503)
+Step... (24275 | Loss: 0.035337090492248535, Learning Rate: 5.1971721404697746e-05, Gradient Norm: 0.35255369544029236)
+Step... (24300 | Loss: 0.03448829799890518, Learning Rate: 5.1921211706940085e-05, Gradient Norm: 0.36964043974876404)
+Step... (24325 | Loss: 0.05197534337639809, Learning Rate: 5.187070200918242e-05, Gradient Norm: 0.45723956823349)
+Step... (24350 | Loss: 0.05632118508219719, Learning Rate: 5.182020686333999e-05, Gradient Norm: 3.6309027671813965)
+Step... (24375 | Loss: 0.04953671619296074, Learning Rate: 5.176969716558233e-05, Gradient Norm: 1.7830593585968018)
+Step... (24400 | Loss: 0.03154705464839935, Learning Rate: 5.171918746782467e-05, Gradient Norm: 0.4155527651309967)
+Step... (24425 | Loss: 0.03356127068400383, Learning Rate: 5.166868504602462e-05, Gradient Norm: 0.31667256355285645)
+Step... (24450 | Loss: 0.035732828080654144, Learning Rate: 5.161818262422457e-05, Gradient Norm: 0.399539977312088)
+Step... (24475 | Loss: 0.036945439875125885, Learning Rate: 5.156767292646691e-05, Gradient Norm: 0.38079139590263367)
+Step... (24500 | Loss: 0.04229236766695976, Learning Rate: 5.1517170504666865e-05, Gradient Norm: 0.48107612133026123)
+Step... (24525 | Loss: 0.02834494039416313, Learning Rate: 5.146666808286682e-05, Gradient Norm: 0.2878149449825287)
+Step... (24550 | Loss: 0.028665006160736084, Learning Rate: 5.1416158385109156e-05, Gradient Norm: 0.44036924839019775)
+Step... (24575 | Loss: 0.04935942962765694, Learning Rate: 5.136565596330911e-05, Gradient Norm: 0.40158262848854065)
+Step... (24600 | Loss: 0.034180428832769394, Learning Rate: 5.1315149903530255e-05, Gradient Norm: 3.239790439605713)
+Step... (24625 | Loss: 0.043823905289173126, Learning Rate: 5.12646438437514e-05, Gradient Norm: 0.3326380252838135)
+Step... (24650 | Loss: 0.035595715045928955, Learning Rate: 5.1214137783972546e-05, Gradient Norm: 0.3283407688140869)
+Step... (24675 | Loss: 0.03204268217086792, Learning Rate: 5.11636353621725e-05, Gradient Norm: 0.3907879590988159)
+Step... (24700 | Loss: 0.0314224548637867, Learning Rate: 5.111312566441484e-05, Gradient Norm: 0.3861796259880066)
+Step... (24725 | Loss: 0.02714911662042141, Learning Rate: 5.106262324261479e-05, Gradient Norm: 0.2878110110759735)
+Step... (24750 | Loss: 0.03068208508193493, Learning Rate: 5.1012120820814744e-05, Gradient Norm: 0.38559654355049133)
+Step... (24775 | Loss: 0.03362426161766052, Learning Rate: 5.096161112305708e-05, Gradient Norm: 0.327539324760437)
+Step... (24800 | Loss: 0.04312548413872719, Learning Rate: 5.0911108701257035e-05, Gradient Norm: 0.5235893130302429)
+Step... (24825 | Loss: 0.055415429174900055, Learning Rate: 5.086060627945699e-05, Gradient Norm: 0.3906141221523285)
+Step... (24850 | Loss: 0.0417981818318367, Learning Rate: 5.081009658169933e-05, Gradient Norm: 0.36772969365119934)
+Step... (24875 | Loss: 0.027505910024046898, Learning Rate: 5.075959415989928e-05, Gradient Norm: 0.48655566573143005)
+Step... (24900 | Loss: 0.04489564523100853, Learning Rate: 5.070909173809923e-05, Gradient Norm: 0.39025452733039856)
+Step... (24925 | Loss: 0.03899327293038368, Learning Rate: 5.065858204034157e-05, Gradient Norm: 0.3016781806945801)
+Step... (24950 | Loss: 0.017189113423228264, Learning Rate: 5.0608079618541524e-05, Gradient Norm: 0.29946625232696533)
+Step... (24975 | Loss: 0.045142460614442825, Learning Rate: 5.055757719674148e-05, Gradient Norm: 0.373174250125885)
+Step... (25000 | Loss: 0.02781319059431553, Learning Rate: 5.0507067498983815e-05, Gradient Norm: 0.31940504908561707)
+Step... (25025 | Loss: 0.03962790593504906, Learning Rate: 5.045656507718377e-05, Gradient Norm: 0.3029711842536926)
+Step... (25050 | Loss: 0.03775897994637489, Learning Rate: 5.040606265538372e-05, Gradient Norm: 0.43249836564064026)
+Step... (25075 | Loss: 0.04975719004869461, Learning Rate: 5.035555295762606e-05, Gradient Norm: 0.33092451095581055)
+Step... (25100 | Loss: 0.05254314839839935, Learning Rate: 5.030505053582601e-05, Gradient Norm: 0.43144869804382324)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Step... (20000/50000 | Eval Loss: 1.021510124206543 | Eval wer: 0.05054961214661226 | Eval cer: 0.0362100285658818 |):  50% 6/12 [35:03:02<35:02:00, 21020.10s/it]
+Step... (25175 | Loss: 0.032494306564331055, Learning Rate: 5.015353599446826e-05, Gradient Norm: 0.25956735014915466)
+Step... (25200 | Loss: 0.027480341494083405, Learning Rate: 5.01030299346894e-05, Gradient Norm: 0.28603485226631165)
+Step... (25225 | Loss: 0.036957938224077225, Learning Rate: 5.005252387491055e-05, Gradient Norm: 0.3290182650089264)
+Step... (25250 | Loss: 0.03833230957388878, Learning Rate: 5.0002017815131694e-05, Gradient Norm: 0.38001635670661926)
+Step... (25275 | Loss: 0.02907697670161724, Learning Rate: 4.995151539333165e-05, Gradient Norm: 0.25869879126548767)
+Step... (25300 | Loss: 0.03300327807664871, Learning Rate: 4.9901005695573986e-05, Gradient Norm: 0.31227272748947144)
+Step... (25325 | Loss: 0.03420973941683769, Learning Rate: 4.985050327377394e-05, Gradient Norm: 0.3454754948616028)
+Step... (25350 | Loss: 0.022977076470851898, Learning Rate: 4.980000085197389e-05, Gradient Norm: 0.37838831543922424)
+Step... (25375 | Loss: 0.025919809937477112, Learning Rate: 4.974949115421623e-05, Gradient Norm: 0.2612961530685425)
+Step... (25400 | Loss: 0.04246160387992859, Learning Rate: 4.969898873241618e-05, Gradient Norm: 0.3997235894203186)
+Step... (25425 | Loss: 0.016840150579810143, Learning Rate: 4.9648486310616136e-05, Gradient Norm: 0.2279246747493744)
+Step... (25450 | Loss: 0.03332209214568138, Learning Rate: 4.9597976612858474e-05, Gradient Norm: 0.41686922311782837)
+Step... (25475 | Loss: 0.03769592195749283, Learning Rate: 4.954747419105843e-05, Gradient Norm: 0.36882728338241577)
+Step... (25500 | Loss: 0.026205237954854965, Learning Rate: 4.949697176925838e-05, Gradient Norm: 0.2956397533416748)
+Step... (25525 | Loss: 0.03775778040289879, Learning Rate: 4.944646207150072e-05, Gradient Norm: 0.41357094049453735)
+Step... (25550 | Loss: 0.042417582124471664, Learning Rate: 4.939595964970067e-05, Gradient Norm: 0.41240692138671875)
+Step... (25575 | Loss: 0.05006255954504013, Learning Rate: 4.9345457227900624e-05, Gradient Norm: 0.34435006976127625)
+Step... (25600 | Loss: 0.03163505718111992, Learning Rate: 4.929494753014296e-05, Gradient Norm: 0.33239689469337463)
+Step... (25625 | Loss: 0.028710873797535896, Learning Rate: 4.9244445108342916e-05, Gradient Norm: 0.3661336600780487)
+Step... (25650 | Loss: 0.026209404692053795, Learning Rate: 4.919394268654287e-05, Gradient Norm: 0.33108606934547424)
+Step... (25675 | Loss: 0.04295961931347847, Learning Rate: 4.914343298878521e-05, Gradient Norm: 0.4011436700820923)
+Step... (25700 | Loss: 0.03574109822511673, Learning Rate: 4.909293056698516e-05, Gradient Norm: 0.36241382360458374)
+Step... (25725 | Loss: 0.04625528305768967, Learning Rate: 4.90424208692275e-05, Gradient Norm: 0.3794752061367035)
+Step... (25750 | Loss: 0.04854767024517059, Learning Rate: 4.899191844742745e-05, Gradient Norm: 0.3879258930683136)
+Step... (25775 | Loss: 0.035202860832214355, Learning Rate: 4.8941416025627404e-05, Gradient Norm: 0.3439372777938843)
+Step... (25800 | Loss: 0.0253062155097723, Learning Rate: 4.889090632786974e-05, Gradient Norm: 0.3394845128059387)
+Step... (25825 | Loss: 0.0289002638310194, Learning Rate: 4.8840403906069696e-05, Gradient Norm: 0.29701897501945496)
+Step... (25850 | Loss: 0.0273846834897995, Learning Rate: 4.878989784629084e-05, Gradient Norm: 0.2742794454097748)
+Step... (25875 | Loss: 0.0390518382191658, Learning Rate: 4.873939178651199e-05, Gradient Norm: 1.3369203805923462)
+Step... (25900 | Loss: 0.03895574063062668, Learning Rate: 4.868888572673313e-05, Gradient Norm: 0.42404425144195557)
+Step... (25925 | Loss: 0.021992146968841553, Learning Rate: 4.8638383304933086e-05, Gradient Norm: 0.28285321593284607)
+Step... (25950 | Loss: 0.03738517314195633, Learning Rate: 4.8587873607175425e-05, Gradient Norm: 0.3228248655796051)
+Step... (25975 | Loss: 0.0590066984295845, Learning Rate: 4.853737118537538e-05, Gradient Norm: 0.40467754006385803)
+Step... (26000 | Loss: 0.018190395087003708, Learning Rate: 4.848686876357533e-05, Gradient Norm: 0.2537316083908081)
+Step... (26025 | Loss: 0.043122727423906326, Learning Rate: 4.843635906581767e-05, Gradient Norm: 0.45839107036590576)
+Step... (26050 | Loss: 0.034402068704366684, Learning Rate: 4.838585664401762e-05, Gradient Norm: 0.3322671949863434)
+Step... (26075 | Loss: 0.03683105483651161, Learning Rate: 4.8335354222217575e-05, Gradient Norm: 0.3035377860069275)
+Step... (26100 | Loss: 0.0251045860350132, Learning Rate: 4.828484452445991e-05, Gradient Norm: 0.35312619805336)
+Step... (26125 | Loss: 0.06226085126399994, Learning Rate: 4.8234342102659866e-05, Gradient Norm: 0.409700870513916)
+Step... (26150 | Loss: 0.060523878782987595, Learning Rate: 4.818383968085982e-05, Gradient Norm: 0.48786985874176025)
+Step... (26175 | Loss: 0.026466630399227142, Learning Rate: 4.813332998310216e-05, Gradient Norm: 0.39357778429985046)
+Step... (26200 | Loss: 0.051018837839365005, Learning Rate: 4.808282756130211e-05, Gradient Norm: 0.49015292525291443)
+Step... (26225 | Loss: 0.04816717654466629, Learning Rate: 4.8032325139502063e-05, Gradient Norm: 0.3389943242073059)
+Step... (26250 | Loss: 0.04806462675333023, Learning Rate: 4.79818154417444e-05, Gradient Norm: 0.39536547660827637)
+Step... (26275 | Loss: 0.03313664346933365, Learning Rate: 4.7931313019944355e-05, Gradient Norm: 0.3253754675388336)
+Step... (26300 | Loss: 0.031168362125754356, Learning Rate: 4.788081059814431e-05, Gradient Norm: 0.3615038990974426)
+Step... (26325 | Loss: 0.02266540192067623, Learning Rate: 4.7830300900386646e-05, Gradient Norm: 0.29001644253730774)
+Step... (26350 | Loss: 0.043101128190755844, Learning Rate: 4.77797984785866e-05, Gradient Norm: 0.5296374559402466)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  40% 1742/4393 [2:17:29<2:51:06,  3.87s/it]
+Step... (26375 | Loss: 0.015395677648484707, Learning Rate: 4.772929605678655e-05, Gradient Norm: 1.1583268642425537)
+Step... (26400 | Loss: 0.009922747500240803, Learning Rate: 4.767878635902889e-05, Gradient Norm: 0.23366865515708923)
+Step... (26425 | Loss: 0.040154799818992615, Learning Rate: 4.7628283937228844e-05, Gradient Norm: 0.45613422989845276)
+Step... (26450 | Loss: 0.02349100075662136, Learning Rate: 4.757777787744999e-05, Gradient Norm: 0.38283392786979675)
+Step... (26475 | Loss: 0.03603263571858406, Learning Rate: 4.7527271817671135e-05, Gradient Norm: 0.3727433979511261)
+Step... (26500 | Loss: 0.018448349088430405, Learning Rate: 4.747676575789228e-05, Gradient Norm: 1.0056108236312866)
+Step... (26525 | Loss: 0.02613736502826214, Learning Rate: 4.7426263336092234e-05, Gradient Norm: 0.33278515934944153)
+Step... (26550 | Loss: 0.021993344649672508, Learning Rate: 4.737575363833457e-05, Gradient Norm: 0.3807253837585449)
+Step... (26575 | Loss: 0.018662679940462112, Learning Rate: 4.7325251216534525e-05, Gradient Norm: 0.2747914493083954)
+Step... (26600 | Loss: 0.015389377251267433, Learning Rate: 4.727474879473448e-05, Gradient Norm: 0.30432483553886414)
+Step... (26625 | Loss: 0.018875302746891975, Learning Rate: 4.722423909697682e-05, Gradient Norm: 0.21066243946552277)
+Step... (26650 | Loss: 0.013026373460888863, Learning Rate: 4.717373667517677e-05, Gradient Norm: 0.33200427889823914)
+Step... (26675 | Loss: 0.03511090576648712, Learning Rate: 4.712323425337672e-05, Gradient Norm: 0.4361608326435089)
+Step... (26700 | Loss: 0.019190248101949692, Learning Rate: 4.707272455561906e-05, Gradient Norm: 0.9028236865997314)
+Step... (26725 | Loss: 0.03395620360970497, Learning Rate: 4.7022222133819014e-05, Gradient Norm: 0.3553292155265808)
+Step... (26750 | Loss: 0.014008201658725739, Learning Rate: 4.697171243606135e-05, Gradient Norm: 0.3180076479911804)
+Step... (26775 | Loss: 0.021352039650082588, Learning Rate: 4.6921210014261305e-05, Gradient Norm: 0.34052228927612305)
+Step... (26800 | Loss: 0.01608300395309925, Learning Rate: 4.687070759246126e-05, Gradient Norm: 0.43110185861587524)
+Step... (26825 | Loss: 0.04146311432123184, Learning Rate: 4.68201978947036e-05, Gradient Norm: 0.3803328573703766)
+Step... (26850 | Loss: 0.01878603734076023, Learning Rate: 4.676969547290355e-05, Gradient Norm: 1.2297662496566772)
+Step... (26875 | Loss: 0.024104727432131767, Learning Rate: 4.67191930511035e-05, Gradient Norm: 0.32827281951904297)
+Step... (26900 | Loss: 0.006425368599593639, Learning Rate: 4.666868335334584e-05, Gradient Norm: 0.19023503363132477)
+Step... (26925 | Loss: 0.03244623914361, Learning Rate: 4.6618180931545794e-05, Gradient Norm: 0.4636324346065521)
+Step... (26950 | Loss: 0.018195796757936478, Learning Rate: 4.656767850974575e-05, Gradient Norm: 0.3885265588760376)
+Step... (26975 | Loss: 0.018303479999303818, Learning Rate: 4.6517168811988086e-05, Gradient Norm: 0.22915388643741608)
+Step... (27000 | Loss: 0.01953260228037834, Learning Rate: 4.646666639018804e-05, Gradient Norm: 0.3849063813686371)
+Step... (27025 | Loss: 0.024017462506890297, Learning Rate: 4.641616396838799e-05, Gradient Norm: 0.29185715317726135)
+Step... (27050 | Loss: 0.02381003648042679, Learning Rate: 4.636565427063033e-05, Gradient Norm: 0.3834833800792694)
+Step... (27075 | Loss: 0.02851175330579281, Learning Rate: 4.631515184883028e-05, Gradient Norm: 0.33930420875549316)
+Step... (27100 | Loss: 0.029051026329398155, Learning Rate: 4.626464578905143e-05, Gradient Norm: 0.4754555821418762)
+Step... (27125 | Loss: 0.027669742703437805, Learning Rate: 4.6214139729272574e-05, Gradient Norm: 0.23227062821388245)
+Step... (27150 | Loss: 0.02711222507059574, Learning Rate: 4.616363366949372e-05, Gradient Norm: 0.3472621738910675)
+Step... (27175 | Loss: 0.030751440674066544, Learning Rate: 4.611313124769367e-05, Gradient Norm: 0.37772148847579956)
+Step... (27200 | Loss: 0.02963493950664997, Learning Rate: 4.606262154993601e-05, Gradient Norm: 0.3551703989505768)
+Step... (27225 | Loss: 0.034281305968761444, Learning Rate: 4.6012119128135964e-05, Gradient Norm: 0.25948455929756165)
+Step... (27250 | Loss: 0.01915004849433899, Learning Rate: 4.596161670633592e-05, Gradient Norm: 0.30437567830085754)
+Step... (27275 | Loss: 0.026445778086781502, Learning Rate: 4.5911107008578256e-05, Gradient Norm: 0.348853200674057)
+Step... (27300 | Loss: 0.018407680094242096, Learning Rate: 4.586060458677821e-05, Gradient Norm: 0.40228012204170227)
+Step... (27325 | Loss: 0.016526585444808006, Learning Rate: 4.581010216497816e-05, Gradient Norm: 0.2467903345823288)
+Step... (27350 | Loss: 0.03916558250784874, Learning Rate: 4.57595924672205e-05, Gradient Norm: 0.3872826099395752)
+Step... (27375 | Loss: 0.027904687449336052, Learning Rate: 4.570909004542045e-05, Gradient Norm: 0.29256585240364075)
+Step... (27400 | Loss: 0.013314536772668362, Learning Rate: 4.5658587623620406e-05, Gradient Norm: 0.49819448590278625)
+Step... (27425 | Loss: 0.02740517258644104, Learning Rate: 4.5608077925862744e-05, Gradient Norm: 0.3470984995365143)
+Step... (27450 | Loss: 0.011915629729628563, Learning Rate: 4.55575755040627e-05, Gradient Norm: 0.28347650170326233)
+Step... (27475 | Loss: 0.037782225757837296, Learning Rate: 4.550707308226265e-05, Gradient Norm: 0.603645384311676)
+Step... (27500 | Loss: 0.020660242065787315, Learning Rate: 4.545656338450499e-05, Gradient Norm: 0.40294796228408813)
+Step... (27525 | Loss: 0.019880592823028564, Learning Rate: 4.540606096270494e-05, Gradient Norm: 0.262466698884964)
+Step... (27550 | Loss: 0.02121094986796379, Learning Rate: 4.5355558540904894e-05, Gradient Norm: 0.3522772789001465)
+Step... (27575 | Loss: 0.020857077091932297, Learning Rate: 4.530504884314723e-05, Gradient Norm: 0.35182562470436096)
+Step... (27600 | Loss: 0.01385863684117794, Learning Rate: 4.5254546421347186e-05, Gradient Norm: 0.2593011260032654)
+Step... (27625 | Loss: 0.027512112632393837, Learning Rate: 4.520404399954714e-05, Gradient Norm: 0.4048057496547699)
+Step... (27650 | Loss: 0.04779520258307457, Learning Rate: 4.515353430178948e-05, Gradient Norm: 0.5996983051300049)
+Step... (27675 | Loss: 0.037076059728860855, Learning Rate: 4.510303187998943e-05, Gradient Norm: 0.4130019247531891)
+Step... (27700 | Loss: 0.0145907336845994, Learning Rate: 4.5052525820210576e-05, Gradient Norm: 0.26569056510925293)
+Step... (27725 | Loss: 0.026986515149474144, Learning Rate: 4.500201976043172e-05, Gradient Norm: 0.3900778591632843)
+Step... (27750 | Loss: 0.021214786916971207, Learning Rate: 4.495151370065287e-05, Gradient Norm: 0.35070163011550903)
+Step... (27775 | Loss: 0.020700503140687943, Learning Rate: 4.490100764087401e-05, Gradient Norm: 0.23256829380989075)
+Step... (27800 | Loss: 0.011971977539360523, Learning Rate: 4.485050158109516e-05, Gradient Norm: 0.3890122175216675)
+Step... (27825 | Loss: 0.029216863214969635, Learning Rate: 4.479999915929511e-05, Gradient Norm: 0.3727366030216217)
+Step... (27850 | Loss: 0.021556446328759193, Learning Rate: 4.474948946153745e-05, Gradient Norm: 0.3240383565425873)
+Step... (27875 | Loss: 0.024696046486496925, Learning Rate: 4.4698987039737403e-05, Gradient Norm: 0.30146247148513794)
+Step... (27900 | Loss: 0.0251696165651083, Learning Rate: 4.4648484617937356e-05, Gradient Norm: 0.37362998723983765)
+Step... (27925 | Loss: 0.015774782747030258, Learning Rate: 4.4597974920179695e-05, Gradient Norm: 0.24134306609630585)
+Step... (27950 | Loss: 0.014067770913243294, Learning Rate: 4.454747249837965e-05, Gradient Norm: 0.3240377604961395)
+Step... (27975 | Loss: 0.019330274313688278, Learning Rate: 4.44969700765796e-05, Gradient Norm: 0.27255779504776)
+Step... (28000 | Loss: 0.022736601531505585, Learning Rate: 4.444646037882194e-05, Gradient Norm: 0.36978673934936523)
+Step... (28025 | Loss: 0.030382484197616577, Learning Rate: 4.439595795702189e-05, Gradient Norm: 0.45521456003189087)
+Step... (28050 | Loss: 0.025521015748381615, Learning Rate: 4.4345455535221845e-05, Gradient Norm: 0.40030258893966675)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  79% 3466/4393 [4:33:16<1:29:13,  5.77s/it]
+Step... (28100 | Loss: 0.017779188230633736, Learning Rate: 4.4244443415664136e-05, Gradient Norm: 0.3853922486305237)
+Step... (28125 | Loss: 0.029410462826490402, Learning Rate: 4.419394099386409e-05, Gradient Norm: 0.41001659631729126)
+Step... (28150 | Loss: 0.016860324889421463, Learning Rate: 4.414343129610643e-05, Gradient Norm: 0.3353402614593506)
+Step... (28175 | Loss: 0.1199326440691948, Learning Rate: 4.409292887430638e-05, Gradient Norm: 1.177916407585144)
+Step... (28200 | Loss: 0.009715979918837547, Learning Rate: 4.4042426452506334e-05, Gradient Norm: 0.23744286596775055)
+Step... (28225 | Loss: 0.023713188245892525, Learning Rate: 4.399191675474867e-05, Gradient Norm: 0.27059558033943176)
+Step... (28250 | Loss: 0.015286052599549294, Learning Rate: 4.3941414332948625e-05, Gradient Norm: 0.3356326222419739)
+Step... (28275 | Loss: 0.03733113408088684, Learning Rate: 4.389091191114858e-05, Gradient Norm: 0.5076051950454712)
+Step... (28300 | Loss: 0.024148428812623024, Learning Rate: 4.3840402213390917e-05, Gradient Norm: 0.40085557103157043)
+Step... (28325 | Loss: 0.024370620027184486, Learning Rate: 4.378989979159087e-05, Gradient Norm: 0.27008843421936035)
+Step... (28350 | Loss: 0.12309176474809647, Learning Rate: 4.3739393731812015e-05, Gradient Norm: 0.5491603016853333)
+Step... (28375 | Loss: 0.020386263728141785, Learning Rate: 4.368888767203316e-05, Gradient Norm: 0.2690469026565552)
+Step... (28400 | Loss: 0.018853355199098587, Learning Rate: 4.363838161225431e-05, Gradient Norm: 0.40446701645851135)
+Step... (28425 | Loss: 0.029719140380620956, Learning Rate: 4.358787919045426e-05, Gradient Norm: 0.30341053009033203)
+Step... (28450 | Loss: 0.03215150907635689, Learning Rate: 4.35373694926966e-05, Gradient Norm: 0.6584086418151855)
+Step... (28475 | Loss: 0.019242705777287483, Learning Rate: 4.348686707089655e-05, Gradient Norm: 0.3172548711299896)
+Step... (28500 | Loss: 0.034587059170007706, Learning Rate: 4.3436364649096504e-05, Gradient Norm: 0.4526202380657196)
+Step... (28525 | Loss: 0.03248746320605278, Learning Rate: 4.338585495133884e-05, Gradient Norm: 0.3442862331867218)
+Step... (28550 | Loss: 0.017472591251134872, Learning Rate: 4.3335352529538795e-05, Gradient Norm: 0.33737820386886597)
+Step... (28575 | Loss: 0.03616875782608986, Learning Rate: 4.328485010773875e-05, Gradient Norm: 0.44748085737228394)
+Step... (28600 | Loss: 0.010958717204630375, Learning Rate: 4.323434040998109e-05, Gradient Norm: 0.3348379135131836)
+Step... (28625 | Loss: 0.021889686584472656, Learning Rate: 4.318383798818104e-05, Gradient Norm: 0.291972815990448)
+Step... (28650 | Loss: 0.018253032118082047, Learning Rate: 4.313333556638099e-05, Gradient Norm: 0.39204180240631104)
+Step... (28675 | Loss: 0.03230177238583565, Learning Rate: 4.308282586862333e-05, Gradient Norm: 0.31189021468162537)
+Step... (28700 | Loss: 0.025849582627415657, Learning Rate: 4.3032323446823284e-05, Gradient Norm: 0.5001385807991028)
+Step... (28725 | Loss: 0.02958354540169239, Learning Rate: 4.298182102502324e-05, Gradient Norm: 0.3614922761917114)
+Step... (28750 | Loss: 0.02733941376209259, Learning Rate: 4.2931311327265576e-05, Gradient Norm: 0.3756287693977356)
+Step... (28775 | Loss: 0.029227962717413902, Learning Rate: 4.288080890546553e-05, Gradient Norm: 0.37987276911735535)
+Step... (28800 | Loss: 0.02478295937180519, Learning Rate: 4.283030648366548e-05, Gradient Norm: 0.43760961294174194)
+Step... (28825 | Loss: 0.02181088924407959, Learning Rate: 4.277979678590782e-05, Gradient Norm: 0.3122001886367798)
+Step... (28850 | Loss: 0.027777288109064102, Learning Rate: 4.272929436410777e-05, Gradient Norm: 0.39810433983802795)
+Step... (28875 | Loss: 0.038595400750637054, Learning Rate: 4.267878466635011e-05, Gradient Norm: 0.38269856572151184)
+Step... (28900 | Loss: 0.03247010335326195, Learning Rate: 4.2628282244550064e-05, Gradient Norm: 0.4291834831237793)
+Step... (28925 | Loss: 0.014019368216395378, Learning Rate: 4.257777982275002e-05, Gradient Norm: 0.2331201434135437)
+Step... (28950 | Loss: 0.017961695790290833, Learning Rate: 4.2527270124992356e-05, Gradient Norm: 0.3652317225933075)
+Step... (28975 | Loss: 0.03309032693505287, Learning Rate: 4.247676770319231e-05, Gradient Norm: 0.43095332384109497)
+Step... (29000 | Loss: 0.01431436650454998, Learning Rate: 4.2426261643413454e-05, Gradient Norm: 0.3215678930282593)
+Step... (29025 | Loss: 0.030018476769328117, Learning Rate: 4.23757555836346e-05, Gradient Norm: 0.41584596037864685)
+Step... (29050 | Loss: 0.02051413059234619, Learning Rate: 4.2325249523855746e-05, Gradient Norm: 0.43135276436805725)
+Step... (29075 | Loss: 0.03962719812989235, Learning Rate: 4.22747471020557e-05, Gradient Norm: 0.46567052602767944)
+Step... (29100 | Loss: 0.02515358477830887, Learning Rate: 4.2224241042276844e-05, Gradient Norm: 1.7703990936279297)
+Step... (29125 | Loss: 0.0183755811303854, Learning Rate: 4.217373498249799e-05, Gradient Norm: 0.46105656027793884)
+Step... (29150 | Loss: 0.023963920772075653, Learning Rate: 4.212323256069794e-05, Gradient Norm: 0.3423953354358673)
+Step... (29175 | Loss: 0.02140207402408123, Learning Rate: 4.207272286294028e-05, Gradient Norm: 0.29520702362060547)
+Step... (29200 | Loss: 0.03680615872144699, Learning Rate: 4.2022220441140234e-05, Gradient Norm: 1.6429659128189087)
+Step... (29225 | Loss: 0.03201254457235336, Learning Rate: 4.197171801934019e-05, Gradient Norm: 0.355790913105011)
+Step... (29250 | Loss: 0.01865505427122116, Learning Rate: 4.1921208321582526e-05, Gradient Norm: 0.37935853004455566)
+Step... (29275 | Loss: 0.04450344666838646, Learning Rate: 4.187070589978248e-05, Gradient Norm: 0.42739537358283997)
+Step... (29300 | Loss: 0.02708597481250763, Learning Rate: 4.182020347798243e-05, Gradient Norm: 0.4142377972602844)
+Step... (29325 | Loss: 0.017092134803533554, Learning Rate: 4.176969378022477e-05, Gradient Norm: 0.2180645912885666)
+Step... (29350 | Loss: 0.02320004440844059, Learning Rate: 4.171919135842472e-05, Gradient Norm: 0.4315558671951294)
+Step... (29375 | Loss: 0.022841814905405045, Learning Rate: 4.1668688936624676e-05, Gradient Norm: 0.2843684256076813)
+Step... (29400 | Loss: 0.012752613984048367, Learning Rate: 4.1618179238867015e-05, Gradient Norm: 0.2700633704662323)
+Step... (29425 | Loss: 0.032406099140644073, Learning Rate: 4.156767681706697e-05, Gradient Norm: 0.3672173023223877)
+Step... (29450 | Loss: 0.013478636741638184, Learning Rate: 4.151717439526692e-05, Gradient Norm: 0.36269333958625793)
+Step... (29475 | Loss: 0.02877821959555149, Learning Rate: 4.146666469750926e-05, Gradient Norm: 1.2450404167175293)
+Step... (29500 | Loss: 0.026545371860265732, Learning Rate: 4.141616227570921e-05, Gradient Norm: 0.7389810085296631)
+Step... (29525 | Loss: 0.04219873994588852, Learning Rate: 4.1365659853909165e-05, Gradient Norm: 0.48379987478256226)
+Step... (29550 | Loss: 0.025202322751283646, Learning Rate: 4.13151501561515e-05, Gradient Norm: 0.36057695746421814)
+Step... (29575 | Loss: 0.036751795560121536, Learning Rate: 4.1264647734351456e-05, Gradient Norm: 0.3288840651512146)
+Step... (29600 | Loss: 0.04314383491873741, Learning Rate: 4.12141416745726e-05, Gradient Norm: 1.1218085289001465)
+Step... (29625 | Loss: 0.02685457281768322, Learning Rate: 4.116363561479375e-05, Gradient Norm: 0.3273230791091919)
+Step... (29650 | Loss: 0.017800813540816307, Learning Rate: 4.1113129555014893e-05, Gradient Norm: 0.41265296936035156)
+Step... (29675 | Loss: 0.023478511720895767, Learning Rate: 4.1062627133214846e-05, Gradient Norm: 0.2817476987838745)
+Step... (29700 | Loss: 0.018604634329676628, Learning Rate: 4.1012117435457185e-05, Gradient Norm: 0.3600842356681824)
+Step... (29725 | Loss: 0.01911950670182705, Learning Rate: 4.096161501365714e-05, Gradient Norm: 0.24088819324970245)
+Step... (29750 | Loss: 0.010041565634310246, Learning Rate: 4.091111259185709e-05, Gradient Norm: 0.26652786135673523)
+Step... (29775 | Loss: 0.020387249067425728, Learning Rate: 4.086060289409943e-05, Gradient Norm: 0.2445187121629715)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  83% 3641/4393 [4:47:16<48:57,  3.91s/it]
+Evaluating ...:   0% 0/85 [00:00<?, ?it/s]
+Step... (29825 | Loss: 0.01673395000398159, Learning Rate: 4.0759598050499335e-05, Gradient Norm: 0.21883606910705566)
+Step... (29850 | Loss: 0.03240591287612915, Learning Rate: 4.0709088352741674e-05, Gradient Norm: 0.40065884590148926)
+Step... (29875 | Loss: 0.019736234098672867, Learning Rate: 4.0658585930941626e-05, Gradient Norm: 0.32982364296913147)
+Step... (29900 | Loss: 0.018150361254811287, Learning Rate: 4.0608076233183965e-05, Gradient Norm: 0.387714684009552)
+Step... (29925 | Loss: 0.018563171848654747, Learning Rate: 4.055757381138392e-05, Gradient Norm: 0.22470852732658386)
+Step... (29950 | Loss: 0.01995537057518959, Learning Rate: 4.050707138958387e-05, Gradient Norm: 0.373696506023407)
+Step... (29975 | Loss: 0.02121199481189251, Learning Rate: 4.045656169182621e-05, Gradient Norm: 0.24509090185165405)
+/home/sanchitgandhi/hf/lib/python3.8/site-packages/flax/jax_utils.py:312: FutureWarning: jax.tree_map is deprecated, and will be removed in a future release. Use jax.tree_util.tree_map instead.
+  return jax.tree_map(pad, tree)
+/home/sanchitgandhi/hf/lib/python3.8/site-packages/flax/jax_utils.py:321: FutureWarning: jax.tree_map is deprecated, and will be removed in a future release. Use jax.tree_util.tree_map instead.
+  return out if static_return else jax.tree_map(unpad, out)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  device_metrics = jax.tree_map(lambda x: x[0], device_metrics)
+/home/sanchitgandhi/hf/lib/python3.8/site-packages/flax/training/common_utils.py:45: FutureWarning: jax.tree_map is deprecated, and will be removed in a future release. Use jax.tree_util.tree_map instead.
+  return jax.tree_map(stack_args, *forest)
+run_flax_speech_recognition_seq2seq.py:1392: FutureWarning: jax.tree_map is deprecated, and will be removed in a future release. Use jax.tree_util.tree_map instead.
+  eval_metrics = jax.tree_map(jnp.mean, eval_metrics)
+Step... (20000/50000 | Eval Loss: 1.021510124206543 | run_flax_speech_recognition_seq2seq.py:1425: FutureWarning: jax.tree_map is deprecated, and will be removed in a future release. Use jax.tree_util.tree_map instead.
+  params = jax.device_get(jax.tree_map(lambda x: x[0], state.params))
+Configuration saved in /home/sanchitgandhi/flax-wav2vec2-2-bart-large-ls-960h-black-box/config.json
+tcmalloc: large alloc 2586787840 bytes == 0x36da2e000 @  0x7f7cba873680 0x7f7cba893bdd 0x7f7b690721ff 0x7f7b6908142c 0x7f7b6908241d 0x7f7b6908241d 0x7f7b6908241d 0x7f7b6908241d 0x7f7b6908241d 0x7f7b6908241d 0x7f7b6908241d 0x7f7b6907c164 0x7f7b6907c91e 0x505166 0x56bbfa 0x569dba 0x5f6eb3 0x56cc1f 0x569dba 0x5f6eb3 0x56cc1f 0x5f6cd6 0x56bacd 0x569dba 0x50bca0 0x56cc1f 0x569dba 0x5f6eb3 0x56bacd 0x569dba 0x5f6eb3
+Model weights saved in /home/sanchitgandhi/flax-wav2vec2-2-bart-large-ls-960h-black-box/flax_model.msgpack
+tokenizer config file saved in ./tokenizer_config.json
+Special tokens file saved in ./special_tokens_map.json
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
 To disable this warning, you can either:
 	- Avoid using `tokenizers` before the fork if possible

wandb/run-20220828_085247-2hx8pk65/files/wandb-summary.json

@@ -1 +1 @@
-{"train/decoder_grad_norm": 0.24299649894237518, "train/decoder_param_norm": 1060.7144775390625, "train/encoder_grad_norm": 0.20758813619613647, "train/encoder_param_norm": 2320.3076171875, "train/grad_norm": 0.3195936977863312, "layer_grad_norm/": {"decoder": {"model": {"decoder": {"embed_positions": {"embedding": 0.015636462718248367}, "embed_tokens": {"embedding": 0.08405420929193497}, "layernorm_embedding": {"bias": 0.006145514082163572, "scale": 0.004968359600752592}, "layers": {"FlaxBartDecoderLayers": {"encoder_attn": {"k_proj": {"bias": 6.667326488241088e-06, "kernel": 0.014865289442241192}, "out_proj": {"bias": 0.01036052592098713, "kernel": 0.049787431955337524}, "q_proj": {"bias": 0.0007108663558028638, "kernel": 0.015181932598352432}, "v_proj": {"bias": 0.019716547802090645, "kernel": 0.03999572619795799}}, "encoder_attn_layer_norm": {"bias": 0.015086976811289787, "scale": 0.016625721007585526}, "fc1": {"bias": 0.0057020955719053745, "kernel": 0.14046090841293335}, "fc2": {"bias": 0.0147400489076972, "kernel": 0.12780840694904327}, "final_layer_norm": {"bias": 0.032790299504995346, "scale": 0.03732256218791008}, "self_attn": {"k_proj": {"bias": 2.526882781239692e-06, "kernel": 0.01344823744148016}, "out_proj": {"bias": 0.020945662632584572, "kernel": 0.0472058430314064}, "q_proj": {"bias": 0.001085575670003891, "kernel": 0.014002838172018528}, "v_proj": {"bias": 0.020695650950074196, "kernel": 0.06064840778708458}}, "self_attn_layer_norm": {"bias": 0.009373994544148445, "scale": 0.011083677411079407}}}}}}, "encoder": {"adapter": {"layers": {"0": {"conv": {"bias": 0.02808896079659462, "kernel": 0.06710191071033478}}, "1": {"conv": {"bias": 0.02258674055337906, "kernel": 0.046897199004888535}}, "2": {"conv": {"bias": 0.02590387687087059, "kernel": 0.07143399119377136}}}}, "encoder": {"layer_norm": {"bias": 0.09637241810560226, "scale": 0.0566645972430706}, "layers": {"FlaxWav2Vec2EncoderLayers": {"attention": {"k_proj": {"bias": 2.509430260033696e-06, "kernel": 0.019797371700406075}, "out_proj": {"bias": 0.0026633520610630512, "kernel": 0.042062435299158096}, "q_proj": {"bias": 0.003061411203816533, "kernel": 0.01928599737584591}, "v_proj": {"bias": 0.011404206976294518, "kernel": 0.04114246740937233}}, "feed_forward": {"intermediate_dense": {"bias": 0.006081512663513422, "kernel": 0.05225696042180061}, "output_dense": {"bias": 0.002437157789245248, "kernel": 0.045792415738105774}}, "final_layer_norm": {"bias": 0.031238090246915817, "scale": 0.03307477384805679}, "layer_norm": {"bias": 0.04994071274995804, "scale": 0.04129469022154808}}}, "pos_conv_embed": {"conv": {"bias": 0.0008167774649336934, "weight_g": 0.0036110610235482454, "weight_v": 0.012701401486992836}}}, "feature_extractor": {"conv_layers": {"0": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "1": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "2": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "3": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "4": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "5": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "6": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}}}, "feature_projection": {"layer_norm": {"bias": 0.0036688754335045815, "scale": 0.004511996638029814}, "projection": {"bias": 0.0011850270675495267, "kernel": 0.033120427280664444}}, "masked_spec_embed": 0.0}}, "layer_param_norm/": {"decoder": {"model": {"decoder": {"embed_positions": {"embedding": 58.625732421875}, "embed_tokens": {"embedding": 628.6316528320312}, "layernorm_embedding": {"bias": 2.4100914001464844, "scale": 13.898597717285156}, "layers": {"FlaxBartDecoderLayers": {"encoder_attn": {"k_proj": {"bias": 47.98827362060547, "kernel": 330.9457092285156}, "out_proj": {"bias": 6.15840482711792, "kernel": 227.3690948486328}, "q_proj": {"bias": 20.846906661987305, "kernel": 337.8941650390625}, "v_proj": {"bias": 3.635528802871704, "kernel": 231.59678649902344}}, "encoder_attn_layer_norm": {"bias": 10.790207862854004, "scale": 57.176551818847656}, "fc1": {"bias": 25.858234405517578, "kernel": 343.7656555175781}, "fc2": {"bias": 7.86165714263916, "kernel": 246.748291015625}, "final_layer_norm": {"bias": 3.946004867553711, "scale": 63.51469421386719}, "self_attn": {"k_proj": {"bias": 59.53274917602539, "kernel": 279.572998046875}, "out_proj": {"bias": 3.8042359352111816, "kernel": 132.37606811523438}, "q_proj": {"bias": 32.1674919128418, "kernel": 282.6742248535156}, "v_proj": {"bias": 2.5869321823120117, "kernel": 140.7692108154297}}, "self_attn_layer_norm": {"bias": 8.874273300170898, "scale": 84.6916732788086}}}}}}, "encoder": {"adapter": {"layers": {"0": {"conv": {"bias": 0.9373133182525635, "kernel": 60.65754699707031}}, "1": {"conv": {"bias": 1.0780301094055176, "kernel": 58.4447135925293}}, "2": {"conv": {"bias": 1.3075517416000366, "kernel": 58.37184524536133}}}}, "encoder": {"layer_norm": {"bias": 0.29216912388801575, "scale": 4.3043694496154785}, "layers": {"FlaxWav2Vec2EncoderLayers": {"attention": {"k_proj": {"bias": 19.379070281982422, "kernel": 552.49365234375}, "out_proj": {"bias": 16.84808349609375, "kernel": 704.6502075195312}, "q_proj": {"bias": 40.86162567138672, "kernel": 545.0123291015625}, "v_proj": {"bias": 15.593074798583984, "kernel": 696.20166015625}}, "feed_forward": {"intermediate_dense": {"bias": 24.4946231842041, "kernel": 1376.0777587890625}, "output_dense": {"bias": 20.806180953979492, "kernel": 1302.1185302734375}}, "final_layer_norm": {"bias": 32.5095329284668, "scale": 141.84854125976562}, "layer_norm": {"bias": 7.295251846313477, "scale": 45.60984420776367}}}, "pos_conv_embed": {"conv": {"bias": 15.245718002319336, "weight_g": 21.039236068725586, "weight_v": 213.549072265625}}}, "feature_extractor": {"conv_layers": {"0": {"conv": {"bias": 0.5982058644294739, "kernel": 8.08896541595459}, "layer_norm": {"bias": 10.069783210754395, "scale": 10.451257705688477}}, "1": {"conv": {"bias": 4.74075174331665, "kernel": 90.8435287475586}, "layer_norm": {"bias": 6.922820091247559, "scale": 19.5467586517334}}, "2": {"conv": {"bias": 6.7732415199279785, "kernel": 146.13897705078125}, "layer_norm": {"bias": 9.044225692749023, "scale": 19.424888610839844}}, "3": {"conv": {"bias": 5.224758148193359, "kernel": 159.10508728027344}, "layer_norm": {"bias": 8.319666862487793, "scale": 17.64743423461914}}, "4": {"conv": {"bias": 4.434978008270264, "kernel": 157.35813903808594}, "layer_norm": {"bias": 9.193974494934082, "scale": 15.562357902526855}}, "5": {"conv": {"bias": 5.297643661499023, "kernel": 131.1835174560547}, "layer_norm": {"bias": 10.735219955444336, "scale": 13.812533378601074}}, "6": {"conv": {"bias": 5.615579128265381, "kernel": 136.41822814941406}, "layer_norm": {"bias": 12.515308380126953, "scale": 11.152680397033691}}}}, "feature_projection": {"layer_norm": {"bias": 9.315728187561035, "scale": 27.725435256958008}, "projection": {"bias": 4.307735443115234, "kernel": 88.24262237548828}}, "masked_spec_embed": 26.247730255126953}}, "train/learning_rate": 6.065858542569913e-05, "train/loss": 0.04709053412079811, "train/param_norm": 2551.262939453125, "_timestamp": 1661775899, "_runtime": 99132, "_step": 19975, "eval/loss": 0.6138997077941895, "eval/wer": 0.05543913826697548, "eval/cer": 0.039964500651745845, "eval/step_10k": {"_type": "table-file", "sha256": "8b44e8a00a036a18ffdf81b4d076c8bf849ea6649001c69e94fa439b14f110ee", "size": 26434, "artifact_path": "wandb-client-artifact://18m0dj4hts3yiat04x5pvmncavkjapd5wb8bznb37vw8c0lqna3m2yjd1wtdrfstuoo7ejt2sphvjo0zuw1e5ne5d3qbkd7c1fylclfggig6us5tsmsj2uum5pchx48n:latest/eval/step_10k.table.json", "_latest_artifact_path": "wandb-client-artifact://18m0dj4hts3yiat04x5pvmncavkjapd5wb8bznb37vw8c0lqna3m2yjd1wtdrfstuoo7ejt2sphvjo0zuw1e5ne5d3qbkd7c1fylclfggig6us5tsmsj2uum5pchx48n:latest/eval/step_10k.table.json", "path": "media/table/eval/step_10k_10000_8b44e8a00a036a18ffdf.table.json", "ncols": 7, "nrows": 50}}
+{"train/decoder_grad_norm": 0.17611896991729736, "train/decoder_param_norm": 1062.4339599609375, "train/encoder_grad_norm": 0.1704455018043518, "train/encoder_param_norm": 2322.47119140625, "train/grad_norm": 0.24509090185165405, "layer_grad_norm/": {"decoder": {"model": {"decoder": {"embed_positions": {"embedding": 0.006716505624353886}, "embed_tokens": {"embedding": 0.0642298087477684}, "layernorm_embedding": {"bias": 0.002760232426226139, "scale": 0.002035953802987933}, "layers": {"FlaxBartDecoderLayers": {"encoder_attn": {"k_proj": {"bias": 4.814476142200874e-06, "kernel": 0.009996717795729637}, "out_proj": {"bias": 0.007056824862957001, "kernel": 0.035733651369810104}, "q_proj": {"bias": 0.00045872520422562957, "kernel": 0.010119627229869366}, "v_proj": {"bias": 0.014002962969243526, "kernel": 0.028897128999233246}}, "encoder_attn_layer_norm": {"bias": 0.010591115802526474, "scale": 0.01078032236546278}, "fc1": {"bias": 0.004002838861197233, "kernel": 0.09873048961162567}, "fc2": {"bias": 0.010417253710329533, "kernel": 0.09346094727516174}, "final_layer_norm": {"bias": 0.02925114333629608, "scale": 0.025131691247224808}, "self_attn": {"k_proj": {"bias": 1.495998731115833e-06, "kernel": 0.009474096819758415}, "out_proj": {"bias": 0.016716178506612778, "kernel": 0.03129136934876442}, "q_proj": {"bias": 0.0007107162964530289, "kernel": 0.00879898015409708}, "v_proj": {"bias": 0.01823665015399456, "kernel": 0.04575955122709274}}, "self_attn_layer_norm": {"bias": 0.006624831352382898, "scale": 0.0074623520486056805}}}}}}, "encoder": {"adapter": {"layers": {"0": {"conv": {"bias": 0.024443458765745163, "kernel": 0.04797496274113655}}, "1": {"conv": {"bias": 0.018976185470819473, "kernel": 0.03335544094443321}}, "2": {"conv": {"bias": 0.02025711163878441, "kernel": 0.05017269030213356}}}}, "encoder": {"layer_norm": {"bias": 0.09079575538635254, "scale": 0.0370769128203392}, "layers": {"FlaxWav2Vec2EncoderLayers": {"attention": {"k_proj": {"bias": 2.546356427046703e-06, "kernel": 0.018089286983013153}, "out_proj": {"bias": 0.002156489295884967, "kernel": 0.034899111837148666}, "q_proj": {"bias": 0.0026696091517806053, "kernel": 0.017847701907157898}, "v_proj": {"bias": 0.009628637693822384, "kernel": 0.03381096571683884}}, "feed_forward": {"intermediate_dense": {"bias": 0.004856355953961611, "kernel": 0.04390779137611389}, "output_dense": {"bias": 0.001986933406442404, "kernel": 0.03847503662109375}}, "final_layer_norm": {"bias": 0.02509871870279312, "scale": 0.028251897543668747}, "layer_norm": {"bias": 0.043583061546087265, "scale": 0.03362990543246269}}}, "pos_conv_embed": {"conv": {"bias": 0.0006979878526180983, "weight_g": 0.002805550117045641, "weight_v": 0.011207042261958122}}}, "feature_extractor": {"conv_layers": {"0": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "1": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "2": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "3": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "4": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "5": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "6": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}}}, "feature_projection": {"layer_norm": {"bias": 0.0032904886174947023, "scale": 0.003584273625165224}, "projection": {"bias": 0.0010678826365619898, "kernel": 0.03113219328224659}}, "masked_spec_embed": 0.0}}, "layer_param_norm/": {"decoder": {"model": {"decoder": {"embed_positions": {"embedding": 58.647159576416016}, "embed_tokens": {"embedding": 628.4832763671875}, "layernorm_embedding": {"bias": 2.4181270599365234, "scale": 13.876160621643066}, "layers": {"FlaxBartDecoderLayers": {"encoder_attn": {"k_proj": {"bias": 47.96188735961914, "kernel": 331.3609313964844}, "out_proj": {"bias": 6.137172698974609, "kernel": 227.73728942871094}, "q_proj": {"bias": 20.867891311645508, "kernel": 338.29461669921875}, "v_proj": {"bias": 3.5909812450408936, "kernel": 231.93850708007812}}, "encoder_attn_layer_norm": {"bias": 10.96327018737793, "scale": 57.53877639770508}, "fc1": {"bias": 26.014301300048828, "kernel": 345.999755859375}, "fc2": {"bias": 7.84496545791626, "kernel": 248.36624145507812}, "final_layer_norm": {"bias": 3.921175003051758, "scale": 63.50761413574219}, "self_attn": {"k_proj": {"bias": 59.543113708496094, "kernel": 279.90435791015625}, "out_proj": {"bias": 3.776594638824463, "kernel": 132.7462158203125}, "q_proj": {"bias": 32.19286346435547, "kernel": 283.0003662109375}, "v_proj": {"bias": 2.568603992462158, "kernel": 141.117431640625}}, "self_attn_layer_norm": {"bias": 8.895291328430176, "scale": 84.70443725585938}}}}}}, "encoder": {"adapter": {"layers": {"0": {"conv": {"bias": 1.1401727199554443, "kernel": 62.00471878051758}}, "1": {"conv": {"bias": 1.3068256378173828, "kernel": 59.942413330078125}}, "2": {"conv": {"bias": 1.5166486501693726, "kernel": 59.78715133666992}}}}, "encoder": {"layer_norm": {"bias": 0.2932465970516205, "scale": 4.228818893432617}, "layers": {"FlaxWav2Vec2EncoderLayers": {"attention": {"k_proj": {"bias": 19.381807327270508, "kernel": 553.2210083007812}, "out_proj": {"bias": 16.85451316833496, "kernel": 705.1119995117188}, "q_proj": {"bias": 40.90638732910156, "kernel": 545.735107421875}, "v_proj": {"bias": 15.575094223022461, "kernel": 696.6250610351562}}, "feed_forward": {"intermediate_dense": {"bias": 24.463790893554688, "kernel": 1377.1727294921875}, "output_dense": {"bias": 20.80949592590332, "kernel": 1303.4677734375}}, "final_layer_norm": {"bias": 32.52007293701172, "scale": 141.95835876464844}, "layer_norm": {"bias": 7.280069828033447, "scale": 45.696510314941406}}}, "pos_conv_embed": {"conv": {"bias": 15.224443435668945, "weight_g": 21.051162719726562, "weight_v": 213.89393615722656}}}, "feature_extractor": {"conv_layers": {"0": {"conv": {"bias": 0.5982058644294739, "kernel": 8.08896541595459}, "layer_norm": {"bias": 10.069783210754395, "scale": 10.451257705688477}}, "1": {"conv": {"bias": 4.74075174331665, "kernel": 90.8435287475586}, "layer_norm": {"bias": 6.922820091247559, "scale": 19.5467586517334}}, "2": {"conv": {"bias": 6.7732415199279785, "kernel": 146.13897705078125}, "layer_norm": {"bias": 9.044225692749023, "scale": 19.424888610839844}}, "3": {"conv": {"bias": 5.224758148193359, "kernel": 159.10508728027344}, "layer_norm": {"bias": 8.319666862487793, "scale": 17.64743423461914}}, "4": {"conv": {"bias": 4.434978008270264, "kernel": 157.35813903808594}, "layer_norm": {"bias": 9.193974494934082, "scale": 15.562357902526855}}, "5": {"conv": {"bias": 5.297643661499023, "kernel": 131.1835174560547}, "layer_norm": {"bias": 10.735219955444336, "scale": 13.812533378601074}}, "6": {"conv": {"bias": 5.615579128265381, "kernel": 136.41822814941406}, "layer_norm": {"bias": 12.515308380126953, "scale": 11.152680397033691}}}}, "feature_projection": {"layer_norm": {"bias": 9.262188911437988, "scale": 27.640396118164062}, "projection": {"bias": 4.317654132843018, "kernel": 88.17610931396484}}, "masked_spec_embed": 26.247730255126953}}, "train/learning_rate": 4.045656169182621e-05, "train/loss": 0.02121199481189251, "train/param_norm": 2553.945556640625, "_timestamp": 1661823749, "_runtime": 146982, "_step": 29975, "eval/loss": 1.021510124206543, "eval/wer": 0.05054961214661226, "eval/cer": 0.0362100285658818, "eval/step_10k": {"_type": "table-file", "sha256": "8b44e8a00a036a18ffdf81b4d076c8bf849ea6649001c69e94fa439b14f110ee", "size": 26434, "artifact_path": "wandb-client-artifact://18m0dj4hts3yiat04x5pvmncavkjapd5wb8bznb37vw8c0lqna3m2yjd1wtdrfstuoo7ejt2sphvjo0zuw1e5ne5d3qbkd7c1fylclfggig6us5tsmsj2uum5pchx48n:latest/eval/step_10k.table.json", "_latest_artifact_path": "wandb-client-artifact://18m0dj4hts3yiat04x5pvmncavkjapd5wb8bznb37vw8c0lqna3m2yjd1wtdrfstuoo7ejt2sphvjo0zuw1e5ne5d3qbkd7c1fylclfggig6us5tsmsj2uum5pchx48n:latest/eval/step_10k.table.json", "path": "media/table/eval/step_10k_10000_8b44e8a00a036a18ffdf.table.json", "ncols": 7, "nrows": 50}, "eval/step_20k": {"_type": "table-file", "sha256": "a0a50c5d8793ca99e4646f70c3624f8742c5285825bc1c59ab4083ac4de9d6e3", "size": 26657, "artifact_path": "wandb-client-artifact://13ri9hnxp93kf7dsdol2hs1j0v7bpkwwvujpi27awdck0fjm6vfog0dun9k9toif5xrt3cijlotddakikiw0bnbo3go679b4d2spq9c0w865vq0k9auiszkkbvev62fc:latest/eval/step_20k.table.json", "_latest_artifact_path": "wandb-client-artifact://13ri9hnxp93kf7dsdol2hs1j0v7bpkwwvujpi27awdck0fjm6vfog0dun9k9toif5xrt3cijlotddakikiw0bnbo3go679b4d2spq9c0w865vq0k9auiszkkbvev62fc:latest/eval/step_20k.table.json", "path": "media/table/eval/step_20k_20000_a0a50c5d8793ca99e464.table.json", "ncols": 7, "nrows": 50}}

wandb/run-20220828_085247-2hx8pk65/run-2hx8pk65.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b64db33311cf3882e158023908e58b8d0a72a15e4fe5d8fa86d30c5ba22a41e2
-size 8691435
+oid sha256:dbd52b7487584ca71d80fed0e3182d1bd195ab787ea787199645db598d730074
+size 12917901