2hx8pk65: saving weights and logs of step 40k

.gitattributes

@@ -29,4 +29,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
-*.wandb filter=lfs diff=lfs merge=lfs -text
+*.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20220828_085247-2hx8pk65/logs/debug-internal.log filter=lfs diff=lfs merge=lfs -text

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:32e9052e8f2cf429458f533122ce50b5a0fc3cfc6e3096288daefda2766c5f0b
+oid sha256:8387a279a508da48c4b9b41b29ad79013e3f168689d639a6ed6588c9f0a4ed1d
 size 2353616717

wandb/run-20220828_085247-2hx8pk65/files/media/table/eval/step_40k_40000_1c187b6d789cca3b4ec0.table.json

@@ -0,0 +1 @@
+{"columns": ["id", "label_str", "beam_1", "beam_2", "beam_3", "beam_4", "beam_5"], "data": [["2277-149896-0000", "he was in a fevered state of mind owing to the blight his wife's action threatened to cast upon his entire future", "he was in a fevered state of mind owing to the blight his wife's action threatened to cast upon his entire future", "he was in a fevered state of mind owing to the blight his wife's action threatened to cast upon his entire future", "he was in a fevered state of mind owing to the blight his wife's action threatn'd to cast upon his entire future", "he was in a fovered state of mind owing to the blight his wife's action threatened to cast upon his entire future", "he was in a heavered state of mind owing to the blight his wife's action threatened to cast upon his entire future"], ["2277-149896-0001", "he would have to pay her the money which she would now regularly demand or there would be trouble it did not matter what he did", "he would have to pay her the money which she would now regularly demand or there would be trouble it did not matter what he did", "he would have to pay her the money which she would now regularly demand or there would be trouble it did not matter what he did it", "he would have to pay her the money which she would now regularly demand or there would be trouble it did not matter what he did", "he would have to pay her her the money which she would now regularly demand or there would be trouble it did not matter what he did", "he would have to pay her the money which she would now regularly demand or there would be trouble it did not matter what he said"], ["2277-149896-0002", "hurstwood walked the floor mentally arranging the chief points of his situation", "hurstwood walked to the floor mentally arranging the chief points of his situation", "hurstwood walked the floor mentally arranging the chief points of his situation", "hurstwood walked to the floor mentally arranging the chief points of his situation", "hurstwood walked to the floor mentally arranging the chief points in his situation", "hurstwood walked the floor mentally arranging the chief points of his situation"], ["2277-149896-0003", "he also thought of his managerial position", "he also thought of his managerial position", "he also thought of this managerial position", "he also thought of his managemental position", "he also thought of the managerial position", "he also thought of his managerial position and"], ["2277-149896-0004", "how would the papers talk about it", "how would the papers talk about it", "how would the papers talk about it and", "how would the papers talk about it you", "how would the papers talk about it can't be", "how would the papers talk about it can't you"], ["2277-149896-0005", "many little wrinkles gathered between his eyes as he contemplated this and his brow moistened", "many little wrinkles gathered between his eyes as he contemplated this and his brow moistened", "many little wrinkles gathered between his eyes as he considered this and his brow moistened", "many little wrinkles gathering between his eyes as he contemplated this and his brow moistened", "many little wrinkles gathered between his eyes as he contemented this and his brow moistened", "many little wrinkles gather between his eyes as he contemplated this and his brow moistened"], ["2277-149896-0006", "he could arrange that satisfactorily for carrie would be glad to wait if necessary", "he could arrange that satisfactorily for carrie would be glad to wait if necessary", "he could arrange that satisfactorily for carry would be glad to wait if necessary", "he could arrange that satisfactorily for cary would be glad to wait if necessary", "he could arrange this satisfactorily for carrie would be glad to wait if necessary", "he could arrange the satisfactorily for carrie would be glad to wait if necessary"], ["2277-149896-0007", "he would see how things turned out to morrow and then he would talk to her they were going to meet as usual", "he would see how things turned out to morrow and then he would talk to her they were going to meet as usual", "he would see how things turned out tomorrow and then he would talk to her they were going to meet as usual", "he would see how things turned out to morrow and then he would talk to her they were going to meet as usual a", "he would see how things turned out to morrow and then he would talk to her they were looking to meet as usual", "he could see how things turned out to morrow and then he would talk to her they were going to meet as usual"], ["2277-149896-0008", "for some reason he felt as if something might come that way and was relieved when all the envelopes had been scanned and nothing suspicious noticed", "for some reason he felt as if something might come that way and was relieved when all the envelopes had been scanned and nothing suspicious noticed", "for some reason he felt as if something might come that way and was relieved while all the envelopes had been scanned and nothing suspicious noticed", "for some reason he felt as if something might come that way and was relieved when all the envelopes had been screened and nothing suspicious noticed", "for some reason he felt as if something might come that way and was relieved when all the envelopes have been scanned and nothing suspicious noticed", "for some reason he felt as if something might come that way and was relieved when all the envelopes had been scanned and nothing suspicious notice"], ["2277-149896-0009", "while the danger had not lessened it had not as yet materialised and with him no news was good news", "while the danger had not lessened it had not as yet materialized and with him no news was good news", "while the danger had not lessened it had not as yet materialised and with him no news was good news", "while the danger had not lessened it had not as yet materialized and with him no news was good news and", "while the danger had not lessened it had not as yet materialized and with him no news was good news a", "while the danger had not lessened it had not as yet materialized and with him no news was good news i"], ["2277-149896-0010", "so little did he consider drouet that it never once occurred to him to worry about his finding out", "so little did he consider drouet that it never once occurred to him to worry about his finding out", "so little did he consider drue that it never once occurred to him to worry about his finding out", "so little did he consider dru that it never once occurred to him to worry about his finding out", "so little did he consider druma that it never once occurred to him to worry about his finding out", "so little did he consider druda that it never once occurred to him to worry about his finding out"], ["2277-149896-0011", "he grew restless as he ruminated and then decided that perhaps it was nothing", "he grew restless as he ruminated and then decided that perhaps it was nothing", "he grew restless as he ruminated and then decided that perhaps it was nothing", "he grew restless as he ruminated and then decided that perhaps it was nothing and", "he grew restless as he ruminated and then decided that perhaps it was nothing i", "he grew restless as he ruminated and then decided that perhaps it was nothing had"], ["2277-149896-0012", "she had not been able to get away this morning", "she had not been able to get away this morning", "she had not been able to get away this morning and", "she had not been able to get away this morning i", "she had not been able to get away this morning a", "she had not been able to get away this morning to"], ["2277-149896-0013", "he would get one to day it would probably be on his desk when he got back he would look for it at once", "he would get one to day it would probably be on his desk when he got back he would look for it at once", "he would get one to day it would probably be on his desk when he got back he would look for it at once and", "he would get one to day it could probably be on his desk when he got back he would look for it at once", "he would get one to day it would probably be upon his desk when he got back he would look for it at once", "he would get one to day it would probably be on his desk when we got back he would look for it at once"], ["2277-149896-0014", "after a time he gave up waiting and drearily headed for the madison car", "after a time he gave up waiting and drearily headed for the madison car", "after a time he gave up waiting and drearily heading for the madison car", "after a time he gave up waiting and desirily headed for the madison car", "after a time he gave up waiting and drearily headed for the madison car and", "after a time he gave up waiting and dearily headed for the madison car"], ["2277-149896-0015", "he went in and examined his letters but there was nothing from carrie", "he went in and examined his letters but there was nothing from carrie", "he went in and examined his letters but there was nothing from kerry", "he went in and examined his letters but there was nothing from carry", "he went in and examined his letters but there was nothing from carey", "he went in and examined his letters but there was nothing from perry"], ["2277-149896-0016", "fortunately there was nothing from his wife either", "fortunately there was nothing from his wife either", "fortunately there was nothing from his wife either and", "fortunately there was nothing from his wife either", "fortunately there was nothing from this wife either", "fortunately there was nothing from his wife neither"], ["2277-149896-0017", "at one thirty he went to rector's for lunch and when he returned a messenger was waiting for him", "at one thirty he went to rector's for lunch and when he returned a messenger was waiting for him", "at one thirty he went to rectors for lunch and when he returned a messenger was waiting for him", "at one thirty he went to rectors for lunch and when he returned a messenger was waiting for him", "at one thirty he went to restor's for lunch and when he returned a messenger was waiting for him", "at one thirty he went to repertors for lunch and when he returned a messenger was waiting for him"], ["2277-149896-0018", "his first impulse was to write but four words in reply go to the devil", "his first impulse was to write but four words in reply go to the devil", "his first impulse was to write but four words in reply go to the devil and", "his first impulse was to write but four words in reply go to the devil'", "his first instinct was to write but four words in reply go to the devil", "his first impulse was to write but fourwords in reply go to the devil"], ["2277-149896-0019", "but he compromised by telling the boy that there would be no reply", "but he compromised by telling the boy that there would be no reply", "but he compromised by telling the boy that there would be no reply i", "but he compromised by telling the boy there would be no reply", "but he compromised by telling the boy that there would be no reply and", "but hecompromised by telling the boy that there would be no reply"], ["2277-149896-0020", "then he sat down in his chair and gazed without seeing contemplating the result of his work", "then he sat down in his chair and gazed without seeing contemplating the result of his work", "then he sat down in his chair and gazed without seeing contemplating the results of his work", "then he sat down in his chair and gazed without seeing contemplating the result of his works", "then he sat down in his chair and gazed without seeing contemplated the result of his work", "than he sat down in his chair and gazed without seeing contemplating the result of his work"], ["2277-149896-0021", "what would she do about that the confounded wretch", "what would she do about that the confounded wretch", "what would she do about that that confounded wretch", "what would she do about that the confounded wretch and", "what would she do about that the confounded wretch that", "what would she do about that the con founded wretch"], ["2277-149896-0022", "later however his old discretion asserted itself", "later however his old discretion asserted itself", "later however his old discretion asserted itself and", "later however his old discretion ascertained itself", "later however his old discretion asserted itself", "later however his old discretion asserted itself if"], ["2277-149896-0023", "something had to be done a climax was near and she would not sit idle", "something had to be done a climax was near and she would not sit idle", "something had to be done a nexus was near and she would not sit idle", "something had to be done a climax was near as she would not sit idle", "something had to be done a climax was near she would not sit idle", "something had to be done a climax was near and she would not sit idle"], ["2277-149896-0024", "he knew her well enough to know that when she had decided upon a plan she would follow it up", "he knew her well enough to know that when she had decided upon a plan she would follow it up", "he knew her well enough to know that when she had decided upon the plan she would follow it up", "he knew her well enough to know that when she had decided upon a plan she would follow it up i", "he knew her well enough to know that when she had decided upon a plan she would follow it up and", "he knew her well enough to know that when she had decided upon a plan she would follow it up to"], ["2277-149896-0025", "he arose from his chair and went and looked out into the street", "he arose from his chair and went and looked out into the street", "he arose from his chair and went and looked out into the street and", "he arose from his chair and went and looked out into the streets", "he rose from his chair and went and looked out into the street", "he arose from his chair and went and looked out into the street a"], ["2277-149896-0026", "the long drizzle had begun pedestrians had turned up collars and trousers at the bottom", "the long drizzle had begun pedestrians had turned up collars and trousers at the bottom", "the long drizzle had begun pedestrian had turned up collars and trousers at the bottom", "the long drizzle had begun pedestrians had turned up collars and trousers at the bottom", "the long drizzle had begun domestians had turned up collars and trousers at the bottom", "the long drizzle had begun vegetables had turned up collars and trousers at the bottom"], ["2277-149896-0027", "hurstwood almost exclaimed out loud at the insistency of this thing", "hurstwood almost exclaimed out loud at the insistence of this thing", "hurstwood almost exclaimed out loud at the insistency of this thing", "hurstwood almost exclaimed out loud at the insistency of this thing", "hurstwood almost exclaimed out loud at the insistance of this thing", "hurstwood almost exclaimed out loud at the insistence of this thing"], ["2277-149896-0028", "he put on his hat and looked around for his umbrella", "he put on his hat and looked around for his umbrella", "he put on his hat and looked round for his umbrella", "he put on his hat and looked around for his umbrella and", "he put on his hat and looked toward for his umbrella", "he put on his hat and looked about for his umbrella"], ["2277-149896-0029", "he would have some arrangement of this thing", "he would have some arrangement of this thing", "he would have some arrangement of the thing", "he would have some arrangement of this thing and", "he would have some arrangement of the thing and", "he would have some arrangement of this thing i"], ["2277-149896-0030", "he began to wish that he had compromised in some way or other that he had sent the money perhaps he could do it up here", "he began to wish that he had compromised in some way or other that he had sent the money perhaps he could do it up here", "he began to wish that he had compromised in some way or other that he had sent the money perhaps he could do it up here and", "he began to wish that he had compromised in some way or other that he had set the money perhaps he could do it up here", "he began to wish that he had compromised in some way or other that he had sented the money perhaps he could do it up here", "he began to wish that he had compromised in some way or other that he had sent the money perhaps he could do it up here but"], ["2277-149896-0031", "he would go in and see anyhow he would have no row", "he would go in and see anyhow he would have no row", "he would go in and see anyhow he would have no row and", "he would go in and see anyhow he would have no row he", "he would go in and see anyhow he would have no row he would", "he would go in and see anyhow he would have no row a"], ["2277-149896-0032", "by the time he reached his own street he was keenly alive to the difficulties of his situation and wished over and over that some solution would offer itself that he could see his way out", "by the time he reached his own street he was keenly alive to the difficulties of his situation and wished over and over that some solution would offer itself that he could see his way out", "by the time he reached his own street he was keenly alive to the difficulties of this situation and wished over and over that some solution would offer itself that he could see his way out", "by the time we reached his own street he was keenly alive to the difficulties of his situation and wished over and over that some solution would offer itself that he could see his way out", "by the time he reached this own street he was keenly alive to the difficulties of his situation and wished over and over that some solution would offer itself that he could see his way out", ""], ["2277-149896-0033", "then he rang the bell no answer", "then he rang the bell no answer", "then he rang the bell no answer i", "than he rang the bell no answer", "then he rang the bell no answer to", "then he rang the bell no answer and"], ["2277-149896-0034", "he rang again this time harder still no answer", "he rang again this time harder still no answer", "he rang again this time harder still no answer i", "he rang again this time harder still no answer and", "he rang again this time harder still no answer a", "he ring again this time harder still no answer"], ["2277-149897-0000", "when hurstwood got back to his office again he was in a greater quandary than ever", "when hurstwood got back to his office again he was in a greater quandary than ever", "when hurstwood got back to his office again he was in a greater quadrary than ever", "when hurstwood went back to his office again he was in a greater quandary than ever", "when hurstwood got back to his office again he was in a greater quadrille than ever", "when hurstwood got back to his office again he was in a greater quandary than ever and"], ["2277-149897-0001", "he could hardly realise how it had all come about", "he could hardly realize how it had all come about", "he could hardly realise how it had all come about", "he could hardly realize how it has all come about", "he could hardly realize how it had all come about and", "he could hardly realize how it had all done about"], ["2277-149897-0002", "no letter had come no word of any kind and yet here it was late in the evening and she had agreed to meet him that morning", "no letter had come no word of any kind and yet here it was late in the evening and she had agreed to meet him that morning", "no letter had come no word of any kind and yet here it was late in the evening as she had agreed to meet him that morning", "no letter had come no word of any kind and yet here it was late in the evening and she had agreed to meet him that morning and", "no letter had come no word of any kind and yet here it was late in the afternoon and she had agreed to meet him that morning", "no letter had come no word of any kind and yet here it was late in the evening and she had agreed to meet him that morning i"], ["2277-149897-0003", "he saw that in the excitement of recent events he had not formulated a plan upon that score", "he saw that in the excitement of recent events he had not formulated a plan upon that score", "he saw that in the excitement of recent events he had not formulated a plan upon the score", "he saw that in the excitement of recent events he had not formulated a plan upon that score and", "he saw that in the excitement of recent events he had not formulated a plan upon that score a", "he saw that in the excitement of recent events he had not demonstrated a plan upon that score"], ["2277-149897-0004", "he was getting some vague comfort out of a good cigar but it was no panacea for the ill which affected him", "he was getting some vague comfort out of a good cigar but it was no panacea for the ill which affected him", "he was getting some vague comfort out of a good cigar but it was no panatia for the ill which affected him", "he was getting some vague comfort out of a good cigar but it was no panatism for the ill which affected him", "he was getting some vague comfort out of a good cigar but it was no panegy for the ill which affected him", "he was getting some vague comfort out of a good cigar but it was no pannity for the ill which affected him"], ["2277-149897-0005", "it was with great opposition after two or three hours of the most urgent mental affirmation and denial that at last he got an envelope placed in it the requested amount and slowly sealed it up", "it was with great opposition after two or three hours of the most urgent mental affirmation and denial that at last he got an envelope placed in it the requested amount and slowly sealed it up", "it was with great opposition after two or three hours of the most urgent mental affirmation and denial that at last he got an envelope placed in it the requested amount and slowly sealed it up and", "it was with great opposition after two or three hours of the most urgent mental acknowledgement and denial that at last he got an envelope placed in it the requested amount and slowly sealed it up", "it was with great opposition after two or three hours of the most urgent personal affirmation and denial that at last he got an envelope placed in it the requested amount and slowly sealed it up", "it was with great opposition after two or three hours of the most urgent mental affirmation and denial that at last he got an envelope placed in it the requested amount and slowly sealed it up a"], ["2277-149897-0006", "then he called harry the boy of all work around the place", "then he called harry the boy of all work around the place", "then he called harry the boy of all work around the place and", "then called harry the boy of all work around the place", "then he called harry the boy of all work around the place i", "then he called harry the boy of all work round the place"], ["2277-149897-0007", "you take this to this address he said handing him the envelope and give it to missus hurstwood yes sir said the boy", "you take this to this address he said handing him the envelope and give it to missus hurstwood yes sir said the boy", "you take this to this address he said handing him the envelope and give it to missus hurstwood yes sir said the boy i", "you take this to this address he said handing him the envelope and give it to missus hurstwood yes sir said the boy oh", "you take this to this address he said handing him the envelope and give it to missus hurstwood yes sir said the boy a", "you take this to this address he said handing him an envelope and give it to missus hurstwood yes sir said the boy"], ["2277-149897-0008", "any answer i guess not", "any answer i guess not", "any answer i guess not he", "any answer i guess not you", "any answer i guess not i", "any answer i guess not to"], ["2277-149897-0009", "the boy hastened away and the manager fell to his musings", "the boy hastened away and the manager fell to his musings", "the boy hastened away and the director fell to his musings", "the boy hastened away and the major fell to his musings", "the boy hurried away and the manager fell to his musings", "the boy hastily away and the manager fell to his musings"], ["2277-149897-0010", "he was beaten for to night and he might just as well make the best of it", "he was beaten for to night and he might just as well make the best of it", "he was beaten for to night and we might just as well make the best of it", "he was beaten for to night and i might just as well make the best of it", "he was beaten for tonight and he might just as well make the best of it", "he was beaten for to night and the might just as well make the best of it"], ["2277-149897-0011", "she would take the envelope and know that she had triumphed", "she would take the envelope and know that she had triumphed", "she would take the envelope and know that she had triumphed and", "she would take the envelope and know that she had triumphed eight", "she would take a envelope and know that she had triumphed", "she would take the envelope and know that she had triumphed i"], ["2277-149897-0012", "if he only had that letter back he wouldn't send it", "if he only had that letter back he wouldn't send it", "if he only had that letter back he wouldn't send it if", "if he only had that letter back he wouldn't send itif", "if he only had that letter back he wouldn't send it", "if he only held that letter back he wouldn't send it"], ["2277-149897-0013", "for relief he arose and joined in conversation with a few friends who were drinking", "for relief he arose and joined in the conversation with a few friends who were drinking", "for relief he arose and joined in the conversation with the few friends who were drinking", "for relief he arose and rejoined in the conversation with a few friends who were drinking", "for relief he arose and joined in the conversation with a few friends who were drinking and", "for relief he arose and joined in the conversation with a few friends who were drinking for"], ["2277-149897-0014", "all the time his thoughts would run out to his home and see the scene being therein enacted", "all the time his thoughts would run out to his home and see the scene being therein enacted", "all the time his thoughts would run out to his house and see the scene being therein enacted", "all this time his thoughts would run out to his home and see the scene being therein enacted", "all the time his thought would run out to his home and see the scene being therein enacted", "all the time his thoughts would run out to his home and see this scene being therein enacted"]]}

wandb/run-20220828_085247-2hx8pk65/files/output.log

@@ -33613,5 +33613,10358 @@ To disable this warning, you can either:
 	- Avoid using `tokenizers` before the fork if possible
 	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
 huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+Training...:  83% 3642/4393 [4:55:39<32:14:49, 154.58s/it]
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+  return jax.tree_map(/4393 [4:55:39<32:14:49, 154.58s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  return jax.tree_map(lambda x: x[0], tree)7,  5.87s/it]
+run_flax_speech_recognition_seq2seq.py:336: FutureWarning: jax.tree_map is deprecated, and will be removed in a future release. Use jax.tree_util.tree_map instead.
+  return jax.tree_map(lambda x: x.astype(jnp.float32) if x.dtype == jnp.bfloat16 else x, t)
+Step... (30000/50000 | Eval Loss: 0.9876799583435059 | Eval wer: 0.04759016212639241 | Eval cer: 0.035079873533571844 |):  50% 6/12 [40:00:43<35:02:00, 21020.10s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Step... (30000/50000 | Eval Loss: 0.9876799583435059 | Eval wer: 0.04759016212639241 | Eval cer: 0.035079873533571844 |):  58% 7/12 [40:58:25<29:19:56, 21119.31s/it]
+Step... (30000/50000 | Eval Loss: 0.9876799583435059 | Eval wer: 0.04759016212639241 | Eval cer: 0.035079873533571844 |)
+Step... (30025 | Loss: 0.01562521792948246, Learning Rate: 4.0355556848226115e-05, Gradient Norm: 0.265262246131897)
+Step... (30050 | Loss: 0.016519010066986084, Learning Rate: 4.0305047150468454e-05, Gradient Norm: 0.39625251293182373)
+Step... (30075 | Loss: 0.016090065240859985, Learning Rate: 4.0254544728668407e-05, Gradient Norm: 0.24512924253940582)
+Step... (30100 | Loss: 0.020717907696962357, Learning Rate: 4.020404230686836e-05, Gradient Norm: 0.36649051308631897)
+Step... (30125 | Loss: 0.023311305791139603, Learning Rate: 4.01535326091107e-05, Gradient Norm: 0.48252689838409424)
+Step... (30150 | Loss: 0.020284006372094154, Learning Rate: 4.010303018731065e-05, Gradient Norm: 0.4051229953765869)
+Step... (30175 | Loss: 0.03670956566929817, Learning Rate: 4.0052527765510604e-05, Gradient Norm: 0.3055880665779114)
+Step... (30200 | Loss: 0.016665274277329445, Learning Rate: 4.000201806775294e-05, Gradient Norm: 0.3359684348106384)
+Step... (30225 | Loss: 0.021576257422566414, Learning Rate: 3.9951515645952895e-05, Gradient Norm: 0.2677457928657532)
+Step... (30250 | Loss: 0.018543612211942673, Learning Rate: 3.990100958617404e-05, Gradient Norm: 0.3655332922935486)
+Step... (30275 | Loss: 0.02315855212509632, Learning Rate: 3.985050352639519e-05, Gradient Norm: 0.2440711408853531)
+Step... (30300 | Loss: 0.012810557149350643, Learning Rate: 3.979999746661633e-05, Gradient Norm: 0.29266688227653503)
+Step... (30325 | Loss: 0.016653001308441162, Learning Rate: 3.9749495044816285e-05, Gradient Norm: 0.22393402457237244)
+Step... (30350 | Loss: 0.01203943882137537, Learning Rate: 3.969898898503743e-05, Gradient Norm: 0.2736056447029114)
+Step... (30375 | Loss: 0.018077773973345757, Learning Rate: 3.964848292525858e-05, Gradient Norm: 0.2008410394191742)
+Step... (30400 | Loss: 0.03966773301362991, Learning Rate: 3.959798050345853e-05, Gradient Norm: 0.3925747871398926)
+Step... (30425 | Loss: 0.031882643699645996, Learning Rate: 3.954747080570087e-05, Gradient Norm: 0.39469194412231445)
+Step... (30450 | Loss: 0.014567309990525246, Learning Rate: 3.949696838390082e-05, Gradient Norm: 0.2972516715526581)
+Step... (30475 | Loss: 0.026796311140060425, Learning Rate: 3.9446465962100774e-05, Gradient Norm: 0.3355867564678192)
+Step... (30500 | Loss: 0.0361606702208519, Learning Rate: 3.939595626434311e-05, Gradient Norm: 0.407830148935318)
+Step... (30525 | Loss: 0.03200415521860123, Learning Rate: 3.9345453842543066e-05, Gradient Norm: 0.5616704821586609)
+Step... (30550 | Loss: 0.014776033349335194, Learning Rate: 3.929495142074302e-05, Gradient Norm: 0.3266984522342682)
+Step... (30575 | Loss: 0.012889200821518898, Learning Rate: 3.924444172298536e-05, Gradient Norm: 0.1923571079969406)
+Step... (30600 | Loss: 0.013984130695462227, Learning Rate: 3.919393930118531e-05, Gradient Norm: 0.3364371657371521)
+Step... (30625 | Loss: 0.029455211013555527, Learning Rate: 3.914343687938526e-05, Gradient Norm: 0.34443628787994385)
+Step... (30650 | Loss: 0.03519538417458534, Learning Rate: 3.90929271816276e-05, Gradient Norm: 0.5291494131088257)
+Step... (30675 | Loss: 0.02681455761194229, Learning Rate: 3.9042424759827554e-05, Gradient Norm: 0.30602386593818665)
+Step... (30700 | Loss: 0.02662944421172142, Learning Rate: 3.899192233802751e-05, Gradient Norm: 0.36101633310317993)
+Step... (30725 | Loss: 0.01868010126054287, Learning Rate: 3.8941412640269846e-05, Gradient Norm: 0.336725115776062)
+Step... (30750 | Loss: 0.05492250248789787, Learning Rate: 3.88909102184698e-05, Gradient Norm: 0.7654430270195007)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  40% 1748/4393 [2:19:42<1:36:57,  2.20s/it]
+Step... (30775 | Loss: 0.020074449479579926, Learning Rate: 3.884040779666975e-05, Gradient Norm: 0.35103708505630493)
+Step... (30800 | Loss: 0.009022093378007412, Learning Rate: 3.878989809891209e-05, Gradient Norm: 0.4046609401702881)
+Step... (30825 | Loss: 0.010481682606041431, Learning Rate: 3.873939567711204e-05, Gradient Norm: 0.39897775650024414)
+Step... (30850 | Loss: 0.0015329535817727447, Learning Rate: 3.868888961733319e-05, Gradient Norm: 0.08926476538181305)
+Step... (30875 | Loss: 0.018157506361603737, Learning Rate: 3.8638383557554334e-05, Gradient Norm: 0.33348050713539124)
+Step... (30900 | Loss: 0.0076045086607337, Learning Rate: 3.858787749777548e-05, Gradient Norm: 0.35109609365463257)
+Step... (30925 | Loss: 0.018489006906747818, Learning Rate: 3.8537371437996626e-05, Gradient Norm: 0.2586105465888977)
+Step... (30950 | Loss: 0.029866378754377365, Learning Rate: 3.848686537821777e-05, Gradient Norm: 0.9838259220123291)
+Step... (30975 | Loss: 0.025535719469189644, Learning Rate: 3.8436362956417724e-05, Gradient Norm: 0.26122814416885376)
+Step... (31000 | Loss: 0.010486635379493237, Learning Rate: 3.838585689663887e-05, Gradient Norm: 0.8068628311157227)
+Step... (31025 | Loss: 0.015530271455645561, Learning Rate: 3.8335350836860016e-05, Gradient Norm: 0.1934879869222641)
+Step... (31050 | Loss: 0.010574892163276672, Learning Rate: 3.828484841505997e-05, Gradient Norm: 0.5335045456886292)
+Step... (31075 | Loss: 0.02766241505742073, Learning Rate: 3.823433871730231e-05, Gradient Norm: 0.33427202701568604)
+Step... (31100 | Loss: 0.004194389563053846, Learning Rate: 3.818383629550226e-05, Gradient Norm: 0.21223211288452148)
+Step... (31125 | Loss: 0.012914509512484074, Learning Rate: 3.813333387370221e-05, Gradient Norm: 0.22572757303714752)
+Step... (31150 | Loss: 0.002646281151100993, Learning Rate: 3.808282417594455e-05, Gradient Norm: 0.20129801332950592)
+Step... (31175 | Loss: 0.018720488995313644, Learning Rate: 3.8032321754144505e-05, Gradient Norm: 0.33454686403274536)
+Step... (31200 | Loss: 0.0021898970007896423, Learning Rate: 3.798181933234446e-05, Gradient Norm: 0.2922018766403198)
+Step... (31225 | Loss: 0.018139390274882317, Learning Rate: 3.7931309634586796e-05, Gradient Norm: 0.284376323223114)
+Step... (31250 | Loss: 0.017836367711424828, Learning Rate: 3.788080721278675e-05, Gradient Norm: 0.7448089718818665)
+Step... (31275 | Loss: 0.009362353943288326, Learning Rate: 3.78303047909867e-05, Gradient Norm: 0.23159761726856232)
+Step... (31300 | Loss: 0.005309374537318945, Learning Rate: 3.777979509322904e-05, Gradient Norm: 0.3577989637851715)
+Step... (31325 | Loss: 0.023352516815066338, Learning Rate: 3.772929267142899e-05, Gradient Norm: 0.3282865583896637)
+Step... (31350 | Loss: 0.011359583586454391, Learning Rate: 3.7678790249628946e-05, Gradient Norm: 0.4796968102455139)
+Step... (31375 | Loss: 0.015349440276622772, Learning Rate: 3.7628280551871285e-05, Gradient Norm: 0.23296594619750977)
+Step... (31400 | Loss: 0.002398452255874872, Learning Rate: 3.757777813007124e-05, Gradient Norm: 0.17549781501293182)
+Step... (31425 | Loss: 0.01950138248503208, Learning Rate: 3.752727570827119e-05, Gradient Norm: 0.30373281240463257)
+Step... (31450 | Loss: 0.006116131786257029, Learning Rate: 3.747676601051353e-05, Gradient Norm: 0.35958558320999146)
+Step... (31475 | Loss: 0.022649751976132393, Learning Rate: 3.742626358871348e-05, Gradient Norm: 0.29181864857673645)
+Step... (31500 | Loss: 0.005045859608799219, Learning Rate: 3.737575752893463e-05, Gradient Norm: 0.2876945436000824)
+Step... (31525 | Loss: 0.011433024890720844, Learning Rate: 3.7325251469155774e-05, Gradient Norm: 0.2105947732925415)
+Step... (31550 | Loss: 0.017296401783823967, Learning Rate: 3.727474540937692e-05, Gradient Norm: 0.5196598172187805)
+Step... (31575 | Loss: 0.021925507113337517, Learning Rate: 3.722424298757687e-05, Gradient Norm: 0.3805218040943146)
+Step... (31600 | Loss: 0.00693158432841301, Learning Rate: 3.717373692779802e-05, Gradient Norm: 0.9232239127159119)
+Step... (31625 | Loss: 0.011155126616358757, Learning Rate: 3.7123230868019164e-05, Gradient Norm: 0.2306728959083557)
+Step... (31650 | Loss: 0.0032064933329820633, Learning Rate: 3.7072728446219116e-05, Gradient Norm: 0.18296311795711517)
+Step... (31675 | Loss: 0.015599323436617851, Learning Rate: 3.7022218748461455e-05, Gradient Norm: 0.2689945101737976)
+Step... (31700 | Loss: 0.003943402785807848, Learning Rate: 3.697171632666141e-05, Gradient Norm: 0.2862476110458374)
+Step... (31725 | Loss: 0.014135945588350296, Learning Rate: 3.692121390486136e-05, Gradient Norm: 0.29760271310806274)
+Step... (31750 | Loss: 0.001614759094081819, Learning Rate: 3.68707042071037e-05, Gradient Norm: 0.11080089956521988)
+Step... (31775 | Loss: 0.024603702127933502, Learning Rate: 3.682020178530365e-05, Gradient Norm: 0.485177606344223)
+Step... (31800 | Loss: 0.004279211163520813, Learning Rate: 3.6769699363503605e-05, Gradient Norm: 0.2789261043071747)
+Step... (31825 | Loss: 0.008554738014936447, Learning Rate: 3.6719189665745944e-05, Gradient Norm: 0.17841897904872894)
+Step... (31850 | Loss: 0.004655478522181511, Learning Rate: 3.66686872439459e-05, Gradient Norm: 0.441360741853714)
+Step... (31875 | Loss: 0.010068932548165321, Learning Rate: 3.661818482214585e-05, Gradient Norm: 0.18529221415519714)
+Step... (31900 | Loss: 0.01598585955798626, Learning Rate: 3.656767512438819e-05, Gradient Norm: 0.7631728649139404)
+Step... (31925 | Loss: 0.01710684411227703, Learning Rate: 3.651717270258814e-05, Gradient Norm: 0.2801308333873749)
+Step... (31950 | Loss: 0.005609472282230854, Learning Rate: 3.646666300483048e-05, Gradient Norm: 0.4520111083984375)
+Step... (31975 | Loss: 0.010884484276175499, Learning Rate: 3.641616058303043e-05, Gradient Norm: 0.21241894364356995)
+Step... (32000 | Loss: 0.009471398778259754, Learning Rate: 3.6365658161230385e-05, Gradient Norm: 0.5816971659660339)
+Step... (32025 | Loss: 0.021492963656783104, Learning Rate: 3.6315148463472724e-05, Gradient Norm: 0.2569616734981537)
+Step... (32050 | Loss: 0.002289405558258295, Learning Rate: 3.626464604167268e-05, Gradient Norm: 0.11815373599529266)
+Step... (32075 | Loss: 0.02590622566640377, Learning Rate: 3.621414361987263e-05, Gradient Norm: 0.3947508633136749)
+Step... (32100 | Loss: 0.007713155820965767, Learning Rate: 3.616363392211497e-05, Gradient Norm: 0.457485556602478)
+Step... (32125 | Loss: 0.02284613437950611, Learning Rate: 3.611313150031492e-05, Gradient Norm: 0.31568753719329834)
+Step... (32150 | Loss: 0.006654662545770407, Learning Rate: 3.606262544053607e-05, Gradient Norm: 0.39326000213623047)
+Step... (32175 | Loss: 0.01354904007166624, Learning Rate: 3.601211938075721e-05, Gradient Norm: 0.2626039683818817)
+Step... (32200 | Loss: 0.01198299415409565, Learning Rate: 3.596161332097836e-05, Gradient Norm: 0.5191692113876343)
+Step... (32225 | Loss: 0.0203788373619318, Learning Rate: 3.591111089917831e-05, Gradient Norm: 0.29200589656829834)
+Step... (32250 | Loss: 0.011300805024802685, Learning Rate: 3.586060483939946e-05, Gradient Norm: 0.3954571485519409)
+Step... (32275 | Loss: 0.02609907276928425, Learning Rate: 3.58100987796206e-05, Gradient Norm: 0.4498150944709778)
+Step... (32300 | Loss: 0.006780035328119993, Learning Rate: 3.5759596357820556e-05, Gradient Norm: 0.4498085379600525)
+Step... (32325 | Loss: 0.021178090944886208, Learning Rate: 3.5709086660062894e-05, Gradient Norm: 0.39719340205192566)
+Step... (32350 | Loss: 0.016519315540790558, Learning Rate: 3.565858423826285e-05, Gradient Norm: 0.5781014561653137)
+Step... (32375 | Loss: 0.016620468348264694, Learning Rate: 3.56080818164628e-05, Gradient Norm: 0.2802444398403168)
+Step... (32400 | Loss: 0.022222617641091347, Learning Rate: 3.555757211870514e-05, Gradient Norm: 0.7580689787864685)
+Step... (32425 | Loss: 0.0225432887673378, Learning Rate: 3.550706969690509e-05, Gradient Norm: 0.288802832365036)
+Step... (32450 | Loss: 0.0061545781791210175, Learning Rate: 3.5456567275105044e-05, Gradient Norm: 0.3461533784866333)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  79% 3474/4393 [4:37:39<1:27:37,  5.72s/it]
+Step... (32500 | Loss: 0.003240741789340973, Learning Rate: 3.5355555155547336e-05, Gradient Norm: 0.2734076678752899)
+Step... (32525 | Loss: 0.022595498710870743, Learning Rate: 3.530505273374729e-05, Gradient Norm: 0.2993549406528473)
+Step... (32550 | Loss: 0.004729455336928368, Learning Rate: 3.525454303598963e-05, Gradient Norm: 0.36609122157096863)
+Step... (32575 | Loss: 0.018012160435318947, Learning Rate: 3.520404061418958e-05, Gradient Norm: 0.3467451333999634)
+Step... (32600 | Loss: 0.003656194545328617, Learning Rate: 3.515353819238953e-05, Gradient Norm: 0.1817796230316162)
+Step... (32625 | Loss: 0.02160341665148735, Learning Rate: 3.510302849463187e-05, Gradient Norm: 0.2465333789587021)
+Step... (32650 | Loss: 0.006482965312898159, Learning Rate: 3.5052526072831824e-05, Gradient Norm: 0.4499094486236572)
+Step... (32675 | Loss: 0.020122623071074486, Learning Rate: 3.500202365103178e-05, Gradient Norm: 0.38773903250694275)
+Step... (32700 | Loss: 0.008678543381392956, Learning Rate: 3.4951513953274116e-05, Gradient Norm: 0.5408302545547485)
+Step... (32725 | Loss: 0.030759282410144806, Learning Rate: 3.490101153147407e-05, Gradient Norm: 0.468974232673645)
+Step... (32750 | Loss: 0.017723778262734413, Learning Rate: 3.4850505471695215e-05, Gradient Norm: 1.1028417348861694)
+Step... (32775 | Loss: 0.019855264574289322, Learning Rate: 3.479999941191636e-05, Gradient Norm: 0.7205868363380432)
+Step... (32800 | Loss: 0.009921908378601074, Learning Rate: 3.4749493352137506e-05, Gradient Norm: 0.6433566808700562)
+Step... (32825 | Loss: 0.008554498665034771, Learning Rate: 3.469899093033746e-05, Gradient Norm: 0.17429663240909576)
+Step... (32850 | Loss: 0.015867536887526512, Learning Rate: 3.4648484870558605e-05, Gradient Norm: 1.0128272771835327)
+Step... (32875 | Loss: 0.01661548763513565, Learning Rate: 3.459797881077975e-05, Gradient Norm: 0.27105745673179626)
+Step... (32900 | Loss: 0.012580855749547482, Learning Rate: 3.45474763889797e-05, Gradient Norm: 0.6393226385116577)
+Step... (32925 | Loss: 0.02611595205962658, Learning Rate: 3.449696669122204e-05, Gradient Norm: 0.34206777811050415)
+Step... (32950 | Loss: 0.01171598955988884, Learning Rate: 3.4446464269421995e-05, Gradient Norm: 0.6070415377616882)
+Step... (32975 | Loss: 0.0251434575766325, Learning Rate: 3.439595457166433e-05, Gradient Norm: 0.315672367811203)
+Step... (33000 | Loss: 0.0021658502519130707, Learning Rate: 3.4345452149864286e-05, Gradient Norm: 0.10065610706806183)
+Step... (33025 | Loss: 0.01859498955309391, Learning Rate: 3.429494972806424e-05, Gradient Norm: 0.31704211235046387)
+Step... (33050 | Loss: 0.009487439878284931, Learning Rate: 3.424444003030658e-05, Gradient Norm: 1.1141239404678345)
+Step... (33075 | Loss: 0.019629456102848053, Learning Rate: 3.419393760850653e-05, Gradient Norm: 0.3015685975551605)
+Step... (33100 | Loss: 0.01052027940750122, Learning Rate: 3.414343518670648e-05, Gradient Norm: 0.6721656918525696)
+Step... (33125 | Loss: 0.016070451587438583, Learning Rate: 3.409292548894882e-05, Gradient Norm: 0.2360723912715912)
+Step... (33150 | Loss: 0.005491696763783693, Learning Rate: 3.4042423067148775e-05, Gradient Norm: 0.2844001054763794)
+Step... (33175 | Loss: 0.0110780606046319, Learning Rate: 3.399192064534873e-05, Gradient Norm: 0.22559884190559387)
+Step... (33200 | Loss: 0.012108217924833298, Learning Rate: 3.3941410947591066e-05, Gradient Norm: 0.6203787326812744)
+Step... (33225 | Loss: 0.016109203919768333, Learning Rate: 3.389090852579102e-05, Gradient Norm: 0.2231585830450058)
+Step... (33250 | Loss: 0.02048683539032936, Learning Rate: 3.384040610399097e-05, Gradient Norm: 1.3308199644088745)
+Step... (33275 | Loss: 0.007731268182396889, Learning Rate: 3.378989640623331e-05, Gradient Norm: 0.16010697185993195)
+Step... (33300 | Loss: 0.004303706344217062, Learning Rate: 3.3739393984433264e-05, Gradient Norm: 0.34767386317253113)
+Step... (33325 | Loss: 0.01468588039278984, Learning Rate: 3.3688891562633216e-05, Gradient Norm: 0.24376437067985535)
+Step... (33350 | Loss: 0.004447631072252989, Learning Rate: 3.3638381864875555e-05, Gradient Norm: 0.6799517869949341)
+Step... (33375 | Loss: 0.02182612009346485, Learning Rate: 3.358787944307551e-05, Gradient Norm: 0.2627018392086029)
+Step... (33400 | Loss: 0.007039026822894812, Learning Rate: 3.3537373383296654e-05, Gradient Norm: 0.5744321942329407)
+Step... (33425 | Loss: 0.020231476053595543, Learning Rate: 3.34868673235178e-05, Gradient Norm: 0.31140726804733276)
+Step... (33450 | Loss: 0.019542209804058075, Learning Rate: 3.343636490171775e-05, Gradient Norm: 0.839359700679779)
+Step... (33475 | Loss: 0.012925517745316029, Learning Rate: 3.33858588419389e-05, Gradient Norm: 0.2737915813922882)
+Step... (33500 | Loss: 0.012760810554027557, Learning Rate: 3.3335352782160044e-05, Gradient Norm: 0.8377636075019836)
+Step... (33525 | Loss: 0.020903266966342926, Learning Rate: 3.328484672238119e-05, Gradient Norm: 0.5242891311645508)
+Step... (33550 | Loss: 0.008186263032257557, Learning Rate: 3.323434430058114e-05, Gradient Norm: 0.4458857774734497)
+Step... (33575 | Loss: 0.008030146360397339, Learning Rate: 3.318383460282348e-05, Gradient Norm: 0.15442919731140137)
+Step... (33600 | Loss: 0.009593255817890167, Learning Rate: 3.3133332181023434e-05, Gradient Norm: 0.4650934636592865)
+Step... (33625 | Loss: 0.027050640434026718, Learning Rate: 3.308282975922339e-05, Gradient Norm: 0.30105096101760864)
+Step... (33650 | Loss: 0.013253802433609962, Learning Rate: 3.3032320061465725e-05, Gradient Norm: 0.6894952058792114)
+Step... (33675 | Loss: 0.00922253169119358, Learning Rate: 3.298181763966568e-05, Gradient Norm: 0.20608296990394592)
+Step... (33700 | Loss: 0.005614001303911209, Learning Rate: 3.293131521786563e-05, Gradient Norm: 0.4169163405895233)
+Step... (33725 | Loss: 0.026930734515190125, Learning Rate: 3.288080552010797e-05, Gradient Norm: 0.4555964469909668)
+Step... (33750 | Loss: 0.0053314161486923695, Learning Rate: 3.283030309830792e-05, Gradient Norm: 0.3469780683517456)
+Step... (33775 | Loss: 0.0124602559953928, Learning Rate: 3.2779800676507875e-05, Gradient Norm: 0.2316969931125641)
+Step... (33800 | Loss: 0.004806062206625938, Learning Rate: 3.2729290978750214e-05, Gradient Norm: 0.4230194687843323)
+Step... (33825 | Loss: 0.02302425727248192, Learning Rate: 3.267878855695017e-05, Gradient Norm: 0.31783807277679443)
+Step... (33850 | Loss: 0.009264129213988781, Learning Rate: 3.262828613515012e-05, Gradient Norm: 0.4420391917228699)
+Step... (33875 | Loss: 0.015245446003973484, Learning Rate: 3.257777643739246e-05, Gradient Norm: 0.19797156751155853)
+Step... (33900 | Loss: 0.011483004316687584, Learning Rate: 3.252727401559241e-05, Gradient Norm: 0.6684927940368652)
+Step... (33925 | Loss: 0.014820012263953686, Learning Rate: 3.2476771593792364e-05, Gradient Norm: 0.2446964681148529)
+Step... (33950 | Loss: 0.011935788206756115, Learning Rate: 3.24262618960347e-05, Gradient Norm: 0.6529338359832764)
+Step... (33975 | Loss: 0.030441801995038986, Learning Rate: 3.2375759474234655e-05, Gradient Norm: 0.4021615982055664)
+Step... (34000 | Loss: 0.004538531880825758, Learning Rate: 3.2325249776476994e-05, Gradient Norm: 0.24466592073440552)
+Step... (34025 | Loss: 0.02973652072250843, Learning Rate: 3.227474735467695e-05, Gradient Norm: 0.5382617115974426)
+Step... (34050 | Loss: 0.0059531498700380325, Learning Rate: 3.222424129489809e-05, Gradient Norm: 0.5172224640846252)
+Step... (34075 | Loss: 0.021076254546642303, Learning Rate: 3.217373523511924e-05, Gradient Norm: 0.3070044219493866)
+Step... (34100 | Loss: 0.014772494323551655, Learning Rate: 3.212323281331919e-05, Gradient Norm: 0.5799047946929932)
+Step... (34125 | Loss: 0.019663726910948753, Learning Rate: 3.207272675354034e-05, Gradient Norm: 0.27861204743385315)
+Step... (34150 | Loss: 0.005554813891649246, Learning Rate: 3.202222069376148e-05, Gradient Norm: 0.4016672372817993)
+Step... (34175 | Loss: 0.02114006131887436, Learning Rate: 3.197171463398263e-05, Gradient Norm: 0.3323518931865692)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Step... (30000/50000 | Eval Loss: 0.9876799583435059 | Eval wer: 0.04759016212639241 | Eval cer: 0.035079873533571844 |):  67% 8/12 [46:48:52<23:25:59, 21089.92s/it]
+Step... (34225 | Loss: 0.01761227659881115, Learning Rate: 3.187070251442492e-05, Gradient Norm: 0.3862716555595398)
+Step... (34250 | Loss: 0.0024810461327433586, Learning Rate: 3.182020009262487e-05, Gradient Norm: 0.16885358095169067)
+Step... (34275 | Loss: 0.020597655326128006, Learning Rate: 3.1769697670824826e-05, Gradient Norm: 0.26309433579444885)
+Step... (34300 | Loss: 0.002395742107182741, Learning Rate: 3.1719187973067164e-05, Gradient Norm: 0.09939858317375183)
+Step... (34325 | Loss: 0.01582499034702778, Learning Rate: 3.166868555126712e-05, Gradient Norm: 0.40641799569129944)
+Step... (34350 | Loss: 0.006080330815166235, Learning Rate: 3.161818312946707e-05, Gradient Norm: 0.4112752676010132)
+Step... (34375 | Loss: 0.012621372006833553, Learning Rate: 3.156767343170941e-05, Gradient Norm: 0.20963077247142792)
+Step... (34400 | Loss: 0.020280960947275162, Learning Rate: 3.151717100990936e-05, Gradient Norm: 0.8154411315917969)
+Step... (34425 | Loss: 0.02049107477068901, Learning Rate: 3.1466668588109314e-05, Gradient Norm: 0.27243077754974365)
+Step... (34450 | Loss: 0.008309825323522091, Learning Rate: 3.141615889035165e-05, Gradient Norm: 0.4491531252861023)
+Step... (34475 | Loss: 0.021880406886339188, Learning Rate: 3.1365656468551606e-05, Gradient Norm: 3.2739105224609375)
+Step... (34500 | Loss: 0.007439988665282726, Learning Rate: 3.131515404675156e-05, Gradient Norm: 0.544802725315094)
+Step... (34525 | Loss: 0.02172919735312462, Learning Rate: 3.12646443489939e-05, Gradient Norm: 0.28622761368751526)
+Step... (34550 | Loss: 0.006562023889273405, Learning Rate: 3.121414192719385e-05, Gradient Norm: 0.3183799386024475)
+Step... (34575 | Loss: 0.010621322318911552, Learning Rate: 3.11636395053938e-05, Gradient Norm: 0.3131429851055145)
+Step... (34600 | Loss: 0.013357987627387047, Learning Rate: 3.111312980763614e-05, Gradient Norm: 0.48718786239624023)
+Step... (34625 | Loss: 0.017189813777804375, Learning Rate: 3.1062627385836095e-05, Gradient Norm: 0.2213728129863739)
+Step... (34650 | Loss: 0.002611755859106779, Learning Rate: 3.101212132605724e-05, Gradient Norm: 0.1833813488483429)
+Step... (34675 | Loss: 0.026818081736564636, Learning Rate: 3.0961615266278386e-05, Gradient Norm: 0.32487785816192627)
+Step... (34700 | Loss: 0.009743205271661282, Learning Rate: 3.091111284447834e-05, Gradient Norm: 0.591576337814331)
+Step... (34725 | Loss: 0.015863729640841484, Learning Rate: 3.0860606784699485e-05, Gradient Norm: 0.3375481367111206)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  39% 1705/4393 [2:15:22<4:00:59,  5.38s/it]
+Step... (34775 | Loss: 0.018730444833636284, Learning Rate: 3.0759594665141776e-05, Gradient Norm: 0.3146304488182068)
+Step... (34800 | Loss: 0.0131835350766778, Learning Rate: 3.070909224334173e-05, Gradient Norm: 0.6344761252403259)
+Step... (34825 | Loss: 0.02017289213836193, Learning Rate: 3.065858254558407e-05, Gradient Norm: 0.2658047080039978)
+Step... (34850 | Loss: 0.011610460467636585, Learning Rate: 3.060808012378402e-05, Gradient Norm: 0.5844695568084717)
+Step... (34875 | Loss: 0.020308807492256165, Learning Rate: 3.055757770198397e-05, Gradient Norm: 0.3984432518482208)
+Step... (34900 | Loss: 0.008994593285024166, Learning Rate: 3.0507069823215716e-05, Gradient Norm: 0.6693220138549805)
+Step... (34925 | Loss: 0.024976585060358047, Learning Rate: 3.0456565582426265e-05, Gradient Norm: 0.31148794293403625)
+Step... (34950 | Loss: 0.01379681471735239, Learning Rate: 3.0406063160626218e-05, Gradient Norm: 0.6182345151901245)
+Step... (34975 | Loss: 0.015182623639702797, Learning Rate: 3.0355553462868556e-05, Gradient Norm: 0.24722100794315338)
+Step... (35000 | Loss: 0.0074134087190032005, Learning Rate: 3.030505104106851e-05, Gradient Norm: 0.35679754614830017)
+Step... (35025 | Loss: 0.012640486471354961, Learning Rate: 3.0254548619268462e-05, Gradient Norm: 0.3233649730682373)
+Step... (35050 | Loss: 0.003796222386881709, Learning Rate: 3.02040389215108e-05, Gradient Norm: 0.23635932803153992)
+Step... (35075 | Loss: 0.015530078671872616, Learning Rate: 3.0153536499710754e-05, Gradient Norm: 0.3063891530036926)
+Step... (35100 | Loss: 0.00287941237911582, Learning Rate: 3.0103026801953092e-05, Gradient Norm: 0.12883423268795013)
+Step... (35125 | Loss: 0.018001878634095192, Learning Rate: 3.0052524380153045e-05, Gradient Norm: 0.2948234975337982)
+Step... (35150 | Loss: 0.01038078498095274, Learning Rate: 3.0002020139363594e-05, Gradient Norm: 0.16426806151866913)
+Step... (35175 | Loss: 0.010202213190495968, Learning Rate: 2.9951512260595337e-05, Gradient Norm: 0.24624192714691162)
+Step... (35200 | Loss: 0.014662917703390121, Learning Rate: 2.990100983879529e-05, Gradient Norm: 0.9740239977836609)
+Step... (35225 | Loss: 0.010558203794062138, Learning Rate: 2.985050559800584e-05, Gradient Norm: 0.2638781666755676)
+Step... (35250 | Loss: 0.015992023050785065, Learning Rate: 2.979999771923758e-05, Gradient Norm: 0.25827088952064514)
+Step... (35275 | Loss: 0.005929975304752588, Learning Rate: 2.974949347844813e-05, Gradient Norm: 0.20532174408435822)
+Step... (35300 | Loss: 0.01789812184870243, Learning Rate: 2.9698991056648083e-05, Gradient Norm: 0.22047241032123566)
+Step... (35325 | Loss: 0.01096875499933958, Learning Rate: 2.964848135889042e-05, Gradient Norm: 0.7241865396499634)
+Step... (35350 | Loss: 0.013526245020329952, Learning Rate: 2.9597978937090375e-05, Gradient Norm: 0.2711643576622009)
+Step... (35375 | Loss: 0.00827824231237173, Learning Rate: 2.9547476515290327e-05, Gradient Norm: 0.18412812054157257)
+Step... (35400 | Loss: 0.01867770217359066, Learning Rate: 2.9496966817532666e-05, Gradient Norm: 0.32241126894950867)
+Step... (35425 | Loss: 0.007050258107483387, Learning Rate: 2.944646439573262e-05, Gradient Norm: 0.15257257223129272)
+Step... (35450 | Loss: 0.017707733437418938, Learning Rate: 2.9395960154943168e-05, Gradient Norm: 0.2753700911998749)
+Step... (35475 | Loss: 0.005322085693478584, Learning Rate: 2.934545227617491e-05, Gradient Norm: 0.17496798932552338)
+Step... (35500 | Loss: 0.009057429619133472, Learning Rate: 2.9294949854374863e-05, Gradient Norm: 0.25817573070526123)
+Step... (35525 | Loss: 0.013614678755402565, Learning Rate: 2.9244445613585413e-05, Gradient Norm: 0.26426923274993896)
+Step... (35550 | Loss: 0.010442844592034817, Learning Rate: 2.9193937734817155e-05, Gradient Norm: 0.2709563076496124)
+Step... (35575 | Loss: 0.005399641115218401, Learning Rate: 2.9143433494027704e-05, Gradient Norm: 0.1638725847005844)
+Step... (35600 | Loss: 0.00854143314063549, Learning Rate: 2.9092931072227657e-05, Gradient Norm: 0.14298127591609955)
+Step... (35625 | Loss: 0.010740221478044987, Learning Rate: 2.9042421374469995e-05, Gradient Norm: 0.21739308536052704)
+Step... (35650 | Loss: 0.013804024085402489, Learning Rate: 2.899191895266995e-05, Gradient Norm: 0.2226768285036087)
+Step... (35675 | Loss: 0.0050653815269470215, Learning Rate: 2.89414165308699e-05, Gradient Norm: 0.12603415548801422)
+Step... (35700 | Loss: 0.0073559898883104324, Learning Rate: 2.889090683311224e-05, Gradient Norm: 0.18728569149971008)
+Step... (35725 | Loss: 0.014252002350986004, Learning Rate: 2.8840404411312193e-05, Gradient Norm: 0.2678786516189575)
+Step... (35750 | Loss: 0.01553369965404272, Learning Rate: 2.8789900170522742e-05, Gradient Norm: 0.2801755666732788)
+Step... (35775 | Loss: 0.007932987995445728, Learning Rate: 2.8739392291754484e-05, Gradient Norm: 0.15107274055480957)
+Step... (35800 | Loss: 0.009900908917188644, Learning Rate: 2.8688889869954437e-05, Gradient Norm: 0.18677286803722382)
+Step... (35825 | Loss: 0.008868378587067127, Learning Rate: 2.8638385629164986e-05, Gradient Norm: 0.21733544766902924)
+Step... (35850 | Loss: 0.014373034238815308, Learning Rate: 2.858787775039673e-05, Gradient Norm: 0.321470707654953)
+Step... (35875 | Loss: 0.008444367907941341, Learning Rate: 2.8537373509607278e-05, Gradient Norm: 0.20683307945728302)
+Step... (35900 | Loss: 0.008994083851575851, Learning Rate: 2.848687108780723e-05, Gradient Norm: 0.15626375377178192)
+Step... (35925 | Loss: 0.008749552071094513, Learning Rate: 2.843636139004957e-05, Gradient Norm: 0.5908063054084778)
+Step... (35950 | Loss: 0.011478588916361332, Learning Rate: 2.8385858968249522e-05, Gradient Norm: 0.2266269028186798)
+Step... (35975 | Loss: 0.01196364313364029, Learning Rate: 2.8335356546449475e-05, Gradient Norm: 0.29479607939720154)
+Step... (36000 | Loss: 0.011318921111524105, Learning Rate: 2.8284846848691814e-05, Gradient Norm: 0.17441242933273315)
+Step... (36025 | Loss: 0.015252816490828991, Learning Rate: 2.8234344426891766e-05, Gradient Norm: 0.3211612403392792)
+Step... (36050 | Loss: 0.0149453179910779, Learning Rate: 2.8183840186102316e-05, Gradient Norm: 0.2780107855796814)
+Step... (36075 | Loss: 0.013392834924161434, Learning Rate: 2.8133332307334058e-05, Gradient Norm: 0.30323371291160583)
+Step... (36100 | Loss: 0.008990200236439705, Learning Rate: 2.808282988553401e-05, Gradient Norm: 0.1986197680234909)
+Step... (36125 | Loss: 0.007589063141494989, Learning Rate: 2.803232018777635e-05, Gradient Norm: 0.16976489126682281)
+Step... (36150 | Loss: 0.018465880304574966, Learning Rate: 2.7981817765976302e-05, Gradient Norm: 3.223573923110962)
+Step... (36175 | Loss: 0.017651699483394623, Learning Rate: 2.793131352518685e-05, Gradient Norm: 0.3118066191673279)
+Step... (36200 | Loss: 0.006813254207372665, Learning Rate: 2.7880805646418594e-05, Gradient Norm: 0.17273718118667603)
+Step... (36225 | Loss: 0.017117872834205627, Learning Rate: 2.7830301405629143e-05, Gradient Norm: 0.25965094566345215)
+Step... (36250 | Loss: 0.0120441485196352, Learning Rate: 2.7779798983829096e-05, Gradient Norm: 0.3207753300666809)
+Step... (36275 | Loss: 0.011607665568590164, Learning Rate: 2.7729289286071435e-05, Gradient Norm: 0.22257204353809357)
+Step... (36300 | Loss: 0.020630035549402237, Learning Rate: 2.7678786864271387e-05, Gradient Norm: 0.45128902792930603)
+Step... (36325 | Loss: 0.005392152350395918, Learning Rate: 2.762828444247134e-05, Gradient Norm: 0.1612943410873413)
+Step... (36350 | Loss: 0.014236119575798512, Learning Rate: 2.757777474471368e-05, Gradient Norm: 0.37850603461265564)
+Step... (36375 | Loss: 0.015626532956957817, Learning Rate: 2.7527272322913632e-05, Gradient Norm: 2.135969638824463)
+Step... (36400 | Loss: 0.011072629131376743, Learning Rate: 2.7476769901113585e-05, Gradient Norm: 0.16433218121528625)
+Step... (36425 | Loss: 0.014065195806324482, Learning Rate: 2.7426260203355923e-05, Gradient Norm: 0.26176708936691284)
+Step... (36450 | Loss: 0.017493167892098427, Learning Rate: 2.7375757781555876e-05, Gradient Norm: 0.24977616965770721)
+Step... (36475 | Loss: 0.009039318189024925, Learning Rate: 2.7325253540766425e-05, Gradient Norm: 0.1969795823097229)
+Step... (36500 | Loss: 0.020493078976869583, Learning Rate: 2.7274745661998168e-05, Gradient Norm: 0.3413366675376892)
+Step... (36525 | Loss: 0.011266868561506271, Learning Rate: 2.7224241421208717e-05, Gradient Norm: 0.2362344115972519)
+Step... (36550 | Loss: 0.010742920450866222, Learning Rate: 2.717373899940867e-05, Gradient Norm: 0.23466497659683228)
+Step... (36575 | Loss: 0.011458583176136017, Learning Rate: 2.712322930165101e-05, Gradient Norm: 0.20105010271072388)
+Step... (36600 | Loss: 0.00982366781681776, Learning Rate: 2.707272687985096e-05, Gradient Norm: 0.19666042923927307)
+Step... (36625 | Loss: 0.010288187302649021, Learning Rate: 2.7022224458050914e-05, Gradient Norm: 0.26822808384895325)
+Step... (36650 | Loss: 0.011828682385385036, Learning Rate: 2.6971714760293253e-05, Gradient Norm: 0.20932193100452423)
+Step... (36675 | Loss: 0.01743757352232933, Learning Rate: 2.6921212338493206e-05, Gradient Norm: 0.33511021733283997)
+Step... (36700 | Loss: 0.02315790392458439, Learning Rate: 2.6870708097703755e-05, Gradient Norm: 0.41521745920181274)
+Step... (36725 | Loss: 0.01171206310391426, Learning Rate: 2.6820200218935497e-05, Gradient Norm: 0.30450528860092163)
+Step... (36750 | Loss: 0.01481856219470501, Learning Rate: 2.676969779713545e-05, Gradient Norm: 0.6513252854347229)
+Step... (36775 | Loss: 0.010270772501826286, Learning Rate: 2.6719193556346e-05, Gradient Norm: 0.44218534231185913)
+Step... (36800 | Loss: 0.01345136296004057, Learning Rate: 2.666868567757774e-05, Gradient Norm: 0.2729993760585785)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  78% 3405/4393 [4:30:03<1:28:38,  5.38s/it]
+Step... (36850 | Loss: 0.020339932292699814, Learning Rate: 2.6567679014988244e-05, Gradient Norm: 0.28321829438209534)
+Step... (36875 | Loss: 0.010295059531927109, Learning Rate: 2.6517169317230582e-05, Gradient Norm: 0.22561118006706238)
+Step... (36900 | Loss: 0.02300870791077614, Learning Rate: 2.6466666895430535e-05, Gradient Norm: 0.3506458103656769)
+Step... (36925 | Loss: 0.008988190442323685, Learning Rate: 2.6416164473630488e-05, Gradient Norm: 0.22951672971248627)
+Step... (36950 | Loss: 0.017710519954562187, Learning Rate: 2.6365654775872827e-05, Gradient Norm: 0.2859334647655487)
+Step... (36975 | Loss: 0.011881349608302116, Learning Rate: 2.631515235407278e-05, Gradient Norm: 0.28052568435668945)
+Step... (37000 | Loss: 0.016069943085312843, Learning Rate: 2.626464811328333e-05, Gradient Norm: 0.22248497605323792)
+Step... (37025 | Loss: 0.008865948766469955, Learning Rate: 2.621414023451507e-05, Gradient Norm: 0.223648339509964)
+Step... (37050 | Loss: 0.013166670687496662, Learning Rate: 2.6163637812715024e-05, Gradient Norm: 0.24581901729106903)
+Step... (37075 | Loss: 0.010862416587769985, Learning Rate: 2.6113133571925573e-05, Gradient Norm: 0.21010161936283112)
+Step... (37100 | Loss: 0.012013047933578491, Learning Rate: 2.6062625693157315e-05, Gradient Norm: 0.2584199607372284)
+Step... (37125 | Loss: 0.007369975559413433, Learning Rate: 2.6012121452367865e-05, Gradient Norm: 0.32532358169555664)
+Step... (37150 | Loss: 0.018205538392066956, Learning Rate: 2.5961613573599607e-05, Gradient Norm: 0.3200891315937042)
+Step... (37175 | Loss: 0.008243363350629807, Learning Rate: 2.5911109332810156e-05, Gradient Norm: 0.2219228297472)
+Step... (37200 | Loss: 0.007743628229945898, Learning Rate: 2.586060691101011e-05, Gradient Norm: 0.20197027921676636)
+Step... (37225 | Loss: 0.009823272004723549, Learning Rate: 2.5810097213252448e-05, Gradient Norm: 0.16621631383895874)
+Step... (37250 | Loss: 0.01478399708867073, Learning Rate: 2.57595947914524e-05, Gradient Norm: 0.21830250322818756)
+Step... (37275 | Loss: 0.009942734614014626, Learning Rate: 2.5709092369652353e-05, Gradient Norm: 0.23925411701202393)
+Step... (37300 | Loss: 0.02040395885705948, Learning Rate: 2.5658582671894692e-05, Gradient Norm: 0.27813541889190674)
+Step... (37325 | Loss: 0.00806287582963705, Learning Rate: 2.5608080250094645e-05, Gradient Norm: 0.18820592761039734)
+Step... (37350 | Loss: 0.012719778344035149, Learning Rate: 2.5557577828294598e-05, Gradient Norm: 0.22716785967350006)
+Step... (37375 | Loss: 0.01238447055220604, Learning Rate: 2.5507068130536936e-05, Gradient Norm: 0.3181142807006836)
+Step... (37400 | Loss: 0.008483604528009892, Learning Rate: 2.545656570873689e-05, Gradient Norm: 0.14901268482208252)
+Step... (37425 | Loss: 0.009233653545379639, Learning Rate: 2.540606146794744e-05, Gradient Norm: 0.23794111609458923)
+Step... (37450 | Loss: 0.018127061426639557, Learning Rate: 2.535555358917918e-05, Gradient Norm: 0.27508270740509033)
+Step... (37475 | Loss: 0.010165028274059296, Learning Rate: 2.530504934838973e-05, Gradient Norm: 0.22427453100681305)
+Step... (37500 | Loss: 0.010916335508227348, Learning Rate: 2.5254546926589683e-05, Gradient Norm: 0.18438170850276947)
+Step... (37525 | Loss: 0.009773856028914452, Learning Rate: 2.520403722883202e-05, Gradient Norm: 0.23069195449352264)
+Step... (37550 | Loss: 0.016379348933696747, Learning Rate: 2.5153534807031974e-05, Gradient Norm: 0.2591167390346527)
+Step... (37575 | Loss: 0.014079141430556774, Learning Rate: 2.5103032385231927e-05, Gradient Norm: 0.2219134420156479)
+Step... (37600 | Loss: 0.010436130687594414, Learning Rate: 2.5052522687474266e-05, Gradient Norm: 0.20184342563152313)
+Step... (37625 | Loss: 0.02243305929005146, Learning Rate: 2.500202026567422e-05, Gradient Norm: 0.385381817817688)
+Step... (37650 | Loss: 0.009557241573929787, Learning Rate: 2.495151784387417e-05, Gradient Norm: 0.18861576914787292)
+Step... (37675 | Loss: 0.02001246064901352, Learning Rate: 2.490100814611651e-05, Gradient Norm: 0.3057927191257477)
+Step... (37700 | Loss: 0.015565990470349789, Learning Rate: 2.4850505724316463e-05, Gradient Norm: 0.26939934492111206)
+Step... (37725 | Loss: 0.010391747578978539, Learning Rate: 2.4800001483527012e-05, Gradient Norm: 0.2478579878807068)
+Step... (37750 | Loss: 0.012761175632476807, Learning Rate: 2.4749493604758754e-05, Gradient Norm: 0.2503322958946228)
+Step... (37775 | Loss: 0.007083362899720669, Learning Rate: 2.4698989363969304e-05, Gradient Norm: 0.20680712163448334)
+Step... (37800 | Loss: 0.01457024272531271, Learning Rate: 2.4648486942169257e-05, Gradient Norm: 0.21773581206798553)
+Step... (37825 | Loss: 0.016917739063501358, Learning Rate: 2.4597977244411595e-05, Gradient Norm: 0.4584696590900421)
+Step... (37850 | Loss: 0.020319297909736633, Learning Rate: 2.4547474822611548e-05, Gradient Norm: 0.28668245673179626)
+Step... (37875 | Loss: 0.007732781581580639, Learning Rate: 2.44969724008115e-05, Gradient Norm: 0.22831612825393677)
+Step... (37900 | Loss: 0.010798927396535873, Learning Rate: 2.444646270305384e-05, Gradient Norm: 0.19562046229839325)
+Step... (37925 | Loss: 0.0118245929479599, Learning Rate: 2.4395960281253792e-05, Gradient Norm: 0.378909707069397)
+Step... (37950 | Loss: 0.011480288580060005, Learning Rate: 2.434545604046434e-05, Gradient Norm: 0.18696899712085724)
+Step... (37975 | Loss: 0.01095297746360302, Learning Rate: 2.4294948161696084e-05, Gradient Norm: 0.20788218080997467)
+Step... (38000 | Loss: 0.014074883423745632, Learning Rate: 2.4244445739896037e-05, Gradient Norm: 0.2136598378419876)
+Step... (38025 | Loss: 0.00818438921123743, Learning Rate: 2.4193941499106586e-05, Gradient Norm: 0.18253900110721588)
+Step... (38050 | Loss: 0.016727542504668236, Learning Rate: 2.4143433620338328e-05, Gradient Norm: 0.2901366949081421)
+Step... (38075 | Loss: 0.007837343961000443, Learning Rate: 2.4092929379548877e-05, Gradient Norm: 0.19501838088035583)
+Step... (38100 | Loss: 0.008517307229340076, Learning Rate: 2.404242695774883e-05, Gradient Norm: 0.2721792459487915)
+Step... (38125 | Loss: 0.009104839526116848, Learning Rate: 2.399191725999117e-05, Gradient Norm: 0.4130169749259949)
+Step... (38150 | Loss: 0.0076513695530593395, Learning Rate: 2.3941414838191122e-05, Gradient Norm: 0.15972954034805298)
+Step... (38175 | Loss: 0.012398313730955124, Learning Rate: 2.389090514043346e-05, Gradient Norm: 0.2811894714832306)
+Step... (38200 | Loss: 0.01311738695949316, Learning Rate: 2.3840402718633413e-05, Gradient Norm: 0.20744727551937103)
+Step... (38225 | Loss: 0.013343630358576775, Learning Rate: 2.3789900296833366e-05, Gradient Norm: 0.23322346806526184)
+Step... (38250 | Loss: 0.013739443384110928, Learning Rate: 2.3739390599075705e-05, Gradient Norm: 0.3291863799095154)
+Step... (38275 | Loss: 0.008902786299586296, Learning Rate: 2.3688888177275658e-05, Gradient Norm: 0.18816299736499786)
+Step... (38300 | Loss: 0.029258565977215767, Learning Rate: 2.363838575547561e-05, Gradient Norm: 0.3861159086227417)
+Step... (38325 | Loss: 0.009011116810142994, Learning Rate: 2.358787605771795e-05, Gradient Norm: 0.34315818548202515)
+Step... (38350 | Loss: 0.012569929473102093, Learning Rate: 2.3537373635917902e-05, Gradient Norm: 0.23500430583953857)
+Step... (38375 | Loss: 0.005166625138372183, Learning Rate: 2.348686939512845e-05, Gradient Norm: 0.17557674646377563)
+Step... (38400 | Loss: 0.01499476283788681, Learning Rate: 2.3436361516360193e-05, Gradient Norm: 0.2289804071187973)
+Step... (38425 | Loss: 0.006382503546774387, Learning Rate: 2.3385857275570743e-05, Gradient Norm: 0.1602679044008255)
+Step... (38450 | Loss: 0.010056969709694386, Learning Rate: 2.3335354853770696e-05, Gradient Norm: 0.18118983507156372)
+Step... (38475 | Loss: 0.008315986953675747, Learning Rate: 2.3284845156013034e-05, Gradient Norm: 0.18881313502788544)
+Step... (38500 | Loss: 0.01467866450548172, Learning Rate: 2.3234342734212987e-05, Gradient Norm: 0.5424960255622864)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Step... (30000/50000 | Eval Loss: 0.9876799583435059 | Eval wer: 0.04759016212639241 | Eval cer: 0.035079873533571844 |):  75% 9/12 [52:36:57<17:31:17, 21025.84s/it]
+Training...:   0% 0/4393 [00:00<?, ?it/s]
+Step... (38550 | Loss: 0.010347037576138973, Learning Rate: 2.313333061465528e-05, Gradient Norm: 0.2250284105539322)
+Step... (38575 | Loss: 0.0063611362129449844, Learning Rate: 2.308282819285523e-05, Gradient Norm: 0.1974717080593109)
+Step... (38600 | Loss: 0.004289443604648113, Learning Rate: 2.3032325771055184e-05, Gradient Norm: 0.09970905631780624)
+Step... (38625 | Loss: 0.011062589474022388, Learning Rate: 2.2981816073297523e-05, Gradient Norm: 0.2628202438354492)
+Step... (38650 | Loss: 0.010620958171784878, Learning Rate: 2.2931313651497476e-05, Gradient Norm: 0.21071377396583557)
+Step... (38675 | Loss: 0.007905133068561554, Learning Rate: 2.2880809410708025e-05, Gradient Norm: 0.1927347332239151)
+Step... (38700 | Loss: 0.021522454917430878, Learning Rate: 2.2830301531939767e-05, Gradient Norm: 0.26387181878089905)
+Step... (38725 | Loss: 0.01568610407412052, Learning Rate: 2.2779797291150317e-05, Gradient Norm: 0.3414469063282013)
+Step... (38750 | Loss: 0.007739794906228781, Learning Rate: 2.272929486935027e-05, Gradient Norm: 0.20845206081867218)
+Step... (38775 | Loss: 0.01500839926302433, Learning Rate: 2.2678785171592608e-05, Gradient Norm: 0.366758793592453)
+Step... (38800 | Loss: 0.01572476327419281, Learning Rate: 2.262828274979256e-05, Gradient Norm: 0.23734962940216064)
+Step... (38825 | Loss: 0.01844528131186962, Learning Rate: 2.2577780327992514e-05, Gradient Norm: 0.3101344704627991)
+Step... (38850 | Loss: 0.022008061408996582, Learning Rate: 2.2527270630234852e-05, Gradient Norm: 0.8459998965263367)
+Step... (38875 | Loss: 0.01427079550921917, Learning Rate: 2.2476768208434805e-05, Gradient Norm: 0.23986835777759552)
+Step... (38900 | Loss: 0.010430174879729748, Learning Rate: 2.2426265786634758e-05, Gradient Norm: 0.2690977454185486)
+Step... (38925 | Loss: 0.013039502315223217, Learning Rate: 2.2375756088877097e-05, Gradient Norm: 0.26950889825820923)
+Step... (38950 | Loss: 0.011772986501455307, Learning Rate: 2.232525366707705e-05, Gradient Norm: 0.22706244885921478)
+Step... (38975 | Loss: 0.012624364346265793, Learning Rate: 2.22747494262876e-05, Gradient Norm: 0.35651257634162903)
+Step... (39000 | Loss: 0.01489238440990448, Learning Rate: 2.222424154751934e-05, Gradient Norm: 0.2117861658334732)
+Step... (39025 | Loss: 0.02081136405467987, Learning Rate: 2.217373730672989e-05, Gradient Norm: 0.3005424439907074)
+Step... (39050 | Loss: 0.007413056679069996, Learning Rate: 2.2123234884929843e-05, Gradient Norm: 0.14772680401802063)
+Step... (39075 | Loss: 0.011335279792547226, Learning Rate: 2.2072725187172182e-05, Gradient Norm: 0.23882412910461426)
+Step... (39100 | Loss: 0.020389100536704063, Learning Rate: 2.2022222765372135e-05, Gradient Norm: 0.2939780652523041)
+Step... (39125 | Loss: 0.016199955716729164, Learning Rate: 2.1971720343572088e-05, Gradient Norm: 0.2736057639122009)
+Step... (39150 | Loss: 0.01839425601065159, Learning Rate: 2.1921210645814426e-05, Gradient Norm: 0.3050045967102051)
+Step... (39175 | Loss: 0.008332288824021816, Learning Rate: 2.187070822401438e-05, Gradient Norm: 0.18911497294902802)
+Step... (39200 | Loss: 0.0066859484650194645, Learning Rate: 2.1820198526256718e-05, Gradient Norm: 0.16664181649684906)
+Step... (39225 | Loss: 0.010230396874248981, Learning Rate: 2.176969610445667e-05, Gradient Norm: 0.29577866196632385)
+Step... (39250 | Loss: 0.009517351165413857, Learning Rate: 2.1719193682656623e-05, Gradient Norm: 0.17758044600486755)
+Step... (39275 | Loss: 0.006245317868888378, Learning Rate: 2.1668683984898962e-05, Gradient Norm: 0.19972631335258484)
+Step... (39300 | Loss: 0.013634399510920048, Learning Rate: 2.1618181563098915e-05, Gradient Norm: 0.22748829424381256)
+Step... (39325 | Loss: 0.006963997147977352, Learning Rate: 2.1567677322309464e-05, Gradient Norm: 0.2040238231420517)
+Step... (39350 | Loss: 0.022512728348374367, Learning Rate: 2.1517169443541206e-05, Gradient Norm: 0.30836740136146545)
+Step... (39375 | Loss: 0.011416936293244362, Learning Rate: 2.1466665202751756e-05, Gradient Norm: 0.2511473000049591)
+Step... (39400 | Loss: 0.016352824866771698, Learning Rate: 2.141616278095171e-05, Gradient Norm: 0.2511434555053711)
+Step... (39425 | Loss: 0.009906467981636524, Learning Rate: 2.1365653083194047e-05, Gradient Norm: 0.2822321951389313)
+Step... (39450 | Loss: 0.011319736018776894, Learning Rate: 2.1315150661394e-05, Gradient Norm: 0.2532801330089569)
+Step... (39475 | Loss: 0.008581236936151981, Learning Rate: 2.1264648239593953e-05, Gradient Norm: 0.1751236915588379)
+Step... (39500 | Loss: 0.01655127853155136, Learning Rate: 2.121413854183629e-05, Gradient Norm: 0.23636147379875183)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  11% 462/4393 [37:00<6:26:59,  5.91s/it]
+Evaluating ...:   0% 0/85 [00:00<?, ?it/s]
+Step... (39550 | Loss: 0.013957838527858257, Learning Rate: 2.1113133698236197e-05, Gradient Norm: 0.25816094875335693)
+Step... (39575 | Loss: 0.003900324460119009, Learning Rate: 2.1062624000478536e-05, Gradient Norm: 0.15411750972270966)
+Step... (39600 | Loss: 0.004219492897391319, Learning Rate: 2.101212157867849e-05, Gradient Norm: 0.1492801159620285)
+Step... (39625 | Loss: 0.007128617260605097, Learning Rate: 2.0961617337889038e-05, Gradient Norm: 0.21547576785087585)
+Step... (39650 | Loss: 0.012958021834492683, Learning Rate: 2.091110945912078e-05, Gradient Norm: 0.28005728125572205)
+Step... (39675 | Loss: 0.006572413258254528, Learning Rate: 2.086060521833133e-05, Gradient Norm: 0.3049915134906769)
+Step... (39700 | Loss: 0.008246519602835178, Learning Rate: 2.0810102796531282e-05, Gradient Norm: 0.2068808376789093)
+Step... (39725 | Loss: 0.004701581783592701, Learning Rate: 2.075959309877362e-05, Gradient Norm: 0.23778201639652252)
+Step... (39750 | Loss: 0.010084073059260845, Learning Rate: 2.0709090676973574e-05, Gradient Norm: 0.1955907642841339)
+Step... (39775 | Loss: 0.0037632507737725973, Learning Rate: 2.0658588255173527e-05, Gradient Norm: 0.18372221291065216)
+Step... (39800 | Loss: 0.009699777700006962, Learning Rate: 2.0608078557415865e-05, Gradient Norm: 0.19054998457431793)
+Step... (39825 | Loss: 0.0035127829760313034, Learning Rate: 2.0557576135615818e-05, Gradient Norm: 0.17126810550689697)
+Step... (39850 | Loss: 0.007305247243493795, Learning Rate: 2.050707371381577e-05, Gradient Norm: 0.15133488178253174)
+Step... (39875 | Loss: 0.007563646882772446, Learning Rate: 2.045656401605811e-05, Gradient Norm: 0.2595842778682709)
+Step... (39900 | Loss: 0.009252386167645454, Learning Rate: 2.0406061594258063e-05, Gradient Norm: 0.16340221464633942)
+Step... (39925 | Loss: 0.009359311312437057, Learning Rate: 2.0355557353468612e-05, Gradient Norm: 0.15072058141231537)
+Step... (39950 | Loss: 0.005342613440006971, Learning Rate: 2.0305049474700354e-05, Gradient Norm: 0.1631428301334381)
+Step... (39975 | Loss: 0.008378183469176292, Learning Rate: 2.0254545233910903e-05, Gradient Norm: 0.23551413416862488)
+/home/sanchitgandhi/hf/lib/python3.8/site-packages/flax/jax_utils.py:312: FutureWarning: jax.tree_map is deprecated, and will be removed in a future release. Use jax.tree_util.tree_map instead.
+  return jax.tree_map(pad, tree)
+/home/sanchitgandhi/hf/lib/python3.8/site-packages/flax/jax_utils.py:321: FutureWarning: jax.tree_map is deprecated, and will be removed in a future release. Use jax.tree_util.tree_map instead.
+  return out if static_return else jax.tree_map(unpad, out)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  device_metrics = jax.tree_map(lambda x: x[0], device_metrics)
+/home/sanchitgandhi/hf/lib/python3.8/site-packages/flax/training/common_utils.py:45: FutureWarning: jax.tree_map is deprecated, and will be removed in a future release. Use jax.tree_util.tree_map instead.
+  return jax.tree_map(stack_args, *forest)
+run_flax_speech_recognition_seq2seq.py:1392: FutureWarning: jax.tree_map is deprecated, and will be removed in a future release. Use jax.tree_util.tree_map instead.
+  eval_metrics = jax.tree_map(jnp.mean, eval_metrics)
+Step... (30000/50000 | Eval Loss: 0.9876799583435059 | Eval wer: 0.04759016212639241 | Eval cer: 0.035079873533571844 |):  75% 9/12 [53:21:16<17:31:17, 21025.84s/it]
+  params = jax.device_get(jax.tree_map(lambda x: x[0], state.params))
+Configuration saved in /home/sanchitgandhi/flax-wav2vec2-2-bart-large-ls-960h-black-box/config.json
+tcmalloc: large alloc 2586787840 bytes == 0x46a526000 @  0x7f7cba873680 0x7f7cba893bdd 0x7f7b690721ff 0x7f7b6908142c 0x7f7b6908241d 0x7f7b6908241d 0x7f7b6908241d 0x7f7b6908241d 0x7f7b6908241d 0x7f7b6908241d 0x7f7b6908241d 0x7f7b6907c164 0x7f7b6907c91e 0x505166 0x56bbfa 0x569dba 0x5f6eb3 0x56cc1f 0x569dba 0x5f6eb3 0x56cc1f 0x5f6cd6 0x56bacd 0x569dba 0x50bca0 0x56cc1f 0x569dba 0x5f6eb3 0x56bacd 0x569dba 0x5f6eb3
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+Model weights saved in /home/sanchitgandhi/flax-wav2vec2-2-bart-large-ls-960h-black-box/flax_model.msgpack
+tokenizer config file saved in ./tokenizer_config.json
+Special tokens file saved in ./special_tokens_map.json
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+Adding files tracked by Git LFS: ['wandb/run-20220828_085247-2hx8pk65/logs/debug-internal.log']. This may take a bit of time if the files are large.
+WARNING:huggingface_hub.repository:Adding files tracked by Git LFS: ['wandb/run-20220828_085247-2hx8pk65/logs/debug-internal.log']. This may take a bit of time if the files are large.
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
+To disable this warning, you can either:
+	- Avoid using `tokenizers` before the fork if possible
+	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
+huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
 To disable this warning, you can either:
 	- Avoid using `tokenizers` before the fork if possible

wandb/run-20220828_085247-2hx8pk65/files/wandb-summary.json

@@ -1 +1 @@
-{"train/decoder_grad_norm": 0.17611896991729736, "train/decoder_param_norm": 1062.4339599609375, "train/encoder_grad_norm": 0.1704455018043518, "train/encoder_param_norm": 2322.47119140625, "train/grad_norm": 0.24509090185165405, "layer_grad_norm/": {"decoder": {"model": {"decoder": {"embed_positions": {"embedding": 0.006716505624353886}, "embed_tokens": {"embedding": 0.0642298087477684}, "layernorm_embedding": {"bias": 0.002760232426226139, "scale": 0.002035953802987933}, "layers": {"FlaxBartDecoderLayers": {"encoder_attn": {"k_proj": {"bias": 4.814476142200874e-06, "kernel": 0.009996717795729637}, "out_proj": {"bias": 0.007056824862957001, "kernel": 0.035733651369810104}, "q_proj": {"bias": 0.00045872520422562957, "kernel": 0.010119627229869366}, "v_proj": {"bias": 0.014002962969243526, "kernel": 0.028897128999233246}}, "encoder_attn_layer_norm": {"bias": 0.010591115802526474, "scale": 0.01078032236546278}, "fc1": {"bias": 0.004002838861197233, "kernel": 0.09873048961162567}, "fc2": {"bias": 0.010417253710329533, "kernel": 0.09346094727516174}, "final_layer_norm": {"bias": 0.02925114333629608, "scale": 0.025131691247224808}, "self_attn": {"k_proj": {"bias": 1.495998731115833e-06, "kernel": 0.009474096819758415}, "out_proj": {"bias": 0.016716178506612778, "kernel": 0.03129136934876442}, "q_proj": {"bias": 0.0007107162964530289, "kernel": 0.00879898015409708}, "v_proj": {"bias": 0.01823665015399456, "kernel": 0.04575955122709274}}, "self_attn_layer_norm": {"bias": 0.006624831352382898, "scale": 0.0074623520486056805}}}}}}, "encoder": {"adapter": {"layers": {"0": {"conv": {"bias": 0.024443458765745163, "kernel": 0.04797496274113655}}, "1": {"conv": {"bias": 0.018976185470819473, "kernel": 0.03335544094443321}}, "2": {"conv": {"bias": 0.02025711163878441, "kernel": 0.05017269030213356}}}}, "encoder": {"layer_norm": {"bias": 0.09079575538635254, "scale": 0.0370769128203392}, "layers": {"FlaxWav2Vec2EncoderLayers": {"attention": {"k_proj": {"bias": 2.546356427046703e-06, "kernel": 0.018089286983013153}, "out_proj": {"bias": 0.002156489295884967, "kernel": 0.034899111837148666}, "q_proj": {"bias": 0.0026696091517806053, "kernel": 0.017847701907157898}, "v_proj": {"bias": 0.009628637693822384, "kernel": 0.03381096571683884}}, "feed_forward": {"intermediate_dense": {"bias": 0.004856355953961611, "kernel": 0.04390779137611389}, "output_dense": {"bias": 0.001986933406442404, "kernel": 0.03847503662109375}}, "final_layer_norm": {"bias": 0.02509871870279312, "scale": 0.028251897543668747}, "layer_norm": {"bias": 0.043583061546087265, "scale": 0.03362990543246269}}}, "pos_conv_embed": {"conv": {"bias": 0.0006979878526180983, "weight_g": 0.002805550117045641, "weight_v": 0.011207042261958122}}}, "feature_extractor": {"conv_layers": {"0": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "1": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "2": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "3": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "4": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "5": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "6": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}}}, "feature_projection": {"layer_norm": {"bias": 0.0032904886174947023, "scale": 0.003584273625165224}, "projection": {"bias": 0.0010678826365619898, "kernel": 0.03113219328224659}}, "masked_spec_embed": 0.0}}, "layer_param_norm/": {"decoder": {"model": {"decoder": {"embed_positions": {"embedding": 58.647159576416016}, "embed_tokens": {"embedding": 628.4832763671875}, "layernorm_embedding": {"bias": 2.4181270599365234, "scale": 13.876160621643066}, "layers": {"FlaxBartDecoderLayers": {"encoder_attn": {"k_proj": {"bias": 47.96188735961914, "kernel": 331.3609313964844}, "out_proj": {"bias": 6.137172698974609, "kernel": 227.73728942871094}, "q_proj": {"bias": 20.867891311645508, "kernel": 338.29461669921875}, "v_proj": {"bias": 3.5909812450408936, "kernel": 231.93850708007812}}, "encoder_attn_layer_norm": {"bias": 10.96327018737793, "scale": 57.53877639770508}, "fc1": {"bias": 26.014301300048828, "kernel": 345.999755859375}, "fc2": {"bias": 7.84496545791626, "kernel": 248.36624145507812}, "final_layer_norm": {"bias": 3.921175003051758, "scale": 63.50761413574219}, "self_attn": {"k_proj": {"bias": 59.543113708496094, "kernel": 279.90435791015625}, "out_proj": {"bias": 3.776594638824463, "kernel": 132.7462158203125}, "q_proj": {"bias": 32.19286346435547, "kernel": 283.0003662109375}, "v_proj": {"bias": 2.568603992462158, "kernel": 141.117431640625}}, "self_attn_layer_norm": {"bias": 8.895291328430176, "scale": 84.70443725585938}}}}}}, "encoder": {"adapter": {"layers": {"0": {"conv": {"bias": 1.1401727199554443, "kernel": 62.00471878051758}}, "1": {"conv": {"bias": 1.3068256378173828, "kernel": 59.942413330078125}}, "2": {"conv": {"bias": 1.5166486501693726, "kernel": 59.78715133666992}}}}, "encoder": {"layer_norm": {"bias": 0.2932465970516205, "scale": 4.228818893432617}, "layers": {"FlaxWav2Vec2EncoderLayers": {"attention": {"k_proj": {"bias": 19.381807327270508, "kernel": 553.2210083007812}, "out_proj": {"bias": 16.85451316833496, "kernel": 705.1119995117188}, "q_proj": {"bias": 40.90638732910156, "kernel": 545.735107421875}, "v_proj": {"bias": 15.575094223022461, "kernel": 696.6250610351562}}, "feed_forward": {"intermediate_dense": {"bias": 24.463790893554688, "kernel": 1377.1727294921875}, "output_dense": {"bias": 20.80949592590332, "kernel": 1303.4677734375}}, "final_layer_norm": {"bias": 32.52007293701172, "scale": 141.95835876464844}, "layer_norm": {"bias": 7.280069828033447, "scale": 45.696510314941406}}}, "pos_conv_embed": {"conv": {"bias": 15.224443435668945, "weight_g": 21.051162719726562, "weight_v": 213.89393615722656}}}, "feature_extractor": {"conv_layers": {"0": {"conv": {"bias": 0.5982058644294739, "kernel": 8.08896541595459}, "layer_norm": {"bias": 10.069783210754395, "scale": 10.451257705688477}}, "1": {"conv": {"bias": 4.74075174331665, "kernel": 90.8435287475586}, "layer_norm": {"bias": 6.922820091247559, "scale": 19.5467586517334}}, "2": {"conv": {"bias": 6.7732415199279785, "kernel": 146.13897705078125}, "layer_norm": {"bias": 9.044225692749023, "scale": 19.424888610839844}}, "3": {"conv": {"bias": 5.224758148193359, "kernel": 159.10508728027344}, "layer_norm": {"bias": 8.319666862487793, "scale": 17.64743423461914}}, "4": {"conv": {"bias": 4.434978008270264, "kernel": 157.35813903808594}, "layer_norm": {"bias": 9.193974494934082, "scale": 15.562357902526855}}, "5": {"conv": {"bias": 5.297643661499023, "kernel": 131.1835174560547}, "layer_norm": {"bias": 10.735219955444336, "scale": 13.812533378601074}}, "6": {"conv": {"bias": 5.615579128265381, "kernel": 136.41822814941406}, "layer_norm": {"bias": 12.515308380126953, "scale": 11.152680397033691}}}}, "feature_projection": {"layer_norm": {"bias": 9.262188911437988, "scale": 27.640396118164062}, "projection": {"bias": 4.317654132843018, "kernel": 88.17610931396484}}, "masked_spec_embed": 26.247730255126953}}, "train/learning_rate": 4.045656169182621e-05, "train/loss": 0.02121199481189251, "train/param_norm": 2553.945556640625, "_timestamp": 1661823749, "_runtime": 146982, "_step": 29975, "eval/loss": 1.021510124206543, "eval/wer": 0.05054961214661226, "eval/cer": 0.0362100285658818, "eval/step_10k": {"_type": "table-file", "sha256": "8b44e8a00a036a18ffdf81b4d076c8bf849ea6649001c69e94fa439b14f110ee", "size": 26434, "artifact_path": "wandb-client-artifact://18m0dj4hts3yiat04x5pvmncavkjapd5wb8bznb37vw8c0lqna3m2yjd1wtdrfstuoo7ejt2sphvjo0zuw1e5ne5d3qbkd7c1fylclfggig6us5tsmsj2uum5pchx48n:latest/eval/step_10k.table.json", "_latest_artifact_path": "wandb-client-artifact://18m0dj4hts3yiat04x5pvmncavkjapd5wb8bznb37vw8c0lqna3m2yjd1wtdrfstuoo7ejt2sphvjo0zuw1e5ne5d3qbkd7c1fylclfggig6us5tsmsj2uum5pchx48n:latest/eval/step_10k.table.json", "path": "media/table/eval/step_10k_10000_8b44e8a00a036a18ffdf.table.json", "ncols": 7, "nrows": 50}, "eval/step_20k": {"_type": "table-file", "sha256": "a0a50c5d8793ca99e4646f70c3624f8742c5285825bc1c59ab4083ac4de9d6e3", "size": 26657, "artifact_path": "wandb-client-artifact://13ri9hnxp93kf7dsdol2hs1j0v7bpkwwvujpi27awdck0fjm6vfog0dun9k9toif5xrt3cijlotddakikiw0bnbo3go679b4d2spq9c0w865vq0k9auiszkkbvev62fc:latest/eval/step_20k.table.json", "_latest_artifact_path": "wandb-client-artifact://13ri9hnxp93kf7dsdol2hs1j0v7bpkwwvujpi27awdck0fjm6vfog0dun9k9toif5xrt3cijlotddakikiw0bnbo3go679b4d2spq9c0w865vq0k9auiszkkbvev62fc:latest/eval/step_20k.table.json", "path": "media/table/eval/step_20k_20000_a0a50c5d8793ca99e464.table.json", "ncols": 7, "nrows": 50}}
+{"train/decoder_grad_norm": 0.18293321132659912, "train/decoder_param_norm": 1063.0654296875, "train/encoder_grad_norm": 0.148331880569458, "train/encoder_param_norm": 2323.336669921875, "train/grad_norm": 0.23551413416862488, "layer_grad_norm/": {"decoder": {"model": {"decoder": {"embed_positions": {"embedding": 0.008046639151871204}, "embed_tokens": {"embedding": 0.060666970908641815}, "layernorm_embedding": {"bias": 0.003096886444836855, "scale": 0.0024350089952349663}, "layers": {"FlaxBartDecoderLayers": {"encoder_attn": {"k_proj": {"bias": 5.256703389022732e-06, "kernel": 0.011271456256508827}, "out_proj": {"bias": 0.007880721241235733, "kernel": 0.03872065246105194}, "q_proj": {"bias": 0.0004976371419616044, "kernel": 0.01105893962085247}, "v_proj": {"bias": 0.015226359479129314, "kernel": 0.030586158856749535}}, "encoder_attn_layer_norm": {"bias": 0.01159473042935133, "scale": 0.012393548153340816}, "fc1": {"bias": 0.004439335782080889, "kernel": 0.10113218426704407}, "fc2": {"bias": 0.011387723498046398, "kernel": 0.09879624843597412}, "final_layer_norm": {"bias": 0.026725012809038162, "scale": 0.026244478300213814}, "self_attn": {"k_proj": {"bias": 1.8891130366682773e-06, "kernel": 0.010783703066408634}, "out_proj": {"bias": 0.016153542324900627, "kernel": 0.039619628340005875}, "q_proj": {"bias": 0.0008692306582815945, "kernel": 0.01054653525352478}, "v_proj": {"bias": 0.01716933213174343, "kernel": 0.04919762164354324}}, "self_attn_layer_norm": {"bias": 0.007215098012238741, "scale": 0.008393414318561554}}}}}}, "encoder": {"adapter": {"layers": {"0": {"conv": {"bias": 0.022141050547361374, "kernel": 0.049536462873220444}}, "1": {"conv": {"bias": 0.017486272379755974, "kernel": 0.03386249393224716}}, "2": {"conv": {"bias": 0.020221581682562828, "kernel": 0.05212334543466568}}}}, "encoder": {"layer_norm": {"bias": 0.07752241939306259, "scale": 0.03339512273669243}, "layers": {"FlaxWav2Vec2EncoderLayers": {"attention": {"k_proj": {"bias": 1.787670839803468e-06, "kernel": 0.016130739822983742}, "out_proj": {"bias": 0.001517767203040421, "kernel": 0.0303343553096056}, "q_proj": {"bias": 0.002082030288875103, "kernel": 0.01582256518304348}, "v_proj": {"bias": 0.00666964752599597, "kernel": 0.027823949232697487}}, "feed_forward": {"intermediate_dense": {"bias": 0.002947209170088172, "kernel": 0.03786110132932663}, "output_dense": {"bias": 0.0014471356989815831, "kernel": 0.03480111435055733}}, "final_layer_norm": {"bias": 0.014765393920242786, "scale": 0.01634989120066166}, "layer_norm": {"bias": 0.03281298279762268, "scale": 0.024696579203009605}}}, "pos_conv_embed": {"conv": {"bias": 0.0003536914009600878, "weight_g": 0.0017076540971174836, "weight_v": 0.0059150331653654575}}}, "feature_extractor": {"conv_layers": {"0": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "1": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "2": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "3": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "4": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "5": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}, "6": {"conv": {"bias": 0.0, "kernel": 0.0}, "layer_norm": {"bias": 0.0, "scale": 0.0}}}}, "feature_projection": {"layer_norm": {"bias": 0.00199363986030221, "scale": 0.0019663891289383173}, "projection": {"bias": 0.000548368610907346, "kernel": 0.015020000748336315}}, "masked_spec_embed": 0.0}}, "layer_param_norm/": {"decoder": {"model": {"decoder": {"embed_positions": {"embedding": 58.655296325683594}, "embed_tokens": {"embedding": 628.4320678710938}, "layernorm_embedding": {"bias": 2.423579692840576, "scale": 13.864307403564453}, "layers": {"FlaxBartDecoderLayers": {"encoder_attn": {"k_proj": {"bias": 47.95212173461914, "kernel": 331.52020263671875}, "out_proj": {"bias": 6.132379531860352, "kernel": 227.88941955566406}, "q_proj": {"bias": 20.880966186523438, "kernel": 338.4452819824219}, "v_proj": {"bias": 3.570829391479492, "kernel": 232.0838623046875}}, "encoder_attn_layer_norm": {"bias": 11.024534225463867, "scale": 57.68368148803711}, "fc1": {"bias": 26.054283142089844, "kernel": 346.79296875}, "fc2": {"bias": 7.841187953948975, "kernel": 248.94561767578125}, "final_layer_norm": {"bias": 3.914222002029419, "scale": 63.57343673706055}, "self_attn": {"k_proj": {"bias": 59.552547454833984, "kernel": 280.0233459472656}, "out_proj": {"bias": 3.7681872844696045, "kernel": 132.86544799804688}, "q_proj": {"bias": 32.200714111328125, "kernel": 283.1105651855469}, "v_proj": {"bias": 2.561887741088867, "kernel": 141.2297821044922}}, "self_attn_layer_norm": {"bias": 8.89633846282959, "scale": 84.71704864501953}}}}}}, "encoder": {"adapter": {"layers": {"0": {"conv": {"bias": 1.1991273164749146, "kernel": 62.47041320800781}}, "1": {"conv": {"bias": 1.3806315660476685, "kernel": 60.54581069946289}}, "2": {"conv": {"bias": 1.5864932537078857, "kernel": 60.361602783203125}}}}, "encoder": {"layer_norm": {"bias": 0.2923617959022522, "scale": 4.219600677490234}, "layers": {"FlaxWav2Vec2EncoderLayers": {"attention": {"k_proj": {"bias": 19.385311126708984, "kernel": 553.5303955078125}, "out_proj": {"bias": 16.84536361694336, "kernel": 705.300537109375}, "q_proj": {"bias": 40.93048858642578, "kernel": 546.0474243164062}, "v_proj": {"bias": 15.559945106506348, "kernel": 696.8070068359375}}, "feed_forward": {"intermediate_dense": {"bias": 24.44249725341797, "kernel": 1377.5845947265625}, "output_dense": {"bias": 20.795345306396484, "kernel": 1303.997314453125}}, "final_layer_norm": {"bias": 32.52366256713867, "scale": 142.01145935058594}, "layer_norm": {"bias": 7.272017478942871, "scale": 45.770084381103516}}}, "pos_conv_embed": {"conv": {"bias": 15.211658477783203, "weight_g": 21.0489501953125, "weight_v": 214.11346435546875}}}, "feature_extractor": {"conv_layers": {"0": {"conv": {"bias": 0.5982058644294739, "kernel": 8.08896541595459}, "layer_norm": {"bias": 10.069783210754395, "scale": 10.451257705688477}}, "1": {"conv": {"bias": 4.74075174331665, "kernel": 90.8435287475586}, "layer_norm": {"bias": 6.922820091247559, "scale": 19.5467586517334}}, "2": {"conv": {"bias": 6.7732415199279785, "kernel": 146.13897705078125}, "layer_norm": {"bias": 9.044225692749023, "scale": 19.424888610839844}}, "3": {"conv": {"bias": 5.224758148193359, "kernel": 159.10508728027344}, "layer_norm": {"bias": 8.319666862487793, "scale": 17.64743423461914}}, "4": {"conv": {"bias": 4.434978008270264, "kernel": 157.35813903808594}, "layer_norm": {"bias": 9.193974494934082, "scale": 15.562357902526855}}, "5": {"conv": {"bias": 5.297643661499023, "kernel": 131.1835174560547}, "layer_norm": {"bias": 10.735219955444336, "scale": 13.812533378601074}}, "6": {"conv": {"bias": 5.615579128265381, "kernel": 136.41822814941406}, "layer_norm": {"bias": 12.515308380126953, "scale": 11.152680397033691}}}}, "feature_projection": {"layer_norm": {"bias": 9.234997749328613, "scale": 27.593135833740234}, "projection": {"bias": 4.323590278625488, "kernel": 88.11954498291016}}, "masked_spec_embed": 26.247730255126953}}, "train/learning_rate": 2.0254545233910903e-05, "train/loss": 0.008378183469176292, "train/param_norm": 2554.995361328125, "_timestamp": 1661871981, "_runtime": 195214, "_step": 39975, "eval/loss": 0.9876799583435059, "eval/wer": 0.04759016212639241, "eval/cer": 0.035079873533571844, "eval/step_10k": {"_type": "table-file", "sha256": "8b44e8a00a036a18ffdf81b4d076c8bf849ea6649001c69e94fa439b14f110ee", "size": 26434, "artifact_path": "wandb-client-artifact://18m0dj4hts3yiat04x5pvmncavkjapd5wb8bznb37vw8c0lqna3m2yjd1wtdrfstuoo7ejt2sphvjo0zuw1e5ne5d3qbkd7c1fylclfggig6us5tsmsj2uum5pchx48n:latest/eval/step_10k.table.json", "_latest_artifact_path": "wandb-client-artifact://18m0dj4hts3yiat04x5pvmncavkjapd5wb8bznb37vw8c0lqna3m2yjd1wtdrfstuoo7ejt2sphvjo0zuw1e5ne5d3qbkd7c1fylclfggig6us5tsmsj2uum5pchx48n:latest/eval/step_10k.table.json", "path": "media/table/eval/step_10k_10000_8b44e8a00a036a18ffdf.table.json", "ncols": 7, "nrows": 50}, "eval/step_20k": {"_type": "table-file", "sha256": "a0a50c5d8793ca99e4646f70c3624f8742c5285825bc1c59ab4083ac4de9d6e3", "size": 26657, "artifact_path": "wandb-client-artifact://13ri9hnxp93kf7dsdol2hs1j0v7bpkwwvujpi27awdck0fjm6vfog0dun9k9toif5xrt3cijlotddakikiw0bnbo3go679b4d2spq9c0w865vq0k9auiszkkbvev62fc:latest/eval/step_20k.table.json", "_latest_artifact_path": "wandb-client-artifact://13ri9hnxp93kf7dsdol2hs1j0v7bpkwwvujpi27awdck0fjm6vfog0dun9k9toif5xrt3cijlotddakikiw0bnbo3go679b4d2spq9c0w865vq0k9auiszkkbvev62fc:latest/eval/step_20k.table.json", "path": "media/table/eval/step_20k_20000_a0a50c5d8793ca99e464.table.json", "ncols": 7, "nrows": 50}, "eval/step_30k": {"_type": "table-file", "sha256": "509ad8614e16ae2800f100e82fd40356d3559e0425cf50e8e19c3155dc6c604e", "size": 26610, "artifact_path": "wandb-client-artifact://1v7mr89p2ldfoyyvrj6x6g0s8qpvgnioclm4puk5i0wphipeujkyna0xf7y63nt4a5qivkdzytt4lzhwuraoe53a5683ln1yzntyw9kh7vhcxz7oci70rpw62o1tz6ho:latest/eval/step_30k.table.json", "_latest_artifact_path": "wandb-client-artifact://1v7mr89p2ldfoyyvrj6x6g0s8qpvgnioclm4puk5i0wphipeujkyna0xf7y63nt4a5qivkdzytt4lzhwuraoe53a5683ln1yzntyw9kh7vhcxz7oci70rpw62o1tz6ho:latest/eval/step_30k.table.json", "path": "media/table/eval/step_30k_30000_509ad8614e16ae2800f1.table.json", "ncols": 7, "nrows": 50}}

wandb/run-20220828_085247-2hx8pk65/run-2hx8pk65.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dbd52b7487584ca71d80fed0e3182d1bd195ab787ea787199645db598d730074
-size 12917901
+oid sha256:704bf667df7d300df731d8d0bb3301cfbf4b05488ba632c3bc0bcf3bd69bea8c
+size 17153865