Training in progress, step 1500

.ipynb_checkpoints/eval-checkpoint.py

@@ -85,7 +85,7 @@ def create_vocabulary_from_data(
     )
 
 
-    vocab_dict = {v: k for k, v in enumerate(sorted(list(vocabs["test"]["vocab"])))}
+    vocab_dict = {v: k for k, v in enumerate(sorted(vocabs["test"]["vocab"][0]))}
 
     # replace white space with delimiter token
     if word_delimiter_token is not None:
@@ -150,9 +150,9 @@ def main(args):
                     pad_token=pad_token,
                 )
         print(vocab_dict)
-        print(set(vocab_dict) - set(tokenizer.get_vocab()))
-    
+        print("OOV chars:", set(vocab_dict) - set(tokenizer.get_vocab()))    
 
+        
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
 

eval.py

@@ -85,7 +85,7 @@ def create_vocabulary_from_data(
     )
 
 
-    vocab_dict = {v: k for k, v in enumerate(sorted(list(vocabs["test"]["vocab"])))}
+    vocab_dict = {v: k for k, v in enumerate(sorted(vocabs["test"]["vocab"][0]))}
 
     # replace white space with delimiter token
     if word_delimiter_token is not None:
@@ -150,9 +150,9 @@ def main(args):
                     pad_token=pad_token,
                 )
         print(vocab_dict)
-        print(set(vocab_dict) - set(tokenizer.get_vocab()))
-    
+        print("OOV chars:", set(vocab_dict) - set(tokenizer.get_vocab()))    
 
+        
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
 

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0f8b5632f9e07f55fba70e10da1295b474bf5e556941e6a29a922295b5be21de
+oid sha256:424c9eb48f2b29370f0999f3880f59229a4354c0e426a6210488c3332d18a561
 size 1262054897

runs/Feb02_06-54-25_job-699ba53c-fea9-4eb2-81af-a97f440eaa45/events.out.tfevents.1643785646.job-699ba53c-fea9-4eb2-81af-a97f440eaa45.33872.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f4b9f85ce3736a50be3ac6b8b235dbf084d34a5ecbdea0c0d946a7ab0368daf4
-size 6955
+oid sha256:b4f9ad2746642360fd16b28a7e680fa5af325c16f6c7f11b572ffb75b09c8480
+size 8058