Repository: flairNLP/flair

Issue:
Loading a saved LM model forgets previous best validation score
When i load a previous trained LM model, and run few more epochs even if my model gets worst score then before it will update best-lm.pt

as code suggests, as trainig start  Best val score is set to 10000 by default even if my previous score was far less than that. 


Discussion:
Good point, we need to serialize also the best score to enable better training continuation from a saved model! 

Resolving patch:
diff --git a/flair/models/language_model.py b/flair/models/language_model.py
--- a/flair/models/language_model.py
+++ b/flair/models/language_model.py
@@ -16,7 +16,8 @@ def __init__(self,
                  nlayers: int,
                  embedding_size: int = 100,
                  nout=None,
-                 dropout=0.5):
+                 dropout=0.5,
+                 best_score=None):
 
         super(LanguageModel, self).__init__()
 
@@ -49,6 +50,8 @@ def __init__(self,
 
         self.init_weights()
 
+        self.best_score = best_score
+
         # auto-spawn on GPU if available
         if torch.cuda.is_available():
             self.cuda()
@@ -123,13 +126,16 @@ def load_language_model(cls, model_file):
         else:
             state = torch.load(model_file)
 
+        best_score = state['best_score'] if 'best_score' in state else None
+
         model = LanguageModel(state['dictionary'],
                                              state['is_forward_lm'],
                                              state['hidden_size'],
                                              state['nlayers'],
                                              state['embedding_size'],
                                              state['nout'],
-                                             state['dropout'])
+                                             state['dropout'],
+                                             best_score)
         model.load_state_dict(state['state_dict'])
         model.eval()
         if torch.cuda.is_available():
@@ -145,7 +151,8 @@ def save(self, file):
             'nlayers': self.nlayers,
             'embedding_size': self.embedding_size,
             'nout': self.nout,
-            'dropout': self.dropout
+            'dropout': self.dropout,
+            'best_score': self.best_score
         }
         torch.save(model_state, file, pickle_protocol=4)
 
diff --git a/flair/trainers/language_model_trainer.py b/flair/trainers/language_model_trainer.py
--- a/flair/trainers/language_model_trainer.py
+++ b/flair/trainers/language_model_trainer.py
@@ -185,7 +185,7 @@ def train(self,
         try:
 
             epoch = 0
-            best_val_loss = 100000000
+            best_val_loss = self.model.best_score if self.model.best_score is not None else 100000000
             optimizer = torch.optim.SGD(self.model.parameters(), lr=learning_rate)
             scheduler: ReduceLROnPlateau = ReduceLROnPlateau(optimizer, verbose=True, factor=anneal_factor,
                                                              patience=patience)
@@ -269,6 +269,7 @@ def train(self,
 
                 # Save the model if the validation loss is the best we've seen so far.
                 if val_loss < best_val_loss:
+                    self.model.best_score = best_val_loss
                     self.model.save(savefile)
                     best_val_loss = val_loss
 


Test patch:
diff --git a/tests/test_model_integration.py b/tests/test_model_integration.py
--- a/tests/test_model_integration.py
+++ b/tests/test_model_integration.py
@@ -302,5 +302,8 @@ def test_train_language_model(results_base_path, resources_path):
     assert (text is not None)
     assert (len(text) == 100)
 
+    loaded_language_model = LanguageModel.load_language_model(str(results_base_path / 'best-lm.pt'))
+    assert (loaded_language_model.best_score < 100)
+
     # clean up results directory
     shutil.rmtree(results_base_path, ignore_errors=True)