Repository: espnet/espnet

Issue:
Initializaing Language model for RNN-Transducer 
I am trying to initialise the  RNN-decoder using pre-trained language model with a similar architecture (dlayers, dunits) . 
I get following warning messages : 
```
2021-02-12 13:34:53,311 (asr_init:259) WARNING: model(s) found for pre-initialization
2021-02-12 13:34:53,315 (asr_init:203) WARNING: reading model parameters from /exp/train_rnnlm_pytorch_rnnt4_rnnlm_char/rnnlm.model.20
2021-02-12 13:34:58,336 (asr_init:150) WARNING: module(s) ['att.', ' dec.'] don't match or (partially match) available modules in model.
2021-02-12 13:34:58,337 (asr_init:152) WARNING: for information, the existing modules in model are:
2021-02-12 13:34:58,337 (asr_init:153) WARNING: ['predictor.embed.weight', 'predictor.rnn.0.weight_ih', 'predictor.rnn.0.weight_hh', 'predictor.rnn.0.bias_ih', 'predictor.rnn.0.bias_hh', 'predictor.lo.weight', 'predictor.lo.bias']
2021-02-12 13:34:58,337 (asr_init:243) WARNING: loading [] from model: exp/train_rnnlm_pytorch_rnnt4_rnnlm_char/rnnlm.model.20
e
```
It seems the initalization is not correct, It will be great if somebody can point to sample configuration to do the same.  
Thanks


Discussion:
Hi,

Sorry, we made some changes in the transducer decoder and decoder initialization with a pre-trained LM is disabled currently. I'll fix that as soon as possible.

Resolving patch:
diff --git a/espnet/asr/pytorch_backend/asr_init.py b/espnet/asr/pytorch_backend/asr_init.py
--- a/espnet/asr/pytorch_backend/asr_init.py
+++ b/espnet/asr/pytorch_backend/asr_init.py
@@ -92,34 +92,28 @@ def get_partial_state_dict(model_state_dict, modules):
     return new_state_dict
 
 
-def get_partial_lm_state_dict(model_state_dict, modules):
-    """Create compatible ASR state_dict from model_state_dict (LM).
-
-    The keys for specified modules are modified to match ASR decoder modules keys.
+def get_lm_state_dict(lm_state_dict):
+    """Create compatible ASR decoder state dict from LM state dict.
 
     Args:
-        model_state_dict (OrderedDict): trained model state_dict
-        modules (list): specified module list for transfer
+        lm_state_dict (OrderedDict): pre-trained LM state_dict
 
     Return:
-        new_state_dict (OrderedDict): the updated state_dict
-        new_mods (list): the updated module list
+        new_state_dict (OrderedDict): LM state_dict with updated keys
 
     """
     new_state_dict = OrderedDict()
-    new_modules = []
 
-    for key, value in list(model_state_dict.items()):
-        if key == "predictor.embed.weight" and "predictor.embed." in modules:
-            new_key = "dec.embed.weight"
-            new_state_dict[new_key] = value
-            new_modules += [new_key]
-        elif "predictor.rnn." in key and "predictor.rnn." in modules:
-            new_key = "dec.decoder." + key.split("predictor.rnn.", 1)[1]
+    for key, value in list(lm_state_dict.items()):
+        if key == "predictor.embed.weight":
+            new_state_dict["dec.embed.weight"] = value
+        elif key.startswith("predictor.rnn."):
+            _split = key.split(".")
+
+            new_key = "dec.decoder." + _split[2] + "." + _split[3] + "_l0"
             new_state_dict[new_key] = value
-            new_modules += [new_key]
 
-    return new_state_dict, new_modules
+    return new_state_dict
 
 
 def filter_modules(model_state_dict, modules):
@@ -202,7 +196,7 @@ def get_trained_model_state_dict(model_path):
     if "rnnlm" in model_path:
         logging.warning("reading model parameters from %s", model_path)
 
-        return torch.load(model_path), True
+        return get_lm_state_dict(torch.load(model_path))
 
     idim, odim, args = get_model_conf(model_path, conf_path)
 
@@ -222,7 +216,7 @@ def get_trained_model_state_dict(model_path):
         or isinstance(model, TTSInterface)
     )
 
-    return model.state_dict(), False
+    return model.state_dict()
 
 
 def load_trained_modules(idim, odim, args, interface=ASRInterface):
@@ -263,30 +257,23 @@ def print_new_keys(state_dict, modules, model_path):
     ]:
         if model_path is not None:
             if os.path.isfile(model_path):
-                model_state_dict, is_lm = get_trained_model_state_dict(model_path)
+                model_state_dict = get_trained_model_state_dict(model_path)
 
                 modules = filter_modules(model_state_dict, modules)
-                if is_lm:
-                    partial_state_dict, modules = get_partial_lm_state_dict(
-                        model_state_dict, modules
-                    )
-                    print_new_keys(partial_state_dict, modules, model_path)
-                else:
-                    partial_state_dict = get_partial_state_dict(
-                        model_state_dict, modules
-                    )
-
-                    if partial_state_dict:
-                        if transfer_verification(
-                            main_state_dict, partial_state_dict, modules
-                        ):
-                            print_new_keys(partial_state_dict, modules, model_path)
-                            main_state_dict.update(partial_state_dict)
-                        else:
-                            logging.warning(
-                                f"modules {modules} in model {model_path} "
-                                f"don't match your training config",
-                            )
+
+                partial_state_dict = get_partial_state_dict(model_state_dict, modules)
+
+                if partial_state_dict:
+                    if transfer_verification(
+                        main_state_dict, partial_state_dict, modules
+                    ):
+                        print_new_keys(partial_state_dict, modules, model_path)
+                        main_state_dict.update(partial_state_dict)
+                    else:
+                        logging.warning(
+                            f"modules {modules} in model {model_path} "
+                            f"don't match your training config",
+                        )
             else:
                 logging.warning("model was not found : %s", model_path)
 
diff --git a/espnet/bin/asr_train.py b/espnet/bin/asr_train.py
--- a/espnet/bin/asr_train.py
+++ b/espnet/bin/asr_train.py
@@ -379,7 +379,7 @@ def get_parser(parser=None, required=True):
     )
     parser.add_argument(
         "--dec-init-mods",
-        default="att., dec.",
+        default="att.,dec.",
         type=lambda s: [str(mod) for mod in s.split(",") if s != ""],
         help="List of decoder modules to initialize, separated by a comma.",
     )


Test patch:
diff --git a/ci/test_integration.sh b/ci/test_integration.sh
--- a/ci/test_integration.sh
+++ b/ci/test_integration.sh
@@ -67,6 +67,16 @@ echo "=== ASR (backend=pytorch, model=conformer-transducer) ==="
 ./run.sh --python "${python}" --stage 4 --train-config conf/train_conformer_transducer.yaml \
         --decode-config conf/decode_transducer.yaml
 
+# test finetuning
+## test transfer learning
+echo "=== ASR (backend=pytorch, model=rnnt, transfer_learning=enc) ==="
+./run.sh --python "${python}" --stage 4 --train-config conf/train_transducer_pre_init_enc.yaml \
+         --decode-config conf/decode_transducer.yaml
+echo "=== ASR (backend=pytorch, model=rnnt, transfer_learning=LM) ==="
+./run.sh --python "${python}" --stage 4 --train-config conf/train_transducer_pre_init_lm.yaml \
+         --decode-config conf/decode_transducer.yaml
+## to do: cover all tasks + freezing option
+
 echo "==== ASR (backend=pytorch num-encs 2) ==="
 ./run.sh --python "${python}" --stage 2 --train-config ./conf/train_mulenc2.yaml --decode-config ./conf/decode_mulenc2.yaml --mulenc true
 # Remove generated files in order to reduce the disk usage