Repository: nyu-mll/jiant

Issue:
target_train_val_interval is ignored.
Varying it doesn't work, even for tasks without task-specific overrides, and I don't see any code that uses it.




Discussion:
It looks like get_task_attr falls back on max_vals when it doesn't find a task-specific attribute.

Resolving patch:
diff --git a/src/models.py b/src/models.py
--- a/src/models.py
+++ b/src/models.py
@@ -587,8 +587,8 @@ def get_task_specific_params(args, task_name):
         AllenNLP Params object of task-specific params.
     """
 
-    def _get_task_attr(attr_name):
-        return config.get_task_attr(args, task_name, attr_name)
+    def _get_task_attr(attr_name, default=None):
+        return config.get_task_attr(args, task_name, attr_name, default)
 
     params = {}
     params["cls_type"] = _get_task_attr("classifier")
diff --git a/src/trainer.py b/src/trainer.py
--- a/src/trainer.py
+++ b/src/trainer.py
@@ -33,8 +33,16 @@ def build_trainer_params(args, task_names, phase="pretrain"):
     In particular, we want to search args for task specific training parameters.
     """
 
-    def _get_task_attr(attr_name):
-        return config.get_task_attr(args, task_names, attr_name)
+    def _get_attr(attr_name, default=None):
+        if phase == "pretrain":
+            # We ignore task-specific trainer attributes during pretraining.
+            if default is not None:
+                return default
+            return args[attr_name]
+        else:
+            # During target task training, we get task-specific attributes if available.
+            assert len(task_names) == 1
+            return config.get_task_attr(args, task_names[0], attr_name, default)
 
     params = {}
     train_opts = [
@@ -45,8 +53,6 @@ def _get_task_attr(attr_name):
         "lr_patience",
         "patience",
         "scheduler_threshold",
-    ]
-    extra_opts = [
         "sent_enc",
         "d_hid",
         "warmup",
@@ -57,21 +63,25 @@ def _get_task_attr(attr_name):
         "keep_all_checkpoints",
         "val_data_limit",
         "max_epochs",
-        "training_data_fraction",
+        "dec_val_scale",
     ]
     for attr in train_opts:
-        params[attr] = _get_task_attr(attr)
-    for attr in extra_opts:
-        if attr == "training_data_fraction":
-            if phase == "pretrain":
-                params[attr] = args.pretrain_data_fraction
-            else:
-                params[attr] = args.target_train_data_fraction
-        else:
-            params[attr] = getattr(args, attr)
-    params["max_vals"] = _get_task_attr("max_vals")
-    params["val_interval"] = _get_task_attr("val_interval")
-    params["dec_val_scale"] = _get_task_attr("dec_val_scale")
+        params[attr] = _get_attr(attr)
+
+    # Special case: If no task-specific arg is found for these, we'll need to fall back to a
+    # phase-specific default.
+    params["max_vals"] = _get_attr(
+        "max_vals", default=args.target_train_max_vals if phase == "target_train" else None
+    )
+    params["val_interval"] = _get_attr(
+        "val_interval", default=args.target_train_val_interval if phase == "target_train" else None
+    )
+    params["training_data_fraction"] = _get_attr(
+        "training_data_fraction",
+        default=args.target_train_data_fraction
+        if phase == "target_train"
+        else args.pretrain_data_fraction,
+    )
 
     return Params(params)
 
@@ -1171,19 +1181,19 @@ def _save_target_train_checkpoints(
         self, epoch, best_str, new_best_macro, task_states, model_state, training_state
     ):
         """
-        Saves task specific checkpoints. If transfer_paradigm=finetune, then each task-specific checkpoint 
-        will contain different weights for BERT. If transfer_paradigm=frozen, the only difference will be 
+        Saves task specific checkpoints. If transfer_paradigm=finetune, then each task-specific checkpoint
+        will contain different weights for BERT. If transfer_paradigm=frozen, the only difference will be
         in the weights for the task-specific modules.
 
-        Parameters 
+        Parameters
         --------------------
             - epoch: int
             - phase: str
             - new_best_macro: bool
             - task_states: dict
-            - model_state: dict of weights 
+            - model_state: dict of weights
             - model_state: experiment model
-        
+
         Returns
         --------------------
         None
diff --git a/src/utils/config.py b/src/utils/config.py
--- a/src/utils/config.py
+++ b/src/utils/config.py
@@ -100,25 +100,17 @@ def __str__(self):
         return json.dumps(self.as_dict(), indent=2, sort_keys=True)
 
 
-def get_task_attr(
-    args: Type[Params], task_names: Union[str, Sequence[str]], attr_name: str, default=None
-):
+def get_task_attr(args: Type[Params], task_name: str, attr_name: str, default=None):
     """ Get a task-specific param.
 
-    Look in args.task_name.attr_name, then args.task_name_attr_name,
+    Look in args.task_name.attr_name, then fall back to default (if provided),
     then fall back to args.attr_name.
     """
-    if isinstance(task_names, str):
-        task_names = [task_names]
-
-    for task_name in task_names:
-        if task_name in args and (attr_name in args[task_name]):
-            return args[task_name][attr_name]
-        compound_key = "%s_%s" % (task_name, attr_name)
-        if compound_key in args:
-            return args[compound_key]
-    #  return args[attr_name]
-    return args.get(attr_name, default)
+    if task_name in args and (attr_name in args[task_name]):
+        return args[task_name][attr_name]
+    if default is not None:
+        return default
+    return args[attr_name]
 
 
 def params_from_file(config_files: Union[str, Iterable[str]], overrides: str = None):


Test patch: