Repository: facebookresearch/ParlAI

Issue:
Using world-logs while evaluating a model with more than one task, only keeps the world log for the last task.
Trying to evaluate one mode with two tasks, while keeping the model outputs in world logs, I noticed that there was no result from one of the tasks in the world log. To reproduce this one may try running
```
 parlai eval_model -t wizard_of_wikipedia,babi \
--world-logs /some/path/world-log \
--num-examples 1 --model repeat_label
```
Running this, there is only a single line in the `world-log.json` file. Checking the file you can see `"id": "babi:Task1k:1"` which may mean that parlai is generating separate world log files for each task, but assumes same name for all of them and writes over the previous ones.


Discussion:
This issue has not had activity in 30 days. Please feel free to reopen if you have more issues. You may apply the "never-stale" tag to prevent this from happening.

Resolving patch:
diff --git a/parlai/scripts/eval_model.py b/parlai/scripts/eval_model.py
--- a/parlai/scripts/eval_model.py
+++ b/parlai/scripts/eval_model.py
@@ -118,13 +118,27 @@ def _save_eval_stats(opt, report):
         f.write("\n")  # for jq
 
 
+def get_task_world_logs(task, world_logs, is_multitask=False):
+    if not is_multitask:
+        return world_logs
+    else:
+        base_outfile, extension = os.path.splitext(world_logs)
+        return f'{base_outfile}_{task}{extension}'
+
+
 def _eval_single_world(opt, agent, task):
     logging.info(f'Evaluating task {task} using datatype {opt.get("datatype")}.')
     # set up world logger
-    world_logger = WorldLogger(opt) if opt['world_logs'] else None
-
     task_opt = opt.copy()  # copy opt since we're editing the task
     task_opt['task'] = task
+    # add task suffix in case of multi-tasking
+    if opt['world_logs']:
+        task_opt['world_logs'] = get_task_world_logs(
+            task, task_opt['world_logs'], is_multitask=len(opt['task'].split(',')) > 1
+        )
+
+    world_logger = WorldLogger(task_opt) if task_opt['world_logs'] else None
+
     world = create_task(task_opt, agent)  # create worlds for tasks
 
     # set up logging
@@ -161,10 +175,10 @@ def _eval_single_world(opt, agent, task):
         world_logger.reset()  # add final acts to logs
         if is_distributed():
             rank = get_rank()
-            base_outfile, extension = os.path.splitext(opt['world_logs'])
+            base_outfile, extension = os.path.splitext(task_opt['world_logs'])
             outfile = base_outfile + f'_{rank}' + extension
         else:
-            outfile = opt['world_logs']
+            outfile = task_opt['world_logs']
         world_logger.write(outfile, world, file_format=opt['save_format'])
 
     report = aggregate_unnamed_reports(all_gather_list(world.report()))


Test patch:
diff --git a/tests/test_eval_model.py b/tests/test_eval_model.py
--- a/tests/test_eval_model.py
+++ b/tests/test_eval_model.py
@@ -8,6 +8,7 @@
 import pytest
 import unittest
 import parlai.utils.testing as testing_utils
+from parlai.scripts.eval_model import get_task_world_logs
 
 
 class TestEvalModel(unittest.TestCase):
@@ -227,6 +228,34 @@ def test_save_report(self):
                 json_lines = f.readlines()
             assert len(json_lines) == 100
 
+    def test_save_multiple_logs(self):
+        """
+        Test that we can save multiple world_logs from eval model on multiple tasks.
+        """
+        with testing_utils.tempdir() as tmpdir:
+            log_report = os.path.join(tmpdir, 'world_logs.jsonl')
+            save_report = os.path.join(tmpdir, 'report')
+            multitask = 'integration_tests,blended_skill_talk'
+            opt = dict(
+                task=multitask,
+                model='repeat_label',
+                datatype='valid',
+                batchsize=97,
+                num_examples=100,
+                display_examples=False,
+                world_logs=log_report,
+                report_filename=save_report,
+            )
+            valid, test = testing_utils.eval_model(opt)
+
+            for task in multitask.split(','):
+                task_log_report = get_task_world_logs(
+                    task, log_report, is_multitask=True
+                )
+                with PathManager.open(task_log_report) as f:
+                    json_lines = f.readlines()
+                assert len(json_lines) == 100
+
 
 if __name__ == '__main__':
     unittest.main()