Repository: facebookresearch/ParlAI

Issue:
World logging not working with eval_model.py with dynamic batching
Use this to open other questions or issues, and provide context here.
When I tried `multiprocessing_eval.py --dynamic-batching full`, it gave me this error `AttributeError: 'DynamicBatchWorld' object has no attribute 'acts'`. It seems dynamic batching is not supported in `world_logging`?




Discussion:

Resolving patch:
diff --git a/parlai/core/worlds.py b/parlai/core/worlds.py
--- a/parlai/core/worlds.py
+++ b/parlai/core/worlds.py
@@ -1110,8 +1110,10 @@ def __init__(self, opt: Opt, world: Union[DialogPartnerWorld, MultiWorld]):
 
     def reset(self):
         super().reset()
+        self._task_acts = [None for _ in range(self._BUFFER_SIZE)]
         self._obs = [None for _ in range(self._BUFFER_SIZE)]
         self._scores = [None for _ in range(self._BUFFER_SIZE)]
+        self.acts = [None, None]
 
         self.number_parleys = 0
         self.total_exs = 0
@@ -1183,6 +1185,12 @@ def parley(self):
                 self.worlds[i].parley_init()
 
             act = self.worlds[i].get_task_agent().act()
+
+            # we log the task act and the index of the act
+            # in the buffer for world logging purposes
+            self._task_acts[i] = act  # for world logging
+            self._task_acts[i]['dyn_batch_idx'] = i
+
             obs = self.worlds[i].get_model_agent().observe(act)
             self._obs[i] = obs
 
@@ -1245,15 +1253,20 @@ def parley(self):
 
         # great, this batch is good to go! let's run it!
         acts = self.world.get_model_agent().batch_act([self._obs[i] for i in batch])
+        self.acts = [[self._task_acts[i] for i in batch], acts]
         # broadcast the results back to all the models
         for i, act in zip(batch, acts):
             # we need to make sure that the teachers saw the result
             self.worlds[i].get_task_agent().observe(act)
             # and that the agent copies saw their own voice
             self.worlds[i].get_model_agent().self_observe(act)
-
             # move these worlds forward
             act = self.worlds[i].get_task_agent().act()
+            # we log the task act and the index of the act
+            # in the buffer for world logging purposes
+            self._task_acts[i] = act
+            self._task_acts[i]['dyn_batch_idx'] = i
+            # save the observations to form a batch
             obs = self.worlds[i].get_model_agent().observe(act)
             self._scores[i] = self._score(obs)
             self._obs[i] = obs
diff --git a/parlai/utils/world_logging.py b/parlai/utils/world_logging.py
--- a/parlai/utils/world_logging.py
+++ b/parlai/utils/world_logging.py
@@ -8,7 +8,7 @@
 Useful utilities for logging actions/observations in a world.
 """
 
-from parlai.core.worlds import BatchWorld
+from parlai.core.worlds import BatchWorld, DynamicBatchWorld
 from parlai.utils.misc import msg_to_str
 from parlai.utils.conversations import Conversations
 import parlai.utils.logging as logging
@@ -81,15 +81,21 @@ def _add_episode(self, episode):
         self._logs.append(episode)
 
     def _is_batch_world(self, world):
-        return isinstance(world, BatchWorld) and len(world.worlds) > 1
+        return (
+            isinstance(world, BatchWorld) or isinstance(world, DynamicBatchWorld)
+        ) and len(world.worlds) > 1
 
     def _log_batch(self, world):
         batch_act = world.get_acts()
         parleys = zip(*batch_act)
         for i, parley in enumerate(parleys):
-            self._add_msgs(parley, idx=i)
-            if world.worlds[i].episode_done():
-                self.reset_world(idx=i)
+            # in dynamic batching, we only return `batchsize` acts, but the
+            # 'dyn_batch_idx' key in the task act corresponds the episode index
+            # in the buffer
+            idx = parley[0]['dyn_batch_idx'] if 'dyn_batch_idx' in parley[0] else i
+            self._add_msgs(parley, idx=idx)
+            if world.worlds[idx].episode_done():
+                self.reset_world(idx=idx)
 
     def log(self, world):
         """
@@ -140,6 +146,7 @@ def write_parlai_format(self, outfile):
                 fw.write('\n')
 
     def write_conversations_format(self, outfile, world):
+        logging.info(f'Saving log to {outfile} in Conversations format')
         Conversations.save_conversations(
             self._logs,
             outfile,


Test patch:
diff --git a/tests/test_dynamicbatching.py b/tests/test_dynamicbatching.py
--- a/tests/test_dynamicbatching.py
+++ b/tests/test_dynamicbatching.py
@@ -4,12 +4,14 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
-from typing import Dict, Any
-
-import unittest
 from parlai.core.opt import Opt
-import parlai.utils.testing as testing_utils
 from parlai.tasks.integration_tests.agents import NUM_TEST, EXAMPLE_SIZE
+from parlai.utils.conversations import Conversations
+import parlai.utils.testing as testing_utils
+
+import os
+from typing import Dict, Any
+import unittest
 
 _TASK = 'integration_tests:variable_length'
 
@@ -97,6 +99,34 @@ def test_multiworld_stream(self):
             datatype='train:stream',
         )
 
+    def test_world_logging(self):
+        with testing_utils.tempdir() as tmpdir:
+            save_report = os.path.join(tmpdir, 'report')
+            testing_utils.eval_model(
+                dict(
+                    model_file='zoo:unittest/transformer_generator2/model',
+                    task='integration_tests:multiturn_candidate',
+                    save_world_logs=True,
+                    report_filename=save_report,
+                    truncate=1024,
+                    dynamic_batching='full',
+                    batchsize=4,
+                )
+            )
+            convo_fle = (
+                str(save_report)
+                + '_integration_tests:multiturn_candidate_replies.jsonl'
+            )
+            convos = Conversations(convo_fle)
+            for convo in convos:
+                self.assertEquals(len(convo), 2 * 4)  # each episode is 4 turns
+                # now assert that they are all from the same dynamic batch index
+                dyn_batch_idx = convo[0]['dyn_batch_idx']
+                for i, turn in enumerate(convo):
+                    if i % 2 == 0 and i > 0:
+                        # we log the batch index in the teacher acts only
+                        self.assertEquals(dyn_batch_idx, turn['dyn_batch_idx'])
+
     def test_weird_batchsize(self):
         # intentionally a difficult number
         self._test_correct_processed(NUM_TEST, batchsize=7)