Repository: facebookresearch/ParlAI

Issue:
[Seeker self chat] RuntimeError: Message already contains key `knowledge_response`. If this was intentional, please use the function `force_set(key, value)`.
Hi, I'm still trying to figure out how to run Seeker in self chat with a search engine, I'm facing the following issue.

```bash
parlai self_chat --model-file zoo:seeker/seeker_dialogue_400M/model  --num-self-chats 200 --display-examples True --seed_messages_from_file="./seeds.txt" --outfile="./out"  --init-opt gen/seeker_dialogue --search-server 0.0.0.0:8083
```

```
Current ParlAI commit: a84d28c4f9e9a56ddf9f8c3b124b61ac5ae991f5
Current internal commit: a84d28c4f9e9a56ddf9f8c3b124b61ac5ae991f5
Current fb commit: a84d28c4f9e9a56ddf9f8c3b124b61ac5ae991f5
```


>...
>20:40:13 | creating task(s): self_chat
[context]: Hi!
/home/gpu/ParlAI/parlai/core/torch_generator_agent.py:1728: UserWarning: __floordiv__ is deprecated, and its behavior will change in a future version of pytorch. It currently rounds toward 0 (like the 'trunc' function NOT 'floor'). This results in incorrect rounding for negative values. To keep the current behavior, use torch.div(a, b, rounding_mode='trunc'), or for actual floor division, use torch.div(a, b, rounding_mode='floor').
  hyp_ids = best_idxs // voc_size
20:40:13 | Search Queries: ['conscious experiences']
20:40:13 | sending search request to http://0.0.0.0:8083
20:40:23 | URLS:
https://www.sciencedirect.com/topics/computer-science/conscious-experience
https://nobaproject.com/modules/consciousness
https://www.open.edu/openlearn/mod/resource/view.php?id=25779
https://www.verywellmind.com/what-is-consciousness-2795922
https://www.jstor.org/stable/2253868
20:40:23 | Generated knowledge: ['1. Browse Content\n2. ScienceDirect help']
Traceback (most recent call last):
  File "/home/gpu/env/bin/parlai", line 33, in <module>
    sys.exit(load_entry_point('parlai', 'console_scripts', 'parlai')())
  File "/home/gpu/ParlAI/parlai/__main__.py", line 14, in main
    superscript_main()
  File "/home/gpu/ParlAI/parlai/core/script.py", line 325, in superscript_main
    return SCRIPT_REGISTRY[cmd].klass._run_from_parser_and_opt(opt, parser)
  File "/home/gpu/ParlAI/parlai/core/script.py", line 108, in _run_from_parser_and_opt
    return script.run()
  File "/home/gpu/ParlAI/parlai/scripts/self_chat.py", line 176, in run
    return self_chat(self.opt)
  File "/home/gpu/ParlAI/parlai/scripts/self_chat.py", line 146, in self_chat
    _run_self_chat_episode(opt, world, logger)
  File "/home/gpu/ParlAI/parlai/scripts/self_chat.py", line 93, in _run_self_chat_episode
    world.parley()
  File "/home/gpu/ParlAI/parlai/tasks/self_chat/worlds.py", line 178, in parley
    self.agents[1 - i].observe(validate(self.acts[i]))
  File "/home/gpu/ParlAI/projects/seeker/agents/seeker.py", line 566, in observe
    observation['knowledge_response'] = observation.get('checked_sentence', '')
  File "/home/gpu/ParlAI/parlai/core/message.py", line 30, in __setitem__
    raise RuntimeError(
RuntimeError: Message already contains key `knowledge_response`. If this was intentional, please use the function `force_set(key, value)`.

Should I tweak the knowledge params? Or could you give me the CLI to run Seeker in self chat with a search engine please?

Thanks a lot 😇


Discussion:

Resolving patch:
diff --git a/projects/seeker/agents/seeker.py b/projects/seeker/agents/seeker.py
--- a/projects/seeker/agents/seeker.py
+++ b/projects/seeker/agents/seeker.py
@@ -81,6 +81,12 @@ def add_cmdline_args(
             default=False,
             help='Whether to make model act output fully serializable.',
         )
+        combo_fid.add_argument(
+            '--force-skip-retrieval',
+            type='bool',
+            default=False,
+            help='If True, we force skip retrieval on any/all incoming examples',
+        )
 
     def build_model(self) -> ComboFidModel:
         """
@@ -107,9 +113,12 @@ def batchify(self, obs_batch: List[Message], sort: bool = False) -> Batch:
         batch = super().batchify(obs_batch, sort)
         valid_exs = [ex for ex in obs_batch if self.is_valid(ex)]
         if valid_exs:
-            skip_retrieval = [
-                ex.get(self.opt['skip_retrieval_key'], False) for ex in valid_exs
-            ]
+            if self.opt.get('force_skip_retrieval', False):
+                skip_retrieval = [True] * len(valid_exs)
+            else:
+                skip_retrieval = [
+                    ex.get(self.opt['skip_retrieval_key'], False) for ex in valid_exs
+                ]
             batch.skip_retrieval_vec = torch.BoolTensor(skip_retrieval)
             if any(ex.get('prior_knowledge_responses') for ex in valid_exs):
                 vecs, _lens = self._pad_tensor(
@@ -563,7 +572,9 @@ def observe(self, observation: Message) -> Dict[str, Message]:
         for key in ['label_candidates', 'knowledge']:
             # Delete unnecessarily large keys
             observation.pop(key, '')
-        observation['knowledge_response'] = observation.get('checked_sentence', '')
+        observation.force_set(
+            'knowledge_response', observation.get('checked_sentence', '')
+        )
 
         raw_observation = copy.deepcopy(observation)
         # This part is *specifically* for document chunking.


Test patch:
diff --git a/tests/nightly/gpu/test_seeker.py b/tests/nightly/gpu/test_seeker.py
--- a/tests/nightly/gpu/test_seeker.py
+++ b/tests/nightly/gpu/test_seeker.py
@@ -6,6 +6,7 @@
 import unittest
 
 import parlai.scripts.eval_model as ems
+from parlai.scripts.self_chat import SelfChat
 import parlai.utils.testing as testing_utils
 
 R2C2_BASE_400M = 'zoo:seeker/r2c2_base_400M/model'
@@ -111,3 +112,15 @@ def test_blenderbot(self):
             'datatype': 'valid',
         }
         ems.EvalModel.main(**opt)
+
+
+class TestSeekerSelfChat(unittest.TestCase):
+    def test_400m(self):
+        SelfChat.main(
+            model_file='zoo:seeker/seeker_dialogue_400M/model',
+            num_self_chats=1,
+            init_opt='gen/seeker_dialogue',
+            search_decision='never',
+            search_server='none',
+            krm_force_skip_retrieval=True,
+        )