working logging, readme

LICENSE.md

@@ -0,0 +1,7 @@
+Copyright 2021 Google LLC. SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+
+https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

README.md

@@ -10,4 +10,48 @@ pinned: false
 license: apache-2.0
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Code as Policies Tabletop Manipulation Interactive Demo
+
+This notebook is a part of the open-source code release associated with the paper:
+
+[Code as Policies: Language Model Programs for Embodied Control](https://code-as-policies.github.io/)
+
+This notebook gives an interactive demo for the simulated tabletop manipulation domain, seen in the paper section IV.D
+
+## Preparations: 
+
+1) Obtain an [OpenAI API Key](https://openai.com/blog/openai-api/)
+
+2) Gain Codex access by [joining the waitlist](https://openai.com/blog/openai-codex/)
+
+Once you have Codex access you can use `code-davinci-002`. Using the GPT-3 model (`text-dainvci-002`) is also ok, but performance won't be as good (there will be more code logic errors).
+
+## Instructions: 
+
+1. Fill in the API Key, model name, and how many blocks and bowls to be spawned in the environment. 
+2. Click Setup/Reset Env
+3. Based on the new randomly sampled object names, input an instruction and click Run Instruction. If successful, this will render a video and update the simulation environment visualization.
+
+You can run instructions in sequence and refer back to previous commands (e.g. do the same with other blocks, move the same block to the other bowl, etc). Click Setup/Reset Env to reset, and this will clear the current instruction history.
+
+Supported commands:
+* Spatial reasoning (e.g. to the left of the red block, the closest corner, the farthest bowl, the second block from the right)
+* Sequential actions (e.g. put blocks in matching bowls, stack blocks on the bottom right corner)
+* Contextual commands (e.g. do the same with the blue block, undo that)
+* Language-based reasoning (e.g. put the forest-colored block on the ocean-colored bowl).
+* Simple Q&A (e.g. how many blocks are to the left of the blue bowl?)
+
+Example commands (note object names may need to be changed depending the sampled object names):
+* put the sun-colored block on the bowl closest to it
+* stack the blocks on the bottom most bowl
+* arrange the blocks as a square in the middle
+* move the square 5cm to the right
+* how many blocks are to the right of the orange bowl?
+* pick up the block closest to the top left corner and place it on the bottom right corner
+
+Known limitations:
+* In simulation we're using ground truth object poses instead of using vision models. This means that commands the require knowledge of visual apperances (e.g. darkest bowl, largest object) are not supported.
+* Currently, the low-level pick place primitive does not do collision checking, so if there are many objects on the table, placing actions may incur collisions.
+* Prompt saturation - if too many commands (10+) are executed in a row, then the LLM may start to ignore examples in the early parts of the prompt.
+* Ambiguous instructions - if a given instruction doesn't lead to the desired actions, try rephrasing it to remove ambiguities (e.g. place the block on the closest bowl -> place the block on its closest bowl)
+

app.py

@@ -9,10 +9,10 @@ from omegaconf import OmegaConf
 from moviepy.editor import ImageSequenceClip
 import gradio as gr
 
-
 from lmp import LMP, LMPFGen
 from sim import PickPlaceEnv, LMP_wrapper
 from consts import ALL_BLOCKS, ALL_BOWLS
+from md_logger import MarkdownLogger
 
 
 class DemoRunner:
@@ -21,6 +21,7 @@ class DemoRunner:
         self._cfg = OmegaConf.to_container(OmegaConf.load('cfg.yaml'), resolve=True)
         self._env = None
         self._model_name = ''
+        self._md_logger = MarkdownLogger()
 
     def make_LMP(self, env):
         # LMP env wrapper
@@ -49,20 +50,20 @@ class DemoRunner:
                 'get_corner_name', 'get_side_name',
             ]
         }
-        variable_vars['say'] = lambda msg: print(f'robot says: {msg}')
+        variable_vars['say'] = lambda msg: self._md_logger.log_text(f'Robot says: "{msg}"')
 
         # creating the function-generating LMP
-        lmp_fgen = LMPFGen(cfg['lmps']['fgen'], fixed_vars, variable_vars)
+        lmp_fgen = LMPFGen(cfg['lmps']['fgen'], fixed_vars, variable_vars, self._md_logger)
 
         # creating other low-level LMPs
         variable_vars.update({
-            k: LMP(k, cfg['lmps'][k], lmp_fgen, fixed_vars, variable_vars)
+            k: LMP(k, cfg['lmps'][k], lmp_fgen, fixed_vars, variable_vars, self._md_logger)
             for k in ['parse_obj_name', 'parse_position', 'parse_question', 'transform_shape_pts']
         })
 
         # creating the LMP that deals w/ high-level language commands
         lmp_tabletop_ui = LMP(
-            'tabletop_ui', cfg['lmps']['tabletop_ui'], lmp_fgen, fixed_vars, variable_vars
+            'tabletop_ui', cfg['lmps']['tabletop_ui'], lmp_fgen, fixed_vars, variable_vars, self._md_logger
         )
 
         return lmp_tabletop_ui
@@ -89,45 +90,56 @@ class DemoRunner:
             return 'Please run setup first'
 
         self._env.cache_video = []
+        self._md_logger.clear()
 
-        self._lmp_tabletop_ui(instruction, f'objects = {self._env.object_list}')
+        try:
+            self._lmp_tabletop_ui(instruction, f'objects = {self._env.object_list}')
+            run_info = self._md_logger.get_log()
+        except Exception as e:
+            run_info = f'Error: {e}'
 
-        video_file_name = ''
+        video_file_name = None
         if self._env.cache_video:
             rendered_clip = ImageSequenceClip(self._env.cache_video, fps=25)
-            video_file_name = NamedTemporaryFile(suffix='.mp4', delete=False).name
+            video_file_name = NamedTemporaryFile(suffix='.mp4').name
             rendered_clip.write_videofile(video_file_name, fps=25)
 
-        return 'Done', video_file_name
+        return run_info, self._env.get_camera_image(), video_file_name
 
 
 if __name__ == '__main__':
     demo_runner = DemoRunner()
     demo = gr.Blocks()
 
+    with open('README.md', 'r') as f:
+        for _ in range(12):
+            next(f)
+        readme_text = f.read()
+
     with demo:
+        gr.Markdown(readme_text)
         with gr.Row():
             with gr.Column():
                 with gr.Row():
-                    inp_api_key = gr.Textbox(label='OpenAI API Key', lines=1, value='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')
+                    inp_api_key = gr.Textbox(label='OpenAI API Key', lines=1)
                     inp_model_name = gr.Dropdown(label='Model Name', choices=['code-davinci-002', 'text-davinci-002'], value='code-davinci-002')
                 with gr.Row():
                     inp_n_blocks = gr.Slider(label='Num Blocks', minimum=0, maximum=3, value=3, step=1)
                     inp_n_bowls = gr.Slider(label='Num Bowls', minimum=0, maximum=3, value=3, step=1)
             
-                btn_setup = gr.Button("1) Setup/Reset Env")
+                btn_setup = gr.Button("Setup/Reset Env")
                 info_setup = gr.Markdown(label='Setup Info')
             with gr.Column():
-                img_setup = gr.Image(label='Setup Image')
+                img_setup = gr.Image(label='Current Simulation')
 
         with gr.Row():
             with gr.Column():
                 
                 inp_instruction = gr.Textbox(label='Instruction', lines=1) 
-                btn_run = gr.Button("2) Run Instruction")
-                info_run = gr.Label(label='Run Info')
+                btn_run = gr.Button("Run Instruction")
+                info_run = gr.Markdown(label='Generated Code')
             with gr.Column():
-                video_run = gr.Video(label='Run Video')
+                video_run = gr.Video(label='Video of Last Instruction')
         
         btn_setup.click(
             demo_runner.setup, 
@@ -137,7 +149,7 @@ if __name__ == '__main__':
         btn_run.click(
             demo_runner.run, 
             inputs=[inp_instruction], 
-            outputs=[info_run, video_run]
+            outputs=[info_run, img_setup, video_run]
         )
         
     demo.launch()

lmp.py

@@ -10,9 +10,10 @@ from pygments.formatters import TerminalFormatter
 
 class LMP:
 
-    def __init__(self, name, cfg, lmp_fgen, fixed_vars, variable_vars):
+    def __init__(self, name, cfg, lmp_fgen, fixed_vars, variable_vars, md_logger):
         self._name = name
         self._cfg = cfg
+        self._md_logger = md_logger
 
         with open(self._cfg['prompt_path'], 'r') as f:
             self._base_prompt = f.read()
@@ -72,7 +73,9 @@ class LMP:
             to_log = f'{use_query}\n{to_exec}'
 
         to_log_pretty = highlight(to_log, PythonLexer(), TerminalFormatter())
-        print(f'LMP {self._name} exec:\n\n{to_log_pretty}\n')
+        print(f'LMP {self._name} generated code:\n{to_log_pretty}')
+        self._md_logger.log_text(f'LMP {self._name} Generated Code:')
+        self._md_logger.log_code(to_log)
 
         new_fs = self._lmp_fgen.create_new_fs_from_code(code_str)
         self._variable_vars.update(new_fs)
@@ -94,12 +97,13 @@ class LMP:
 
 class LMPFGen:
 
-    def __init__(self, cfg, fixed_vars, variable_vars):
+    def __init__(self, cfg, fixed_vars, variable_vars, md_logger):
         self._cfg = cfg
 
         self._stop_tokens = list(self._cfg['stop'])
         self._fixed_vars = fixed_vars
         self._variable_vars = variable_vars
+        self._md_logger = md_logger
 
         with open(self._cfg['prompt_path'], 'r') as f:
             self._base_prompt = f.read()
@@ -142,8 +146,11 @@ class LMPFGen:
 
         f = lvars[f_name]
 
-        to_print = highlight(f'{use_query}\n{f_src}', PythonLexer(), TerminalFormatter())
-        print(f'LMP FGEN created:\n\n{to_print}\n')
+        to_print = f'{use_query}\n{f_src}'
+        to_print_pretty = highlight(to_print, PythonLexer(), TerminalFormatter())
+        print(f'LMPFGen generated code:\n{to_print_pretty}')
+        self._md_logger.log_text('Generated Function:')
+        self._md_logger.log_code(to_print)
 
         if return_src:
             return f, f_src

md_logger.py

@@ -0,0 +1,16 @@
+class MarkdownLogger:
+
+    def __init__(self):
+        self._log = ''
+
+    def log_text(self, text):
+        self._log += '\n' + text + '\n'
+    
+    def log_code(self, code):
+        self._log += f'\n```python\n{code}\n```\n'
+
+    def clear(self):
+        self._log = ''
+
+    def get_log(self):
+        return self._log

prompts/parse_obj_name.py

@@ -5,8 +5,7 @@ from utils import get_obj_positions_np
 objects = ['blue block', 'cyan block', 'purple bowl', 'gray bowl', 'brown bowl', 'pink block', 'purple block']
 # the block closest to the purple bowl.
 block_names = ['blue block', 'cyan block', 'purple block']
-block_positions = get_obj_positions_np(block_names)
-closest_block_idx = get_closest_idx(points=block_positions, point=get_obj_pos('purple bowl'))
+closest_block_idx = get_closest_idx(points=get_obj_positions_np(block_names), point=get_obj_pos('purple bowl'))
 closest_block_name = block_names[closest_block_idx]
 ret_val = closest_block_name
 objects = ['brown bowl', 'banana', 'brown block', 'apple', 'blue bowl', 'blue block']
@@ -37,28 +36,24 @@ objects = ['blue block', 'cyan block', 'purple bowl', 'brown bowl', 'purple bloc
 # the block closest to the bottom right corner.
 corner_pos = parse_position('bottom right corner')
 block_names = ['blue block', 'cyan block', 'purple block']
-block_positions = get_obj_positions_np(block_names)
-closest_block_idx = get_closest_idx(points=block_positions, point=corner_pos)
+closest_block_idx = get_closest_idx(points=get_obj_positions_np(block_names), point=corner_pos)
 closest_block_name = block_names[closest_block_idx]
 ret_val = closest_block_name
 objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
 # the left most block.
 block_names = ['green block', 'brown block', 'blue block']
-block_positions = get_obj_positions_np(block_names)
-left_block_idx = np.argsort(block_positions[:, 0])[0]
+left_block_idx = np.argsort(get_obj_positions_np(block_names)[:, 0])[0]
 left_block_name = block_names[left_block_idx]
 ret_val = left_block_name
 objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
 # the bowl on near the top.
 bowl_names = ['brown bowl', 'green bowl', 'blue bowl']
-bowl_positions = get_obj_positions_np(bowl_names)
-top_bowl_idx = np.argsort(block_positions[:, 1])[-1]
+top_bowl_idx = np.argsort(get_obj_positions_np(bowl_names)[:, 1])[-1]
 top_bowl_name = bowl_names[top_bowl_idx]
 ret_val = top_bowl_name
 objects = ['yellow bowl', 'purple block', 'yellow block', 'purple bowl', 'pink bowl', 'pink block']
 # the third bowl from the right.
 bowl_names = ['yellow bowl', 'purple bowl', 'pink bowl']
-bowl_positions = get_obj_positions_np(bowl_names)
-bowl_idx = np.argsort(block_positions[:, 0])[-3]
+bowl_idx = np.argsort(get_obj_positions_np(bowl_names)[:, 0])[-3]
 bowl_name = bowl_names[bowl_idx]
 ret_val = bowl_name