reduce prompt size

readme updates
reduced max tokens for tabletop_ui

README.md

@@ -22,7 +22,7 @@ Below is an interactive demo for the simulated tabletop manipulation domain, see
 1. Obtain an [OpenAI API Key](https://openai.com/blog/openai-api/)
 2. Gain Codex access by [joining the waitlist](https://openai.com/blog/openai-codex/) 
 
-Once you have Codex access you can use `code-davinci-002`. Using the GPT-3 model (`text-dainvci-002`) is also ok, but performance won't be as good (there will be more code logic errors).
+Once you have Codex access you can use code-davinci-002. Using the GPT-3 model (text-dainvci-002) is also ok, but performance won't be as good (there will be more code logic errors).
 
 ## Usage 
 1. Fill in the API Key, model name, and how many blocks and bowls to be spawned in the environment. 
@@ -48,9 +48,10 @@ Note object names may need to be changed depending the sampled object names.
 * pick up the block closest to the top left corner and place it on the bottom right corner
 
 ## Known Limitations
-* In simulation we're using ground truth object poses instead of using vision models. This means that instructions the require knowledge of visual apperances (e.g. darkest bowl, largest object) are not supported.
+* In simulation we're using ground truth object poses instead of using vision models. This means that instructions the require knowledge of visual apperances (e.g. darkest bowl, largest object, empty bowls) are not supported.
 * Currently, the low-level pick place primitive does not do collision checking, so if there are many objects on the table, placing actions may incur collisions.
 * The pick place primitive is also unable to pick up bowls.
 * Prompt saturation - if too many instructions (10+) are executed in a row, then the LLM may start to ignore examples in the early parts of the prompt.
 * Ambiguous instructions - if a given instruction doesn't lead to the desired actions, try rephrasing it to remove ambiguities (e.g. place the block on the closest bowl -> place the block on its closest bowl)
 * Maximum token length - you may hit the maximum token length if running multiple commands in sequence. Please reset the simulation when this happens.
+* If you get the error "That model does not exist" it means you're trying to use Codex w/o Codex access. Switch to text-davinci-002 instead.

app.py

@@ -95,7 +95,7 @@ class DemoRunner:
         try:
             self._lmp_tabletop_ui(instruction, f'objects = {self._env.object_list}')
         except Exception as e:
-            return str(e), None, None
+            return f'Error: {e}', None, None
 
         video_file_name = None
         if self._env.cache_video:
@@ -140,7 +140,7 @@ if __name__ == '__main__':
             with gr.Column():
                 with gr.Row():
                     inp_api_key = gr.Textbox(label='OpenAI API Key (this is not stored anywhere)', lines=1)
-                    inp_model_name = gr.Dropdown(label='Model Name', choices=['code-davinci-002', 'text-davinci-002'], value='code-davinci-002')
+                    inp_model_name = gr.Dropdown(label='Model Name', choices=['code-davinci-002', 'text-davinci-002'], value='text-davinci-002')
                 with gr.Row():
                     inp_n_blocks = gr.Slider(label='Number of Blocks', minimum=0, maximum=4, value=3, step=1)
                     inp_n_bowls = gr.Slider(label='Number of Bowls', minimum=0, maximum=4, value=3, step=1)
@@ -154,7 +154,7 @@ if __name__ == '__main__':
             with gr.Column():
                 
                 inp_instruction = gr.Textbox(label='Instruction', lines=1) 
-                btn_run = gr.Button("Run (this may take 20+ seconds)")
+                btn_run = gr.Button("Run (this may take 30+ seconds)")
                 info_run = gr.Markdown(label='Generated Code')
             with gr.Column():
                 video_run = gr.Video(label='Video of Last Instruction')

cfg.yaml

@@ -3,7 +3,7 @@ lmps:
   tabletop_ui:
     prompt_path: prompts/tabletop_ui.py
     engine: model_name
-    max_tokens: 512
+    max_tokens: 256
     temperature: 0
     query_prefix: '# '
     query_suffix: '.'

prompts/parse_question.py

@@ -8,9 +8,6 @@ objects = ['yellow bowl', 'blue block', 'yellow block', 'blue bowl', 'fruit', 'g
 yellow_object_names = parse_obj_name('the yellow objects', f'objects = {get_obj_names()}')
 ret_val = len(yellow_object_names)
 objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
-# is the pink block on the green bowl?
-ret_val = bbox_contains_pt(container_name='green bowl', obj_name='pink block')
-objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
 # what are the blocks left of the green bowl?
 block_names = parse_obj_name('the blocks', f'objects = {get_obj_names()}')
 green_bowl_pos = get_obj_pos('green bowl')

prompts/tabletop_ui.py

@@ -15,11 +15,6 @@ objects = ['yellow block', 'green block', 'yellow bowl', 'blue block', 'blue bow
 say('Got it - putting the green block on the top right corner')
 corner_pos = parse_position('top right corner')
 put_first_on_second('green block', corner_pos)
-objects = ['yellow block', 'green block', 'yellow bowl', 'blue block', 'blue bowl', 'green bowl']
-# stack the blue bowl on the yellow bowl on the green block.
-order_bottom_to_top = ['green block', 'yellow block', 'blue bowl']
-say(f'Sure - stacking from top to bottom: {", ".join(order_bottom_to_top)}')
-stack_objects_in_order(object_names=order_bottom_to_top)
 objects = ['cyan block', 'white block', 'cyan bowl', 'blue block', 'blue bowl', 'white bowl']
 # move the cyan block into its corresponding bowl.
 matches = {'cyan block': 'cyan bowl'}
@@ -39,7 +34,7 @@ say('Sure thing - putting the yellow block between the blue bowl and the green b
 target_pos = parse_position('a point in the middle betweeen the blue bowl and the green block')
 put_first_on_second('yellow block', target_pos)
 objects = ['yellow block', 'red block', 'yellow bowl', 'gray block', 'gray bowl', 'red bowl']
-# can you cut the bowls in half.
+# can you cut the blocks in half.
 say('no, I can only move objects around')
 objects = ['yellow block', 'green block', 'yellow bowl', 'gray block', 'gray bowl', 'green bowl']
 # stack the blocks on the right side with the gray one on the bottom.
@@ -54,18 +49,18 @@ say('Ok! Stacking everything with the green block on the top')
 order_bottom_to_top = ['blue bowl', 'pink bowl', 'green bowl', 'pink block', 'blue block', 'green block']
 stack_objects_in_order(object_names=order_bottom_to_top)
 objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
-# move the grass-colored bowl to the left.
-say('Sure - moving the green bowl left by 10 centimeters')
-left_pos = parse_position('a point 10cm left of the green bowl')
-put_first_on_second('green bowl', left_pos)
+# move the grass-colored block to the left.
+say('Sure - moving the green block left by 10 centimeters')
+left_pos = parse_position('a point 10cm left of the green block')
+put_first_on_second('green block', left_pos)
 objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
-# why did you move the red bowl.
-say(f'I did not move the red bowl')
+# why did you move the red block.
+say(f'I did not move the red block')
 objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
 # undo that.
-say('Sure - moving the green bowl right by 10 centimeters')
-left_pos = parse_position('a point 10cm right of the green bowl')
-put_first_on_second('green bowl', left_pos)
+say('Sure - moving the green block right by 10 centimeters')
+left_pos = parse_position('a point 10cm right of the green block')
+put_first_on_second('green block', left_pos)
 objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
 # place the top most block to the corner closest to the bottom most block.
 top_block_name = parse_obj_name('top most block', f'objects = {get_obj_names()}')
@@ -74,38 +69,16 @@ closest_corner_pos = parse_position(f'the corner closest to the {bottom_block_na
 say(f'Putting the {top_block_name} on the {get_corner_name(closest_corner_pos)}')
 put_first_on_second(top_block_name, closest_corner_pos)
 objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
-# move the brown bowl to the side closest to the green block.
-closest_side_position = parse_position('the side closest to the green block')
-say(f'Got it - putting the brown bowl on the {get_side_name(closest_side_position)}')
-put_first_on_second('brown bowl', closest_side_position)
-objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
-# place the green block to the right of the bowl that has the blue block.
-bowl_name = parse_obj_name('the bowl that has the blue block', f'objects = {get_obj_names()}')
-if bowl_name:
-  target_pos = parse_position(f'a point 10cm to the right of the {bowl_name}')
-  say(f'No problem - placing the green block to the right of the {bowl_name}')
-  put_first_on_second('green block', target_pos)
-else:
-  say('There are no bowls that has the blue block')
-objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
-# place the blue block in the empty bowl.
-empty_bowl_name = parse_obj_name('the empty bowl', f'objects = {get_obj_names()}')
-if empty_bowl_name:
-  say(f'Ok! Putting the blue block on the {empty_bowl_name}')
-  put_first_on_second('blue block', empty_bowl_name)
-else:
-  say('There are no empty bowls')
+# move the brown block to the side closest to the green bowl.
+closest_side_position = parse_position('the side closest to the green bowl')
+say(f'Got it - putting the brown block on the {get_side_name(closest_side_position)}')
+put_first_on_second('brown block', closest_side_position)
 objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
 # move the other blocks to the bottom corners.
-block_names = parse_obj_name('blocks other than the blue block', f'objects = {get_obj_names()}')
+block_names = parse_obj_name('blocks other than the green block', f'objects = {get_obj_names()}')
 corners = parse_position('the bottom corners')
 for block_name, pos in zip(block_names, corners):
   put_first_on_second(block_name, pos)
-objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
-# move the red bowl a lot to the left of the blocks.
-say('Sure! Moving the red bowl to a point left of the blocks')
-left_pos = parse_position('a point 20cm left of the blocks')
-put_first_on_second('red bowl', left_pos)
 objects = ['pink block', 'gray block', 'orange block']
 # move the pinkish colored block on the bottom side.
 say('Ok - putting the pink block on the bottom side')
@@ -134,25 +107,12 @@ say(f'Ok - moving the other {len(block_names)} blocks to different corners')
 for block_name, pos in zip(block_names, corners):
   put_first_on_second(block_name, pos)
 objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
-# is the pink block on the green bowl.
-if parse_question('is the pink block on the green bowl', f'objects = {get_obj_names()}'):
-  say('Yes - the pink block is on the green bowl.')
-else:
-  say('No - the pink block is not on the green bowl.')
-objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
 # what are the blocks left of the green bowl.
 left_block_names =  parse_question('what are the blocks left of the green bowl', f'objects = {get_obj_names()}')
 if len(left_block_names) > 0:
   say(f'These blocks are left of the green bowl: {", ".join(left_block_names)}')
 else:
   say('There are no blocks left of the green bowl')
-objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
-# if you see a purple bowl put it on the blue bowl
-if is_obj_visible('purple bowl'):
-  say('Putting the purple bowl on the pink bowl')
-  put_first_on_second('purple bowl', 'pink bowl')
-else:
-  say('I don\'t see a purple bowl')
 objects = ['yellow block', 'green block', 'yellow bowl', 'blue block', 'blue bowl', 'green bowl']
 # imagine that the bowls are different biomes on earth and imagine that the blocks are parts of a building.
 say('ok')
@@ -161,10 +121,6 @@ objects = ['yellow block', 'green block', 'yellow bowl', 'blue block', 'blue bow
 order_bottom_to_top = ['green bowl', 'blue block', 'green block', 'yellow block']
 say('stacking the blocks on the green bowl')
 stack_objects_in_order(object_names=order_bottom_to_top)
-objects = ['yellow block', 'green block', 'yellow bowl', 'gray block', 'gray bowl', 'green bowl']
-# show me what happens when the desert gets flooded by the ocean.
-say('putting the yellow bowl on the blue bowl')
-put_first_on_second('yellow bowl', 'blue bowl')
 objects = ['pink block', 'gray block', 'orange block']
 # move all blocks 5cm toward the top.
 say('Ok - moving all blocks 5cm toward the top')