jackyliang42 commited on
Commit
8a9e562
1 Parent(s): 2a6f170

updated model to use text-davinci-003

Browse files
Files changed (4) hide show
  1. README.md +1 -5
  2. app.py +4 -9
  3. cfg.yaml +6 -6
  4. prompts/tabletop_ui.py +45 -21
README.md CHANGED
@@ -20,12 +20,9 @@ Below is an interactive demo for the simulated tabletop manipulation domain, see
20
 
21
  ## Preparations
22
  1. Obtain an [OpenAI API Key](https://openai.com/blog/openai-api/)
23
- 2. Gain Codex access by [joining the waitlist](https://openai.com/blog/openai-codex/)
24
-
25
- Once you have Codex access you can use code-davinci-002. Using the GPT-3 model (text-dainvci-002) is also ok, but performance won't be as good (there will be more code logic errors).
26
 
27
  ## Usage
28
- 1. Fill in the API Key, model name, and how many blocks and bowls to be spawned in the environment.
29
  2. Click Setup/Reset Simulation
30
  3. Based on the new randomly sampled object names, input an instruction and click Run Instruction. If successful, this will render a video and update the simulation environment visualization.
31
 
@@ -54,4 +51,3 @@ Note object names may need to be changed depending the sampled object names.
54
  * Prompt saturation - if too many instructions (10+) are executed in a row, then the LLM may start to ignore examples in the early parts of the prompt.
55
  * Ambiguous instructions - if a given instruction doesn't lead to the desired actions, try rephrasing it to remove ambiguities (e.g. place the block on the closest bowl -> place the block on its closest bowl)
56
  * Maximum token length - you may hit the maximum token length if running multiple commands in sequence. Please reset the simulation when this happens.
57
- * If you get the error "That model does not exist" it means you're trying to use Codex w/o Codex access. Switch to text-davinci-002 instead.
 
20
 
21
  ## Preparations
22
  1. Obtain an [OpenAI API Key](https://openai.com/blog/openai-api/)
 
 
 
23
 
24
  ## Usage
25
+ 1. Fill in the API Key and how many blocks and bowls to be spawned in the environment.
26
  2. Click Setup/Reset Simulation
27
  3. Based on the new randomly sampled object names, input an instruction and click Run Instruction. If successful, this will render a video and update the simulation environment visualization.
28
 
 
51
  * Prompt saturation - if too many instructions (10+) are executed in a row, then the LLM may start to ignore examples in the early parts of the prompt.
52
  * Ambiguous instructions - if a given instruction doesn't lead to the desired actions, try rephrasing it to remove ambiguities (e.g. place the block on the closest bowl -> place the block on its closest bowl)
53
  * Maximum token length - you may hit the maximum token length if running multiple commands in sequence. Please reset the simulation when this happens.
 
app.py CHANGED
@@ -20,7 +20,6 @@ class DemoRunner:
20
  def __init__(self):
21
  self._cfg = OmegaConf.to_container(OmegaConf.load('cfg.yaml'), resolve=True)
22
  self._env = None
23
- self._model_name = ''
24
  self._md_logger = MarkdownLogger()
25
 
26
  def make_LMP(self, env):
@@ -30,8 +29,6 @@ class DemoRunner:
30
  'init_objs': list(env.obj_name_to_id.keys()),
31
  'coords': cfg['tabletop_coords']
32
  }
33
- for vs in cfg['lmps'].values():
34
- vs['engine'] = self._model_name
35
 
36
  LMP_env = LMP_wrapper(env, cfg)
37
  # creating APIs that the LMPs can interact with
@@ -68,9 +65,8 @@ class DemoRunner:
68
 
69
  return lmp_tabletop_ui
70
 
71
- def setup(self, api_key, model_name, n_blocks, n_bowls):
72
  openai.api_key = api_key
73
- self._model_name = model_name
74
 
75
  self._env = PickPlaceEnv(render=True, high_res=True, high_frame_rate=False)
76
  list_idxs = np.random.choice(len(ALL_BLOCKS), size=max(n_blocks, n_bowls), replace=False)
@@ -107,7 +103,7 @@ class DemoRunner:
107
  return self._md_logger.get_log(), self._env.get_camera_image(), video_file_name
108
 
109
 
110
- def setup(api_key, model_name, n_blocks, n_bowls):
111
  if not api_key:
112
  return 'Please enter your OpenAI API key!', None, None
113
 
@@ -116,7 +112,7 @@ def setup(api_key, model_name, n_blocks, n_bowls):
116
 
117
  demo_runner = DemoRunner()
118
 
119
- info, img = demo_runner.setup(api_key, model_name, n_blocks, n_bowls)
120
  return info, img, demo_runner
121
 
122
 
@@ -141,7 +137,6 @@ if __name__ == '__main__':
141
  with gr.Column():
142
  with gr.Row():
143
  inp_api_key = gr.Textbox(label='OpenAI API Key (this is not stored anywhere)', lines=1)
144
- inp_model_name = gr.Dropdown(label='Model Name', choices=['code-davinci-002', 'text-davinci-002'], value='code-davinci-002')
145
  with gr.Row():
146
  inp_n_blocks = gr.Slider(label='Number of Blocks', minimum=0, maximum=4, value=3, step=1)
147
  inp_n_bowls = gr.Slider(label='Number of Bowls', minimum=0, maximum=4, value=3, step=1)
@@ -162,7 +157,7 @@ if __name__ == '__main__':
162
 
163
  btn_setup.click(
164
  setup,
165
- inputs=[inp_api_key, inp_model_name, inp_n_blocks, inp_n_bowls],
166
  outputs=[info_setup, img_setup, state]
167
  )
168
  btn_run.click(
 
20
  def __init__(self):
21
  self._cfg = OmegaConf.to_container(OmegaConf.load('cfg.yaml'), resolve=True)
22
  self._env = None
 
23
  self._md_logger = MarkdownLogger()
24
 
25
  def make_LMP(self, env):
 
29
  'init_objs': list(env.obj_name_to_id.keys()),
30
  'coords': cfg['tabletop_coords']
31
  }
 
 
32
 
33
  LMP_env = LMP_wrapper(env, cfg)
34
  # creating APIs that the LMPs can interact with
 
65
 
66
  return lmp_tabletop_ui
67
 
68
+ def setup(self, api_key, n_blocks, n_bowls):
69
  openai.api_key = api_key
 
70
 
71
  self._env = PickPlaceEnv(render=True, high_res=True, high_frame_rate=False)
72
  list_idxs = np.random.choice(len(ALL_BLOCKS), size=max(n_blocks, n_bowls), replace=False)
 
103
  return self._md_logger.get_log(), self._env.get_camera_image(), video_file_name
104
 
105
 
106
+ def setup(api_key, n_blocks, n_bowls):
107
  if not api_key:
108
  return 'Please enter your OpenAI API key!', None, None
109
 
 
112
 
113
  demo_runner = DemoRunner()
114
 
115
+ info, img = demo_runner.setup(api_key, n_blocks, n_bowls)
116
  return info, img, demo_runner
117
 
118
 
 
137
  with gr.Column():
138
  with gr.Row():
139
  inp_api_key = gr.Textbox(label='OpenAI API Key (this is not stored anywhere)', lines=1)
 
140
  with gr.Row():
141
  inp_n_blocks = gr.Slider(label='Number of Blocks', minimum=0, maximum=4, value=3, step=1)
142
  inp_n_bowls = gr.Slider(label='Number of Bowls', minimum=0, maximum=4, value=3, step=1)
 
157
 
158
  btn_setup.click(
159
  setup,
160
+ inputs=[inp_api_key, inp_n_blocks, inp_n_bowls],
161
  outputs=[info_setup, img_setup, state]
162
  )
163
  btn_run.click(
cfg.yaml CHANGED
@@ -2,7 +2,7 @@
2
  lmps:
3
  tabletop_ui:
4
  prompt_path: prompts/tabletop_ui.py
5
- engine: model_name
6
  max_tokens: 256
7
  temperature: 0
8
  query_prefix: '# '
@@ -15,7 +15,7 @@ lmps:
15
  return_val_name: ret_val
16
  parse_obj_name:
17
  prompt_path: prompts/parse_obj_name.py
18
- engine: model_name
19
  max_tokens: 512
20
  temperature: 0
21
  query_prefix: '# '
@@ -28,7 +28,7 @@ lmps:
28
  return_val_name: ret_val
29
  parse_position:
30
  prompt_path: prompts/parse_position.py
31
- engine: model_name
32
  max_tokens: 512
33
  temperature: 0
34
  query_prefix: '# '
@@ -41,7 +41,7 @@ lmps:
41
  return_val_name: ret_val
42
  parse_question:
43
  prompt_path: prompts/parse_question.py
44
- engine: model_name
45
  max_tokens: 512
46
  temperature: 0
47
  query_prefix: '# '
@@ -54,7 +54,7 @@ lmps:
54
  return_val_name: ret_val
55
  transform_shape_pts:
56
  prompt_path: prompts/transform_shape_pts.py
57
- engine: model_name
58
  max_tokens: 512
59
  temperature: 0
60
  query_prefix: '# '
@@ -67,7 +67,7 @@ lmps:
67
  return_val_name: new_shape_pts
68
  fgen:
69
  prompt_path: prompts/fgen.py
70
- engine: model_name
71
  max_tokens: 512
72
  temperature: 0
73
  query_prefix: '# define function: '
 
2
  lmps:
3
  tabletop_ui:
4
  prompt_path: prompts/tabletop_ui.py
5
+ engine: text-davinci-003
6
  max_tokens: 256
7
  temperature: 0
8
  query_prefix: '# '
 
15
  return_val_name: ret_val
16
  parse_obj_name:
17
  prompt_path: prompts/parse_obj_name.py
18
+ engine: text-davinci-003
19
  max_tokens: 512
20
  temperature: 0
21
  query_prefix: '# '
 
28
  return_val_name: ret_val
29
  parse_position:
30
  prompt_path: prompts/parse_position.py
31
+ engine: text-davinci-003
32
  max_tokens: 512
33
  temperature: 0
34
  query_prefix: '# '
 
41
  return_val_name: ret_val
42
  parse_question:
43
  prompt_path: prompts/parse_question.py
44
+ engine: text-davinci-003
45
  max_tokens: 512
46
  temperature: 0
47
  query_prefix: '# '
 
54
  return_val_name: ret_val
55
  transform_shape_pts:
56
  prompt_path: prompts/transform_shape_pts.py
57
+ engine: text-davinci-003
58
  max_tokens: 512
59
  temperature: 0
60
  query_prefix: '# '
 
67
  return_val_name: new_shape_pts
68
  fgen:
69
  prompt_path: prompts/fgen.py
70
+ engine: text-davinci-003
71
  max_tokens: 512
72
  temperature: 0
73
  query_prefix: '# define function: '
prompts/tabletop_ui.py CHANGED
@@ -4,7 +4,7 @@ from env_utils import put_first_on_second, get_obj_pos, get_obj_names, say, get_
4
  from plan_utils import parse_obj_name, parse_position, parse_question, transform_shape_pts
5
 
6
  objects = ['yellow block', 'green block', 'yellow bowl', 'blue block', 'blue bowl', 'green bowl']
7
- # the yellow block on the yellow bowl.
8
  say('Ok - putting the yellow block on the yellow bowl')
9
  put_first_on_second('yellow block', 'yellow bowl')
10
  objects = ['yellow block', 'green block', 'yellow bowl', 'blue block', 'blue bowl', 'green bowl']
@@ -15,6 +15,11 @@ objects = ['yellow block', 'green block', 'yellow bowl', 'blue block', 'blue bow
15
  say('Got it - putting the green block on the top right corner')
16
  corner_pos = parse_position('top right corner')
17
  put_first_on_second('green block', corner_pos)
 
 
 
 
 
18
  objects = ['cyan block', 'white block', 'cyan bowl', 'blue block', 'blue bowl', 'white bowl']
19
  # move the cyan block into its corresponding bowl.
20
  matches = {'cyan block': 'cyan bowl'}
@@ -34,7 +39,7 @@ say('Sure thing - putting the yellow block between the blue bowl and the green b
34
  target_pos = parse_position('a point in the middle betweeen the blue bowl and the green block')
35
  put_first_on_second('yellow block', target_pos)
36
  objects = ['yellow block', 'red block', 'yellow bowl', 'gray block', 'gray bowl', 'red bowl']
37
- # can you cut the blocks in half.
38
  say('no, I can only move objects around')
39
  objects = ['yellow block', 'green block', 'yellow bowl', 'gray block', 'gray bowl', 'green bowl']
40
  # stack the blocks on the right side with the gray one on the bottom.
@@ -43,24 +48,24 @@ right_side = parse_position('the right side')
43
  put_first_on_second('gray block', right_side)
44
  order_bottom_to_top = ['gray block', 'green block', 'yellow block']
45
  stack_objects_in_order(object_names=order_bottom_to_top)
 
 
 
 
 
46
  objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
47
- # stack everything with the green block on top.
48
- say('Ok! Stacking everything with the green block on the top')
49
- order_bottom_to_top = ['blue bowl', 'pink bowl', 'green bowl', 'pink block', 'blue block', 'green block']
50
- stack_objects_in_order(object_names=order_bottom_to_top)
51
- objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
52
- # move the grass-colored block to the left.
53
- say('Sure - moving the green block left by 10 centimeters')
54
- left_pos = parse_position('a point 10cm left of the green block')
55
- put_first_on_second('green block', left_pos)
56
  objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
57
- # why did you move the red block.
58
- say(f'I did not move the red block')
59
  objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
60
  # undo that.
61
- say('Sure - moving the green block right by 10 centimeters')
62
- left_pos = parse_position('a point 10cm right of the green block')
63
- put_first_on_second('green block', left_pos)
64
  objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
65
  # place the top most block to the corner closest to the bottom most block.
66
  top_block_name = parse_obj_name('top most block', f'objects = {get_obj_names()}')
@@ -69,13 +74,22 @@ closest_corner_pos = parse_position(f'the corner closest to the {bottom_block_na
69
  say(f'Putting the {top_block_name} on the {get_corner_name(closest_corner_pos)}')
70
  put_first_on_second(top_block_name, closest_corner_pos)
71
  objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
72
- # move the brown block to the side closest to the green bowl.
73
- closest_side_position = parse_position('the side closest to the green bowl')
74
- say(f'Got it - putting the brown block on the {get_side_name(closest_side_position)}')
75
- put_first_on_second('brown block', closest_side_position)
 
 
 
 
 
 
 
 
 
76
  objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
77
  # move the other blocks to the bottom corners.
78
- block_names = parse_obj_name('blocks other than the green block', f'objects = {get_obj_names()}')
79
  corners = parse_position('the bottom corners')
80
  for block_name, pos in zip(block_names, corners):
81
  put_first_on_second(block_name, pos)
@@ -107,6 +121,12 @@ say(f'Ok - moving the other {len(block_names)} blocks to different corners')
107
  for block_name, pos in zip(block_names, corners):
108
  put_first_on_second(block_name, pos)
109
  objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
 
 
 
 
 
 
110
  # what are the blocks left of the green bowl.
111
  left_block_names = parse_question('what are the blocks left of the green bowl', f'objects = {get_obj_names()}')
112
  if len(left_block_names) > 0:
@@ -121,6 +141,10 @@ objects = ['yellow block', 'green block', 'yellow bowl', 'blue block', 'blue bow
121
  order_bottom_to_top = ['green bowl', 'blue block', 'green block', 'yellow block']
122
  say('stacking the blocks on the green bowl')
123
  stack_objects_in_order(object_names=order_bottom_to_top)
 
 
 
 
124
  objects = ['pink block', 'gray block', 'orange block']
125
  # move all blocks 5cm toward the top.
126
  say('Ok - moving all blocks 5cm toward the top')
 
4
  from plan_utils import parse_obj_name, parse_position, parse_question, transform_shape_pts
5
 
6
  objects = ['yellow block', 'green block', 'yellow bowl', 'blue block', 'blue bowl', 'green bowl']
7
+ # place the yellow block on the yellow bowl.
8
  say('Ok - putting the yellow block on the yellow bowl')
9
  put_first_on_second('yellow block', 'yellow bowl')
10
  objects = ['yellow block', 'green block', 'yellow bowl', 'blue block', 'blue bowl', 'green bowl']
 
15
  say('Got it - putting the green block on the top right corner')
16
  corner_pos = parse_position('top right corner')
17
  put_first_on_second('green block', corner_pos)
18
+ objects = ['yellow block', 'green block', 'yellow bowl', 'blue block', 'blue bowl', 'green bowl']
19
+ # stack the blue bowl on the yellow bowl on the green block.
20
+ order_bottom_to_top = ['green block', 'yellow block', 'blue bowl']
21
+ say(f'Sure - stacking from top to bottom: {", ".join(order_bottom_to_top)}')
22
+ stack_objects_in_order(object_names=order_bottom_to_top)
23
  objects = ['cyan block', 'white block', 'cyan bowl', 'blue block', 'blue bowl', 'white bowl']
24
  # move the cyan block into its corresponding bowl.
25
  matches = {'cyan block': 'cyan bowl'}
 
39
  target_pos = parse_position('a point in the middle betweeen the blue bowl and the green block')
40
  put_first_on_second('yellow block', target_pos)
41
  objects = ['yellow block', 'red block', 'yellow bowl', 'gray block', 'gray bowl', 'red bowl']
42
+ # can you cut the bowls in half.
43
  say('no, I can only move objects around')
44
  objects = ['yellow block', 'green block', 'yellow bowl', 'gray block', 'gray bowl', 'green bowl']
45
  # stack the blocks on the right side with the gray one on the bottom.
 
48
  put_first_on_second('gray block', right_side)
49
  order_bottom_to_top = ['gray block', 'green block', 'yellow block']
50
  stack_objects_in_order(object_names=order_bottom_to_top)
51
+ objects = ['yellow block', 'green block', 'yellow bowl', 'blue block', 'blue bowl', 'green bowl']
52
+ # hide the blue bowl.
53
+ bowl_name = np.random.choice(['yellow bowl', 'green bowl'])
54
+ say(f'Sounds good! Hiding the blue bowl under the {bowl_name}')
55
+ put_first_on_second(bowl_name, 'blue bowl')
56
  objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
57
+ # move the grass-colored bowl to the left.
58
+ say('Sure - moving the green bowl left by 10 centimeters')
59
+ left_pos = parse_position('a point 10cm left of the green bowl')
60
+ put_first_on_second('green bowl', left_pos)
 
 
 
 
 
61
  objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
62
+ # why did you move the red bowl.
63
+ say(f'I did not move the red bowl')
64
  objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
65
  # undo that.
66
+ say('Sure - moving the green bowl right by 10 centimeters')
67
+ left_pos = parse_position('a point 10cm right of the green bowl')
68
+ put_first_on_second('green bowl', left_pos)
69
  objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
70
  # place the top most block to the corner closest to the bottom most block.
71
  top_block_name = parse_obj_name('top most block', f'objects = {get_obj_names()}')
 
74
  say(f'Putting the {top_block_name} on the {get_corner_name(closest_corner_pos)}')
75
  put_first_on_second(top_block_name, closest_corner_pos)
76
  objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
77
+ # move the brown bowl to the side closest to the green block.
78
+ closest_side_position = parse_position('the side closest to the green block')
79
+ say(f'Got it - putting the brown bowl on the {get_side_name(closest_side_position)}')
80
+ put_first_on_second('brown bowl', closest_side_position)
81
+ objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
82
+ # place the green block to the right of the bowl that has the blue block.
83
+ bowl_name = parse_obj_name('the bowl that has the blue block', f'objects = {get_obj_names()}')
84
+ if bowl_name:
85
+ target_pos = parse_position(f'a point 10cm to the right of the {bowl_name}')
86
+ say(f'No problem - placing the green block to the right of the {bowl_name}')
87
+ put_first_on_second('green block', target_pos)
88
+ else:
89
+ say('There are no bowls that has the blue block')
90
  objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
91
  # move the other blocks to the bottom corners.
92
+ block_names = parse_obj_name('blocks other than the blue block', f'objects = {get_obj_names()}')
93
  corners = parse_position('the bottom corners')
94
  for block_name, pos in zip(block_names, corners):
95
  put_first_on_second(block_name, pos)
 
121
  for block_name, pos in zip(block_names, corners):
122
  put_first_on_second(block_name, pos)
123
  objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
124
+ # is the pink block on the green bowl.
125
+ if parse_question('is the pink block on the green bowl', f'objects = {get_obj_names()}'):
126
+ say('Yes - the pink block is on the green bowl.')
127
+ else:
128
+ say('No - the pink block is not on the green bowl.')
129
+ objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
130
  # what are the blocks left of the green bowl.
131
  left_block_names = parse_question('what are the blocks left of the green bowl', f'objects = {get_obj_names()}')
132
  if len(left_block_names) > 0:
 
141
  order_bottom_to_top = ['green bowl', 'blue block', 'green block', 'yellow block']
142
  say('stacking the blocks on the green bowl')
143
  stack_objects_in_order(object_names=order_bottom_to_top)
144
+ objects = ['yellow block', 'green block', 'yellow bowl', 'gray block', 'gray bowl', 'green bowl']
145
+ # show me what happens when the desert gets flooded by the ocean.
146
+ say('putting the yellow bowl on the blue bowl')
147
+ put_first_on_second('yellow bowl', 'blue bowl')
148
  objects = ['pink block', 'gray block', 'orange block']
149
  # move all blocks 5cm toward the top.
150
  say('Ok - moving all blocks 5cm toward the top')