jackyliang42 commited on
Commit
47097db
1 Parent(s): 9a40e4f

working logging, readme

Browse files
Files changed (6) hide show
  1. LICENSE.md +7 -0
  2. README.md +45 -1
  3. app.py +28 -16
  4. lmp.py +12 -5
  5. md_logger.py +16 -0
  6. prompts/parse_obj_name.py +5 -10
LICENSE.md ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ Copyright 2021 Google LLC. SPDX-License-Identifier: Apache-2.0
2
+
3
+ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
4
+
5
+ https://www.apache.org/licenses/LICENSE-2.0
6
+
7
+ Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
README.md CHANGED
@@ -10,4 +10,48 @@ pinned: false
10
  license: apache-2.0
11
  ---
12
 
13
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10
  license: apache-2.0
11
  ---
12
 
13
+ # Code as Policies Tabletop Manipulation Interactive Demo
14
+
15
+ This notebook is a part of the open-source code release associated with the paper:
16
+
17
+ [Code as Policies: Language Model Programs for Embodied Control](https://code-as-policies.github.io/)
18
+
19
+ This notebook gives an interactive demo for the simulated tabletop manipulation domain, seen in the paper section IV.D
20
+
21
+ ## Preparations:
22
+
23
+ 1) Obtain an [OpenAI API Key](https://openai.com/blog/openai-api/)
24
+
25
+ 2) Gain Codex access by [joining the waitlist](https://openai.com/blog/openai-codex/)
26
+
27
+ Once you have Codex access you can use `code-davinci-002`. Using the GPT-3 model (`text-dainvci-002`) is also ok, but performance won't be as good (there will be more code logic errors).
28
+
29
+ ## Instructions:
30
+
31
+ 1. Fill in the API Key, model name, and how many blocks and bowls to be spawned in the environment.
32
+ 2. Click Setup/Reset Env
33
+ 3. Based on the new randomly sampled object names, input an instruction and click Run Instruction. If successful, this will render a video and update the simulation environment visualization.
34
+
35
+ You can run instructions in sequence and refer back to previous commands (e.g. do the same with other blocks, move the same block to the other bowl, etc). Click Setup/Reset Env to reset, and this will clear the current instruction history.
36
+
37
+ Supported commands:
38
+ * Spatial reasoning (e.g. to the left of the red block, the closest corner, the farthest bowl, the second block from the right)
39
+ * Sequential actions (e.g. put blocks in matching bowls, stack blocks on the bottom right corner)
40
+ * Contextual commands (e.g. do the same with the blue block, undo that)
41
+ * Language-based reasoning (e.g. put the forest-colored block on the ocean-colored bowl).
42
+ * Simple Q&A (e.g. how many blocks are to the left of the blue bowl?)
43
+
44
+ Example commands (note object names may need to be changed depending the sampled object names):
45
+ * put the sun-colored block on the bowl closest to it
46
+ * stack the blocks on the bottom most bowl
47
+ * arrange the blocks as a square in the middle
48
+ * move the square 5cm to the right
49
+ * how many blocks are to the right of the orange bowl?
50
+ * pick up the block closest to the top left corner and place it on the bottom right corner
51
+
52
+ Known limitations:
53
+ * In simulation we're using ground truth object poses instead of using vision models. This means that commands the require knowledge of visual apperances (e.g. darkest bowl, largest object) are not supported.
54
+ * Currently, the low-level pick place primitive does not do collision checking, so if there are many objects on the table, placing actions may incur collisions.
55
+ * Prompt saturation - if too many commands (10+) are executed in a row, then the LLM may start to ignore examples in the early parts of the prompt.
56
+ * Ambiguous instructions - if a given instruction doesn't lead to the desired actions, try rephrasing it to remove ambiguities (e.g. place the block on the closest bowl -> place the block on its closest bowl)
57
+
app.py CHANGED
@@ -9,10 +9,10 @@ from omegaconf import OmegaConf
9
  from moviepy.editor import ImageSequenceClip
10
  import gradio as gr
11
 
12
-
13
  from lmp import LMP, LMPFGen
14
  from sim import PickPlaceEnv, LMP_wrapper
15
  from consts import ALL_BLOCKS, ALL_BOWLS
 
16
 
17
 
18
  class DemoRunner:
@@ -21,6 +21,7 @@ class DemoRunner:
21
  self._cfg = OmegaConf.to_container(OmegaConf.load('cfg.yaml'), resolve=True)
22
  self._env = None
23
  self._model_name = ''
 
24
 
25
  def make_LMP(self, env):
26
  # LMP env wrapper
@@ -49,20 +50,20 @@ class DemoRunner:
49
  'get_corner_name', 'get_side_name',
50
  ]
51
  }
52
- variable_vars['say'] = lambda msg: print(f'robot says: {msg}')
53
 
54
  # creating the function-generating LMP
55
- lmp_fgen = LMPFGen(cfg['lmps']['fgen'], fixed_vars, variable_vars)
56
 
57
  # creating other low-level LMPs
58
  variable_vars.update({
59
- k: LMP(k, cfg['lmps'][k], lmp_fgen, fixed_vars, variable_vars)
60
  for k in ['parse_obj_name', 'parse_position', 'parse_question', 'transform_shape_pts']
61
  })
62
 
63
  # creating the LMP that deals w/ high-level language commands
64
  lmp_tabletop_ui = LMP(
65
- 'tabletop_ui', cfg['lmps']['tabletop_ui'], lmp_fgen, fixed_vars, variable_vars
66
  )
67
 
68
  return lmp_tabletop_ui
@@ -89,45 +90,56 @@ class DemoRunner:
89
  return 'Please run setup first'
90
 
91
  self._env.cache_video = []
 
92
 
93
- self._lmp_tabletop_ui(instruction, f'objects = {self._env.object_list}')
 
 
 
 
94
 
95
- video_file_name = ''
96
  if self._env.cache_video:
97
  rendered_clip = ImageSequenceClip(self._env.cache_video, fps=25)
98
- video_file_name = NamedTemporaryFile(suffix='.mp4', delete=False).name
99
  rendered_clip.write_videofile(video_file_name, fps=25)
100
 
101
- return 'Done', video_file_name
102
 
103
 
104
  if __name__ == '__main__':
105
  demo_runner = DemoRunner()
106
  demo = gr.Blocks()
107
 
 
 
 
 
 
108
  with demo:
 
109
  with gr.Row():
110
  with gr.Column():
111
  with gr.Row():
112
- inp_api_key = gr.Textbox(label='OpenAI API Key', lines=1, value='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')
113
  inp_model_name = gr.Dropdown(label='Model Name', choices=['code-davinci-002', 'text-davinci-002'], value='code-davinci-002')
114
  with gr.Row():
115
  inp_n_blocks = gr.Slider(label='Num Blocks', minimum=0, maximum=3, value=3, step=1)
116
  inp_n_bowls = gr.Slider(label='Num Bowls', minimum=0, maximum=3, value=3, step=1)
117
 
118
- btn_setup = gr.Button("1) Setup/Reset Env")
119
  info_setup = gr.Markdown(label='Setup Info')
120
  with gr.Column():
121
- img_setup = gr.Image(label='Setup Image')
122
 
123
  with gr.Row():
124
  with gr.Column():
125
 
126
  inp_instruction = gr.Textbox(label='Instruction', lines=1)
127
- btn_run = gr.Button("2) Run Instruction")
128
- info_run = gr.Label(label='Run Info')
129
  with gr.Column():
130
- video_run = gr.Video(label='Run Video')
131
 
132
  btn_setup.click(
133
  demo_runner.setup,
@@ -137,7 +149,7 @@ if __name__ == '__main__':
137
  btn_run.click(
138
  demo_runner.run,
139
  inputs=[inp_instruction],
140
- outputs=[info_run, video_run]
141
  )
142
 
143
  demo.launch()
 
9
  from moviepy.editor import ImageSequenceClip
10
  import gradio as gr
11
 
 
12
  from lmp import LMP, LMPFGen
13
  from sim import PickPlaceEnv, LMP_wrapper
14
  from consts import ALL_BLOCKS, ALL_BOWLS
15
+ from md_logger import MarkdownLogger
16
 
17
 
18
  class DemoRunner:
 
21
  self._cfg = OmegaConf.to_container(OmegaConf.load('cfg.yaml'), resolve=True)
22
  self._env = None
23
  self._model_name = ''
24
+ self._md_logger = MarkdownLogger()
25
 
26
  def make_LMP(self, env):
27
  # LMP env wrapper
 
50
  'get_corner_name', 'get_side_name',
51
  ]
52
  }
53
+ variable_vars['say'] = lambda msg: self._md_logger.log_text(f'Robot says: "{msg}"')
54
 
55
  # creating the function-generating LMP
56
+ lmp_fgen = LMPFGen(cfg['lmps']['fgen'], fixed_vars, variable_vars, self._md_logger)
57
 
58
  # creating other low-level LMPs
59
  variable_vars.update({
60
+ k: LMP(k, cfg['lmps'][k], lmp_fgen, fixed_vars, variable_vars, self._md_logger)
61
  for k in ['parse_obj_name', 'parse_position', 'parse_question', 'transform_shape_pts']
62
  })
63
 
64
  # creating the LMP that deals w/ high-level language commands
65
  lmp_tabletop_ui = LMP(
66
+ 'tabletop_ui', cfg['lmps']['tabletop_ui'], lmp_fgen, fixed_vars, variable_vars, self._md_logger
67
  )
68
 
69
  return lmp_tabletop_ui
 
90
  return 'Please run setup first'
91
 
92
  self._env.cache_video = []
93
+ self._md_logger.clear()
94
 
95
+ try:
96
+ self._lmp_tabletop_ui(instruction, f'objects = {self._env.object_list}')
97
+ run_info = self._md_logger.get_log()
98
+ except Exception as e:
99
+ run_info = f'Error: {e}'
100
 
101
+ video_file_name = None
102
  if self._env.cache_video:
103
  rendered_clip = ImageSequenceClip(self._env.cache_video, fps=25)
104
+ video_file_name = NamedTemporaryFile(suffix='.mp4').name
105
  rendered_clip.write_videofile(video_file_name, fps=25)
106
 
107
+ return run_info, self._env.get_camera_image(), video_file_name
108
 
109
 
110
  if __name__ == '__main__':
111
  demo_runner = DemoRunner()
112
  demo = gr.Blocks()
113
 
114
+ with open('README.md', 'r') as f:
115
+ for _ in range(12):
116
+ next(f)
117
+ readme_text = f.read()
118
+
119
  with demo:
120
+ gr.Markdown(readme_text)
121
  with gr.Row():
122
  with gr.Column():
123
  with gr.Row():
124
+ inp_api_key = gr.Textbox(label='OpenAI API Key', lines=1)
125
  inp_model_name = gr.Dropdown(label='Model Name', choices=['code-davinci-002', 'text-davinci-002'], value='code-davinci-002')
126
  with gr.Row():
127
  inp_n_blocks = gr.Slider(label='Num Blocks', minimum=0, maximum=3, value=3, step=1)
128
  inp_n_bowls = gr.Slider(label='Num Bowls', minimum=0, maximum=3, value=3, step=1)
129
 
130
+ btn_setup = gr.Button("Setup/Reset Env")
131
  info_setup = gr.Markdown(label='Setup Info')
132
  with gr.Column():
133
+ img_setup = gr.Image(label='Current Simulation')
134
 
135
  with gr.Row():
136
  with gr.Column():
137
 
138
  inp_instruction = gr.Textbox(label='Instruction', lines=1)
139
+ btn_run = gr.Button("Run Instruction")
140
+ info_run = gr.Markdown(label='Generated Code')
141
  with gr.Column():
142
+ video_run = gr.Video(label='Video of Last Instruction')
143
 
144
  btn_setup.click(
145
  demo_runner.setup,
 
149
  btn_run.click(
150
  demo_runner.run,
151
  inputs=[inp_instruction],
152
+ outputs=[info_run, img_setup, video_run]
153
  )
154
 
155
  demo.launch()
lmp.py CHANGED
@@ -10,9 +10,10 @@ from pygments.formatters import TerminalFormatter
10
 
11
  class LMP:
12
 
13
- def __init__(self, name, cfg, lmp_fgen, fixed_vars, variable_vars):
14
  self._name = name
15
  self._cfg = cfg
 
16
 
17
  with open(self._cfg['prompt_path'], 'r') as f:
18
  self._base_prompt = f.read()
@@ -72,7 +73,9 @@ class LMP:
72
  to_log = f'{use_query}\n{to_exec}'
73
 
74
  to_log_pretty = highlight(to_log, PythonLexer(), TerminalFormatter())
75
- print(f'LMP {self._name} exec:\n\n{to_log_pretty}\n')
 
 
76
 
77
  new_fs = self._lmp_fgen.create_new_fs_from_code(code_str)
78
  self._variable_vars.update(new_fs)
@@ -94,12 +97,13 @@ class LMP:
94
 
95
  class LMPFGen:
96
 
97
- def __init__(self, cfg, fixed_vars, variable_vars):
98
  self._cfg = cfg
99
 
100
  self._stop_tokens = list(self._cfg['stop'])
101
  self._fixed_vars = fixed_vars
102
  self._variable_vars = variable_vars
 
103
 
104
  with open(self._cfg['prompt_path'], 'r') as f:
105
  self._base_prompt = f.read()
@@ -142,8 +146,11 @@ class LMPFGen:
142
 
143
  f = lvars[f_name]
144
 
145
- to_print = highlight(f'{use_query}\n{f_src}', PythonLexer(), TerminalFormatter())
146
- print(f'LMP FGEN created:\n\n{to_print}\n')
 
 
 
147
 
148
  if return_src:
149
  return f, f_src
 
10
 
11
  class LMP:
12
 
13
+ def __init__(self, name, cfg, lmp_fgen, fixed_vars, variable_vars, md_logger):
14
  self._name = name
15
  self._cfg = cfg
16
+ self._md_logger = md_logger
17
 
18
  with open(self._cfg['prompt_path'], 'r') as f:
19
  self._base_prompt = f.read()
 
73
  to_log = f'{use_query}\n{to_exec}'
74
 
75
  to_log_pretty = highlight(to_log, PythonLexer(), TerminalFormatter())
76
+ print(f'LMP {self._name} generated code:\n{to_log_pretty}')
77
+ self._md_logger.log_text(f'LMP {self._name} Generated Code:')
78
+ self._md_logger.log_code(to_log)
79
 
80
  new_fs = self._lmp_fgen.create_new_fs_from_code(code_str)
81
  self._variable_vars.update(new_fs)
 
97
 
98
  class LMPFGen:
99
 
100
+ def __init__(self, cfg, fixed_vars, variable_vars, md_logger):
101
  self._cfg = cfg
102
 
103
  self._stop_tokens = list(self._cfg['stop'])
104
  self._fixed_vars = fixed_vars
105
  self._variable_vars = variable_vars
106
+ self._md_logger = md_logger
107
 
108
  with open(self._cfg['prompt_path'], 'r') as f:
109
  self._base_prompt = f.read()
 
146
 
147
  f = lvars[f_name]
148
 
149
+ to_print = f'{use_query}\n{f_src}'
150
+ to_print_pretty = highlight(to_print, PythonLexer(), TerminalFormatter())
151
+ print(f'LMPFGen generated code:\n{to_print_pretty}')
152
+ self._md_logger.log_text('Generated Function:')
153
+ self._md_logger.log_code(to_print)
154
 
155
  if return_src:
156
  return f, f_src
md_logger.py ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ class MarkdownLogger:
2
+
3
+ def __init__(self):
4
+ self._log = ''
5
+
6
+ def log_text(self, text):
7
+ self._log += '\n' + text + '\n'
8
+
9
+ def log_code(self, code):
10
+ self._log += f'\n```python\n{code}\n```\n'
11
+
12
+ def clear(self):
13
+ self._log = ''
14
+
15
+ def get_log(self):
16
+ return self._log
prompts/parse_obj_name.py CHANGED
@@ -5,8 +5,7 @@ from utils import get_obj_positions_np
5
  objects = ['blue block', 'cyan block', 'purple bowl', 'gray bowl', 'brown bowl', 'pink block', 'purple block']
6
  # the block closest to the purple bowl.
7
  block_names = ['blue block', 'cyan block', 'purple block']
8
- block_positions = get_obj_positions_np(block_names)
9
- closest_block_idx = get_closest_idx(points=block_positions, point=get_obj_pos('purple bowl'))
10
  closest_block_name = block_names[closest_block_idx]
11
  ret_val = closest_block_name
12
  objects = ['brown bowl', 'banana', 'brown block', 'apple', 'blue bowl', 'blue block']
@@ -37,28 +36,24 @@ objects = ['blue block', 'cyan block', 'purple bowl', 'brown bowl', 'purple bloc
37
  # the block closest to the bottom right corner.
38
  corner_pos = parse_position('bottom right corner')
39
  block_names = ['blue block', 'cyan block', 'purple block']
40
- block_positions = get_obj_positions_np(block_names)
41
- closest_block_idx = get_closest_idx(points=block_positions, point=corner_pos)
42
  closest_block_name = block_names[closest_block_idx]
43
  ret_val = closest_block_name
44
  objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
45
  # the left most block.
46
  block_names = ['green block', 'brown block', 'blue block']
47
- block_positions = get_obj_positions_np(block_names)
48
- left_block_idx = np.argsort(block_positions[:, 0])[0]
49
  left_block_name = block_names[left_block_idx]
50
  ret_val = left_block_name
51
  objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
52
  # the bowl on near the top.
53
  bowl_names = ['brown bowl', 'green bowl', 'blue bowl']
54
- bowl_positions = get_obj_positions_np(bowl_names)
55
- top_bowl_idx = np.argsort(block_positions[:, 1])[-1]
56
  top_bowl_name = bowl_names[top_bowl_idx]
57
  ret_val = top_bowl_name
58
  objects = ['yellow bowl', 'purple block', 'yellow block', 'purple bowl', 'pink bowl', 'pink block']
59
  # the third bowl from the right.
60
  bowl_names = ['yellow bowl', 'purple bowl', 'pink bowl']
61
- bowl_positions = get_obj_positions_np(bowl_names)
62
- bowl_idx = np.argsort(block_positions[:, 0])[-3]
63
  bowl_name = bowl_names[bowl_idx]
64
  ret_val = bowl_name
 
5
  objects = ['blue block', 'cyan block', 'purple bowl', 'gray bowl', 'brown bowl', 'pink block', 'purple block']
6
  # the block closest to the purple bowl.
7
  block_names = ['blue block', 'cyan block', 'purple block']
8
+ closest_block_idx = get_closest_idx(points=get_obj_positions_np(block_names), point=get_obj_pos('purple bowl'))
 
9
  closest_block_name = block_names[closest_block_idx]
10
  ret_val = closest_block_name
11
  objects = ['brown bowl', 'banana', 'brown block', 'apple', 'blue bowl', 'blue block']
 
36
  # the block closest to the bottom right corner.
37
  corner_pos = parse_position('bottom right corner')
38
  block_names = ['blue block', 'cyan block', 'purple block']
39
+ closest_block_idx = get_closest_idx(points=get_obj_positions_np(block_names), point=corner_pos)
 
40
  closest_block_name = block_names[closest_block_idx]
41
  ret_val = closest_block_name
42
  objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
43
  # the left most block.
44
  block_names = ['green block', 'brown block', 'blue block']
45
+ left_block_idx = np.argsort(get_obj_positions_np(block_names)[:, 0])[0]
 
46
  left_block_name = block_names[left_block_idx]
47
  ret_val = left_block_name
48
  objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
49
  # the bowl on near the top.
50
  bowl_names = ['brown bowl', 'green bowl', 'blue bowl']
51
+ top_bowl_idx = np.argsort(get_obj_positions_np(bowl_names)[:, 1])[-1]
 
52
  top_bowl_name = bowl_names[top_bowl_idx]
53
  ret_val = top_bowl_name
54
  objects = ['yellow bowl', 'purple block', 'yellow block', 'purple bowl', 'pink bowl', 'pink block']
55
  # the third bowl from the right.
56
  bowl_names = ['yellow bowl', 'purple bowl', 'pink bowl']
57
+ bowl_idx = np.argsort(get_obj_positions_np(bowl_names)[:, 0])[-3]
 
58
  bowl_name = bowl_names[bowl_idx]
59
  ret_val = bowl_name