SceneDiffuser commited on
Commit
75dc06d
β€’
1 Parent(s): dd12e04
Files changed (4) hide show
  1. README.md +11 -4
  2. app.py +14 -24
  3. interface.py +64 -18
  4. scenediffuser +1 -1
README.md CHANGED
@@ -1,11 +1,18 @@
1
  ---
2
  title: SceneDiffuserDemo
3
- emoji: 🌍
4
- colorFrom: green
5
- colorTo: indigo
6
  sdk: gradio
7
- sdk_version: 3.16.1
8
  app_file: app.py
 
 
 
 
 
 
 
9
  pinned: false
10
  ---
11
 
 
1
  ---
2
  title: SceneDiffuserDemo
3
+ emoji: πŸ›‹
4
+ colorFrom: yellow
5
+ colorTo: pink
6
  sdk: gradio
7
+ sdk_version: 3.18.0
8
  app_file: app.py
9
+ tags:
10
+ - 3D
11
+ - Scene Understanding
12
+ - Diffusion
13
+ - Generation
14
+ - Optimization
15
+ - Planning
16
  pinned: false
17
  ---
18
 
app.py CHANGED
@@ -13,7 +13,7 @@ with gr.Blocks(css='style.css') as demo:
13
  gr.HTML(value="<p align='center' style='font-size: 1.2em; color: #485fc7;'><a href='https://arxiv.org/abs/2301.06015' target='_blank'>arXiv</a> | <a href='https://scenediffuser.github.io/' target='_blank'>Project Page</a> | <a href='https://github.com/scenediffuser/Scene-Diffuser' target='_blank'>Code</a></p>")
14
  gr.Markdown("<p align='center'><i>\"SceneDiffuser provides a unified model for solving scene-conditioned generation, optimization, and planning.\"</i></p>")
15
 
16
- ## five task
17
  ## pose generation
18
  with gr.Tab("Pose Generation"):
19
  with gr.Row():
@@ -32,33 +32,23 @@ with gr.Blocks(css='style.css') as demo:
32
  button1.click(IF.pose_generation, inputs=input1, outputs=[image1])
33
 
34
  ## motion generation
35
- # with gr.Tab("Motion Generation"):
36
- # with gr.Row():
37
- # with gr.Column(scale=2):
38
- # selector2 = gr.Dropdown(choices=['MPH16', 'MPH1Library', 'N0SittingBooth', 'N3OpenArea'], label='Scenes', value='MPH16', interactive=True)
39
- # with gr.Row():
40
- # sample2 = gr.Slider(minimum=1, maximum=8, step=1, label='Count', interactive=True, value=1)
41
- # seed2 = gr.Slider(minimum=0, maximum=2 ** 16, step=1, label='Seed', interactive=True, value=2023)
42
- # with gr.Row():
43
- # withstart = gr.Checkbox(label='With Start', interactive=True, value=False)
44
- # opt2 = gr.Checkbox(label='Optimizer Guidance', interactive=True, value=True)
45
- # scale_opt2 = gr.Slider(minimum=0.1, maximum=9.9, step=0.1, label='Scale', interactive=True, value=1.1)
46
- # button2 = gr.Button("Run")
47
- # with gr.Column(scale=3):
48
- # image2 = gr.Image(label="Result")
49
- # input2 = [selector2, sample2, seed2, withstart, opt2, scale_opt2]
50
- # button2.click(IF.motion_generation, inputs=input2, outputs=image2)
51
  with gr.Tab("Motion Generation"):
52
  with gr.Row():
53
  with gr.Column(scale=2):
54
- input2 = [
55
- gr.Dropdown(choices=['MPH16', 'MPH1Library', 'N0SittingBooth', 'N3OpenArea'], label='Scenes')
56
- ]
57
- button2 = gr.Button("Generate")
58
- gr.HTML("<p style='font-size: 0.9em; color: #555555;'>Notes: the output results are pre-sampled results. We will deploy a real-time model for this task soon.</p>")
 
 
 
 
59
  with gr.Column(scale=3):
60
- output2 = gr.Image(label="Result")
61
- button2.click(IF.motion_generation, inputs=input2, outputs=output2)
 
 
62
 
63
  ## grasp generation
64
  with gr.Tab("Grasp Generation"):
 
13
  gr.HTML(value="<p align='center' style='font-size: 1.2em; color: #485fc7;'><a href='https://arxiv.org/abs/2301.06015' target='_blank'>arXiv</a> | <a href='https://scenediffuser.github.io/' target='_blank'>Project Page</a> | <a href='https://github.com/scenediffuser/Scene-Diffuser' target='_blank'>Code</a></p>")
14
  gr.Markdown("<p align='center'><i>\"SceneDiffuser provides a unified model for solving scene-conditioned generation, optimization, and planning.\"</i></p>")
15
 
16
+ ## five tasks
17
  ## pose generation
18
  with gr.Tab("Pose Generation"):
19
  with gr.Row():
 
32
  button1.click(IF.pose_generation, inputs=input1, outputs=[image1])
33
 
34
  ## motion generation
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
35
  with gr.Tab("Motion Generation"):
36
  with gr.Row():
37
  with gr.Column(scale=2):
38
+ selector2 = gr.Dropdown(choices=['MPH16', 'MPH1Library', 'N0SittingBooth', 'N3OpenArea'], label='Scenes', value='MPH16', interactive=True)
39
+ with gr.Row():
40
+ sample2 = gr.Slider(minimum=1, maximum=2, step=1, label='Count', interactive=True, value=1)
41
+ seed2 = gr.Slider(minimum=0, maximum=2 ** 16, step=1, label='Seed', interactive=True, value=2023)
42
+ with gr.Row():
43
+ withstart = gr.Checkbox(label='With Start', interactive=True, value=False)
44
+ opt2 = gr.Checkbox(label='Optimizer Guidance', interactive=True, value=False)
45
+ scale_opt2 = gr.Slider(minimum=0.1, maximum=9.9, step=0.1, label='Scale', interactive=True, value=1.1)
46
+ button2 = gr.Button("Run")
47
  with gr.Column(scale=3):
48
+ image2 = gr.Gallery(label="Image [Result]").style(grid=[1], height="50")
49
+ gr.HTML("<p style='font-size: 0.9em; color: #555555;'>Notes: For motion generation, it will take a long time to do sampleing and rendering, especifically when you tick optimizer guidance.</p>")
50
+ input2 = [selector2, sample2, seed2, withstart, opt2, scale_opt2]
51
+ button2.click(IF.motion_generation, inputs=input2, outputs=image2)
52
 
53
  ## grasp generation
54
  with gr.Tab("Grasp Generation"):
interface.py CHANGED
@@ -4,6 +4,8 @@ import torch
4
  import hydra
5
  import numpy as np
6
  import zipfile
 
 
7
 
8
  from typing import Any
9
  from hydra import compose, initialize
@@ -24,9 +26,9 @@ def model_weight_path(task, has_observation=False):
24
  if task == 'pose_gen':
25
  return hf_hub_download('SceneDiffuser/SceneDiffuser', 'weights/2022-11-09_11-22-52_PoseGen_ddm4_lr1e-4_ep100/ckpts/model.pth')
26
  elif task == 'motion_gen' and has_observation == True:
27
- return hf_hub_download('SceneDiffuser/SceneDiffuser', 'weights//ckpts/model.pth')
28
  elif task == 'motion_gen' and has_observation == False:
29
- return hf_hub_download('SceneDiffuser/SceneDiffuser', 'weights//ckpts/model.pth')
30
  elif task == 'path_planning':
31
  return hf_hub_download('SceneDiffuser/SceneDiffuser', 'weights/2022-11-25_20-57-28_Path_ddm4_LR1e-4_E100_REL/ckpts/model.pth')
32
  else:
@@ -140,7 +142,10 @@ def _planning(cfg: DictConfig, scene: str) -> Any:
140
 
141
 
142
  ## interface for five task
143
- ## real-time model: pose generation, path planning
 
 
 
144
  def pose_generation(scene, count, seed, opt, scale) -> Any:
145
  scene_model_weight_path = pretrain_pointtrans_weight_path()
146
  data_dir, smpl_dir, prox_dir, vposer_dir = pose_motion_data_path()
@@ -181,22 +186,63 @@ def pose_generation(scene, count, seed, opt, scale) -> Any:
181
  hydra.core.global_hydra.GlobalHydra.instance().clear()
182
  return res
183
 
184
- def motion_generation(scene):
185
- assert isinstance(scene, str)
186
- cnt = {
187
- 'MPH1Library': 3,
188
- 'MPH16': 6,
189
- 'N0SittingBooth': 7,
190
- 'N3OpenArea': 5
191
- }[scene]
192
-
193
- res = f"./results/motion_generation/results/{scene}/{random.randint(0, cnt-1)}.gif"
194
- if not os.path.exists(res):
195
- results_path = hf_hub_download('SceneDiffuser/SceneDiffuser', 'results/motion_generation/results.zip')
196
- os.makedirs('./results/motion_generation/', exist_ok=True)
197
- with zipfile.ZipFile(results_path, 'r') as zip_ref:
198
- zip_ref.extractall('./results/motion_generation/')
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
199
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
200
  return res
201
 
202
  def grasp_generation(case_id):
 
4
  import hydra
5
  import numpy as np
6
  import zipfile
7
+ import time
8
+ import uuid
9
 
10
  from typing import Any
11
  from hydra import compose, initialize
 
26
  if task == 'pose_gen':
27
  return hf_hub_download('SceneDiffuser/SceneDiffuser', 'weights/2022-11-09_11-22-52_PoseGen_ddm4_lr1e-4_ep100/ckpts/model.pth')
28
  elif task == 'motion_gen' and has_observation == True:
29
+ return hf_hub_download('SceneDiffuser/SceneDiffuser', 'weights/2022-11-09_14-28-12_MotionGen_ddm_T200_lr1e-4_ep300_obser/ckpts/model.pth')
30
  elif task == 'motion_gen' and has_observation == False:
31
+ return hf_hub_download('SceneDiffuser/SceneDiffuser', 'weights/2022-11-09_12-54-50_MotionGen_ddm_T200_lr1e-4_ep300/ckpts/model.pth')
32
  elif task == 'path_planning':
33
  return hf_hub_download('SceneDiffuser/SceneDiffuser', 'weights/2022-11-25_20-57-28_Path_ddm4_LR1e-4_E100_REL/ckpts/model.pth')
34
  else:
 
142
 
143
 
144
  ## interface for five task
145
+ ## real-time model:
146
+ ## - pose generation
147
+ ## - motion generation
148
+ ## - path planning
149
  def pose_generation(scene, count, seed, opt, scale) -> Any:
150
  scene_model_weight_path = pretrain_pointtrans_weight_path()
151
  data_dir, smpl_dir, prox_dir, vposer_dir = pose_motion_data_path()
 
186
  hydra.core.global_hydra.GlobalHydra.instance().clear()
187
  return res
188
 
189
+ def motion_generation(scene, count, seed, withstart, opt, scale) -> Any:
190
+ scene_model_weight_path = pretrain_pointtrans_weight_path()
191
+ data_dir, smpl_dir, prox_dir, vposer_dir = pose_motion_data_path()
192
+ override_config = [
193
+ "diffuser=ddpm",
194
+ "diffuser.steps=200",
195
+ "model=unet",
196
+ "model.use_position_embedding=true",
197
+ f"model.scene_model.pretrained_weights={scene_model_weight_path}",
198
+ "task=motion_gen",
199
+ f"task.has_observation={withstart}",
200
+ "task.dataset.repr_type=absolute",
201
+ "task.dataset.frame_interval_test=20",
202
+ "task.visualizer.name=MotionGenVisualizerHF",
203
+ f"task.visualizer.ksample={count}",
204
+ f"task.dataset.data_dir={data_dir}",
205
+ f"task.dataset.smpl_dir={smpl_dir}",
206
+ f"task.dataset.prox_dir={prox_dir}",
207
+ f"task.dataset.vposer_dir={vposer_dir}",
208
+ ]
209
+ if opt == True:
210
+ override_config += [
211
+ "optimizer=motion_in_scene",
212
+ "optimizer.scale_type=div_var",
213
+ f"optimizer.scale={scale}",
214
+ "optimizer.vposer=false",
215
+ "optimizer.contact_weight=0.02",
216
+ "optimizer.collision_weight=1.0",
217
+ "optimizer.smoothness_weight=0.001",
218
+ "optimizer.frame_interval=1",
219
+ ]
220
 
221
+ initialize(config_path="./scenediffuser/configs", version_base=None)
222
+ config = compose(config_name="default", overrides=override_config)
223
+
224
+ random.seed(seed)
225
+ np.random.seed(seed)
226
+ torch.manual_seed(seed)
227
+ torch.cuda.manual_seed(seed)
228
+ torch.cuda.manual_seed_all(seed)
229
+
230
+ res_gifs = _sampling(config, scene)
231
+
232
+ ## save sampled motion as .gif file
233
+ datestr = time.strftime("%Y-%m-%d", time.localtime(time.time()))
234
+ target_dir = os.path.join('./results/motion_generation/', f'd-{datestr}')
235
+ os.makedirs(target_dir, exist_ok=True)
236
+ res = []
237
+ uuid_str = uuid.uuid4()
238
+ for i, imgs in enumerate(res_gifs):
239
+ target_path = os.path.join(target_dir, f'{uuid_str}--{i}.gif')
240
+ imgs = [im.resize((720, 405)) for im in imgs] # resize image for low resolution to save space
241
+ img, *img_rest = imgs
242
+ img.save(fp=target_path, format='GIF', append_images=img_rest, save_all=True, duration=33.33, loop=0)
243
+ res.append(target_path)
244
+
245
+ hydra.core.global_hydra.GlobalHydra.instance().clear()
246
  return res
247
 
248
  def grasp_generation(case_id):
scenediffuser CHANGED
@@ -1 +1 @@
1
- Subproject commit 2dfda611966515f5d2aff8ac45225f7f77bf9541
 
1
+ Subproject commit ddcba15d05dcb52f3f3b576f7c60e5e255baa584