================= Task and Asset Design! >>> Prompt: You are an AI in robot simulation code and task design. I will provide you some example tasks, code implementation, and some guidelines for how to generate tasks and then you will help me generate a new task `TARGET_TASK_NAME`. My goal is to design creative and feasible tasks for tabletop manipulation. I will first ask you to describe the task in natural languages and then will let you write the code for it. ========= Here are all the assets. Use only these assets in the task and code design. """ insertion/: ell.urdf fixture.urdf bowl/: bowl.urdf box/: box-template.urdf stacking/: block.urdf stand.urdf zone/: zone.obj zone.urdf pallet/: pallet.obj pallet.urdf ball/: ball-template.urdf cylinder/: cylinder-template.urdf bowl/: bowl.urdf # assets not for picking corner/: corner-template.urdf line/: single-green-line-template.urdf container/: container-template.urdf """ There are certain rules on the asset usage. 1. Sweeping piles task must have small blocks `block/small.urdf` and zones `zone.urdf`. Only the piles can be swept in all assets 2. Insertion tasks must have `insertion/ell.urdf` and `insertion/fixture.urdf`. Only the fixture can be inserted in all assets. ========= Here are some examples of good tasks. Try to be creative and high standard, and avoid overlapping with these tasks. TASK_DESCRIPTION_PROMPT ========= Now let's design the task `TARGET_TASK_NAME`. Please describe the new task `TARGET_TASK_NAME` in natural languages in a clear and detailed way. Format the answer in a python dictionary with keys "task-name" and value type string with lower-case and separated by hyphens, "task-description" (one sentence and do not mention urdf paths) and value type string, and "assets-used" and value type list of strings. Note: - Do not use assets that are not in the list above. - Tasks that have more colors and shapes are interesting. - Be as specific as possible about the number, shape, and color of each asset in the task descriptions. - The task need to obey physics and remain feasible. ========= Now I will provide you some reference code and you can write the code for the task. """ import os import numpy as np from cliport.tasks.task import Task from cliport.utils import utils class PackingShapes(Task): """pick up randomly sized shapes and place them tightly into a container.""" def __init__(self): super().__init__() self.max_steps = 1 self.homogeneous = False self.lang_template = "pack the {obj} in the brown box" self.task_completed_desc = "done packing shapes." self.additional_reset() def reset(self, env): super().reset(env) # Shape Names: shapes = utils.assembling_kit_shapes n_objects = 5 if self.mode == 'train': obj_shapes = np.random.choice(self.train_set, n_objects, replace=False) else: if self.homogeneous: obj_shapes = [np.random.choice(self.test_set, replace=False)] * n_objects else: obj_shapes = np.random.choice(self.test_set, n_objects, replace=False) # Shuffle colors to avoid always picking an object of the same color colors, color_names = utils.get_colors(mode=self.mode) # Add container box. zone_size = self.get_random_size(0.1, 0.15, 0.1, 0.15, 0.05, 0.05) zone_pose = self.get_random_pose(env, zone_size) container_template = 'container/container-template.urdf' replace = {'DIM': zone_size, 'HALF': (zone_size[0] / 2, zone_size[1] / 2, zone_size[2] / 2)} # IMPORTANT: REPLACE THE TEMPLATE URDF with `fill_template` container_urdf = self.fill_template(container_template, replace) env.add_object(container_urdf, zone_pose, 'fixed') # Add objects. objects = [] template = 'kitting/object-template.urdf' for i in range(n_objects): shape = obj_shapes[i] # x, y, z dimensions for the asset size size = (0.08, 0.08, 0.02) pose= self.get_random_pose(env, size) fname = f'{shape:02d}.obj' fname = os.path.join(self.assets_root, 'kitting', fname) scale = [0.003, 0.003, 0.001] replace = {'FNAME': (fname,), 'SCALE': scale, 'COLOR': colors[i]} # IMPORTANT: REPLACE THE TEMPLATE URDF urdf = self.fill_template(template, replace) block_id = env.add_object(urdf, pose) objects.append(block_id) # Pick the first shape. num_objects_to_pick = 1 for i in range(num_objects_to_pick): # IMPORTANT: Specify (obj_pts, [(zone_pose, zone_size)]) for target `zone`. obj_pts is a dict self.add_goal(objs=[objects[i]], matches=np.int32([[1]]), targ_poses=[zone_pose], replace=False, rotations=True, metric='zone', params=[(zone_pose, zone_size)], step_max_reward=1 / num_objects_to_pick) self.lang_goals.append(self.lang_template.format(obj=shapes[obj_shapes[i]])) """ """ import numpy as np from cliport.tasks.task import Task from cliport.utils import utils import pybullet as p class PlaceRedInGreen(Task): """pick up the red blocks and place them into the green bowls amidst other objects.""" def __init__(self): super().__init__() self.max_steps = 10 self.lang_template = "put the red blocks in a green bowl" self.task_completed_desc = "done placing blocks in bowls." self.additional_reset() def reset(self, env): super().reset(env) n_bowls = np.random.randint(1, 4) n_blocks = np.random.randint(1, n_bowls + 1) # Add bowls. # x, y, z dimensions for the asset size bowl_size = (0.12, 0.12, 0) bowl_urdf = 'bowl/bowl.urdf' bowl_poses = [] for _ in range(n_bowls): bowl_pose = self.get_random_pose(env, obj_size=bowl_size) env.add_object(urdf=bowl_urdf, pose=bowl_pose, category='fixed') bowl_poses.append(bowl_pose) # Add blocks. # x, y, z dimensions for the asset size blocks = [] block_size = (0.04, 0.04, 0.04) block_urdf = 'stacking/block.urdf' for _ in range(n_blocks): block_pose = self.get_random_pose(env, obj_size=block_size) block_id = env.add_object(block_urdf, block_pose) blocks.append(block_id) # Goal: each red block is in a different green bowl. self.add_goal(objs=blocks, matches=np.ones((len(blocks), len(bowl_poses))), targ_poses=bowl_poses, replace=False, rotations=True, metric='pose', params=None, step_max_reward=1) self.lang_goals.append(self.lang_template) # Colors of distractor objects. # IMPORTANT: RETRIEVE THE ACTUAL COLOR VALUES bowl_colors = [utils.COLORS[c] for c in utils.COLORS if c != 'green'] block_colors = [utils.COLORS[c] for c in utils.COLORS if c != 'red'] # Add distractors. n_distractors = 0 while n_distractors < 6: is_block = np.random.rand() > 0.5 urdf = block_urdf if is_block else bowl_urdf size = block_size if is_block else bowl_size colors = block_colors if is_block else bowl_colors pose = self.get_random_pose(env, obj_size=size) color = colors[n_distractors % len(colors)] obj_id = env.add_object(urdf, pose, color=color) n_distractors += 1 """ ========= Do not use libraries, functions, and assets that you don't know. For each object, try to describe its color, size, category in the task first before you write the code. You do not need extra helper functions. Comment the code liberally to explain what each piece does and why it's written that way. Now write the code for the task "TASK_NAME_TEMPLATE" in python code block starting with ```python. Reminder: TASK_STRING_TEMPLATE >>> Answer: The task we will design is called "insert-and-stack". In this task, the robot is required to first insert the ell into the fixture and then stack a number of blocks on top of the inserted fixture. The task should be completed within a specified zone. The blocks can be of multiple colors to make the task more interesting and appealing. Here is how we define it: ```python { "task-name": "insert-and-stack", "task-description": "Insert the ell into the fixture and then stack three blocks of different colors on top of the inserted fixture within the specified zone.", "assets-used": ["insertion/ell.urdf", "insertion/fixture.urdf", "stacking/block.urdf", "zone/urdf"] } ``` Now let's write the task code: ```python import numpy as np from cliport.tasks.task import Task from cliport.utils import utils import os class InsertAndStack(Task): """Insert ell into fixture and then stack blocks on top of it.""" def __init__(self): super().__init__() self.max_steps = 10 self.metric = 'pose' self.lang_template = "insert the ell into the fixture and then stack blocks on top of it" self.task_completed_desc = "done insert-and-stack." def reset(self, env): super().reset(env) # Add ell ell_size = (0.1, 0.1, 0.1) ell_pose = self.get_random_pose(env, ell_size) ell_urdf = 'insertion/ell.urdf' ell_id = env.add_object(ell_urdf, ell_pose) self.color_random_bright(ell_id) # Add fixture fixture_size = (0.12, 0.12, 0.1) fixture_pose = self.get_random_pose(env, fixture_size) fixture_urdf = 'insertion/fixture.urdf' fixture_id = env.add_object(fixture_urdf, fixture_pose) self.color_random_bright(fixture_id) # Add blocks block_size = (0.04, 0.04, 0.04) block_urdf = 'stacking/block.urdf' blocks = [] for _ in range(3): # We want 3 blocks block_pose = self.get_random_pose(env, block_size) block_id = env.add_object(block_urdf, block_pose) self.color_random_bright(block_id) # Randomly color the blocks blocks.append(block_id) # Define the zone zone_size = (0.1, 0.1, 0.1) zone_pose = self.get_random_pose(env, zone_size) zone_urdf = 'zone/zone.urdf' env.add_object(zone_urdf, zone_pose, 'fixed') # Zone is static # Set task goals objs = [ell_id] + blocks goal_poses = [fixture_pose] * (len(objs)) self.goals.append((objs, goal_poses, False)) self.lang_goals.append(self.lang_template) ``` In this code, we first add the ell and fixture into the environment. They are randomly colored for aesthetic purposes. Next, we add three blocks into the environment, which are also randomly colored. We then define a zone in which the task needs to be performed and add it to the environment. Finally, we set the goal of the task, which is for the ell and blocks to be placed in the fixture's position. This effectively means that the blocks must be stacked on top of the ell that has been inserted into the fixture. Note: The color_random_bright() method is assumed to be a method of the Task class which sets a random bright color to the object. If such method does not exist, the color assignment lines can be removed. ================= Code Generation! >>> Prompt: Now I will provide you some reference code and you can write the code for the task "insert-and-stack". """ import os import numpy as np from cliport.tasks.task import Task from cliport.utils import utils class PackingShapes(Task): """pick up randomly sized shapes and place them tightly into a container.""" def __init__(self): super().__init__() self.max_steps = 1 self.homogeneous = False self.lang_template = "pack the {obj} in the brown box" self.task_completed_desc = "done packing shapes." self.additional_reset() def reset(self, env): super().reset(env) # Shape Names: shapes = utils.assembling_kit_shapes n_objects = 5 if self.mode == 'train': obj_shapes = np.random.choice(self.train_set, n_objects, replace=False) else: if self.homogeneous: obj_shapes = [np.random.choice(self.test_set, replace=False)] * n_objects else: obj_shapes = np.random.choice(self.test_set, n_objects, replace=False) # Shuffle colors to avoid always picking an object of the same color colors, color_names = utils.get_colors(mode=self.mode) # Add container box. zone_size = self.get_random_size(0.1, 0.15, 0.1, 0.15, 0.05, 0.05) zone_pose = self.get_random_pose(env, zone_size) container_template = 'container/container-template.urdf' replace = {'DIM': zone_size, 'HALF': (zone_size[0] / 2, zone_size[1] / 2, zone_size[2] / 2)} # IMPORTANT: REPLACE THE TEMPLATE URDF with `fill_template` container_urdf = self.fill_template(container_template, replace) env.add_object(container_urdf, zone_pose, 'fixed') # Add objects. objects = [] template = 'kitting/object-template.urdf' for i in range(n_objects): shape = obj_shapes[i] # x, y, z dimensions for the asset size size = (0.08, 0.08, 0.02) pose= self.get_random_pose(env, size) fname = f'{shape:02d}.obj' fname = os.path.join(self.assets_root, 'kitting', fname) scale = [0.003, 0.003, 0.001] # .0005 replace = {'FNAME': (fname,), 'SCALE': scale, 'COLOR': colors[i]} # IMPORTANT: REPLACE THE TEMPLATE URDF urdf = self.fill_template(template, replace) block_id = env.add_object(urdf, pose) objects.append(block_id) # Pick the first shape. num_objects_to_pick = 1 for i in range(num_objects_to_pick): # IMPORTANT: Specify (obj_pts, [(zone_pose, zone_size)]) for target `zone`. obj_pts is a dict language_goal = self.lang_template.format(obj=shapes[obj_shapes[i]]) self.add_goal(objs=[objects[i]], matches=np.int32([[1]]), targ_poses=[zone_pose], replace=False, rotations=True, metric='zone', params=[(zone_pose, zone_size)], step_max_reward=1 / num_objects_to_pick, language_goal=language_goal) """ """ import numpy as np from cliport.tasks.task import Task from cliport.utils import utils import pybullet as p class PlaceRedInGreen(Task): """pick up the red blocks and place them into the green bowls amidst other objects.""" def __init__(self): super().__init__() self.max_steps = 10 self.lang_template = "put the red blocks in a green bowl" self.task_completed_desc = "done placing blocks in bowls." self.additional_reset() def reset(self, env): super().reset(env) n_bowls = np.random.randint(1, 4) n_blocks = np.random.randint(1, n_bowls + 1) # Add bowls. # x, y, z dimensions for the asset size bowl_size = (0.12, 0.12, 0) bowl_urdf = 'bowl/bowl.urdf' bowl_poses = [] for _ in range(n_bowls): bowl_pose = self.get_random_pose(env, obj_size=bowl_size) env.add_object(urdf=bowl_urdf, pose=bowl_pose, category='fixed') bowl_poses.append(bowl_pose) # Add blocks. # x, y, z dimensions for the asset size blocks = [] block_size = (0.04, 0.04, 0.04) block_urdf = 'stacking/block.urdf' for _ in range(n_blocks): block_pose = self.get_random_pose(env, obj_size=block_size) block_id = env.add_object(block_urdf, block_pose) blocks.append(block_id) # Goal: each red block is in a different green bowl. self.add_goal(objs=blocks, matches=np.ones((len(blocks), len(bowl_poses))), targ_poses=bowl_poses, replace=False, rotations=True, metric='pose', params=None, step_max_reward=1, language_goal=self.lang_template) # Colors of distractor objects. bowl_colors = [utils.COLORS[c] for c in utils.COLORS if c != 'green'] block_colors = [utils.COLORS[c] for c in utils.COLORS if c != 'red'] # Add distractors. n_distractors = 0 while n_distractors < 6: is_block = np.random.rand() > 0.5 urdf = block_urdf if is_block else bowl_urdf size = block_size if is_block else bowl_size colors = block_colors if is_block else bowl_colors pose = self.get_random_pose(env, obj_size=size) color = colors[n_distractors % len(colors)] obj_id = env.add_object(urdf, pose, color=color) n_distractors += 1 """ """ import numpy as np from cliport.tasks import primitives from cliport.tasks.grippers import Spatula from cliport.tasks.task import Task from cliport.utils import utils class SweepingPiles(Task): """Push piles of small objects into a target goal zone marked on the tabletop.""" def __init__(self): super().__init__() self.max_steps = 20 self.lang_template = "push the pile of blocks into the green square" self.task_completed_desc = "done sweeping." self.primitive = primitives.push self.ee = Spatula self.additional_reset() def reset(self, env): super().reset(env) # Add goal zone. zone_size = (0.12, 0.12, 0) zone_pose = self.get_random_pose(env, zone_size) env.add_object('zone/zone.urdf', zone_pose, 'fixed') # Add pile of small blocks with `make_piles` function obj_ids = self.make_piles(env) # Add goal self.add_goal(objs=obj_ids, matches=np.ones((50, 1)), targ_poses=[zone_pose], replace=True, rotations=False, metric='zone', params=[(zone_pose, zone_size)], step_max_reward=1, language_goal=self.lang_template) """ """ import numpy as np from cliport.tasks.task import Task from cliport.utils import utils import pybullet as p class StackBlockPyramid(Task): """Build a pyramid of colored blocks in a color sequence""" def __init__(self): super().__init__() self.max_steps = 12 self.lang_template = "make the {row} row with {blocks}" self.task_completed_desc = "done stacking block pyramid." self.additional_reset() def reset(self, env): super().reset(env) # Add base. base_size = (0.05, 0.15, 0.005) base_urdf = 'stacking/stand.urdf' base_pose = self.get_random_pose(env, base_size) env.add_object(base_urdf, base_pose, category='fixed') # Block colors. colors = [ utils.COLORS['purple'], utils.COLORS['blue'], utils.COLORS['green'], utils.COLORS['yellow'], utils.COLORS['orange'], utils.COLORS['red'] ] # Add blocks. block_size = (0.04, 0.04, 0.04) block_urdf = 'stacking/block.urdf' objs = [] for i in range(6): block_pose = self.get_random_pose(env, block_size) block_id = env.add_object(block_urdf, block_pose, color=colors[i]) objs.append(block_id) # IMPORTANT Associate placement locations for goals. place_pos = [(0, -0.05, 0.03), (0, 0, 0.03), (0, 0.05, 0.03), (0, -0.025, 0.08), (0, 0.025, 0.08), (0, 0, 0.13)] targs = [(utils.apply(base_pose, i), base_pose[1]) for i in place_pos] # Goal: blocks are stacked in a pyramid (bottom row: green, blue, purple). language_goal = self.lang_template.format(blocks="the green, blue and purple blocks", row="bottom") self.add_goal(objs=objs[:3], matches=np.ones((3, 3)), targ_poses=targs[:3], replace=False, rotations=True, metric='pose', params=None, step_max_reward=1 / 2, symmetries=[np.pi/2]*3, language_goal=language_goal) # Goal: blocks are stacked in a pyramid (middle row: yellow, orange). language_goal = self.lang_template.format(blocks="the yellow and orange blocks", row="middle") self.add_goal(objs=objs[3:5], matches=np.ones((2, 2)), targ_poses=targs[3:5], replace=False, rotations=True, metric='pose', params=None, step_max_reward=1 / 3, symmetries=[np.pi/2]*2, language_goal=language_goal) # Goal: blocks are stacked in a pyramid (top row: red). language_goal = self.lang_template.format(blocks="the red block", row="top") self.add_goal(objs=objs[5:], matches=np.ones((1, 1)), targ_poses=targs[5:], replace=False, rotations=True, metric='pose', params=None, step_max_reward=1 / 6, symmetries=[np.pi/2]*1, language_goal=language_goal) """ Do not use libraries, functions, and assets that you don't know. For each object, try to describe its color, size, category in the task first before you write the code. You do not need extra helper functions. Comment the code liberally to explain what each piece does and why it's written that way. If you have only one goal, `step_max_reward` in `add_goal` should be 1. Use functions `make_piles` and `make_ropes` for creating piles and cables. To use spatula together with the push primitives, import the libraries """ from cliport.tasks import primitives; from cliport.tasks.grippers import Spatula """ and then use `self.primitive = primitives.push` and `self.ee = Spatula`. Note that the number of language goals usually match the number of motion goals, since they should correspond to each other. Now write the code for the task "insert-and-stack" in python code block starting with ```python. Reminder: {'task-name': 'insert-and-stack', 'task-description': 'Insert the ell into the fixture and then stack three blocks of different colors on top of the inserted fixture within the specified zone.', 'assets-used': ['insertion/ell.urdf', 'insertion/fixture.urdf', 'stacking/block.urdf', 'zone/urdf']} >>> Answer: Here is the code for the task "insert-and-stack": ```python import numpy as np from cliport.tasks.task import Task from cliport.utils import utils class InsertAndStack(Task): """Insert the ell into the fixture and then stack blocks on top of it.""" def __init__(self): super().__init__() self.max_steps = 10 self.metric = 'pose' self.lang_template = "insert the ell into the fixture and then stack blocks on top of it" self.task_completed_desc = "done insert-and-stack." def reset(self, env): super().reset(env) # Add ell ell_size = (0.1, 0.1, 0.1) ell_pose = self.get_random_pose(env, ell_size) ell_urdf = 'insertion/ell.urdf' ell_id = env.add_object(ell_urdf, ell_pose) self.color_random_bright(ell_id) # Add fixture fixture_size = (0.12, 0.12, 0.1) fixture_pose = self.get_random_pose(env, fixture_size) fixture_urdf = 'insertion/fixture.urdf' fixture_id = env.add_object(fixture_urdf, fixture_pose) self.color_random_bright(fixture_id) # Add blocks block_size = (0.04, 0.04, 0.04) block_urdf = 'stacking/block.urdf' blocks = [] for _ in range(3): # We want 3 blocks block_pose = self.get_random_pose(env, block_size) block_id = env.add_object(block_urdf, block_pose) self.color_random_bright(block_id) # Randomly color the blocks blocks.append(block_id) # Define the zone zone_size = (0.1, 0.1, 0.1) zone_pose = self.get_random_pose(env, zone_size) zone_urdf = 'zone/zone.urdf' env.add_object(zone_urdf, zone_pose, 'fixed') # Zone is static # Set task goals objs = [ell_id] + blocks goal_poses = [fixture_pose] * (len(objs)) self.add_goal(objs=objs, matches=np.ones((len(objs), 1)), targ_poses=goal_poses, replace=False, rotations=True, metric='pose', params=None, step_max_reward=1, language_goal=self.lang_template) ``` In this code, we first add the ell and fixture into the environment. They are randomly colored for aesthetic purposes. Next, we add three blocks into the environment, which are also randomly colored. We then define a zone in which the task needs to be performed and add it to the environment. Finally, we set the goal of the task, which is for the ell and blocks to be placed in the fixture's position. This effectively means that the blocks must be stacked on top of the ell that has been inserted into the fixture. Note: The color_random_bright() method is assumed to be a method of the Task class which sets a random bright color to the object. If such method does not exist, the color assignment lines can be removed. ================= TRIAL: 1